From 6c31f15bc6f7bef5efc739344eb5bab3f67cf98b Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 6 Nov 2023 15:03:33 +0100 Subject: [PATCH 001/339] Read config files from new location after they have been migrated to after https://github.com/pi-hole/pi-hole/pull/5479 Signed-off-by: DL6ER --- src/config/dnsmasq_config.c | 12 ------------ src/config/dnsmasq_config.h | 4 ++-- 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/src/config/dnsmasq_config.c b/src/config/dnsmasq_config.c index e727e3a7..d580cc25 100644 --- a/src/config/dnsmasq_config.c +++ b/src/config/dnsmasq_config.c @@ -595,7 +595,6 @@ bool read_legacy_dhcp_static_config(void) { // Check if file exists, if not, there is nothing to do const char *path = DNSMASQ_STATIC_LEASES; - const char *target = DNSMASQ_STATIC_LEASES".bck"; if(!file_exists(path)) return true; @@ -645,11 +644,6 @@ bool read_legacy_dhcp_static_config(void) return false; } - // Move file to backup location - log_info("Moving %s to %s", path, target); - if(rename(path, target) != 0) - log_warn("Unable to move %s to %s: %s", path, target, strerror(errno)); - return true; } @@ -658,7 +652,6 @@ bool read_legacy_cnames_config(void) { // Check if file exists, if not, there is nothing to do const char *path = DNSMASQ_CNAMES; - const char *target = DNSMASQ_CNAMES".bck"; if(!file_exists(path)) return true; @@ -708,11 +701,6 @@ bool read_legacy_cnames_config(void) return false; } - // Move file to backup location - log_info("Moving %s to %s", path, target); - if(rename(path, target) != 0) - log_warn("Unable to move %s to %s: %s", path, target, strerror(errno)); - return true; } diff --git a/src/config/dnsmasq_config.h b/src/config/dnsmasq_config.h index 8552169c..d9607c99 100644 --- a/src/config/dnsmasq_config.h +++ b/src/config/dnsmasq_config.h @@ -24,8 +24,8 @@ bool write_custom_list(void); #define DNSMASQ_PH_CONFIG "/etc/pihole/dnsmasq.conf" #define DNSMASQ_TEMP_CONF "/etc/pihole/dnsmasq.conf.temp" -#define DNSMASQ_STATIC_LEASES "/etc/pihole/04-pihole-static-dhcp.conf" -#define DNSMASQ_CNAMES "/etc/pihole/05-pihole-custom-cname.conf" +#define DNSMASQ_STATIC_LEASES "/etc/pihole/migration_backup_v6/04-pihole-static-dhcp.conf" +#define DNSMASQ_CNAMES "/etc/pihole/migration_backup_v6/05-pihole-custom-cname.conf" #define DNSMASQ_CUSTOM_LIST "/etc/pihole/custom.list" #define DHCPLEASESFILE "/etc/pihole/dhcp.leases" From 714f14babb7695b9b514c985c5a347daedd67667 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 3 Feb 2024 19:33:34 +0100 Subject: [PATCH 002/339] Add optional JSON object .import to POST /api/teleporter that allows the user to pick what is to be restored Signed-off-by: DL6ER --- src/api/docs/content/specs/teleporter.yaml | 44 +++++++++ src/api/teleporter.c | 108 +++++++++++++++++---- src/webserver/json_macros.h | 6 ++ src/zip/teleporter.c | 72 ++++++++++++-- src/zip/teleporter.h | 2 +- 5 files changed, 205 insertions(+), 27 deletions(-) diff --git a/src/api/docs/content/specs/teleporter.yaml b/src/api/docs/content/specs/teleporter.yaml index 90132e2d..5a52fbf0 100644 --- a/src/api/docs/content/specs/teleporter.yaml +++ b/src/api/docs/content/specs/teleporter.yaml @@ -42,6 +42,50 @@ components: file: type: string format: binary + import: + type: object + nullable: true + properties: + config: + type: boolean + description: "Import Pi-hole configuration" + example: true + dhcp_leases: + type: boolean + description: "Import Pi-hole DHCP leases" + example: true + gravity: + type: object + properties: + group: + type: boolean + description: "Import Pi-hole's groups table" + example: true + adlist: + type: boolean + description: "Import Pi-hole's adlist table" + example: true + adlist_by_group: + type: boolean + description: "Import Pi-hole's table relating adlist entries to groups" + example: true + domainlist: + type: boolean + description: "Import Pi-hole's domainlist table" + example: true + domainlist_by_group: + type: boolean + description: "Import Pi-hole's table relating domainlist entries to groups" + example: true + client: + type: boolean + description: "Import Pi-hole's client table" + example: true + client_by_group: + type: boolean + description: "Import Pi-hole's table relating client entries to groups" + example: true + description: "A JSON object of files to import. If omitted, all files will be imported." responses: '200': description: OK diff --git a/src/api/teleporter.c b/src/api/teleporter.c index 12eb2ad1..a595cdc5 100644 --- a/src/api/teleporter.c +++ b/src/api/teleporter.c @@ -68,14 +68,18 @@ static int api_teleporter_GET(struct ftl_conn *api) struct upload_data { bool too_large; char *sid; + cJSON *import; uint8_t *data; char *filename; size_t filesize; + struct { + bool file; + bool sid; + bool import; + } field; }; // Callback function for CivetWeb to determine which fields we want to receive -static bool is_file = false; -static bool is_sid = false; static int field_found(const char *key, const char *filename, char *path, @@ -85,17 +89,22 @@ static int field_found(const char *key, struct upload_data *data = (struct upload_data *)user_data; log_debug(DEBUG_API, "Found field: \"%s\", filename: \"%s\"", key, filename); - is_file = false; - is_sid = false; + // Set all fields to false + memset(&data->field, false, sizeof(data->field)); if(strcasecmp(key, "file") == 0 && filename && *filename) { data->filename = strdup(filename); - is_file = true; + data->field.file = true; return MG_FORM_FIELD_STORAGE_GET; } else if(strcasecmp(key, "sid") == 0) { - is_sid = true; + data->field.sid = true; + return MG_FORM_FIELD_STORAGE_GET; + } + else if(strcasecmp(key, "import") == 0) + { + data->field.import = true; return MG_FORM_FIELD_STORAGE_GET; } @@ -111,7 +120,7 @@ static int field_get(const char *key, const char *value, size_t valuelen, void * struct upload_data *data = (struct upload_data *)user_data; log_debug(DEBUG_API, "Received field: \"%s\" (length %zu bytes)", key, valuelen); - if(is_file) + if(data->field.file) { if(data->filesize + valuelen > MAXFILESIZE) { @@ -129,7 +138,7 @@ static int field_get(const char *key, const char *value, size_t valuelen, void * log_debug(DEBUG_API, "Received file (%zu bytes, buffer is now %zu bytes)", valuelen, data->filesize); } - else if(is_sid) + else if(data->field.sid) { // Allocate memory for the SID data->sid = calloc(valuelen + 1, sizeof(char)); @@ -138,6 +147,27 @@ static int field_get(const char *key, const char *value, size_t valuelen, void * // Add terminating NULL byte (memcpy does not do this) data->sid[valuelen] = '\0'; } + else if(data->field.import) + { + // Try to parse the JSON data + cJSON *json = cJSON_ParseWithLength(value, valuelen); + if(json == NULL) + { + log_err("Unable to parse JSON data in API request: %s", cJSON_GetErrorPtr()); + return MG_FORM_FIELD_HANDLE_ABORT; + } + + // Check if the JSON data is an object + if(!cJSON_IsObject(json)) + { + log_err("JSON data in API request is not an object"); + cJSON_Delete(json); + return MG_FORM_FIELD_HANDLE_ABORT; + } + + // Store the parsed JSON data + data->import = json; + } // If there is more data in this field, get the next chunk. // Otherwise: handle the next field. @@ -168,6 +198,11 @@ static int free_upload_data(struct upload_data *data) free(data->data); data->data = NULL; } + if(data->import) + { + cJSON_Delete(data->import); + data->import = NULL; + } return 0; } @@ -262,7 +297,7 @@ static int process_received_zip(struct ftl_conn *api, struct upload_data *data) char hint[ERRBUF_SIZE]; memset(hint, 0, sizeof(hint)); cJSON *json_files = JSON_NEW_ARRAY(); - const char *error = read_teleporter_zip(data->data, data->filesize, hint, json_files); + const char *error = read_teleporter_zip(data->data, data->filesize, hint, data->import, json_files); if(error != NULL) { const size_t msglen = strlen(error) + strlen(hint) + 4; @@ -277,7 +312,7 @@ static int process_received_zip(struct ftl_conn *api, struct upload_data *data) free_upload_data(data); return send_json_error_free(api, 400, "bad_request", - "Invalid ZIP archive", + "Invalid request", msg, true); } @@ -632,14 +667,34 @@ static int process_received_tar_gz(struct ftl_conn *api, struct upload_data *dat // Parse JSON files in the TAR archive cJSON *imported_files = JSON_NEW_ARRAY(); - for(size_t i = 0; i < sizeof(teleporter_v5_files) / sizeof(struct teleporter_files); i++) + + // Check if the archive contains gravity tables + cJSON *gravity = data->import != NULL ? cJSON_GetObjectItemCaseSensitive(data->import, "gravity") : NULL; + if(data->import == NULL || gravity != NULL) { - size_t fileSize = 0u; - cJSON *json = NULL; - const char *file = find_file_in_tar(archive, archive_size, teleporter_v5_files[i].filename, &fileSize); - if(file != NULL && fileSize > 0u && (json = cJSON_ParseWithLength(file, fileSize)) != NULL) - if(import_json_table(json, &teleporter_v5_files[i])) - JSON_COPY_STR_TO_ARRAY(imported_files, teleporter_v5_files[i].filename); + for(size_t i = 0; i < sizeof(teleporter_v5_files) / sizeof(struct teleporter_files); i++) + { + // - if import is NULL we import all files/tables + // - if import is non-NULL, but gravity is NULL we skip + // the import of gravity tables + // - if import is non-NULL, and gravity is non-NULL, we + // import the file/table if it is in the object, a + // boolean and true + if(data->import != NULL || gravity == NULL || !JSON_KEY_TRUE(gravity, teleporter_v5_files[i].table_name)) + { + log_info("Skipping import of \"%s\" as it was not requested for import", + teleporter_v5_files[i].filename); + continue; + } + + // Import the JSON file + size_t fileSize = 0u; + cJSON *json = NULL; + const char *file = find_file_in_tar(archive, archive_size, teleporter_v5_files[i].filename, &fileSize); + if(file != NULL && fileSize > 0u && (json = cJSON_ParseWithLength(file, fileSize)) != NULL) + if(import_json_table(json, &teleporter_v5_files[i])) + JSON_COPY_STR_TO_ARRAY(imported_files, teleporter_v5_files[i].filename); + } } // Temporarily write further files to to disk so we can import them on restart @@ -648,15 +703,19 @@ static int process_received_tar_gz(struct ftl_conn *api, struct upload_data *dat const char *destination; } extract_files[] = { { + // i = 0 .archive_name = "custom.list", .destination = DNSMASQ_CUSTOM_LIST_LEGACY },{ + // i = 1 .archive_name = "dhcp.leases", .destination = DHCPLEASESFILE },{ + // i = 2 .archive_name = "pihole-FTL.conf", .destination = GLOBALCONFFILE_LEGACY },{ + // i = 3 .archive_name = "setupVars.conf", .destination = config.files.setupVars.v.s } @@ -665,6 +724,21 @@ static int process_received_tar_gz(struct ftl_conn *api, struct upload_data *dat { size_t fileSize = 0u; const char *file = find_file_in_tar(archive, archive_size, extract_files[i].archive_name, &fileSize); + + if(data->import != NULL && i == 1 && !JSON_KEY_TRUE(data->import, "dhcp_leases")) + { + log_info("Skipping import of \"%s\" as it was not requested for import", + extract_files[i].archive_name); + continue; + } + // all other values of i belong to config files + else if(data->import != NULL && !JSON_KEY_TRUE(data->import, "config")) + { + log_info("Skipping import of \"%s\" as it was not requested for import", + extract_files[i].archive_name); + continue; + } + if(file != NULL && fileSize > 0u) { // Write file to disk diff --git a/src/webserver/json_macros.h b/src/webserver/json_macros.h index f3966f08..a5ee25a7 100644 --- a/src/webserver/json_macros.h +++ b/src/webserver/json_macros.h @@ -254,3 +254,9 @@ #define JSON_INCREMENT_NUMBER(number_obj, inc)({ \ cJSON_SetNumberHelper(number_obj, number_obj->valuedouble + inc); \ }) + +// Returns true if the key exists and is true, otherwise false +#define JSON_KEY_TRUE(obj, key)({ \ + cJSON *elem = cJSON_GetObjectItemCaseSensitive(obj, key); \ + elem != NULL ? cJSON_IsTrue(elem) : false; \ +}) diff --git a/src/zip/teleporter.c b/src/zip/teleporter.c index b9970e22..690b6f2a 100644 --- a/src/zip/teleporter.c +++ b/src/zip/teleporter.c @@ -37,11 +37,11 @@ #include "webserver/cJSON/cJSON.h" // set_event() #include "events.h" - +// JSON_KEY_TRUE +#include "webserver/json_macros.h" // Tables to copy from the gravity database to the Teleporter database static const char *gravity_tables[] = { - "info", "group", "adlist", "adlist_by_group", @@ -365,7 +365,7 @@ static const char *import_dhcp_leases(void *ptr, size_t size, char * const hint) } static const char *test_and_import_database(void *ptr, size_t size, const char *destination, - const char **tables, const unsigned int num_tables, + const char **tables, const size_t num_tables, char * const hint) { // Check if the file is empty @@ -523,7 +523,7 @@ static const char *test_and_import_database(void *ptr, size_t size, const char * return NULL; } -const char *read_teleporter_zip(uint8_t *buffer, const size_t buflen, char * const hint, cJSON *imported_files) +const char *read_teleporter_zip(uint8_t *buffer, const size_t buflen, char * const hint, cJSON *import, cJSON *imported_files) { // Initialize ZIP archive mz_zip_archive zip = { 0 }; @@ -585,8 +585,16 @@ const char *read_teleporter_zip(uint8_t *buffer, const size_t buflen, char * con // Process file // Is this "etc/pihole/pihole.toml" ? - if(strcmp(file_stat.m_filename, "etc/pihole/pihole.toml") == 0) + if(strcmp(file_stat.m_filename, extract_files[0]) == 0) { + // Check whether we should import this file + if(import != NULL && !JSON_KEY_TRUE(import, "config")) + { + log_info("Ignoring file %s in Teleporter archive (not in import list)", file_stat.m_filename); + free(ptr); + continue; + } + // Import Pi-hole configuration memset(hint, 0, ERRBUF_SIZE); const char *err = test_and_import_pihole_toml(ptr, file_stat.m_uncomp_size, hint); @@ -598,8 +606,16 @@ const char *read_teleporter_zip(uint8_t *buffer, const size_t buflen, char * con log_debug(DEBUG_CONFIG, "Imported Pi-hole configuration: %s", file_stat.m_filename); } // Is this "etc/pihole/dhcp.leases"? - else if(strcmp(file_stat.m_filename, "etc/pihole/dhcp.leases") == 0) + else if(strcmp(file_stat.m_filename, extract_files[1]) == 0) { + // Check whether we should import this file + if(import != NULL && !JSON_KEY_TRUE(import, "dhcp_leases")) + { + log_info("Ignoring file %s in Teleporter archive (not in import list)", file_stat.m_filename); + free(ptr); + continue; + } + // Import DHCP leases memset(hint, 0, ERRBUF_SIZE); const char *err = import_dhcp_leases(ptr, file_stat.m_uncomp_size, hint); @@ -610,12 +626,50 @@ const char *read_teleporter_zip(uint8_t *buffer, const size_t buflen, char * con } log_debug(DEBUG_CONFIG, "Imported DHCP leases: %s", file_stat.m_filename); } - else if(strcmp(file_stat.m_filename, "etc/pihole/gravity.db") == 0) + // Is this "etc/pihole/gravity.db"? + else if(strcmp(file_stat.m_filename, extract_files[2]) == 0) { + // Check whether we should import this file + if(import != NULL && !cJSON_HasObjectItem(import, "gravity")) + { + log_info("Ignoring file %s in Teleporter archive (not in import list)", file_stat.m_filename); + free(ptr); + continue; + } + + const char *import_tables[ArraySize(gravity_tables)] = { NULL }; + size_t num_tables = 0u; + if(import == NULL) + { + // Import all tables + num_tables = ArraySize(gravity_tables); + memcpy(import_tables, gravity_tables, sizeof(gravity_tables)); + } + else + { + // Get object at import.gravity + cJSON *import_gravity = cJSON_GetObjectItem(import, "gravity"); + + // Check if import.gravity is a JSON object + if(import_gravity == NULL || !cJSON_IsObject(import_gravity)) + { + log_warn("Ignoring file %s in Teleporter archive (import.gravity is not a JSON object)", file_stat.m_filename); + free(ptr); + continue; + } + + // Import selected tables + for(size_t j = 0; j < ArraySize(gravity_tables); j++) + { + if(JSON_KEY_TRUE(import, gravity_tables[j])) + import_tables[num_tables++] = gravity_tables[j]; + } + } + // Import gravity database memset(hint, 0, ERRBUF_SIZE); const char *err = test_and_import_database(ptr, file_stat.m_uncomp_size, config.files.gravity.v.s, - gravity_tables, ArraySize(gravity_tables), hint); + import_tables, num_tables, hint); if(err != NULL) { free(ptr); @@ -730,7 +784,7 @@ bool read_teleporter_zip_from_disk(const char *filename) // Process ZIP archive char hint[ERRBUF_SIZE] = ""; cJSON *imported_files = cJSON_CreateArray(); - const char *error = read_teleporter_zip(ptr, size, hint, imported_files); + const char *error = read_teleporter_zip(ptr, size, hint, NULL, imported_files); if(error != NULL) { diff --git a/src/zip/teleporter.h b/src/zip/teleporter.h index a5743028..0394648f 100644 --- a/src/zip/teleporter.h +++ b/src/zip/teleporter.h @@ -15,7 +15,7 @@ const char *generate_teleporter_zip(mz_zip_archive *zip, char filename[128], void **ptr, size_t *size); bool free_teleporter_zip(mz_zip_archive *zip); -const char *read_teleporter_zip(uint8_t *buffer, const size_t buflen, char *hint, cJSON *json_files); +const char *read_teleporter_zip(uint8_t *buffer, const size_t buflen, char *hint, cJSON *import, cJSON *json_files); bool write_teleporter_zip_to_disk(void); bool read_teleporter_zip_from_disk(const char *filename); From b650631d6e9fa0c54b080009dc99ddbf36d27e4a Mon Sep 17 00:00:00 2001 From: Simon Kelley Date: Wed, 17 May 2023 23:19:30 +0100 Subject: [PATCH 003/339] Log truncated DNS replies. Signed-off-by: DL6ER --- src/dnsmasq/cache.c | 16 +++++++++++----- src/dnsmasq/forward.c | 30 ++++++++++++++++++++---------- src/dnsmasq/rfc1035.c | 5 ++++- 3 files changed, 35 insertions(+), 16 deletions(-) diff --git a/src/dnsmasq/cache.c b/src/dnsmasq/cache.c index 0816cb54..6620a825 100644 --- a/src/dnsmasq/cache.c +++ b/src/dnsmasq/cache.c @@ -2068,9 +2068,10 @@ const char *edestr(int ede) /**** P-hole modified: Added file and line and serve log_query via macro defined in dnsmasq.h ****/ void _log_query(unsigned int flags, char *name, union all_addr *addr, char *arg, unsigned short type, const char *file, const int line) { - char *source, *dest = arg; + char *source, *dest; char *verb = "is"; char *extra = ""; + char *gap = " "; char portstring[7]; /* space for # */ FTL_hook(flags, name, addr, arg, daemon->log_display_id, type, file, line); @@ -2082,6 +2083,8 @@ void _log_query(unsigned int flags, char *name, union all_addr *addr, char *arg, if (!(flags & (F_SERVER | F_IPSET)) && type > 0) arg = querystr(arg, type); + dest = arg; + #ifdef HAVE_DNSSEC if ((flags & F_DNSSECOK) && option_bool(OPT_EXTRALOG)) extra = " (DNSSEC signed)"; @@ -2202,18 +2205,21 @@ void _log_query(unsigned int flags, char *name, union all_addr *addr, char *arg, else source = "cached"; - if (name && !name[0]) + if (!name) + gap = name = ""; + else if (!name[0]) name = "."; + if (option_bool(OPT_EXTRALOG)) { if (flags & F_NOEXTRA) - my_syslog(LOG_INFO, "%u %s %s %s %s%s", daemon->log_display_id, source, name, verb, dest, extra); + my_syslog(LOG_INFO, "%u %s %s%s%s %s%s", daemon->log_display_id, source, name, gap, verb, dest, extra); else { int port = prettyprint_addr(daemon->log_source_addr, daemon->addrbuff2); - my_syslog(LOG_INFO, "%u %s/%u %s %s %s %s%s", daemon->log_display_id, daemon->addrbuff2, port, source, name, verb, dest, extra); + my_syslog(LOG_INFO, "%u %s/%u %s %s%s%s %s%s", daemon->log_display_id, daemon->addrbuff2, port, source, name, gap, verb, dest, extra); } } else - my_syslog(LOG_INFO, "%s %s %s %s%s", source, name, verb, dest, extra); + my_syslog(LOG_INFO, "%s %s%s%s %s%s", source, name, gap, verb, dest, extra); } diff --git a/src/dnsmasq/forward.c b/src/dnsmasq/forward.c index 60e26764..9229f5b9 100644 --- a/src/dnsmasq/forward.c +++ b/src/dnsmasq/forward.c @@ -944,17 +944,24 @@ static void dnssec_validate(struct frec *forward, struct dns_header *header, if (forward->blocking_query) return; - /* Truncated answer can't be validated. - If this is an answer to a DNSSEC-generated query, we still - need to get the client to retry over TCP, so return - an answer with the TC bit set, even if the actual answer fits. - */ - if (header->hb3 & HB3_TC) - status = STAT_TRUNCATED; - /* If all replies to a query are REFUSED, give up. */ if (RCODE(header) == REFUSED) status = STAT_ABANDONED; + else if (header->hb3 & HB3_TC) + { + /* Truncated answer can't be validated. + If this is an answer to a DNSSEC-generated query, we still + need to get the client to retry over TCP, so return + an answer with the TC bit set, even if the actual answer fits. + */ + status = STAT_TRUNCATED; + if (forward->flags & (FREC_DNSKEY_QUERY | FREC_DS_QUERY)) + { + unsigned char *p = (unsigned char *)(header+1); + if (extract_name(header, plen, &p, daemon->namebuff, 0, 4) == 1) + log_query(F_UPSTREAM | F_NOEXTRA, daemon->namebuff, NULL, "truncated", (forward->flags & FREC_DNSKEY_QUERY) ? T_DNSKEY : T_DS); + } + } /* As soon as anything returns BOGUS, we stop and unwind, to do otherwise would invite infinite loops, since the answers to DNSKEY and DS queries @@ -1345,7 +1352,10 @@ static void return_reply(time_t now, struct frec *forward, struct dns_header *he no_cache_dnssec = 0; if (STAT_ISEQUAL(status, STAT_TRUNCATED)) - header->hb3 |= HB3_TC; + { + header->hb3 |= HB3_TC; + log_query(F_SECSTAT, "result", NULL, "TRUNCATED", 0); + } else { char *result, *domain = "result"; @@ -1371,7 +1381,7 @@ static void return_reply(time_t now, struct frec *forward, struct dns_header *he if (extract_request(header, n, daemon->namebuff, NULL)) domain = daemon->namebuff; } - + log_query(F_SECSTAT, domain, &a, result, 0); } } diff --git a/src/dnsmasq/rfc1035.c b/src/dnsmasq/rfc1035.c index 3b4cc340..523a8dbc 100644 --- a/src/dnsmasq/rfc1035.c +++ b/src/dnsmasq/rfc1035.c @@ -936,7 +936,10 @@ int extract_addresses(struct dns_header *header, size_t qlen, char *name, time_t } } } - + + if (header->hb3 & HB3_TC) + log_query(F_UPSTREAM, NULL, NULL, "truncated", 0); + /* Don't put stuff from a truncated packet into the cache. Don't cache replies from non-recursive nameservers, since we may get a reply containing a CNAME but not its target, even though the target From d38a0a6dcd160124cbc018d6afdaaf86217860bd Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 27 May 2023 12:45:01 +0200 Subject: [PATCH 004/339] Necessary changed to handle the most recent dnsmasq changes in FTL Signed-off-by: DL6ER --- src/dnsmasq_interface.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c index 9f2adca1..8a30d61b 100644 --- a/src/dnsmasq_interface.c +++ b/src/dnsmasq_interface.c @@ -165,6 +165,14 @@ void FTL_hook(unsigned int flags, const char *name, union all_addr *addr, char * ; // Ignored else if(flags & F_IPSET) ; // Ignored + else if(flags == F_UPSTREAM && strcmp(arg, "truncated") == 0) + ; // Ignored - truncated reply + // + // flags will by (F_UPSTREAM | F_NOEXTRA) with type being + // T_DNSKEY or T_DS when this is a truncated DNSSEC reply + // + // otherwise, flags will be F_UPSTREAM and the type is not set + // (== 0) else FTL_reply(flags, name, addr, arg, id, path, line); } From 6b48e6d063569fc114310adad6e4b9d1dd57904e Mon Sep 17 00:00:00 2001 From: Simon Kelley Date: Fri, 26 May 2023 17:55:35 +0100 Subject: [PATCH 005/339] Behave better when attempting to contact unresponsive TCP servers. By default TCP connect takes minutes to fail when trying to connect a server which is not responding and for which the network layer doesn't generate HOSTUNREACH errors. This is doubled because having failed to connect in FASTOPEN mode, the code then tries again with a call to connect(). We set TCP_SYNCNT to 2, which make the timeout about 10 seconds. This in an unportable Linux feature, so it doesn't work on other platforms. No longer try connect() if sendmsg in fastopen mode fails with ETIMEDOUT or EHOSTUNREACH since the story will just be the same. Signed-off-by: DL6ER --- src/dnsmasq/forward.c | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/dnsmasq/forward.c b/src/dnsmasq/forward.c index 9229f5b9..9b4b2551 100644 --- a/src/dnsmasq/forward.c +++ b/src/dnsmasq/forward.c @@ -2002,7 +2002,7 @@ static ssize_t tcp_talk(int first, int last, int start, unsigned char *packet, while (1) { - int data_sent = 0; + int data_sent = 0, timedout = 0; struct server *serv; if (firstsendto == -1) @@ -2040,15 +2040,27 @@ static ssize_t tcp_talk(int first, int last, int start, unsigned char *packet, serv->tcpfd = -1; continue; } + +#ifdef TCP_SYNCNT + /* TCP connections by default take ages to time out. + At least on Linux, we can reduce that to only two attempts + to get a reply. For DNS, that's more sensible. */ + mark = 2; + setsockopt(serv->tcpfd, IPPROTO_TCP, TCP_SYNCNT, &mark, sizeof(unsigned int)); +#endif #ifdef MSG_FASTOPEN server_send(serv, serv->tcpfd, packet, qsize + sizeof(u16), MSG_FASTOPEN); if (errno == 0) data_sent = 1; + else if (errno = ETIMEDOUT || errno == EHOSTUNREACH) + timedout = 1; #endif - if (!data_sent && connect(serv->tcpfd, &serv->addr.sa, sa_len(&serv->addr)) == -1) + /* If fastopen failed due to lack of reply, then there's no point in + trying again in non-FASTOPEN mode. */ + if (timedout || (!data_sent && connect(serv->tcpfd, &serv->addr.sa, sa_len(&serv->addr)) == -1)) { close(serv->tcpfd); serv->tcpfd = -1; From 6cc10f72edd598fb092f1bcca2de1a76b3b10d72 Mon Sep 17 00:00:00 2001 From: Simon Kelley Date: Fri, 26 May 2023 18:19:15 +0100 Subject: [PATCH 006/339] =/== typo in last commit. Signed-off-by: DL6ER --- src/dnsmasq/forward.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dnsmasq/forward.c b/src/dnsmasq/forward.c index 9b4b2551..8043aa71 100644 --- a/src/dnsmasq/forward.c +++ b/src/dnsmasq/forward.c @@ -2054,7 +2054,7 @@ static ssize_t tcp_talk(int first, int last, int start, unsigned char *packet, if (errno == 0) data_sent = 1; - else if (errno = ETIMEDOUT || errno == EHOSTUNREACH) + else if (errno == ETIMEDOUT || errno == EHOSTUNREACH) timedout = 1; #endif From 0a90f07d688e38c123a30f228d5f3402a3321c46 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 27 May 2023 12:52:26 +0200 Subject: [PATCH 007/339] Update changed indentation of known DNSMASQ warning Signed-off-by: DL6ER --- test/dnsmasq_warnings | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/dnsmasq_warnings b/test/dnsmasq_warnings index f1256266..94b23792 100644 --- a/test/dnsmasq_warnings +++ b/test/dnsmasq_warnings @@ -75,7 +75,7 @@ src/dnsmasq/dnsmasq.c src/dnsmasq/dnsmasq.c my_syslog(LOG_WARNING, _("no servers found in %s, will retry"), latest->name); src/dnsmasq/dnssec.c - my_syslog(LOG_WARNING, _("Insecure DS reply received for %s, check domain configuration and upstream DNS server DNSSEC support"), name); + my_syslog(LOG_WARNING, _("Insecure DS reply received for %s, check domain configuration and upstream DNS server DNSSEC support"), name); src/dnsmasq/forward.c my_syslog(LOG_WARNING, _("discarding DNS reply: subnet option mismatch")); src/dnsmasq/forward.c From 45c342af051a6283628886d5e4d67fdc768ef1b0 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 8 Feb 2024 21:13:56 +0100 Subject: [PATCH 008/339] Force-update embedded dnsmasq version. We are loosing the individual dnsmasq history of the ~ last year, however, given the multitude of merge conflicts and the fact that this code will soon(ish) be replaced by development-v6 (where the history is 100% intact), this isn't much of an issue Signed-off-by: DL6ER --- src/args.c | 2 +- src/dnsmasq/CMakeLists.txt | 10 + src/dnsmasq/arp.c | 2 +- src/dnsmasq/auth.c | 2 +- src/dnsmasq/blockdata.c | 125 ++- src/dnsmasq/bpf.c | 2 +- src/dnsmasq/cache.c | 218 +++-- src/dnsmasq/config.h | 4 +- src/dnsmasq/conntrack.c | 2 +- src/dnsmasq/crypto.c | 2 +- src/dnsmasq/dbus.c | 63 +- src/dnsmasq/dhcp-common.c | 6 +- src/dnsmasq/dhcp-protocol.h | 2 +- src/dnsmasq/dhcp.c | 4 +- src/dnsmasq/dhcp6-protocol.h | 2 +- src/dnsmasq/dhcp6.c | 40 +- src/dnsmasq/dns-protocol.h | 2 +- src/dnsmasq/dnsmasq.c | 166 ++-- src/dnsmasq/dnsmasq.h | 84 +- src/dnsmasq/dnssec.c | 154 +-- src/dnsmasq/domain-match.c | 2 +- src/dnsmasq/domain.c | 24 +- src/dnsmasq/dump.c | 2 +- src/dnsmasq/edns0.c | 21 +- src/dnsmasq/forward.c | 347 +++---- src/dnsmasq/hash-questions.c | 4 +- src/dnsmasq/helper.c | 4 +- src/dnsmasq/inotify.c | 4 +- src/dnsmasq/ip6addr.h | 2 +- src/dnsmasq/lease.c | 47 +- src/dnsmasq/log.c | 10 +- src/dnsmasq/loop.c | 2 +- src/dnsmasq/metrics.c | 3 +- src/dnsmasq/metrics.h | 3 +- src/dnsmasq/netlink.c | 2 +- src/dnsmasq/network.c | 66 +- src/dnsmasq/nftset.c | 18 +- src/dnsmasq/option.c | 528 ++++++---- src/dnsmasq/outpacket.c | 2 +- src/dnsmasq/pattern.c | 2 +- src/dnsmasq/poll.c | 2 +- src/dnsmasq/radv-protocol.h | 2 +- src/dnsmasq/radv.c | 11 +- src/dnsmasq/rfc1035.c | 1792 +++++++++++++++++++--------------- src/dnsmasq/rfc2131.c | 18 +- src/dnsmasq/rfc3315.c | 60 +- src/dnsmasq/rrfilter.c | 143 ++- src/dnsmasq/slaac.c | 2 +- src/dnsmasq/tftp.c | 10 +- src/dnsmasq/ubus.c | 2 +- src/dnsmasq/util.c | 19 +- src/main.c | 2 +- src/main.h | 2 +- src/signals.h | 2 + test/dnsmasq_warnings | 8 +- test/test_suite.bats | 16 +- 56 files changed, 2373 insertions(+), 1703 deletions(-) diff --git a/src/args.c b/src/args.c index 2a570114..19e1544a 100644 --- a/src/args.c +++ b/src/args.c @@ -276,7 +276,7 @@ void parse_args(int argc, char* argv[]) const char *arg[2]; arg[0] = ""; arg[1] = "--test"; - exit(main_dnsmasq(2, arg)); + exit(main_dnsmasq(2, (char**)arg)); } // If we find "--" we collect everything behind that for dnsmasq diff --git a/src/dnsmasq/CMakeLists.txt b/src/dnsmasq/CMakeLists.txt index ff880d32..7d5f6ae4 100644 --- a/src/dnsmasq/CMakeLists.txt +++ b/src/dnsmasq/CMakeLists.txt @@ -1,3 +1,13 @@ +# Pi-hole: A black hole for Internet advertisements +# (c) 2020 Pi-hole, LLC (https://pi-hole.net) +# Network-wide ad blocking via your own hardware. +# +# FTL Engine +# /src/dnsmasq/CMakeList.txt +# +# This file is copyright under the latest version of the EUPL. +# Please see LICENSE file for your rights under this license. + set(sources arp.c auth.c diff --git a/src/dnsmasq/arp.c b/src/dnsmasq/arp.c index eda165f6..0a5a9bfa 100644 --- a/src/dnsmasq/arp.c +++ b/src/dnsmasq/arp.c @@ -1,4 +1,4 @@ -/* dnsmasq is Copyright (c) 2000-2022 Simon Kelley +/* dnsmasq is Copyright (c) 2000-2024 Simon Kelley This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by diff --git a/src/dnsmasq/auth.c b/src/dnsmasq/auth.c index 7088d677..e6adc379 100644 --- a/src/dnsmasq/auth.c +++ b/src/dnsmasq/auth.c @@ -1,4 +1,4 @@ -/* dnsmasq is Copyright (c) 2000-2022 Simon Kelley +/* dnsmasq is Copyright (c) 2000-2024 Simon Kelley This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by diff --git a/src/dnsmasq/blockdata.c b/src/dnsmasq/blockdata.c index 4c26155f..96698760 100644 --- a/src/dnsmasq/blockdata.c +++ b/src/dnsmasq/blockdata.c @@ -1,4 +1,4 @@ -/* dnsmasq is Copyright (c) 2000-2022 Simon Kelley +/* dnsmasq is Copyright (c) 2000-2024 Simon Kelley This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -19,7 +19,7 @@ static struct blockdata *keyblock_free; static unsigned int blockdata_count, blockdata_hwm, blockdata_alloced; -static void blockdata_expand(int n) +static void add_blocks(int n) { struct blockdata *new = whine_malloc(n * sizeof(struct blockdata)); @@ -47,7 +47,7 @@ void blockdata_init(void) /* Note that daemon->cachesize is enforced to have non-zero size if OPT_DNSSEC_VALID is set */ if (option_bool(OPT_DNSSEC_VALID)) - blockdata_expand(daemon->cachesize); + add_blocks(daemon->cachesize); } void blockdata_report(void) @@ -58,50 +58,61 @@ void blockdata_report(void) blockdata_alloced * sizeof(struct blockdata)); } +static struct blockdata *new_block(void) +{ + struct blockdata *block; + + if (!keyblock_free) + add_blocks(50); + + if (keyblock_free) + { + block = keyblock_free; + keyblock_free = block->next; + blockdata_count++; + if (blockdata_hwm < blockdata_count) + blockdata_hwm = blockdata_count; + block->next = NULL; + return block; + } + + return NULL; +} + static struct blockdata *blockdata_alloc_real(int fd, char *data, size_t len) { struct blockdata *block, *ret = NULL; struct blockdata **prev = &ret; size_t blen; - while (len > 0) + do { - if (!keyblock_free) - blockdata_expand(50); - - if (keyblock_free) - { - block = keyblock_free; - keyblock_free = block->next; - blockdata_count++; - } - else + if (!(block = new_block())) { /* failed to alloc, free partial chain */ blockdata_free(ret); return NULL; } - - if (blockdata_hwm < blockdata_count) - blockdata_hwm = blockdata_count; + + if ((blen = len > KEYBLOCK_LEN ? KEYBLOCK_LEN : len) > 0) + { + if (data) + { + memcpy(block->key, data, blen); + data += blen; + } + else if (!read_write(fd, block->key, blen, 1)) + { + /* failed read free partial chain */ + blockdata_free(ret); + return NULL; + } + } - blen = len > KEYBLOCK_LEN ? KEYBLOCK_LEN : len; - if (data) - { - memcpy(block->key, data, blen); - data += blen; - } - else if (!read_write(fd, block->key, blen, 1)) - { - /* failed read free partial chain */ - blockdata_free(ret); - return NULL; - } len -= blen; *prev = block; prev = &block->next; - block->next = NULL; - } + } while (len != 0); return ret; } @@ -111,6 +122,58 @@ struct blockdata *blockdata_alloc(char *data, size_t len) return blockdata_alloc_real(0, data, len); } +/* Add data to the end of the block. + newlen is length of new data, NOT total new length. + Use blockdata_alloc(NULL, 0) to make empty block to add to. */ +int blockdata_expand(struct blockdata *block, size_t oldlen, char *data, size_t newlen) +{ + struct blockdata *b; + + /* find size of current final block */ + for (b = block; oldlen > KEYBLOCK_LEN && b; b = b->next, oldlen -= KEYBLOCK_LEN); + + /* chain to short for length, something is broken */ + if (oldlen > KEYBLOCK_LEN) + { + blockdata_free(block); + return 0; + } + + while (1) + { + struct blockdata *new; + size_t blocksize = KEYBLOCK_LEN - oldlen; + size_t size = (newlen <= blocksize) ? newlen : blocksize; + + if (size != 0) + { + memcpy(&b->key[oldlen], data, size); + data += size; + newlen -= size; + } + + /* full blocks from now on. */ + oldlen = 0; + + if (newlen == 0) + break; + + if ((new = new_block())) + { + b->next = new; + b = new; + } + else + { + /* failed to alloc, free partial chain */ + blockdata_free(block); + return 0; + } + } + + return 1; +} + void blockdata_free(struct blockdata *blocks) { struct blockdata *tmp; diff --git a/src/dnsmasq/bpf.c b/src/dnsmasq/bpf.c index 4dd97c0e..62b589c2 100644 --- a/src/dnsmasq/bpf.c +++ b/src/dnsmasq/bpf.c @@ -1,4 +1,4 @@ -/* dnsmasq is Copyright (c) 2000-2022 Simon Kelley +/* dnsmasq is Copyright (c) 2000-2024 Simon Kelley This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by diff --git a/src/dnsmasq/cache.c b/src/dnsmasq/cache.c index 6620a825..dff3485d 100644 --- a/src/dnsmasq/cache.c +++ b/src/dnsmasq/cache.c @@ -1,4 +1,4 @@ -/* dnsmasq is Copyright (c) 2000-2022 Simon Kelley +/* dnsmasq is Copyright (c) 2000-2024 Simon Kelley This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -15,7 +15,7 @@ */ #include "dnsmasq.h" -#include "../dnsmasq_interface.h" +#include "dnsmasq_interface.h" static struct crec *cache_head = NULL, *cache_tail = NULL, **hash_table = NULL; #ifdef HAVE_DHCP @@ -30,6 +30,7 @@ static void make_non_terminals(struct crec *source); static struct crec *really_insert(char *name, union all_addr *addr, unsigned short class, time_t now, unsigned long ttl, unsigned int flags); static void dump_cache_entry(struct crec *cache, time_t now); +char *querystr(char *desc, unsigned short type); /* type->string mapping: this is also used by the name-hash function as a mixing table. */ /* taken from https://www.iana.org/assignments/dns-parameters/dns-parameters.xhtml */ @@ -124,6 +125,7 @@ static const struct { { 258, "AVC" }, /* Application Visibility and Control [Wolfgang_Riedel] AVC/avc-completed-template 2016-02-26*/ { 259, "DOA" }, /* Digital Object Architecture [draft-durand-doa-over-dns] DOA/doa-completed-template 2017-08-30*/ { 260, "AMTRELAY" }, /* Automatic Multicast Tunneling Relay [RFC8777] AMTRELAY/amtrelay-completed-template 2019-02-06*/ + { 261, "RESINFO" }, /* Resolver Information as Key/Value Pairs https://datatracker.ietf.org/doc/draft-ietf-add-resolver-info/06/ */ { 32768, "TA" }, /* DNSSEC Trust Authorities [Sam_Weiler][http://cameo.library.cmu.edu/][ Deploying DNSSEC Without a Signed Root. Technical Report 1999-19, Information Networking Institute, Carnegie Mellon University, April 2004.] 2005-12-13*/ { 32769, "DLV" }, /* DNSSEC Lookaside Validation (OBSOLETE) [RFC8749][RFC4431] */ }; @@ -134,6 +136,32 @@ static void cache_link(struct crec *crecp); void rehash(int size); static void cache_hash(struct crec *crecp); +unsigned short rrtype(char *in) +{ + unsigned int i; + + for (i = 0; i < (sizeof(typestr)/sizeof(typestr[0])); i++) + if (strcasecmp(in, typestr[i].name) == 0) + return typestr[i].type; + + return 0; +} + +/* Pi-hole function: return name of RR type */ +const char *rrtype_name(unsigned short type) +{ + unsigned int i; + + if(type == 0) + return "OTHER"; + + for (i = 0; i < (sizeof(typestr)/sizeof(typestr[0])); i++) + if (typestr[i].type == type) + return typestr[i].name; + + return NULL; +} + void next_uid(struct crec *crecp) { static unsigned int uid = 0; @@ -264,8 +292,8 @@ static void cache_blockdata_free(struct crec *crecp) { if (!(crecp->flags & F_NEG)) { - if (crecp->flags & F_SRV) - blockdata_free(crecp->addr.srv.target); + if ((crecp->flags & F_RR) && (crecp->flags & F_KEYTAG)) + blockdata_free(crecp->addr.rrblock.rrdata); #ifdef HAVE_DNSSEC else if (crecp->flags & F_DNSKEY) blockdata_free(crecp->addr.key.keydata); @@ -413,18 +441,21 @@ unsigned int cache_remove_uid(const unsigned int uid) { int i; unsigned int removed = 0; - struct crec *crecp, **up; + struct crec *crecp, *tmp, **up; for (i = 0; i < hash_size; i++) - for (crecp = hash_table[i], up = &hash_table[i]; crecp; crecp = crecp->hash_next) - if ((crecp->flags & (F_HOSTS | F_DHCP | F_CONFIG)) && crecp->uid == uid) - { - *up = crecp->hash_next; - free(crecp); - removed++; - } - else - up = &crecp->hash_next; + for (crecp = hash_table[i], up = &hash_table[i]; crecp; crecp = tmp) + { + tmp = crecp->hash_next; + if ((crecp->flags & (F_HOSTS | F_DHCP | F_CONFIG)) && crecp->uid == uid) + { + *up = tmp; + free(crecp); + removed++; + } + else + up = &crecp->hash_next; + } return removed; } @@ -458,9 +489,20 @@ static struct crec *cache_scan_free(char *name, union all_addr *addr, unsigned s { if ((crecp->flags & F_FORWARD) && hostname_isequal(cache_get_name(crecp), name)) { + int rrmatch = 0; + if (crecp->flags & flags & F_RR) + { + unsigned short rrc = (crecp->flags & F_KEYTAG) ? crecp->addr.rrblock.rrtype : crecp->addr.rrdata.rrtype; + unsigned short rra = (flags & F_KEYTAG) ? addr->rrblock.rrtype : addr->rrdata.rrtype; + + if (rrc == rra) + rrmatch = 1; + } + /* Don't delete DNSSEC in favour of a CNAME, they can co-exist */ - if ((flags & crecp->flags & (F_IPV4 | F_IPV6 | F_SRV | F_NXDOMAIN)) || - (((crecp->flags | flags) & F_CNAME) && !(crecp->flags & (F_DNSKEY | F_DS)))) + if ((flags & crecp->flags & (F_IPV4 | F_IPV6 | F_NXDOMAIN)) || + (((crecp->flags | flags) & F_CNAME) && !(crecp->flags & (F_DNSKEY | F_DS))) || + rrmatch) { if (crecp->flags & (F_HOSTS | F_DHCP | F_CONFIG)) return crecp; @@ -607,8 +649,8 @@ static struct crec *really_insert(char *name, union all_addr *addr, unsigned sho if (insert_error) return NULL; - /* we don't cache zero-TTL records. */ - if (ttl == 0) + /* we don't cache zero-TTL records unless we're doing stale-caching. */ + if (daemon->cache_max_expiry == 0 && ttl == 0) { insert_error = 1; return NULL; @@ -776,14 +818,13 @@ void cache_end_insert(void) read_write(daemon->pipe_to_parent, (unsigned char *)name, m, 0); read_write(daemon->pipe_to_parent, (unsigned char *)&new_chain->ttd, sizeof(new_chain->ttd), 0); read_write(daemon->pipe_to_parent, (unsigned char *)&flags, sizeof(flags), 0); - - if (flags & (F_IPV4 | F_IPV6 | F_DNSKEY | F_DS | F_SRV)) - read_write(daemon->pipe_to_parent, (unsigned char *)&new_chain->addr, sizeof(new_chain->addr), 0); - if (flags & F_SRV) + read_write(daemon->pipe_to_parent, (unsigned char *)&new_chain->addr, sizeof(new_chain->addr), 0); + + if (flags & F_RR) { - /* A negative SRV entry is possible and has no data, obviously. */ - if (!(flags & F_NEG)) - blockdata_write(new_chain->addr.srv.target, new_chain->addr.srv.targetlen, daemon->pipe_to_parent); + /* A negative RR entry is possible and has no data, obviously. */ + if (!(flags & F_NEG) && (flags & F_KEYTAG)) + blockdata_write(new_chain->addr.rrblock.rrdata, new_chain->addr.rrblock.datalen, daemon->pipe_to_parent); } #ifdef HAVE_DNSSEC if (flags & F_DNSKEY) @@ -842,41 +883,15 @@ int cache_recv_insert(time_t now, int fd) if (!read_write(fd, (unsigned char *)daemon->namebuff, m, 1) || !read_write(fd, (unsigned char *)&ttd, sizeof(ttd), 1) || - !read_write(fd, (unsigned char *)&flags, sizeof(flags), 1)) + !read_write(fd, (unsigned char *)&flags, sizeof(flags), 1) || + !read_write(fd, (unsigned char *)&addr, sizeof(addr), 1)) return 0; daemon->namebuff[m] = 0; ttl = difftime(ttd, now); - if (flags & (F_IPV4 | F_IPV6 | F_DNSKEY | F_DS | F_SRV)) - { - unsigned short class = C_IN; - - if (!read_write(fd, (unsigned char *)&addr, sizeof(addr), 1)) - return 0; - - if ((flags & F_SRV) && !(flags & F_NEG) && !(addr.srv.target = blockdata_read(fd, addr.srv.targetlen))) - return 0; - -#ifdef HAVE_DNSSEC - if (flags & F_DNSKEY) - { - if (!read_write(fd, (unsigned char *)&class, sizeof(class), 1) || - !(addr.key.keydata = blockdata_read(fd, addr.key.keylen))) - return 0; - } - else if (flags & F_DS) - { - if (!read_write(fd, (unsigned char *)&class, sizeof(class), 1) || - (!(flags & F_NEG) && !(addr.key.keydata = blockdata_read(fd, addr.key.keylen)))) - return 0; - } -#endif - - crecp = really_insert(daemon->namebuff, &addr, class, now, ttl, flags); - } - else if (flags & F_CNAME) + if (flags & F_CNAME) { struct crec *newc = really_insert(daemon->namebuff, NULL, C_IN, now, ttl, flags); /* This relies on the fact that the target of a CNAME immediately precedes @@ -884,11 +899,11 @@ int cache_recv_insert(time_t now, int fd) the order reversal on the new_chain. */ if (newc) { - newc->addr.cname.is_name_ptr = 0; - - if (!crecp) - newc->addr.cname.target.cache = NULL; - else + newc->addr.cname.is_name_ptr = 0; + + if (!crecp) + newc->addr.cname.target.cache = NULL; + else { next_uid(crecp); newc->addr.cname.target.cache = crecp; @@ -896,6 +911,29 @@ int cache_recv_insert(time_t now, int fd) } } } + else + { + unsigned short class = C_IN; + + if ((flags & F_RR) && !(flags & F_NEG) && (flags & F_KEYTAG) + && !(addr.rrblock.rrdata = blockdata_read(fd, addr.rrblock.datalen))) + return 0; +#ifdef HAVE_DNSSEC + if (flags & F_DNSKEY) + { + if (!read_write(fd, (unsigned char *)&class, sizeof(class), 1) || + !(addr.key.keydata = blockdata_read(fd, addr.key.keylen))) + return 0; + } + else if (flags & F_DS) + { + if (!read_write(fd, (unsigned char *)&class, sizeof(class), 1) || + (!(flags & F_NEG) && !(addr.key.keydata = blockdata_read(fd, addr.key.keylen)))) + return 0; + } +#endif + crecp = really_insert(daemon->namebuff, &addr, class, now, ttl, flags); + } } } @@ -1588,7 +1626,7 @@ static void make_non_terminals(struct crec *source) if (!is_outdated_cname_pointer(crecp) && (crecp->flags & F_FORWARD) && (crecp->flags & type) && - !(crecp->flags & (F_IPV4 | F_IPV6 | F_CNAME | F_SRV | F_DNSKEY | F_DS)) && + !(crecp->flags & (F_IPV4 | F_IPV6 | F_CNAME | F_DNSKEY | F_DS | F_RR)) && hostname_isequal(name, cache_get_name(crecp))) { *up = crecp->hash_next; @@ -1645,7 +1683,7 @@ static void make_non_terminals(struct crec *source) if (crecp) { - crecp->flags = (source->flags | F_NAMEP) & ~(F_IPV4 | F_IPV6 | F_CNAME | F_SRV | F_DNSKEY | F_DS | F_REVERSE); + crecp->flags = (source->flags | F_NAMEP) & ~(F_IPV4 | F_IPV6 | F_CNAME | F_RR | F_DNSKEY | F_DS | F_REVERSE); if (!(crecp->flags & F_IMMORTAL)) crecp->ttd = source->ttd; crecp->name.namep = name; @@ -1697,12 +1735,6 @@ int cache_make_stat(struct txt_record *t) break; #endif - /* Pi-hole modification */ - case TXT_PRIVACYLEVEL: - sprintf(buff+1, "%d", *pihole_privacylevel); - break; - /* -------------------- */ - case TXT_STAT_SERVERS: /* sum counts from different records for same server */ for (serv = daemon->servers; serv; serv = serv->next) @@ -1783,21 +1815,27 @@ static void dump_cache_entry(struct crec *cache, time_t now) p = buff; *a = 0; - if (strlen(n) == 0 && !(cache->flags & F_REVERSE)) - n = ""; + + if (cache->flags & F_REVERSE) + { + if ((cache->flags & F_NEG)) + n = ""; + } + else + { + if (strlen(n) == 0) + n = ""; + } + p += sprintf(p, "%-30.30s ", sanitise(n)); if ((cache->flags & F_CNAME) && !is_outdated_cname_pointer(cache)) a = sanitise(cache_get_cname_target(cache)); - else if ((cache->flags & F_SRV) && !(cache->flags & F_NEG)) + else if (cache->flags & F_RR) { - int targetlen = cache->addr.srv.targetlen; - ssize_t len = sprintf(a, "%u %u %u ", cache->addr.srv.priority, - cache->addr.srv.weight, cache->addr.srv.srvport); - - if (targetlen > (40 - len)) - targetlen = 40 - len; - blockdata_retrieve(cache->addr.srv.target, targetlen, a + len); - a[len + targetlen] = 0; + if (cache->flags & F_KEYTAG) + sprintf(a, "%s", querystr(NULL, cache->addr.rrblock.rrtype)); + else + sprintf(a, "%s", querystr(NULL, cache->addr.rrdata.rrtype)); } #ifdef HAVE_DNSSEC else if (cache->flags & F_DS) @@ -1825,8 +1863,8 @@ static void dump_cache_entry(struct crec *cache, time_t now) t = "6"; else if (cache->flags & F_CNAME) t = "C"; - else if (cache->flags & F_SRV) - t = "V"; + else if (cache->flags & F_RR) + t = "T"; #ifdef HAVE_DNSSEC else if (cache->flags & F_DS) t = "S"; @@ -1872,8 +1910,6 @@ void get_dnsmasq_cache_info(struct cache_info *ci) ci->valid.ipv6++; else if (cache->flags & F_CNAME) ci->valid.cname++; - else if (cache->flags & F_SRV) - ci->valid.srv++; #ifdef HAVE_DNSSEC else if (cache->flags & F_DS) ci->valid.ds++; @@ -1907,7 +1943,12 @@ void dump_cache(time_t now) #endif blockdata_report(); - + my_syslog(LOG_INFO, _("child processes for TCP requests: in use %zu, highest since last SIGUSR1 %zu, max allowed %zu."), + daemon->metrics[METRIC_TCP_CONNECTIONS], + daemon->max_procs_used, + daemon->max_procs); + daemon->max_procs_used = daemon->metrics[METRIC_TCP_CONNECTIONS]; + /* sum counts from different records for same server */ for (serv = daemon->servers; serv; serv = serv->next) serv->flags &= ~SERV_MARK; @@ -2096,7 +2137,14 @@ void _log_query(unsigned int flags, char *name, union all_addr *addr, char *arg, { dest = daemon->addrbuff; - if (flags & F_KEYTAG) + if (flags & F_RR) + { + if (flags & F_KEYTAG) + dest = querystr(NULL, addr->rrblock.rrtype); + else + dest = querystr(NULL, addr->rrdata.rrtype); + } + else if (flags & F_KEYTAG) sprintf(daemon->addrbuff, arg, addr->log.keytag, addr->log.algo, addr->log.digest); else if (flags & F_RCODE) { @@ -2153,8 +2201,6 @@ void _log_query(unsigned int flags, char *name, union all_addr *addr, char *arg, } else if (flags & F_CNAME) dest = ""; - else if (flags & F_SRV) - dest = ""; else if (flags & F_RRNAME) dest = arg; diff --git a/src/dnsmasq/config.h b/src/dnsmasq/config.h index 12562901..f545176f 100644 --- a/src/dnsmasq/config.h +++ b/src/dnsmasq/config.h @@ -1,4 +1,4 @@ -/* dnsmasq is Copyright (c) 2000-2022 Simon Kelley +/* dnsmasq is Copyright (c) 2000-2024 Simon Kelley This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -15,7 +15,7 @@ */ #define FTABSIZ 150 /* max number of outstanding requests (default) */ -#define MAX_PROCS 60 /* max no children for TCP requests */ +#define MAX_PROCS 60 /* default max no children for TCP requests */ #define CHILD_LIFETIME 300 /* secs 'till terminated (RFC1035 suggests > 120s) */ #define TCP_MAX_QUERIES 100 /* Maximum number of queries per incoming TCP connection */ #define TCP_BACKLOG 32 /* kernel backlog limit for TCP connections */ diff --git a/src/dnsmasq/conntrack.c b/src/dnsmasq/conntrack.c index fe48f2b1..5cc81680 100644 --- a/src/dnsmasq/conntrack.c +++ b/src/dnsmasq/conntrack.c @@ -1,4 +1,4 @@ -/* dnsmasq is Copyright (c) 2000-2022 Simon Kelley +/* dnsmasq is Copyright (c) 2000-2024 Simon Kelley This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by diff --git a/src/dnsmasq/crypto.c b/src/dnsmasq/crypto.c index 2678683e..abc744a6 100644 --- a/src/dnsmasq/crypto.c +++ b/src/dnsmasq/crypto.c @@ -1,4 +1,4 @@ -/* dnsmasq is Copyright (c) 2000-2022 Simon Kelley +/* dnsmasq is Copyright (c) 2000-2024 Simon Kelley This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by diff --git a/src/dnsmasq/dbus.c b/src/dnsmasq/dbus.c index 4366b7ea..ad6a4f37 100644 --- a/src/dnsmasq/dbus.c +++ b/src/dnsmasq/dbus.c @@ -1,4 +1,4 @@ -/* dnsmasq is Copyright (c) 2000-2022 Simon Kelley +/* dnsmasq is Copyright (c) 2000-2024 Simon Kelley This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -106,6 +106,7 @@ const char* introspection_xml_template = "\n"; static char *introspection_xml = NULL; +static int watches_modified = 0; struct watch { DBusWatch *watch; @@ -127,6 +128,7 @@ static dbus_bool_t add_watch(DBusWatch *watch, void *data) w->watch = watch; w->next = daemon->watches; daemon->watches = w; + watches_modified++; (void)data; /* no warning */ return TRUE; @@ -134,7 +136,7 @@ static dbus_bool_t add_watch(DBusWatch *watch, void *data) static void remove_watch(DBusWatch *watch, void *data) { - struct watch **up, *w, *tmp; + struct watch **up, *w, *tmp; for (up = &(daemon->watches), w = daemon->watches; w; w = tmp) { @@ -143,6 +145,7 @@ static void remove_watch(DBusWatch *watch, void *data) { *up = tmp; free(w); + watches_modified++; } else up = &(w->next); @@ -825,11 +828,25 @@ DBusHandlerResult message_handler(DBusConnection *connection, } else if (strcmp(method, "SetFilterA") == 0) { - reply = dbus_set_bool(message, OPT_FILTER_A, "filter-A"); + static int done = 0; + static struct rrlist list = { T_A, NULL }; + + if (!done) + { + list.next = daemon->filter_rr; + daemon->filter_rr = &list; + } } else if (strcmp(method, "SetFilterAAAA") == 0) { - reply = dbus_set_bool(message, OPT_FILTER_AAAA, "filter-AAAA"); + static int done = 0; + static struct rrlist list = { T_AAAA, NULL }; + + if (!done) + { + list.next = daemon->filter_rr; + daemon->filter_rr = &list; + } } else if (strcmp(method, "SetLocaliseQueriesOption") == 0) { @@ -941,41 +958,53 @@ void set_dbus_listeners(void) { unsigned int flags = dbus_watch_get_flags(w->watch); int fd = dbus_watch_get_unix_fd(w->watch); + int poll_flags = POLLERR; if (flags & DBUS_WATCH_READABLE) - poll_listen(fd, POLLIN); - + poll_flags |= POLLIN; if (flags & DBUS_WATCH_WRITABLE) - poll_listen(fd, POLLOUT); + poll_flags |= POLLOUT; - poll_listen(fd, POLLERR); + poll_listen(fd, poll_flags); } } -void check_dbus_listeners() +static int check_dbus_watches() { - DBusConnection *connection = (DBusConnection *)daemon->dbus; struct watch *w; + watches_modified = 0; for (w = daemon->watches; w; w = w->next) if (dbus_watch_get_enabled(w->watch)) { unsigned int flags = 0; int fd = dbus_watch_get_unix_fd(w->watch); - - if (poll_check(fd, POLLIN)) + int poll_flags = poll_check(fd, POLLIN|POLLOUT|POLLERR); + + if ((poll_flags & POLLIN) != 0) flags |= DBUS_WATCH_READABLE; - - if (poll_check(fd, POLLOUT)) + if ((poll_flags & POLLOUT) != 0) flags |= DBUS_WATCH_WRITABLE; - - if (poll_check(fd, POLLERR)) + if ((poll_flags & POLLERR) != 0) flags |= DBUS_WATCH_ERROR; if (flags != 0) - dbus_watch_handle(w->watch, flags); + { + dbus_watch_handle(w->watch, flags); + if (watches_modified) + return 0; + } } + return 1; +} + +void check_dbus_listeners() +{ + DBusConnection *connection = (DBusConnection *)daemon->dbus; + + while (!check_dbus_watches()) ; + if (connection) { dbus_connection_ref (connection); diff --git a/src/dnsmasq/dhcp-common.c b/src/dnsmasq/dhcp-common.c index 7e2abef4..48d6563d 100644 --- a/src/dnsmasq/dhcp-common.c +++ b/src/dnsmasq/dhcp-common.c @@ -1,4 +1,4 @@ -/* dnsmasq is Copyright (c) 2000-2022 Simon Kelley +/* dnsmasq is Copyright (c) 2000-2024 Simon Kelley This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -553,11 +553,11 @@ char *whichdevice(void) return NULL; for (if_tmp = daemon->if_names; if_tmp; if_tmp = if_tmp->next) - if (if_tmp->name && (!if_tmp->used || strchr(if_tmp->name, '*'))) + if (if_tmp->name && (!(if_tmp->flags & INAME_USED) || strchr(if_tmp->name, '*'))) return NULL; for (found = NULL, iface = daemon->interfaces; iface; iface = iface->next) - if (iface->dhcp_ok) + if (iface->dhcp4_ok || iface->dhcp6_ok) { if (!found) found = iface; diff --git a/src/dnsmasq/dhcp-protocol.h b/src/dnsmasq/dhcp-protocol.h index e281143a..3dde3543 100644 --- a/src/dnsmasq/dhcp-protocol.h +++ b/src/dnsmasq/dhcp-protocol.h @@ -1,4 +1,4 @@ -/* dnsmasq is Copyright (c) 2000-2022 Simon Kelley +/* dnsmasq is Copyright (c) 2000-2024 Simon Kelley This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by diff --git a/src/dnsmasq/dhcp.c b/src/dnsmasq/dhcp.c index e5783918..b65facd8 100644 --- a/src/dnsmasq/dhcp.c +++ b/src/dnsmasq/dhcp.c @@ -1,4 +1,4 @@ -/* dnsmasq is Copyright (c) 2000-2022 Simon Kelley +/* dnsmasq is Copyright (c) 2000-2024 Simon Kelley This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -297,7 +297,7 @@ void dhcp_packet(time_t now, int pxe_fd) } for (tmp = daemon->dhcp_except; tmp; tmp = tmp->next) - if (tmp->name && wildcard_match(tmp->name, ifr.ifr_name)) + if (tmp->name && (tmp->flags & INAME_4) && wildcard_match(tmp->name, ifr.ifr_name)) return; /* unlinked contexts/relays are marked by context->current == context */ diff --git a/src/dnsmasq/dhcp6-protocol.h b/src/dnsmasq/dhcp6-protocol.h index ce166037..a23adac5 100644 --- a/src/dnsmasq/dhcp6-protocol.h +++ b/src/dnsmasq/dhcp6-protocol.h @@ -1,4 +1,4 @@ -/* dnsmasq is Copyright (c) 2000-2022 Simon Kelley +/* dnsmasq is Copyright (c) 2000-2024 Simon Kelley This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by diff --git a/src/dnsmasq/dhcp6.c b/src/dnsmasq/dhcp6.c index 1c8c6794..c9d54dcd 100644 --- a/src/dnsmasq/dhcp6.c +++ b/src/dnsmasq/dhcp6.c @@ -1,4 +1,4 @@ -/* dnsmasq is Copyright (c) 2000-2022 Simon Kelley +/* dnsmasq is Copyright (c) 2000-2024 Simon Kelley This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -92,7 +92,7 @@ void dhcp6_packet(time_t now) struct iface_param parm; struct cmsghdr *cmptr; struct msghdr msg; - int if_index = 0; + uint32_t if_index = 0; union { struct cmsghdr align; /* this ensures alignment */ char control6[CMSG_SPACE(sizeof(struct in6_pktinfo))]; @@ -118,11 +118,6 @@ void dhcp6_packet(time_t now) if ((sz = recv_dhcp_packet(daemon->dhcp6fd, &msg)) == -1) return; -#ifdef HAVE_DUMPFILE - dump_packet_udp(DUMP_DHCPV6, (void *)daemon->dhcp_packet.iov_base, sz, - (union mysockaddr *)&from, NULL, daemon->dhcp6fd); -#endif - for (cmptr = CMSG_FIRSTHDR(&msg); cmptr; cmptr = CMSG_NXTHDR(&msg, cmptr)) if (cmptr->cmsg_level == IPPROTO_IPV6 && cmptr->cmsg_type == daemon->v6pktinfo) { @@ -138,6 +133,34 @@ void dhcp6_packet(time_t now) if (!indextoname(daemon->dhcp6fd, if_index, ifr.ifr_name)) return; + +#ifdef HAVE_LINUX_NETWORK + /* This works around a possible Linux kernel bug when using interfaces + enslaved to a VRF. The scope_id in the source address gets set + to the index of the VRF interface, not the slave. Fortunately, + the interface index returned by packetinfo is correct so we use + that instead. Log this once, so if it triggers in other circumstances + we've not anticipated and breaks things, we get some clues. */ + if (from.sin6_scope_id != if_index) + { + static int logged = 0; + + if (!logged) + { + my_syslog(MS_DHCP | LOG_WARNING, + _("Working around kernel bug: faulty source address scope for VRF slave %s"), + ifr.ifr_name); + logged = 1; + } + + from.sin6_scope_id = if_index; + } +#endif + +#ifdef HAVE_DUMPFILE + dump_packet_udp(DUMP_DHCPV6, (void *)daemon->dhcp_packet.iov_base, sz, + (union mysockaddr *)&from, NULL, daemon->dhcp6fd); +#endif if (relay_reply6(&from, sz, ifr.ifr_name)) { @@ -159,7 +182,8 @@ void dhcp6_packet(time_t now) return; for (tmp = daemon->dhcp_except; tmp; tmp = tmp->next) - if (tmp->name && wildcard_match(tmp->name, ifr.ifr_name)) + if (tmp->name && (tmp->flags & INAME_6) && + wildcard_match(tmp->name, ifr.ifr_name)) return; parm.current = NULL; diff --git a/src/dnsmasq/dns-protocol.h b/src/dnsmasq/dns-protocol.h index 8558c33b..0671adf2 100644 --- a/src/dnsmasq/dns-protocol.h +++ b/src/dnsmasq/dns-protocol.h @@ -1,4 +1,4 @@ -/* dnsmasq is Copyright (c) 2000-2022 Simon Kelley +/* dnsmasq is Copyright (c) 2000-2024 Simon Kelley This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by diff --git a/src/dnsmasq/dnsmasq.c b/src/dnsmasq/dnsmasq.c index 59bd4c15..dccbb7ab 100644 --- a/src/dnsmasq/dnsmasq.c +++ b/src/dnsmasq/dnsmasq.c @@ -1,4 +1,4 @@ -/* dnsmasq is Copyright (c) 2000-2022 Simon Kelley +/* dnsmasq is Copyright (c) 2000-2024 Simon Kelley This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -23,23 +23,27 @@ #if defined(HAVE_IDN) || defined(HAVE_LIBIDN2) || defined(LOCALEDIR) #include #endif -#include "../dnsmasq_interface.h" +#include "dnsmasq_interface.h" // killed -#include "../signals.h" +#include "signals.h" +// FTL_fork_and_bind_sockets() +#include "main.h" struct daemon *daemon; static volatile pid_t pid = 0; static volatile int pipewrite; -static char terminate = 0; +volatile char FTL_terminate = 0; static void set_dns_listeners(void); +static void set_tftp_listeners(void); static void check_dns_listeners(time_t now); static void sig_handler(int sig); static void async_event(int pipe, time_t now); static void fatal_event(struct event_desc *ev, char *msg); static int read_event(int fd, struct event_desc *evp, char **msg); static void poll_resolv(int force, int do_reload, time_t now); +static void tcp_init(void); int main_dnsmasq (int argc, char **argv) { @@ -79,10 +83,7 @@ int main_dnsmasq (int argc, char **argv) #endif #if defined(HAVE_IDN) || defined(HAVE_LIBIDN2) || defined(LOCALEDIR) - setlocale(LC_ALL, ""); - /*** Pi-hole modification ***/ - setlocale(LC_NUMERIC, "C"); - /****************************/ + /*** Pi-hole modification: Locale is already initialized in main.c ***/ #endif #ifdef LOCALEDIR bindtextdomain("dnsmasq", LOCALEDIR); @@ -95,7 +96,7 @@ int main_dnsmasq (int argc, char **argv) sigaction(SIGUSR1, &sigact, NULL); sigaction(SIGUSR2, &sigact, NULL); sigaction(SIGHUP, &sigact, NULL); - sigaction(SIGTERM, &sigact, NULL); + sigaction(SIGUSR6, &sigact, NULL); // Pi-hole modification sigaction(SIGALRM, &sigact, NULL); sigaction(SIGCHLD, &sigact, NULL); sigaction(SIGINT, &sigact, NULL); @@ -132,29 +133,15 @@ int main_dnsmasq (int argc, char **argv) { /* Note that both /000 and '.' are allowed within labels. These get represented in presentation format using NAME_ESCAPE as an escape - character when in DNSSEC mode. - In theory, if all the characters in a name were /000 or + character. In theory, if all the characters in a name were /000 or '.' or NAME_ESCAPE then all would have to be escaped, so the - presentation format would be twice as long as the spec. - - daemon->namebuff was previously allocated by the option-reading - code before we knew if we're in DNSSEC mode, so reallocate here. */ - free(daemon->namebuff); - daemon->namebuff = safe_malloc(MAXDNAME * 2); - daemon->keyname = safe_malloc(MAXDNAME * 2); - daemon->workspacename = safe_malloc(MAXDNAME * 2); + presentation format would be twice as long as the spec. */ + daemon->keyname = safe_malloc((MAXDNAME * 2) + 1); /* one char flag per possible RR in answer section (may get extended). */ daemon->rr_status_sz = 64; daemon->rr_status = safe_malloc(sizeof(*daemon->rr_status) * daemon->rr_status_sz); } #endif - -#if defined(HAVE_CONNTRACK) && defined(HAVE_UBUS) - /* CONNTRACK UBUS code uses this buffer, so if not allocated above, - we need to allocate it here. */ - if (option_bool(OPT_CMARK_ALST_EN) && !daemon->workspacename) - daemon->workspacename = safe_malloc(MAXDNAME); -#endif #ifdef HAVE_DHCP if (!daemon->lease_file) @@ -385,6 +372,13 @@ int main_dnsmasq (int argc, char **argv) if (!enumerate_interfaces(1) || !enumerate_interfaces(0)) die(_("failed to find list of interfaces: %s"), NULL, EC_MISC); + +#ifdef HAVE_DHCP + /* Determine lease FQDNs after enumerate_interfaces() call, since it needs + to call get_domain and that's only valid for some domain configs once we + have interface addresses. */ + lease_calc_fqdns(); +#endif if (option_bool(OPT_NOWILD) || option_bool(OPT_CLEVERBIND)) { @@ -392,7 +386,7 @@ int main_dnsmasq (int argc, char **argv) if (!option_bool(OPT_CLEVERBIND)) for (if_tmp = daemon->if_names; if_tmp; if_tmp = if_tmp->next) - if (if_tmp->name && !if_tmp->used) + if (if_tmp->name && !(if_tmp->flags & INAME_USED)) die(_("unknown interface %s"), if_tmp->name, EC_BADNET); #if defined(HAVE_LINUX_NETWORK) && defined(HAVE_DHCP) @@ -428,11 +422,13 @@ int main_dnsmasq (int argc, char **argv) daemon->numrrand = max_fd/3; /* safe_malloc returns zero'd memory */ daemon->randomsocks = safe_malloc(daemon->numrrand * sizeof(struct randfd)); + + tcp_init(); } #ifdef HAVE_INOTIFY - if ((daemon->port != 0 || daemon->dhcp || daemon->doing_dhcp6) - && (!option_bool(OPT_NO_RESOLV) || daemon->dynamic_dirs)) + if ((daemon->port != 0 && !option_bool(OPT_NO_RESOLV)) || + daemon->dynamic_dirs) inotify_dnsmasq_init(); else daemon->inotifyfd = -1; @@ -872,6 +868,8 @@ int main_dnsmasq (int argc, char **argv) if (option_bool(OPT_LOCAL_SERVICE)) my_syslog(LOG_INFO, _("DNS service limited to local subnets")); + else if (option_bool(OPT_LOCALHOST_SERVICE)) + my_syslog(LOG_INFO, _("DNS service limited to localhost")); } my_syslog(LOG_INFO, _("compile time options: %s"), compile_opts); @@ -950,7 +948,7 @@ int main_dnsmasq (int argc, char **argv) if (!option_bool(OPT_NOWILD)) for (if_tmp = daemon->if_names; if_tmp; if_tmp = if_tmp->next) - if (if_tmp->name && !if_tmp->used) + if (if_tmp->name && !(if_tmp->flags & INAME_USED)) my_syslog(LOG_WARNING, _("warning: interface %s does not currently exist"), if_tmp->name); if (daemon->port != 0 && option_bool(OPT_NO_RESOLV)) @@ -1058,8 +1056,10 @@ int main_dnsmasq (int argc, char **argv) pid = getpid(); daemon->pipe_to_parent = -1; - for (i = 0; i < MAX_PROCS; i++) - daemon->tcp_pipes[i] = -1; + + if (daemon->port != 0) + for (i = 0; i < daemon->max_procs; i++) + daemon->tcp_pipes[i] = -1; #ifdef HAVE_INOTIFY /* Using inotify, have to select a resolv file at startup */ @@ -1067,10 +1067,10 @@ int main_dnsmasq (int argc, char **argv) #endif /*** Pi-hole modification ***/ - terminate = killed; + FTL_terminate = killed; /****************************/ - while (!terminate) + while (!FTL_terminate) { int timeout = fast_retry(now); @@ -1086,7 +1086,12 @@ int main_dnsmasq (int argc, char **argv) (timeout == -1 || timeout > 1000)) timeout = 1000; - set_dns_listeners(); + if (daemon->port != 0) + set_dns_listeners(); + +#ifdef HAVE_TFTP + set_tftp_listeners(); +#endif #ifdef HAVE_DBUS if (option_bool(OPT_DBUS)) @@ -1271,8 +1276,9 @@ int main_dnsmasq (int argc, char **argv) check_ubus_listeners(); } #endif - - check_dns_listeners(now); + + if (daemon->port != 0) + check_dns_listeners(now); #ifdef HAVE_TFTP check_tftp_listeners(now); @@ -1337,7 +1343,7 @@ static void sig_handler(int sig) event = EVENT_CHILD; else if (sig == SIGALRM) event = EVENT_ALARM; - else if (sig == SIGTERM) + else if (sig == SIGUSR6) // Pi-hole modified event = EVENT_TERM; else if (sig == SIGUSR1) event = EVENT_DUMP; @@ -1545,10 +1551,15 @@ static void async_event(int pipe, time_t now) if (errno != EINTR) break; } - else - for (i = 0 ; i < MAX_PROCS; i++) + else if (daemon->port != 0) + for (i = 0 ; i < daemon->max_procs; i++) if (daemon->tcp_pids[i] == p) - daemon->tcp_pids[i] = 0; + { + daemon->tcp_pids[i] = 0; + /* tcp_pipes == -1 && tcp_pids == 0 required to free slot */ + if (daemon->tcp_pipes[i] == -1) + daemon->metrics[METRIC_TCP_CONNECTIONS]--; + } break; #if defined(HAVE_SCRIPT) @@ -1611,9 +1622,10 @@ static void async_event(int pipe, time_t now) case EVENT_TERM: /* Knock all our children on the head. */ - for (i = 0; i < MAX_PROCS; i++) - if (daemon->tcp_pids[i] != 0) - kill(daemon->tcp_pids[i], SIGALRM); + if (daemon->port != 0) + for (i = 0; i < daemon->max_procs; i++) + if (daemon->tcp_pids[i] != 0) + kill(daemon->tcp_pids[i], SIGALRM); #if defined(HAVE_SCRIPT) && defined(HAVE_DHCP) /* handle pending lease transitions */ @@ -1653,7 +1665,7 @@ static void async_event(int pipe, time_t now) flush_log(); /*** Pi-hole modification ***/ // exit(EC_GOOD); - terminate = 1; + FTL_terminate = 1; /*** Pi-hole modification ***/ } } @@ -1763,23 +1775,33 @@ void clear_cache_and_reload(time_t now) #endif } -static void set_dns_listeners(void) -{ - struct serverfd *serverfdp; - struct listener *listener; - struct randfd_list *rfl; - int i; - #ifdef HAVE_TFTP +static void set_tftp_listeners(void) +{ int tftp = 0; struct tftp_transfer *transfer; + struct listener *listener; + if (!option_bool(OPT_SINGLE_PORT)) for (transfer = daemon->tftp_trans; transfer; transfer = transfer->next) { tftp++; poll_listen(transfer->sockfd, POLLIN); } + + for (listener = daemon->listeners; listener; listener = listener->next) + /* tftp == 0 in single-port mode. */ + if (tftp <= daemon->tftp_max && listener->tftpfd != -1) + poll_listen(listener->tftpfd, POLLIN); +} #endif + +static void set_dns_listeners(void) +{ + struct serverfd *serverfdp; + struct listener *listener; + struct randfd_list *rfl; + int i; for (serverfdp = daemon->sfds; serverfdp; serverfdp = serverfdp->next) poll_listen(serverfdp->fd, POLLIN); @@ -1793,7 +1815,7 @@ static void set_dns_listeners(void) poll_listen(rfl->rfd->fd, POLLIN); /* check to see if we have free tcp process slots. */ - for (i = MAX_PROCS - 1; i >= 0; i--) + for (i = daemon->max_procs - 1; i >= 0; i--) if (daemon->tcp_pids[i] == 0 && daemon->tcp_pipes[i] == -1) break; @@ -1808,16 +1830,10 @@ static void set_dns_listeners(void) we'll be called again when a slot becomes available. */ if (listener->tcpfd != -1 && i >= 0) poll_listen(listener->tcpfd, POLLIN); - -#ifdef HAVE_TFTP - /* tftp == 0 in single-port mode. */ - if (tftp <= daemon->tftp_max && listener->tftpfd != -1) - poll_listen(listener->tftpfd, POLLIN); -#endif } if (!option_bool(OPT_DEBUG)) - for (i = 0; i < MAX_PROCS; i++) + for (i = 0; i < daemon->max_procs; i++) if (daemon->tcp_pipes[i] != -1) poll_listen(daemon->tcp_pipes[i], POLLIN); } @@ -1852,13 +1868,16 @@ static void check_dns_listeners(time_t now) to free the process slot. Once the child process has gone, poll() returns POLLHUP, not POLLIN, so have to check for both here. */ if (!option_bool(OPT_DEBUG)) - for (i = 0; i < MAX_PROCS; i++) + for (i = 0; i < daemon->max_procs; i++) if (daemon->tcp_pipes[i] != -1 && poll_check(daemon->tcp_pipes[i], POLLIN | POLLHUP) && !cache_recv_insert(now, daemon->tcp_pipes[i])) { close(daemon->tcp_pipes[i]); daemon->tcp_pipes[i] = -1; + /* tcp_pipes == -1 && tcp_pids == 0 required to free slot */ + if (daemon->tcp_pids[i] == 0) + daemon->metrics[METRIC_TCP_CONNECTIONS]--; } for (listener = daemon->listeners; listener; listener = listener->next) @@ -1867,17 +1886,12 @@ static void check_dns_listeners(time_t now) if (listener->fd != -1 && poll_check(listener->fd, POLLIN)) receive_query(listener, now); -#ifdef HAVE_TFTP - if (listener->tftpfd != -1 && poll_check(listener->tftpfd, POLLIN)) - tftp_request(listener, now); -#endif - /* check to see if we have a free tcp process slot. Note that we can't assume that because we had at least one a poll() time, that we still do. There may be more waiting connections after poll() returns then free process slots. */ - for (i = MAX_PROCS - 1; i >= 0; i--) + for (i = daemon->max_procs - 1; i >= 0; i--) if (daemon->tcp_pids[i] == 0 && daemon->tcp_pipes[i] == -1) break; @@ -1993,6 +2007,9 @@ static void check_dns_listeners(time_t now) /* i holds index of free slot */ daemon->tcp_pids[i] = p; daemon->tcp_pipes[i] = pipefd[0]; + daemon->metrics[METRIC_TCP_CONNECTIONS]++; + if (daemon->metrics[METRIC_TCP_CONNECTIONS] > daemon->max_procs_used) + daemon->max_procs_used = daemon->metrics[METRIC_TCP_CONNECTIONS]; } close(confd); @@ -2178,7 +2195,11 @@ int delay_dhcp(time_t start, int sec, int fd, uint32_t addr, unsigned short id) poll_reset(); if (fd != -1) poll_listen(fd, POLLIN); - set_dns_listeners(); + if (daemon->port != 0) + set_dns_listeners(); +#ifdef HAVE_TFTP + set_tftp_listeners(); +#endif set_log_writer(); #ifdef HAVE_DHCP6 @@ -2196,7 +2217,8 @@ int delay_dhcp(time_t start, int sec, int fd, uint32_t addr, unsigned short id) now = dnsmasq_time(); check_log_writer(0); - check_dns_listeners(now); + if (daemon->port != 0) + check_dns_listeners(now); #ifdef HAVE_DHCP6 if (daemon->doing_ra && poll_check(daemon->icmp6fd, POLLIN)) @@ -2239,3 +2261,9 @@ void print_dnsmasq_version(const char *yellow, const char *green, const char *bo printf(_("Features: %s\n\n"), compile_opts); } /**************************************************************************************/ + +void tcp_init(void) +{ + daemon->tcp_pids = safe_malloc(daemon->max_procs*sizeof(pid_t)); + daemon->tcp_pipes = safe_malloc(daemon->max_procs*sizeof(int)); +} diff --git a/src/dnsmasq/dnsmasq.h b/src/dnsmasq/dnsmasq.h index 2883b8d6..8447206d 100644 --- a/src/dnsmasq/dnsmasq.h +++ b/src/dnsmasq/dnsmasq.h @@ -1,4 +1,4 @@ -/* dnsmasq is Copyright (c) 2000-2022 Simon Kelley +/* dnsmasq is Copyright (c) 2000-2024 Simon Kelley This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -14,7 +14,7 @@ along with this program. If not, see . */ -#define COPYRIGHT "Copyright (c) 2000-2022 Simon Kelley" +#define COPYRIGHT "Copyright (c) 2000-2024 Simon Kelley" /* We do defines that influence behavior of stdio.h, so complain if included too early. */ @@ -276,12 +276,12 @@ struct event_desc { #define OPT_UMBRELLA_DEVID 64 #define OPT_CMARK_ALST_EN 65 #define OPT_QUIET_TFTP 66 -#define OPT_FILTER_A 67 -#define OPT_FILTER_AAAA 68 -#define OPT_STRIP_ECS 69 -#define OPT_STRIP_MAC 70 -#define OPT_NORR 71 -#define OPT_NO_IDENT 72 +#define OPT_STRIP_ECS 67 +#define OPT_STRIP_MAC 68 +#define OPT_NORR 69 +#define OPT_NO_IDENT 70 +#define OPT_CACHE_RR 71 +#define OPT_LOCALHOST_SERVICE 72 #define OPT_LAST 73 #define OPTION_BITS (sizeof(unsigned int)*8) @@ -325,17 +325,28 @@ union all_addr { unsigned char algo; unsigned char digest; } ds; - struct { - struct blockdata *target; - unsigned short targetlen, srvport, priority, weight; - } srv; /* for log_query */ struct { unsigned short keytag, algo, digest, rcode; int ede; } log; + /* for arbitrary RR record stored in block */ + struct { + unsigned short rrtype; + unsigned short datalen; + struct blockdata *rrdata; + } rrblock; + /* for arbitrary RR record small enough to go in addr. + NOTE: rrblock and rrdata are discriminated by the F_KEYTAG bit + in the cache flags. */ + struct datablock { + unsigned short rrtype; + unsigned char datalen; /* also length of SOA in negative records. */ + char data[]; + } rrdata; }; +#define RR_IMDATALEN (sizeof(union all_addr) - offsetof(struct datablock, data)) struct bogus_addr { int is6, prefix; @@ -371,7 +382,8 @@ struct naptr { #define TXT_STAT_AUTH 6 #define TXT_STAT_SERVERS 7 /* Pi-hole modification */ -#define TXT_PRIVACYLEVEL 123 +#define TXT_API_DOMAIN 124 +#define TXT_API_LOCAL 125 /************************/ #endif @@ -515,7 +527,7 @@ struct crec { #define F_NOEXTRA (1u<<27) #define F_DOMAINSRV (1u<<28) #define F_RCODE (1u<<29) -#define F_SRV (1u<<30) +#define F_RR (1u<<30) #define F_STALE (1u<<31) #define UID_NONE 0 @@ -640,7 +652,8 @@ struct allowlist { struct irec { union mysockaddr addr; struct in_addr netmask; /* only valid for IPv4 */ - int tftp_ok, dhcp_ok, mtu, done, warned, dad, dns_auth, index, multicast_done, found, label; + int tftp_ok, dhcp4_ok, dhcp6_ok, mtu, done, warned, dad; + int dns_auth, index, multicast_done, found, label; char *name; /* Pi-hole modification */ char *slabel; @@ -659,10 +672,19 @@ struct listener { struct iname { char *name; union mysockaddr addr; - int used; + int flags; struct iname *next; }; +#define INAME_USED 1 +#define INAME_4 2 +#define INAME_6 4 + +struct rrlist { + unsigned short rr; + struct rrlist *next; +}; + /* subnet parameters from command line */ struct mysubnet { union mysockaddr addr; @@ -1128,6 +1150,7 @@ extern struct daemon { struct naptr *naptr; struct txt_record *txt, *rr; struct ptr_record *ptr; + struct rrlist *cache_rr, *filter_rr; struct host_record *host_records, *host_records_tail; struct cname *cnames; struct auth_zone *auth_zones; @@ -1216,10 +1239,7 @@ extern struct daemon { char *packet; /* packet buffer */ int packet_buff_sz; /* size of above */ char *namebuff; /* MAXDNAME size buffer */ -#if (defined(HAVE_CONNTRACK) && defined(HAVE_UBUS)) || defined(HAVE_DNSSEC) - /* CONNTRACK UBUS code uses this buffer, as well as DNSSEC code. */ char *workspacename; -#endif #ifdef HAVE_DNSSEC char *keyname; /* MAXDNAME size buffer */ unsigned long *rr_status; /* ceiling in TTL from DNSSEC or zero for insecure */ @@ -1236,8 +1256,8 @@ extern struct daemon { struct server *srv_save; /* Used for resend on DoD */ size_t packet_len; /* " " */ int fd_save; /* " " */ - pid_t tcp_pids[MAX_PROCS]; - int tcp_pipes[MAX_PROCS]; + pid_t *tcp_pids; + int *tcp_pipes; int pipe_to_parent; int numrrand; struct randfd *randomsocks; @@ -1297,6 +1317,8 @@ extern struct daemon { /* file for packet dumps. */ int dumpfd; #endif + int max_procs; + uint max_procs_used; } *daemon; struct server_details { @@ -1309,6 +1331,7 @@ struct server_details { /* cache.c */ void cache_init(void); +unsigned short rrtype(char *in); void next_uid(struct crec *crecp); /********************************************* Pi-hole modification ***********************************************/ #define log_query(flags,name,addr,arg,type) _log_query(flags, name, addr, arg, type, __FILE__, __LINE__) @@ -1359,6 +1382,8 @@ int read_hostsfile(char *filename, unsigned int index, int cache_size, void blockdata_init(void); void blockdata_report(void); struct blockdata *blockdata_alloc(char *data, size_t len); +int blockdata_expand(struct blockdata *block, size_t oldlen, + char *data, size_t newlen); void *blockdata_retrieve(struct blockdata *block, size_t len, void *data); struct blockdata *blockdata_read(int fd, size_t len); void blockdata_write(struct blockdata *block, size_t len, int fd); @@ -1371,6 +1396,7 @@ int is_name_synthetic(int flags, char *name, union all_addr *addr); int is_rev_synth(int flag, union all_addr *addr, char *name); /* rfc1035.c */ +int do_doctor(struct dns_header *header, size_t qlen, char *namebuff); int extract_name(struct dns_header *header, size_t plen, unsigned char **pp, char *name, int isExtract, int extrabytes); unsigned char *skip_name(unsigned char *ansp, struct dns_header *header, size_t plen, int extrabytes); @@ -1381,14 +1407,14 @@ unsigned int extract_request(struct dns_header *header, size_t qlen, void setup_reply(struct dns_header *header, unsigned int flags, int ede); int extract_addresses(struct dns_header *header, size_t qlen, char *name, time_t now, struct ipsets *ipsets, struct ipsets *nftsets, int is_sign, - int check_rebind, int no_cache_dnssec, int secure, int *doctored); + int check_rebind, int no_cache_dnssec, int secure); #if defined(HAVE_CONNTRACK) && defined(HAVE_UBUS) void report_addresses(struct dns_header *header, size_t len, u32 mark); #endif size_t answer_request(struct dns_header *header, char *limit, size_t qlen, struct in_addr local_addr, struct in_addr local_netmask, time_t now, int ad_reqd, int do_bit, int have_pseudoheader, - int *stale); + int *stale, int *filtered); int check_for_bogus_wildcard(struct dns_header *header, size_t qlen, char *name, time_t now); int check_for_ignored_address(struct dns_header *header, size_t qlen); @@ -1440,6 +1466,7 @@ void rand_init(void); unsigned short rand16(void); u32 rand32(void); u64 rand64(void); +int rr_on_list(struct rrlist *list, unsigned short rr); int legal_hostname(char *name); char *canonicalise(char *in, int *nomem); unsigned char *do_rfc1035_name(unsigned char *p, char *sval, char *limit); @@ -1600,6 +1627,7 @@ void lease_update_from_configs(void); int do_script_run(time_t now); void rerun_scripts(void); void lease_find_interfaces(time_t now); +void lease_calc_fqdns(void); #ifdef HAVE_SCRIPT void lease_add_extradata(struct dhcp_lease *lease, unsigned char *data, unsigned int len, int delim); @@ -1844,14 +1872,16 @@ void poll_listen(int fd, short event); int do_poll(int timeout); /* rrfilter.c */ -size_t rrfilter(struct dns_header *header, size_t plen, int mode); -u16 *rrfilter_desc(int type); +size_t rrfilter(struct dns_header *header, size_t *plen, int mode); +short *rrfilter_desc(int type); int expand_workspace(unsigned char ***wkspc, int *szp, int new); +int to_wire(char *name); +void from_wire(char *name); /* modes. */ #define RRFILTER_EDNS0 0 #define RRFILTER_DNSSEC 1 -#define RRFILTER_A 2 -#define RRFILTER_AAAA 3 +#define RRFILTER_CONF 2 + /* edns0.c */ unsigned char *find_pseudoheader(struct dns_header *header, size_t plen, size_t *len, unsigned char **p, int *is_sign, int *is_last); diff --git a/src/dnsmasq/dnssec.c b/src/dnsmasq/dnssec.c index 219ba9af..29a8e7a7 100644 --- a/src/dnsmasq/dnssec.c +++ b/src/dnsmasq/dnssec.c @@ -1,5 +1,5 @@ /* dnssec.c is Copyright (c) 2012 Giovanni Bajo - and Copyright (c) 2012-2020 Simon Kelley + and Copyright (c) 2012-2023 Simon Kelley This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -24,81 +24,6 @@ #define SERIAL_LT -1 #define SERIAL_GT 1 -/* Convert from presentation format to wire format, in place. - Also map UC -> LC. - Note that using extract_name to get presentation format - then calling to_wire() removes compression and maps case, - thus generating names in canonical form. - Calling to_wire followed by from_wire is almost an identity, - except that the UC remains mapped to LC. - - Note that both /000 and '.' are allowed within labels. These get - represented in presentation format using NAME_ESCAPE as an escape - character. In theory, if all the characters in a name were /000 or - '.' or NAME_ESCAPE then all would have to be escaped, so the - presentation format would be twice as long as the spec (1024). - The buffers are all declared as 2049 (allowing for the trailing zero) - for this reason. -*/ -static int to_wire(char *name) -{ - unsigned char *l, *p, *q, term; - int len; - - for (l = (unsigned char*)name; *l != 0; l = p) - { - for (p = l; *p != '.' && *p != 0; p++) - if (*p >= 'A' && *p <= 'Z') - *p = *p - 'A' + 'a'; - else if (*p == NAME_ESCAPE) - { - for (q = p; *q; q++) - *q = *(q+1); - (*p)--; - } - term = *p; - - if ((len = p - l) != 0) - memmove(l+1, l, len); - *l = len; - - p++; - - if (term == 0) - *p = 0; - } - - return l + 1 - (unsigned char *)name; -} - -/* Note: no compression allowed in input. */ -static void from_wire(char *name) -{ - unsigned char *l, *p, *last; - int len; - - for (last = (unsigned char *)name; *last != 0; last += *last+1); - - for (l = (unsigned char *)name; *l != 0; l += len+1) - { - len = *l; - memmove(l, l+1, len); - for (p = l; p < l + len; p++) - if (*p == '.' || *p == 0 || *p == NAME_ESCAPE) - { - memmove(p+1, p, 1 + last - p); - len++; - *p++ = NAME_ESCAPE; - (*p)++; - } - - l[len] = '.'; - } - - if ((char *)l != name) - *(l-1) = 0; -} - /* Input in presentation format */ static int count_labels(char *name) { @@ -225,7 +150,7 @@ static int is_check_date(unsigned long curtime) On returning 0, the end has been reached. */ struct rdata_state { - u16 *desc; + short *desc; size_t c; unsigned char *end, *ip, *op; char *buff; @@ -246,7 +171,7 @@ static int get_rdata(struct dns_header *header, size_t plen, struct rdata_state { d = *(state->desc); - if (d == (u16)-1) + if (d == -1) { /* all the bytes to the end. */ if ((state->c = state->end - state->ip) != 0) @@ -294,7 +219,7 @@ static int get_rdata(struct dns_header *header, size_t plen, struct rdata_state /* Bubble sort the RRset into the canonical order. */ -static int sort_rrset(struct dns_header *header, size_t plen, u16 *rr_desc, int rrsetidx, +static int sort_rrset(struct dns_header *header, size_t plen, short *rr_desc, int rrsetidx, unsigned char **rrset, char *buff1, char *buff2) { int swap, i, j; @@ -331,7 +256,7 @@ static int sort_rrset(struct dns_header *header, size_t plen, u16 *rr_desc, int is the identity function and we can compare the RRs directly. If not we compare the canonicalised RRs one byte at a time. */ - if (*rr_desc == (u16)-1) + if (*rr_desc == -1) { int rdmin = rdlen1 > rdlen2 ? rdlen2 : rdlen1; int cmp = memcmp(state1.ip, state2.ip, rdmin); @@ -524,7 +449,7 @@ static int validate_rrset(time_t now, struct dns_header *header, size_t plen, in unsigned char *p; int rdlen, j, name_labels, algo, labels, key_tag; struct crec *crecp = NULL; - u16 *rr_desc = rrfilter_desc(type); + short *rr_desc = rrfilter_desc(type); u32 sig_expiration, sig_inception; int failflags = DNSSEC_FAIL_NOSIG | DNSSEC_FAIL_NYV | DNSSEC_FAIL_EXP | DNSSEC_FAIL_NOKEYSUP; @@ -671,7 +596,7 @@ static int validate_rrset(time_t now, struct dns_header *header, size_t plen, in If canonicalisation is not needed, a simple insertion into the hash works. */ - if (*rr_desc == (u16)-1) + if (*rr_desc == -1) { len = htons(rdlen); hash->update(ctx, 2, (unsigned char *)&len); @@ -996,7 +921,7 @@ int dnssec_validate_by_ds(time_t now, struct dns_header *header, size_t plen, ch int dnssec_validate_ds(time_t now, struct dns_header *header, size_t plen, char *name, char *keyname, int class) { unsigned char *p = (unsigned char *)(header+1); - int qtype, qclass, rc, i, neganswer, nons, neg_ttl = 0, found_supported = 0; + int qtype, qclass, rc, i, neganswer = 0, nons = 0, servfail = 0, neg_ttl = 0, found_supported = 0; int aclass, atype, rdlen, flags; unsigned long ttl; union all_addr a; @@ -1009,35 +934,43 @@ int dnssec_validate_ds(time_t now, struct dns_header *header, size_t plen, char GETSHORT(qclass, p); if (qtype != T_DS || qclass != class) - rc = STAT_BOGUS; - else - rc = dnssec_validate_reply(now, header, plen, name, keyname, NULL, 0, &neganswer, &nons, &neg_ttl); - - if (STAT_ISEQUAL(rc, STAT_INSECURE)) - { - my_syslog(LOG_WARNING, _("Insecure DS reply received for %s, check domain configuration and upstream DNS server DNSSEC support"), name); - log_query(F_NOEXTRA | F_UPSTREAM, name, NULL, "BOGUS DS - not secure", 0); - return STAT_BOGUS | DNSSEC_FAIL_INDET; - } - - p = (unsigned char *)(header+1); - if (!extract_name(header, plen, &p, name, 1, 4)) - return STAT_BOGUS; + return STAT_BOGUS; - p += 4; /* qtype, qclass */ - - /* If the key needed to validate the DS is on the same domain as the DS, we'll - loop getting nowhere. Stop that now. This can happen of the DS answer comes - from the DS's zone, and not the parent zone. */ - if (STAT_ISEQUAL(rc, STAT_NEED_KEY) && hostname_isequal(name, keyname)) + /* A SERVFAIL answer has been seen to a DS query not at start of authority, + so treat it as such and continue to search for a DS or proof of no existence + further down the tree. */ + if (RCODE(header) == SERVFAIL) + servfail = neganswer = nons = 1; + else { - log_query(F_NOEXTRA | F_UPSTREAM, name, NULL, "BOGUS DS", 0); - return STAT_BOGUS; + rc = dnssec_validate_reply(now, header, plen, name, keyname, NULL, 0, &neganswer, &nons, &neg_ttl); + + if (STAT_ISEQUAL(rc, STAT_INSECURE)) + { + my_syslog(LOG_WARNING, _("Insecure DS reply received for %s, check domain configuration and upstream DNS server DNSSEC support"), name); + log_query(F_NOEXTRA | F_UPSTREAM, name, NULL, "BOGUS DS - not secure", 0); + return STAT_BOGUS | DNSSEC_FAIL_INDET; + } + + p = (unsigned char *)(header+1); + if (!extract_name(header, plen, &p, name, 1, 4)) + return STAT_BOGUS; + + p += 4; /* qtype, qclass */ + + /* If the key needed to validate the DS is on the same domain as the DS, we'll + loop getting nowhere. Stop that now. This can happen of the DS answer comes + from the DS's zone, and not the parent zone. */ + if (STAT_ISEQUAL(rc, STAT_NEED_KEY) && hostname_isequal(name, keyname)) + { + log_query(F_NOEXTRA | F_UPSTREAM, name, NULL, "BOGUS DS", 0); + return STAT_BOGUS; + } + + if (!STAT_ISEQUAL(rc, STAT_SECURE)) + return rc; } - if (!STAT_ISEQUAL(rc, STAT_SECURE)) - return rc; - if (!neganswer) { cache_start_insert(); @@ -1135,7 +1068,8 @@ int dnssec_validate_ds(time_t now, struct dns_header *header, size_t plen, char cache_end_insert(); if (neganswer) - log_query(F_NOEXTRA | F_UPSTREAM, name, NULL, nons ? "no DS/cut" : "no DS", 0); + log_query(F_NOEXTRA | F_UPSTREAM, name, NULL, + servfail ? "SERVFAIL" : (nons ? "no DS/cut" : "no DS"), 0); return STAT_OK; } @@ -1870,7 +1804,7 @@ static int zone_status(char *name, int class, char *keyname, time_t now) When validating replies to DS records, we're only interested in the NSEC{3} RRs in the auth section. Other RRs in that section missing sigs will not cause am INSECURE reply. We determine this mode - is the nons argument is non-NULL. + if the nons argument is non-NULL. */ int dnssec_validate_reply(time_t now, struct dns_header *header, size_t plen, char *name, char *keyname, int *class, int check_unsigned, int *neganswer, int *nons, int *nsec_ttl) diff --git a/src/dnsmasq/domain-match.c b/src/dnsmasq/domain-match.c index 9cc51e68..cf2da770 100644 --- a/src/dnsmasq/domain-match.c +++ b/src/dnsmasq/domain-match.c @@ -1,4 +1,4 @@ -/* dnsmasq is Copyright (c) 2000-2022 Simon Kelley +/* dnsmasq is Copyright (c) 2000-2024 Simon Kelley This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by diff --git a/src/dnsmasq/domain.c b/src/dnsmasq/domain.c index a893ce5a..f4c0bf71 100644 --- a/src/dnsmasq/domain.c +++ b/src/dnsmasq/domain.c @@ -1,4 +1,4 @@ -/* dnsmasq is Copyright (c) 2000-2022 Simon Kelley +/* dnsmasq is Copyright (c) 2000-2024 Simon Kelley This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -22,12 +22,13 @@ static int match_domain(struct in_addr addr, struct cond_domain *c); static struct cond_domain *search_domain6(struct in6_addr *addr, struct cond_domain *c); static int match_domain6(struct in6_addr *addr, struct cond_domain *c); -int is_name_synthetic(int flags, char *name, union all_addr *addr) +int is_name_synthetic(int flags, char *name, union all_addr *addrp) { char *p; struct cond_domain *c = NULL; int prot = (flags & F_IPV6) ? AF_INET6 : AF_INET; - + union all_addr addr; + for (c = daemon->synth_domains; c; c = c->next) { int found = 0; @@ -74,7 +75,7 @@ int is_name_synthetic(int flags, char *name, union all_addr *addr) if (!c->is6 && index <= ntohl(c->end.s_addr) - ntohl(c->start.s_addr)) { - addr->addr4.s_addr = htonl(ntohl(c->start.s_addr) + index); + addr.addr4.s_addr = htonl(ntohl(c->start.s_addr) + index); found = 1; } } @@ -86,8 +87,8 @@ int is_name_synthetic(int flags, char *name, union all_addr *addr) index <= addr6part(&c->end6) - addr6part(&c->start6)) { u64 start = addr6part(&c->start6); - addr->addr6 = c->start6; - setaddr6part(&addr->addr6, start + index); + addr.addr6 = c->start6; + setaddr6part(&addr.addr6, start + index); found = 1; } } @@ -135,8 +136,8 @@ int is_name_synthetic(int flags, char *name, union all_addr *addr) } } - if (hostname_isequal(c->domain, p+1) && inet_pton(prot, tail, addr)) - found = (prot == AF_INET) ? match_domain(addr->addr4, c) : match_domain6(&addr->addr6, c); + if (hostname_isequal(c->domain, p+1) && inet_pton(prot, tail, &addr)) + found = (prot == AF_INET) ? match_domain(addr.addr4, c) : match_domain6(&addr.addr6, c); } /* restore name */ @@ -148,7 +149,12 @@ int is_name_synthetic(int flags, char *name, union all_addr *addr) if (found) - return 1; + { + if (addrp) + *addrp = addr; + + return 1; + } } return 0; diff --git a/src/dnsmasq/dump.c b/src/dnsmasq/dump.c index 57352a9d..d1442f0f 100644 --- a/src/dnsmasq/dump.c +++ b/src/dnsmasq/dump.c @@ -1,4 +1,4 @@ -/* dnsmasq is Copyright (c) 2000-2022 Simon Kelley +/* dnsmasq is Copyright (c) 2000-2024 Simon Kelley This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by diff --git a/src/dnsmasq/edns0.c b/src/dnsmasq/edns0.c index c498eb12..598478fa 100644 --- a/src/dnsmasq/edns0.c +++ b/src/dnsmasq/edns0.c @@ -1,4 +1,4 @@ -/* dnsmasq is Copyright (c) 2000-2022 Simon Kelley +/* dnsmasq is Copyright (c) 2000-2024 Simon Kelley This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -178,7 +178,7 @@ size_t add_pseudoheader(struct dns_header *header, size_t plen, unsigned char *l memcpy(buff, datap, rdlen); /* now, delete OPT RR */ - plen = rrfilter(header, plen, RRFILTER_EDNS0); + rrfilter(header, &plen, RRFILTER_EDNS0); /* Now, force addition of a new one */ p = NULL; @@ -191,16 +191,13 @@ size_t add_pseudoheader(struct dns_header *header, size_t plen, unsigned char *l if (!(p = skip_questions(header, plen)) || !(p = skip_section(p, ntohs(header->ancount) + ntohs(header->nscount) + ntohs(header->arcount), - header, plen))) - { - free(buff); - return plen; - } - if (p + 11 > limit) - { - free(buff); - return plen; /* Too big */ - } + header, plen)) || + p + 11 > limit) + { + free(buff); + return plen; /* bad packet */ + } + *p++ = 0; /* empty name */ PUTSHORT(T_OPT, p); PUTSHORT(udp_sz, p); /* max packet length, 512 if not given in EDNS0 header */ diff --git a/src/dnsmasq/forward.c b/src/dnsmasq/forward.c index 8043aa71..59bb91ed 100644 --- a/src/dnsmasq/forward.c +++ b/src/dnsmasq/forward.c @@ -1,4 +1,4 @@ -/* dnsmasq is Copyright (c) 2000-2022 Simon Kelley +/* dnsmasq is Copyright (c) 2000-2024 Simon Kelley This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -697,7 +697,7 @@ static size_t process_reply(struct dns_header *header, time_t now, struct server { unsigned char *pheader, *sizep; struct ipsets *ipsets = NULL, *nftsets = NULL; - int munged = 0, is_sign; + int is_sign; unsigned int rcode = RCODE(header); size_t plen; /******** Pi-hole modification ********/ @@ -706,8 +706,7 @@ static size_t process_reply(struct dns_header *header, time_t now, struct server (void)ad_reqd; (void)do_bit; - (void)bogusanswer; - + #ifdef HAVE_IPSET if (daemon->ipsets && extract_request(header, n, daemon->namebuff, NULL)) ipsets = domain_find_sets(daemon->ipsets, daemon->namebuff); @@ -738,7 +737,7 @@ static size_t process_reply(struct dns_header *header, time_t now, struct server if (added_pheader) { /* client didn't send EDNS0, we added one, strip it off before returning answer. */ - n = rrfilter(header, n, RRFILTER_EDNS0); + rrfilter(header, &n, RRFILTER_EDNS0); pheader = NULL; } else @@ -801,119 +800,118 @@ static size_t process_reply(struct dns_header *header, time_t now, struct server server->flags |= SERV_WARNED_RECURSIVE; } - if (daemon->bogus_addr && rcode != NXDOMAIN && - check_for_bogus_wildcard(header, n, daemon->namebuff, now)) + if (header->hb3 & HB3_TC) { - munged = 1; - SET_RCODE(header, NXDOMAIN); - header->hb3 &= ~HB3_AA; - cache_secure = 0; - ede = EDE_BLOCKED; + log_query(F_UPSTREAM, NULL, NULL, "truncated", 0); + header->ancount = htons(0); + header->nscount = htons(0); + header->arcount = htons(0); } - else + + if (!(header->hb3 & HB3_TC) && (!bogusanswer || (header->hb4 & HB4_CD))) { - int doctored = 0; + if (rcode == NXDOMAIN && extract_request(header, n, daemon->namebuff, NULL) && + (check_for_local_domain(daemon->namebuff, now) || lookup_domain(daemon->namebuff, F_CONFIG, NULL, NULL))) + { + /* if we forwarded a query for a locally known name (because it was for + an unknown type) and the answer is NXDOMAIN, convert that to NODATA, + since we know that the domain exists, even if upstream doesn't */ + header->hb3 |= HB3_AA; + SET_RCODE(header, NOERROR); + cache_secure = 0; + } - if (rcode == NXDOMAIN && - extract_request(header, n, daemon->namebuff, NULL)) + if (daemon->doctors && do_doctor(header, n, daemon->namebuff)) + cache_secure = 0; + + /* check_for_bogus_wildcard() does it's own caching, so + don't call extract_addresses() if it triggers. */ + if (daemon->bogus_addr && rcode != NXDOMAIN && + check_for_bogus_wildcard(header, n, daemon->namebuff, now)) { - if (check_for_local_domain(daemon->namebuff, now) || - lookup_domain(daemon->namebuff, F_CONFIG, NULL, NULL)) - { - /* if we forwarded a query for a locally known name (because it was for - an unknown type) and the answer is NXDOMAIN, convert that to NODATA, - since we know that the domain exists, even if upstream doesn't */ - munged = 1; - header->hb3 |= HB3_AA; - SET_RCODE(header, NOERROR); - cache_secure = 0; - } - } - - /* Before extract_addresses() */ - if (rcode == NOERROR) - { - if (option_bool(OPT_FILTER_A)) - n = rrfilter(header, n, RRFILTER_A); - - if (option_bool(OPT_FILTER_AAAA)) - n = rrfilter(header, n, RRFILTER_AAAA); - } - - switch (extract_addresses(header, n, daemon->namebuff, now, ipsets, nftsets, is_sign, check_rebind, no_cache, cache_secure, &doctored)) - { - case 1: - my_syslog(LOG_WARNING, _("possible DNS-rebind attack detected: %s"), daemon->namebuff); - munged = 1; + header->ancount = htons(0); + header->nscount = htons(0); + header->arcount = htons(0); + SET_RCODE(header, NXDOMAIN); + header->hb3 &= ~HB3_AA; cache_secure = 0; ede = EDE_BLOCKED; - break; - - /* extract_addresses() found a malformed answer. */ - case 2: - munged = 1; - SET_RCODE(header, SERVFAIL); - cache_secure = 0; - ede = EDE_OTHER; - break; - - /* Pi-hole modification */ - case 99: - cache_secure = 0; - // Make a private copy of the pheader to ensure - // we are not accidentially rewriting what is in - // the pheader when we're creating a crafted reply - // further below (when a query is to be blocked) - if (pheader) - { - pheader_copy = calloc(1, plen); - memcpy(pheader_copy, pheader, plen); - } - - // Generate DNS packet for reply, a possibly existing pseudo header - // will be restored later inside resize_packet() - n = FTL_make_answer(header, ((char *) header) + 65536, n, &ede); - break; } + else + { + int rc = extract_addresses(header, n, daemon->namebuff, now, ipsets, nftsets, is_sign, check_rebind, no_cache, cache_secure); - if (doctored) - cache_secure = 0; + if (rc != 0) + { + header->ancount = htons(0); + header->nscount = htons(0); + header->arcount = htons(0); + cache_secure = 0; + } + + if (rc == 1) + { + my_syslog(LOG_WARNING, _("possible DNS-rebind attack detected: %s"), daemon->namebuff); + ede = EDE_BLOCKED; + } + + if (rc == 2) + { + /* extract_addresses() found a malformed answer. */ + SET_RCODE(header, SERVFAIL); + ede = EDE_OTHER; + } + + /* Pi-hole modification */ + if(rc == 99) + { + cache_secure = 0; + // Make a private copy of the pheader to ensure + // we are not accidentially rewriting what is in + // the pheader when we're creating a crafted reply + // further below (when a query is to be blocked) + if (pheader) + { + pheader_copy = calloc(1, plen); + memcpy(pheader_copy, pheader, plen); + } + + // Generate DNS packet for reply, a possibly existing pseudo header + // will be restored later inside resize_packet() + n = FTL_make_answer(header, ((char *) header) + 65536, n, &ede); + } + } + + if (RCODE(header) == NOERROR && rrfilter(header, &n, RRFILTER_CONF) > 0) + ede = EDE_FILTERED; } #ifdef HAVE_DNSSEC - if (bogusanswer && !(header->hb4 & HB4_CD) && !option_bool(OPT_DNSSEC_DEBUG)) - { - /* Bogus reply, turn into SERVFAIL */ - SET_RCODE(header, SERVFAIL); - munged = 1; - } - if (option_bool(OPT_DNSSEC_VALID)) { - header->hb4 &= ~HB4_AD; - - if (!(header->hb4 & HB4_CD) && ad_reqd && cache_secure) + if (bogusanswer) + { + if (!(header->hb4 & HB4_CD) && !option_bool(OPT_DNSSEC_DEBUG)) + { + /* Bogus reply, turn into SERVFAIL */ + SET_RCODE(header, SERVFAIL); + header->ancount = htons(0); + header->nscount = htons(0); + header->arcount = htons(0); + ede = EDE_DNSSEC_BOGUS; + } + } + else if (!(header->hb4 & HB4_CD) && ad_reqd && cache_secure) header->hb4 |= HB4_AD; /* If the requestor didn't set the DO bit, don't return DNSSEC info. */ if (!do_bit) - n = rrfilter(header, n, RRFILTER_DNSSEC); + rrfilter(header, &n, RRFILTER_DNSSEC); } #endif - - /* do this after extract_addresses. Ensure NODATA reply and remove - nameserver info. */ - if (munged) - { - header->ancount = htons(0); - header->nscount = htons(0); - header->arcount = htons(0); - header->hb3 &= ~HB3_TC; - } - /* the bogus-nxdomain stuff, doctor and NXDOMAIN->NODATA munging can all elide - sections of the packet. Find the new length here and put back pseudoheader - if it was removed. */ + /* the code above can elide sections of the packet. Find the new length here + and put back pseudoheader if it was removed. */ n = resize_packet(header, n, pheader_copy ? pheader_copy : pheader, plen); /******** Pi-hole modification ********/ // The line above was modified to use @@ -1882,10 +1880,10 @@ void receive_query(struct listener *listen, time_t now) #endif else { - int stale; + int stale, filtered; int ad_reqd = do_bit; - u16 hb3 = header->hb3, hb4 = header->hb4; int fd = listen->fd; + struct blockdata *saved_question = blockdata_alloc((char *) header, (size_t)n); /* RFC 6840 5.7 */ if (header->hb4 & HB4_AD) @@ -1922,17 +1920,27 @@ void receive_query(struct listener *listen, time_t now) /**********************************************/ m = answer_request(header, ((char *) header) + udp_size, (size_t)n, - dst_addr_4, netmask, now, ad_reqd, do_bit, have_pseudoheader, &stale); + dst_addr_4, netmask, now, ad_reqd, do_bit, have_pseudoheader, &stale, &filtered); if (m >= 1) { - if (stale && have_pseudoheader) + if (have_pseudoheader) { - u16 swap = htons(EDE_STALE); - - m = add_pseudoheader(header, m, ((unsigned char *) header) + udp_size, daemon->edns_pktsz, - EDNS0_OPTION_EDE, (unsigned char *)&swap, 2, do_bit, 0); + int ede = EDE_UNSET; + if (filtered) + ede = EDE_FILTERED; + else if (stale) + ede = EDE_STALE; + + if (ede != EDE_UNSET) + { + u16 swap = htons(ede); + + m = add_pseudoheader(header, m, ((unsigned char *) header) + udp_size, daemon->edns_pktsz, + EDNS0_OPTION_EDE, (unsigned char *)&swap, 2, do_bit, 0); + } } + #ifdef HAVE_DUMPFILE dump_packet_udp(DUMP_REPLY, daemon->packet, m, NULL, &source_addr, listen->fd); #endif @@ -1947,34 +1955,31 @@ void receive_query(struct listener *listen, time_t now) daemon->metrics[METRIC_DNS_STALE_ANSWERED]++; } - if (m == 0 || stale) + if (stale) { - if (m != 0) + /* We answered with stale cache data, so forward the query anyway to + refresh that. */ + m = 0; + + /* We've already answered the client, so don't send it the answer + when it comes back. */ + fd = -1; + } + + if (saved_question) + { + if (m == 0) { - size_t plen; + blockdata_retrieve(saved_question, (size_t)n, header); - /* We answered with stale cache data, so forward the query anyway to - refresh that. Restore the query from the answer packet. */ - pheader = find_pseudoheader(header, (size_t)m, &plen, NULL, NULL, NULL); - - header->hb3 = hb3; - header->hb4 = hb4; - header->ancount = htons(0); - header->nscount = htons(0); - header->arcount = htons(0); - - m = resize_packet(header, m, pheader, plen); - - /* We've already answered the client, so don't send it the answer - when it comes back. */ - fd = -1; + if (forward_query(fd, &source_addr, &dst_addr, if_index, + header, (size_t)n, ((char *) header) + udp_size, now, NULL, ad_reqd, do_bit, 0)) + daemon->metrics[METRIC_DNS_QUERIES_FORWARDED]++; + else + daemon->metrics[METRIC_DNS_LOCAL_ANSWERED]++; } - if (forward_query(fd, &source_addr, &dst_addr, if_index, - header, (size_t)n, ((char *) header) + udp_size, now, NULL, ad_reqd, do_bit, 0)) - daemon->metrics[METRIC_DNS_QUERIES_FORWARDED]++; - else - daemon->metrics[METRIC_DNS_LOCAL_ANSWERED]++; + blockdata_free(saved_question); } } } @@ -2163,7 +2168,7 @@ static int tcp_key_recurse(time_t now, int status, struct dns_header *header, si daemon->log_display_id = ++daemon->log_id; log_query_mysockaddr(F_NOEXTRA | F_DNSSEC | F_SERVER, keyname, &server->addr, - STAT_ISEQUAL(status, STAT_NEED_KEY) ? "dnssec-query[DNSKEY]" : "dnssec-query[DS]", 0); + STAT_ISEQUAL(new_status, STAT_NEED_KEY) ? "dnssec-query[DNSKEY]" : "dnssec-query[DS]", 0); new_status = tcp_key_recurse(now, new_status, new_header, m, class, name, keyname, server, have_mark, mark, keycount); @@ -2188,7 +2193,7 @@ static int tcp_key_recurse(time_t now, int status, struct dns_header *header, si unsigned char *tcp_request(int confd, time_t now, union mysockaddr *local_addr, struct in_addr netmask, int auth_dns) { - size_t size = 0; + size_t size = 0, saved_size = 0; int norebind; #ifdef HAVE_CONNTRACK int is_single_query = 0, allowed = 1; @@ -2199,6 +2204,7 @@ unsigned char *tcp_request(int confd, time_t now, int checking_disabled, do_bit, added_pheader = 0, have_pseudoheader = 0; int cacheable, no_cache_dnssec = 0, cache_secure = 0, bogusanswer = 0; size_t m; + struct blockdata *saved_question = NULL; unsigned short qtype; unsigned int gotname; /* Max TCP packet + slop + size */ @@ -2216,9 +2222,8 @@ unsigned char *tcp_request(int confd, time_t now, unsigned char *pheader; unsigned int mark = 0; int have_mark = 0; - int first, last, stale, do_stale = 0; + int first, last, filtered, stale, do_stale = 0; unsigned int flags = 0; - u16 hb3, hb4; /************ Pi-hole modification ************/ bool piholeblocked = false; @@ -2277,35 +2282,15 @@ unsigned char *tcp_request(int confd, time_t now, { int ede = EDE_UNSET; - if (query_count == TCP_MAX_QUERIES) - return packet; - - if (do_stale) + if (!do_stale) { - size_t plen; - - /* We answered the last query with stale data. Now try and get fresh data. - Restore query from answer. */ - pheader = find_pseudoheader(header, m, &plen, NULL, NULL, NULL); + if (query_count == TCP_MAX_QUERIES) + break; - header->hb3 = hb3; - header->hb4 = hb4; - header->ancount = htons(0); - header->nscount = htons(0); - header->arcount = htons(0); - - size = resize_packet(header, m, pheader, plen); - } - else - { if (!read_write(confd, &c1, 1, 1) || !read_write(confd, &c2, 1, 1) || !(size = c1 << 8 | c2) || !read_write(confd, payload, size, 1)) - return packet; - - /* for stale-answer processing. */ - hb3 = header->hb3; - hb4 = header->hb4; + break; } if (size < (int)sizeof(struct dns_header)) @@ -2327,7 +2312,6 @@ unsigned char *tcp_request(int confd, time_t now, no_cache_dnssec = 1; //********************** Pi-hole modification **********************// - unsigned char *pheader = NULL; pheader = find_pseudoheader(header, (size_t)size, NULL, &pheader, NULL, NULL); FTL_parse_pseudoheaders(pheader, (size_t)size); //******************************************************************// @@ -2447,18 +2431,28 @@ unsigned char *tcp_request(int confd, time_t now, if (do_stale) m = 0; else - /* m > 0 if answered from cache */ - m = answer_request(header, ((char *) header) + 65536, (size_t)size, - dst_addr_4, netmask, now, ad_reqd, do_bit, have_pseudoheader, &stale); - + { + if (saved_question) + blockdata_free(saved_question); + + saved_question = blockdata_alloc((char *) header, (size_t)size); + saved_size = size; + + /* m > 0 if answered from cache */ + m = answer_request(header, ((char *) header) + 65536, (size_t)size, + dst_addr_4, netmask, now, ad_reqd, do_bit, have_pseudoheader, &stale, &filtered); + } /* Do this by steam now we're not in the select() loop */ check_log_writer(1); - if (m == 0) + if (m == 0 && saved_question) { struct server *master; int start; + blockdata_retrieve(saved_question, (size_t)saved_size, header); + size = saved_size; + if (lookup_domain(daemon->namebuff, gotname, &first, &last)) flags = is_local_answer(now, first, daemon->namebuff); else @@ -2591,13 +2585,23 @@ unsigned char *tcp_request(int confd, time_t now, m = add_pseudoheader(header, m, ((unsigned char *) header) + 65536, daemon->edns_pktsz, 0, NULL, 0, do_bit, 0); } } - else if (stale) - { - u16 swap = htons((u16)EDE_STALE); - - m = add_pseudoheader(header, m, ((unsigned char *) header) + 65536, daemon->edns_pktsz, EDNS0_OPTION_EDE, (unsigned char *)&swap, 2, do_bit, 0); - } - + else if (have_pseudoheader) + { + ede = EDE_UNSET; + + if (filtered) + ede = EDE_FILTERED; + else if (stale) + ede = EDE_STALE; + + if (ede != EDE_UNSET) + { + u16 swap = htons((u16)ede); + + m = add_pseudoheader(header, m, ((unsigned char *) header) + 65536, daemon->edns_pktsz, EDNS0_OPTION_EDE, (unsigned char *)&swap, 2, do_bit, 0); + } + } + check_log_writer(1); *length = htons(m); @@ -2613,7 +2617,7 @@ unsigned char *tcp_request(int confd, time_t now, break; /* If we answered with stale data, this process will now try and get fresh data into - the cache then and cannot therefore accept new queries. Close the incoming + the cache and cannot therefore accept new queries. Close the incoming connection to signal that to the client. Then set do_stale and loop round once more to try and get fresh data, after which we exit. */ if (stale) @@ -2631,6 +2635,9 @@ unsigned char *tcp_request(int confd, time_t now, close(confd); } + if (saved_question) + blockdata_free(saved_question); + return packet; } diff --git a/src/dnsmasq/hash-questions.c b/src/dnsmasq/hash-questions.c index adcf62c8..e6304ac8 100644 --- a/src/dnsmasq/hash-questions.c +++ b/src/dnsmasq/hash-questions.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2012-2020 Simon Kelley +/* Copyright (c) 2012-2023 Simon Kelley This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -165,7 +165,7 @@ static void sha256_transform(SHA256_CTX *ctx, const BYTE data[]) WORD a, b, c, d, e, f, g, h, i, j, t1, t2, m[64]; for (i = 0, j = 0; i < 16; ++i, j += 4) - m[i] = (data[j] << 24) | (data[j + 1] << 16) | (data[j + 2] << 8) | (data[j + 3]); + m[i] = (((WORD)data[j]) << 24) | (((WORD)data[j + 1]) << 16) | (((WORD)data[j + 2]) << 8) | (((WORD)data[j + 3])); for ( ; i < 64; ++i) m[i] = SIG1(m[i - 2]) + m[i - 7] + SIG0(m[i - 15]) + m[i - 16]; diff --git a/src/dnsmasq/helper.c b/src/dnsmasq/helper.c index a96788b3..93a2b4a2 100644 --- a/src/dnsmasq/helper.c +++ b/src/dnsmasq/helper.c @@ -1,4 +1,4 @@ -/* dnsmasq is Copyright (c) 2000-2022 Simon Kelley +/* dnsmasq is Copyright (c) 2000-2024 Simon Kelley This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -99,7 +99,7 @@ int create_helper(int event_fd, int err_fd, uid_t uid, gid_t gid, long max_fd) } /**** Pi-hole modification ****/ - logg("Started dnsmasq helper"); + logg("Started script helper"); /******************************/ /* ignore SIGTERM and SIGINT, so that we can clean up when the main process gets hit diff --git a/src/dnsmasq/inotify.c b/src/dnsmasq/inotify.c index d3c8277b..0c775de8 100644 --- a/src/dnsmasq/inotify.c +++ b/src/dnsmasq/inotify.c @@ -1,4 +1,4 @@ -/* dnsmasq is Copyright (c) 2000-2022 Simon Kelley +/* dnsmasq is Copyright (c) 2000-2024 Simon Kelley This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -94,7 +94,7 @@ void inotify_dnsmasq_init() if (daemon->inotifyfd == -1) die(_("failed to create inotify: %s"), NULL, EC_MISC); - if (option_bool(OPT_NO_RESOLV)) + if (daemon->port == 0 || option_bool(OPT_NO_RESOLV)) return; for (res = daemon->resolv_files; res; res = res->next) diff --git a/src/dnsmasq/ip6addr.h b/src/dnsmasq/ip6addr.h index 977e6840..39dc8e2a 100644 --- a/src/dnsmasq/ip6addr.h +++ b/src/dnsmasq/ip6addr.h @@ -1,4 +1,4 @@ -/* dnsmasq is Copyright (c) 2000-2022 Simon Kelley +/* dnsmasq is Copyright (c) 2000-2024 Simon Kelley This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by diff --git a/src/dnsmasq/lease.c b/src/dnsmasq/lease.c index 8a7b9756..55e8443b 100644 --- a/src/dnsmasq/lease.c +++ b/src/dnsmasq/lease.c @@ -1,4 +1,4 @@ -/* dnsmasq is Copyright (c) 2000-2022 Simon Kelley +/* dnsmasq is Copyright (c) 2000-2024 Simon Kelley This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -15,7 +15,6 @@ */ #include "dnsmasq.h" - #ifdef HAVE_DHCP static struct dhcp_lease *leases = NULL, *old_leases = NULL; @@ -28,8 +27,7 @@ static int read_leases(time_t now, FILE *leasestream) struct dhcp_lease *lease; int clid_len, hw_len, hw_type; int items; - char *domain = NULL; - + *daemon->dhcp_buff3 = *daemon->dhcp_buff2 = '\0'; /* client-id max length is 255 which is 255*2 digits + 254 colons @@ -69,8 +67,8 @@ static int read_leases(time_t now, FILE *leasestream) if (inet_pton(AF_INET, daemon->namebuff, &addr.addr4)) { - if ((lease = lease4_allocate(addr.addr4))) - domain = get_domain(lease->addr); + lease = lease4_allocate(addr.addr4); + hw_len = parse_hex(daemon->dhcp_buff2, (unsigned char *)daemon->dhcp_buff2, DHCP_CHADDR_MAX, NULL, &hw_type); /* For backwards compatibility, no explicit MAC address type means ether. */ @@ -90,10 +88,7 @@ static int read_leases(time_t now, FILE *leasestream) } if ((lease = lease6_allocate(&addr.addr6, lease_type))) - { - lease_set_iaid(lease, strtoul(s, NULL, 10)); - domain = get_domain6(&lease->addr6); - } + lease_set_iaid(lease, strtoul(s, NULL, 10)); } #endif else @@ -114,7 +109,7 @@ static int read_leases(time_t now, FILE *leasestream) hw_len, hw_type, clid_len, now, 0); if (strcmp(daemon->dhcp_buff, "*") != 0) - lease_set_hostname(lease, daemon->dhcp_buff, 0, domain, NULL); + lease_set_hostname(lease, daemon->dhcp_buff, 0, NULL, NULL); ei = atol(daemon->dhcp_buff3); @@ -946,6 +941,36 @@ static void kill_name(struct dhcp_lease *lease) lease->hostname = lease->fqdn = NULL; } +void lease_calc_fqdns(void) +{ + struct dhcp_lease *lease; + + for (lease = leases; lease; lease = lease->next) + { + char *domain; + + if (lease->hostname) + { +#ifdef HAVE_DHCP6 + if (lease->flags & (LEASE_TA | LEASE_NA)) + domain = get_domain6(&lease->addr6); + else +#endif + domain = get_domain(lease->addr); + + if (domain) + { + /* This is called only during startup, before forking, hence safe_malloc() */ + lease->fqdn = safe_malloc(strlen(lease->hostname) + strlen(domain) + 2); + + strcpy(lease->fqdn, lease->hostname); + strcat(lease->fqdn, "."); + strcat(lease->fqdn, domain); + } + } + } +} + void lease_set_hostname(struct dhcp_lease *lease, const char *name, int auth, char *domain, char *config_domain) { struct dhcp_lease *lease_tmp; diff --git a/src/dnsmasq/log.c b/src/dnsmasq/log.c index c38ed445..661f077f 100644 --- a/src/dnsmasq/log.c +++ b/src/dnsmasq/log.c @@ -1,4 +1,4 @@ -/* dnsmasq is Copyright (c) 2000-2022 Simon Kelley +/* dnsmasq is Copyright (c) 2000-2024 Simon Kelley This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -15,7 +15,11 @@ */ #include "dnsmasq.h" -#include "../log.h" +/******* Pi-hole modification *******/ +#include "log.h" +#include "dnsmasq_interface.h" +#include "main.h" +/************************************/ #ifdef __ANDROID__ # include @@ -507,6 +511,4 @@ void die(char *message, char *arg1, int exit_code) /********** Pi-hole modification *************/ FTL_log_dnsmasq_fatal(message, arg1, errmess); /*********************************************/ - - exit(exit_code); } diff --git a/src/dnsmasq/loop.c b/src/dnsmasq/loop.c index 19bfae0d..f87293f8 100644 --- a/src/dnsmasq/loop.c +++ b/src/dnsmasq/loop.c @@ -1,4 +1,4 @@ -/* dnsmasq is Copyright (c) 2000-2022 Simon Kelley +/* dnsmasq is Copyright (c) 2000-2024 Simon Kelley This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by diff --git a/src/dnsmasq/metrics.c b/src/dnsmasq/metrics.c index f3e6728a..f8b8d9c5 100644 --- a/src/dnsmasq/metrics.c +++ b/src/dnsmasq/metrics.c @@ -1,4 +1,4 @@ -/* dnsmasq is Copyright (c) 2000-2022 Simon Kelley +/* dnsmasq is Copyright (c) 2000-2024 Simon Kelley This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -39,6 +39,7 @@ const char * metric_names[] = { "leases_pruned_4", "leases_allocated_6", "leases_pruned_6", + "tcp_connections", }; const char* get_metric_name(int i) { diff --git a/src/dnsmasq/metrics.h b/src/dnsmasq/metrics.h index 6f62a406..839e01dd 100644 --- a/src/dnsmasq/metrics.h +++ b/src/dnsmasq/metrics.h @@ -1,4 +1,4 @@ -/* dnsmasq is Copyright (c) 2000-2022 Simon Kelley +/* dnsmasq is Copyright (c) 2000-2024 Simon Kelley This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -38,6 +38,7 @@ enum { METRIC_LEASES_PRUNED_4, METRIC_LEASES_ALLOCATED_6, METRIC_LEASES_PRUNED_6, + METRIC_TCP_CONNECTIONS, __METRIC_MAX, }; diff --git a/src/dnsmasq/netlink.c b/src/dnsmasq/netlink.c index c156cde3..ef4b5fec 100644 --- a/src/dnsmasq/netlink.c +++ b/src/dnsmasq/netlink.c @@ -1,4 +1,4 @@ -/* dnsmasq is Copyright (c) 2000-2022 Simon Kelley +/* dnsmasq is Copyright (c) 2000-2024 Simon Kelley This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by diff --git a/src/dnsmasq/network.c b/src/dnsmasq/network.c index 7217495b..b37d43e8 100644 --- a/src/dnsmasq/network.c +++ b/src/dnsmasq/network.c @@ -1,4 +1,4 @@ -/* dnsmasq is Copyright (c) 2000-2022 Simon Kelley +/* dnsmasq is Copyright (c) 2000-2024 Simon Kelley This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -125,7 +125,10 @@ int iface_check(int family, union all_addr *addr, char *name, int *auth) for (tmp = daemon->if_names; tmp; tmp = tmp->next) if (tmp->name && wildcard_match(tmp->name, name)) - ret = tmp->used = 1; + { + tmp->flags |= INAME_USED; + ret = 1; + } if (addr) for (tmp = daemon->if_addrs; tmp; tmp = tmp->next) @@ -133,11 +136,17 @@ int iface_check(int family, union all_addr *addr, char *name, int *auth) { if (family == AF_INET && tmp->addr.in.sin_addr.s_addr == addr->addr4.s_addr) - ret = match_addr = tmp->used = 1; + { + tmp->flags |= INAME_USED; + ret = match_addr = 1; + } else if (family == AF_INET6 && IN6_ARE_ADDR_EQUAL(&tmp->addr.in6.sin6_addr, &addr->addr6)) - ret = match_addr = tmp->used = 1; + { + tmp->flags |= INAME_USED; + ret = match_addr = 1; + } } } @@ -237,7 +246,8 @@ static int iface_allowed(struct iface_param *param, int if_index, char *label, int loopback; struct ifreq ifr; int tftp_ok = !!option_bool(OPT_TFTP); - int dhcp_ok = 1; + int dhcp4_ok = 1; + int dhcp6_ok = 1; int auth_dns = 0; int is_label = 0; #if defined(HAVE_DHCP) || defined(HAVE_TFTP) @@ -253,7 +263,7 @@ static int iface_allowed(struct iface_param *param, int if_index, char *label, loopback = ifr.ifr_flags & IFF_LOOPBACK; if (loopback) - dhcp_ok = 0; + dhcp4_ok = dhcp6_ok = 0; if (!label) label = ifr.ifr_name; @@ -503,7 +513,7 @@ static int iface_allowed(struct iface_param *param, int if_index, char *label, if ((lo->name = whine_malloc(strlen(ifr.ifr_name)+1))) { strcpy(lo->name, ifr.ifr_name); - lo->used = 1; + lo->flags |= INAME_USED; lo->next = daemon->if_names; daemon->if_names = lo; } @@ -525,14 +535,17 @@ static int iface_allowed(struct iface_param *param, int if_index, char *label, if (auth_dns) { tftp_ok = 0; - dhcp_ok = 0; + dhcp4_ok = dhcp6_ok = 0; } else for (tmp = daemon->dhcp_except; tmp; tmp = tmp->next) if (tmp->name && wildcard_match(tmp->name, ifr.ifr_name)) { tftp_ok = 0; - dhcp_ok = 0; + if (tmp->flags & INAME_4) + dhcp4_ok = 0; + if (tmp->flags & INAME_6) + dhcp6_ok = 0; } #endif @@ -559,7 +572,8 @@ static int iface_allowed(struct iface_param *param, int if_index, char *label, iface->addr = *addr; iface->netmask = netmask; iface->tftp_ok = tftp_ok; - iface->dhcp_ok = dhcp_ok; + iface->dhcp4_ok = dhcp4_ok; + iface->dhcp6_ok = dhcp6_ok; iface->dns_auth = auth_dns; iface->mtu = mtu; iface->dad = !!(iface_flags & IFACE_TENTATIVE); @@ -699,8 +713,7 @@ static int release_listener(struct listener *l) /* In case it ever returns */ l->iface->done = 0; // Pi-hole modification - logg("stopped listening on %s(#%d): %s port %d", - l->iface->name, l->iface->index, daemon->addrbuff, port); + logg("stopped listening on %s(#%d): %s port %d", l->iface->name, l->iface->index, daemon->addrbuff, port); } if (l->fd != -1) @@ -915,15 +928,24 @@ static int make_sock(union mysockaddr *addr, int type, int dienow) errno = errsave; - if (dienow) + /* Failure to bind addresses given by --listen-address at this point + because there's no interface with the address is OK if we're doing bind-dynamic. + If/when an interface is created with the relevant address we'll notice + and attempt to bind it then. This is in the generic error path so we close the socket, + but EADDRNOTAVAIL is only a possible error from bind() + + When a new address is created and we call this code again (dienow == 0) there + may still be configured addresses when don't exist, (consider >1 --listen-address, + when the first is created, the second will still be missing) so we suppress + EADDRNOTAVAIL even in that case to avoid confusing log entries. + */ + if (!option_bool(OPT_CLEVERBIND) || errno != EADDRNOTAVAIL) { - /* failure to bind addresses given by --listen-address at this point - is OK if we're doing bind-dynamic */ - if (!option_bool(OPT_CLEVERBIND)) + if (dienow) die(s, daemon->addrbuff, EC_BADNET); + else + my_syslog(LOG_WARNING, s, daemon->addrbuff, strerror(errno)); } - else - my_syslog(LOG_WARNING, s, daemon->addrbuff, strerror(errno)); return -1; } @@ -1199,7 +1221,7 @@ void create_bound_listeners(int dienow) // Pi-hole modification const int port = prettyprint_addr(&iface->addr, daemon->addrbuff); logg("listening on %s(#%d): %s port %d", - iface->name, iface->index, daemon->addrbuff, port); + iface->name, iface->index, daemon->addrbuff, port); } } @@ -1215,7 +1237,7 @@ void create_bound_listeners(int dienow) (no netmask) and some MTU login the tftp code. */ for (if_tmp = daemon->if_addrs; if_tmp; if_tmp = if_tmp->next) - if (!if_tmp->used && + if (!(if_tmp->flags & INAME_USED) && (new = create_listeners(&if_tmp->addr, !!option_bool(OPT_TFTP), dienow))) { new->next = daemon->listeners; @@ -1227,7 +1249,7 @@ void create_bound_listeners(int dienow) my_syslog(LOG_DEBUG|MS_DEBUG, _("listening on %s port %d"), daemon->addrbuff, port); } // Pi-hole modification - const int port = prettyprint_addr(&if_tmp->addr, daemon->addrbuff); + const int port = prettyprint_addr(&if_tmp->addr, daemon->addrbuff); logg("listening on %s port %d", daemon->addrbuff, port); } } @@ -1306,7 +1328,7 @@ void join_multicast(int dienow) struct irec *iface, *tmp; for (iface = daemon->interfaces; iface; iface = iface->next) - if (iface->addr.sa.sa_family == AF_INET6 && iface->dhcp_ok && !iface->multicast_done) + if (iface->addr.sa.sa_family == AF_INET6 && iface->dhcp6_ok && !iface->multicast_done) { /* There's an irec per address but we only want to join for multicast once per interface. Weed out duplicates. */ diff --git a/src/dnsmasq/nftset.c b/src/dnsmasq/nftset.c index 4e152dc1..123326ca 100644 --- a/src/dnsmasq/nftset.c +++ b/src/dnsmasq/nftset.c @@ -1,4 +1,4 @@ -/* dnsmasq is Copyright (c) 2000-2022 Simon Kelley +/* dnsmasq is Copyright (c) 2000-2024 Simon Kelley This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -43,7 +43,8 @@ int add_to_nftset(const char *setname, const union all_addr *ipaddr, int flags, const char *cmd = remove ? cmd_del : cmd_add; int ret, af = (flags & F_IPV4) ? AF_INET : AF_INET6; size_t new_sz; - char *new, *err, *nl; + char *err_str, *new, *nl; + const char *err; static char *cmd_buf = NULL; static size_t cmd_buf_sz = 0; @@ -78,14 +79,19 @@ int add_to_nftset(const char *setname, const union all_addr *ipaddr, int flags, } ret = nft_run_cmd_from_buffer(ctx, cmd_buf); - err = (char *)nft_ctx_get_error_buffer(ctx); + err = nft_ctx_get_error_buffer(ctx); if (ret != 0) { /* Log only first line of error return. */ - if ((nl = strchr(err, '\n'))) - *nl = 0; - my_syslog(LOG_ERR, "nftset %s %s", setname, err); + if ((err_str = whine_malloc(strlen(err) + 1))) + { + strcpy(err_str, err); + if ((nl = strchr(err_str, '\n'))) + *nl = 0; + my_syslog(LOG_ERR, "nftset %s %s", setname, err_str); + free(err_str); + } } return ret; diff --git a/src/dnsmasq/option.c b/src/dnsmasq/option.c index 8f738995..39956385 100644 --- a/src/dnsmasq/option.c +++ b/src/dnsmasq/option.c @@ -1,4 +1,4 @@ -/* dnsmasq is Copyright (c) 2000-2022 Simon Kelley +/* dnsmasq is Copyright (c) 2000-2024 Simon Kelley This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -190,6 +190,11 @@ struct myoption { #define LOPT_STALE_CACHE 377 #define LOPT_NORR 378 #define LOPT_NO_IDENT 379 +#define LOPT_CACHE_RR 380 +#define LOPT_FILTER_RR 381 +#define LOPT_NO_DHCP6 382 +#define LOPT_NO_DHCP4 383 +#define LOPT_MAX_PROCS 384 #ifdef HAVE_GETOPT_LONG static const struct option opts[] = @@ -221,7 +226,7 @@ static const struct myoption opts[] = { "domain-suffix", 1, 0, 's' }, { "interface", 1, 0, 'i' }, { "listen-address", 1, 0, 'a' }, - { "local-service", 0, 0, LOPT_LOCAL_SERVICE }, + { "local-service", 2, 0, LOPT_LOCAL_SERVICE }, { "bogus-priv", 0, 0, 'b' }, { "bogus-nxdomain", 1, 0, 'B' }, { "ignore-address", 1, 0, LOPT_IGNORE_ADDR }, @@ -229,6 +234,7 @@ static const struct myoption opts[] = { "filterwin2k", 0, 0, 'f' }, { "filter-A", 0, 0, LOPT_FILTER_A }, { "filter-AAAA", 0, 0, LOPT_FILTER_AAAA }, + { "filter-rr", 1, 0, LOPT_FILTER_RR }, { "pid-file", 2, 0, 'x' }, { "strict-order", 0, 0, 'o' }, { "server", 1, 0, 'S' }, @@ -243,11 +249,14 @@ static const struct myoption opts[] = { "local-ttl", 1, 0, 'T' }, { "no-negcache", 0, 0, 'N' }, { "no-round-robin", 0, 0, LOPT_NORR }, + { "cache-rr", 1, 0, LOPT_CACHE_RR }, { "addn-hosts", 1, 0, 'H' }, { "hostsdir", 1, 0, LOPT_HOST_INOTIFY }, { "query-port", 1, 0, 'Q' }, { "except-interface", 1, 0, 'I' }, { "no-dhcp-interface", 1, 0, '2' }, + { "no-dhcpv4-interface", 1, 0, LOPT_NO_DHCP4 }, + { "no-dhcpv6-interface", 1, 0, LOPT_NO_DHCP6 }, { "domain-needed", 0, 0, 'D' }, { "dhcp-lease-max", 1, 0, 'X' }, { "bind-interfaces", 0, 0, 'z' }, @@ -380,6 +389,7 @@ static const struct myoption opts[] = { "fast-dns-retry", 2, 0, LOPT_FAST_RETRY }, { "use-stale-cache", 2, 0 , LOPT_STALE_CACHE }, { "no-ident", 0, 0, LOPT_NO_IDENT }, + { "max-tcp-connections", 1, 0, LOPT_MAX_PROCS }, { NULL, 0, 0, 0 } }; @@ -407,8 +417,9 @@ static struct { { 'e', OPT_SELFMX, NULL, gettext_noop("Return self-pointing MX records for local hosts."), NULL }, { 'E', OPT_EXPAND, NULL, gettext_noop("Expand simple names in /etc/hosts with domain-suffix."), NULL }, { 'f', OPT_FILTER, NULL, gettext_noop("Don't forward spurious DNS requests from Windows hosts."), NULL }, - { LOPT_FILTER_A, OPT_FILTER_A, NULL, gettext_noop("Don't include IPv4 addresses in DNS answers."), NULL }, - { LOPT_FILTER_AAAA, OPT_FILTER_AAAA, NULL, gettext_noop("Don't include IPv6 addresses in DNS answers."), NULL }, + { LOPT_FILTER_A, ARG_DUP, NULL, gettext_noop("Don't include IPv4 addresses in DNS answers."), NULL }, + { LOPT_FILTER_AAAA, ARG_DUP, NULL, gettext_noop("Don't include IPv6 addresses in DNS answers."), NULL }, + { LOPT_FILTER_RR, ARG_DUP, "", gettext_noop("Don't include resource records of the given type in DNS answers."), NULL }, { 'F', ARG_DUP, ",...", gettext_noop("Enable DHCP in the range given with lease duration."), NULL }, { 'g', ARG_ONE, "", gettext_noop("Change to this group after startup (defaults to %s)."), CHGRP }, { 'G', ARG_DUP, "", gettext_noop("Set address or hostname for a specified machine."), NULL }, @@ -477,6 +488,8 @@ static struct { { '1', ARG_ONE, "[=]", gettext_noop("Enable the DBus interface for setting upstream servers, etc."), NULL }, { LOPT_UBUS, ARG_ONE, "[=]", gettext_noop("Enable the UBus interface."), NULL }, { '2', ARG_DUP, "", gettext_noop("Do not provide DHCP on this interface, only provide DNS."), NULL }, + { LOPT_NO_DHCP6, ARG_DUP, "", gettext_noop("Do not provide DHCPv6 on this interface."), NULL }, + { LOPT_NO_DHCP4, ARG_DUP, "", gettext_noop("Do not provide DHCPv4 on this interface."), NULL }, { '3', ARG_DUP, "[=tag:]...", gettext_noop("Enable dynamic address allocation for bootp."), NULL }, { '4', ARG_DUP, "set:,", gettext_noop("Map MAC address (with wildcards) to option set."), NULL }, { LOPT_BRIDGE, ARG_DUP, ",..", gettext_noop("Treat DHCP requests on aliases as arriving from interface."), NULL }, @@ -564,19 +577,21 @@ static struct { { LOPT_QUIET_DHCP6, OPT_QUIET_DHCP6, NULL, gettext_noop("Do not log routine DHCPv6."), NULL }, { LOPT_QUIET_RA, OPT_QUIET_RA, NULL, gettext_noop("Do not log RA."), NULL }, { LOPT_LOG_DEBUG, OPT_LOG_DEBUG, NULL, gettext_noop("Log debugging information."), NULL }, - { LOPT_LOCAL_SERVICE, OPT_LOCAL_SERVICE, NULL, gettext_noop("Accept queries only from directly-connected networks."), NULL }, + { LOPT_LOCAL_SERVICE, ARG_ONE, NULL, gettext_noop("Accept queries only from directly-connected networks."), NULL }, { LOPT_LOOP_DETECT, OPT_LOOP_DETECT, NULL, gettext_noop("Detect and remove DNS forwarding loops."), NULL }, { LOPT_IGNORE_ADDR, ARG_DUP, "", gettext_noop("Ignore DNS responses containing ipaddr."), NULL }, { LOPT_DHCPTTL, ARG_ONE, "", gettext_noop("Set TTL in DNS responses with DHCP-derived addresses."), NULL }, { LOPT_REPLY_DELAY, ARG_ONE, "", gettext_noop("Delay DHCP replies for at least number of seconds."), NULL }, { LOPT_RAPID_COMMIT, OPT_RAPID_COMMIT, NULL, gettext_noop("Enables DHCPv4 Rapid Commit option."), NULL }, - { LOPT_DUMPFILE, ARG_ONE, "", gettext_noop("Path to debug packet dump file"), NULL }, - { LOPT_DUMPMASK, ARG_ONE, "", gettext_noop("Mask which packets to dump"), NULL }, + { LOPT_DUMPFILE, ARG_ONE, "", gettext_noop("Path to debug packet dump file."), NULL }, + { LOPT_DUMPMASK, ARG_ONE, "", gettext_noop("Mask which packets to dump."), NULL }, { LOPT_SCRIPT_TIME, OPT_LEASE_RENEW, NULL, gettext_noop("Call dhcp-script when lease expiry changes."), NULL }, { LOPT_UMBRELLA, ARG_ONE, "[=]", gettext_noop("Send Cisco Umbrella identifiers including remote IP."), NULL }, { LOPT_QUIET_TFTP, OPT_QUIET_TFTP, NULL, gettext_noop("Do not log routine TFTP."), NULL }, { LOPT_NORR, OPT_NORR, NULL, gettext_noop("Suppress round-robin ordering of DNS records."), NULL }, { LOPT_NO_IDENT, OPT_NO_IDENT, NULL, gettext_noop("Do not add CHAOS TXT records."), NULL }, + { LOPT_CACHE_RR, ARG_DUP, "", gettext_noop("Cache this DNS resource record type."), NULL }, + { LOPT_MAX_PROCS, ARG_ONE, "", gettext_noop("Maximum number of concurrent tcp connections."), NULL }, { 0, 0, NULL, NULL, NULL } }; @@ -1271,6 +1286,17 @@ static char *domain_rev6(int from_file, char *server, struct in6_addr *addr6, in return NULL; } +static void if_names_add(const char *ifname) +{ + struct iname *new = opt_malloc(sizeof(struct iname)); + new->next = daemon->if_names; + daemon->if_names = new; + /* new->name may be NULL if someone does + "interface=" to disable all interfaces except loop. */ + new->name = opt_string_alloc(ifname); + new->flags = 0; +} + #ifdef HAVE_DHCP static int is_tag_prefix(char *arg) @@ -1400,7 +1426,6 @@ static void dhcp_opt_free(struct dhcp_opt *opt) free(opt); } - /* This is too insanely large to keep in-line in the switch */ static int parse_dhcp_opt(char *errstr, char *arg, int flags) { @@ -2566,179 +2591,182 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma case 's': /* --domain */ case LOPT_SYNTH: /* --synth-domain */ - if (strcmp (arg, "#") == 0) - set_option_bool(OPT_RESOLV_DOMAIN); - else - { - char *d, *d_raw = arg; - comma = split(arg); - if (!(d = canonicalise_opt(d_raw))) - ret_err(gen_err); - else - { - free(d); /* allocate this again below. */ - if (comma) - { - struct cond_domain *new = opt_malloc(sizeof(struct cond_domain)); - char *netpart; - - new->prefix = NULL; - new->indexed = 0; - new->prefixlen = 0; - - unhide_metas(comma); - if ((netpart = split_chr(comma, '/'))) - { - int msize; - - arg = split(netpart); - if (!atoi_check(netpart, &msize)) - ret_err_free(gen_err, new); - else if (inet_pton(AF_INET, comma, &new->start)) - { - int mask; - - if (msize > 32) - ret_err_free(_("bad prefix length"), new); + { + char *d, *d_raw = arg; + comma = split(arg); + if (!(d = canonicalise_opt(d_raw))) + ret_err(gen_err); + else + { + free(d); /* allocate this again below. */ + if (comma) + { + struct cond_domain *new = opt_malloc(sizeof(struct cond_domain)); + char *netpart; + + new->prefix = NULL; + new->indexed = 0; + new->prefixlen = 0; + + unhide_metas(comma); + if ((netpart = split_chr(comma, '/'))) + { + int msize; + + arg = split(netpart); + if (!atoi_check(netpart, &msize)) + ret_err_free(gen_err, new); + else if (inet_pton(AF_INET, comma, &new->start)) + { + int mask; + + if (msize > 32) + ret_err_free(_("bad prefix length"), new); + + mask = (1 << (32 - msize)) - 1; + new->is6 = 0; + new->start.s_addr = ntohl(htonl(new->start.s_addr) & ~mask); + new->end.s_addr = new->start.s_addr | htonl(mask); + if (arg) + { + if (option != 's') + { + if (!(new->prefix = canonicalise_opt(arg)) || + strlen(new->prefix) > MAXLABEL - INET_ADDRSTRLEN) + ret_err_free(_("bad prefix"), new); + } + else if (strcmp(arg, "local") != 0) + ret_err_free(gen_err, new); + else + { + /* local=/xxx.yyy.zzz.in-addr.arpa/ */ + domain_rev4(0, NULL, &new->start, msize); + + /* local=// */ + /* d_raw can't failed to canonicalise here, checked above. */ + add_update_server(SERV_LITERAL_ADDRESS, NULL, NULL, NULL, d_raw, NULL); + } + } + } + else if (inet_pton(AF_INET6, comma, &new->start6)) + { + u64 mask, addrpart = addr6part(&new->start6); + + if (msize > 128) + ret_err_free(_("bad prefix length"), new); + + mask = (1LLU << (128 - msize)) - 1LLU; + + new->is6 = 1; + new->prefixlen = msize; + + /* prefix==64 overflows the mask calculation above */ + if (msize <= 64) + mask = (u64)-1LL; - mask = (1 << (32 - msize)) - 1; - new->is6 = 0; - new->start.s_addr = ntohl(htonl(new->start.s_addr) & ~mask); - new->end.s_addr = new->start.s_addr | htonl(mask); - if (arg) - { - if (option != 's') - { - if (!(new->prefix = canonicalise_opt(arg)) || - strlen(new->prefix) > MAXLABEL - INET_ADDRSTRLEN) - ret_err_free(_("bad prefix"), new); - } - else if (strcmp(arg, "local") != 0) - ret_err_free(gen_err, new); - else - { - /* local=/xxx.yyy.zzz.in-addr.arpa/ */ - domain_rev4(0, NULL, &new->start, msize); - - /* local=// */ - /* d_raw can't failed to canonicalise here, checked above. */ - add_update_server(SERV_LITERAL_ADDRESS, NULL, NULL, NULL, d_raw, NULL); - } - } - } - else if (inet_pton(AF_INET6, comma, &new->start6)) - { - u64 mask, addrpart = addr6part(&new->start6); - - if (msize > 128) - ret_err_free(_("bad prefix length"), new); - - mask = (1LLU << (128 - msize)) - 1LLU; - - new->is6 = 1; - new->prefixlen = msize; - - /* prefix==64 overflows the mask calculation above */ - if (msize <= 64) - mask = (u64)-1LL; - - new->end6 = new->start6; - setaddr6part(&new->start6, addrpart & ~mask); - setaddr6part(&new->end6, addrpart | mask); - - if (arg) - { - if (option != 's') - { - if (!(new->prefix = canonicalise_opt(arg)) || - strlen(new->prefix) > MAXLABEL - INET6_ADDRSTRLEN) - ret_err_free(_("bad prefix"), new); - } - else if (strcmp(arg, "local") != 0) - ret_err_free(gen_err, new); - else - { - /* generate the equivalent of - local=/xxx.yyy.zzz.ip6.arpa/ */ - domain_rev6(0, NULL, &new->start6, msize); - - /* local=// */ - /* d_raw can't failed to canonicalise here, checked above. */ - add_update_server(SERV_LITERAL_ADDRESS, NULL, NULL, NULL, d_raw, NULL); - } - } - } - else - ret_err_free(gen_err, new); - } - else - { - char *prefstr; - arg = split(comma); - prefstr = split(arg); - - if (inet_pton(AF_INET, comma, &new->start)) - { - new->is6 = 0; - if (!arg) - new->end.s_addr = new->start.s_addr; - else if (!inet_pton(AF_INET, arg, &new->end)) - ret_err_free(gen_err, new); - } - else if (inet_pton(AF_INET6, comma, &new->start6)) - { - new->is6 = 1; - if (!arg) - memcpy(&new->end6, &new->start6, IN6ADDRSZ); - else if (!inet_pton(AF_INET6, arg, &new->end6)) - ret_err_free(gen_err, new); - } - else if (option == 's') - { - /* subnet from interface. */ - new->interface = opt_string_alloc(comma); - new->al = NULL; - } - else - ret_err_free(gen_err, new); - - if (option != 's' && prefstr) - { - if (!(new->prefix = canonicalise_opt(prefstr)) || - strlen(new->prefix) > MAXLABEL - INET_ADDRSTRLEN) - ret_err_free(_("bad prefix"), new); - } - } - - new->domain = canonicalise_opt(d_raw); - if (option == 's') - { - new->next = daemon->cond_domain; - daemon->cond_domain = new; - } - else - { - char *star; - if (new->prefix && - (star = strrchr(new->prefix, '*')) - && *(star+1) == 0) - { - *star = 0; - new->indexed = 1; - if (new->is6 && new->prefixlen < 64) - ret_err_free(_("prefix length too small"), new); - } - new->next = daemon->synth_domains; - daemon->synth_domains = new; - } - } - else if (option == 's') - daemon->domain_suffix = canonicalise_opt(d_raw); - else - ret_err(gen_err); - } - } - break; + new->end6 = new->start6; + setaddr6part(&new->start6, addrpart & ~mask); + setaddr6part(&new->end6, addrpart | mask); + + if (arg) + { + if (option != 's') + { + if (!(new->prefix = canonicalise_opt(arg)) || + strlen(new->prefix) > MAXLABEL - INET6_ADDRSTRLEN) + ret_err_free(_("bad prefix"), new); + } + else if (strcmp(arg, "local") != 0) + ret_err_free(gen_err, new); + else + { + /* generate the equivalent of + local=/xxx.yyy.zzz.ip6.arpa/ */ + domain_rev6(0, NULL, &new->start6, msize); + + /* local=// */ + /* d_raw can't failed to canonicalise here, checked above. */ + add_update_server(SERV_LITERAL_ADDRESS, NULL, NULL, NULL, d_raw, NULL); + } + } + } + else + ret_err_free(gen_err, new); + } + else + { + char *prefstr; + arg = split(comma); + prefstr = split(arg); + + if (inet_pton(AF_INET, comma, &new->start)) + { + new->is6 = 0; + if (!arg) + new->end.s_addr = new->start.s_addr; + else if (!inet_pton(AF_INET, arg, &new->end)) + ret_err_free(gen_err, new); + } + else if (inet_pton(AF_INET6, comma, &new->start6)) + { + new->is6 = 1; + if (!arg) + memcpy(&new->end6, &new->start6, IN6ADDRSZ); + else if (!inet_pton(AF_INET6, arg, &new->end6)) + ret_err_free(gen_err, new); + } + else if (option == 's') + { + /* subnet from interface. */ + new->interface = opt_string_alloc(comma); + new->al = NULL; + } + else + ret_err_free(gen_err, new); + + if (option != 's' && prefstr) + { + if (!(new->prefix = canonicalise_opt(prefstr)) || + strlen(new->prefix) > MAXLABEL - INET_ADDRSTRLEN) + ret_err_free(_("bad prefix"), new); + } + } + + new->domain = canonicalise_opt(d_raw); + if (option == 's') + { + new->next = daemon->cond_domain; + daemon->cond_domain = new; + } + else + { + char *star; + if (new->prefix && + (star = strrchr(new->prefix, '*')) + && *(star+1) == 0) + { + *star = 0; + new->indexed = 1; + if (new->is6 && new->prefixlen < 64) + ret_err_free(_("prefix length too small"), new); + } + new->next = daemon->synth_domains; + daemon->synth_domains = new; + } + } + else if (option == 's') + { + if (strcmp (arg, "#") == 0) + set_option_bool(OPT_RESOLV_DOMAIN); + else + daemon->domain_suffix = canonicalise_opt(d_raw); + } + else + ret_err(gen_err); + } + + break; + } case LOPT_CPE_ID: /* --add-dns-client */ if (arg) @@ -2821,14 +2849,8 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma case 'i': /* --interface */ do { - struct iname *new = opt_malloc(sizeof(struct iname)); - comma = split(arg); - new->next = daemon->if_names; - daemon->if_names = new; - /* new->name may be NULL if someone does - "interface=" to disable all interfaces except loop. */ - new->name = opt_string_alloc(arg); - new->used = 0; + comma = split(arg); + if_names_add(arg); arg = comma; } while (arg); break; @@ -2841,10 +2863,13 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma case 'I': /* --except-interface */ case '2': /* --no-dhcp-interface */ + case LOPT_NO_DHCP6: /* --no-dhcpv6-interface */ + case LOPT_NO_DHCP4: /* --no-dhcpv4-interface */ do { struct iname *new = opt_malloc(sizeof(struct iname)); comma = split(arg); new->name = opt_string_alloc(arg); + new->flags = INAME_4 | INAME_6; if (option == 'I') { new->next = daemon->if_except; @@ -2857,6 +2882,10 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma } else { + if (option == LOPT_NO_DHCP6) + new->flags &= ~INAME_4; + if (option == LOPT_NO_DHCP4) + new->flags &= ~INAME_6; new->next = daemon->dhcp_except; daemon->dhcp_except = new; } @@ -2938,7 +2967,7 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma else ret_err_free(gen_err, new); - new->used = 0; + new->flags = 0; if (option == 'a') { new->next = daemon->if_addrs; @@ -3053,8 +3082,8 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma else flags &= ~SERV_FOR_NODOTS; - /* address=/#/ matches the same as without domain */ - if (option == 'A' && cur_domain[0] == '#' && cur_domain[1] == 0) + /* address=/#/ matches the same as without domain, as does server=/#/.... for consistency. */ + if (cur_domain[0] == '#' && cur_domain[1] == 0) cur_domain[0] = 0; } @@ -3387,6 +3416,15 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma ret_err(gen_err); else if (daemon->max_logs > 100) daemon->max_logs = 100; + break; + + case LOPT_LOCAL_SERVICE: /* --local-service */ + if (!arg || !strcmp(arg, "net")) + set_option_bool(OPT_LOCAL_SERVICE); + else if (!strcmp(arg, "host")) + set_option_bool(OPT_LOCALHOST_SERVICE); + else + ret_err(gen_err); break; case 'P': /* --edns-packet-max */ @@ -3447,7 +3485,7 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma break; } - case LOPT_FAST_RETRY: + case LOPT_FAST_RETRY: /* --fast-dns-retry */ daemon->fast_retry_timeout = TIMEOUT; if (!arg) @@ -3469,6 +3507,47 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma } } break; + + case LOPT_CACHE_RR: /* --cache-rr */ + case LOPT_FILTER_RR: /* --filter-rr */ + case LOPT_FILTER_A: /* --filter-A */ + case LOPT_FILTER_AAAA: /* --filter-AAAA */ + while (1) { + int type; + struct rrlist *new; + + comma = NULL; + + if (option == LOPT_FILTER_A) + type = T_A; + else if (option == LOPT_FILTER_AAAA) + type = T_AAAA; + else + { + comma = split(arg); + if (!atoi_check(arg, &type) && (type = rrtype(arg)) == 0) + ret_err(_("bad RR type")); + } + + new = opt_malloc(sizeof(struct rrlist)); + new->rr = type; + + if (option == LOPT_CACHE_RR) + { + new->next = daemon->cache_rr; + daemon->cache_rr = new; + } + else + { + new->next = daemon->filter_rr; + daemon->filter_rr = new; + } + + if (!comma) break; + arg = comma; + } + break; + #ifdef HAVE_DHCP case 'X': /* --dhcp-lease-max */ @@ -5164,7 +5243,7 @@ err: break; } - case LOPT_STALE_CACHE: + case LOPT_STALE_CACHE: /* --use-stale-cache */ { int max_expiry = STALE_CACHE_EXPIRY; if (arg) @@ -5254,7 +5333,17 @@ err: break; } #endif - + + case LOPT_MAX_PROCS: /* --max-tcp-connections */ + { + int max_procs; + /* Don't accept numbers less than 1. */ + if (!atoi_check(arg, &max_procs) || max_procs < 1) + ret_err(gen_err); + daemon->max_procs = max_procs; + break; + } + default: ret_err(_("unsupported option (check that dnsmasq was compiled with DHCP/TFTP/DNSSEC/DBus support)")); @@ -5675,11 +5764,11 @@ static void clear_dynamic_conf(void) } } -static void clear_dynamic_opt(void) +static void clear_dhcp_opt(struct dhcp_opt **dhcp_opts) { struct dhcp_opt *opts, *cp, **up; - for (up = &daemon->dhcp_opts, opts = daemon->dhcp_opts; opts; opts = cp) + for (up = dhcp_opts, opts = *dhcp_opts; opts; opts = cp) { cp = opts->next; @@ -5693,6 +5782,14 @@ static void clear_dynamic_opt(void) } } +static void clear_dynamic_opt(void) +{ + clear_dhcp_opt(&daemon->dhcp_opts); +#ifdef HAVE_DHCP6 + clear_dhcp_opt(&daemon->dhcp_opts6); +#endif +} + void reread_dhcp(void) { struct hostsfile *hf; @@ -5737,15 +5834,21 @@ void read_opts(int argc, char **argv, char *compile_opts) { size_t argbuf_size = MAXDNAME; char *argbuf = opt_malloc(argbuf_size); - char *buff = opt_malloc(MAXDNAME); + /* Note that both /000 and '.' are allowed within labels. These get + represented in presentation format using NAME_ESCAPE as an escape + character. In theory, if all the characters in a name were /000 or + '.' or NAME_ESCAPE then all would have to be escaped, so the + presentation format would be twice as long as the spec. */ + char *buff = opt_malloc((MAXDNAME * 2) + 1); int option, testmode = 0; char *arg, *conffile = NULL; - + opterr = 0; daemon = opt_malloc(sizeof(struct daemon)); memset(daemon, 0, sizeof(struct daemon)); daemon->namebuff = buff; + daemon->workspacename = safe_malloc((MAXDNAME * 2) + 1); daemon->addrbuff = safe_malloc(ADDRSTRLEN); /* Set defaults - everything else is zero or NULL */ @@ -5769,6 +5872,8 @@ void read_opts(int argc, char **argv, char *compile_opts) daemon->soa_expiry = SOA_EXPIRY; daemon->randport_limit = 1; daemon->host_index = SRC_AH; + daemon->max_procs = MAX_PROCS; + daemon->max_procs_used = 0; /* See comment above make_servers(). Optimises server-read code. */ mark_servers(0); @@ -5884,8 +5989,10 @@ void read_opts(int argc, char **argv, char *compile_opts) #endif add_txt("servers.bind", NULL, TXT_STAT_SERVERS); /* Pi-hole modification */ - add_txt("privacylevel.pihole", NULL, TXT_PRIVACYLEVEL); - add_txt("version.FTL", (char*)get_FTL_version(), 0 ); + add_txt("version.ftl", (char*)get_FTL_version(), 0 ); + add_txt("api.ftl", NULL, TXT_API_DOMAIN); + add_txt("domain.api.ftl", NULL, TXT_API_DOMAIN); + add_txt("local.api.ftl", NULL, TXT_API_LOCAL); /************************/ } #endif @@ -6070,7 +6177,16 @@ void read_opts(int argc, char **argv, char *compile_opts) /* If there's access-control config, then ignore --local-service, it's intended as a system default to keep otherwise unconfigured installations safe. */ if (daemon->if_names || daemon->if_except || daemon->if_addrs || daemon->authserver) - reset_option_bool(OPT_LOCAL_SERVICE); + { + reset_option_bool(OPT_LOCAL_SERVICE); + reset_option_bool(OPT_LOCALHOST_SERVICE); + } + else if (option_bool(OPT_LOCALHOST_SERVICE) && !option_bool(OPT_LOCAL_SERVICE)) + { + /* listen only on localhost, emulate --interface=lo --bind-interfaces */ + if_names_add(NULL); + set_option_bool(OPT_NOWILD); + } if (testmode) { @@ -6078,3 +6194,13 @@ void read_opts(int argc, char **argv, char *compile_opts) exit(0); } } + +/******************** Pi-hole extension ********************/ +void reset_usage_indicator(void) +{ + for (unsigned int i = 0; usage[i].opt != 0; i++) + if(usage[i].rept == ARG_USED_CL || + usage[i].rept == ARG_USED_FILE) + usage[i].rept = ARG_ONE; +} +/**********************************************************/ diff --git a/src/dnsmasq/outpacket.c b/src/dnsmasq/outpacket.c index abb3a3a4..1a29f613 100644 --- a/src/dnsmasq/outpacket.c +++ b/src/dnsmasq/outpacket.c @@ -1,4 +1,4 @@ -/* dnsmasq is Copyright (c) 2000-2022 Simon Kelley +/* dnsmasq is Copyright (c) 2000-2024 Simon Kelley This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by diff --git a/src/dnsmasq/pattern.c b/src/dnsmasq/pattern.c index e56e4956..cf19eb0f 100644 --- a/src/dnsmasq/pattern.c +++ b/src/dnsmasq/pattern.c @@ -1,4 +1,4 @@ -/* dnsmasq is Copyright (c) 2000-2022 Simon Kelley +/* dnsmasq is Copyright (c) 2000-2024 Simon Kelley This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by diff --git a/src/dnsmasq/poll.c b/src/dnsmasq/poll.c index bbb9009b..24568cc4 100644 --- a/src/dnsmasq/poll.c +++ b/src/dnsmasq/poll.c @@ -1,4 +1,4 @@ -/* dnsmasq is Copyright (c) 2000-2022 Simon Kelley +/* dnsmasq is Copyright (c) 2000-2024 Simon Kelley This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by diff --git a/src/dnsmasq/radv-protocol.h b/src/dnsmasq/radv-protocol.h index 7fb6bd82..1e77f2c0 100644 --- a/src/dnsmasq/radv-protocol.h +++ b/src/dnsmasq/radv-protocol.h @@ -1,4 +1,4 @@ -/* dnsmasq is Copyright (c) 2000-2022 Simon Kelley +/* dnsmasq is Copyright (c) 2000-2024 Simon Kelley This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by diff --git a/src/dnsmasq/radv.c b/src/dnsmasq/radv.c index 5820f4a6..d2d33905 100644 --- a/src/dnsmasq/radv.c +++ b/src/dnsmasq/radv.c @@ -1,4 +1,4 @@ -/* dnsmasq is Copyright (c) 2000-2022 Simon Kelley +/* dnsmasq is Copyright (c) 2000-2024 Simon Kelley This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -186,7 +186,8 @@ void icmp6_packet(time_t now) return; for (tmp = daemon->dhcp_except; tmp; tmp = tmp->next) - if (tmp->name && wildcard_match(tmp->name, interface)) + if (tmp->name && (tmp->flags & INAME_6) && + wildcard_match(tmp->name, interface)) return; if (packet[1] != 0) @@ -835,7 +836,8 @@ time_t periodic_ra(time_t now) { struct iname *tmp; for (tmp = daemon->dhcp_except; tmp; tmp = tmp->next) - if (tmp->name && wildcard_match(tmp->name, param.name)) + if (tmp->name && (tmp->flags & INAME_6) && + wildcard_match(tmp->name, param.name)) break; if (!tmp) { @@ -934,7 +936,8 @@ static int iface_search(struct in6_addr *local, int prefix, return 1; for (tmp = daemon->dhcp_except; tmp; tmp = tmp->next) - if (tmp->name && wildcard_match(tmp->name, param->name)) + if (tmp->name && (tmp->flags & INAME_6) && + wildcard_match(tmp->name, param->name)) return 1; for (context = daemon->dhcp6; context; context = context->next) diff --git a/src/dnsmasq/rfc1035.c b/src/dnsmasq/rfc1035.c index 523a8dbc..8146886c 100644 --- a/src/dnsmasq/rfc1035.c +++ b/src/dnsmasq/rfc1035.c @@ -1,4 +1,4 @@ -/* dnsmasq is Copyright (c) 2000-2022 Simon Kelley +/* dnsmasq is Copyright (c) 2000-2024 Simon Kelley This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -90,23 +90,14 @@ int extract_name(struct dns_header *header, size_t plen, unsigned char **pp, if (isExtract) { unsigned char c = *p; -#ifdef HAVE_DNSSEC - if (option_bool(OPT_DNSSEC_VALID)) + + if (c == 0 || c == '.' || c == NAME_ESCAPE) { - if (c == 0 || c == '.' || c == NAME_ESCAPE) - { - *cp++ = NAME_ESCAPE; - *cp++ = c+1; - } - else - *cp++ = c; + *cp++ = NAME_ESCAPE; + *cp++ = c+1; } else -#endif - if (c != 0 && c != '.') - *cp++ = c; - else - return 0; + *cp++ = c; } else { @@ -119,10 +110,9 @@ int extract_name(struct dns_header *header, size_t plen, unsigned char **pp, cp++; if (c1 >= 'A' && c1 <= 'Z') c1 += 'a' - 'A'; -#ifdef HAVE_DNSSEC - if (option_bool(OPT_DNSSEC_VALID) && c1 == NAME_ESCAPE) + + if (c1 == NAME_ESCAPE) c1 = (*cp++)-1; -#endif if (c2 >= 'A' && c2 <= 'Z') c2 += 'a' - 'A'; @@ -395,14 +385,23 @@ static int private_net6(struct in6_addr *a, int ban_localhost) ((u32 *)a)[0] == htonl(0x20010db8); /* RFC 6303 4.6 */ } -static unsigned char *do_doctor(unsigned char *p, int count, struct dns_header *header, size_t qlen, int *doctored) +int do_doctor(struct dns_header *header, size_t qlen, char *namebuff) { + unsigned char *p; int i, qtype, qclass, rdlen; - - for (i = count; i != 0; i--) + int done = 0; + + if (!(p = skip_questions(header, qlen))) + return done; + + for (i = 0; i < ntohs(header->ancount) + ntohs(header->arcount); i++) { - if (!(p = skip_name(p, header, qlen, 10))) - return 0; /* bad packet */ + /* Skip over auth section */ + if (i == ntohs(header->ancount) && !(p = skip_section(p, ntohs(header->nscount), header, qlen))) + return done; + + if (!extract_name(header, qlen, &p, namebuff, 1, 10)) + return done; /* bad packet */ GETSHORT(qtype, p); GETSHORT(qclass, p); @@ -412,103 +411,193 @@ static unsigned char *do_doctor(unsigned char *p, int count, struct dns_header * if (qclass == C_IN && qtype == T_A) { struct doctor *doctor; - struct in_addr addr; + union all_addr addr; if (!CHECK_LEN(header, p, qlen, INADDRSZ)) - return 0; + return done; /* alignment */ - memcpy(&addr, p, INADDRSZ); + memcpy(&addr.addr4, p, INADDRSZ); for (doctor = daemon->doctors; doctor; doctor = doctor->next) { if (doctor->end.s_addr == 0) { - if (!is_same_net(doctor->in, addr, doctor->mask)) + if (!is_same_net(doctor->in, addr.addr4, doctor->mask)) continue; } - else if (ntohl(doctor->in.s_addr) > ntohl(addr.s_addr) || - ntohl(doctor->end.s_addr) < ntohl(addr.s_addr)) + else if (ntohl(doctor->in.s_addr) > ntohl(addr.addr4.s_addr) || + ntohl(doctor->end.s_addr) < ntohl(addr.addr4.s_addr)) continue; - addr.s_addr &= ~doctor->mask.s_addr; - addr.s_addr |= (doctor->out.s_addr & doctor->mask.s_addr); + addr.addr4.s_addr &= ~doctor->mask.s_addr; + addr.addr4.s_addr |= (doctor->out.s_addr & doctor->mask.s_addr); /* Since we munged the data, the server it came from is no longer authoritative */ header->hb3 &= ~HB3_AA; - *doctored = 1; - memcpy(p, &addr, INADDRSZ); +#ifdef HAVE_DNSSEC + /* remove validated flag from this RR, since we changed it! */ + if (option_bool(OPT_DNSSEC_VALID) && i < ntohs(header->ancount)) + daemon->rr_status[i] = 0; +#endif + done = 1; + memcpy(p, &addr.addr4, INADDRSZ); + log_query(F_FORWARD | F_CONFIG | F_IPV4, namebuff, &addr, NULL, 0); break; } } if (!ADD_RDLEN(header, p, qlen, rdlen)) - return 0; /* bad packet */ + return done; /* bad packet */ } - - return p; + + return done; } -static int find_soa(struct dns_header *header, size_t qlen, int *doctored) +/* Find SOA RR in auth section to get TTL for negative caching of name. + Cache said SOA and return the difference in length between name and the name of the + SOA RR so we can look it up again. +*/ +static int find_soa(struct dns_header *header, size_t qlen, char *name, int *substring, unsigned long *ttlp, int no_cache, time_t now) { - unsigned char *p; + unsigned char *p, *psave; int qtype, qclass, rdlen; - unsigned long ttl, minttl = ULONG_MAX; - int i, found_soa = 0; - - /* first move to NS section and find TTL from any SOA section */ + unsigned long ttl, minttl; + int i, j; + size_t name_len, soa_len, len; + union all_addr addr; + + /* first move to NS section and find TTL from SOA RR */ if (!(p = skip_questions(header, qlen)) || - !(p = do_doctor(p, ntohs(header->ancount), header, qlen, doctored))) + !(p = skip_section(p, ntohs(header->ancount), header, qlen))) return 0; /* bad packet */ + + name_len = strlen(name); - for (i = ntohs(header->nscount); i != 0; i--) + if (substring) + *substring = name_len; + + if (ttlp) + *ttlp = daemon->neg_ttl; + + for (i = 0; i < ntohs(header->nscount); i++) { - if (!(p = skip_name(p, header, qlen, 10))) + if (!extract_name(header, qlen, &p, daemon->workspacename, 1, 0)) return 0; /* bad packet */ GETSHORT(qtype, p); GETSHORT(qclass, p); GETLONG(ttl, p); GETSHORT(rdlen, p); + + psave = p; if ((qclass == C_IN) && (qtype == T_SOA)) { - found_soa = 1; - if (ttl < minttl) - minttl = ttl; + soa_len = strlen(daemon->workspacename); - /* MNAME */ - if (!(p = skip_name(p, header, qlen, 0))) - return 0; - /* RNAME */ - if (!(p = skip_name(p, header, qlen, 20))) - return 0; - p += 16; /* SERIAL REFRESH RETRY EXPIRE */ - - GETLONG(ttl, p); /* minTTL */ - if (ttl < minttl) - minttl = ttl; + /* SOA must be for the name we're interested in. */ + if (soa_len <= name_len && memcmp(daemon->workspacename, name + name_len - soa_len, soa_len) == 0) + { + int prefix = name_len - soa_len; + + if (!no_cache) + { + if (!(addr.rrblock.rrdata = blockdata_alloc(NULL, 0))) + return 0; + addr.rrblock.rrtype = T_SOA; + addr.rrblock.datalen = 0; + } + + for (j = 0; j < 2; j++) /* MNAME, RNAME */ + { + if (!extract_name(header, qlen, &p, daemon->workspacename, 1, 0)) + { + if (!no_cache) + blockdata_free(addr.rrblock.rrdata); + return 0; + } + + if (!no_cache) + { + len = to_wire(daemon->workspacename); + if (!blockdata_expand(addr.rrblock.rrdata, addr.rrblock.datalen, daemon->workspacename, len)) + { + blockdata_free(addr.rrblock.rrdata); + return 0; + } + + addr.rrblock.datalen += len; + } + } + + if (!CHECK_LEN(header, p, qlen, 20)) + { + if (!no_cache) + blockdata_free(addr.rrblock.rrdata); + return 0; + } + + /* rest of RR */ + if (!no_cache && !blockdata_expand(addr.rrblock.rrdata, addr.rrblock.datalen, (char *)p, 20)) + { + blockdata_free(addr.rrblock.rrdata); + return 0; + } + + addr.rrblock.datalen += 20; + + if (!no_cache) + { + int secflag = 0; + +#ifdef HAVE_DNSSEC + if (option_bool(OPT_DNSSEC_VALID) && daemon->rr_status[i + ntohs(header->ancount)] != 0) + { + secflag = F_DNSSECOK; + + /* limit TTL based on signature. */ + if (daemon->rr_status[i + ntohs(header->ancount)] < ttl) + ttl = daemon->rr_status[i + ntohs(header->ancount)]; + } +#endif + + if (!cache_insert(name + prefix, &addr, C_IN, now, ttl, F_FORWARD | F_RR | F_KEYTAG | secflag)) + { + blockdata_free(addr.rrblock.rrdata); + return 0; + } + } + + p += 16; /* SERIAL REFRESH RETRY EXPIRE */ + + GETLONG(minttl, p); /* minTTL */ + if (ttl < minttl) + minttl = ttl; + + if (substring) + *substring = prefix; + + if (ttlp) + *ttlp = minttl; + + return 1; + } } - else if (!ADD_RDLEN(header, p, qlen, rdlen)) + + p = psave; + + if (!ADD_RDLEN(header, p, qlen, rdlen)) return 0; /* bad packet */ } - /* rewrite addresses in additional section too */ - if (!do_doctor(p, ntohs(header->arcount), header, qlen, doctored)) - return 0; - - if (!found_soa) - minttl = daemon->neg_ttl; - - return minttl; + return 0; } /* Print TXT reply to log */ -static int print_txt(struct dns_header *header, const size_t qlen, char *name, - unsigned char *p, const int ardlen, int secflag) +static int log_txt(char *name, unsigned char *p, const int ardlen, int secflag) { unsigned char *p1 = p; - if (!CHECK_LEN(header, p1, qlen, ardlen)) - return 0; + /* Loop over TXT payload */ while ((p1 - p) < ardlen) { @@ -527,7 +616,7 @@ static int print_txt(struct dns_header *header, const size_t qlen, char *name, } *p3 = 0; - log_query(secflag | F_FORWARD | F_UPSTREAM, name, NULL, (char*)p1, 0); + log_query(secflag | F_FORWARD, name, NULL, (char*)p1, 0); /* restore */ memmove(p1 + 1, p1, i); *p1 = len; @@ -545,10 +634,10 @@ static int print_txt(struct dns_header *header, const size_t qlen, char *name, */ int extract_addresses(struct dns_header *header, size_t qlen, char *name, time_t now, struct ipsets *ipsets, struct ipsets *nftsets, int is_sign, int check_rebind, - int no_cache_dnssec, int secure, int *doctored) + int no_cache_dnssec, int secure) { unsigned char *p, *p1, *endrr, *namep; - int j, qtype, qclass, aqtype, aqclass, ardlen, res, searched_soa = 0; + int j, qtype, qclass, aqtype, aqclass, ardlen, res; unsigned long ttl = 0; union all_addr addr; #ifdef HAVE_IPSET @@ -568,28 +657,9 @@ int extract_addresses(struct dns_header *header, size_t qlen, char *name, time_t int cname_short = 0; #endif unsigned long cttl = ULONG_MAX, attl; - + cache_start_insert(); - /* find_soa is needed for dns_doctor side effects, so don't call it lazily if there are any. */ - if (daemon->doctors || option_bool(OPT_DNSSEC_VALID)) - { - searched_soa = 1; - ttl = find_soa(header, qlen, doctored); - - if (*doctored) - { - if (secure) - return 0; -#ifdef HAVE_DNSSEC - if (option_bool(OPT_DNSSEC_VALID)) - for (j = 0; j < ntohs(header->ancount); j++) - if (daemon->rr_status[j] != 0) - return 0; -#endif - } - } - namep = p = (unsigned char *)(header+1); if (ntohs(header->qdcount) != 1 || !extract_name(header, qlen, &p, name, 1, 4)) @@ -638,7 +708,7 @@ int extract_addresses(struct dns_header *header, size_t qlen, char *name, time_t if (aqclass == C_IN && res != 2 && (aqtype == T_CNAME || aqtype == T_PTR)) { #ifdef HAVE_DNSSEC - if (option_bool(OPT_DNSSEC_VALID) && !no_cache_dnssec && daemon->rr_status[j] != 0) + if (option_bool(OPT_DNSSEC_VALID) && daemon->rr_status[j] != 0) { /* validated RR anywhere in CNAME chain, don't cache. */ if (cname_short || aqtype == T_CNAME) @@ -686,17 +756,16 @@ int extract_addresses(struct dns_header *header, size_t qlen, char *name, time_t if (!found && !option_bool(OPT_NO_NEG)) { - if (!searched_soa) - { - searched_soa = 1; - ttl = find_soa(header, qlen, doctored); - } + /* For reverse records, we use the name field to store the SOA name. */ + int substring, have_soa = find_soa(header, qlen, name, &substring, &ttl, no_cache_dnssec, now); flags |= F_NEG | (secure ? F_DNSSECOK : 0); if (name_encoding && ttl) { flags |= F_REVERSE | name_encoding; - cache_insert(NULL, &addr, C_IN, now, ttl, flags); + if (!have_soa) + flags |= F_NO_RR; /* Marks no SOA found. */ + cache_insert(name + substring, &addr, C_IN, now, ttl, flags); } log_query(flags | F_UPSTREAM, name, &addr, NULL, 0); @@ -718,8 +787,8 @@ int extract_addresses(struct dns_header *header, size_t qlen, char *name, time_t addrlen = IN6ADDRSZ; flags |= F_IPV6; } - else if (qtype == T_SRV) - flags |= F_SRV; + else if (qtype != T_CNAME && (qtype == T_SRV || rr_on_list(daemon->cache_rr, qtype))) + flags |= F_RR; else insert = 0; /* NOTE: do not cache data from CNAME queries. */ @@ -755,7 +824,7 @@ int extract_addresses(struct dns_header *header, size_t qlen, char *name, time_t } #ifdef HAVE_DNSSEC - if (option_bool(OPT_DNSSEC_VALID) && !no_cache_dnssec && daemon->rr_status[j] != 0) + if (option_bool(OPT_DNSSEC_VALID) && daemon->rr_status[j] != 0) { secflag = F_DNSSECOK; @@ -817,31 +886,103 @@ int extract_addresses(struct dns_header *header, size_t qlen, char *name, time_t #ifdef HAVE_DNSSEC if (!option_bool(OPT_DNSSEC_VALID) || aqtype != T_RRSIG) #endif - log_query(secflag | F_FORWARD | F_UPSTREAM, name, NULL, NULL, aqtype); + log_query(secflag | F_FORWARD | F_UPSTREAM | F_RRNAME, name, NULL, NULL, aqtype); } else if (!(flags & F_NXDOMAIN)) { found = 1; - if (flags & F_SRV) + if (flags & F_RR) { + short desc, *rrdesc = rrfilter_desc(aqtype); unsigned char *tmp = namep; - if (!CHECK_LEN(header, p1, qlen, 6)) + if (!CHECK_LEN(header, p1, qlen, ardlen)) return 2; /* bad packet */ - GETSHORT(addr.srv.priority, p1); - GETSHORT(addr.srv.weight, p1); - GETSHORT(addr.srv.srvport, p1); - if (!extract_name(header, qlen, &p1, name, 1, 0)) - return 2; - addr.srv.targetlen = strlen(name) + 1; /* include terminating zero */ - if (!(addr.srv.target = blockdata_alloc(name, addr.srv.targetlen))) - return 0; - /* we overwrote the original name, so get it back here. */ - if (!extract_name(header, qlen, &tmp, name, 1, 0)) - return 2; - } + /* If the data has no names and is small enough, store it in + the crec address field rather than allocate a block. */ + if (*rrdesc == -1 && ardlen <= (int)RR_IMDATALEN) + { + addr.rrdata.rrtype = aqtype; + addr.rrdata.datalen = (char)ardlen; + flags &= ~F_KEYTAG; /* in case of >1 answer, not all the same. */ + if (ardlen != 0) + memcpy(addr.rrdata.data, p1, ardlen); + } + else + { + addr.rrblock.rrtype = aqtype; + addr.rrblock.datalen = 0; + flags |= F_KEYTAG; /* discriminates between rrdata and rrblock */ + + /* The RR data may include names, and those names may include + compression, which will be rendered meaningless when + copied into another packet. + Here we go through a description of the packet type to + find the names, and extract them to a c-string and then + re-encode them to standalone DNS format without compression. */ + if (!(addr.rrblock.rrdata = blockdata_alloc(NULL, 0))) + return 0; + do + { + desc = *rrdesc++; + + if (desc == -1) + { + /* Copy the rest of the RR and end. */ + if (!blockdata_expand(addr.rrblock.rrdata, addr.rrblock.datalen, (char *)p1, endrr - p1)) + { + blockdata_free(addr.rrblock.rrdata); + return 0; + } + addr.rrblock.datalen += endrr - p1; + } + else if (desc == 0) + { + /* Name, extract it then re-encode. */ + int len; + + if (!extract_name(header, qlen, &p1, name, 1, 0)) + { + blockdata_free(addr.rrblock.rrdata); + return 2; + } + + len = to_wire(name); + if (!blockdata_expand(addr.rrblock.rrdata, addr.rrblock.datalen, name, len)) + { + blockdata_free(addr.rrblock.rrdata); + return 0; + } + + addr.rrblock.datalen += len; + } + else + { + /* desc is length of a block of data to be used as-is */ + if (desc > endrr - p1) + desc = endrr - p1; + + if (!blockdata_expand(addr.rrblock.rrdata, addr.rrblock.datalen, (char *)p1, desc)) + { + blockdata_free(addr.rrblock.rrdata); + return 0; + } + + addr.rrblock.datalen += desc; + p1 += desc; + } + } while (desc != -1); + + /* we overwrote the original name, so get it back here. */ + if (!extract_name(header, qlen, &tmp, name, 1, 0)) + { + blockdata_free(addr.rrblock.rrdata); + return 2; + } + } + } else if (flags & (F_IPV4 | F_IPV6)) { /* copy address into aligned storage */ @@ -885,15 +1026,32 @@ int extract_addresses(struct dns_header *header, size_t qlen, char *name, time_t cpp->addr.cname.uid = newc->uid; } cpp = NULL; + + /* cache insert failed, don't leak blockdata. */ + if (!newc && (flags & F_RR) && (flags & F_KEYTAG)) + blockdata_free(addr.rrblock.rrdata); } if (aqtype == T_TXT) { - if (!print_txt(header, qlen, name, p1, ardlen, secflag)) - return 2; + if (!CHECK_LEN(header, p1, qlen, ardlen)) + return 2; + + log_txt(name, p1, ardlen, secflag | F_UPSTREAM); } else - log_query(flags | F_FORWARD | secflag | F_UPSTREAM, name, &addr, NULL, aqtype); + { + int negflag = F_UPSTREAM; + + /* We're filtering this RRtype. It will be removed from the + returned packet in process_reply() but gets cached here anyway + and will be filtered again on the way out of the cache. Here, + we just need to alter the logging. */ + if (rr_on_list(daemon->filter_rr, qtype)) + negflag = F_NEG | F_CONFIG; + + log_query(negflag | flags | F_FORWARD | secflag, name, &addr, NULL, aqtype); + } } p1 = endrr; @@ -905,7 +1063,7 @@ int extract_addresses(struct dns_header *header, size_t qlen, char *name, time_t { if (flags & F_NXDOMAIN) { - flags &= ~(F_IPV4 | F_IPV6 | F_SRV); + flags &= ~(F_IPV4 | F_IPV6 | F_RR); /* Can store NXDOMAIN reply for any qtype. */ insert = 1; @@ -913,20 +1071,25 @@ int extract_addresses(struct dns_header *header, size_t qlen, char *name, time_t log_query(F_UPSTREAM | F_FORWARD | F_NEG | flags | (secure ? F_DNSSECOK : 0), name, NULL, NULL, 0); - if (!searched_soa) + if (insert && !option_bool(OPT_NO_NEG)) { - searched_soa = 1; - ttl = find_soa(header, qlen, doctored); - } - - /* If there's no SOA to get the TTL from, but there is a CNAME - pointing at this, inherit its TTL */ - if (insert && !option_bool(OPT_NO_NEG) && (ttl || cpp)) - { - if (ttl == 0) - ttl = cttl; + int substring, have_soa = find_soa(header, qlen, name, &substring, &ttl, no_cache_dnssec, now); - newc = cache_insert(name, NULL, C_IN, now, ttl, F_FORWARD | F_NEG | flags | (secure ? F_DNSSECOK : 0)); + /* If there's no SOA to get the TTL from, but there is a CNAME + pointing at this, inherit its TTL */ + if (ttl || cpp) + { + if (!ttl) + ttl = cttl; + + addr.rrdata.datalen = substring; + addr.rrdata.rrtype = qtype; + + if (!have_soa) + flags |= F_NO_RR; /* Marks no SOA found. */ + } + + newc = cache_insert(name, &addr, C_IN, now, ttl, F_FORWARD | F_NEG | flags | (secure ? F_DNSSECOK : 0)); if (newc && cpp) { next_uid(newc); @@ -937,15 +1100,10 @@ int extract_addresses(struct dns_header *header, size_t qlen, char *name, time_t } } - if (header->hb3 & HB3_TC) - log_query(F_UPSTREAM, NULL, NULL, "truncated", 0); - - /* Don't put stuff from a truncated packet into the cache. - Don't cache replies from non-recursive nameservers, since we may get a + /* Don't cache replies from non-recursive nameservers, since we may get a reply containing a CNAME but not its target, even though the target does exist. */ - if (!(header->hb3 & HB3_TC) && - !(header->hb4 & HB4_CD) && + if (!(header->hb4 & HB4_CD) && (header->hb4 & HB4_RA) && !no_cache_dnssec) cache_end_insert(); @@ -1149,6 +1307,10 @@ int check_for_local_domain(char *name, time_t now) if (cache_find_non_terminal(name, now)) return 1; + if (is_name_synthetic(F_IPV4, name, NULL) || + is_name_synthetic(F_IPV6, name, NULL)) + return 1; + return 0; } @@ -1174,8 +1336,7 @@ static int check_bad_address(struct dns_header *header, size_t qlen, struct bogu GETSHORT(qtype, p); GETSHORT(qclass, p); GETLONG(ttl, p); - GETSHORT(rdlen, p); - + GETSHORT(rdlen, p) if (ttlp) *ttlp = ttl; @@ -1228,8 +1389,9 @@ int check_for_bogus_wildcard(struct dns_header *header, size_t qlen, char *name, /* Found a bogus address. Insert that info here, since there no SOA record to get the ttl from in the normal processing */ cache_start_insert(); - cache_insert(name, NULL, C_IN, now, ttl, F_IPV4 | F_FORWARD | F_NEG | F_NXDOMAIN); + cache_insert(name, NULL, C_IN, now, ttl, F_FORWARD | F_NEG | F_NXDOMAIN); cache_end_insert(); + log_query(F_CONFIG | F_FORWARD | F_NEG | F_NXDOMAIN, name, NULL, NULL, 0); return 1; } @@ -1424,7 +1586,7 @@ static int cache_validated(const struct crec *crecp) size_t answer_request(struct dns_header *header, char *limit, size_t qlen, struct in_addr local_addr, struct in_addr local_netmask, time_t now, int ad_reqd, int do_bit, int have_pseudoheader, - int *stale) + int *stale, int *filtered) { char *name = daemon->namebuff; unsigned char *p, *ansp; @@ -1432,19 +1594,23 @@ size_t answer_request(struct dns_header *header, char *limit, size_t qlen, union all_addr addr; int nameoffset; unsigned short flag; - int q, ans, anscount = 0, addncount = 0; - int dryrun = 0; - struct crec *crecp; + int ans, anscount = 0, nscount = 0, addncount = 0; + struct crec *crecp, *soa_lookup = NULL; int nxdomain = 0, notimp = 0, auth = 1, trunc = 0, sec_data = 1; struct mx_srv_record *rec; size_t len; int rd_bit = (header->hb3 & HB3_RD); - + int count = 255; /* catch loops */ + if (stale) *stale = 0; + + if (filtered) + *filtered = 0; /* never answer queries with RD unset, to avoid cache snooping. */ - if (ntohs(header->ancount) != 0 || + if ( ntohs(header->qdcount) != 1 || + ntohs(header->ancount) != 0 || ntohs(header->nscount) != 0 || ntohs(header->qdcount) == 0 || OPCODE(header) != QUERY ) @@ -1454,16 +1620,9 @@ size_t answer_request(struct dns_header *header, char *limit, size_t qlen, if (header->hb4 & HB4_CD) sec_data = 0; - /* If there is an additional data section then it will be overwritten by - partial replies, so we have to do a dry run to see if we can answer - the query. */ - if (ntohs(header->arcount) != 0) - dryrun = 1; - for (rec = daemon->mxnames; rec; rec = rec->next) rec->offset = 0; - rerun: /* determine end of question section (we put answers there) */ if (!(ansp = skip_questions(header, qlen))) return 0; /* bad packet */ @@ -1471,670 +1630,695 @@ size_t answer_request(struct dns_header *header, char *limit, size_t qlen, /* now process each question, answers go in RRs after the question */ p = (unsigned char *)(header+1); - for (q = ntohs(header->qdcount); q != 0; q--) - { - int count = 255; /* catch loops */ - - /* save pointer to name for copying into answers */ - nameoffset = p - (unsigned char *)header; - - /* now extract name as .-concatenated string into name */ - if (!extract_name(header, qlen, &p, name, 1, 4)) - return 0; /* bad packet */ - - GETSHORT(qtype, p); - GETSHORT(qclass, p); - - ans = 0; /* have we answered this question */ - - if (qclass == C_IN) - while (--count != 0 && (crecp = cache_find_by_name(NULL, name, now, F_CNAME | F_NXDOMAIN))) + /* save pointer to name for copying into answers */ + nameoffset = p - (unsigned char *)header; + + /* now extract name as .-concatenated string into name */ + if (!extract_name(header, qlen, &p, name, 1, 4)) + return 0; /* bad packet */ + + GETSHORT(qtype, p); + GETSHORT(qclass, p); + + ans = 0; /* have we answered this question */ + + if (qclass == C_IN) + while (--count != 0 && (crecp = cache_find_by_name(NULL, name, now, F_CNAME | F_NXDOMAIN))) + { + char *cname_target; + int stale_flag = 0; + + if (crec_isstale(crecp, now)) { - char *cname_target; - int stale_flag = 0; - - if (crec_isstale(crecp, now)) - { - if (stale) - *stale = 1; - - stale_flag = F_STALE; - } - - if (crecp->flags & F_NXDOMAIN) - { - if (qtype == T_CNAME) - { - if (!dryrun) - log_query(stale_flag | crecp->flags, name, NULL, record_source(crecp->uid), 0); - auth = 0; - nxdomain = 1; - ans = 1; - } - break; - } - - cname_target = cache_get_cname_target(crecp); - - /* If the client asked for DNSSEC don't use cached data. */ - if ((crecp->flags & (F_HOSTS | F_DHCP | F_CONFIG)) || - (rd_bit && (!do_bit || cache_validated(crecp)))) - { - if (crecp->flags & F_CONFIG || qtype == T_CNAME) - ans = 1; - - if (!(crecp->flags & F_DNSSECOK)) - sec_data = 0; - - if (!dryrun) - { - log_query(stale_flag | crecp->flags, name, NULL, record_source(crecp->uid), 0); - if (add_resource_record(header, limit, &trunc, nameoffset, &ansp, - crec_ttl(crecp, now), &nameoffset, - T_CNAME, C_IN, "d", cname_target)) - anscount++; - } - - } - else - return 0; /* give up if any cached CNAME in chain can't be used for DNSSEC reasons. */ + if (stale) + *stale = 1; + stale_flag = F_STALE; + } + + if (crecp->flags & F_NEG) + soa_lookup = crecp; + + if (crecp->flags & F_NXDOMAIN) + { if (qtype == T_CNAME) - break; + { + log_query(stale_flag | crecp->flags, name, NULL, record_source(crecp->uid), 0); + auth = 0; + nxdomain = 1; + ans = 1; + } + break; + } + + cname_target = cache_get_cname_target(crecp); + + /* If the client asked for DNSSEC don't use cached data. */ + if ((crecp->flags & (F_HOSTS | F_DHCP | F_CONFIG)) || + (rd_bit && (!do_bit || cache_validated(crecp)))) + { + if (crecp->flags & F_CONFIG || qtype == T_CNAME) + ans = 1; - strcpy(name, cname_target); + if (!(crecp->flags & F_DNSSECOK)) + sec_data = 0; + + log_query(stale_flag | crecp->flags, name, NULL, record_source(crecp->uid), 0); + if (add_resource_record(header, limit, &trunc, nameoffset, &ansp, + crec_ttl(crecp, now), &nameoffset, + T_CNAME, C_IN, "d", cname_target)) + anscount++; + } + else + return 0; /* give up if any cached CNAME in chain can't be used for DNSSEC reasons. */ + + if (qtype == T_CNAME) + break; + + strcpy(name, cname_target); + } + + if (qtype == T_TXT || qtype == T_ANY) + { + struct txt_record *t; + for(t = daemon->txt; t ; t = t->next) + { + if (t->class == qclass && hostname_isequal(name, t->name)) + { + unsigned long ttl = daemon->local_ttl; + int ok = 1; + + ans = 1, sec_data = 0; +#ifndef NO_ID + /* Dynamically generate stat record */ + if (t->stat != 0) + { + ttl = 0; + if (!cache_make_stat(t)) + ok = 0; + } +#endif + if (ok) + { + log_query(F_CONFIG | F_RRNAME, name, NULL, "", 0); + if (add_resource_record(header, limit, &trunc, nameoffset, &ansp, + ttl, NULL, + T_TXT, t->class, "t", t->len, t->txt)) + anscount++; + } + } + } + } + + if (qclass == C_CHAOS) + { + /* don't forward *.bind and *.server chaos queries - always reply with NOTIMP */ + if (hostname_issubdomain("bind", name) || hostname_issubdomain("server", name)) + { + if (!ans) + { + notimp = 1, auth = 0; + + addr.log.rcode = NOTIMP; + log_query(F_CONFIG | F_RCODE, name, &addr, NULL, 0); + + ans = 1, sec_data = 0; + } + } + } + + if (qclass == C_IN) + { + struct txt_record *t; + + for (t = daemon->rr; t; t = t->next) + if ((t->class == qtype || qtype == T_ANY) && hostname_isequal(name, t->name)) + { + ans = 1; + sec_data = 0; + log_query(F_CONFIG | F_RRNAME, name, NULL, NULL, t->class); + if (add_resource_record(header, limit, &trunc, nameoffset, &ansp, + daemon->local_ttl, NULL, + t->class, C_IN, "t", t->len, t->txt)) + anscount++; } - if (qtype == T_TXT || qtype == T_ANY) + if (qtype == T_PTR || qtype == T_ANY) { - struct txt_record *t; - for(t = daemon->txt; t ; t = t->next) - { - if (t->class == qclass && hostname_isequal(name, t->name)) - { - ans = 1, sec_data = 0; - if (!dryrun) - { - unsigned long ttl = daemon->local_ttl; - int ok = 1; -#ifndef NO_ID - /* Dynamically generate stat record */ - if (t->stat != 0) - { - ttl = 0; - if (!cache_make_stat(t)) - ok = 0; - } -#endif - if (ok) - { - log_query(F_CONFIG | F_RRNAME, name, NULL, "", 0); - if (add_resource_record(header, limit, &trunc, nameoffset, &ansp, - ttl, NULL, - T_TXT, t->class, "t", t->len, t->txt)) - anscount++; - } - } - } - } - } - - if (qclass == C_CHAOS) - { - /* don't forward *.bind and *.server chaos queries - always reply with NOTIMP */ - if (hostname_issubdomain("bind", name) || hostname_issubdomain("server", name)) - { - if (!ans) - { - notimp = 1, auth = 0; - if (!dryrun) - { - addr.log.rcode = NOTIMP; - log_query(F_CONFIG | F_RCODE, name, &addr, NULL, 0); - } - ans = 1, sec_data = 0; - } - } - } - - if (qclass == C_IN) - { - struct txt_record *t; - - for (t = daemon->rr; t; t = t->next) - if ((t->class == qtype || qtype == T_ANY) && hostname_isequal(name, t->name)) - { - ans = 1; - sec_data = 0; - if (!dryrun) - { - log_query(F_CONFIG | F_RRNAME, name, NULL, NULL, t->class); - if (add_resource_record(header, limit, &trunc, nameoffset, &ansp, - daemon->local_ttl, NULL, - t->class, C_IN, "t", t->len, t->txt)) - anscount++; - } - } - - if (qtype == T_PTR || qtype == T_ANY) - { - /* see if it's w.z.y.z.in-addr.arpa format */ - int is_arpa = in_arpa_name_2_addr(name, &addr); - struct ptr_record *ptr; - struct interface_name* intr = NULL; - - for (ptr = daemon->ptr; ptr; ptr = ptr->next) - if (hostname_isequal(name, ptr->name)) - break; - - if (is_arpa == F_IPV4) - for (intr = daemon->int_names; intr; intr = intr->next) - { - struct addrlist *addrlist; - - for (addrlist = intr->addr; addrlist; addrlist = addrlist->next) - if (!(addrlist->flags & ADDRLIST_IPV6) && addr.addr4.s_addr == addrlist->addr.addr4.s_addr) - break; - - if (addrlist) - break; - else if (!(intr->flags & INP4)) - while (intr->next && strcmp(intr->intr, intr->next->intr) == 0) - intr = intr->next; - } - else if (is_arpa == F_IPV6) - for (intr = daemon->int_names; intr; intr = intr->next) - { - struct addrlist *addrlist; - - for (addrlist = intr->addr; addrlist; addrlist = addrlist->next) - if ((addrlist->flags & ADDRLIST_IPV6) && IN6_ARE_ADDR_EQUAL(&addr.addr6, &addrlist->addr.addr6)) - break; - - if (addrlist) - break; - else if (!(intr->flags & INP6)) - while (intr->next && strcmp(intr->intr, intr->next->intr) == 0) - intr = intr->next; - } - - if (intr) - { - sec_data = 0; - ans = 1; - if (!dryrun) - { - log_query(is_arpa | F_REVERSE | F_CONFIG, intr->name, &addr, NULL, 0); - if (add_resource_record(header, limit, &trunc, nameoffset, &ansp, - daemon->local_ttl, NULL, - T_PTR, C_IN, "d", intr->name)) - anscount++; - } - } - else if (ptr) - { - ans = 1; - sec_data = 0; - if (!dryrun) - { - log_query(F_CONFIG | F_RRNAME, name, NULL, "", 0); - for (ptr = daemon->ptr; ptr; ptr = ptr->next) - if (hostname_isequal(name, ptr->name) && - add_resource_record(header, limit, &trunc, nameoffset, &ansp, - daemon->local_ttl, NULL, - T_PTR, C_IN, "d", ptr->ptr)) - anscount++; - - } - } - else if (is_arpa && (crecp = cache_find_by_addr(NULL, &addr, now, is_arpa))) - { - /* Don't use cache when DNSSEC data required, unless we know that - the zone is unsigned, which implies that we're doing - validation. */ - if ((crecp->flags & (F_HOSTS | F_DHCP | F_CONFIG)) || - (rd_bit && (!do_bit || cache_validated(crecp)) )) - { - do - { - int stale_flag = 0; - - if (crec_isstale(crecp, now)) - { - if (stale) - *stale = 1; - - stale_flag = F_STALE; - } - - /* don't answer wildcard queries with data not from /etc/hosts or dhcp leases */ - if (qtype == T_ANY && !(crecp->flags & (F_HOSTS | F_DHCP))) - continue; - - - if (!(crecp->flags & F_DNSSECOK)) - sec_data = 0; - - ans = 1; - - if (crecp->flags & F_NEG) - { - auth = 0; - if (crecp->flags & F_NXDOMAIN) - nxdomain = 1; - if (!dryrun) - log_query(stale_flag | (crecp->flags & ~F_FORWARD), name, &addr, NULL, 0); - } - else - { - if (!(crecp->flags & (F_HOSTS | F_DHCP))) - auth = 0; - if (!dryrun) - { - log_query(stale_flag | (crecp->flags & ~F_FORWARD), cache_get_name(crecp), &addr, - record_source(crecp->uid), 0); - - if (add_resource_record(header, limit, &trunc, nameoffset, &ansp, - crec_ttl(crecp, now), NULL, - T_PTR, C_IN, "d", cache_get_name(crecp))) - anscount++; - } - } - } while ((crecp = cache_find_by_addr(crecp, &addr, now, is_arpa))); - } - } - else if (is_rev_synth(is_arpa, &addr, name)) - { - ans = 1; - sec_data = 0; - if (!dryrun) - { - log_query(F_CONFIG | F_REVERSE | is_arpa, name, &addr, NULL, 0); - - if (add_resource_record(header, limit, &trunc, nameoffset, &ansp, - daemon->local_ttl, NULL, - T_PTR, C_IN, "d", name)) - anscount++; - } - } - else if (option_bool(OPT_BOGUSPRIV) && - ((is_arpa == F_IPV6 && private_net6(&addr.addr6, 1)) || (is_arpa == F_IPV4 && private_net(addr.addr4, 1))) && - !lookup_domain(name, F_DOMAINSRV, NULL, NULL)) - { - /* if no configured server, not in cache, enabled and private IPV4 address, return NXDOMAIN */ - ans = 1; - sec_data = 0; - nxdomain = 1; - if (!dryrun) - log_query(F_CONFIG | F_REVERSE | is_arpa | F_NEG | F_NXDOMAIN, - name, &addr, NULL, 0); - } - } - - for (flag = F_IPV4; flag; flag = (flag == F_IPV4) ? F_IPV6 : 0) - { - unsigned short type = (flag == F_IPV6) ? T_AAAA : T_A; - struct interface_name *intr; - - if (qtype != type && qtype != T_ANY) - continue; - - /* interface name stuff */ - for (intr = daemon->int_names; intr; intr = intr->next) - if (hostname_isequal(name, intr->name)) - break; - - if (intr) - { - struct addrlist *addrlist; - int gotit = 0, localise = 0; - - enumerate_interfaces(0); - - /* See if a putative address is on the network from which we received - the query, is so we'll filter other answers. */ - if (local_addr.s_addr != 0 && option_bool(OPT_LOCALISE) && type == T_A) - for (intr = daemon->int_names; intr; intr = intr->next) - if (hostname_isequal(name, intr->name)) - for (addrlist = intr->addr; addrlist; addrlist = addrlist->next) - if (!(addrlist->flags & ADDRLIST_IPV6) && - is_same_net(addrlist->addr.addr4, local_addr, local_netmask)) - { - localise = 1; - break; - } - - for (intr = daemon->int_names; intr; intr = intr->next) - if (hostname_isequal(name, intr->name)) - { - for (addrlist = intr->addr; addrlist; addrlist = addrlist->next) - if (((addrlist->flags & ADDRLIST_IPV6) ? T_AAAA : T_A) == type) - { - if (localise && - !is_same_net(addrlist->addr.addr4, local_addr, local_netmask)) - continue; - - if (addrlist->flags & ADDRLIST_REVONLY) - continue; - - ans = 1; - sec_data = 0; - if (!dryrun) - { - gotit = 1; - log_query(F_FORWARD | F_CONFIG | flag, name, &addrlist->addr, NULL, 0); - if (add_resource_record(header, limit, &trunc, nameoffset, &ansp, - daemon->local_ttl, NULL, type, C_IN, - type == T_A ? "4" : "6", &addrlist->addr)) - anscount++; - } - } - } - - if (!dryrun && !gotit) - log_query(F_FORWARD | F_CONFIG | flag | F_NEG, name, NULL, NULL, 0); - - continue; - } - - if ((crecp = cache_find_by_name(NULL, name, now, flag | F_NXDOMAIN | (dryrun ? F_NO_RR : 0)))) - { - int localise = 0; - - /* See if a putative address is on the network from which we received - the query, is so we'll filter other answers. */ - if (local_addr.s_addr != 0 && option_bool(OPT_LOCALISE) && flag == F_IPV4) - { - struct crec *save = crecp; - do { - if ((crecp->flags & F_HOSTS) && - is_same_net(crecp->addr.addr4, local_addr, local_netmask)) - { - localise = 1; - break; - } - } while ((crecp = cache_find_by_name(crecp, name, now, flag))); - crecp = save; - } - - /* If the client asked for DNSSEC don't use cached data. */ - if ((crecp->flags & (F_HOSTS | F_DHCP | F_CONFIG)) || - (rd_bit && (!do_bit || cache_validated(crecp)) )) - do - { - int stale_flag = 0; - - if (crec_isstale(crecp, now)) - { - if (stale) - *stale = 1; - - stale_flag = F_STALE; - } - - /* don't answer wildcard queries with data not from /etc/hosts - or DHCP leases */ - if (qtype == T_ANY && !(crecp->flags & (F_HOSTS | F_DHCP | F_CONFIG))) - break; - - if (!(crecp->flags & F_DNSSECOK)) - sec_data = 0; - - if (crecp->flags & F_NEG) - { - ans = 1; - auth = 0; - if (crecp->flags & F_NXDOMAIN) - nxdomain = 1; - if (!dryrun) - // Pi-hole modification: Added record_source(crecp->uid) such that the subroutines know - // where the reply came from (e.g. gravity.list) - log_query(stale_flag | crecp->flags, name, NULL, record_source(crecp->uid), 0); - } - else - { - /* If we are returning local answers depending on network, - filter here. */ - if (localise && - (crecp->flags & F_HOSTS) && - !is_same_net(crecp->addr.addr4, local_addr, local_netmask)) - continue; - - if (!(crecp->flags & (F_HOSTS | F_DHCP))) - auth = 0; - - ans = 1; - if (!dryrun) - { - log_query(stale_flag | (crecp->flags & ~F_REVERSE), name, &crecp->addr, - record_source(crecp->uid), 0); - // ****************************** Pi-hole modification ****************************** - const char *src = crecp != NULL ? crecp->flags & F_BIGNAME ? crecp->name.bname->name : crecp->name.sname : NULL; - if(FTL_CNAME(name, src, daemon->log_display_id)) - { - // Served from cache. This can happen if a domain hidden in the CNAME path - // is only blocked for some but not all clients. In this case, the entire - // CNAME path may already be in the cache. - // This query is to be blocked as we found a blocked domain while walking the CNAME path. - // Log to pihole.log: "cached domainabc.com is blocked during CNAME inspection" - log_query(F_UPSTREAM, name, NULL, "blocked during CNAME inspection", 0); - break; - } - // ********************************************************************************** - - if (add_resource_record(header, limit, &trunc, nameoffset, &ansp, - crec_ttl(crecp, now), NULL, type, C_IN, - type == T_A ? "4" : "6", &crecp->addr)) - anscount++; - } - } - } while ((crecp = cache_find_by_name(crecp, name, now, flag))); - } - else if (is_name_synthetic(flag, name, &addr)) - { - ans = 1, sec_data = 0; - if (!dryrun) - { - log_query(F_FORWARD | F_CONFIG | flag, name, &addr, NULL, 0); - if (add_resource_record(header, limit, &trunc, nameoffset, &ansp, - daemon->local_ttl, NULL, type, C_IN, type == T_A ? "4" : "6", &addr)) - anscount++; - } - } - } - - if (qtype == T_MX || qtype == T_ANY) - { - int found = 0; - for (rec = daemon->mxnames; rec; rec = rec->next) - if (!rec->issrv && hostname_isequal(name, rec->name)) - { - ans = found = 1; - sec_data = 0; - if (!dryrun) - { - int offset; - log_query(F_CONFIG | F_RRNAME, name, NULL, "", 0); - if (add_resource_record(header, limit, &trunc, nameoffset, &ansp, daemon->local_ttl, - &offset, T_MX, C_IN, "sd", rec->weight, rec->target)) - { - anscount++; - if (rec->target) - rec->offset = offset; - } - } - } - - if (!found && (option_bool(OPT_SELFMX) || option_bool(OPT_LOCALMX)) && - cache_find_by_name(NULL, name, now, F_HOSTS | F_DHCP | F_NO_RR)) - { - ans = 1; - sec_data = 0; - if (!dryrun) - { - log_query(F_CONFIG | F_RRNAME, name, NULL, "", 0); - if (add_resource_record(header, limit, &trunc, nameoffset, &ansp, daemon->local_ttl, NULL, - T_MX, C_IN, "sd", 1, - option_bool(OPT_SELFMX) ? name : daemon->mxtarget)) - anscount++; - } - } - } - - if (qtype == T_SRV || qtype == T_ANY) - { - int found = 0; - struct mx_srv_record *move = NULL, **up = &daemon->mxnames; - - for (rec = daemon->mxnames; rec; rec = rec->next) - if (rec->issrv && hostname_isequal(name, rec->name)) - { - found = ans = 1; - sec_data = 0; - if (!dryrun) - { - int offset; - log_query(F_CONFIG | F_RRNAME, name, NULL, "", 0); - if (add_resource_record(header, limit, &trunc, nameoffset, &ansp, daemon->local_ttl, - &offset, T_SRV, C_IN, "sssd", - rec->priority, rec->weight, rec->srvport, rec->target)) - { - anscount++; - if (rec->target) - rec->offset = offset; - } - } - - /* unlink first SRV record found */ - if (!move) - { - move = rec; - *up = rec->next; - } - else - up = &rec->next; - } - else - up = &rec->next; - - /* put first SRV record back at the end. */ - if (move) - { - *up = move; - move->next = NULL; - } - - if (!found) - { - if ((crecp = cache_find_by_name(NULL, name, now, F_SRV | F_NXDOMAIN | (dryrun ? F_NO_RR : 0))) && - rd_bit && (!do_bit || (option_bool(OPT_DNSSEC_VALID) && !(crecp->flags & F_DNSSECOK)))) - do - { - int stale_flag = 0; - - if (crec_isstale(crecp, now)) - { - if (stale) - *stale = 1; - - stale_flag = F_STALE; - } - /* don't answer wildcard queries with data not from /etc/hosts or dhcp leases, except for NXDOMAIN */ - if (qtype == T_ANY && !(crecp->flags & (F_NXDOMAIN))) - break; - - if (!(crecp->flags & F_DNSSECOK)) - sec_data = 0; - - auth = 0; - found = ans = 1; - - if (crecp->flags & F_NEG) - { - if (crecp->flags & F_NXDOMAIN) - nxdomain = 1; - if (!dryrun) - log_query(stale_flag | crecp->flags, name, NULL, NULL, 0); - } - else if (!dryrun) - { - char *target = blockdata_retrieve(crecp->addr.srv.target, crecp->addr.srv.targetlen, NULL); - log_query(stale_flag | crecp->flags, name, NULL, NULL, 0); - - if (add_resource_record(header, limit, &trunc, nameoffset, &ansp, - crec_ttl(crecp, now), NULL, T_SRV, C_IN, "sssd", - crecp->addr.srv.priority, crecp->addr.srv.weight, crecp->addr.srv.srvport, - target)) - anscount++; - } - } while ((crecp = cache_find_by_name(crecp, name, now, F_SRV))); - } - - if (!found && option_bool(OPT_FILTER) && (qtype == T_SRV || (qtype == T_ANY && strchr(name, '_')))) - { - ans = 1; - sec_data = 0; - if (!dryrun) - log_query(F_CONFIG | F_NEG, name, NULL, NULL, 0); - } - } - - if (qtype == T_NAPTR || qtype == T_ANY) - { - struct naptr *na; - for (na = daemon->naptr; na; na = na->next) - if (hostname_isequal(name, na->name)) - { - ans = 1; - sec_data = 0; - if (!dryrun) - { - log_query(F_CONFIG | F_RRNAME, name, NULL, "", 0); - if (add_resource_record(header, limit, &trunc, nameoffset, &ansp, daemon->local_ttl, - NULL, T_NAPTR, C_IN, "sszzzd", - na->order, na->pref, na->flags, na->services, na->regexp, na->replace)) - anscount++; - } - } - } + /* see if it's w.z.y.z.in-addr.arpa format */ + int is_arpa = in_arpa_name_2_addr(name, &addr); + struct ptr_record *ptr; + struct interface_name* intr = NULL; - if (qtype == T_MAILB) - ans = 1, nxdomain = 1, sec_data = 0; - - if (qtype == T_SOA && option_bool(OPT_FILTER)) + for (ptr = daemon->ptr; ptr; ptr = ptr->next) + if (hostname_isequal(name, ptr->name)) + break; + + if (is_arpa == F_IPV4) + for (intr = daemon->int_names; intr; intr = intr->next) + { + struct addrlist *addrlist; + + for (addrlist = intr->addr; addrlist; addrlist = addrlist->next) + if (!(addrlist->flags & ADDRLIST_IPV6) && addr.addr4.s_addr == addrlist->addr.addr4.s_addr) + break; + + if (addrlist) + break; + else if (!(intr->flags & INP4)) + while (intr->next && strcmp(intr->intr, intr->next->intr) == 0) + intr = intr->next; + } + else if (is_arpa == F_IPV6) + for (intr = daemon->int_names; intr; intr = intr->next) + { + struct addrlist *addrlist; + + for (addrlist = intr->addr; addrlist; addrlist = addrlist->next) + if ((addrlist->flags & ADDRLIST_IPV6) && IN6_ARE_ADDR_EQUAL(&addr.addr6, &addrlist->addr.addr6)) + break; + + if (addrlist) + break; + else if (!(intr->flags & INP6)) + while (intr->next && strcmp(intr->intr, intr->next->intr) == 0) + intr = intr->next; + } + + if (intr) + { + sec_data = 0; + ans = 1; + log_query(is_arpa | F_REVERSE | F_CONFIG, intr->name, &addr, NULL, 0); + if (add_resource_record(header, limit, &trunc, nameoffset, &ansp, + daemon->local_ttl, NULL, + T_PTR, C_IN, "d", intr->name)) + anscount++; + } + else if (ptr) { ans = 1; sec_data = 0; - if (!dryrun) - log_query(F_CONFIG | F_NEG, name, &addr, NULL, 0); + log_query(F_CONFIG | F_RRNAME, name, NULL, "", 0); + for (ptr = daemon->ptr; ptr; ptr = ptr->next) + if (hostname_isequal(name, ptr->name) && + add_resource_record(header, limit, &trunc, nameoffset, &ansp, + daemon->local_ttl, NULL, + T_PTR, C_IN, "d", ptr->ptr)) + anscount++; + + } + else if (is_arpa && (crecp = cache_find_by_addr(NULL, &addr, now, is_arpa))) + { + /* Don't use cache when DNSSEC data required, unless we know that + the zone is unsigned, which implies that we're doing + validation. */ + if ((crecp->flags & (F_HOSTS | F_DHCP | F_CONFIG)) || + (rd_bit && (!do_bit || cache_validated(crecp)) )) + { + do + { + int stale_flag = 0; + + if (crec_isstale(crecp, now)) + { + if (stale) + *stale = 1; + + stale_flag = F_STALE; + } + + /* don't answer wildcard queries with data not from /etc/hosts or dhcp leases */ + if (qtype == T_ANY && !(crecp->flags & (F_HOSTS | F_DHCP))) + continue; + + if (!(crecp->flags & F_DNSSECOK)) + sec_data = 0; + + ans = 1; + + if (crecp->flags & F_NEG) + { + auth = 0; + if (crecp->flags & F_NXDOMAIN) + nxdomain = 1; + log_query(stale_flag | (crecp->flags & ~F_FORWARD), name, &addr, NULL, 0); + soa_lookup = crecp; + } + else + { + if (!(crecp->flags & (F_HOSTS | F_DHCP))) + auth = 0; + + log_query(stale_flag | (crecp->flags & ~F_FORWARD), cache_get_name(crecp), &addr, + record_source(crecp->uid), 0); + + if (add_resource_record(header, limit, &trunc, nameoffset, &ansp, + crec_ttl(crecp, now), NULL, + T_PTR, C_IN, "d", cache_get_name(crecp))) + anscount++; + } + } while ((crecp = cache_find_by_addr(crecp, &addr, now, is_arpa))); + } + } + else if (is_rev_synth(is_arpa, &addr, name)) + { + ans = 1; + sec_data = 0; + log_query(F_CONFIG | F_REVERSE | is_arpa, name, &addr, NULL, 0); + + if (add_resource_record(header, limit, &trunc, nameoffset, &ansp, + daemon->local_ttl, NULL, + T_PTR, C_IN, "d", name)) + anscount++; + } + else if (option_bool(OPT_BOGUSPRIV) && + ((is_arpa == F_IPV6 && private_net6(&addr.addr6, 1)) || (is_arpa == F_IPV4 && private_net(addr.addr4, 1))) && + !lookup_domain(name, F_DOMAINSRV, NULL, NULL)) + { + /* if no configured server, not in cache, enabled and private IPV4 address, return NXDOMAIN */ + ans = 1; + sec_data = 0; + nxdomain = 1; + log_query(F_CONFIG | F_REVERSE | is_arpa | F_NEG | F_NXDOMAIN, + name, &addr, NULL, 0); } } - + + for (flag = F_IPV4; flag; flag = (flag == F_IPV4) ? F_IPV6 : 0) + { + unsigned short type = (flag == F_IPV6) ? T_AAAA : T_A; + struct interface_name *intr; + + if (qtype != type && qtype != T_ANY) + continue; + + /* interface name stuff */ + for (intr = daemon->int_names; intr; intr = intr->next) + if (hostname_isequal(name, intr->name)) + break; + + if (intr) + { + struct addrlist *addrlist; + int gotit = 0, localise = 0; + + enumerate_interfaces(0); + + /* See if a putative address is on the network from which we received + the query, is so we'll filter other answers. */ + if (local_addr.s_addr != 0 && option_bool(OPT_LOCALISE) && type == T_A) + for (intr = daemon->int_names; intr; intr = intr->next) + if (hostname_isequal(name, intr->name)) + for (addrlist = intr->addr; addrlist; addrlist = addrlist->next) + if (!(addrlist->flags & ADDRLIST_IPV6) && + is_same_net(addrlist->addr.addr4, local_addr, local_netmask)) + { + localise = 1; + break; + } + + for (intr = daemon->int_names; intr; intr = intr->next) + if (hostname_isequal(name, intr->name)) + { + for (addrlist = intr->addr; addrlist; addrlist = addrlist->next) + if (((addrlist->flags & ADDRLIST_IPV6) ? T_AAAA : T_A) == type) + { + if (localise && + !is_same_net(addrlist->addr.addr4, local_addr, local_netmask)) + continue; + + if (addrlist->flags & ADDRLIST_REVONLY) + continue; + + ans = 1; + sec_data = 0; + gotit = 1; + log_query(F_FORWARD | F_CONFIG | flag, name, &addrlist->addr, NULL, 0); + if (add_resource_record(header, limit, &trunc, nameoffset, &ansp, + daemon->local_ttl, NULL, type, C_IN, + type == T_A ? "4" : "6", &addrlist->addr)) + anscount++; + } + } + + if (!gotit) + log_query(F_FORWARD | F_CONFIG | flag | F_NEG, name, NULL, NULL, 0); + + continue; + } + + if ((crecp = cache_find_by_name(NULL, name, now, flag))) + { + int localise = 0; + + /* See if a putative address is on the network from which we received + the query, is so we'll filter other answers. */ + if (!(crecp->flags & F_NEG) && local_addr.s_addr != 0 && option_bool(OPT_LOCALISE) && flag == F_IPV4) + { + struct crec *save = crecp; + do { + if ((crecp->flags & F_HOSTS) && + is_same_net(crecp->addr.addr4, local_addr, local_netmask)) + { + localise = 1; + break; + } + } while ((crecp = cache_find_by_name(crecp, name, now, flag))); + crecp = save; + } + + /* If the client asked for DNSSEC don't use cached data. */ + if ((crecp->flags & (F_HOSTS | F_DHCP | F_CONFIG)) || + (rd_bit && (!do_bit || cache_validated(crecp)) )) + do + { + int stale_flag = 0; + + if (crec_isstale(crecp, now)) + { + if (stale) + *stale = 1; + + stale_flag = F_STALE; + } + + /* don't answer wildcard queries with data not from /etc/hosts + or DHCP leases */ + if (qtype == T_ANY && !(crecp->flags & (F_HOSTS | F_DHCP | F_CONFIG))) + break; + + if (!(crecp->flags & F_DNSSECOK)) + sec_data = 0; + + if (!(crecp->flags & (F_HOSTS | F_DHCP))) + auth = 0; + + if (rr_on_list(daemon->filter_rr, qtype) && + !(crecp->flags & (F_HOSTS | F_DHCP | F_CONFIG | F_NEG))) + { + /* We have a cached answer but we're filtering it. */ + ans = 1; + sec_data = 0; + + log_query(F_NEG | F_CONFIG | flag, name, NULL, NULL, 0); + + if (filtered) + *filtered = 1; + } + else if (crecp->flags & F_NEG) + { + ans = 1; + auth = 0; + soa_lookup = crecp; + if (crecp->flags & F_NXDOMAIN) + nxdomain = 1; + + // Pi-hole modification: Added record_source(crecp->uid) such that the subroutines know + // where the reply came from (e.g. gravity.list) + log_query(stale_flag | crecp->flags, name, NULL, record_source(crecp->uid), 0); + } + else + { + /* If we are returning local answers depending on network, + filter here. */ + if (localise && + (crecp->flags & F_HOSTS) && + !is_same_net(crecp->addr.addr4, local_addr, local_netmask)) + continue; + + ans = 1; + log_query(stale_flag | (crecp->flags & ~F_REVERSE), name, &crecp->addr, + record_source(crecp->uid), 0); + // ****************************** Pi-hole modification ****************************** + const char *src = crecp != NULL ? crecp->flags & F_BIGNAME ? crecp->name.bname->name : crecp->name.sname : NULL; + if(FTL_CNAME(name, src, daemon->log_display_id)) + { + // Served from cache. This can happen if a domain hidden in the CNAME path + // is only blocked for some but not all clients. In this case, the entire + // CNAME path may already be in the cache. + // This query is to be blocked as we found a blocked domain while walking the CNAME path. + // Log to pihole.log: "cached domainabc.com is blocked during CNAME inspection" + log_query(F_UPSTREAM, name, NULL, "blocked during CNAME inspection", 0); + break; + } + // ********************************************************************************** + + if (add_resource_record(header, limit, &trunc, nameoffset, &ansp, + crec_ttl(crecp, now), NULL, type, C_IN, + type == T_A ? "4" : "6", &crecp->addr)) + anscount++; + } + } while ((crecp = cache_find_by_name(crecp, name, now, flag))); + } + else if (is_name_synthetic(flag, name, &addr)) + { + ans = 1, sec_data = 0; + log_query(F_FORWARD | F_CONFIG | flag, name, &addr, NULL, 0); + if (add_resource_record(header, limit, &trunc, nameoffset, &ansp, + daemon->local_ttl, NULL, type, C_IN, type == T_A ? "4" : "6", &addr)) + anscount++; + } + } + + if (qtype == T_MX || qtype == T_ANY) + { + int found = 0; + for (rec = daemon->mxnames; rec; rec = rec->next) + if (!rec->issrv && hostname_isequal(name, rec->name)) + { + int offset; + + ans = found = 1; + sec_data = 0; + + log_query(F_CONFIG | F_RRNAME, name, NULL, "", 0); + if (add_resource_record(header, limit, &trunc, nameoffset, &ansp, daemon->local_ttl, + &offset, T_MX, C_IN, "sd", rec->weight, rec->target)) + { + anscount++; + if (rec->target) + rec->offset = offset; + } + } + + if (!found && (option_bool(OPT_SELFMX) || option_bool(OPT_LOCALMX)) && + cache_find_by_name(NULL, name, now, F_HOSTS | F_DHCP | F_NO_RR)) + { + ans = 1; + sec_data = 0; + log_query(F_CONFIG | F_RRNAME, name, NULL, "", 0); + if (add_resource_record(header, limit, &trunc, nameoffset, &ansp, daemon->local_ttl, NULL, + T_MX, C_IN, "sd", 1, + option_bool(OPT_SELFMX) ? name : daemon->mxtarget)) + anscount++; + } + } + + if (qtype == T_SRV || qtype == T_ANY) + { + struct mx_srv_record *move = NULL, **up = &daemon->mxnames; + + for (rec = daemon->mxnames; rec; rec = rec->next) + if (rec->issrv && hostname_isequal(name, rec->name)) + { + int offset; + + ans = 1; + sec_data = 0; + log_query(F_CONFIG | F_RRNAME, name, NULL, "", 0); + if (add_resource_record(header, limit, &trunc, nameoffset, &ansp, daemon->local_ttl, + &offset, T_SRV, C_IN, "sssd", + rec->priority, rec->weight, rec->srvport, rec->target)) + { + anscount++; + if (rec->target) + rec->offset = offset; + } + + /* unlink first SRV record found */ + if (!move) + { + move = rec; + *up = rec->next; + } + else + up = &rec->next; + } + else + up = &rec->next; + + /* put first SRV record back at the end. */ + if (move) + { + *up = move; + move->next = NULL; + } + } + + if (qtype == T_NAPTR || qtype == T_ANY) + { + struct naptr *na; + for (na = daemon->naptr; na; na = na->next) + if (hostname_isequal(name, na->name)) + { + ans = 1; + sec_data = 0; + log_query(F_CONFIG | F_RRNAME, name, NULL, "", 0); + if (add_resource_record(header, limit, &trunc, nameoffset, &ansp, daemon->local_ttl, + NULL, T_NAPTR, C_IN, "sszzzd", + na->order, na->pref, na->flags, na->services, na->regexp, na->replace)) + anscount++; + } + } + + if (qtype == T_MAILB) + ans = 1, nxdomain = 1, sec_data = 0; + + if (qtype == T_SOA && option_bool(OPT_FILTER)) + { + ans = 1; + sec_data = 0; + log_query(F_CONFIG | F_NEG, name, &addr, NULL, 0); + } + if (!ans) { - /* We may know that the domain doesn't exist for any RRtype. */ - if ((crecp = cache_find_by_name(NULL, name, now, F_NXDOMAIN))) - { - ans = nxdomain = 1; - auth = 0; + if ((crecp = cache_find_by_name(NULL, name, now, F_RR | F_NXDOMAIN)) && + rd_bit && (!do_bit || cache_validated(crecp))) + do + { + int flags = crecp->flags; + unsigned short rrtype; + + if (flags & F_KEYTAG) + rrtype = crecp->addr.rrblock.rrtype; + else + rrtype = crecp->addr.rrdata.rrtype; + + if ((flags & F_NXDOMAIN) || rrtype == qtype) + { + char *rrdata = NULL; + unsigned short rrlen = 0; + + if (crec_isstale(crecp, now)) + { + if (stale) + *stale = 1; + + flags |= F_STALE; + } + + if (!(flags & F_DNSSECOK)) + sec_data = 0; + + if (flags & F_NXDOMAIN) + nxdomain = 1; + else if (rr_on_list(daemon->filter_rr, qtype)) + flags |= F_NEG | F_CONFIG; + + auth = 0; + ans = 1; - if (!(crecp->flags & F_DNSSECOK)) - sec_data = 0; + if (flags & F_NEG) + soa_lookup = crecp; + + if (!(flags & F_NEG)) + { + if (flags & F_KEYTAG) + { + rrlen = crecp->addr.rrblock.datalen; + rrdata = blockdata_retrieve(crecp->addr.rrblock.rrdata, crecp->addr.rrblock.datalen, NULL); + } + else + { + rrlen = crecp->addr.rrdata.datalen; + rrdata = crecp->addr.rrdata.data; + } + } + + if (!(flags & F_NEG) && add_resource_record(header, limit, &trunc, nameoffset, &ansp, + crec_ttl(crecp, now), NULL, qtype, C_IN, "t", + rrlen, rrdata)) + anscount++; + + /* log after cache insertion as log_txt mangles rrdata */ + if (qtype == T_TXT && !(crecp->flags & F_NEG)) + log_txt(name, (unsigned char *)rrdata, rrlen, crecp->flags & F_DNSSECOK); + else + log_query(flags, name, &crecp->addr, NULL, 0); + } + } while ((crecp = cache_find_by_name(crecp, name, now, F_RR))); + } + + if (!ans && option_bool(OPT_FILTER) && (qtype == T_SRV || (qtype == T_ANY && strchr(name, '_')))) + { + ans = 1; + sec_data = 0; + log_query(F_CONFIG | F_NEG, name, NULL, NULL, 0); + } + + + if (!ans && rr_on_list(daemon->filter_rr, qtype)) + { + /* We don't have a cached answer and when we get an answer from upstream we're going to + filter it anyway. If we have a cached answer for the domain for another RRtype then + that may be enough to tell us if the answer should be NODATA and save the round trip. + Cached NXDOMAIN has already been handled, so here we look for any record for the domain, + since its existence allows us to return a NODATA answer. Note that we never set the AD flag, + since we didn't authenticate the record. */ + + if (cache_find_by_name(NULL, name, now, F_IPV4 | F_IPV6 | F_RR | F_CNAME)) + { + ans = 1; + sec_data = auth = 0; - if (!dryrun) - log_query(F_NXDOMAIN | F_NEG, name, NULL, NULL, 0); + log_query(F_NEG | F_CONFIG | flag, name, NULL, NULL, 0); + + if (filtered) + *filtered = 1; } - else - return 0; /* failed to answer a question */ } } - if (dryrun) + if (!ans) + return 0; /* failed to answer a question */ + + /* We found a negative record. See if we have an SOA record to + return in the AUTH section. + + For FORWARD NEG records, the addr.rrdata.datalen field of the othewise + empty addr is used to held an offset in to the name which yields the SOA + name. For REVERSE NEG records, the otherwise empty name field holds the + SOA name. If soa_name has zero length, then no SOA is known. soa_lookup + MUST be a neg record here. + + If the F_NO_RR flag is set, there was no SOA record supplied with the RR. */ + if (soa_lookup && !(soa_lookup->flags & F_NO_RR)) { - dryrun = 0; - goto rerun; + char *soa_name = soa_lookup->flags & F_REVERSE ? cache_get_name(soa_lookup) : name + soa_lookup->addr.rrdata.datalen; + + crecp = NULL; + while ((crecp = cache_find_by_name(crecp, soa_name, now, F_RR))) + if (crecp->addr.rrblock.rrtype == T_SOA) + { + char *rrdata; + + if (!(crecp->flags & F_NEG) && + (rrdata = blockdata_retrieve(crecp->addr.rrblock.rrdata, crecp->addr.rrblock.datalen, NULL)) && + add_resource_record(header, limit, &trunc, 0, &ansp, + crec_ttl(crecp, now), NULL, T_SOA, C_IN, "t", + soa_name, crecp->addr.rrblock.datalen, rrdata)) + { + nscount++; + + if (!(crecp->flags & F_DNSSECOK)) + sec_data = 0; + } + break; + } } - + /* create an additional data section, for stuff in SRV and MX record replies. */ for (rec = daemon->mxnames; rec; rec = rec->next) if (rec->offset != 0) @@ -2156,7 +2340,11 @@ size_t answer_request(struct dns_header *header, char *limit, size_t qlen, if (add_resource_record(header, limit, NULL, rec->offset, &ansp, crec_ttl(crecp, now), NULL, type, C_IN, crecp->flags & F_IPV4 ? "4" : "6", &crecp->addr)) - addncount++; + { + addncount++; + if (!(crecp->flags & F_DNSSECOK)) + sec_data = 0; + } } } @@ -2181,7 +2369,7 @@ size_t answer_request(struct dns_header *header, char *limit, size_t qlen, else SET_RCODE(header, NOERROR); /* no error */ header->ancount = htons(anscount); - header->nscount = htons(0); + header->nscount = htons(nscount); header->arcount = htons(addncount); len = ansp - (unsigned char *)header; diff --git a/src/dnsmasq/rfc2131.c b/src/dnsmasq/rfc2131.c index 5190982d..42d148a0 100644 --- a/src/dnsmasq/rfc2131.c +++ b/src/dnsmasq/rfc2131.c @@ -1,4 +1,4 @@ -/* dnsmasq is Copyright (c) 2000-2022 Simon Kelley +/* dnsmasq is Copyright (c) 2000-2024 Simon Kelley This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -77,7 +77,7 @@ size_t dhcp_reply(struct dhcp_context *context, char *iface_name, int int_index, struct dhcp_vendor *vendor; struct dhcp_mac *mac; struct dhcp_netid_list *id_list; - int clid_len = 0, ignore = 0, do_classes = 0, rapid_commit = 0, selecting = 0, pxearch = -1; + int clid_len = 0, ignore = 0, do_classes = 0, rapidCommit = 0, selecting = 0, pxearch = -1; const char *pxevendor = NULL; struct dhcp_packet *mess = (struct dhcp_packet *)daemon->dhcp_packet.iov_base; unsigned char *end = (unsigned char *)(mess + 1); @@ -1157,14 +1157,14 @@ size_t dhcp_reply(struct dhcp_context *context, char *iface_name, int int_index, if (option_bool(OPT_RAPID_COMMIT) && option_find(mess, sz, OPTION_RAPID_COMMIT, 0)) { - rapid_commit = 1; + rapidCommit = 1; /* If a lease exists for this host and another address, squash it. */ if (lease && lease->addr.s_addr != mess->yiaddr.s_addr) { lease_prune(lease, now); lease = NULL; } - goto rapid_commit; + goto rapidCommit; } log_tags(tagif_netid, ntohl(mess->xid)); @@ -1285,7 +1285,7 @@ size_t dhcp_reply(struct dhcp_context *context, char *iface_name, int int_index, daemon->metrics[METRIC_DHCPREQUEST]++; log_packet("DHCPREQUEST", &mess->yiaddr, emac, emac_len, iface_name, NULL, NULL, mess->xid); - rapid_commit: + rapidCommit: if (!message) { struct dhcp_config *addr_config; @@ -1357,11 +1357,11 @@ size_t dhcp_reply(struct dhcp_context *context, char *iface_name, int int_index, if (message) { - daemon->metrics[rapid_commit ? METRIC_NOANSWER : METRIC_DHCPNAK]++; - log_packet(rapid_commit ? "NOANSWER" : "DHCPNAK", &mess->yiaddr, emac, emac_len, iface_name, NULL, message, mess->xid); + daemon->metrics[rapidCommit ? METRIC_NOANSWER : METRIC_DHCPNAK]++; + log_packet(rapidCommit ? "NOANSWER" : "DHCPNAK", &mess->yiaddr, emac, emac_len, iface_name, NULL, message, mess->xid); /* rapid commit case: lease allocate failed but don't send DHCPNAK */ - if (rapid_commit) + if (rapidCommit) return 0; mess->yiaddr.s_addr = 0; @@ -1523,7 +1523,7 @@ size_t dhcp_reply(struct dhcp_context *context, char *iface_name, int int_index, option_put(mess, end, OPTION_MESSAGE_TYPE, 1, DHCPACK); option_put(mess, end, OPTION_SERVER_IDENTIFIER, INADDRSZ, ntohl(server_id(context, override, fallback).s_addr)); option_put(mess, end, OPTION_LEASE_TIME, 4, time); - if (rapid_commit) + if (rapidCommit) option_put(mess, end, OPTION_RAPID_COMMIT, 0, 0); do_options(context, mess, end, req_options, hostname, get_domain(mess->yiaddr), netid, subnet_addr, fqdn_flags, borken_opt, pxearch, uuid, vendor_class_len, now, time, fuzz, pxevendor); diff --git a/src/dnsmasq/rfc3315.c b/src/dnsmasq/rfc3315.c index 477df91c..400d9396 100644 --- a/src/dnsmasq/rfc3315.c +++ b/src/dnsmasq/rfc3315.c @@ -1,4 +1,4 @@ -/* dnsmasq is Copyright (c) 2000-2022 Simon Kelley +/* dnsmasq is Copyright (c) 2000-2024 Simon Kelley This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -333,12 +333,29 @@ static int dhcp6_no_relay(struct state *state, int msg_type, unsigned char *inbu else if (msg_type != DHCP6IREQ) return 0; - /* server-id must match except for SOLICIT, CONFIRM and REBIND messages */ - if (msg_type != DHCP6SOLICIT && msg_type != DHCP6CONFIRM && msg_type != DHCP6IREQ && msg_type != DHCP6REBIND && - (!(opt = opt6_find(state->packet_options, state->end, OPTION6_SERVER_ID, 1)) || - opt6_len(opt) != daemon->duid_len || - memcmp(opt6_ptr(opt, 0), daemon->duid, daemon->duid_len) != 0)) - return 0; + /* server-id must match except for SOLICIT, CONFIRM and REBIND messages, which MUST NOT + have a server-id. 3315 para 15.x */ + opt = opt6_find(state->packet_options, state->end, OPTION6_SERVER_ID, 1); + + if (msg_type == DHCP6SOLICIT || msg_type == DHCP6CONFIRM || msg_type == DHCP6REBIND) + { + if (opt) + return 0; + } + else if (msg_type == DHCP6IREQ) + { + /* If server-id provided, it must match. */ + if (opt && (opt6_len(opt) != daemon->duid_len || + memcmp(opt6_ptr(opt, 0), daemon->duid, daemon->duid_len) != 0)) + return 0; + } + else + { + /* Everything else MUST have a server-id that matches ours. */ + if (!opt || opt6_len(opt) != daemon->duid_len || + memcmp(opt6_ptr(opt, 0), daemon->duid, daemon->duid_len) != 0) + return 0; + } o = new_opt6(OPTION6_SERVER_ID); put_opt6(daemon->duid, daemon->duid_len); @@ -457,6 +474,8 @@ static int dhcp6_no_relay(struct state *state, int msg_type, unsigned char *inbu state->tags = &mac_opt->netid; } } + else if (option_bool(OPT_LOG_OPTS)) + my_syslog(MS_DHCP | LOG_INFO, _("%u cannot determine client MAC address"), state->xid); if ((opt = opt6_find(state->packet_options, state->end, OPTION6_FQDN, 1))) { @@ -1055,7 +1074,7 @@ static int dhcp6_no_relay(struct state *state, int msg_type, unsigned char *inbu case DHCP6CONFIRM: { - int good_addr = 0; + int good_addr = 0, bad_addr = 0; /* set reply message type */ outmsgtype = DHCP6REPLY; @@ -1077,32 +1096,35 @@ static int dhcp6_no_relay(struct state *state, int msg_type, unsigned char *inbu if (!address6_valid(state->context, &req_addr, tagif, 1)) { - o1 = new_opt6(OPTION6_STATUS_CODE); - put_opt6_short(DHCP6NOTONLINK); - put_opt6_string(_("confirm failed")); - end_opt6(o1); + bad_addr = 1; log6_quiet(state, "DHCPREPLY", &req_addr, _("confirm failed")); - return 1; } - - good_addr = 1; - log6_quiet(state, "DHCPREPLY", &req_addr, state->hostname); + else + { + good_addr = 1; + log6_quiet(state, "DHCPREPLY", &req_addr, state->hostname); + } } } /* No addresses, no reply: RFC 3315 18.2.2 */ - if (!good_addr) + if (!good_addr && !bad_addr) return 0; o1 = new_opt6(OPTION6_STATUS_CODE); - put_opt6_short(DHCP6SUCCESS ); - put_opt6_string(_("all addresses still on link")); + put_opt6_short(bad_addr ? DHCP6NOTONLINK : DHCP6SUCCESS); + put_opt6_string(bad_addr ? (_("confirm failed")) : (_("all addresses still on link"))); end_opt6(o1); break; } case DHCP6IREQ: { + /* 3315 para 15.12 */ + if (opt6_find(state->packet_options, state->end, OPTION6_IA_NA, 1) || + opt6_find(state->packet_options, state->end, OPTION6_IA_TA, 1)) + return 0; + /* We can't discriminate contexts based on address, as we don't know it. If there is only one possible context, we can use its tags */ if (state->context && state->context->netid.net && !state->context->current) diff --git a/src/dnsmasq/rrfilter.c b/src/dnsmasq/rrfilter.c index 42d9c210..7c277fa4 100644 --- a/src/dnsmasq/rrfilter.c +++ b/src/dnsmasq/rrfilter.c @@ -1,4 +1,4 @@ -/* dnsmasq is Copyright (c) 2000-2022 Simon Kelley +/* dnsmasq is Copyright (c) 2000-2024 Simon Kelley This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -136,9 +136,9 @@ static int check_rrs(unsigned char *p, struct dns_header *header, size_t plen, i if (class == C_IN) { - u16 *d; + short *d; - for (pp = p, d = rrfilter_desc(type); *d != (u16)-1; d++) + for (pp = p, d = rrfilter_desc(type); *d != -1; d++) { if (*d != 0) pp += *d; @@ -156,41 +156,46 @@ static int check_rrs(unsigned char *p, struct dns_header *header, size_t plen, i } -/* mode may be remove EDNS0 or DNSSEC RRs or remove A or AAAA from answer section. */ -size_t rrfilter(struct dns_header *header, size_t plen, int mode) +/* mode may be remove EDNS0 or DNSSEC RRs or remove A or AAAA from answer section. + * returns number of modified records. */ +size_t rrfilter(struct dns_header *header, size_t *plen, int mode) { static unsigned char **rrs = NULL; static int rr_sz = 0; unsigned char *p = (unsigned char *)(header+1); - int i, rdlen, qtype, qclass, rr_found, chop_an, chop_ns, chop_ar; + size_t rr_found = 0; + int i, rdlen, qtype, qclass, chop_an, chop_ns, chop_ar; + if (mode == RRFILTER_CONF && !daemon->filter_rr) + return 0; + if (ntohs(header->qdcount) != 1 || - !(p = skip_name(p, header, plen, 4))) - return plen; + !(p = skip_name(p, header, *plen, 4))) + return 0; GETSHORT(qtype, p); GETSHORT(qclass, p); /* First pass, find pointers to start and end of all the records we wish to elide: records added for DNSSEC, unless explicitly queried for */ - for (rr_found = 0, chop_ns = 0, chop_an = 0, chop_ar = 0, i = 0; + for (chop_ns = 0, chop_an = 0, chop_ar = 0, i = 0; i < ntohs(header->ancount) + ntohs(header->nscount) + ntohs(header->arcount); i++) { unsigned char *pstart = p; int type, class; - if (!(p = skip_name(p, header, plen, 10))) - return plen; + if (!(p = skip_name(p, header, *plen, 10))) + return rr_found; GETSHORT(type, p); GETSHORT(class, p); p += 4; /* TTL */ GETSHORT(rdlen, p); - if (!ADD_RDLEN(header, p, plen, rdlen)) - return plen; + if (!ADD_RDLEN(header, p, *plen, rdlen)) + return rr_found; if (mode == RRFILTER_EDNS0) /* EDNS */ { @@ -217,15 +222,12 @@ size_t rrfilter(struct dns_header *header, size_t plen, int mode) if (class != C_IN) continue; - if (mode == RRFILTER_A && type != T_A) - continue; - - if (mode == RRFILTER_AAAA && type != T_AAAA) + if (!rr_on_list(daemon->filter_rr, type)) continue; } if (!expand_workspace(&rrs, &rr_sz, rr_found + 1)) - return plen; + return rr_found; rrs[rr_found++] = pstart; rrs[rr_found++] = p; @@ -240,7 +242,7 @@ size_t rrfilter(struct dns_header *header, size_t plen, int mode) /* Nothing to do. */ if (rr_found == 0) - return plen; + return rr_found; /* Second pass, look for pointers in names in the records we're keeping and make sure they don't point to records we're going to elide. This is theoretically possible, but unlikely. If @@ -248,42 +250,42 @@ size_t rrfilter(struct dns_header *header, size_t plen, int mode) p = (unsigned char *)(header+1); /* question first */ - if (!check_name(&p, header, plen, 0, rrs, rr_found)) - return plen; + if (!check_name(&p, header, *plen, 0, rrs, rr_found)) + return rr_found; p += 4; /* qclass, qtype */ /* Now answers and NS */ - if (!check_rrs(p, header, plen, 0, rrs, rr_found)) - return plen; + if (!check_rrs(p, header, *plen, 0, rrs, rr_found)) + return rr_found; /* Third pass, actually fix up pointers in the records */ p = (unsigned char *)(header+1); - check_name(&p, header, plen, 1, rrs, rr_found); + check_name(&p, header, *plen, 1, rrs, rr_found); p += 4; /* qclass, qtype */ - check_rrs(p, header, plen, 1, rrs, rr_found); + check_rrs(p, header, *plen, 1, rrs, rr_found); /* Fourth pass, elide records */ - for (p = rrs[0], i = 1; i < rr_found; i += 2) + for (p = rrs[0], i = 1; (unsigned)i < rr_found; i += 2) { unsigned char *start = rrs[i]; - unsigned char *end = (i != rr_found - 1) ? rrs[i+1] : ((unsigned char *)header) + plen; + unsigned char *end = ((unsigned)i != rr_found - 1) ? rrs[i+1] : ((unsigned char *)header) + *plen; memmove(p, start, end-start); p += end-start; } - plen = p - (unsigned char *)header; + *plen = p - (unsigned char *)header; header->ancount = htons(ntohs(header->ancount) - chop_an); header->nscount = htons(ntohs(header->nscount) - chop_ns); header->arcount = htons(ntohs(header->arcount) - chop_ar); - return plen; + return rr_found; } /* This is used in the DNSSEC code too, hence it's exported */ -u16 *rrfilter_desc(int type) +short *rrfilter_desc(int type) { /* List of RRtypes which include domains in the data. 0 -> domain @@ -294,7 +296,7 @@ u16 *rrfilter_desc(int type) anything which needs no mangling. */ - static u16 rr_desc[] = + static short rr_desc[] = { T_NS, 0, -1, T_MD, 0, -1, @@ -319,10 +321,10 @@ u16 *rrfilter_desc(int type) 0, -1 /* wildcard/catchall */ }; - u16 *p = rr_desc; + short *p = rr_desc; while (*p != type && *p != 0) - while (*p++ != (u16)-1); + while (*p++ != -1); return p+1; } @@ -350,3 +352,78 @@ int expand_workspace(unsigned char ***wkspc, int *szp, int new) return 1; } + +/* Convert from presentation format to wire format, in place. + Also map UC -> LC. + Note that using extract_name to get presentation format + then calling to_wire() removes compression and maps case, + thus generating names in canonical form. + Calling to_wire followed by from_wire is almost an identity, + except that the UC remains mapped to LC. + + Note that both /000 and '.' are allowed within labels. These get + represented in presentation format using NAME_ESCAPE as an escape + character. In theory, if all the characters in a name were /000 or + '.' or NAME_ESCAPE then all would have to be escaped, so the + presentation format would be twice as long as the spec (1024). + The buffers are all declared as 2049 (allowing for the trailing zero) + for this reason. +*/ +int to_wire(char *name) +{ + unsigned char *l, *p, *q, term; + int len; + + for (l = (unsigned char*)name; *l != 0; l = p) + { + for (p = l; *p != '.' && *p != 0; p++) + if (*p >= 'A' && *p <= 'Z') + *p = *p - 'A' + 'a'; + else if (*p == NAME_ESCAPE) + { + for (q = p; *q; q++) + *q = *(q+1); + (*p)--; + } + term = *p; + + if ((len = p - l) != 0) + memmove(l+1, l, len); + *l = len; + + p++; + + if (term == 0) + *p = 0; + } + + return l + 1 - (unsigned char *)name; +} + +/* Note: no compression allowed in input. */ +void from_wire(char *name) +{ + unsigned char *l, *p, *last; + int len; + + for (last = (unsigned char *)name; *last != 0; last += *last+1); + + for (l = (unsigned char *)name; *l != 0; l += len+1) + { + len = *l; + memmove(l, l+1, len); + for (p = l; p < l + len; p++) + if (*p == '.' || *p == 0 || *p == NAME_ESCAPE) + { + memmove(p+1, p, 1 + last - p); + len++; + *p++ = NAME_ESCAPE; + (*p)++; + } + + l[len] = '.'; + } + + if ((char *)l != name) + *(l-1) = 0; +} diff --git a/src/dnsmasq/slaac.c b/src/dnsmasq/slaac.c index 7d3fce48..c37e7ff2 100644 --- a/src/dnsmasq/slaac.c +++ b/src/dnsmasq/slaac.c @@ -1,4 +1,4 @@ -/* dnsmasq is Copyright (c) 2000-2022 Simon Kelley +/* dnsmasq is Copyright (c) 2000-2024 Simon Kelley This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by diff --git a/src/dnsmasq/tftp.c b/src/dnsmasq/tftp.c index 8e1dc4ae..4421cf93 100644 --- a/src/dnsmasq/tftp.c +++ b/src/dnsmasq/tftp.c @@ -1,4 +1,4 @@ -/* dnsmasq is Copyright (c) 2000-2022 Simon Kelley +/* dnsmasq is Copyright (c) 2000-2024 Simon Kelley This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -228,7 +228,8 @@ void tftp_request(struct listener *listen, time_t now) #ifdef HAVE_DHCP /* allowed interfaces are the same as for DHCP */ for (tmp = daemon->dhcp_except; tmp; tmp = tmp->next) - if (tmp->name && wildcard_match(tmp->name, name)) + if (tmp->name && (tmp->flags & INAME_4) && (tmp->flags & INAME_6) && + wildcard_match(tmp->name, name)) return; #endif } @@ -584,8 +585,13 @@ static struct tftp_file *check_tftp_fileperm(ssize_t *len, char *prefix, char *c void check_tftp_listeners(time_t now) { + struct listener *listener; struct tftp_transfer *transfer, *tmp, **up; + for (listener = daemon->listeners; listener; listener = listener->next) + if (listener->tftpfd != -1 && poll_check(listener->tftpfd, POLLIN)) + tftp_request(listener, now); + /* In single port mode, all packets come via port 69 and tftp_request() */ if (!option_bool(OPT_SINGLE_PORT)) for (transfer = daemon->tftp_trans; transfer; transfer = transfer->next) diff --git a/src/dnsmasq/ubus.c b/src/dnsmasq/ubus.c index 09071cfc..a5758e77 100644 --- a/src/dnsmasq/ubus.c +++ b/src/dnsmasq/ubus.c @@ -1,4 +1,4 @@ -/* dnsmasq is Copyright (c) 2000-2022 Simon Kelley +/* dnsmasq is Copyright (c) 2000-2024 Simon Kelley This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by diff --git a/src/dnsmasq/util.c b/src/dnsmasq/util.c index e0ce67d3..3ac88354 100644 --- a/src/dnsmasq/util.c +++ b/src/dnsmasq/util.c @@ -1,4 +1,4 @@ -/* dnsmasq is Copyright (c) 2000-2022 Simon Kelley +/* dnsmasq is Copyright (c) 2000-2024 Simon Kelley This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -115,6 +115,19 @@ u64 rand64(void) return (u64)out[outleft+1] + (((u64)out[outleft]) << 32); } +int rr_on_list(struct rrlist *list, unsigned short rr) +{ + while (list) + { + if (list->rr == rr || list->rr == T_ANY) + return 1; + + list = list->next; + } + + return 0; +} + /* returns 1 if name is OK and ascii printable * returns 2 if name should be processed by IDN */ static int check_name(char *in) @@ -280,11 +293,9 @@ unsigned char *do_rfc1035_name(unsigned char *p, char *sval, char *limit) if (limit && p + 1 > (unsigned char*)limit) return NULL; -#ifdef HAVE_DNSSEC - if (option_bool(OPT_DNSSEC_VALID) && *sval == NAME_ESCAPE) + if (*sval == NAME_ESCAPE) *p++ = (*(++sval))-1; else -#endif *p++ = *sval; } diff --git a/src/main.c b/src/main.c index 1f7727af..f55c5ba9 100644 --- a/src/main.c +++ b/src/main.c @@ -113,7 +113,7 @@ int main (int argc, char* argv[]) for(int i = 0; i < argc_dnsmasq; i++) logg("DEBUG: argv[%i] = \"%s\"", i, argv_dnsmasq[i]); } - main_dnsmasq(argc_dnsmasq, argv_dnsmasq); + main_dnsmasq(argc_dnsmasq, (char**)argv_dnsmasq); logg("Shutting down..."); // Extra grace time is needed as dnsmasq script-helpers may not be diff --git a/src/main.h b/src/main.h index 0c3a5007..536000ae 100644 --- a/src/main.h +++ b/src/main.h @@ -10,7 +10,7 @@ #ifndef MAIN_H #define MAIN_H -int main_dnsmasq(int argc, const char ** argv); +int main_dnsmasq(int argc, char ** argv); extern char *username; extern bool startup; diff --git a/src/signals.h b/src/signals.h index defb7c78..7bacbc27 100644 --- a/src/signals.h +++ b/src/signals.h @@ -12,6 +12,8 @@ #include "enums.h" +#define SIGUSR6 (SIGRTMIN + 6) + void handle_signals(void); void handle_realtime_signals(void); pid_t main_pid(void); diff --git a/test/dnsmasq_warnings b/test/dnsmasq_warnings index 94b23792..3becd253 100644 --- a/test/dnsmasq_warnings +++ b/test/dnsmasq_warnings @@ -17,6 +17,10 @@ src/dnsmasq/cache.c "the name exists in %s with address %s"), host_name, daemon->addrbuff, record_source(fail_crec->uid), daemon->namebuff); +src/dnsmasq/dhcp6.c + my_syslog(MS_DHCP | LOG_WARNING, + _("Working around kernel bug: faulty source address scope for VRF slave %s"), + ifr.ifr_name); src/dnsmasq/dhcp6.c my_syslog(MS_DHCP | LOG_WARNING, _("unknown interface %s in bridge-interface"), @@ -81,7 +85,7 @@ src/dnsmasq/forward.c src/dnsmasq/forward.c my_syslog(LOG_WARNING, _("nameserver %s refused to do a recursive query"), daemon->namebuff); src/dnsmasq/forward.c - my_syslog(LOG_WARNING, _("possible DNS-rebind attack detected: %s"), daemon->namebuff); + my_syslog(LOG_WARNING, _("possible DNS-rebind attack detected: %s"), daemon->namebuff); src/dnsmasq/forward.c my_syslog(LOG_WARNING, _("reducing DNS packet size for nameserver %s to %d"), daemon->addrbuff, SAFE_PKTSZ); src/dnsmasq/forward.c @@ -104,7 +108,7 @@ src/dnsmasq/lease.c src/dnsmasq/log.c my_syslog(LOG_WARNING, _("overflow: %d log entries lost"), e); src/dnsmasq/network.c - my_syslog(LOG_WARNING, s, daemon->addrbuff, strerror(errno)); + my_syslog(LOG_WARNING, s, daemon->addrbuff, strerror(errno)); src/dnsmasq/network.c my_syslog(LOG_WARNING, _("LOUD WARNING: listening on %s may accept requests via interfaces other than %s"), diff --git a/test/test_suite.bats b/test/test_suite.bats index 700263c1..892f209c 100644 --- a/test/test_suite.bats +++ b/test/test_suite.bats @@ -453,20 +453,20 @@ #[[ ${lines[8]} == "clients_ever_seen 8" ]] #[[ ${lines[9]} == "unique_clients 8" ]] [[ ${lines[10]} == "dns_queries_all_types 54" ]] - [[ ${lines[11]} == "reply_UNKNOWN 0" ]] + [[ ${lines[11]} == "reply_UNKNOWN 1" ]] [[ ${lines[12]} == "reply_NODATA 0" ]] [[ ${lines[13]} == "reply_NXDOMAIN 1" ]] [[ ${lines[14]} == "reply_CNAME 7" ]] - [[ ${lines[15]} == "reply_IP 25" ]] + [[ ${lines[15]} == "reply_IP 24" ]] [[ ${lines[16]} == "reply_DOMAIN 0" ]] - [[ ${lines[17]} == "reply_RRNAME 5" ]] + [[ ${lines[17]} == "reply_RRNAME 6" ]] [[ ${lines[18]} == "reply_SERVFAIL 0" ]] [[ ${lines[19]} == "reply_REFUSED 0" ]] [[ ${lines[20]} == "reply_NOTIMP 0" ]] [[ ${lines[21]} == "reply_OTHER 0" ]] - [[ ${lines[22]} == "reply_DNSSEC 6" ]] + [[ ${lines[22]} == "reply_DNSSEC 7" ]] [[ ${lines[23]} == "reply_NONE 0" ]] - [[ ${lines[24]} == "reply_BLOB 10" ]] + [[ ${lines[24]} == "reply_BLOB 8" ]] [[ ${lines[25]} == "dns_queries_all_replies 54" ]] [[ ${lines[26]} == "privacy_level 0" ]] [[ ${lines[27]} == "status enabled" ]] @@ -617,9 +617,9 @@ [[ ${lines[25]} == *" A use-application-dns.net 127.0.0.1 16 2 2 "*" N/A -1 N/A#0 \"\" \"24\""* ]] [[ ${lines[26]} == *" A a.ftl 127.0.0.1 3 2 4 "*" N/A -1 N/A#0 \"\" \"25\""* ]] [[ ${lines[27]} == *" AAAA aaaa.ftl 127.0.0.1 3 2 4 "*" N/A -1 N/A#0 \"\" \"26\""* ]] - [[ ${lines[28]} == *" ANY any.ftl 127.0.0.1 2 2 13 "*" N/A -1 127.0.0.1#5555 \"\" \"27\""* ]] + [[ ${lines[28]} == *" ANY any.ftl 127.0.0.1 2 2 6 "*" N/A -1 127.0.0.1#5555 \"\" \"27\""* ]] [[ ${lines[29]} == *" [CNAME] cname-ok.ftl 127.0.0.1 2 2 3 "*" N/A -1 127.0.0.1#5555 \"\" \"28\""* ]] - [[ ${lines[30]} == *" SRV srv.ftl 127.0.0.1 2 2 13 "*" N/A -1 127.0.0.1#5555 \"\" \"29\""* ]] + [[ ${lines[30]} == *" SRV srv.ftl 127.0.0.1 2 2 11 "*" N/A -1 127.0.0.1#5555 \"\" \"29\""* ]] [[ ${lines[31]} == *" SOA ftl 127.0.0.1 2 2 13 "*" N/A -1 127.0.0.1#5555 \"\" \"30\""* ]] [[ ${lines[32]} == *" PTR ptr.ftl 127.0.0.1 2 2 13 "*" N/A -1 127.0.0.1#5555 \"\" \"31\""* ]] [[ ${lines[33]} == *" TXT txt.ftl 127.0.0.1 2 2 13 "*" N/A -1 127.0.0.1#5555 \"\" \"32\""* ]] @@ -640,7 +640,7 @@ [[ ${lines[48]} == *" DS dnssec.works :: 2 1 11 "*" N/A -1 127.0.0.1#5555 \"\" \"47\""* ]] [[ ${lines[49]} == *" DNSKEY works :: 2 1 11 "*" N/A -1 127.0.0.1#5555 \"\" \"48\""* ]] [[ ${lines[50]} == *" DNSKEY dnssec.works :: 2 1 11 "*" N/A -1 127.0.0.1#5555 \"\" \"49\""* ]] - [[ ${lines[51]} == *" A fail01.dnssec.works 127.0.0.1 2 3 4 "*" N/A -1 127.0.0.1#5555 \"RRSIG missing\" \"50\""* ]] + [[ ${lines[51]} == *" A fail01.dnssec.works 127.0.0.1 2 3 0 "*" N/A -1 127.0.0.1#5555 \"RRSIG missing\" \"50\""* ]] [[ ${lines[52]} == *" DS fail01.dnssec.works :: 2 1 11 "*" N/A -1 127.0.0.1#5555 \"\" \"51\""* ]] [[ ${lines[53]} == *" A special.gravity.ftl 127.0.0.1 1 2 4 "*" N/A -1 N/A#0 \"\" \"52\""* ]] [[ ${lines[54]} == *" A a.b.c.d.special.gravity.ftl 127.0.0.1 1 2 4 "*" N/A -1 N/A#0 \"\" \"53\""* ]] From cc98853d1948344588b448566cf636b8075e8833 Mon Sep 17 00:00:00 2001 From: Simon Kelley Date: Mon, 12 Feb 2024 13:42:07 +0000 Subject: [PATCH 009/339] Tweak logging and special handling of T_ANY in rr-filter code. Signed-off-by: DL6ER --- src/dnsmasq/rfc1035.c | 44 +++++++++++++++++++------------------------ src/dnsmasq/util.c | 2 +- 2 files changed, 20 insertions(+), 26 deletions(-) diff --git a/src/dnsmasq/rfc1035.c b/src/dnsmasq/rfc1035.c index 8146886c..2a70e6eb 100644 --- a/src/dnsmasq/rfc1035.c +++ b/src/dnsmasq/rfc1035.c @@ -594,7 +594,7 @@ static int find_soa(struct dns_header *header, size_t qlen, char *name, int *sub } /* Print TXT reply to log */ -static int log_txt(char *name, unsigned char *p, const int ardlen, int secflag) +static int log_txt(char *name, unsigned char *p, const int ardlen, int flag) { unsigned char *p1 = p; @@ -616,7 +616,7 @@ static int log_txt(char *name, unsigned char *p, const int ardlen, int secflag) } *p3 = 0; - log_query(secflag | F_FORWARD, name, NULL, (char*)p1, 0); + log_query(flag, name, NULL, (char*)p1, 0); /* restore */ memmove(p1 + 1, p1, i); *p1 = len; @@ -787,7 +787,8 @@ int extract_addresses(struct dns_header *header, size_t qlen, char *name, time_t addrlen = IN6ADDRSZ; flags |= F_IPV6; } - else if (qtype != T_CNAME && (qtype == T_SRV || rr_on_list(daemon->cache_rr, qtype))) + else if (qtype != T_CNAME && + (qtype == T_SRV || rr_on_list(daemon->cache_rr, qtype) || rr_on_list(daemon->cache_rr, T_ANY))) flags |= F_RR; else insert = 0; /* NOTE: do not cache data from CNAME queries. */ @@ -813,13 +814,14 @@ int extract_addresses(struct dns_header *header, size_t qlen, char *name, time_t } GETSHORT(ardlen, p1); endrr = p1+ardlen; + + if (!CHECK_LEN(header, endrr, qlen, 0)) + return 2; /* bad packet */ /* Not what we're looking for? */ if (aqclass != C_IN || res == 2) { p1 = endrr; - if (!CHECK_LEN(header, p1, qlen, 0)) - return 2; /* bad packet */ continue; } @@ -881,12 +883,13 @@ int extract_addresses(struct dns_header *header, size_t qlen, char *name, time_t found = 1; } - else if (aqtype != qtype) + else if (qtype == T_ANY || aqtype != qtype) { #ifdef HAVE_DNSSEC if (!option_bool(OPT_DNSSEC_VALID) || aqtype != T_RRSIG) #endif - log_query(secflag | F_FORWARD | F_UPSTREAM | F_RRNAME, name, NULL, NULL, aqtype); + if (qtype != T_ANY) + log_query(secflag | F_FORWARD | F_UPSTREAM | F_RRNAME, name, NULL, NULL, aqtype); } else if (!(flags & F_NXDOMAIN)) { @@ -1032,26 +1035,17 @@ int extract_addresses(struct dns_header *header, size_t qlen, char *name, time_t blockdata_free(addr.rrblock.rrdata); } + /* We're filtering this RRtype. It will be removed from the + returned packet in process_reply() but gets cached here anyway + and will be filtered again on the way out of the cache. Here, + we just need to alter the logging. */ + if (rr_on_list(daemon->filter_rr, qtype)) + secflag = F_NEG | F_CONFIG; + if (aqtype == T_TXT) - { - if (!CHECK_LEN(header, p1, qlen, ardlen)) - return 2; - - log_txt(name, p1, ardlen, secflag | F_UPSTREAM); - } + log_txt(name, p1, ardlen, flags | F_FORWARD | F_UPSTREAM | secflag); else - { - int negflag = F_UPSTREAM; - - /* We're filtering this RRtype. It will be removed from the - returned packet in process_reply() but gets cached here anyway - and will be filtered again on the way out of the cache. Here, - we just need to alter the logging. */ - if (rr_on_list(daemon->filter_rr, qtype)) - negflag = F_NEG | F_CONFIG; - - log_query(negflag | flags | F_FORWARD | secflag, name, &addr, NULL, aqtype); - } + log_query(flags | F_FORWARD | F_UPSTREAM | secflag, name, &addr, NULL, aqtype); } p1 = endrr; diff --git a/src/dnsmasq/util.c b/src/dnsmasq/util.c index 3ac88354..0c7de444 100644 --- a/src/dnsmasq/util.c +++ b/src/dnsmasq/util.c @@ -119,7 +119,7 @@ int rr_on_list(struct rrlist *list, unsigned short rr) { while (list) { - if (list->rr == rr || list->rr == T_ANY) + if (list->rr == rr) return 1; list = list->next; From 9091f18f3fa804c90121563d36fb1a88798324f5 Mon Sep 17 00:00:00 2001 From: Simon Kelley Date: Mon, 12 Feb 2024 16:14:06 +0000 Subject: [PATCH 010/339] Make --filter-rr=ANY filter the answer to ANY queries. Thanks to Dominik Derigs for an earlier patch which inspired this. Signed-off-by: DL6ER --- src/dnsmasq/rfc1035.c | 21 ++++++++++++--------- src/dnsmasq/rrfilter.c | 8 ++++++++ 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/src/dnsmasq/rfc1035.c b/src/dnsmasq/rfc1035.c index 2a70e6eb..e3222eac 100644 --- a/src/dnsmasq/rfc1035.c +++ b/src/dnsmasq/rfc1035.c @@ -1039,7 +1039,7 @@ int extract_addresses(struct dns_header *header, size_t qlen, char *name, time_t returned packet in process_reply() but gets cached here anyway and will be filtered again on the way out of the cache. Here, we just need to alter the logging. */ - if (rr_on_list(daemon->filter_rr, qtype)) + if (qtype != T_ANY && rr_on_list(daemon->filter_rr, qtype)) secflag = F_NEG | F_CONFIG; if (aqtype == T_TXT) @@ -2008,7 +2008,7 @@ size_t answer_request(struct dns_header *header, char *limit, size_t qlen, if (!(crecp->flags & (F_HOSTS | F_DHCP))) auth = 0; - if (rr_on_list(daemon->filter_rr, qtype) && + if (qtype != T_ANY && rr_on_list(daemon->filter_rr, qtype) && !(crecp->flags & (F_HOSTS | F_DHCP | F_CONFIG | F_NEG))) { /* We have a cached answer but we're filtering it. */ @@ -2022,15 +2022,18 @@ size_t answer_request(struct dns_header *header, char *limit, size_t qlen, } else if (crecp->flags & F_NEG) { - ans = 1; - auth = 0; - soa_lookup = crecp; - if (crecp->flags & F_NXDOMAIN) - nxdomain = 1; + if (qtype != T_ANY) + { + ans = 1; + auth = 0; + soa_lookup = crecp; + if (crecp->flags & F_NXDOMAIN) + nxdomain = 1; // Pi-hole modification: Added record_source(crecp->uid) such that the subroutines know // where the reply came from (e.g. gravity.list) log_query(stale_flag | crecp->flags, name, NULL, record_source(crecp->uid), 0); + } } else { @@ -2208,7 +2211,7 @@ size_t answer_request(struct dns_header *header, char *limit, size_t qlen, if (flags & F_NXDOMAIN) nxdomain = 1; - else if (rr_on_list(daemon->filter_rr, qtype)) + else if (qtype != T_ANY && rr_on_list(daemon->filter_rr, qtype)) flags |= F_NEG | F_CONFIG; auth = 0; @@ -2253,7 +2256,7 @@ size_t answer_request(struct dns_header *header, char *limit, size_t qlen, } - if (!ans && rr_on_list(daemon->filter_rr, qtype)) + if (qtype != T_ANY && !ans && rr_on_list(daemon->filter_rr, qtype)) { /* We don't have a cached answer and when we get an answer from upstream we're going to filter it anyway. If we have a cached answer for the domain for another RRtype then diff --git a/src/dnsmasq/rrfilter.c b/src/dnsmasq/rrfilter.c index 7c277fa4..33d385c4 100644 --- a/src/dnsmasq/rrfilter.c +++ b/src/dnsmasq/rrfilter.c @@ -213,6 +213,14 @@ size_t rrfilter(struct dns_header *header, size_t *plen, int mode) if (i < ntohs(header->ancount) && type == qtype && class == qclass) continue; } + else if (qtype == T_ANY && rr_on_list(daemon->filter_rr, T_ANY)) + { + /* Filter replies to ANY queries in the spirit of + RFC RFC 8482 para 4.3 */ + if (class != C_IN || + type == T_A || type == T_AAAA || type == T_MX || type == T_CNAME) + continue; + } else { /* Only looking at answer section now. */ From 91b924d2693abacf304b05df15c5787347acd97e Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 13 Feb 2024 07:19:15 +0100 Subject: [PATCH 011/339] Update embedded dnsmasq version to 2.90test4 Signed-off-by: DL6ER --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6bf4dedd..7e0bd3a5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -11,6 +11,6 @@ cmake_minimum_required(VERSION 2.8.12) project(PIHOLE_FTL C) -set(DNSMASQ_VERSION pi-hole-v2.89-9461807) +set(DNSMASQ_VERSION pi-hole-v2.90test4) add_subdirectory(src) From 108ab67dc9fc276a21b23d064c2178ef255968c2 Mon Sep 17 00:00:00 2001 From: Simon Kelley Date: Sat, 30 Dec 2023 21:01:05 +0000 Subject: [PATCH 012/339] Protection against pathalogical DNSSEC domains. An attacker can create DNSSEC signed domains which need a lot of work to verfify. We limit the number of crypto operations to avoid DoS attacks by CPU exhaustion. Signed-off-by: DL6ER --- src/dnsmasq/dnssec.c | 95 +++++++++++++++++++++++++++++++------------ src/dnsmasq/forward.c | 31 +++++++++++--- 2 files changed, 93 insertions(+), 33 deletions(-) diff --git a/src/dnsmasq/dnssec.c b/src/dnsmasq/dnssec.c index 29a8e7a7..e02dc5da 100644 --- a/src/dnsmasq/dnssec.c +++ b/src/dnsmasq/dnssec.c @@ -430,6 +430,7 @@ static int explore_rrset(struct dns_header *header, size_t plen, int class, int STAT_SECURE_WILDCARD if it validates and is the result of wildcard expansion. (In this case *wildcard_out points to the "body" of the wildcard within name.) STAT_BOGUS signature is wrong, bad packet. + STAT_ABANDONED validation abandoned do to excess resource usage. STAT_NEED_KEY need DNSKEY to complete validation (name is returned in keyname) STAT_NEED_DS need DS to complete validation (name is returned in keyname) @@ -447,7 +448,7 @@ static int validate_rrset(time_t now, struct dns_header *header, size_t plen, in int algo_in, int keytag_in, unsigned long *ttl_out) { unsigned char *p; - int rdlen, j, name_labels, algo, labels, key_tag; + int rdlen, j, name_labels, algo, labels, key_tag, sig_fail_cnt; struct crec *crecp = NULL; short *rr_desc = rrfilter_desc(type); u32 sig_expiration, sig_inception; @@ -467,7 +468,7 @@ static int validate_rrset(time_t now, struct dns_header *header, size_t plen, in rrsetidx = sort_rrset(header, plen, rr_desc, rrsetidx, rrset, daemon->workspacename, keyname); /* Now try all the sigs to try and find one which validates */ - for (j = 0; j addr.key.algo == algo && crecp->addr.key.keytag == key_tag && - crecp->uid == (unsigned int)class && - verify(crecp->addr.key.keydata, crecp->addr.key.keylen, sig, sig_len, digest, hash->digest_size, algo)) - return (labels < name_labels) ? STAT_SECURE_WILDCARD : STAT_SECURE; + crecp->uid == (unsigned int)class) + { + if (verify(crecp->addr.key.keydata, crecp->addr.key.keylen, sig, sig_len, digest, hash->digest_size, algo)) + return (labels < name_labels) ? STAT_SECURE_WILDCARD : STAT_SECURE; + + /* An attacker can waste a lot of our CPU by setting up a giant DNSKEY RRSET full of failing + keys, all of which we have to try. Since many failing keys is not likely for + a legitimate domain, set a limit on how many can fail. */ + sig_fail_cnt++; + + if (sig_fail_cnt > 10) /* TODO */ + { + my_syslog(LOG_ERR, "sig_fail_cnt"); + return STAT_ABANDONED; + } + } } } @@ -681,6 +695,7 @@ static int validate_rrset(time_t now, struct dns_header *header, size_t plen, in STAT_OK Done, key(s) in cache. STAT_BOGUS No DNSKEYs found, which can be validated with DS, or self-sign for DNSKEY RRset is not valid, bad packet. + STAT_ABANDONED resource exhaustion. STAT_NEED_DS DS records to validate a key not found, name in keyname STAT_NEED_KEY DNSKEY records to validate a key not found, name in keyname */ @@ -688,7 +703,7 @@ int dnssec_validate_by_ds(time_t now, struct dns_header *header, size_t plen, ch { unsigned char *psave, *p = (unsigned char *)(header+1); struct crec *crecp, *recp1; - int rc, j, qtype, qclass, rdlen, flags, algo, valid, keytag; + int rc, j, qtype, qclass, rdlen, flags, algo, valid, keytag, ds_fail_cnt, key_fail_cnt; unsigned long ttl, sig_ttl; struct blockdata *key; union all_addr a; @@ -713,7 +728,7 @@ int dnssec_validate_by_ds(time_t now, struct dns_header *header, size_t plen, ch } /* NOTE, we need to find ONE DNSKEY which matches the DS */ - for (valid = 0, j = ntohs(header->ancount); j != 0 && !valid; j--) + for (key_fail_cnt = 0, valid = 0, j = ntohs(header->ancount); j != 0 && !valid; j--) { /* Ensure we have type, class TTL and length */ if (!(rc = extract_name(header, plen, &p, name, 0, 10))) @@ -762,7 +777,7 @@ int dnssec_validate_by_ds(time_t now, struct dns_header *header, size_t plen, ch if (!key) continue; - for (recp1 = crecp; recp1; recp1 = cache_find_by_name(recp1, name, now, F_DS)) + for (ds_fail_cnt = 0, recp1 = crecp; recp1; recp1 = cache_find_by_name(recp1, name, now, F_DS)) { void *ctx; unsigned char *digest, *ds_digest; @@ -771,7 +786,7 @@ int dnssec_validate_by_ds(time_t now, struct dns_header *header, size_t plen, ch int wire_len; if (recp1->addr.ds.algo == algo && - recp1->addr.ds.keytag == keytag && + recp1->addr.ds.keytag == keytag && recp1->uid == (unsigned int)class) { failflags &= ~DNSSEC_FAIL_NOKEY; @@ -796,30 +811,54 @@ int dnssec_validate_by_ds(time_t now, struct dns_header *header, size_t plen, ch if (!(recp1->flags & F_NEG) && recp1->addr.ds.keylen == (int)hash->digest_size && - (ds_digest = blockdata_retrieve(recp1->addr.ds.keydata, recp1->addr.ds.keylen, NULL)) && - memcmp(ds_digest, digest, recp1->addr.ds.keylen) == 0 && - explore_rrset(header, plen, class, T_DNSKEY, name, keyname, &sigcnt, &rrcnt) && - rrcnt != 0) + (ds_digest = blockdata_retrieve(recp1->addr.ds.keydata, recp1->addr.ds.keylen, NULL))) { - if (sigcnt == 0) - continue; - else - failflags &= ~DNSSEC_FAIL_NOSIG; - - rc = validate_rrset(now, header, plen, class, T_DNSKEY, sigcnt, rrcnt, name, keyname, - NULL, key, rdlen - 4, algo, keytag, &sig_ttl); - - failflags &= rc; - - if (STAT_ISEQUAL(rc, STAT_SECURE)) + if (memcmp(ds_digest, digest, recp1->addr.ds.keylen) != 0) { - valid = 1; - break; + /* limit CPU exhaustion attack from large DS x KEY cross-product. */ + ds_fail_cnt++; + + if (ds_fail_cnt > 5) /* TODO */ + { + my_syslog(LOG_ERR, "ds_fail_cnt"); + return STAT_ABANDONED; + } + } + else if (explore_rrset(header, plen, class, T_DNSKEY, name, keyname, &sigcnt, &rrcnt) && + rrcnt != 0) + { + if (sigcnt == 0) + continue; + else + failflags &= ~DNSSEC_FAIL_NOSIG; + + rc = validate_rrset(now, header, plen, class, T_DNSKEY, sigcnt, rrcnt, name, keyname, + NULL, key, rdlen - 4, algo, keytag, &sig_ttl); + + if (STAT_ISEQUAL(rc, STAT_ABANDONED)) + return STAT_ABANDONED; + + failflags &= rc; + + if (STAT_ISEQUAL(rc, STAT_SECURE)) + { + valid = 1; + break; + } } } } } blockdata_free(key); + + /* limit CPU exhaustion attack from large DS x KEY cross-product. */ + key_fail_cnt++; + + if (key_fail_cnt > 15) /* TODO */ + { + my_syslog(LOG_ERR, "key_fail_cnt"); + return STAT_ABANDONED; + } } if (valid) @@ -916,6 +955,7 @@ int dnssec_validate_by_ds(time_t now, struct dns_header *header, size_t plen, ch STAT_BOGUS no DS in reply or not signed, fails validation, bad packet. STAT_NEED_KEY DNSKEY records to validate a DS not found, name in keyname STAT_NEED_DS DS record needed. + STAT_ABANDONED resource exhaustion. */ int dnssec_validate_ds(time_t now, struct dns_header *header, size_t plen, char *name, char *keyname, int class) @@ -1798,6 +1838,7 @@ static int zone_status(char *name, int class, char *keyname, time_t now) STAT_BOGUS signature is wrong, bad packet, no validation where there should be. STAT_NEED_KEY need DNSKEY to complete validation (name is returned in keyname, class in *class) STAT_NEED_DS need DS to complete validation (name is returned in keyname) + STAT_ABANDONED resource exhaustion. daemon->rr_status points to a char array which corressponds to the RRs in the answer and auth sections. This is set to >1 for each RR which is validated, and 0 for any which aren't. @@ -1984,7 +2025,7 @@ int dnssec_validate_reply(time_t now, struct dns_header *header, size_t plen, ch rc = validate_rrset(now, header, plen, class1, type1, sigcnt, rrcnt, name, keyname, &wildname, NULL, 0, 0, 0, &sig_ttl); - if (STAT_ISEQUAL(rc, STAT_BOGUS) || STAT_ISEQUAL(rc, STAT_NEED_KEY) || STAT_ISEQUAL(rc, STAT_NEED_DS)) + if (STAT_ISEQUAL(rc, STAT_BOGUS) || STAT_ISEQUAL(rc, STAT_NEED_KEY) || STAT_ISEQUAL(rc, STAT_NEED_DS) || STAT_ISEQUAL(rc, STAT_ABANDONED)) { if (class) *class = class1; /* Class for DS or DNSKEY */ diff --git a/src/dnsmasq/forward.c b/src/dnsmasq/forward.c index 59bb91ed..a2e818b7 100644 --- a/src/dnsmasq/forward.c +++ b/src/dnsmasq/forward.c @@ -974,11 +974,15 @@ static void dnssec_validate(struct frec *forward, struct dns_header *header, status = dnssec_validate_reply(now, header, plen, daemon->namebuff, daemon->keyname, &forward->class, !option_bool(OPT_DNSSEC_IGN_NS) && (forward->sentto->flags & SERV_DO_DNSSEC), NULL, NULL, NULL); -#ifdef HAVE_DUMPFILE - if (STAT_ISEQUAL(status, STAT_BOGUS)) - dump_packet_udp((forward->flags & (FREC_DNSKEY_QUERY | FREC_DS_QUERY)) ? DUMP_SEC_BOGUS : DUMP_BOGUS, - header, (size_t)plen, &forward->sentto->addr, NULL, -daemon->port); -#endif + + if (STAT_ISEQUAL(status, STAT_ABANDONED)) + { + /* Log the actual validation that made us barf. */ + unsigned char *p = (unsigned char *)(header+1); + if (extract_name(header, plen, &p, daemon->namebuff, 0, 4) == 1) + my_syslog(LOG_WARNING, _("validation of %s failed: resource limit exceeded."), + daemon->namebuff[0] ? daemon->namebuff : "."); + } } /* Can't validate, as we're missing key data. Put this @@ -1109,6 +1113,12 @@ static void dnssec_validate(struct frec *forward, struct dns_header *header, status = STAT_ABANDONED; } +#ifdef HAVE_DUMPFILE + if (STAT_ISEQUAL(status, STAT_BOGUS) || STAT_ISEQUAL(status, STAT_ABANDONED)) + dump_packet_udp((forward->flags & (FREC_DNSKEY_QUERY | FREC_DS_QUERY)) ? DUMP_SEC_BOGUS : DUMP_BOGUS, + header, (size_t)plen, &forward->sentto->addr, NULL, -daemon->port); +#endif + /* Validated original answer, all done. */ if (!forward->dependent) return_reply(now, forward, header, plen, status); @@ -1117,7 +1127,7 @@ static void dnssec_validate(struct frec *forward, struct dns_header *header, /* validated subsidiary query/queries, (and cached result) pop that and return to the previous query/queries we were working on. */ struct frec *prev, *nxt = forward->dependent; - + free_frec(forward); while ((prev = nxt)) @@ -2137,6 +2147,15 @@ static int tcp_key_recurse(time_t now, int status, struct dns_header *header, si !option_bool(OPT_DNSSEC_IGN_NS) && (server->flags & SERV_DO_DNSSEC), NULL, NULL, NULL); + if (STAT_ISEQUAL(new_status, STAT_ABANDONED)) + { + /* Log the actual validation that made us barf. */ + unsigned char *p = (unsigned char *)(header+1); + if (extract_name(header, n, &p, daemon->namebuff, 0, 4) == 1) + my_syslog(LOG_WARNING, _("validation of %s failed: resource limit exceeded."), + daemon->namebuff[0] ? daemon->namebuff : "."); + } + if (!STAT_ISEQUAL(new_status, STAT_NEED_DS) && !STAT_ISEQUAL(new_status, STAT_NEED_KEY)) break; From bf17dd3c04b01b18174c946619cb9944c172a498 Mon Sep 17 00:00:00 2001 From: Simon Kelley Date: Sun, 31 Dec 2023 15:11:54 +0000 Subject: [PATCH 013/339] Update header with new EDE values. Signed-off-by: DL6ER --- src/dnsmasq/dns-protocol.h | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/dnsmasq/dns-protocol.h b/src/dnsmasq/dns-protocol.h index 0671adf2..2777be93 100644 --- a/src/dnsmasq/dns-protocol.h +++ b/src/dnsmasq/dns-protocol.h @@ -112,8 +112,11 @@ #define EDE_NO_AUTH 22 /* No Reachable Authority */ #define EDE_NETERR 23 /* Network error */ #define EDE_INVALID_DATA 24 /* Invalid Data */ - - +#define EDE_SIG_E_B_V 25 /* Signature Expired before Valid */ +#define EDE_TOO_EARLY 26 /* To Early */ +#define EDE_UNS_NS3_ITER 27 /* Unsupported NSEC3 Iterations Value */ +#define EDE_UNABLE_POLICY 28 /* Unable to conform to policy */ +#define EDE_SYNTHESIZED 29 /* Synthesized */ struct dns_header { From 70b0431919469aea5be01965c0b08dafea588c17 Mon Sep 17 00:00:00 2001 From: Simon Kelley Date: Sun, 31 Dec 2023 23:28:11 +0000 Subject: [PATCH 014/339] Update NSEC3 iterations handling to conform with RFC 9276. Signed-off-by: DL6ER --- src/dnsmasq/dnsmasq.h | 2 + src/dnsmasq/dnssec.c | 109 ++++++++++++++++++++++-------------------- 2 files changed, 59 insertions(+), 52 deletions(-) diff --git a/src/dnsmasq/dnsmasq.h b/src/dnsmasq/dnsmasq.h index 8447206d..63a289f9 100644 --- a/src/dnsmasq/dnsmasq.h +++ b/src/dnsmasq/dnsmasq.h @@ -763,6 +763,8 @@ struct dyndir { #define DNSSEC_FAIL_NONSEC 0x0040 /* No NSEC */ #define DNSSEC_FAIL_NODSSUP 0x0080 /* no supported DS algo. */ #define DNSSEC_FAIL_NOKEY 0x0100 /* no DNSKEY */ +#define DNSSEC_FAIL_NSEC3_ITERS 0x0200 /* too many iterations in NSEC3 */ +#define DNSSEC_FAIL_BADPACKET 0x0400 /* bad packet */ #define STAT_ISEQUAL(a, b) (((a) & 0xffff0000) == (b)) diff --git a/src/dnsmasq/dnssec.c b/src/dnsmasq/dnssec.c index e02dc5da..ceb6a37d 100644 --- a/src/dnsmasq/dnssec.c +++ b/src/dnsmasq/dnssec.c @@ -1179,6 +1179,7 @@ static int hostname_cmp(const char *a, const char *b) } } +/* returns 0 on success, or DNSSEC_FAIL_* value on failure. */ static int prove_non_existence_nsec(struct dns_header *header, size_t plen, unsigned char **nsecs, unsigned char **labels, int nsec_count, char *workspace1_in, char *workspace2, char *name, int type, int *nons) { @@ -1203,7 +1204,7 @@ static int prove_non_existence_nsec(struct dns_header *header, size_t plen, unsi GETSHORT(rdlen, p); psave = p; if (!extract_name(header, plen, &p, workspace2, 1, 10)) - return 0; + return DNSSEC_FAIL_BADPACKET; /* If NSEC comes from wildcard expansion, use original wildcard as name for computation. */ @@ -1231,7 +1232,7 @@ static int prove_non_existence_nsec(struct dns_header *header, size_t plen, unsi { /* 4035 para 5.4. Last sentence */ if (type == T_NSEC || type == T_RRSIG) - return 1; + return 0; /* NSEC with the same name as the RR we're testing, check that the type in question doesn't appear in the type map */ @@ -1247,25 +1248,25 @@ static int prove_non_existence_nsec(struct dns_header *header, size_t plen, unsi /* A CNAME answer would also be valid, so if there's a CNAME is should have been returned. */ if ((p[2] & (0x80 >> T_CNAME)) != 0) - return 0; + return DNSSEC_FAIL_NONSEC; /* If the SOA bit is set for a DS record, then we have the DS from the wrong side of the delegation. For the root DS, this is expected. */ if (name_labels != 0 && type == T_DS && (p[2] & (0x80 >> T_SOA)) != 0) - return 0; + return DNSSEC_FAIL_NONSEC; } while (rdlen >= 2) { if (!CHECK_LEN(header, p, plen, rdlen)) - return 0; + return DNSSEC_FAIL_BADPACKET; if (p[0] == type >> 8) { /* Does the NSEC say our type exists? */ if (offset < p[1] && (p[offset+2] & mask) != 0) - return 0; + return DNSSEC_FAIL_NONSEC; break; /* finished checking */ } @@ -1281,17 +1282,17 @@ static int prove_non_existence_nsec(struct dns_header *header, size_t plen, unsi /* Normal case, name falls between NSEC name and next domain name, wrap around case, name falls between NSEC name (rc == -1) and end */ if (hostname_cmp(workspace2, name) >= 0 || hostname_cmp(workspace1, workspace2) >= 0) - return 1; + return 0; } else { /* wrap around case, name falls between start and next domain name */ if (hostname_cmp(workspace1, workspace2) >= 0 && hostname_cmp(workspace2, name) >=0 ) - return 1; + return 0; } } - return 0; + return DNSSEC_FAIL_NONSEC; } /* return digest length, or zero on error */ @@ -1464,6 +1465,7 @@ static int check_nsec3_coverage(struct dns_header *header, size_t plen, int dige return 0; } +/* returns 0 on success, or DNSSEC_FAIL_* value on failure. */ static int prove_non_existence_nsec3(struct dns_header *header, size_t plen, unsigned char **nsecs, int nsec_count, char *workspace1, char *workspace2, char *name, int type, char *wildname, int *nons) { @@ -1485,9 +1487,9 @@ static int prove_non_existence_nsec3(struct dns_header *header, size_t plen, uns for (i = 0; i < nsec_count; i++) { if (!(p = skip_name(nsecs[i], header, plen, 15))) - return 0; /* bad packet */ + return DNSSEC_FAIL_BADPACKET; /* bad packet */ - p += 10; /* type, class, TTL, rdlen */ + p += 10; /* type, class, TTL, rdlen */ algo = *p++; if ((hash = hash_find(nsec3_digest_name(algo)))) @@ -1496,22 +1498,19 @@ static int prove_non_existence_nsec3(struct dns_header *header, size_t plen, uns /* No usable NSEC3s */ if (i == nsec_count) - return 0; + return DNSSEC_FAIL_NONSEC; p++; /* flags */ GETSHORT (iterations, p); - /* Upper-bound iterations, to avoid DoS. - Strictly, there are lower bounds for small keys, but - since we don't have key size info here, at least limit - to the largest bound, for 4096-bit keys. RFC 5155 10.3 */ - if (iterations > 2500) - return 0; + /* Upper-bound iterations, to avoid DoS. RFC 9276 refers. */ + if (iterations > 150) + return DNSSEC_FAIL_NSEC3_ITERS; salt_len = *p++; salt = p; if (!CHECK_LEN(header, salt, plen, salt_len)) - return 0; /* bad packet */ + return DNSSEC_FAIL_BADPACKET; /* bad packet */ /* Now prune so we only have NSEC3 records with same iterations, salt and algo */ for (i = 0; i < nsec_count; i++) @@ -1543,7 +1542,7 @@ static int prove_non_existence_nsec3(struct dns_header *header, size_t plen, uns continue; if (!CHECK_LEN(header, p, plen, salt_len)) - return 0; /* bad packet */ + return DNSSEC_FAIL_BADPACKET; /* bad packet */ if (memcmp(p, salt, salt_len) != 0) continue; @@ -1553,10 +1552,10 @@ static int prove_non_existence_nsec3(struct dns_header *header, size_t plen, uns } if ((digest_len = hash_name(name, &digest, hash, salt, salt_len, iterations)) == 0) - return 0; + return DNSSEC_FAIL_NONSEC; if (check_nsec3_coverage(header, plen, digest_len, digest, type, workspace1, workspace2, nsecs, nsec_count, nons, count_labels(name))) - return 1; + return 0; /* Can't find an NSEC3 which covers the name directly, we need the "closest encloser NSEC3" or an answer inferred from a wildcard record. */ @@ -1572,14 +1571,16 @@ static int prove_non_existence_nsec3(struct dns_header *header, size_t plen, uns break; if ((digest_len = hash_name(closest_encloser, &digest, hash, salt, salt_len, iterations)) == 0) - return 0; + return DNSSEC_FAIL_NONSEC; for (i = 0; i < nsec_count; i++) if ((p = nsecs[i])) { - if (!extract_name(header, plen, &p, workspace1, 1, 0) || - !(base32_len = base32_decode(workspace1, (unsigned char *)workspace2))) - return 0; + if (!extract_name(header, plen, &p, workspace1, 1, 0)) + return DNSSEC_FAIL_BADPACKET; + + if (!(base32_len = base32_decode(workspace1, (unsigned char *)workspace2))) + return DNSSEC_FAIL_NONSEC; if (digest_len == base32_len && memcmp(digest, workspace2, digest_len) == 0) @@ -1594,14 +1595,14 @@ static int prove_non_existence_nsec3(struct dns_header *header, size_t plen, uns while ((closest_encloser = strchr(closest_encloser, '.'))); if (!closest_encloser || !next_closest) - return 0; + return DNSSEC_FAIL_NONSEC; /* Look for NSEC3 that proves the non-existence of the next-closest encloser */ if ((digest_len = hash_name(next_closest, &digest, hash, salt, salt_len, iterations)) == 0) - return 0; + return DNSSEC_FAIL_NONSEC; if (!check_nsec3_coverage(header, plen, digest_len, digest, type, workspace1, workspace2, nsecs, nsec_count, NULL, 1)) - return 0; + return DNSSEC_FAIL_NONSEC; /* Finally, check that there's no seat of wildcard synthesis */ if (!wildname) @@ -1613,15 +1614,16 @@ static int prove_non_existence_nsec3(struct dns_header *header, size_t plen, uns *wildcard = '*'; if ((digest_len = hash_name(wildcard, &digest, hash, salt, salt_len, iterations)) == 0) - return 0; + return DNSSEC_FAIL_NONSEC; if (!check_nsec3_coverage(header, plen, digest_len, digest, type, workspace1, workspace2, nsecs, nsec_count, NULL, 1)) - return 0; + return DNSSEC_FAIL_NONSEC; } - return 1; + return 0; } +/* returns 0 on success, or DNSSEC_FAIL_* value on failure. */ static int prove_non_existence(struct dns_header *header, size_t plen, char *keyname, char *name, int qtype, int qclass, char *wildname, int *nons, int *nsec_ttl) { static unsigned char **nsecset = NULL, **rrsig_labels = NULL; @@ -1634,7 +1636,7 @@ static int prove_non_existence(struct dns_header *header, size_t plen, char *key /* Move to NS section */ if (!p || !(p = skip_section(p, ntohs(header->ancount), header, plen))) - return 0; + return DNSSEC_FAIL_BADPACKET; auth_start = p; @@ -1643,7 +1645,7 @@ static int prove_non_existence(struct dns_header *header, size_t plen, char *key unsigned char *pstart = p; if (!extract_name(header, plen, &p, daemon->workspacename, 1, 10)) - return 0; + return DNSSEC_FAIL_BADPACKET; GETSHORT(type, p); GETSHORT(class, p); @@ -1662,12 +1664,12 @@ static int prove_non_existence(struct dns_header *header, size_t plen, char *key /* No mixed NSECing 'round here, thankyouverymuch */ if (type_found != 0 && type_found != type) - return 0; + return DNSSEC_FAIL_NONSEC; type_found = type; if (!expand_workspace(&nsecset, &nsecset_sz, nsecs_found)) - return 0; + return DNSSEC_FAIL_BADPACKET; if (type == T_NSEC) { @@ -1682,14 +1684,14 @@ static int prove_non_existence(struct dns_header *header, size_t plen, char *key int res, j, rdlen1, type1, class1; if (!expand_workspace(&rrsig_labels, &rrsig_labels_sz, nsecs_found)) - return 0; + return DNSSEC_FAIL_BADPACKET; rrsig_labels[nsecs_found] = NULL; for (j = ntohs(header->nscount); j != 0; j--) { if (!(res = extract_name(header, plen, &p1, daemon->workspacename, 0, 10))) - return 0; + return DNSSEC_FAIL_BADPACKET; GETSHORT(type1, p1); GETSHORT(class1, p1); @@ -1697,7 +1699,7 @@ static int prove_non_existence(struct dns_header *header, size_t plen, char *key GETSHORT(rdlen1, p1); if (!CHECK_LEN(header, p1, plen, rdlen1)) - return 0; + return DNSSEC_FAIL_BADPACKET; if (res == 1 && class1 == qclass && type1 == T_RRSIG) { @@ -1705,7 +1707,7 @@ static int prove_non_existence(struct dns_header *header, size_t plen, char *key unsigned char *psav = p1; if (rdlen1 < 18) - return 0; /* bad packet */ + return DNSSEC_FAIL_BADPACKET; /* bad packet */ GETSHORT(type_covered, p1); @@ -1717,25 +1719,25 @@ static int prove_non_existence(struct dns_header *header, size_t plen, char *key if (!rrsig_labels[nsecs_found]) rrsig_labels[nsecs_found] = p1; else if (*rrsig_labels[nsecs_found] != *p1) /* algo */ - return 0; + return DNSSEC_FAIL_NONSEC; } p1 = psav; } if (!ADD_RDLEN(header, p1, plen, rdlen1)) - return 0; + return DNSSEC_FAIL_BADPACKET; } /* Must have found at least one sig. */ if (!rrsig_labels[nsecs_found]) - return 0; + return DNSSEC_FAIL_NONSEC; } nsecset[nsecs_found++] = pstart; } if (!ADD_RDLEN(header, p, plen, rdlen)) - return 0; + return DNSSEC_FAIL_BADPACKET; } if (type_found == T_NSEC) @@ -1743,7 +1745,7 @@ static int prove_non_existence(struct dns_header *header, size_t plen, char *key else if (type_found == T_NSEC3) return prove_non_existence_nsec3(header, plen, nsecset, nsecs_found, daemon->workspacename, keyname, name, qtype, wildname, nons); else - return 0; + return DNSSEC_FAIL_NONSEC; } /* Check signing status of name. @@ -1857,7 +1859,7 @@ int dnssec_validate_reply(time_t now, struct dns_header *header, size_t plen, ch int type1, class1, rdlen1 = 0, type2, class2, rdlen2, qclass, qtype, targetidx; int i, j, rc = STAT_INSECURE; int secure = STAT_SECURE; - + int rc_nsec; /* extend rr_status if necessary */ if (daemon->rr_status_sz < ntohs(header->ancount) + ntohs(header->nscount)) { @@ -2059,8 +2061,8 @@ int dnssec_validate_reply(time_t now, struct dns_header *header, size_t plen, ch That's not a problem since if the RRsets later fail we'll return BOGUS then. */ if (STAT_ISEQUAL(rc, STAT_SECURE_WILDCARD) && - !prove_non_existence(header, plen, keyname, name, type1, class1, wildname, NULL, NULL)) - return STAT_BOGUS | DNSSEC_FAIL_NONSEC; + ((rc_nsec = prove_non_existence(header, plen, keyname, name, type1, class1, wildname, NULL, NULL))) != 0) + return STAT_BOGUS | rc_nsec; rc = STAT_SECURE; } @@ -2085,20 +2087,21 @@ int dnssec_validate_reply(time_t now, struct dns_header *header, size_t plen, ch /* For anything other than a DS record, this situation is OK if either the answer is in an unsigned zone, or there's a NSEC records. */ - if (!prove_non_existence(header, plen, keyname, name, qtype, qclass, NULL, nons, nsec_ttl)) + if ((rc_nsec = prove_non_existence(header, plen, keyname, name, qtype, qclass, NULL, nons, nsec_ttl)) != 0) { /* Empty DS without NSECS */ if (qtype == T_DS) - return STAT_BOGUS | DNSSEC_FAIL_NONSEC; + return STAT_BOGUS | rc_nsec; - if (!STAT_ISEQUAL((rc = zone_status(name, qclass, keyname, now)), STAT_SECURE)) + if ((rc_nsec & (DNSSEC_FAIL_NONSEC | DNSSEC_FAIL_NSEC3_ITERS)) && + !STAT_ISEQUAL((rc = zone_status(name, qclass, keyname, now)), STAT_SECURE)) { if (class) *class = qclass; /* Class for NEED_DS or NEED_KEY */ return rc; } - return STAT_BOGUS | DNSSEC_FAIL_NONSEC; /* signed zone, no NSECs */ + return STAT_BOGUS | rc_nsec; /* signed zone, no NSECs */ } } @@ -2180,6 +2183,8 @@ int errflags_to_ede(int status) return EDE_NO_DNSKEY; else if (status & DNSSEC_FAIL_NODSSUP) return EDE_USUPDS; + else if (status & DNSSEC_FAIL_NSEC3_ITERS) + return EDE_UNS_NS3_ITER; else if (status & DNSSEC_FAIL_NONSEC) return EDE_NO_NSEC; else if (status & DNSSEC_FAIL_INDET) From dd11688b8c9c855073e480ba0546c14b5936b1ae Mon Sep 17 00:00:00 2001 From: Simon Kelley Date: Mon, 1 Jan 2024 17:17:25 +0000 Subject: [PATCH 015/339] Measure cryptographic work done by DNSSEC. Signed-off-by: DL6ER --- src/dnsmasq/dnsmasq.h | 10 ++++++---- src/dnsmasq/dnssec.c | 44 +++++++++++++++++++++++++++---------------- src/dnsmasq/forward.c | 42 ++++++++++++++++++++++++++++------------- 3 files changed, 63 insertions(+), 33 deletions(-) diff --git a/src/dnsmasq/dnsmasq.h b/src/dnsmasq/dnsmasq.h index 63a289f9..0a95d40f 100644 --- a/src/dnsmasq/dnsmasq.h +++ b/src/dnsmasq/dnsmasq.h @@ -802,7 +802,7 @@ struct frec { struct blockdata *stash; /* Saved reply, whilst we validate */ size_t stash_len; #ifdef HAVE_DNSSEC - int class, work_counter; + int class, work_counter, validate_counter; struct frec *dependent; /* Query awaiting internally-generated DNSKEY or DS query */ struct frec *next_dependent; /* list of above. */ struct frec *blocking_query; /* Query which is blocking us. */ @@ -1440,10 +1440,12 @@ int in_zone(struct auth_zone *zone, char *name, char **cut); /* dnssec.c */ #ifdef HAVE_DNSSEC size_t dnssec_generate_query(struct dns_header *header, unsigned char *end, char *name, int class, int type, int edns_pktsz); -int dnssec_validate_by_ds(time_t now, struct dns_header *header, size_t plen, char *name, char *keyname, int class); -int dnssec_validate_ds(time_t now, struct dns_header *header, size_t plen, char *name, char *keyname, int class); +int dnssec_validate_by_ds(time_t now, struct dns_header *header, size_t plen, char *name, + char *keyname, int class, int *validate_count); +int dnssec_validate_ds(time_t now, struct dns_header *header, size_t plen, char *name, + char *keyname, int class, int *validate_count); int dnssec_validate_reply(time_t now, struct dns_header *header, size_t plen, char *name, char *keyname, int *class, - int check_unsigned, int *neganswer, int *nons, int *nsec_ttl); + int check_unsigned, int *neganswer, int *nons, int *nsec_ttl, int *validate_count); int dnskey_keytag(int alg, int flags, unsigned char *key, int keylen); size_t filter_rrsigs(struct dns_header *header, size_t plen); int setup_timestamp(void); diff --git a/src/dnsmasq/dnssec.c b/src/dnsmasq/dnssec.c index ceb6a37d..cb35175d 100644 --- a/src/dnsmasq/dnssec.c +++ b/src/dnsmasq/dnssec.c @@ -445,7 +445,7 @@ static int explore_rrset(struct dns_header *header, size_t plen, int class, int */ static int validate_rrset(time_t now, struct dns_header *header, size_t plen, int class, int type, int sigidx, int rrsetidx, char *name, char *keyname, char **wildcard_out, struct blockdata *key, int keylen, - int algo_in, int keytag_in, unsigned long *ttl_out) + int algo_in, int keytag_in, unsigned long *ttl_out, int *validate_counter) { unsigned char *p; int rdlen, j, name_labels, algo, labels, key_tag, sig_fail_cnt; @@ -655,8 +655,10 @@ static int validate_rrset(time_t now, struct dns_header *header, size_t plen, in if (key) { - if (algo_in == algo && keytag_in == key_tag && - verify(key, keylen, sig, sig_len, digest, hash->digest_size, algo)) + if (algo_in == algo && keytag_in == key_tag) + (*validate_counter)++; + + if (verify(key, keylen, sig, sig_len, digest, hash->digest_size, algo)) return STAT_SECURE; } else @@ -667,7 +669,9 @@ static int validate_rrset(time_t now, struct dns_header *header, size_t plen, in crecp->addr.key.keytag == key_tag && crecp->uid == (unsigned int)class) { - if (verify(crecp->addr.key.keydata, crecp->addr.key.keylen, sig, sig_len, digest, hash->digest_size, algo)) + (*validate_counter)++; + + if (verify(crecp->addr.key.keydata, crecp->addr.key.keylen, sig, sig_len, digest, hash->digest_size, algo)) return (labels < name_labels) ? STAT_SECURE_WILDCARD : STAT_SECURE; /* An attacker can waste a lot of our CPU by setting up a giant DNSKEY RRSET full of failing @@ -699,7 +703,8 @@ static int validate_rrset(time_t now, struct dns_header *header, size_t plen, in STAT_NEED_DS DS records to validate a key not found, name in keyname STAT_NEED_KEY DNSKEY records to validate a key not found, name in keyname */ -int dnssec_validate_by_ds(time_t now, struct dns_header *header, size_t plen, char *name, char *keyname, int class) +int dnssec_validate_by_ds(time_t now, struct dns_header *header, size_t plen, char *name, + char *keyname, int class, int *validate_counter) { unsigned char *psave, *p = (unsigned char *)(header+1); struct crec *crecp, *recp1; @@ -806,6 +811,7 @@ int dnssec_validate_by_ds(time_t now, struct dns_header *header, size_t plen, ch hash->update(ctx, (unsigned int)wire_len, (unsigned char *)name); hash->update(ctx, (unsigned int)rdlen, psave); hash->digest(ctx, hash->digest_size, digest); + (*validate_counter)++; /* computing a hash is a unit of crypto work. */ from_wire(name); @@ -833,7 +839,7 @@ int dnssec_validate_by_ds(time_t now, struct dns_header *header, size_t plen, ch failflags &= ~DNSSEC_FAIL_NOSIG; rc = validate_rrset(now, header, plen, class, T_DNSKEY, sigcnt, rrcnt, name, keyname, - NULL, key, rdlen - 4, algo, keytag, &sig_ttl); + NULL, key, rdlen - 4, algo, keytag, &sig_ttl, validate_counter); if (STAT_ISEQUAL(rc, STAT_ABANDONED)) return STAT_ABANDONED; @@ -958,7 +964,8 @@ int dnssec_validate_by_ds(time_t now, struct dns_header *header, size_t plen, ch STAT_ABANDONED resource exhaustion. */ -int dnssec_validate_ds(time_t now, struct dns_header *header, size_t plen, char *name, char *keyname, int class) +int dnssec_validate_ds(time_t now, struct dns_header *header, size_t plen, char *name, + char *keyname, int class, int *validate_counter) { unsigned char *p = (unsigned char *)(header+1); int qtype, qclass, rc, i, neganswer = 0, nons = 0, servfail = 0, neg_ttl = 0, found_supported = 0; @@ -983,7 +990,7 @@ int dnssec_validate_ds(time_t now, struct dns_header *header, size_t plen, char servfail = neganswer = nons = 1; else { - rc = dnssec_validate_reply(now, header, plen, name, keyname, NULL, 0, &neganswer, &nons, &neg_ttl); + rc = dnssec_validate_reply(now, header, plen, name, keyname, NULL, 0, &neganswer, &nons, &neg_ttl, validate_counter); if (STAT_ISEQUAL(rc, STAT_INSECURE)) { @@ -1466,8 +1473,8 @@ static int check_nsec3_coverage(struct dns_header *header, size_t plen, int dige } /* returns 0 on success, or DNSSEC_FAIL_* value on failure. */ -static int prove_non_existence_nsec3(struct dns_header *header, size_t plen, unsigned char **nsecs, int nsec_count, - char *workspace1, char *workspace2, char *name, int type, char *wildname, int *nons) +static int prove_non_existence_nsec3(struct dns_header *header, size_t plen, unsigned char **nsecs, int nsec_count, char *workspace1, + char *workspace2, char *name, int type, char *wildname, int *nons, int *validate_counter) { unsigned char *salt, *p, *digest; int digest_len, i, iterations, salt_len, base32_len, algo = 0; @@ -1551,6 +1558,7 @@ static int prove_non_existence_nsec3(struct dns_header *header, size_t plen, uns nsecs[i] = nsec3p; } + (*validate_counter)++; if ((digest_len = hash_name(name, &digest, hash, salt, salt_len, iterations)) == 0) return DNSSEC_FAIL_NONSEC; @@ -1570,6 +1578,7 @@ static int prove_non_existence_nsec3(struct dns_header *header, size_t plen, uns if (wildname && hostname_isequal(closest_encloser, wildname)) break; + (*validate_counter)++; if ((digest_len = hash_name(closest_encloser, &digest, hash, salt, salt_len, iterations)) == 0) return DNSSEC_FAIL_NONSEC; @@ -1598,6 +1607,7 @@ static int prove_non_existence_nsec3(struct dns_header *header, size_t plen, uns return DNSSEC_FAIL_NONSEC; /* Look for NSEC3 that proves the non-existence of the next-closest encloser */ + (*validate_counter)++; if ((digest_len = hash_name(next_closest, &digest, hash, salt, salt_len, iterations)) == 0) return DNSSEC_FAIL_NONSEC; @@ -1613,6 +1623,7 @@ static int prove_non_existence_nsec3(struct dns_header *header, size_t plen, uns wildcard--; *wildcard = '*'; + (*validate_counter)++; if ((digest_len = hash_name(wildcard, &digest, hash, salt, salt_len, iterations)) == 0) return DNSSEC_FAIL_NONSEC; @@ -1624,7 +1635,8 @@ static int prove_non_existence_nsec3(struct dns_header *header, size_t plen, uns } /* returns 0 on success, or DNSSEC_FAIL_* value on failure. */ -static int prove_non_existence(struct dns_header *header, size_t plen, char *keyname, char *name, int qtype, int qclass, char *wildname, int *nons, int *nsec_ttl) +static int prove_non_existence(struct dns_header *header, size_t plen, char *keyname, char *name, int qtype, int qclass, + char *wildname, int *nons, int *nsec_ttl, int *validate_counter) { static unsigned char **nsecset = NULL, **rrsig_labels = NULL; static int nsecset_sz = 0, rrsig_labels_sz = 0; @@ -1743,7 +1755,7 @@ static int prove_non_existence(struct dns_header *header, size_t plen, char *key if (type_found == T_NSEC) return prove_non_existence_nsec(header, plen, nsecset, rrsig_labels, nsecs_found, daemon->workspacename, keyname, name, qtype, nons); else if (type_found == T_NSEC3) - return prove_non_existence_nsec3(header, plen, nsecset, nsecs_found, daemon->workspacename, keyname, name, qtype, wildname, nons); + return prove_non_existence_nsec3(header, plen, nsecset, nsecs_found, daemon->workspacename, keyname, name, qtype, wildname, nons, validate_counter); else return DNSSEC_FAIL_NONSEC; } @@ -1850,7 +1862,7 @@ static int zone_status(char *name, int class, char *keyname, time_t now) if the nons argument is non-NULL. */ int dnssec_validate_reply(time_t now, struct dns_header *header, size_t plen, char *name, char *keyname, - int *class, int check_unsigned, int *neganswer, int *nons, int *nsec_ttl) + int *class, int check_unsigned, int *neganswer, int *nons, int *nsec_ttl, int *validate_counter) { static unsigned char **targets = NULL; static int target_sz = 0; @@ -2025,7 +2037,7 @@ int dnssec_validate_reply(time_t now, struct dns_header *header, size_t plen, ch { unsigned long sig_ttl; rc = validate_rrset(now, header, plen, class1, type1, sigcnt, - rrcnt, name, keyname, &wildname, NULL, 0, 0, 0, &sig_ttl); + rrcnt, name, keyname, &wildname, NULL, 0, 0, 0, &sig_ttl, validate_counter); if (STAT_ISEQUAL(rc, STAT_BOGUS) || STAT_ISEQUAL(rc, STAT_NEED_KEY) || STAT_ISEQUAL(rc, STAT_NEED_DS) || STAT_ISEQUAL(rc, STAT_ABANDONED)) { @@ -2061,7 +2073,7 @@ int dnssec_validate_reply(time_t now, struct dns_header *header, size_t plen, ch That's not a problem since if the RRsets later fail we'll return BOGUS then. */ if (STAT_ISEQUAL(rc, STAT_SECURE_WILDCARD) && - ((rc_nsec = prove_non_existence(header, plen, keyname, name, type1, class1, wildname, NULL, NULL))) != 0) + ((rc_nsec = prove_non_existence(header, plen, keyname, name, type1, class1, wildname, NULL, NULL, validate_counter))) != 0) return STAT_BOGUS | rc_nsec; rc = STAT_SECURE; @@ -2087,7 +2099,7 @@ int dnssec_validate_reply(time_t now, struct dns_header *header, size_t plen, ch /* For anything other than a DS record, this situation is OK if either the answer is in an unsigned zone, or there's a NSEC records. */ - if ((rc_nsec = prove_non_existence(header, plen, keyname, name, qtype, qclass, NULL, nons, nsec_ttl)) != 0) + if ((rc_nsec = prove_non_existence(header, plen, keyname, name, qtype, qclass, NULL, nons, nsec_ttl, validate_counter)) != 0) { /* Empty DS without NSECS */ if (qtype == T_DS) diff --git a/src/dnsmasq/forward.c b/src/dnsmasq/forward.c index a2e818b7..02be799e 100644 --- a/src/dnsmasq/forward.c +++ b/src/dnsmasq/forward.c @@ -17,6 +17,8 @@ #include "dnsmasq.h" #include "../dnsmasq_interface.h" +static int vchwm = 0; /* TODO */ + static struct frec *get_new_frec(time_t now, struct server *serv, int force); static struct frec *lookup_frec(unsigned short id, int fd, void *hash, int *firstp, int *lastp); static struct frec *lookup_frec_by_query(void *hash, unsigned int flags, unsigned int flagmask); @@ -345,6 +347,7 @@ static int forward_query(int udpfd, union mysockaddr *udpaddr, forward->flags |= FREC_AD_QUESTION; #ifdef HAVE_DNSSEC forward->work_counter = DNSSEC_WORK; + forward->validate_counter = 0; if (do_bit) forward->flags |= FREC_DO_QUESTION; #endif @@ -936,6 +939,8 @@ static size_t process_reply(struct dns_header *header, time_t now, struct server static void dnssec_validate(struct frec *forward, struct dns_header *header, ssize_t plen, int status, time_t now) { + struct frec *orig; + daemon->log_display_id = forward->frec_src.log_id; /* We've had a reply already, which we're validating. Ignore this duplicate */ @@ -960,6 +965,9 @@ static void dnssec_validate(struct frec *forward, struct dns_header *header, log_query(F_UPSTREAM | F_NOEXTRA, daemon->namebuff, NULL, "truncated", (forward->flags & FREC_DNSKEY_QUERY) ? T_DNSKEY : T_DS); } } + + /* Find the original query that started it all.... */ + for (orig = forward; orig->dependent; orig = orig->dependent); /* As soon as anything returns BOGUS, we stop and unwind, to do otherwise would invite infinite loops, since the answers to DNSKEY and DS queries @@ -967,13 +975,13 @@ static void dnssec_validate(struct frec *forward, struct dns_header *header, if (!STAT_ISEQUAL(status, STAT_BOGUS) && !STAT_ISEQUAL(status, STAT_TRUNCATED) && !STAT_ISEQUAL(status, STAT_ABANDONED)) { if (forward->flags & FREC_DNSKEY_QUERY) - status = dnssec_validate_by_ds(now, header, plen, daemon->namebuff, daemon->keyname, forward->class); + status = dnssec_validate_by_ds(now, header, plen, daemon->namebuff, daemon->keyname, forward->class, &orig->validate_counter); else if (forward->flags & FREC_DS_QUERY) - status = dnssec_validate_ds(now, header, plen, daemon->namebuff, daemon->keyname, forward->class); + status = dnssec_validate_ds(now, header, plen, daemon->namebuff, daemon->keyname, forward->class, &orig->validate_counter); else status = dnssec_validate_reply(now, header, plen, daemon->namebuff, daemon->keyname, &forward->class, !option_bool(OPT_DNSSEC_IGN_NS) && (forward->sentto->flags & SERV_DO_DNSSEC), - NULL, NULL, NULL); + NULL, NULL, NULL, &orig->validate_counter); if (STAT_ISEQUAL(status, STAT_ABANDONED)) { @@ -1030,15 +1038,11 @@ static void dnssec_validate(struct frec *forward, struct dns_header *header, else { struct server *server; - struct frec *orig; void *hash; size_t nn; int serverind, fd; struct randfd_list *rfds = NULL; - /* Find the original query that started it all.... */ - for (orig = forward; orig->dependent; orig = orig->dependent); - /* Make sure we don't expire and free the orig frec during the allocation of a new one: third arg of get_new_frec() does that. */ if ((serverind = dnssec_server(forward->sentto, daemon->keyname, NULL, NULL)) != -1 && @@ -1393,6 +1397,11 @@ static void return_reply(time_t now, struct frec *forward, struct dns_header *he log_query(F_SECSTAT, domain, &a, result, 0); } } + + if (forward->validate_counter > vchwm) + vchwm = forward->validate_counter; + if (extract_request(header, n, daemon->namebuff, NULL)) + my_syslog(LOG_INFO, "Validate_counter %s is %d, HWM is %d", daemon->namebuff, forward->validate_counter, vchwm); /* TODO */ #endif if (option_bool(OPT_NO_REBIND)) @@ -2122,7 +2131,7 @@ static ssize_t tcp_talk(int first, int last, int start, unsigned char *packet, /* Recurse down the key hierarchy */ static int tcp_key_recurse(time_t now, int status, struct dns_header *header, size_t n, int class, char *name, char *keyname, struct server *server, - int have_mark, unsigned int mark, int *keycount) + int have_mark, unsigned int mark, int *keycount, int *validatecount) { int first, last, start, new_status; unsigned char *packet = NULL; @@ -2139,13 +2148,13 @@ static int tcp_key_recurse(time_t now, int status, struct dns_header *header, si if (--(*keycount) == 0) new_status = STAT_ABANDONED; else if (STAT_ISEQUAL(status, STAT_NEED_KEY)) - new_status = dnssec_validate_by_ds(now, header, n, name, keyname, class); + new_status = dnssec_validate_by_ds(now, header, n, name, keyname, class, validatecount); else if (STAT_ISEQUAL(status, STAT_NEED_DS)) - new_status = dnssec_validate_ds(now, header, n, name, keyname, class); + new_status = dnssec_validate_ds(now, header, n, name, keyname, class, validatecount); else new_status = dnssec_validate_reply(now, header, n, name, keyname, &class, !option_bool(OPT_DNSSEC_IGN_NS) && (server->flags & SERV_DO_DNSSEC), - NULL, NULL, NULL); + NULL, NULL, NULL, validatecount); if (STAT_ISEQUAL(new_status, STAT_ABANDONED)) { @@ -2189,7 +2198,8 @@ static int tcp_key_recurse(time_t now, int status, struct dns_header *header, si log_query_mysockaddr(F_NOEXTRA | F_DNSSEC | F_SERVER, keyname, &server->addr, STAT_ISEQUAL(new_status, STAT_NEED_KEY) ? "dnssec-query[DNSKEY]" : "dnssec-query[DS]", 0); - new_status = tcp_key_recurse(now, new_status, new_header, m, class, name, keyname, server, have_mark, mark, keycount); + new_status = tcp_key_recurse(now, new_status, new_header, m, class, name, keyname, server, + have_mark, mark, keycount, validatecount); daemon->log_display_id = log_save; @@ -2533,8 +2543,9 @@ unsigned char *tcp_request(int confd, time_t now, if (option_bool(OPT_DNSSEC_VALID) && !checking_disabled && (master->flags & SERV_DO_DNSSEC)) { int keycount = DNSSEC_WORK; /* Limit to number of DNSSEC questions, to catch loops and avoid filling cache. */ + int validatecount = 0; /* How many validations we did */ int status = tcp_key_recurse(now, STAT_OK, header, m, 0, daemon->namebuff, daemon->keyname, - serv, have_mark, mark, &keycount); + serv, have_mark, mark, &keycount, &validatecount); char *result, *domain = "result"; union all_addr a; @@ -2560,6 +2571,11 @@ unsigned char *tcp_request(int confd, time_t now, } log_query(F_SECSTAT, domain, &a, result, 0); + + if (validatecount > vchwm) + vchwm = validatecount; + if (extract_request(header, m, daemon->namebuff, NULL)) + my_syslog(LOG_INFO, "Validate_counter %s is %d, HWM is %d", daemon->namebuff, validatecount, vchwm); /* TODO */ } #endif From 2e0d8fff727cc04b0857a3e21a1e35ca00fcd7b3 Mon Sep 17 00:00:00 2001 From: Simon Kelley Date: Tue, 2 Jan 2024 12:25:44 +0000 Subject: [PATCH 016/339] Fix error introduced in 635bc51cac3d5d7dd49ce9e27149cf7e402b7e79 Signed-off-by: DL6ER --- src/dnsmasq/dnssec.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dnsmasq/dnssec.c b/src/dnsmasq/dnssec.c index cb35175d..bc02dadb 100644 --- a/src/dnsmasq/dnssec.c +++ b/src/dnsmasq/dnssec.c @@ -1282,7 +1282,7 @@ static int prove_non_existence_nsec(struct dns_header *header, size_t plen, unsi p += p[1]; } - return 1; + return 0; } else if (rc == -1) { From a133029e4c255a19036b166d41883d9affc6fe56 Mon Sep 17 00:00:00 2001 From: Simon Kelley Date: Tue, 2 Jan 2024 21:43:04 +0000 Subject: [PATCH 017/339] Parameterise work limits for DNSSEC validation. Signed-off-by: DL6ER --- src/dnsmasq/cache.c | 16 +++++++- src/dnsmasq/config.h | 5 +++ src/dnsmasq/dnsmasq.h | 2 + src/dnsmasq/dnssec.c | 90 +++++++++++++++++++++++++------------------ src/dnsmasq/forward.c | 22 +++++------ src/dnsmasq/metrics.c | 1 + src/dnsmasq/metrics.h | 1 + src/dnsmasq/option.c | 9 ++++- 8 files changed, 94 insertions(+), 52 deletions(-) diff --git a/src/dnsmasq/cache.c b/src/dnsmasq/cache.c index dff3485d..9e608ea0 100644 --- a/src/dnsmasq/cache.c +++ b/src/dnsmasq/cache.c @@ -850,6 +850,12 @@ void cache_end_insert(void) if (daemon->pipe_to_parent != -1) { ssize_t m = -1; + +#ifdef HAVE_DNSSEC + /* Sneak out possibly updated crypto HWM. */ + m = -1 - daemon->metrics[METRIC_CRYTO_HWM]; +#endif + read_write(daemon->pipe_to_parent, (unsigned char *)&m, sizeof(m), 0); } @@ -875,8 +881,13 @@ int cache_recv_insert(time_t now, int fd) if (!read_write(fd, (unsigned char *)&m, sizeof(m), 1)) return 0; - if (m == -1) + if (m < 0) { +#ifdef HAVE_DNSSEC + /* Sneak in possibly updated crypto HWM. */ + if ((-m - 1) > daemon->metrics[METRIC_CRYTO_HWM]) + daemon->metrics[METRIC_CRYTO_HWM] = -m - 1; +#endif cache_end_insert(); return 1; } @@ -1941,6 +1952,9 @@ void dump_cache(time_t now) #ifdef HAVE_AUTH my_syslog(LOG_INFO, _("queries for authoritative zones %u"), daemon->metrics[METRIC_DNS_AUTH_ANSWERED]); #endif +#ifdef HAVE_DNSSEC + my_syslog(LOG_INFO, _("DNSSEC per-query crypto HWM %u"), daemon->metrics[METRIC_CRYTO_HWM]); +#endif blockdata_report(); my_syslog(LOG_INFO, _("child processes for TCP requests: in use %zu, highest since last SIGUSR1 %zu, max allowed %zu."), diff --git a/src/dnsmasq/config.h b/src/dnsmasq/config.h index f545176f..0495895f 100644 --- a/src/dnsmasq/config.h +++ b/src/dnsmasq/config.h @@ -23,6 +23,11 @@ #define SAFE_PKTSZ 1232 /* "go anywhere" UDP packet size, see https://dnsflagday.net/2020/ */ #define KEYBLOCK_LEN 40 /* choose to minimise fragmentation when storing DNSSEC keys */ #define DNSSEC_WORK 50 /* Max number of queries to validate one question */ +#define LIMIT_KEY_FAIL 15 /* Number of keys that can fail DS validate in one an answer. */ +#define LIMIT_DS_FAIL 5 /* Number of DS records that can fail to validate a key in one answer */ +#define LIMIT_SIG_FAIL 10 /* Number of signature that can fail to validate in one answer */ +#define LIMIT_CRYPTO 40 /* max no. of crypto operations to validate one a query. */ +#define LIMIT_NSEC3_ITERS 150 /* Max. number if iterations allow in NSEC3 record. */ #define TIMEOUT 10 /* drop UDP queries after TIMEOUT seconds */ #define SMALL_PORT_RANGE 30 /* If DNS port range is smaller than this, use different allocation. */ #define FORWARD_TEST 1000 /* try all servers every 1000 queries */ diff --git a/src/dnsmasq/dnsmasq.h b/src/dnsmasq/dnsmasq.h index 0a95d40f..cdc67e83 100644 --- a/src/dnsmasq/dnsmasq.h +++ b/src/dnsmasq/dnsmasq.h @@ -765,6 +765,7 @@ struct dyndir { #define DNSSEC_FAIL_NOKEY 0x0100 /* no DNSKEY */ #define DNSSEC_FAIL_NSEC3_ITERS 0x0200 /* too many iterations in NSEC3 */ #define DNSSEC_FAIL_BADPACKET 0x0400 /* bad packet */ +#define DNSSEC_FAIL_WORK 0x0800 /* too much crypto */ #define STAT_ISEQUAL(a, b) (((a) & 0xffff0000) == (b)) @@ -1248,6 +1249,7 @@ extern struct daemon { int rr_status_sz; int dnssec_no_time_check; int back_to_the_future; + int limit_key_fail, limit_ds_fail, limit_sig_fail, limit_crypto, limit_work, limit_nsec3_iters; #endif struct frec *frec_list; struct frec_src *free_frec_src; diff --git a/src/dnsmasq/dnssec.c b/src/dnsmasq/dnssec.c index bc02dadb..1a334ec4 100644 --- a/src/dnsmasq/dnssec.c +++ b/src/dnsmasq/dnssec.c @@ -424,6 +424,17 @@ static int explore_rrset(struct dns_header *header, size_t plen, int class, int return 1; } +int dec_counter(int *counter, char *message) +{ + if ((*counter)-- == 0) + { + my_syslog(LOG_WARNING, "limit exceeded: %s", message ? message : "crypto work"); + return 1; + } + + return 0; +} + /* Validate a single RRset (class, type, name) in the supplied DNS reply Return code: STAT_SECURE if it validates. @@ -468,7 +479,7 @@ static int validate_rrset(time_t now, struct dns_header *header, size_t plen, in rrsetidx = sort_rrset(header, plen, rr_desc, rrsetidx, rrset, daemon->workspacename, keyname); /* Now try all the sigs to try and find one which validates */ - for (sig_fail_cnt = 0, j = 0; j limit_sig_fail, j = 0; j digest_size, algo)) - return STAT_SECURE; + { + if (dec_counter(validate_counter, NULL)) + return STAT_ABANDONED; + + if (verify(key, keylen, sig, sig_len, digest, hash->digest_size, algo)) + return STAT_SECURE; + } } else { @@ -669,21 +683,17 @@ static int validate_rrset(time_t now, struct dns_header *header, size_t plen, in crecp->addr.key.keytag == key_tag && crecp->uid == (unsigned int)class) { - (*validate_counter)++; - + if (dec_counter(validate_counter, NULL)) + return STAT_ABANDONED; + if (verify(crecp->addr.key.keydata, crecp->addr.key.keylen, sig, sig_len, digest, hash->digest_size, algo)) return (labels < name_labels) ? STAT_SECURE_WILDCARD : STAT_SECURE; /* An attacker can waste a lot of our CPU by setting up a giant DNSKEY RRSET full of failing keys, all of which we have to try. Since many failing keys is not likely for a legitimate domain, set a limit on how many can fail. */ - sig_fail_cnt++; - - if (sig_fail_cnt > 10) /* TODO */ - { - my_syslog(LOG_ERR, "sig_fail_cnt"); - return STAT_ABANDONED; - } + if (dec_counter(&sig_fail_cnt, "SIG fail")) + return STAT_ABANDONED; } } } @@ -733,7 +743,7 @@ int dnssec_validate_by_ds(time_t now, struct dns_header *header, size_t plen, ch } /* NOTE, we need to find ONE DNSKEY which matches the DS */ - for (key_fail_cnt = 0, valid = 0, j = ntohs(header->ancount); j != 0 && !valid; j--) + for (key_fail_cnt = daemon->limit_key_fail, valid = 0, j = ntohs(header->ancount); j != 0 && !valid; j--) { /* Ensure we have type, class TTL and length */ if (!(rc = extract_name(header, plen, &p, name, 0, 10))) @@ -781,8 +791,8 @@ int dnssec_validate_by_ds(time_t now, struct dns_header *header, size_t plen, ch /* No zone key flag or malloc failure */ if (!key) continue; - - for (ds_fail_cnt = 0, recp1 = crecp; recp1; recp1 = cache_find_by_name(recp1, name, now, F_DS)) + + for (ds_fail_cnt = daemon->limit_ds_fail, recp1 = crecp; recp1; recp1 = cache_find_by_name(recp1, name, now, F_DS)) { void *ctx; unsigned char *digest, *ds_digest; @@ -801,6 +811,10 @@ int dnssec_validate_by_ds(time_t now, struct dns_header *header, size_t plen, ch else failflags &= ~DNSSEC_FAIL_NODSSUP; + /* computing a hash is a unit of crypto work. */ + if (dec_counter(validate_counter, NULL)) + return STAT_ABANDONED; + if (!hash_init(hash, &ctx, &digest)) continue; @@ -811,7 +825,6 @@ int dnssec_validate_by_ds(time_t now, struct dns_header *header, size_t plen, ch hash->update(ctx, (unsigned int)wire_len, (unsigned char *)name); hash->update(ctx, (unsigned int)rdlen, psave); hash->digest(ctx, hash->digest_size, digest); - (*validate_counter)++; /* computing a hash is a unit of crypto work. */ from_wire(name); @@ -822,13 +835,8 @@ int dnssec_validate_by_ds(time_t now, struct dns_header *header, size_t plen, ch if (memcmp(ds_digest, digest, recp1->addr.ds.keylen) != 0) { /* limit CPU exhaustion attack from large DS x KEY cross-product. */ - ds_fail_cnt++; - - if (ds_fail_cnt > 5) /* TODO */ - { - my_syslog(LOG_ERR, "ds_fail_cnt"); - return STAT_ABANDONED; - } + if (dec_counter(&ds_fail_cnt, "DS fail")) + return STAT_ABANDONED; } else if (explore_rrset(header, plen, class, T_DNSKEY, name, keyname, &sigcnt, &rrcnt) && rrcnt != 0) @@ -858,13 +866,8 @@ int dnssec_validate_by_ds(time_t now, struct dns_header *header, size_t plen, ch blockdata_free(key); /* limit CPU exhaustion attack from large DS x KEY cross-product. */ - key_fail_cnt++; - - if (key_fail_cnt > 15) /* TODO */ - { - my_syslog(LOG_ERR, "key_fail_cnt"); - return STAT_ABANDONED; - } + if (dec_counter(&key_fail_cnt, "KEY fail")) + return STAT_ABANDONED; } if (valid) @@ -1511,7 +1514,7 @@ static int prove_non_existence_nsec3(struct dns_header *header, size_t plen, uns GETSHORT (iterations, p); /* Upper-bound iterations, to avoid DoS. RFC 9276 refers. */ - if (iterations > 150) + if (iterations > daemon->limit_nsec3_iters) return DNSSEC_FAIL_NSEC3_ITERS; salt_len = *p++; @@ -1558,7 +1561,9 @@ static int prove_non_existence_nsec3(struct dns_header *header, size_t plen, uns nsecs[i] = nsec3p; } - (*validate_counter)++; + if (dec_counter(validate_counter, NULL)) + return DNSSEC_FAIL_WORK; + if ((digest_len = hash_name(name, &digest, hash, salt, salt_len, iterations)) == 0) return DNSSEC_FAIL_NONSEC; @@ -1578,7 +1583,9 @@ static int prove_non_existence_nsec3(struct dns_header *header, size_t plen, uns if (wildname && hostname_isequal(closest_encloser, wildname)) break; - (*validate_counter)++; + if (dec_counter(validate_counter, NULL)) + return DNSSEC_FAIL_WORK; + if ((digest_len = hash_name(closest_encloser, &digest, hash, salt, salt_len, iterations)) == 0) return DNSSEC_FAIL_NONSEC; @@ -1607,7 +1614,9 @@ static int prove_non_existence_nsec3(struct dns_header *header, size_t plen, uns return DNSSEC_FAIL_NONSEC; /* Look for NSEC3 that proves the non-existence of the next-closest encloser */ - (*validate_counter)++; + if (dec_counter(validate_counter, NULL)) + return DNSSEC_FAIL_WORK; + if ((digest_len = hash_name(next_closest, &digest, hash, salt, salt_len, iterations)) == 0) return DNSSEC_FAIL_NONSEC; @@ -1623,7 +1632,9 @@ static int prove_non_existence_nsec3(struct dns_header *header, size_t plen, uns wildcard--; *wildcard = '*'; - (*validate_counter)++; + if (dec_counter(validate_counter, NULL)) + return DNSSEC_FAIL_WORK; + if ((digest_len = hash_name(wildcard, &digest, hash, salt, salt_len, iterations)) == 0) return DNSSEC_FAIL_NONSEC; @@ -2074,7 +2085,7 @@ int dnssec_validate_reply(time_t now, struct dns_header *header, size_t plen, ch we'll return BOGUS then. */ if (STAT_ISEQUAL(rc, STAT_SECURE_WILDCARD) && ((rc_nsec = prove_non_existence(header, plen, keyname, name, type1, class1, wildname, NULL, NULL, validate_counter))) != 0) - return STAT_BOGUS | rc_nsec; + return (rc_nsec & DNSSEC_FAIL_WORK) ? STAT_ABANDONED : (STAT_BOGUS | rc_nsec); rc = STAT_SECURE; } @@ -2101,6 +2112,9 @@ int dnssec_validate_reply(time_t now, struct dns_header *header, size_t plen, ch the answer is in an unsigned zone, or there's a NSEC records. */ if ((rc_nsec = prove_non_existence(header, plen, keyname, name, qtype, qclass, NULL, nons, nsec_ttl, validate_counter)) != 0) { + if (rc_nsec & DNSSEC_FAIL_WORK) + return STAT_ABANDONED; + /* Empty DS without NSECS */ if (qtype == T_DS) return STAT_BOGUS | rc_nsec; diff --git a/src/dnsmasq/forward.c b/src/dnsmasq/forward.c index 02be799e..3df8bfe6 100644 --- a/src/dnsmasq/forward.c +++ b/src/dnsmasq/forward.c @@ -17,8 +17,6 @@ #include "dnsmasq.h" #include "../dnsmasq_interface.h" -static int vchwm = 0; /* TODO */ - static struct frec *get_new_frec(time_t now, struct server *serv, int force); static struct frec *lookup_frec(unsigned short id, int fd, void *hash, int *firstp, int *lastp); static struct frec *lookup_frec_by_query(void *hash, unsigned int flags, unsigned int flagmask); @@ -346,8 +344,8 @@ static int forward_query(int udpfd, union mysockaddr *udpaddr, if (ad_reqd) forward->flags |= FREC_AD_QUESTION; #ifdef HAVE_DNSSEC - forward->work_counter = DNSSEC_WORK; - forward->validate_counter = 0; + forward->work_counter = daemon->limit_work; + forward->validate_counter = daemon->limit_crypto; if (do_bit) forward->flags |= FREC_DO_QUESTION; #endif @@ -1398,10 +1396,10 @@ static void return_reply(time_t now, struct frec *forward, struct dns_header *he } } - if (forward->validate_counter > vchwm) - vchwm = forward->validate_counter; + if ((daemon->limit_crypto - forward->validate_counter) > daemon->metrics[METRIC_CRYTO_HWM]) + daemon->metrics[METRIC_CRYTO_HWM] = daemon->limit_crypto - forward->validate_counter; if (extract_request(header, n, daemon->namebuff, NULL)) - my_syslog(LOG_INFO, "Validate_counter %s is %d, HWM is %d", daemon->namebuff, forward->validate_counter, vchwm); /* TODO */ + my_syslog(LOG_INFO, "Validate_counter %s is %d", daemon->namebuff, daemon->limit_crypto - forward->validate_counter); /* TODO */ #endif if (option_bool(OPT_NO_REBIND)) @@ -2542,8 +2540,8 @@ unsigned char *tcp_request(int confd, time_t now, #ifdef HAVE_DNSSEC if (option_bool(OPT_DNSSEC_VALID) && !checking_disabled && (master->flags & SERV_DO_DNSSEC)) { - int keycount = DNSSEC_WORK; /* Limit to number of DNSSEC questions, to catch loops and avoid filling cache. */ - int validatecount = 0; /* How many validations we did */ + int keycount = daemon->limit_work; /* Limit to number of DNSSEC questions, to catch loops and avoid filling cache. */ + int validatecount = daemon->limit_crypto; int status = tcp_key_recurse(now, STAT_OK, header, m, 0, daemon->namebuff, daemon->keyname, serv, have_mark, mark, &keycount, &validatecount); char *result, *domain = "result"; @@ -2572,10 +2570,10 @@ unsigned char *tcp_request(int confd, time_t now, log_query(F_SECSTAT, domain, &a, result, 0); - if (validatecount > vchwm) - vchwm = validatecount; + if ((daemon->limit_crypto - validatecount) > daemon->metrics[METRIC_CRYTO_HWM]) + daemon->metrics[METRIC_CRYTO_HWM] = daemon->limit_crypto - validatecount; if (extract_request(header, m, daemon->namebuff, NULL)) - my_syslog(LOG_INFO, "Validate_counter %s is %d, HWM is %d", daemon->namebuff, validatecount, vchwm); /* TODO */ + my_syslog(LOG_INFO, "Validate_counter %s is %d", daemon->namebuff, daemon->limit_crypto - validatecount); /* TODO */ } #endif diff --git a/src/dnsmasq/metrics.c b/src/dnsmasq/metrics.c index f8b8d9c5..da406149 100644 --- a/src/dnsmasq/metrics.c +++ b/src/dnsmasq/metrics.c @@ -24,6 +24,7 @@ const char * metric_names[] = { "dns_local_answered", "dns_stale_answered", "dns_unanswered", + "max_crypto_use", "bootp", "pxe", "dhcp_ack", diff --git a/src/dnsmasq/metrics.h b/src/dnsmasq/metrics.h index 839e01dd..3ca21734 100644 --- a/src/dnsmasq/metrics.h +++ b/src/dnsmasq/metrics.h @@ -23,6 +23,7 @@ enum { METRIC_DNS_LOCAL_ANSWERED, METRIC_DNS_STALE_ANSWERED, METRIC_DNS_UNANSWERED_QUERY, + METRIC_CRYTO_HWM, METRIC_BOOTP, METRIC_PXE, METRIC_DHCPACK, diff --git a/src/dnsmasq/option.c b/src/dnsmasq/option.c index 39956385..c6c069ba 100644 --- a/src/dnsmasq/option.c +++ b/src/dnsmasq/option.c @@ -5873,7 +5873,14 @@ void read_opts(int argc, char **argv, char *compile_opts) daemon->randport_limit = 1; daemon->host_index = SRC_AH; daemon->max_procs = MAX_PROCS; - daemon->max_procs_used = 0; +#ifdef HAVE_DNSSEC + daemon->limit_key_fail = LIMIT_KEY_FAIL; + daemon->limit_ds_fail = LIMIT_DS_FAIL; + daemon->limit_sig_fail = LIMIT_SIG_FAIL; + daemon->limit_crypto = LIMIT_CRYPTO; + daemon->limit_work = DNSSEC_WORK; + daemon->limit_nsec3_iters = LIMIT_NSEC3_ITERS; +#endif /* See comment above make_servers(). Optimises server-read code. */ mark_servers(0); From 8b9c5d3da8b3d1d73478efa803632408ca9b5a7b Mon Sep 17 00:00:00 2001 From: Simon Kelley Date: Thu, 4 Jan 2024 00:45:31 +0000 Subject: [PATCH 018/339] Update EDE code -> text conversion. Signed-off-by: DL6ER --- src/dnsmasq/cache.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/dnsmasq/cache.c b/src/dnsmasq/cache.c index 9e608ea0..1f7938d6 100644 --- a/src/dnsmasq/cache.c +++ b/src/dnsmasq/cache.c @@ -2116,6 +2116,11 @@ const char *edestr(int ede) case EDE_NO_AUTH: return "no reachable authority"; case EDE_NETERR: return "network error"; case EDE_INVALID_DATA: return "invalid data"; + case EDE_SIG_E_B_V: return "signature expired before valid"; + case EDE_TOO_EARLY: return "too early"; + case EDE_UNS_NS3_ITER: return "unsupported NSEC3 iterations value"; + case EDE_UNABLE_POLICY: return "uanble to conform to policy"; + case EDE_SYNTHESIZED: return "synthesized"; default: return "unknown"; } } From 0ce9541c63603403f00837f22701a16ae3adaf99 Mon Sep 17 00:00:00 2001 From: Simon Kelley Date: Thu, 4 Jan 2024 15:57:43 +0000 Subject: [PATCH 019/339] Rework validate-by-DS to avoid DoS vuln without arbitrary limits. By calculating the hash of a DNSKEY once for each digest algo, we reduce the hashing work from (no. DS) x (no. DNSKEY) to (no. DNSKEY) x (no. distinct digests) The number of distinct digests can never be more than 255 and it's limited by which hashes we implement, so currently only 4. Signed-off-by: DL6ER --- src/dnsmasq/config.h | 6 +- src/dnsmasq/dnsmasq.h | 2 +- src/dnsmasq/dnssec.c | 298 ++++++++++++++++++++---------------------- src/dnsmasq/forward.c | 8 +- src/dnsmasq/option.c | 2 - 5 files changed, 150 insertions(+), 166 deletions(-) diff --git a/src/dnsmasq/config.h b/src/dnsmasq/config.h index 0495895f..382846e9 100644 --- a/src/dnsmasq/config.h +++ b/src/dnsmasq/config.h @@ -23,10 +23,8 @@ #define SAFE_PKTSZ 1232 /* "go anywhere" UDP packet size, see https://dnsflagday.net/2020/ */ #define KEYBLOCK_LEN 40 /* choose to minimise fragmentation when storing DNSSEC keys */ #define DNSSEC_WORK 50 /* Max number of queries to validate one question */ -#define LIMIT_KEY_FAIL 15 /* Number of keys that can fail DS validate in one an answer. */ -#define LIMIT_DS_FAIL 5 /* Number of DS records that can fail to validate a key in one answer */ -#define LIMIT_SIG_FAIL 10 /* Number of signature that can fail to validate in one answer */ -#define LIMIT_CRYPTO 40 /* max no. of crypto operations to validate one a query. */ +#define LIMIT_SIG_FAIL 20 /* Number of signature that can fail to validate in one answer */ +#define LIMIT_CRYPTO 200 /* max no. of crypto operations to validate one a query. */ #define LIMIT_NSEC3_ITERS 150 /* Max. number if iterations allow in NSEC3 record. */ #define TIMEOUT 10 /* drop UDP queries after TIMEOUT seconds */ #define SMALL_PORT_RANGE 30 /* If DNS port range is smaller than this, use different allocation. */ diff --git a/src/dnsmasq/dnsmasq.h b/src/dnsmasq/dnsmasq.h index cdc67e83..66a2d681 100644 --- a/src/dnsmasq/dnsmasq.h +++ b/src/dnsmasq/dnsmasq.h @@ -1249,7 +1249,7 @@ extern struct daemon { int rr_status_sz; int dnssec_no_time_check; int back_to_the_future; - int limit_key_fail, limit_ds_fail, limit_sig_fail, limit_crypto, limit_work, limit_nsec3_iters; + int limit_sig_fail, limit_crypto, limit_work, limit_nsec3_iters; #endif struct frec *frec_list; struct frec_src *free_frec_src; diff --git a/src/dnsmasq/dnssec.c b/src/dnsmasq/dnssec.c index 1a334ec4..036d5609 100644 --- a/src/dnsmasq/dnssec.c +++ b/src/dnsmasq/dnssec.c @@ -711,39 +711,42 @@ static int validate_rrset(time_t now, struct dns_header *header, size_t plen, in or self-sign for DNSKEY RRset is not valid, bad packet. STAT_ABANDONED resource exhaustion. STAT_NEED_DS DS records to validate a key not found, name in keyname - STAT_NEED_KEY DNSKEY records to validate a key not found, name in keyname */ int dnssec_validate_by_ds(time_t now, struct dns_header *header, size_t plen, char *name, char *keyname, int class, int *validate_counter) { - unsigned char *psave, *p = (unsigned char *)(header+1); + unsigned char *psave, *p = (unsigned char *)(header+1), *keyaddr; struct crec *crecp, *recp1; - int rc, j, qtype, qclass, rdlen, flags, algo, valid, keytag, ds_fail_cnt, key_fail_cnt; + int rc, j, qtype, qclass, rdlen, flags, algo, keytag, sigcnt, rrcnt; unsigned long ttl, sig_ttl; - struct blockdata *key; union all_addr a; - int failflags = DNSSEC_FAIL_NOSIG | DNSSEC_FAIL_NODSSUP | DNSSEC_FAIL_NOZONE | DNSSEC_FAIL_NOKEY; + int failflags = DNSSEC_FAIL_NODSSUP | DNSSEC_FAIL_NOZONE; + char valid_digest[255]; + static unsigned char *cached_digest[255]; - if (ntohs(header->qdcount) != 1 || - RCODE(header) == SERVFAIL || RCODE(header) == REFUSED || - !extract_name(header, plen, &p, name, 1, 4)) + if (ntohs(header->qdcount) != 1 || RCODE(header) != NOERROR || !extract_name(header, plen, &p, name, 1, 4)) return STAT_BOGUS | DNSSEC_FAIL_NOKEY; GETSHORT(qtype, p); GETSHORT(qclass, p); - if (qtype != T_DNSKEY || qclass != class || ntohs(header->ancount) == 0) + if (qtype != T_DNSKEY || qclass != class || + !explore_rrset(header, plen, class, T_DNSKEY, name, keyname, &sigcnt, &rrcnt) || + rrcnt == 0) return STAT_BOGUS | DNSSEC_FAIL_NOKEY; + if (sigcnt == 0) + return STAT_BOGUS | DNSSEC_FAIL_NOSIG; + /* See if we have cached a DS record which validates this key */ if (!(crecp = cache_find_by_name(NULL, name, now, F_DS))) { strcpy(keyname, name); return STAT_NEED_DS; } - + /* NOTE, we need to find ONE DNSKEY which matches the DS */ - for (key_fail_cnt = daemon->limit_key_fail, valid = 0, j = ntohs(header->ancount); j != 0 && !valid; j--) + for (j = ntohs(header->ancount); j != 0; j--) { /* Ensure we have type, class TTL and length */ if (!(rc = extract_name(header, plen, &p, name, 0, 10))) @@ -754,7 +757,7 @@ int dnssec_validate_by_ds(time_t now, struct dns_header *header, size_t plen, ch GETLONG(ttl, p); GETSHORT(rdlen, p); - if (!CHECK_LEN(header, p, plen, rdlen) || rdlen < 4) + if (!CHECK_LEN(header, p, plen, rdlen)) return STAT_BOGUS; /* bad packet */ if (qclass != class || qtype != T_DNSKEY || rc == 2) @@ -762,55 +765,59 @@ int dnssec_validate_by_ds(time_t now, struct dns_header *header, size_t plen, ch p += rdlen; continue; } - + + if (rdlen < 5) + return STAT_BOGUS; /* min 1 byte key! */ + psave = p; GETSHORT(flags, p); if (*p++ != 3) - return STAT_BOGUS | DNSSEC_FAIL_NOKEY; + { + p = psave + rdlen; + continue; + } algo = *p++; - keytag = dnskey_keytag(algo, flags, p, rdlen - 4); - key = NULL; + keyaddr = p; + keytag = dnskey_keytag(algo, flags, keyaddr, rdlen - 4); - /* key must have zone key flag set */ - if (flags & 0x100) - { - key = blockdata_alloc((char*)p, rdlen - 4); - failflags &= ~DNSSEC_FAIL_NOZONE; - } - - p = psave; - - if (!ADD_RDLEN(header, p, plen, rdlen)) - { - if (key) - blockdata_free(key); - return STAT_BOGUS; /* bad packet */ - } + p = psave + rdlen; - /* No zone key flag or malloc failure */ - if (!key) + /* key must have zone key flag set */ + if (!(flags & 0x100)) continue; - - for (ds_fail_cnt = daemon->limit_ds_fail, recp1 = crecp; recp1; recp1 = cache_find_by_name(recp1, name, now, F_DS)) + + failflags &= ~DNSSEC_FAIL_NOZONE; + + /* clear digest cache. */ + memset(valid_digest, 0, sizeof(valid_digest)); + + for (recp1 = crecp; recp1; recp1 = cache_find_by_name(recp1, name, now, F_DS)) { void *ctx; unsigned char *digest, *ds_digest; const struct nettle_hash *hash; - int sigcnt, rrcnt; int wire_len; - if (recp1->addr.ds.algo == algo && - recp1->addr.ds.keytag == keytag && - recp1->uid == (unsigned int)class) - { - failflags &= ~DNSSEC_FAIL_NOKEY; + if ((recp1->flags & F_NEG) || + recp1->addr.ds.algo != algo || + recp1->addr.ds.keytag != keytag || + recp1->uid != (unsigned int)class) + continue; + + if (!(hash = hash_find(ds_digest_name(recp1->addr.ds.digest)))) + continue; + + failflags &= ~DNSSEC_FAIL_NODSSUP; - if (!(hash = hash_find(ds_digest_name(recp1->addr.ds.digest)))) - continue; - else - failflags &= ~DNSSEC_FAIL_NODSSUP; + if (recp1->addr.ds.keylen != (int)hash->digest_size || + !(ds_digest = blockdata_retrieve(recp1->addr.ds.keydata, recp1->addr.ds.keylen, NULL))) + continue; + if (valid_digest[recp1->addr.ds.digest]) + digest = cached_digest[recp1->addr.ds.digest]; + else + { /* computing a hash is a unit of crypto work. */ if (dec_counter(validate_counter, NULL)) return STAT_ABANDONED; @@ -821,132 +828,117 @@ int dnssec_validate_by_ds(time_t now, struct dns_header *header, size_t plen, ch wire_len = to_wire(name); /* Note that digest may be different between DSs, so - we can't move this outside the loop. */ + we can't move this outside the loop. We keep + copies of each digest we make for this key, + so maximum digest work is O(keys x digests_types) + rather then O(keys x DSs) */ hash->update(ctx, (unsigned int)wire_len, (unsigned char *)name); hash->update(ctx, (unsigned int)rdlen, psave); hash->digest(ctx, hash->digest_size, digest); from_wire(name); - if (!(recp1->flags & F_NEG) && - recp1->addr.ds.keylen == (int)hash->digest_size && - (ds_digest = blockdata_retrieve(recp1->addr.ds.keydata, recp1->addr.ds.keylen, NULL))) + if (!cached_digest[recp1->addr.ds.digest]) + cached_digest[recp1->addr.ds.digest] = whine_malloc(recp1->addr.ds.keylen); + + if (cached_digest[recp1->addr.ds.digest]) { - if (memcmp(ds_digest, digest, recp1->addr.ds.keylen) != 0) - { - /* limit CPU exhaustion attack from large DS x KEY cross-product. */ - if (dec_counter(&ds_fail_cnt, "DS fail")) - return STAT_ABANDONED; - } - else if (explore_rrset(header, plen, class, T_DNSKEY, name, keyname, &sigcnt, &rrcnt) && - rrcnt != 0) - { - if (sigcnt == 0) - continue; - else - failflags &= ~DNSSEC_FAIL_NOSIG; - - rc = validate_rrset(now, header, plen, class, T_DNSKEY, sigcnt, rrcnt, name, keyname, - NULL, key, rdlen - 4, algo, keytag, &sig_ttl, validate_counter); - - if (STAT_ISEQUAL(rc, STAT_ABANDONED)) - return STAT_ABANDONED; - - failflags &= rc; - - if (STAT_ISEQUAL(rc, STAT_SECURE)) - { - valid = 1; - break; - } - } + memcpy(cached_digest[recp1->addr.ds.digest], digest, recp1->addr.ds.keylen); + valid_digest[recp1->addr.ds.digest] = 1; } } - } - blockdata_free(key); - - /* limit CPU exhaustion attack from large DS x KEY cross-product. */ - if (dec_counter(&key_fail_cnt, "KEY fail")) - return STAT_ABANDONED; - } - - if (valid) - { - /* DNSKEY RRset determined to be OK, now cache it. */ - cache_start_insert(); - - p = skip_questions(header, plen); - - for (j = ntohs(header->ancount); j != 0; j--) - { - /* Ensure we have type, class TTL and length */ - if (!(rc = extract_name(header, plen, &p, name, 0, 10))) - return STAT_BOGUS; /* bad packet */ - GETSHORT(qtype, p); - GETSHORT(qclass, p); - GETLONG(ttl, p); - GETSHORT(rdlen, p); - - /* TTL may be limited by sig. */ - if (sig_ttl < ttl) - ttl = sig_ttl; - - if (!CHECK_LEN(header, p, plen, rdlen)) - return STAT_BOGUS; /* bad packet */ - - if (qclass == class && rc == 1) + if (memcmp(ds_digest, digest, recp1->addr.ds.keylen) == 0) { - psave = p; + /* Found the key validated by a DS record. + Now check the self-sig for the entire key RRset using that key. + Note that validate_rrset() will never return STAT_NEED_KEY here, + since we supply the key it will use as an argument. */ + struct blockdata *key; + + if (!(key = blockdata_alloc((char *)keyaddr, rdlen - 4))) + break; + + rc = validate_rrset(now, header, plen, class, T_DNSKEY, sigcnt, rrcnt, name, keyname, + NULL, key, rdlen - 4, algo, keytag, &sig_ttl, validate_counter); - if (qtype == T_DNSKEY) + blockdata_free(key); + + if (STAT_ISEQUAL(rc, STAT_ABANDONED)) + return rc; + + /* can't validate KEY RRset with this key, see if there's another that + will, which is validated by another DS. */ + if (!STAT_ISEQUAL(rc, STAT_SECURE)) + break; + + /* DNSKEY RRset determined to be OK, now cache it. */ + cache_start_insert(); + + p = skip_questions(header, plen); + + for (j = ntohs(header->ancount); j != 0; j--) { - if (rdlen < 4) + /* Ensure we have type, class TTL and length */ + if (!(rc = extract_name(header, plen, &p, name, 0, 10))) return STAT_BOGUS; /* bad packet */ - GETSHORT(flags, p); - if (*p++ != 3) - return STAT_BOGUS; - algo = *p++; - keytag = dnskey_keytag(algo, flags, p, rdlen - 4); + GETSHORT(qtype, p); + GETSHORT(qclass, p); + GETLONG(ttl, p); + GETSHORT(rdlen, p); - if ((key = blockdata_alloc((char*)p, rdlen - 4))) - { - a.key.keylen = rdlen - 4; - a.key.keydata = key; - a.key.algo = algo; - a.key.keytag = keytag; - a.key.flags = flags; - - if (!cache_insert(name, &a, class, now, ttl, F_FORWARD | F_DNSKEY | F_DNSSECOK)) - { - blockdata_free(key); - return STAT_BOGUS; - } - else - { - a.log.keytag = keytag; - a.log.algo = algo; - if (algo_digest_name(algo)) - log_query(F_NOEXTRA | F_KEYTAG | F_UPSTREAM, name, &a, "DNSKEY keytag %hu, algo %hu", 0); - else - log_query(F_NOEXTRA | F_KEYTAG | F_UPSTREAM, name, &a, "DNSKEY keytag %hu, algo %hu (not supported)", 0); - } - } + /* TTL may be limited by sig. */ + if (sig_ttl < ttl) + ttl = sig_ttl; + + if (!CHECK_LEN(header, p, plen, rdlen)) + return STAT_BOGUS; /* bad packet */ + + psave = p; + + if (qclass == class && rc == 1 && qtype == T_DNSKEY) + { + if (rdlen < 4) + return STAT_BOGUS; /* min 1 byte key! */ + + GETSHORT(flags, p); + if (*p++ == 3) + { + algo = *p++; + keytag = dnskey_keytag(algo, flags, p, rdlen - 4); + + if (!(key = blockdata_alloc((char*)p, rdlen - 4))) + return STAT_BOGUS; + + a.key.keylen = rdlen - 4; + a.key.keydata = key; + a.key.algo = algo; + a.key.keytag = keytag; + a.key.flags = flags; + + if (!cache_insert(name, &a, class, now, ttl, F_FORWARD | F_DNSKEY | F_DNSSECOK)) + return STAT_BOGUS; + + a.log.keytag = keytag; + a.log.algo = algo; + if (algo_digest_name(algo)) + log_query(F_NOEXTRA | F_KEYTAG | F_UPSTREAM, name, &a, "DNSKEY keytag %hu, algo %hu", 0); + else + log_query(F_NOEXTRA | F_KEYTAG | F_UPSTREAM, name, &a, "DNSKEY keytag %hu, algo %hu (not supported)", 0); + } + } + + p = psave + rdlen; } - - p = psave; + + /* commit cache insert. */ + cache_end_insert(); + return STAT_OK; } - - if (!ADD_RDLEN(header, p, plen, rdlen)) - return STAT_BOGUS; /* bad packet */ } - - /* commit cache insert. */ - cache_end_insert(); - return STAT_OK; } - + log_query(F_NOEXTRA | F_UPSTREAM, name, NULL, "BOGUS DNSKEY", 0); return STAT_BOGUS | failflags; } @@ -1056,7 +1048,7 @@ int dnssec_validate_ds(time_t now, struct dns_header *header, size_t plen, char a.log.keytag = keytag; a.log.algo = algo; a.log.digest = digest; - log_query(F_NOEXTRA | F_KEYTAG | F_UPSTREAM, name, &a, "DS keytag %hu, algo %hu, digest %hu (not supported)", 0); + log_query(F_NOEXTRA | F_KEYTAG | F_UPSTREAM, name, &a, "DS for keytag %hu, algo %hu, digest %hu (not supported)", 0); neg_ttl = ttl; } else if ((key = blockdata_alloc((char*)p, rdlen - 4))) @@ -1077,7 +1069,7 @@ int dnssec_validate_ds(time_t now, struct dns_header *header, size_t plen, char a.log.keytag = keytag; a.log.algo = algo; a.log.digest = digest; - log_query(F_NOEXTRA | F_KEYTAG | F_UPSTREAM, name, &a, "DS keytag %hu, algo %hu, digest %hu", 0); + log_query(F_NOEXTRA | F_KEYTAG | F_UPSTREAM, name, &a, "DS for keytag %hu, algo %hu, digest %hu", 0); found_supported = 1; } } diff --git a/src/dnsmasq/forward.c b/src/dnsmasq/forward.c index 3df8bfe6..38939ac5 100644 --- a/src/dnsmasq/forward.c +++ b/src/dnsmasq/forward.c @@ -1396,10 +1396,8 @@ static void return_reply(time_t now, struct frec *forward, struct dns_header *he } } - if ((daemon->limit_crypto - forward->validate_counter) > daemon->metrics[METRIC_CRYTO_HWM]) + if ((daemon->limit_crypto - forward->validate_counter) > (int)daemon->metrics[METRIC_CRYTO_HWM]) daemon->metrics[METRIC_CRYTO_HWM] = daemon->limit_crypto - forward->validate_counter; - if (extract_request(header, n, daemon->namebuff, NULL)) - my_syslog(LOG_INFO, "Validate_counter %s is %d", daemon->namebuff, daemon->limit_crypto - forward->validate_counter); /* TODO */ #endif if (option_bool(OPT_NO_REBIND)) @@ -2570,10 +2568,8 @@ unsigned char *tcp_request(int confd, time_t now, log_query(F_SECSTAT, domain, &a, result, 0); - if ((daemon->limit_crypto - validatecount) > daemon->metrics[METRIC_CRYTO_HWM]) + if ((daemon->limit_crypto - validatecount) > (int)daemon->metrics[METRIC_CRYTO_HWM]) daemon->metrics[METRIC_CRYTO_HWM] = daemon->limit_crypto - validatecount; - if (extract_request(header, m, daemon->namebuff, NULL)) - my_syslog(LOG_INFO, "Validate_counter %s is %d", daemon->namebuff, daemon->limit_crypto - validatecount); /* TODO */ } #endif diff --git a/src/dnsmasq/option.c b/src/dnsmasq/option.c index c6c069ba..120c3406 100644 --- a/src/dnsmasq/option.c +++ b/src/dnsmasq/option.c @@ -5874,8 +5874,6 @@ void read_opts(int argc, char **argv, char *compile_opts) daemon->host_index = SRC_AH; daemon->max_procs = MAX_PROCS; #ifdef HAVE_DNSSEC - daemon->limit_key_fail = LIMIT_KEY_FAIL; - daemon->limit_ds_fail = LIMIT_DS_FAIL; daemon->limit_sig_fail = LIMIT_SIG_FAIL; daemon->limit_crypto = LIMIT_CRYPTO; daemon->limit_work = DNSSEC_WORK; From c32b46772c9ca33622cd97bbbc1e497ca828f189 Mon Sep 17 00:00:00 2001 From: Simon Kelley Date: Fri, 5 Jan 2024 22:56:47 +0000 Subject: [PATCH 020/339] Overhaul data checking in NSEC code. Signed-off-by: DL6ER --- src/dnsmasq/dnssec.c | 102 ++++++++++++++++++++++++++---------------- src/dnsmasq/metrics.c | 4 +- src/dnsmasq/metrics.h | 4 +- 3 files changed, 70 insertions(+), 40 deletions(-) diff --git a/src/dnsmasq/dnssec.c b/src/dnsmasq/dnssec.c index 036d5609..c3246cb6 100644 --- a/src/dnsmasq/dnssec.c +++ b/src/dnsmasq/dnssec.c @@ -918,7 +918,10 @@ int dnssec_validate_by_ds(time_t now, struct dns_header *header, size_t plen, ch a.key.flags = flags; if (!cache_insert(name, &a, class, now, ttl, F_FORWARD | F_DNSKEY | F_DNSSECOK)) - return STAT_BOGUS; + { + blockdata_free(key); + return STAT_BOGUS; + } a.log.keytag = keytag; a.log.algo = algo; @@ -1019,6 +1022,8 @@ int dnssec_validate_ds(time_t now, struct dns_header *header, size_t plen, char for (i = 0; i < ntohs(header->ancount); i++) { + unsigned char *psave; + if (!(rc = extract_name(header, plen, &p, name, 0, 10))) return STAT_BOGUS; /* bad packet */ @@ -1029,15 +1034,16 @@ int dnssec_validate_ds(time_t now, struct dns_header *header, size_t plen, char if (!CHECK_LEN(header, p, plen, rdlen)) return STAT_BOGUS; /* bad packet */ + + psave = p; if (aclass == class && atype == T_DS && rc == 1) { int algo, digest, keytag; - unsigned char *psave = p; struct blockdata *key; - if (rdlen < 4) - return STAT_BOGUS; /* bad packet */ + if (rdlen < 5) + return STAT_BOGUS; /* min 1 byte digest! */ GETSHORT(keytag, p); algo = *p++; @@ -1073,12 +1079,9 @@ int dnssec_validate_ds(time_t now, struct dns_header *header, size_t plen, char found_supported = 1; } } - - p = psave; } - - if (!ADD_RDLEN(header, p, plen, rdlen)) - return STAT_BOGUS; /* bad packet */ + + p = psave + rdlen; } cache_end_insert(); @@ -1201,11 +1204,11 @@ static int prove_non_existence_nsec(struct dns_header *header, size_t plen, unsi p = nsecs[i]; if (!extract_name(header, plen, &p, workspace1, 1, 10)) - return 0; + return DNSSEC_FAIL_BADPACKET; p += 8; /* class, type, TTL */ GETSHORT(rdlen, p); psave = p; - if (!extract_name(header, plen, &p, workspace2, 1, 10)) + if (!extract_name(header, plen, &p, workspace2, 1, 0)) return DNSSEC_FAIL_BADPACKET; /* If NSEC comes from wildcard expansion, use original wildcard @@ -1239,7 +1242,8 @@ static int prove_non_existence_nsec(struct dns_header *header, size_t plen, unsi /* NSEC with the same name as the RR we're testing, check that the type in question doesn't appear in the type map */ rdlen -= p - psave; - /* rdlen is now length of type map, and p points to it */ + /* rdlen is now length of type map, and p points to it + packet checked to be as long as rdlen implies in prove_non_existence() */ /* If we can prove that there's no NS record, return that information. */ if (nons && rdlen >= 2 && p[0] == 0 && (p[2] & (0x80 >> T_NS)) != 0) @@ -1368,23 +1372,23 @@ static int check_nsec3_coverage(struct dns_header *header, size_t plen, int dige for (i = 0; i < nsec_count; i++) if ((p = nsecs[i])) { - if (!extract_name(header, plen, &p, workspace1, 1, 0) || + if (!extract_name(header, plen, &p, workspace1, 1, 10) || !(base32_len = base32_decode(workspace1, (unsigned char *)workspace2))) return 0; p += 8; /* class, type, TTL */ GETSHORT(rdlen, p); + psave = p; + + /* packet checked to be as long as implied by rdlen, salt_len and hash_len in prove_non_existence() */ p++; /* algo */ flags = *p++; /* flags */ p += 2; /* iterations */ salt_len = *p++; /* salt_len */ p += salt_len; /* salt */ hash_len = *p++; /* p now points to next hashed name */ - - if (!CHECK_LEN(header, p, plen, hash_len)) - return 0; - + if (digest_len == base32_len && hash_len == base32_len) { int rc = memcmp(workspace2, digest, digest_len); @@ -1392,7 +1396,8 @@ static int check_nsec3_coverage(struct dns_header *header, size_t plen, int dige if (rc == 0) { /* We found an NSEC3 whose hashed name exactly matches the query, so - we just need to check the type map. p points to the RR data for the record. */ + we just need to check the type map. p points to the RR data for the record. + Note we have packet length up to rdlen bytes checked. */ int offset = (type & 0xff) >> 3; int mask = 0x80 >> (type & 0x07); @@ -1400,15 +1405,12 @@ static int check_nsec3_coverage(struct dns_header *header, size_t plen, int dige p += hash_len; /* skip next-domain hash */ rdlen -= p - psave; - if (!CHECK_LEN(header, p, plen, rdlen)) - return 0; - if (rdlen >= 2 && p[0] == 0) { /* If we can prove that there's no NS record, return that information. */ if (nons && (p[2] & (0x80 >> T_NS)) != 0) *nons = 0; - + /* A CNAME answer would also be valid, so if there's a CNAME is should have been returned. */ if ((p[2] & (0x80 >> T_CNAME)) != 0) @@ -1511,9 +1513,7 @@ static int prove_non_existence_nsec3(struct dns_header *header, size_t plen, uns salt_len = *p++; salt = p; - if (!CHECK_LEN(header, salt, plen, salt_len)) - return DNSSEC_FAIL_BADPACKET; /* bad packet */ - + /* Now prune so we only have NSEC3 records with same iterations, salt and algo */ for (i = 0; i < nsec_count; i++) { @@ -1543,9 +1543,6 @@ static int prove_non_existence_nsec3(struct dns_header *header, size_t plen, uns if (salt_len != *p++) continue; - if (!CHECK_LEN(header, p, plen, salt_len)) - return DNSSEC_FAIL_BADPACKET; /* bad packet */ - if (memcmp(p, salt, salt_len) != 0) continue; @@ -1666,7 +1663,10 @@ static int prove_non_existence(struct dns_header *header, size_t plen, char *key GETSHORT(class, p); GETLONG(ttl, p); GETSHORT(rdlen, p); - + + if (!CHECK_LEN(header, p, plen, rdlen)) + return DNSSEC_FAIL_BADPACKET; + if (class == qclass && (type == T_NSEC || type == T_NSEC3)) { if (nsec_ttl) @@ -1705,22 +1705,25 @@ static int prove_non_existence(struct dns_header *header, size_t plen, char *key for (j = ntohs(header->nscount); j != 0; j--) { + unsigned char *psav; + if (!(res = extract_name(header, plen, &p1, daemon->workspacename, 0, 10))) return DNSSEC_FAIL_BADPACKET; - + GETSHORT(type1, p1); GETSHORT(class1, p1); p1 += 4; /* TTL */ GETSHORT(rdlen1, p1); + psav = p1; + if (!CHECK_LEN(header, p1, plen, rdlen1)) return DNSSEC_FAIL_BADPACKET; if (res == 1 && class1 == qclass && type1 == T_RRSIG) { int type_covered; - unsigned char *psav = p1; - + if (rdlen1 < 18) return DNSSEC_FAIL_BADPACKET; /* bad packet */ @@ -1735,24 +1738,47 @@ static int prove_non_existence(struct dns_header *header, size_t plen, char *key rrsig_labels[nsecs_found] = p1; else if (*rrsig_labels[nsecs_found] != *p1) /* algo */ return DNSSEC_FAIL_NONSEC; - } - p1 = psav; + } } - if (!ADD_RDLEN(header, p1, plen, rdlen1)) - return DNSSEC_FAIL_BADPACKET; + p1 = psav + rdlen1; } /* Must have found at least one sig. */ if (!rrsig_labels[nsecs_found]) return DNSSEC_FAIL_NONSEC; } + else if (type == T_NSEC3) + { + /* Decode the packet structure enough to check that rdlen is big enough + to contain everything other than the type bitmap. + (packet checked to be long enough to contain rdlen above) + We don't need to do any further length checks in check_nes3_coverage() + or prove_non_existence_nsec3() */ + + int salt_len, hash_len; + unsigned char *psav = p; + + if (rdlen < 5) + return DNSSEC_FAIL_BADPACKET; + + p += 4; /* algo, flags, iterations */ + salt_len = *p++; /* salt_len */ + if (rdlen < (6 + salt_len)) + return DNSSEC_FAIL_BADPACKET; /* check up to hash_length */ + + p += salt_len; /* salt */ + hash_len = *p++; + if (rdlen < (6 + salt_len + hash_len)) + return DNSSEC_FAIL_BADPACKET; /* check to end of next hashed name */ + + p = psav; + } nsecset[nsecs_found++] = pstart; } - if (!ADD_RDLEN(header, p, plen, rdlen)) - return DNSSEC_FAIL_BADPACKET; + p += rdlen; } if (type_found == T_NSEC) diff --git a/src/dnsmasq/metrics.c b/src/dnsmasq/metrics.c index da406149..e59e7627 100644 --- a/src/dnsmasq/metrics.c +++ b/src/dnsmasq/metrics.c @@ -24,7 +24,9 @@ const char * metric_names[] = { "dns_local_answered", "dns_stale_answered", "dns_unanswered", - "max_crypto_use", + "dnssec_max_crypto_use", + "dnssec_max_sig_fail", + "dnssec_max_work", "bootp", "pxe", "dhcp_ack", diff --git a/src/dnsmasq/metrics.h b/src/dnsmasq/metrics.h index 3ca21734..cd85e536 100644 --- a/src/dnsmasq/metrics.h +++ b/src/dnsmasq/metrics.h @@ -23,7 +23,9 @@ enum { METRIC_DNS_LOCAL_ANSWERED, METRIC_DNS_STALE_ANSWERED, METRIC_DNS_UNANSWERED_QUERY, - METRIC_CRYTO_HWM, + METRIC_CRYTO_HWM, + METRIC_SIG_FAIL_HWM, + METRIC_WORK_HWM, METRIC_BOOTP, METRIC_PXE, METRIC_DHCPACK, From a389bcca1ae7ca219d0b7a5817d7633143e358bf Mon Sep 17 00:00:00 2001 From: Simon Kelley Date: Sat, 6 Jan 2024 16:13:44 +0000 Subject: [PATCH 021/339] Better stats and logging from DNSSEC resource limiting. Signed-off-by: DL6ER --- src/dnsmasq/cache.c | 37 ++++++++++++++++------- src/dnsmasq/config.h | 4 +-- src/dnsmasq/dnssec.c | 10 ++++--- src/dnsmasq/forward.c | 69 +++++++++++++++++++++++++++---------------- src/dnsmasq/metrics.h | 2 +- 5 files changed, 80 insertions(+), 42 deletions(-) diff --git a/src/dnsmasq/cache.c b/src/dnsmasq/cache.c index 1f7938d6..75cdab40 100644 --- a/src/dnsmasq/cache.c +++ b/src/dnsmasq/cache.c @@ -851,12 +851,17 @@ void cache_end_insert(void) { ssize_t m = -1; -#ifdef HAVE_DNSSEC - /* Sneak out possibly updated crypto HWM. */ - m = -1 - daemon->metrics[METRIC_CRYTO_HWM]; -#endif - read_write(daemon->pipe_to_parent, (unsigned char *)&m, sizeof(m), 0); + +#ifdef HAVE_DNSSEC + /* Sneak out possibly updated crypto HWM values. */ + m = daemon->metrics[METRIC_CRYPTO_HWM]; + read_write(daemon->pipe_to_parent, (unsigned char *)&m, sizeof(m), 0); + m = daemon->metrics[METRIC_SIG_FAIL_HWM]; + read_write(daemon->pipe_to_parent, (unsigned char *)&m, sizeof(m), 0); + m = daemon->metrics[METRIC_WORK_HWM]; + read_write(daemon->pipe_to_parent, (unsigned char *)&m, sizeof(m), 0); +#endif } new_chain = NULL; @@ -875,18 +880,28 @@ int cache_recv_insert(time_t now, int fd) cache_start_insert(); - while(1) + while (1) { if (!read_write(fd, (unsigned char *)&m, sizeof(m), 1)) return 0; - if (m < 0) + if (m == -1) { #ifdef HAVE_DNSSEC /* Sneak in possibly updated crypto HWM. */ - if ((-m - 1) > daemon->metrics[METRIC_CRYTO_HWM]) - daemon->metrics[METRIC_CRYTO_HWM] = -m - 1; + if (!read_write(fd, (unsigned char *)&m, sizeof(m), 1)) + return 0; + if (m > daemon->metrics[METRIC_CRYPTO_HWM]) + daemon->metrics[METRIC_CRYPTO_HWM] = m; + if (!read_write(fd, (unsigned char *)&m, sizeof(m), 1)) + return 0; + if (m > daemon->metrics[METRIC_SIG_FAIL_HWM]) + daemon->metrics[METRIC_SIG_FAIL_HWM] = m; + if (!read_write(fd, (unsigned char *)&m, sizeof(m), 1)) + return 0; + if (m > daemon->metrics[METRIC_WORK_HWM]) + daemon->metrics[METRIC_WORK_HWM] = m; #endif cache_end_insert(); return 1; @@ -1953,7 +1968,9 @@ void dump_cache(time_t now) my_syslog(LOG_INFO, _("queries for authoritative zones %u"), daemon->metrics[METRIC_DNS_AUTH_ANSWERED]); #endif #ifdef HAVE_DNSSEC - my_syslog(LOG_INFO, _("DNSSEC per-query crypto HWM %u"), daemon->metrics[METRIC_CRYTO_HWM]); + my_syslog(LOG_INFO, _("DNSSEC per-query subqueries HWM %u"), daemon->metrics[METRIC_WORK_HWM]); + my_syslog(LOG_INFO, _("DNSSEC per-query crypto work HWM %u"), daemon->metrics[METRIC_CRYPTO_HWM]); + my_syslog(LOG_INFO, _("DNSSEC per-RRSet signature fails HWM %u"), daemon->metrics[METRIC_SIG_FAIL_HWM]); #endif blockdata_report(); diff --git a/src/dnsmasq/config.h b/src/dnsmasq/config.h index 382846e9..eea735c1 100644 --- a/src/dnsmasq/config.h +++ b/src/dnsmasq/config.h @@ -24,8 +24,8 @@ #define KEYBLOCK_LEN 40 /* choose to minimise fragmentation when storing DNSSEC keys */ #define DNSSEC_WORK 50 /* Max number of queries to validate one question */ #define LIMIT_SIG_FAIL 20 /* Number of signature that can fail to validate in one answer */ -#define LIMIT_CRYPTO 200 /* max no. of crypto operations to validate one a query. */ -#define LIMIT_NSEC3_ITERS 150 /* Max. number if iterations allow in NSEC3 record. */ +#define LIMIT_CRYPTO 200 /* max no. of crypto operations to validate one query. */ +#define LIMIT_NSEC3_ITERS 150 /* Max. number if iterations allowed in NSEC3 record. */ #define TIMEOUT 10 /* drop UDP queries after TIMEOUT seconds */ #define SMALL_PORT_RANGE 30 /* If DNS port range is smaller than this, use different allocation. */ #define FORWARD_TEST 1000 /* try all servers every 1000 queries */ diff --git a/src/dnsmasq/dnssec.c b/src/dnsmasq/dnssec.c index c3246cb6..4401908a 100644 --- a/src/dnsmasq/dnssec.c +++ b/src/dnsmasq/dnssec.c @@ -428,7 +428,7 @@ int dec_counter(int *counter, char *message) { if ((*counter)-- == 0) { - my_syslog(LOG_WARNING, "limit exceeded: %s", message ? message : "crypto work"); + my_syslog(LOG_WARNING, "limit exceeded: %s", message ? message : _("per-query crypto work")); return 1; } @@ -686,14 +686,16 @@ static int validate_rrset(time_t now, struct dns_header *header, size_t plen, in if (dec_counter(validate_counter, NULL)) return STAT_ABANDONED; - if (verify(crecp->addr.key.keydata, crecp->addr.key.keylen, sig, sig_len, digest, hash->digest_size, algo)) + if (verify(crecp->addr.key.keydata, crecp->addr.key.keylen, sig, sig_len, digest, hash->digest_size, algo)) return (labels < name_labels) ? STAT_SECURE_WILDCARD : STAT_SECURE; /* An attacker can waste a lot of our CPU by setting up a giant DNSKEY RRSET full of failing keys, all of which we have to try. Since many failing keys is not likely for a legitimate domain, set a limit on how many can fail. */ - if (dec_counter(&sig_fail_cnt, "SIG fail")) - return STAT_ABANDONED; + if ((daemon->limit_sig_fail - (sig_fail_cnt + 1)) > (int)daemon->metrics[METRIC_SIG_FAIL_HWM]) + daemon->metrics[METRIC_SIG_FAIL_HWM] = daemon->limit_sig_fail - (sig_fail_cnt + 1); + if (dec_counter(&sig_fail_cnt, _("per-RRSet signature fails"))) + return STAT_ABANDONED; } } } diff --git a/src/dnsmasq/forward.c b/src/dnsmasq/forward.c index 38939ac5..386e5a82 100644 --- a/src/dnsmasq/forward.c +++ b/src/dnsmasq/forward.c @@ -938,6 +938,7 @@ static void dnssec_validate(struct frec *forward, struct dns_header *header, ssize_t plen, int status, time_t now) { struct frec *orig; + int log_resource = 0; daemon->log_display_id = forward->frec_src.log_id; @@ -980,16 +981,10 @@ static void dnssec_validate(struct frec *forward, struct dns_header *header, status = dnssec_validate_reply(now, header, plen, daemon->namebuff, daemon->keyname, &forward->class, !option_bool(OPT_DNSSEC_IGN_NS) && (forward->sentto->flags & SERV_DO_DNSSEC), NULL, NULL, NULL, &orig->validate_counter); - - if (STAT_ISEQUAL(status, STAT_ABANDONED)) - { - /* Log the actual validation that made us barf. */ - unsigned char *p = (unsigned char *)(header+1); - if (extract_name(header, plen, &p, daemon->namebuff, 0, 4) == 1) - my_syslog(LOG_WARNING, _("validation of %s failed: resource limit exceeded."), - daemon->namebuff[0] ? daemon->namebuff : "."); - } } + + if (STAT_ISEQUAL(status, STAT_ABANDONED)) + log_resource = 1; /* Can't validate, as we're missing key data. Put this answer aside, whilst we get that. */ @@ -1033,6 +1028,11 @@ static void dnssec_validate(struct frec *forward, struct dns_header *header, return; } } + else if (orig->work_counter-- == 0) + { + my_syslog(LOG_WARNING, _("limit exceeded: per-query subqueries")); + log_resource = 1; + } else { struct server *server; @@ -1049,7 +1049,6 @@ static void dnssec_validate(struct frec *forward, struct dns_header *header, daemon->keyname, forward->class, STAT_ISEQUAL(status, STAT_NEED_KEY) ? T_DNSKEY : T_DS, server->edns_pktsz)) && (hash = hash_questions(header, nn, daemon->namebuff)) && - --orig->work_counter != 0 && (fd = allocate_rfd(&rfds, server)) != -1 && (new = get_new_frec(now, server, 1))) { @@ -1115,6 +1114,15 @@ static void dnssec_validate(struct frec *forward, struct dns_header *header, status = STAT_ABANDONED; } + if (log_resource) + { + /* Log the actual validation that made us barf. */ + unsigned char *p = (unsigned char *)(header+1); + if (extract_name(header, plen, &p, daemon->namebuff, 0, 4) == 1) + my_syslog(LOG_WARNING, _("validation of %s failed: resource limit exceeded."), + daemon->namebuff[0] ? daemon->namebuff : "."); + } + #ifdef HAVE_DUMPFILE if (STAT_ISEQUAL(status, STAT_BOGUS) || STAT_ISEQUAL(status, STAT_ABANDONED)) dump_packet_udp((forward->flags & (FREC_DNSKEY_QUERY | FREC_DS_QUERY)) ? DUMP_SEC_BOGUS : DUMP_BOGUS, @@ -1396,8 +1404,11 @@ static void return_reply(time_t now, struct frec *forward, struct dns_header *he } } - if ((daemon->limit_crypto - forward->validate_counter) > (int)daemon->metrics[METRIC_CRYTO_HWM]) - daemon->metrics[METRIC_CRYTO_HWM] = daemon->limit_crypto - forward->validate_counter; + if ((daemon->limit_crypto - forward->validate_counter) > (int)daemon->metrics[METRIC_CRYPTO_HWM]) + daemon->metrics[METRIC_CRYPTO_HWM] = daemon->limit_crypto - forward->validate_counter; + + if ((daemon->limit_work - forward->work_counter) > (int)daemon->metrics[METRIC_WORK_HWM]) + daemon->metrics[METRIC_WORK_HWM] = daemon->limit_work - forward->work_counter; #endif if (option_bool(OPT_NO_REBIND)) @@ -2141,9 +2152,7 @@ static int tcp_key_recurse(time_t now, int status, struct dns_header *header, si int log_save; /* limit the amount of work we do, to avoid cycling forever on loops in the DNS */ - if (--(*keycount) == 0) - new_status = STAT_ABANDONED; - else if (STAT_ISEQUAL(status, STAT_NEED_KEY)) + if (STAT_ISEQUAL(status, STAT_NEED_KEY)) new_status = dnssec_validate_by_ds(now, header, n, name, keyname, class, validatecount); else if (STAT_ISEQUAL(status, STAT_NEED_DS)) new_status = dnssec_validate_ds(now, header, n, name, keyname, class, validatecount); @@ -2152,6 +2161,15 @@ static int tcp_key_recurse(time_t now, int status, struct dns_header *header, si !option_bool(OPT_DNSSEC_IGN_NS) && (server->flags & SERV_DO_DNSSEC), NULL, NULL, NULL, validatecount); + if (!STAT_ISEQUAL(new_status, STAT_NEED_DS) && !STAT_ISEQUAL(new_status, STAT_NEED_KEY) && !STAT_ISEQUAL(new_status, STAT_ABANDONED)) + break; + + if ((*keycount)-- == 0) + { + my_syslog(LOG_WARNING, _("limit exceeded: per-query subqueries")); + new_status = STAT_ABANDONED; + } + if (STAT_ISEQUAL(new_status, STAT_ABANDONED)) { /* Log the actual validation that made us barf. */ @@ -2159,11 +2177,9 @@ static int tcp_key_recurse(time_t now, int status, struct dns_header *header, si if (extract_name(header, n, &p, daemon->namebuff, 0, 4) == 1) my_syslog(LOG_WARNING, _("validation of %s failed: resource limit exceeded."), daemon->namebuff[0] ? daemon->namebuff : "."); + break; } - - if (!STAT_ISEQUAL(new_status, STAT_NEED_DS) && !STAT_ISEQUAL(new_status, STAT_NEED_KEY)) - break; - + /* Can't validate because we need a key/DS whose name now in keyname. Make query for same, and recurse to validate */ if (!packet) @@ -2177,7 +2193,7 @@ static int tcp_key_recurse(time_t now, int status, struct dns_header *header, si new_status = STAT_ABANDONED; break; } - + m = dnssec_generate_query(new_header, ((unsigned char *) new_header) + 65536, keyname, class, STAT_ISEQUAL(new_status, STAT_NEED_KEY) ? T_DNSKEY : T_DS, server->edns_pktsz); @@ -2192,11 +2208,11 @@ static int tcp_key_recurse(time_t now, int status, struct dns_header *header, si daemon->log_display_id = ++daemon->log_id; log_query_mysockaddr(F_NOEXTRA | F_DNSSEC | F_SERVER, keyname, &server->addr, - STAT_ISEQUAL(new_status, STAT_NEED_KEY) ? "dnssec-query[DNSKEY]" : "dnssec-query[DS]", 0); - + STAT_ISEQUAL(new_status, STAT_NEED_KEY) ? "dnssec-query[DNSKEY]" : "dnssec-query[DS]", 0); + new_status = tcp_key_recurse(now, new_status, new_header, m, class, name, keyname, server, have_mark, mark, keycount, validatecount); - + daemon->log_display_id = log_save; if (!STAT_ISEQUAL(new_status, STAT_OK)) @@ -2568,8 +2584,11 @@ unsigned char *tcp_request(int confd, time_t now, log_query(F_SECSTAT, domain, &a, result, 0); - if ((daemon->limit_crypto - validatecount) > (int)daemon->metrics[METRIC_CRYTO_HWM]) - daemon->metrics[METRIC_CRYTO_HWM] = daemon->limit_crypto - validatecount; + if ((daemon->limit_crypto - validatecount) > (int)daemon->metrics[METRIC_CRYPTO_HWM]) + daemon->metrics[METRIC_CRYPTO_HWM] = daemon->limit_crypto - validatecount; + + if ((daemon->limit_work - keycount) > (int)daemon->metrics[METRIC_WORK_HWM]) + daemon->metrics[METRIC_WORK_HWM] = daemon->limit_work - keycount; } #endif diff --git a/src/dnsmasq/metrics.h b/src/dnsmasq/metrics.h index cd85e536..67cb3bfe 100644 --- a/src/dnsmasq/metrics.h +++ b/src/dnsmasq/metrics.h @@ -23,7 +23,7 @@ enum { METRIC_DNS_LOCAL_ANSWERED, METRIC_DNS_STALE_ANSWERED, METRIC_DNS_UNANSWERED_QUERY, - METRIC_CRYTO_HWM, + METRIC_CRYPTO_HWM, METRIC_SIG_FAIL_HWM, METRIC_WORK_HWM, METRIC_BOOTP, From c3bc0f9972dd0bea53ea26778649d9bf07535c5b Mon Sep 17 00:00:00 2001 From: Simon Kelley Date: Sat, 6 Jan 2024 20:51:13 +0000 Subject: [PATCH 022/339] Better allocation code for DS digest cache. Signed-off-by: DL6ER --- src/dnsmasq/dnssec.c | 38 ++++++++++++++++++++++++++++++-------- 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/src/dnsmasq/dnssec.c b/src/dnsmasq/dnssec.c index 4401908a..a3e60ba6 100644 --- a/src/dnsmasq/dnssec.c +++ b/src/dnsmasq/dnssec.c @@ -724,7 +724,8 @@ int dnssec_validate_by_ds(time_t now, struct dns_header *header, size_t plen, ch union all_addr a; int failflags = DNSSEC_FAIL_NODSSUP | DNSSEC_FAIL_NOZONE; char valid_digest[255]; - static unsigned char *cached_digest[255]; + static unsigned char **cached_digest; + static size_t cached_digest_size = 0; if (ntohs(header->qdcount) != 1 || RCODE(header) != NOERROR || !extract_name(header, plen, &p, name, 1, 4)) return STAT_BOGUS | DNSSEC_FAIL_NOKEY; @@ -839,14 +840,35 @@ int dnssec_validate_by_ds(time_t now, struct dns_header *header, size_t plen, ch hash->digest(ctx, hash->digest_size, digest); from_wire(name); - - if (!cached_digest[recp1->addr.ds.digest]) - cached_digest[recp1->addr.ds.digest] = whine_malloc(recp1->addr.ds.keylen); - - if (cached_digest[recp1->addr.ds.digest]) + + if (recp1->addr.ds.digest >= cached_digest_size) { - memcpy(cached_digest[recp1->addr.ds.digest], digest, recp1->addr.ds.keylen); - valid_digest[recp1->addr.ds.digest] = 1; + unsigned char **new; + + /* whine_malloc zeros memory */ + if ((new = whine_malloc((recp1->addr.ds.digest + 5) * sizeof(unsigned char *)))) + { + if (cached_digest_size != 0) + { + memcpy(new, cached_digest, cached_digest_size * sizeof(unsigned char *)); + free(cached_digest); + } + + cached_digest_size = recp1->addr.ds.digest + 5; + cached_digest = new; + } + } + + if (recp1->addr.ds.digest < cached_digest_size) + { + if (!cached_digest[recp1->addr.ds.digest]) + cached_digest[recp1->addr.ds.digest] = whine_malloc(recp1->addr.ds.keylen); + + if (cached_digest[recp1->addr.ds.digest]) + { + memcpy(cached_digest[recp1->addr.ds.digest], digest, recp1->addr.ds.keylen); + valid_digest[recp1->addr.ds.digest] = 1; + } } } From fbc5713104257b39c7e60beeceaef119491947f4 Mon Sep 17 00:00:00 2001 From: Simon Kelley Date: Sun, 7 Jan 2024 22:47:30 +0000 Subject: [PATCH 023/339] Add --dnssec-limits option. Signed-off-by: DL6ER --- src/dnsmasq/config.h | 8 ++++---- src/dnsmasq/dnsmasq.h | 8 +++++++- src/dnsmasq/dnssec.c | 8 ++++---- src/dnsmasq/forward.c | 24 ++++++++++++------------ src/dnsmasq/option.c | 29 +++++++++++++++++++++++++---- 5 files changed, 52 insertions(+), 25 deletions(-) diff --git a/src/dnsmasq/config.h b/src/dnsmasq/config.h index eea735c1..659e068a 100644 --- a/src/dnsmasq/config.h +++ b/src/dnsmasq/config.h @@ -22,10 +22,10 @@ #define EDNS_PKTSZ 1232 /* default max EDNS.0 UDP packet from from /dnsflagday.net/2020 */ #define SAFE_PKTSZ 1232 /* "go anywhere" UDP packet size, see https://dnsflagday.net/2020/ */ #define KEYBLOCK_LEN 40 /* choose to minimise fragmentation when storing DNSSEC keys */ -#define DNSSEC_WORK 50 /* Max number of queries to validate one question */ -#define LIMIT_SIG_FAIL 20 /* Number of signature that can fail to validate in one answer */ -#define LIMIT_CRYPTO 200 /* max no. of crypto operations to validate one query. */ -#define LIMIT_NSEC3_ITERS 150 /* Max. number if iterations allowed in NSEC3 record. */ +#define DNSSEC_LIMIT_WORK 40 /* Max number of queries to validate one question */ +#define DNSSEC_LIMIT_SIG_FAIL 20 /* Number of signature that can fail to validate in one answer */ +#define DNSSEC_LIMIT_CRYPTO 200 /* max no. of crypto operations to validate one query. */ +#define DNSSEC_LIMIT_NSEC3_ITERS 150 /* Max. number if iterations allowed in NSEC3 record. */ #define TIMEOUT 10 /* drop UDP queries after TIMEOUT seconds */ #define SMALL_PORT_RANGE 30 /* If DNS port range is smaller than this, use different allocation. */ #define FORWARD_TEST 1000 /* try all servers every 1000 queries */ diff --git a/src/dnsmasq/dnsmasq.h b/src/dnsmasq/dnsmasq.h index 66a2d681..8a5f0d0b 100644 --- a/src/dnsmasq/dnsmasq.h +++ b/src/dnsmasq/dnsmasq.h @@ -840,6 +840,12 @@ struct frec { #define LEASE_HAVE_HWADDR 128 /* Have set hwaddress */ #define LEASE_EXP_CHANGED 256 /* Lease expiry time changed */ +#define LIMIT_SIG_FAIL 0 +#define LIMIT_CRYPTO 1 +#define LIMIT_WORK 2 +#define LIMIT_NSEC3_ITERS 3 +#define LIMIT_MAX 4 + struct dhcp_lease { int clid_len; /* length of client identifier */ unsigned char *clid; /* clientid */ @@ -1249,7 +1255,7 @@ extern struct daemon { int rr_status_sz; int dnssec_no_time_check; int back_to_the_future; - int limit_sig_fail, limit_crypto, limit_work, limit_nsec3_iters; + int limit[LIMIT_MAX]; #endif struct frec *frec_list; struct frec_src *free_frec_src; diff --git a/src/dnsmasq/dnssec.c b/src/dnsmasq/dnssec.c index a3e60ba6..ed2f53ff 100644 --- a/src/dnsmasq/dnssec.c +++ b/src/dnsmasq/dnssec.c @@ -479,7 +479,7 @@ static int validate_rrset(time_t now, struct dns_header *header, size_t plen, in rrsetidx = sort_rrset(header, plen, rr_desc, rrsetidx, rrset, daemon->workspacename, keyname); /* Now try all the sigs to try and find one which validates */ - for (sig_fail_cnt = daemon->limit_sig_fail, j = 0; j limit[LIMIT_SIG_FAIL], j = 0; j limit_sig_fail - (sig_fail_cnt + 1)) > (int)daemon->metrics[METRIC_SIG_FAIL_HWM]) - daemon->metrics[METRIC_SIG_FAIL_HWM] = daemon->limit_sig_fail - (sig_fail_cnt + 1); + if ((daemon->limit[LIMIT_SIG_FAIL] - (sig_fail_cnt + 1)) > (int)daemon->metrics[METRIC_SIG_FAIL_HWM]) + daemon->metrics[METRIC_SIG_FAIL_HWM] = daemon->limit[LIMIT_SIG_FAIL] - (sig_fail_cnt + 1); if (dec_counter(&sig_fail_cnt, _("per-RRSet signature fails"))) return STAT_ABANDONED; } @@ -1532,7 +1532,7 @@ static int prove_non_existence_nsec3(struct dns_header *header, size_t plen, uns GETSHORT (iterations, p); /* Upper-bound iterations, to avoid DoS. RFC 9276 refers. */ - if (iterations > daemon->limit_nsec3_iters) + if (iterations > daemon->limit[LIMIT_NSEC3_ITERS]) return DNSSEC_FAIL_NSEC3_ITERS; salt_len = *p++; diff --git a/src/dnsmasq/forward.c b/src/dnsmasq/forward.c index 386e5a82..2de082a2 100644 --- a/src/dnsmasq/forward.c +++ b/src/dnsmasq/forward.c @@ -344,8 +344,8 @@ static int forward_query(int udpfd, union mysockaddr *udpaddr, if (ad_reqd) forward->flags |= FREC_AD_QUESTION; #ifdef HAVE_DNSSEC - forward->work_counter = daemon->limit_work; - forward->validate_counter = daemon->limit_crypto; + forward->work_counter = daemon->limit[LIMIT_WORK]; + forward->validate_counter = daemon->limit[LIMIT_CRYPTO]; if (do_bit) forward->flags |= FREC_DO_QUESTION; #endif @@ -1404,11 +1404,11 @@ static void return_reply(time_t now, struct frec *forward, struct dns_header *he } } - if ((daemon->limit_crypto - forward->validate_counter) > (int)daemon->metrics[METRIC_CRYPTO_HWM]) - daemon->metrics[METRIC_CRYPTO_HWM] = daemon->limit_crypto - forward->validate_counter; + if ((daemon->limit[LIMIT_CRYPTO] - forward->validate_counter) > (int)daemon->metrics[METRIC_CRYPTO_HWM]) + daemon->metrics[METRIC_CRYPTO_HWM] = daemon->limit[LIMIT_CRYPTO] - forward->validate_counter; - if ((daemon->limit_work - forward->work_counter) > (int)daemon->metrics[METRIC_WORK_HWM]) - daemon->metrics[METRIC_WORK_HWM] = daemon->limit_work - forward->work_counter; + if ((daemon->limit[LIMIT_WORK] - forward->work_counter) > (int)daemon->metrics[METRIC_WORK_HWM]) + daemon->metrics[METRIC_WORK_HWM] = daemon->limit[LIMIT_WORK] - forward->work_counter; #endif if (option_bool(OPT_NO_REBIND)) @@ -2554,8 +2554,8 @@ unsigned char *tcp_request(int confd, time_t now, #ifdef HAVE_DNSSEC if (option_bool(OPT_DNSSEC_VALID) && !checking_disabled && (master->flags & SERV_DO_DNSSEC)) { - int keycount = daemon->limit_work; /* Limit to number of DNSSEC questions, to catch loops and avoid filling cache. */ - int validatecount = daemon->limit_crypto; + int keycount = daemon->limit[LIMIT_WORK]; /* Limit to number of DNSSEC questions, to catch loops and avoid filling cache. */ + int validatecount = daemon->limit[LIMIT_CRYPTO]; int status = tcp_key_recurse(now, STAT_OK, header, m, 0, daemon->namebuff, daemon->keyname, serv, have_mark, mark, &keycount, &validatecount); char *result, *domain = "result"; @@ -2584,11 +2584,11 @@ unsigned char *tcp_request(int confd, time_t now, log_query(F_SECSTAT, domain, &a, result, 0); - if ((daemon->limit_crypto - validatecount) > (int)daemon->metrics[METRIC_CRYPTO_HWM]) - daemon->metrics[METRIC_CRYPTO_HWM] = daemon->limit_crypto - validatecount; + if ((daemon->limit[LIMIT_CRYPTO] - validatecount) > (int)daemon->metrics[METRIC_CRYPTO_HWM]) + daemon->metrics[METRIC_CRYPTO_HWM] = daemon->limit[LIMIT_CRYPTO] - validatecount; - if ((daemon->limit_work - keycount) > (int)daemon->metrics[METRIC_WORK_HWM]) - daemon->metrics[METRIC_WORK_HWM] = daemon->limit_work - keycount; + if ((daemon->limit[LIMIT_WORK] - keycount) > (int)daemon->metrics[METRIC_WORK_HWM]) + daemon->metrics[METRIC_WORK_HWM] = daemon->limit[LIMIT_WORK] - keycount; } #endif diff --git a/src/dnsmasq/option.c b/src/dnsmasq/option.c index 120c3406..249a6f35 100644 --- a/src/dnsmasq/option.c +++ b/src/dnsmasq/option.c @@ -195,6 +195,7 @@ struct myoption { #define LOPT_NO_DHCP6 382 #define LOPT_NO_DHCP4 383 #define LOPT_MAX_PROCS 384 +#define LOPT_DNSSEC_LIMITS 385 #ifdef HAVE_GETOPT_LONG static const struct option opts[] = @@ -368,6 +369,7 @@ static const struct myoption opts[] = { "dnssec-check-unsigned", 2, 0, LOPT_DNSSEC_CHECK }, { "dnssec-no-timecheck", 0, 0, LOPT_DNSSEC_TIME }, { "dnssec-timestamp", 1, 0, LOPT_DNSSEC_STAMP }, + { "dnssec-limits", 1, 0, LOPT_DNSSEC_LIMITS }, { "dhcp-relay", 1, 0, LOPT_RELAY }, { "ra-param", 1, 0, LOPT_RA_PARAM }, { "quiet-dhcp", 0, 0, LOPT_QUIET_DHCP }, @@ -572,6 +574,7 @@ static struct { { LOPT_DNSSEC_CHECK, ARG_DUP, NULL, gettext_noop("Ensure answers without DNSSEC are in unsigned zones."), NULL }, { LOPT_DNSSEC_TIME, OPT_DNSSEC_TIME, NULL, gettext_noop("Don't check DNSSEC signature timestamps until first cache-reload"), NULL }, { LOPT_DNSSEC_STAMP, ARG_ONE, "", gettext_noop("Timestamp file to verify system clock for DNSSEC"), NULL }, + { LOPT_DNSSEC_LIMITS, ARG_ONE, ",..", gettext_noop("Set resource limits for DNSSEC validation"), NULL }, { LOPT_RA_PARAM, ARG_DUP, ",[mtu:||off,][,][,]", gettext_noop("Set MTU, priority, resend-interval and router-lifetime"), NULL }, { LOPT_QUIET_DHCP, OPT_QUIET_DHCP, NULL, gettext_noop("Do not log routine DHCP."), NULL }, { LOPT_QUIET_DHCP6, OPT_QUIET_DHCP6, NULL, gettext_noop("Do not log routine DHCPv6."), NULL }, @@ -5262,6 +5265,24 @@ err: } #ifdef HAVE_DNSSEC + case LOPT_DNSSEC_LIMITS: + { + int lim, val; + + for (lim = LIMIT_SIG_FAIL; arg && lim < LIMIT_MAX ; lim++, arg = comma) + { + comma = split(arg); + + if (!atoi_check(arg, &val)) + ret_err(gen_err); + + if (val != 0) + daemon->limit[lim] = val; + } + + break; + } + case LOPT_DNSSEC_STAMP: /* --dnssec-timestamp */ daemon->timestamp_file = opt_string_alloc(arg); break; @@ -5874,10 +5895,10 @@ void read_opts(int argc, char **argv, char *compile_opts) daemon->host_index = SRC_AH; daemon->max_procs = MAX_PROCS; #ifdef HAVE_DNSSEC - daemon->limit_sig_fail = LIMIT_SIG_FAIL; - daemon->limit_crypto = LIMIT_CRYPTO; - daemon->limit_work = DNSSEC_WORK; - daemon->limit_nsec3_iters = LIMIT_NSEC3_ITERS; + daemon->limit[LIMIT_SIG_FAIL] = DNSSEC_LIMIT_SIG_FAIL; + daemon->limit[LIMIT_CRYPTO] = DNSSEC_LIMIT_CRYPTO; + daemon->limit[LIMIT_WORK] = DNSSEC_LIMIT_WORK; + daemon->limit[LIMIT_NSEC3_ITERS] = DNSSEC_LIMIT_NSEC3_ITERS; #endif /* See comment above make_servers(). Optimises server-read code. */ From 65402b153176115e165f28cb53e9f22f936f14fd Mon Sep 17 00:00:00 2001 From: Simon Kelley Date: Tue, 13 Feb 2024 13:26:24 +0000 Subject: [PATCH 024/339] Reverse suppression of ANY query answer logging. Signed-off-by: DL6ER --- src/dnsmasq/rfc1035.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/dnsmasq/rfc1035.c b/src/dnsmasq/rfc1035.c index e3222eac..06d3067c 100644 --- a/src/dnsmasq/rfc1035.c +++ b/src/dnsmasq/rfc1035.c @@ -888,8 +888,7 @@ int extract_addresses(struct dns_header *header, size_t qlen, char *name, time_t #ifdef HAVE_DNSSEC if (!option_bool(OPT_DNSSEC_VALID) || aqtype != T_RRSIG) #endif - if (qtype != T_ANY) - log_query(secflag | F_FORWARD | F_UPSTREAM | F_RRNAME, name, NULL, NULL, aqtype); + log_query(secflag | F_FORWARD | F_UPSTREAM | F_RRNAME, name, NULL, NULL, aqtype); } else if (!(flags & F_NXDOMAIN)) { From 3e32d96e329fe061d584ce34ff61728b67d7876a Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 13 Feb 2024 17:07:17 +0100 Subject: [PATCH 025/339] Update expected dnsmasq warnings Signed-off-by: DL6ER --- test/dnsmasq_warnings | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/test/dnsmasq_warnings b/test/dnsmasq_warnings index 3becd253..2e31b3d9 100644 --- a/test/dnsmasq_warnings +++ b/test/dnsmasq_warnings @@ -78,6 +78,8 @@ src/dnsmasq/dnsmasq.c my_syslog(LOG_WARNING, _("failed to access %s: %s"), res->name, strerror(errno)); src/dnsmasq/dnsmasq.c my_syslog(LOG_WARNING, _("no servers found in %s, will retry"), latest->name); +src/dnsmasq/dnssec.c + my_syslog(LOG_WARNING, "limit exceeded: %s", message ? message : _("per-query crypto work")); src/dnsmasq/dnssec.c my_syslog(LOG_WARNING, _("Insecure DS reply received for %s, check domain configuration and upstream DNS server DNSSEC support"), name); src/dnsmasq/forward.c @@ -86,10 +88,20 @@ src/dnsmasq/forward.c my_syslog(LOG_WARNING, _("nameserver %s refused to do a recursive query"), daemon->namebuff); src/dnsmasq/forward.c my_syslog(LOG_WARNING, _("possible DNS-rebind attack detected: %s"), daemon->namebuff); +src/dnsmasq/forward.c + my_syslog(LOG_WARNING, _("limit exceeded: per-query subqueries")); +src/dnsmasq/forward.c + my_syslog(LOG_WARNING, _("validation of %s failed: resource limit exceeded."), + daemon->namebuff[0] ? daemon->namebuff : "."); src/dnsmasq/forward.c my_syslog(LOG_WARNING, _("reducing DNS packet size for nameserver %s to %d"), daemon->addrbuff, SAFE_PKTSZ); src/dnsmasq/forward.c my_syslog(LOG_WARNING, _("ignoring query from non-local network %s (logged only once)"), daemon->addrbuff); +src/dnsmasq/forward.c + my_syslog(LOG_WARNING, _("limit exceeded: per-query subqueries")); +src/dnsmasq/forward.c + my_syslog(LOG_WARNING, _("validation of %s failed: resource limit exceeded."), + daemon->namebuff[0] ? daemon->namebuff : "."); src/dnsmasq/forward.c my_syslog(LOG_WARNING, _("ignoring query from non-local network %s"), daemon->addrbuff); src/dnsmasq/forward.c From 3bb1fcfd3cfa35528ec2978137e50b17a0a02075 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 13 Feb 2024 17:10:01 +0100 Subject: [PATCH 026/339] Update dnsmasq version to 2.90 Signed-off-by: DL6ER --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7e0bd3a5..24fca40d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -11,6 +11,6 @@ cmake_minimum_required(VERSION 2.8.12) project(PIHOLE_FTL C) -set(DNSMASQ_VERSION pi-hole-v2.90test4) +set(DNSMASQ_VERSION pi-hole-v2.90) add_subdirectory(src) From 40886dc78a86171815c70c0a0d6564d8597be421 Mon Sep 17 00:00:00 2001 From: Simon Kelley Date: Mon, 19 Feb 2024 12:22:43 +0000 Subject: [PATCH 027/339] Fix spurious "resource limit exceeded" messages. Replies from upstream with a REFUSED rcode can result in log messages stating that a resource limit has been exceeded, which is not the case. Thanks to Dominik Derigs and the Pi-hole project for spotting this. Signed-off-by: DL6ER --- src/dnsmasq/forward.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/dnsmasq/forward.c b/src/dnsmasq/forward.c index 2de082a2..2176c231 100644 --- a/src/dnsmasq/forward.c +++ b/src/dnsmasq/forward.c @@ -981,10 +981,10 @@ static void dnssec_validate(struct frec *forward, struct dns_header *header, status = dnssec_validate_reply(now, header, plen, daemon->namebuff, daemon->keyname, &forward->class, !option_bool(OPT_DNSSEC_IGN_NS) && (forward->sentto->flags & SERV_DO_DNSSEC), NULL, NULL, NULL, &orig->validate_counter); - } - if (STAT_ISEQUAL(status, STAT_ABANDONED)) - log_resource = 1; + if (STAT_ISEQUAL(status, STAT_ABANDONED)) + log_resource = 1; + } /* Can't validate, as we're missing key data. Put this answer aside, whilst we get that. */ From df58921d47e5cefd9f8d57d36aebaac47a1bf69c Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 19 Feb 2024 13:59:25 +0100 Subject: [PATCH 028/339] Update embedded dnsmasq version to 2.90+1 Signed-off-by: DL6ER --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 24fca40d..39c16d3a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -11,6 +11,6 @@ cmake_minimum_required(VERSION 2.8.12) project(PIHOLE_FTL C) -set(DNSMASQ_VERSION pi-hole-v2.90) +set(DNSMASQ_VERSION pi-hole-v2.90+1) add_subdirectory(src) From ad46a1018a27b17e1320998743f08b90287e8059 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 7 Mar 2024 16:55:11 +0100 Subject: [PATCH 029/339] We should only set local=// if there is no conditional forwarding setting (v6 supports multiple reverse lookup servers), otherwise, this creates a harmless but nonetheless needlessly confusing configuration. Signed-off-by: DL6ER --- src/config/dnsmasq_config.c | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/src/config/dnsmasq_config.c b/src/config/dnsmasq_config.c index 159f34b9..8224fe55 100644 --- a/src/config/dnsmasq_config.c +++ b/src/config/dnsmasq_config.c @@ -449,6 +449,8 @@ bool __attribute__((const)) write_dnsmasq_config(struct config *conf, bool test_ } fputs("\n", pihole_conf); + // Add upstream DNS servers for reverse lookups + bool domain_revServer = false; const unsigned int revServers = cJSON_GetArraySize(conf->dns.revServers.v.json); for(unsigned int i = 0; i < revServers; i++) { @@ -485,8 +487,15 @@ bool __attribute__((const)) write_dnsmasq_config(struct config *conf, bool test_ // If we have a reverse domain, we forward all queries to this domain to // the same destination if(strlen(domain) > 0) + { fprintf(pihole_conf, "server=/%s/%s\n", domain, target); + // Check if the configured domain is the same as the main domain + if(strlen(config.dns.domain.v.s) > 0 && + strcasecmp(domain, config.dns.domain.v.s) == 0) + domain_revServer = true; + } + // Forward unqualified names to the target only when the "never forward // non-FQDN" option is NOT ticked if(!conf->dns.domainNeeded.v.b) @@ -517,7 +526,20 @@ bool __attribute__((const)) write_dnsmasq_config(struct config *conf, bool test_ if(strlen(conf->dns.domain.v.s) > 0) { fputs("# DNS domain for both the DNS and DHCP server\n", pihole_conf); - fprintf(pihole_conf, "domain=%s\n\n", conf->dns.domain.v.s); + if(!domain_revServer) + { + fputs("# This DNS domain in purely local. FTL may answer queries from\n", pihole_conf); + fputs("# /etc/hosts or DHCP but should never forward queries on that\n", pihole_conf); + fputs("# domain to any upstream servers\n", pihole_conf); + fprintf(pihole_conf, "domain=%s\n", conf->dns.domain.v.s); + fprintf(pihole_conf, "local=/%s/\n\n", conf->dns.domain.v.s); + } + else + { + fputs("# This DNS domain is also used for reverse lookups\n", pihole_conf); + fputs("# (see server=//target above)\n", pihole_conf); + fprintf(pihole_conf, "domain=%s\n\n", conf->dns.domain.v.s); + } } if(conf->dhcp.active.v.b) From 231a9853bdca5e4dfdbcde40ea17ec3cdcb7ec46 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 7 Mar 2024 16:58:52 +0100 Subject: [PATCH 030/339] If dns.domainNeeded is set, refuse to send plain domain queries (like laptop) upstream at all. Signed-off-by: DL6ER --- src/config/dnsmasq_config.c | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/src/config/dnsmasq_config.c b/src/config/dnsmasq_config.c index 8224fe55..40019bd5 100644 --- a/src/config/dnsmasq_config.c +++ b/src/config/dnsmasq_config.c @@ -506,19 +506,14 @@ bool __attribute__((const)) write_dnsmasq_config(struct config *conf, bool test_ free(copy); } - // When there is a Pi-hole domain set and "Never forward non-FQDNs" is - // ticked, we add `local=/domain/` to signal that this domain is purely - // local and FTL may answer queries from /etc/hosts or DHCP but should - // never forward queries on that domain to any upstream servers + // When "Never forward non-FQDNs" is ticked, we add `local=//` to signal + // that non-FQDNs queries should never be sent to any upstream servers if(conf->dns.domainNeeded.v.b) { fputs("# Never forward A or AAAA queries for plain names, without\n",pihole_conf); fputs("# dots or domain parts, to upstream nameservers. If the name\n", pihole_conf); - fputs("# is not known from /etc/hosts or DHCP a NXDOMAIN is returned\n", pihole_conf); - if(strlen(conf->dns.domain.v.s)) - fprintf(pihole_conf, "local=/%s/\n\n", conf->dns.domain.v.s); - else - fputs("\n", pihole_conf); + fputs("# is not known from /etc/hosts or DHCP, NXDOMAIN is returned\n", pihole_conf); + fputs("local=//\n\n", pihole_conf); } // Add domain to DNS server. It will also be used for DHCP if the DHCP From d408362efec612194a85865c1df1906ededa8210 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 11 Feb 2024 10:11:52 +0100 Subject: [PATCH 031/339] Remove (undocumented) advanced flag in details config output Signed-off-by: DL6ER --- src/api/config.c | 1 - src/config/config.c | 119 ++++++++++++-------------------------------- src/config/config.h | 11 ++-- 3 files changed, 36 insertions(+), 95 deletions(-) diff --git a/src/api/config.c b/src/api/config.c index 3d3563e9..f7abf159 100644 --- a/src/api/config.c +++ b/src/api/config.c @@ -576,7 +576,6 @@ static int api_config_get(struct ftl_conn *api) // Add config item flags cJSON *flags = JSON_NEW_OBJECT(); JSON_ADD_BOOL_TO_OBJECT(flags, "restart_dnsmasq", conf_item->f & FLAG_RESTART_FTL); - JSON_ADD_BOOL_TO_OBJECT(flags, "advanced", conf_item->f & FLAG_ADVANCED_SETTING); JSON_ADD_BOOL_TO_OBJECT(flags, "session_reset", conf_item->f & FLAG_INVALIDATE_SESSIONS); JSON_ADD_BOOL_TO_OBJECT(flags, "env_var", conf_item->f & FLAG_ENV_VAR); JSON_ADD_ITEM_TO_OBJECT(leaf, "flags", flags); diff --git a/src/config/config.c b/src/config/config.c index 122b26e6..9f4ed785 100644 --- a/src/config/config.c +++ b/src/config/config.c @@ -395,42 +395,36 @@ void initConfig(struct config *conf) conf->dns.CNAMEdeepInspect.k = "dns.CNAMEdeepInspect"; conf->dns.CNAMEdeepInspect.h = "Use this option to control deep CNAME inspection. Disabling it might be beneficial for very low-end devices"; conf->dns.CNAMEdeepInspect.t = CONF_BOOL; - conf->dns.CNAMEdeepInspect.f = FLAG_ADVANCED_SETTING; conf->dns.CNAMEdeepInspect.d.b = true; conf->dns.CNAMEdeepInspect.c = validate_stub; // Only type-based checking conf->dns.blockESNI.k = "dns.blockESNI"; conf->dns.blockESNI.h = "Should _esni. subdomains be blocked by default? Encrypted Server Name Indication (ESNI) is certainly a good step into the right direction to enhance privacy on the web. It prevents on-path observers, including ISPs, coffee shop owners and firewalls, from intercepting the TLS Server Name Indication (SNI) extension by encrypting it. This prevents the SNI from being used to determine which websites users are visiting.\n ESNI will obviously cause issues for pixelserv-tls which will be unable to generate matching certificates on-the-fly when it cannot read the SNI. Cloudflare and Firefox are already enabling ESNI. According to the IEFT draft (link above), we can easily restore piselserv-tls's operation by replying NXDOMAIN to _esni. subdomains of blocked domains as this mimics a \"not configured for this domain\" behavior."; conf->dns.blockESNI.t = CONF_BOOL; - conf->dns.blockESNI.f = FLAG_ADVANCED_SETTING; conf->dns.blockESNI.d.b = true; conf->dns.blockESNI.c = validate_stub; // Only type-based checking conf->dns.EDNS0ECS.k = "dns.EDNS0ECS"; conf->dns.EDNS0ECS.h = "Should we overwrite the query source when client information is provided through EDNS0 client subnet (ECS) information? This allows Pi-hole to obtain client IPs even if they are hidden behind the NAT of a router. This feature has been requested and discussed on Discourse where further information how to use it can be found: https://discourse.pi-hole.net/t/support-for-add-subnet-option-from-dnsmasq-ecs-edns0-client-subnet/35940"; conf->dns.EDNS0ECS.t = CONF_BOOL; - conf->dns.EDNS0ECS.f = FLAG_ADVANCED_SETTING; conf->dns.EDNS0ECS.d.b = true; conf->dns.EDNS0ECS.c = validate_stub; // Only type-based checking conf->dns.ignoreLocalhost.k = "dns.ignoreLocalhost"; conf->dns.ignoreLocalhost.h = "Should FTL hide queries made by localhost?"; conf->dns.ignoreLocalhost.t = CONF_BOOL; - conf->dns.ignoreLocalhost.f = FLAG_ADVANCED_SETTING; conf->dns.ignoreLocalhost.d.b = false; conf->dns.ignoreLocalhost.c = validate_stub; // Only type-based checking conf->dns.showDNSSEC.k = "dns.showDNSSEC"; conf->dns.showDNSSEC.h = "Should FTL should analyze and show internally generated DNSSEC queries?"; conf->dns.showDNSSEC.t = CONF_BOOL; - conf->dns.showDNSSEC.f = FLAG_ADVANCED_SETTING; conf->dns.showDNSSEC.d.b = true; conf->dns.showDNSSEC.c = validate_stub; // Only type-based checking conf->dns.analyzeOnlyAandAAAA.k = "dns.analyzeOnlyAandAAAA"; conf->dns.analyzeOnlyAandAAAA.h = "Should FTL analyze *only* A and AAAA queries?"; conf->dns.analyzeOnlyAandAAAA.t = CONF_BOOL; - conf->dns.analyzeOnlyAandAAAA.f = FLAG_ADVANCED_SETTING; conf->dns.analyzeOnlyAandAAAA.d.b = false; conf->dns.analyzeOnlyAandAAAA.c = validate_stub; // Only type-based checking @@ -447,7 +441,6 @@ void initConfig(struct config *conf) CONFIG_ADD_ENUM_OPTIONS(conf->dns.piholePTR.a, piholePTR); } conf->dns.piholePTR.t = CONF_ENUM_PTR_TYPE; - conf->dns.piholePTR.f = FLAG_ADVANCED_SETTING; conf->dns.piholePTR.d.ptr_type = PTR_PIHOLE; conf->dns.piholePTR.c = validate_stub; // Only type-based checking @@ -464,14 +457,12 @@ void initConfig(struct config *conf) CONFIG_ADD_ENUM_OPTIONS(conf->dns.replyWhenBusy.a, replyWhenBusy); } conf->dns.replyWhenBusy.t = CONF_ENUM_BUSY_TYPE; - conf->dns.replyWhenBusy.f = FLAG_ADVANCED_SETTING; conf->dns.replyWhenBusy.d.busy_reply = BUSY_ALLOW; conf->dns.replyWhenBusy.c = validate_stub; // Only type-based checking conf->dns.blockTTL.k = "dns.blockTTL"; conf->dns.blockTTL.h = "FTL's internal TTL to be handed out for blocked queries in seconds. This settings allows users to select a value different from the dnsmasq config option local-ttl. This is useful in context of locally used hostnames that are known to stay constant over long times (printers, etc.).\n Note that large values may render whitelisting ineffective due to client-side caching of blocked queries."; conf->dns.blockTTL.t = CONF_UINT; - conf->dns.blockTTL.f = FLAG_ADVANCED_SETTING; conf->dns.blockTTL.d.ui = 2; conf->dns.blockTTL.c = validate_stub; // Only type-based checking @@ -479,21 +470,20 @@ void initConfig(struct config *conf) conf->dns.hosts.h = "Array of custom DNS records\n Example: hosts = [ \"127.0.0.1 mylocal\", \"192.168.0.1 therouter\" ]"; conf->dns.hosts.a = cJSON_CreateStringReference("Array of custom DNS records each one in HOSTS form: \"IP HOSTNAME\""); conf->dns.hosts.t = CONF_JSON_STRING_ARRAY; - conf->dns.hosts.f = FLAG_ADVANCED_SETTING; conf->dns.hosts.d.json = cJSON_CreateArray(); conf->dns.hosts.c = validate_dns_hosts; conf->dns.domainNeeded.k = "dns.domainNeeded"; conf->dns.domainNeeded.h = "If set, A and AAAA queries for plain names, without dots or domain parts, are never forwarded to upstream nameservers"; conf->dns.domainNeeded.t = CONF_BOOL; - conf->dns.domainNeeded.f = FLAG_RESTART_FTL | FLAG_ADVANCED_SETTING; + conf->dns.domainNeeded.f = FLAG_RESTART_FTL; conf->dns.domainNeeded.d.b = false; conf->dns.domainNeeded.c = validate_stub; // Only type-based checking conf->dns.expandHosts.k = "dns.expandHosts"; conf->dns.expandHosts.h = "If set, the domain is added to simple names (without a period) in /etc/hosts in the same way as for DHCP-derived names"; conf->dns.expandHosts.t = CONF_BOOL; - conf->dns.expandHosts.f = FLAG_RESTART_FTL | FLAG_ADVANCED_SETTING; + conf->dns.expandHosts.f = FLAG_RESTART_FTL; conf->dns.expandHosts.d.b = false; conf->dns.expandHosts.c = validate_stub; // Only type-based checking @@ -501,14 +491,14 @@ void initConfig(struct config *conf) conf->dns.domain.h = "The DNS domain used by your Pi-hole to expand hosts and for DHCP.\n\n Only if DHCP is enabled below: For DHCP, this has two effects; firstly it causes the DHCP server to return the domain to any hosts which request it, and secondly it sets the domain which it is legal for DHCP-configured hosts to claim. The intention is to constrain hostnames so that an untrusted host on the LAN cannot advertise its name via DHCP as e.g. \"google.com\" and capture traffic not meant for it. If no domain suffix is specified, then any DHCP hostname with a domain part (ie with a period) will be disallowed and logged. If a domain is specified, then hostnames with a domain part are allowed, provided the domain part matches the suffix. In addition, when a suffix is set then hostnames without a domain part have the suffix added as an optional domain part. For instance, we can set domain=mylab.com and have a machine whose DHCP hostname is \"laptop\". The IP address for that machine is available both as \"laptop\" and \"laptop.mylab.com\".\n\n You can disable setting a domain by setting this option to an empty string."; conf->dns.domain.a = cJSON_CreateStringReference(""); conf->dns.domain.t = CONF_STRING; - conf->dns.domain.f = FLAG_RESTART_FTL | FLAG_ADVANCED_SETTING; + conf->dns.domain.f = FLAG_RESTART_FTL; conf->dns.domain.d.s = (char*)"lan"; conf->dns.domain.c = validate_domain; conf->dns.bogusPriv.k = "dns.bogusPriv"; conf->dns.bogusPriv.h = "Should all reverse lookups for private IP ranges (i.e., 192.168.x.y, etc) which are not found in /etc/hosts or the DHCP leases file be answered with \"no such domain\" rather than being forwarded upstream?"; conf->dns.bogusPriv.t = CONF_BOOL; - conf->dns.bogusPriv.f = FLAG_RESTART_FTL | FLAG_ADVANCED_SETTING; + conf->dns.bogusPriv.f = FLAG_RESTART_FTL; conf->dns.bogusPriv.d.b = true; conf->dns.bogusPriv.c = validate_stub; // Only type-based checking @@ -523,7 +513,7 @@ void initConfig(struct config *conf) conf->dns.interface.h = "Interface to use for DNS (see also dnsmasq.listening.mode) and DHCP (if enabled)"; conf->dns.interface.a = cJSON_CreateStringReference("a valid interface name"); conf->dns.interface.t = CONF_STRING; - conf->dns.interface.f = FLAG_RESTART_FTL | FLAG_ADVANCED_SETTING; + conf->dns.interface.f = FLAG_RESTART_FTL; conf->dns.interface.d.s = (char*)""; conf->dns.interface.c = validate_stub; // Type-based checking + dnsmasq syntax checking @@ -531,7 +521,7 @@ void initConfig(struct config *conf) conf->dns.hostRecord.h = "Add A, AAAA and PTR records to the DNS. This adds one or more names to the DNS with associated IPv4 (A) and IPv6 (AAAA) records"; conf->dns.hostRecord.a = cJSON_CreateStringReference("[,....],[],[][,]"); conf->dns.hostRecord.t = CONF_STRING; - conf->dns.hostRecord.f = FLAG_RESTART_FTL | FLAG_ADVANCED_SETTING; + conf->dns.hostRecord.f = FLAG_RESTART_FTL; conf->dns.hostRecord.d.s = (char*)""; conf->dns.hostRecord.c = validate_stub; // Type-based checking + dnsmasq syntax checking @@ -549,7 +539,7 @@ void initConfig(struct config *conf) CONFIG_ADD_ENUM_OPTIONS(conf->dns.listeningMode.a, listeningMode); } conf->dns.listeningMode.t = CONF_ENUM_LISTENING_MODE; - conf->dns.listeningMode.f = FLAG_RESTART_FTL | FLAG_ADVANCED_SETTING; + conf->dns.listeningMode.f = FLAG_RESTART_FTL; conf->dns.listeningMode.d.listeningMode = LISTEN_LOCAL; conf->dns.listeningMode.c = validate_stub; // Only type-based checking @@ -564,14 +554,14 @@ void initConfig(struct config *conf) conf->dns.cnameRecords.h = "List of CNAME records which indicate that is really . If the is given, it overwrites the value of local-ttl"; conf->dns.cnameRecords.a = cJSON_CreateStringReference("Array of CNAMEs each on in one of the following forms: \",[,]\""); conf->dns.cnameRecords.t = CONF_JSON_STRING_ARRAY; - conf->dns.cnameRecords.f = FLAG_RESTART_FTL | FLAG_ADVANCED_SETTING; + conf->dns.cnameRecords.f = FLAG_RESTART_FTL; conf->dns.cnameRecords.d.json = cJSON_CreateArray(); conf->dns.cnameRecords.c = validate_dns_cnames; conf->dns.port.k = "dns.port"; conf->dns.port.h = "Port used by the DNS server"; conf->dns.port.t = CONF_UINT16; - conf->dns.port.f = FLAG_RESTART_FTL | FLAG_ADVANCED_SETTING; + conf->dns.port.f = FLAG_RESTART_FTL; conf->dns.port.d.ui = 53u; conf->dns.port.c = validate_stub; // Only type-based checking @@ -579,14 +569,14 @@ void initConfig(struct config *conf) conf->dns.cache.size.k = "dns.cache.size"; conf->dns.cache.size.h = "Cache size of the DNS server. Note that expiring cache entries naturally make room for new insertions over time. Setting this number too high will have an adverse effect as not only more space is needed, but also lookup speed gets degraded in the 10,000+ range. dnsmasq may issue a warning when you go beyond 10,000+ cache entries."; conf->dns.cache.size.t = CONF_UINT; - conf->dns.cache.size.f = FLAG_RESTART_FTL | FLAG_ADVANCED_SETTING; + conf->dns.cache.size.f = FLAG_RESTART_FTL; conf->dns.cache.size.d.ui = 10000u; conf->dns.cache.size.c = validate_stub; // Only type-based checking conf->dns.cache.optimizer.k = "dns.cache.optimizer"; conf->dns.cache.optimizer.h = "Query cache optimizer: If a DNS name exists in the cache, but its time-to-live has expired only recently, the data will be used anyway (a refreshing from upstream is triggered). This can improve DNS query delays especially over unreliable Internet connections. This feature comes at the expense of possibly sometimes returning out-of-date data and less efficient cache utilization, since old data cannot be flushed when its TTL expires, so the cache becomes mostly least-recently-used. To mitigate issues caused by massively outdated DNS replies, the maximum overaging of cached records is limited. We strongly recommend staying below 86400 (1 day) with this option.\n Setting the TTL excess time to zero will serve stale cache data regardless how long it has expired. This is not recommended as it may lead to stale data being served for a long time. Setting this option to any negative value will disable this feature altogether."; conf->dns.cache.optimizer.t = CONF_INT; - conf->dns.cache.optimizer.f = FLAG_RESTART_FTL | FLAG_ADVANCED_SETTING; + conf->dns.cache.optimizer.f = FLAG_RESTART_FTL; conf->dns.cache.optimizer.d.i = 3600u; conf->dns.cache.optimizer.c = validate_stub; // Only type-based checking @@ -652,7 +642,6 @@ void initConfig(struct config *conf) conf->dns.reply.host.force4.k = "dns.reply.host.force4"; conf->dns.reply.host.force4.h = "Use a specific IPv4 address for the Pi-hole host? By default, FTL determines the address of the interface a query arrived on and uses this address for replying to A queries with the most suitable address for the requesting client. This setting can be used to use a fixed, rather than the dynamically obtained, address when Pi-hole responds to the following names: [ \"pi.hole\", \"\", \"pi.hole.\", \".\" ]"; conf->dns.reply.host.force4.t = CONF_BOOL; - conf->dns.reply.host.force4.f = FLAG_ADVANCED_SETTING; conf->dns.reply.host.force4.d.b = false; conf->dns.reply.host.force4.c = validate_stub; // Only type-based checking @@ -660,14 +649,12 @@ void initConfig(struct config *conf) conf->dns.reply.host.v4.h = "Custom IPv4 address for the Pi-hole host"; conf->dns.reply.host.v4.a = cJSON_CreateStringReference(" or empty string (\"\")"); conf->dns.reply.host.v4.t = CONF_STRUCT_IN_ADDR; - conf->dns.reply.host.v4.f = FLAG_ADVANCED_SETTING; memset(&conf->dns.reply.host.v4.d.in_addr, 0, sizeof(struct in_addr)); conf->dns.reply.host.v4.c = validate_stub; // Only type-based checking conf->dns.reply.host.force6.k = "dns.reply.host.force6"; conf->dns.reply.host.force6.h = "Use a specific IPv6 address for the Pi-hole host? See description for the IPv4 variant above for further details."; conf->dns.reply.host.force6.t = CONF_BOOL; - conf->dns.reply.host.force6.f = FLAG_ADVANCED_SETTING; conf->dns.reply.host.force6.d.b = false; conf->dns.reply.host.force6.c = validate_stub; // Only type-based checking @@ -675,14 +662,12 @@ void initConfig(struct config *conf) conf->dns.reply.host.v6.h = "Custom IPv6 address for the Pi-hole host"; conf->dns.reply.host.v6.a = cJSON_CreateStringReference(" or empty string (\"\")"); conf->dns.reply.host.v6.t = CONF_STRUCT_IN6_ADDR; - conf->dns.reply.host.v6.f = FLAG_ADVANCED_SETTING; memset(&conf->dns.reply.host.v6.d.in6_addr, 0, sizeof(struct in6_addr)); conf->dns.reply.host.v6.c = validate_stub; // Only type-based checking conf->dns.reply.blocking.force4.k = "dns.reply.blocking.force4"; conf->dns.reply.blocking.force4.h = "Use a specific IPv4 address in IP blocking mode? By default, FTL determines the address of the interface a query arrived on and uses this address for replying to A queries with the most suitable address for the requesting client. This setting can be used to use a fixed, rather than the dynamically obtained, address when Pi-hole responds in the following cases: IP blocking mode is used and this query is to be blocked, regular expressions with the ;reply=IP regex extension."; conf->dns.reply.blocking.force4.t = CONF_BOOL; - conf->dns.reply.blocking.force4.f = FLAG_ADVANCED_SETTING; conf->dns.reply.blocking.force4.d.b = false; conf->dns.reply.blocking.force4.c = validate_stub; // Only type-based checking @@ -690,14 +675,12 @@ void initConfig(struct config *conf) conf->dns.reply.blocking.v4.h = "Custom IPv4 address for IP blocking mode"; conf->dns.reply.blocking.v4.a = cJSON_CreateStringReference(" or empty string (\"\")"); conf->dns.reply.blocking.v4.t = CONF_STRUCT_IN_ADDR; - conf->dns.reply.blocking.v4.f = FLAG_ADVANCED_SETTING; memset(&conf->dns.reply.blocking.v4.d.in_addr, 0, sizeof(struct in_addr)); conf->dns.reply.blocking.v4.c = validate_stub; // Only type-based checking conf->dns.reply.blocking.force6.k = "dns.reply.blocking.force6"; conf->dns.reply.blocking.force6.h = "Use a specific IPv6 address in IP blocking mode? See description for the IPv4 variant above for further details."; conf->dns.reply.blocking.force6.t = CONF_BOOL; - conf->dns.reply.blocking.force6.f = FLAG_ADVANCED_SETTING; conf->dns.reply.blocking.force6.d.b = false; conf->dns.reply.blocking.force6.c = validate_stub; // Only type-based checking @@ -705,7 +688,6 @@ void initConfig(struct config *conf) conf->dns.reply.blocking.v6.h = "Custom IPv6 address for IP blocking mode"; conf->dns.reply.blocking.v6.a = cJSON_CreateStringReference(" or empty string (\"\")"); conf->dns.reply.blocking.v6.t = CONF_STRUCT_IN6_ADDR; - conf->dns.reply.blocking.v6.f = FLAG_ADVANCED_SETTING; memset(&conf->dns.reply.blocking.v6.d.in6_addr, 0, sizeof(struct in6_addr)); conf->dns.reply.blocking.v6.c = validate_stub; // Only type-based checking @@ -745,7 +727,7 @@ void initConfig(struct config *conf) conf->dhcp.netmask.h = "The netmask used by your Pi-hole. For directly connected networks (i.e., networks on which the machine running Pi-hole has an interface) the netmask is optional and may be set to an empty string (\"\"): it will then be determined from the interface configuration itself. For networks which receive DHCP service via a relay agent, we cannot determine the netmask itself, so it should explicitly be specified, otherwise Pi-hole guesses based on the class (A, B or C) of the network address."; conf->dhcp.netmask.a = cJSON_CreateStringReference(" (e.g., \"255.255.255.0\") or empty string (\"\") for auto-discovery"); conf->dhcp.netmask.t = CONF_STRUCT_IN_ADDR; - conf->dhcp.netmask.f = FLAG_RESTART_FTL | FLAG_ADVANCED_SETTING; + conf->dhcp.netmask.f = FLAG_RESTART_FTL; memset(&conf->dhcp.netmask.d.in_addr, 0, sizeof(struct in_addr)); conf->dhcp.netmask.c = validate_stub; // Only type-based checking @@ -753,7 +735,7 @@ void initConfig(struct config *conf) conf->dhcp.leaseTime.h = "If the lease time is given, then leases will be given for that length of time. If not given, the default lease time is one hour for IPv4 and one day for IPv6."; conf->dhcp.leaseTime.a = cJSON_CreateStringReference("The lease time can be in seconds, or minutes (e.g., \"45m\") or hours (e.g., \"1h\") or days (like \"2d\") or even weeks (\"1w\"). You may also use \"infinite\" as string but be aware of the drawbacks"); conf->dhcp.leaseTime.t = CONF_STRING; - conf->dhcp.leaseTime.f = FLAG_RESTART_FTL | FLAG_ADVANCED_SETTING; + conf->dhcp.leaseTime.f = FLAG_RESTART_FTL; conf->dhcp.leaseTime.d.s = (char*)""; conf->dhcp.leaseTime.c = validate_stub; // Type-based checking + dnsmasq syntax checking @@ -782,7 +764,7 @@ void initConfig(struct config *conf) conf->dhcp.hosts.h = "Per host parameters for the DHCP server. This allows a machine with a particular hardware address to be always allocated the same hostname, IP address and lease time or to specify static DHCP leases"; conf->dhcp.hosts.a = cJSON_CreateStringReference("Array of static leases each on in one of the following forms: \"[][,id:|*][,set:][,tag:][,][,][,][,ignore]\""); conf->dhcp.hosts.t = CONF_JSON_STRING_ARRAY; - conf->dhcp.hosts.f = FLAG_RESTART_FTL | FLAG_ADVANCED_SETTING; + conf->dhcp.hosts.f = FLAG_RESTART_FTL; conf->dhcp.hosts.d.json = cJSON_CreateArray(); conf->dhcp.hosts.c = validate_stub; // Type-based checking + dnsmasq syntax checking @@ -803,7 +785,6 @@ void initConfig(struct config *conf) conf->resolver.networkNames.k = "resolver.networkNames"; conf->resolver.networkNames.h = "Control whether FTL should use the fallback option to try to obtain client names from checking the network table. This behavior can be disabled with this option.\n Assume an IPv6 client without a host names. However, the network table knows - though the client's MAC address - that this is the same device where we have a host name for another IP address (e.g., a DHCP server managed IPv4 address). In this case, we use the host name associated to the other address as this is the same device."; conf->resolver.networkNames.t = CONF_BOOL; - conf->resolver.networkNames.f = FLAG_ADVANCED_SETTING; conf->resolver.networkNames.d.b = true; conf->resolver.networkNames.c = validate_stub; // Only type-based checking @@ -820,7 +801,6 @@ void initConfig(struct config *conf) CONFIG_ADD_ENUM_OPTIONS(conf->resolver.refreshNames.a, refreshNames); } conf->resolver.refreshNames.t = CONF_ENUM_REFRESH_HOSTNAMES; - conf->resolver.refreshNames.f = FLAG_ADVANCED_SETTING; conf->resolver.refreshNames.d.refresh_hostnames = REFRESH_IPV4_ONLY; conf->resolver.refreshNames.c = validate_stub; // Only type-based checking @@ -864,7 +844,7 @@ void initConfig(struct config *conf) // loss). The gravity database is also not affected as it is only written // to on an individual basis (explicit API calls) and not continuously // (like the query database). - conf->database.useWAL.f = FLAG_ADVANCED_SETTING | FLAG_RESTART_FTL; + conf->database.useWAL.f = FLAG_RESTART_FTL; conf->database.useWAL.d.b = true; conf->database.useWAL.c = validate_stub; // Only type-based checking @@ -872,14 +852,12 @@ void initConfig(struct config *conf) conf->database.network.parseARPcache.k = "database.network.parseARPcache"; conf->database.network.parseARPcache.h = "Should FTL analyze the local ARP cache? When disabled, client identification and the network table will stop working reliably."; conf->database.network.parseARPcache.t = CONF_BOOL; - conf->database.network.parseARPcache.f = FLAG_ADVANCED_SETTING; conf->database.network.parseARPcache.d.b = true; conf->database.network.parseARPcache.c = validate_stub; // Only type-based checking conf->database.network.expire.k = "database.network.expire"; conf->database.network.expire.h = "How long should IP addresses be kept in the network_addresses table [days]? IP addresses (and associated host names) older than the specified number of days are removed to avoid dead entries in the network overview table."; conf->database.network.expire.t = CONF_UINT; - conf->database.network.expire.f = FLAG_ADVANCED_SETTING; conf->database.network.expire.d.ui = conf->database.maxDBdays.d.ui; conf->database.network.expire.c = validate_stub; // Only type-based checking @@ -889,14 +867,14 @@ void initConfig(struct config *conf) conf->webserver.domain.h = "On which domain is the web interface served?"; conf->webserver.domain.a = cJSON_CreateStringReference(""); conf->webserver.domain.t = CONF_STRING; - conf->webserver.domain.f = FLAG_ADVANCED_SETTING | FLAG_RESTART_FTL; + conf->webserver.domain.f = FLAG_RESTART_FTL; conf->webserver.domain.d.s = (char*)"pi.hole"; conf->webserver.domain.c = validate_domain; conf->webserver.acl.k = "webserver.acl"; conf->webserver.acl.h = "Webserver access control list (ACL) allowing for restrictions to be put on the list of IP addresses which have access to the web server. The ACL is a comma separated list of IP subnets, where each subnet is prepended by either a - or a + sign. A plus sign means allow, where a minus sign means deny. If a subnet mask is omitted, such as -1.2.3.4, this means to deny only that single IP address. If this value is not set (empty string), all accesses are allowed. Otherwise, the default setting is to deny all accesses. On each request the full list is traversed, and the last (!) match wins. IPv6 addresses may be specified in CIDR-form [a:b::c]/64.\n\n Example 1: acl = \"+127.0.0.1,+[::1]\"\n ---> deny all access, except from 127.0.0.1 and ::1,\n Example 2: acl = \"+192.168.0.0/16\"\n ---> deny all accesses, except from the 192.168.0.0/16 subnet,\n Example 3: acl = \"+[::]/0\" ---> allow only IPv6 access."; conf->webserver.acl.a = cJSON_CreateStringReference(""); - conf->webserver.acl.f = FLAG_ADVANCED_SETTING | FLAG_RESTART_FTL; + conf->webserver.acl.f = FLAG_RESTART_FTL; conf->webserver.acl.t = CONF_STRING; conf->webserver.acl.d.s = (char*)""; conf->webserver.acl.c = validate_stub; // Type-based checking + civetweb syntax checking @@ -904,14 +882,13 @@ void initConfig(struct config *conf) conf->webserver.port.k = "webserver.port"; conf->webserver.port.h = "Ports to be used by the webserver.\n Comma-separated list of ports to listen on. It is possible to specify an IP address to bind to. In this case, an IP address and a colon must be prepended to the port number. For example, to bind to the loopback interface on port 80 (IPv4) and to all interfaces port 8080 (IPv4), use \"127.0.0.1:80,8080\". \"[::]:80\" can be used to listen to IPv6 connections to port 80. IPv6 addresses of network interfaces can be specified as well, e.g. \"[::1]:80\" for the IPv6 loopback interface. [::]:80 will bind to port 80 IPv6 only.\n In order to use port 80 for all interfaces, both IPv4 and IPv6, use either the configuration \"80,[::]:80\" (create one socket for IPv4 and one for IPv6 only), or \"+80\" (create one socket for both, IPv4 and IPv6). The + notation to use IPv4 and IPv6 will only work if no network interface is specified. Depending on your operating system version and IPv6 network environment, some configurations might not work as expected, so you have to test to find the configuration most suitable for your needs. In case \"+80\" does not work for your environment, you need to use \"80,[::]:80\".\n If the port is TLS/SSL, a letter 's' must be appended, for example, \"80,443s\" will open port 80 and port 443, and connections on port 443 will be encrypted. For non-encrypted ports, it is allowed to append letter 'r' (as in redirect). Redirected ports will redirect all their traffic to the first configured SSL port. For example, if webserver.port is \"80r,443s\", then all HTTP traffic coming at port 80 will be redirected to HTTPS port 443. If this value is not set (empty string), the web server will not be started and, hence, the API will not be available."; conf->webserver.port.a = cJSON_CreateStringReference("comma-separated list of <[ip_address:]port>"); - conf->webserver.port.f = FLAG_ADVANCED_SETTING | FLAG_RESTART_FTL; + conf->webserver.port.f = FLAG_RESTART_FTL; conf->webserver.port.t = CONF_STRING; conf->webserver.port.d.s = (char*)"80,[::]:80,443s,[::]:443s"; conf->webserver.port.c = validate_stub; // Type-based checking + civetweb syntax checking conf->webserver.tls.rev_proxy.k = "webserver.tls.rev_proxy"; conf->webserver.tls.rev_proxy.h = "Is Pi-hole running behind a reverse proxy? If yes, Pi-hole will not consider HTTP-only connections being insecure. This is useful if you are running Pi-hole in a trusted environment, for example, in a local network, and you are using a reverse proxy to provide TLS encryption, e.g., by using Traefik (docker). If you are using a reverse proxy, you can alternatively set webserver.tls.cert to the path of the TLS certificate file and let Pi-hole handle true end-to-end encryption."; - conf->webserver.tls.rev_proxy.f = FLAG_ADVANCED_SETTING; conf->webserver.tls.rev_proxy.t = CONF_BOOL; conf->webserver.tls.rev_proxy.d.b = false; conf->webserver.tls.rev_proxy.c = validate_stub; // Only type-based checking @@ -919,7 +896,7 @@ void initConfig(struct config *conf) conf->webserver.tls.cert.k = "webserver.tls.cert"; conf->webserver.tls.cert.h = "Path to the TLS (SSL) certificate file. This option is only required when at least one of webserver.port is TLS. The file must be in PEM format, and it must have both, private key and certificate (the *.pem file created must contain a 'CERTIFICATE' section as well as a 'RSA PRIVATE KEY' section).\n The *.pem file can be created using\n cp server.crt server.pem\n cat server.key >> server.pem\n if you have these files instead"; conf->webserver.tls.cert.a = cJSON_CreateStringReference(""); - conf->webserver.tls.cert.f = FLAG_ADVANCED_SETTING | FLAG_RESTART_FTL; + conf->webserver.tls.cert.f = FLAG_RESTART_FTL; conf->webserver.tls.cert.t = CONF_STRING; conf->webserver.tls.cert.d.s = (char*)"/etc/pihole/tls.pem"; conf->webserver.tls.cert.c = validate_filepath; @@ -941,7 +918,7 @@ void initConfig(struct config *conf) conf->webserver.paths.webroot.h = "Server root on the host"; conf->webserver.paths.webroot.a = cJSON_CreateStringReference(""); conf->webserver.paths.webroot.t = CONF_STRING; - conf->webserver.paths.webroot.f = FLAG_ADVANCED_SETTING | FLAG_RESTART_FTL; + conf->webserver.paths.webroot.f = FLAG_RESTART_FTL; conf->webserver.paths.webroot.d.s = (char*)"/var/www/html"; conf->webserver.paths.webroot.c = validate_filepath; @@ -949,7 +926,7 @@ void initConfig(struct config *conf) conf->webserver.paths.webhome.h = "Sub-directory of the root containing the web interface"; conf->webserver.paths.webhome.a = cJSON_CreateStringReference(", both slashes are needed!"); conf->webserver.paths.webhome.t = CONF_STRING; - conf->webserver.paths.webhome.f = FLAG_ADVANCED_SETTING | FLAG_RESTART_FTL; + conf->webserver.paths.webhome.f = FLAG_RESTART_FTL; conf->webserver.paths.webhome.d.s = (char*)"/admin/"; conf->webserver.paths.webhome.c = validate_filepath; @@ -992,13 +969,12 @@ void initConfig(struct config *conf) conf->webserver.api.max_sessions.h = "Number of concurrent sessions allowed for the API. If the number of sessions exceeds this value, no new sessions will be allowed until the number of sessions drops due to session expiration or logout. Note that the number of concurrent sessions is irrelevant if authentication is disabled as no sessions are used in this case."; conf->webserver.api.max_sessions.t = CONF_UINT16; conf->webserver.api.max_sessions.d.u16 = 16; - conf->webserver.api.max_sessions.f = FLAG_ADVANCED_SETTING | FLAG_RESTART_FTL; + conf->webserver.api.max_sessions.f = FLAG_RESTART_FTL; conf->webserver.api.max_sessions.c = validate_stub; // Only type-based checking conf->webserver.api.prettyJSON.k = "webserver.api.prettyJSON"; conf->webserver.api.prettyJSON.h = "Should FTL prettify the API output (add extra spaces, newlines and indentation)?"; conf->webserver.api.prettyJSON.t = CONF_BOOL; - conf->webserver.api.prettyJSON.f = FLAG_ADVANCED_SETTING; conf->webserver.api.prettyJSON.d.b = false; conf->webserver.api.prettyJSON.c = validate_stub; // Only type-based checking @@ -1093,7 +1069,7 @@ void initConfig(struct config *conf) conf->files.pid.h = "The file which contains the PID of FTL's main process."; conf->files.pid.a = cJSON_CreateStringReference(""); conf->files.pid.t = CONF_STRING; - conf->files.pid.f = FLAG_ADVANCED_SETTING | FLAG_RESTART_FTL; + conf->files.pid.f = FLAG_RESTART_FTL; conf->files.pid.d.s = (char*)"/run/pihole-FTL.pid"; conf->files.pid.c = validate_filepath; @@ -1101,7 +1077,6 @@ void initConfig(struct config *conf) conf->files.database.h = "The location of FTL's long-term database"; conf->files.database.a = cJSON_CreateStringReference(""); conf->files.database.t = CONF_STRING; - conf->files.database.f = FLAG_ADVANCED_SETTING; conf->files.database.d.s = (char*)"/etc/pihole/pihole-FTL.db"; conf->files.database.c = validate_filepath; @@ -1109,7 +1084,7 @@ void initConfig(struct config *conf) conf->files.gravity.h = "The location of Pi-hole's gravity database"; conf->files.gravity.a = cJSON_CreateStringReference(""); conf->files.gravity.t = CONF_STRING; - conf->files.gravity.f = FLAG_ADVANCED_SETTING | FLAG_RESTART_FTL; + conf->files.gravity.f = FLAG_RESTART_FTL; conf->files.gravity.d.s = (char*)"/etc/pihole/gravity.db"; conf->files.gravity.c = validate_filepath; @@ -1117,7 +1092,7 @@ void initConfig(struct config *conf) conf->files.gravity_tmp.h = "A temporary directory where Pi-hole can store files during gravity updates. This directory must be writable by the user running gravity (typically pihole)."; conf->files.gravity_tmp.a = cJSON_CreateStringReference(""); conf->files.gravity_tmp.t = CONF_STRING; - conf->files.gravity_tmp.f = FLAG_ADVANCED_SETTING | FLAG_RESTART_FTL; + conf->files.gravity_tmp.f = FLAG_RESTART_FTL; conf->files.gravity_tmp.d.s = (char*)"/tmp"; conf->files.gravity_tmp.c = validate_stub; // Only type-based checking @@ -1125,7 +1100,6 @@ void initConfig(struct config *conf) conf->files.macvendor.h = "The database containing MAC -> Vendor information for the network table"; conf->files.macvendor.a = cJSON_CreateStringReference(""); conf->files.macvendor.t = CONF_STRING; - conf->files.macvendor.f = FLAG_ADVANCED_SETTING; conf->files.macvendor.d.s = (char*)"/etc/pihole/macvendor.db"; conf->files.macvendor.c = validate_filepath; @@ -1133,7 +1107,6 @@ void initConfig(struct config *conf) conf->files.setupVars.h = "The old config file of Pi-hole used before v6.0"; conf->files.setupVars.a = cJSON_CreateStringReference(""); conf->files.setupVars.t = CONF_STRING; - conf->files.setupVars.f = FLAG_ADVANCED_SETTING; conf->files.setupVars.d.s = (char*)"/etc/pihole/setupVars.conf"; conf->files.setupVars.c = validate_filepath; @@ -1141,7 +1114,7 @@ void initConfig(struct config *conf) conf->files.pcap.h = "An optional file containing a pcap capture of the network traffic. This file is used for debugging purposes only. If you don't know what this is, you don't need it.\n Setting this to an empty string disables pcap recording. The file must be writable by the user running FTL (typically pihole). Failure to write to this file will prevent the DNS resolver from starting. The file is appended to if it already exists."; conf->files.pcap.a = cJSON_CreateStringReference(""); conf->files.pcap.t = CONF_STRING; - conf->files.pcap.f = FLAG_ADVANCED_SETTING | FLAG_RESTART_FTL; + conf->files.pcap.f = FLAG_RESTART_FTL; conf->files.pcap.d.s = (char*)""; conf->files.pcap.c = validate_filepath_empty; @@ -1152,7 +1125,7 @@ void initConfig(struct config *conf) conf->files.log.webserver.h = "The log file used by the webserver"; conf->files.log.webserver.a = cJSON_CreateStringReference(""); conf->files.log.webserver.t = CONF_STRING; - conf->files.log.webserver.f = FLAG_ADVANCED_SETTING | FLAG_RESTART_FTL; + conf->files.log.webserver.f = FLAG_RESTART_FTL; conf->files.log.webserver.d.s = (char*)"/var/log/pihole/webserver.log"; conf->files.log.webserver.c = validate_filepath; @@ -1160,7 +1133,7 @@ void initConfig(struct config *conf) conf->files.log.dnsmasq.h = "The log file used by the embedded dnsmasq DNS server"; conf->files.log.dnsmasq.a = cJSON_CreateStringReference(""); conf->files.log.dnsmasq.t = CONF_STRING; - conf->files.log.dnsmasq.f = FLAG_ADVANCED_SETTING | FLAG_RESTART_FTL; + conf->files.log.dnsmasq.f = FLAG_RESTART_FTL; conf->files.log.dnsmasq.d.s = (char*)"/var/log/pihole/pihole.log"; conf->files.log.dnsmasq.c = validate_filepath_dash; @@ -1191,21 +1164,20 @@ void initConfig(struct config *conf) conf->misc.nice.k = "misc.nice"; conf->misc.nice.h = "Set niceness of pihole-FTL. Defaults to -10 and can be disabled altogether by setting a value of -999. The nice value is an attribute that can be used to influence the CPU scheduler to favor or disfavor a process in scheduling decisions. The range of the nice value varies across UNIX systems. On modern Linux, the range is -20 (high priority = not very nice to other processes) to +19 (low priority)."; conf->misc.nice.t = CONF_INT; - conf->misc.nice.f = FLAG_ADVANCED_SETTING | FLAG_RESTART_FTL; + conf->misc.nice.f = FLAG_RESTART_FTL; conf->misc.nice.d.i = -10; conf->misc.nice.c = validate_stub; // Only type-based checking conf->misc.addr2line.k = "misc.addr2line"; conf->misc.addr2line.h = "Should FTL translate its own stack addresses into code lines during the bug backtrace? This improves the analysis of crashed significantly. It is recommended to leave the option enabled. This option should only be disabled when addr2line is known to not be working correctly on the machine because, in this case, the malfunctioning addr2line can prevent from generating any backtrace at all."; conf->misc.addr2line.t = CONF_BOOL; - conf->misc.addr2line.f = FLAG_ADVANCED_SETTING; conf->misc.addr2line.d.b = true; conf->misc.addr2line.c = validate_stub; // Only type-based checking conf->misc.etc_dnsmasq_d.k = "misc.etc_dnsmasq_d"; conf->misc.etc_dnsmasq_d.h = "Should FTL load additional dnsmasq configuration files from /etc/dnsmasq.d/?"; conf->misc.etc_dnsmasq_d.t = CONF_BOOL; - conf->misc.etc_dnsmasq_d.f = FLAG_RESTART_FTL | FLAG_ADVANCED_SETTING; + conf->misc.etc_dnsmasq_d.f = FLAG_RESTART_FTL; conf->misc.etc_dnsmasq_d.d.b = false; conf->misc.etc_dnsmasq_d.c = validate_stub; // Only type-based checking @@ -1213,7 +1185,7 @@ void initConfig(struct config *conf) conf->misc.dnsmasq_lines.h = "Additional lines to inject into the generated dnsmasq configuration.\n Warning: This is an advanced setting and should only be used with care. Incorrectly formatted or duplicated lines as well as lines conflicting with the automatic configuration of Pi-hole can break the embedded dnsmasq and will stop DNS resolution from working.\n Use this option with extra care."; conf->misc.dnsmasq_lines.a = cJSON_CreateStringReference("array of valid dnsmasq config line options"); conf->misc.dnsmasq_lines.t = CONF_JSON_STRING_ARRAY; - conf->misc.dnsmasq_lines.f = FLAG_ADVANCED_SETTING | FLAG_RESTART_FTL; + conf->misc.dnsmasq_lines.f = FLAG_RESTART_FTL; conf->misc.dnsmasq_lines.d.json = cJSON_CreateArray(); conf->misc.dnsmasq_lines.c = validate_stub; // Type-based checking + dnsmasq syntax checking @@ -1248,196 +1220,168 @@ void initConfig(struct config *conf) conf->debug.database.k = "debug.database"; conf->debug.database.h = "Print debugging information about database actions. This prints performed SQL statements as well as some general information such as the time it took to store the queries and how many have been saved to the database."; conf->debug.database.t = CONF_BOOL; - conf->debug.database.f = FLAG_ADVANCED_SETTING; conf->debug.database.d.b = false; conf->debug.database.c = validate_stub; // Only type-based checking conf->debug.networking.k = "debug.networking"; conf->debug.networking.h = "Prints a list of the detected interfaces on the startup of pihole-FTL. Also, prints whether these interfaces are IPv4 or IPv6 interfaces."; conf->debug.networking.t = CONF_BOOL; - conf->debug.networking.f = FLAG_ADVANCED_SETTING; conf->debug.networking.d.b = false; conf->debug.networking.c = validate_stub; // Only type-based checking conf->debug.locks.k = "debug.locks"; conf->debug.locks.h = "Print information about shared memory locks. Messages will be generated when waiting, obtaining, and releasing a lock."; conf->debug.locks.t = CONF_BOOL; - conf->debug.locks.f = FLAG_ADVANCED_SETTING; conf->debug.locks.d.b = false; conf->debug.locks.c = validate_stub; // Only type-based checking conf->debug.queries.k = "debug.queries"; conf->debug.queries.h = "Print extensive query information (domains, types, replies, etc.). This has always been part of the legacy debug mode of pihole-FTL."; conf->debug.queries.t = CONF_BOOL; - conf->debug.queries.f = FLAG_ADVANCED_SETTING; conf->debug.queries.d.b = false; conf->debug.queries.c = validate_stub; // Only type-based checking conf->debug.flags.k = "debug.flags"; conf->debug.flags.h = "Print flags of queries received by the DNS hooks. Only effective when DEBUG_QUERIES is enabled as well."; conf->debug.flags.t = CONF_BOOL; - conf->debug.flags.f = FLAG_ADVANCED_SETTING; conf->debug.flags.d.b = false; conf->debug.flags.c = validate_stub; // Only type-based checking conf->debug.shmem.k = "debug.shmem"; conf->debug.shmem.h = "Print information about shared memory buffers. Messages are either about creating or enlarging shmem objects or string injections."; conf->debug.shmem.t = CONF_BOOL; - conf->debug.shmem.f = FLAG_ADVANCED_SETTING; conf->debug.shmem.d.b = false; conf->debug.shmem.c = validate_stub; // Only type-based checking conf->debug.gc.k = "debug.gc"; conf->debug.gc.h = "Print information about garbage collection (GC): What is to be removed, how many have been removed and how long did GC take."; conf->debug.gc.t = CONF_BOOL; - conf->debug.gc.f = FLAG_ADVANCED_SETTING; conf->debug.gc.d.b = false; conf->debug.gc.c = validate_stub; // Only type-based checking conf->debug.arp.k = "debug.arp"; conf->debug.arp.h = "Print information about ARP table processing: How long did parsing take, whether read MAC addresses are valid, and if the macvendor.db file exists."; conf->debug.arp.t = CONF_BOOL; - conf->debug.arp.f = FLAG_ADVANCED_SETTING; conf->debug.arp.d.b = false; conf->debug.arp.c = validate_stub; // Only type-based checking conf->debug.regex.k = "debug.regex"; conf->debug.regex.h = "Controls if FTLDNS should print extended details about regex matching into FTL.log."; conf->debug.regex.t = CONF_BOOL; - conf->debug.regex.f = FLAG_ADVANCED_SETTING; conf->debug.regex.d.b = false; conf->debug.regex.c = validate_stub; // Only type-based checking conf->debug.api.k = "debug.api"; conf->debug.api.h = "Print extra debugging information concerning API calls. This includes the request, the request parameters, and the internal details about how the algorithms decide which data to present and in what form. This very verbose output should only be used when debugging specific API issues and can be helpful, e.g., when a client cannot connect due to an obscure API error. Furthermore, this setting enables logging of all API requests (auth log) and details about user authentication attempts."; conf->debug.api.t = CONF_BOOL; - conf->debug.api.f = FLAG_ADVANCED_SETTING; conf->debug.api.d.b = false; conf->debug.api.c = validate_stub; // Only type-based checking conf->debug.tls.k = "debug.tls"; conf->debug.tls.h = "Print extra debugging information about TLS connections. This includes the TLS version, the cipher suite, the certificate chain and much more. This very verbose output should only be used when debugging specific TLS issues and can be helpful, e.g., when a client cannot connect due to an obscure TLS error as modern browsers do not provide much information about the underlying TLS connection and most often give only very generic error messages without much/any underlying technical information."; conf->debug.tls.t = CONF_BOOL; - conf->debug.tls.f = FLAG_ADVANCED_SETTING; conf->debug.tls.d.b = false; conf->debug.tls.c = validate_stub; // Only type-based checking conf->debug.overtime.k = "debug.overtime"; conf->debug.overtime.h = "Print information about overTime memory operations, such as initializing or moving overTime slots."; conf->debug.overtime.t = CONF_BOOL; - conf->debug.overtime.f = FLAG_ADVANCED_SETTING; conf->debug.overtime.d.b = false; conf->debug.overtime.c = validate_stub; // Only type-based checking conf->debug.status.k = "debug.status"; conf->debug.status.h = "Print information about status changes for individual queries. This can be useful to identify unexpected unknown queries."; conf->debug.status.t = CONF_BOOL; - conf->debug.status.f = FLAG_ADVANCED_SETTING; conf->debug.status.d.b = false; conf->debug.status.c = validate_stub; // Only type-based checking conf->debug.caps.k = "debug.caps"; conf->debug.caps.h = "Print information about capabilities granted to the pihole-FTL process. The current capabilities are printed on receipt of SIGHUP, i.e., the current set of capabilities can be queried without restarting pihole-FTL (by setting DEBUG_CAPS=true and thereafter sending killall -HUP pihole-FTL)."; conf->debug.caps.t = CONF_BOOL; - conf->debug.caps.f = FLAG_ADVANCED_SETTING; conf->debug.caps.d.b = false; conf->debug.caps.c = validate_stub; // Only type-based checking conf->debug.dnssec.k = "debug.dnssec"; conf->debug.dnssec.h = "Print information about DNSSEC activity"; conf->debug.dnssec.t = CONF_BOOL; - conf->debug.dnssec.f = FLAG_ADVANCED_SETTING; conf->debug.dnssec.d.b = false; conf->debug.dnssec.c = validate_stub; // Only type-based checking conf->debug.vectors.k = "debug.vectors"; conf->debug.vectors.h = "FTL uses dynamically allocated vectors for various tasks. This config option enables extensive debugging information such as information about allocation, referencing, deletion, and appending."; conf->debug.vectors.t = CONF_BOOL; - conf->debug.vectors.f = FLAG_ADVANCED_SETTING; conf->debug.vectors.d.b = false; conf->debug.vectors.c = validate_stub; // Only type-based checking conf->debug.resolver.k = "debug.resolver"; conf->debug.resolver.h = "Extensive information about hostname resolution like which DNS servers are used in the first and second hostname resolving tries (only affecting internally generated PTR queries)."; conf->debug.resolver.t = CONF_BOOL; - conf->debug.resolver.f = FLAG_ADVANCED_SETTING; conf->debug.resolver.d.b = false; conf->debug.resolver.c = validate_stub; // Only type-based checking conf->debug.edns0.k = "debug.edns0"; conf->debug.edns0.h = "Print debugging information about received EDNS(0) data."; conf->debug.edns0.t = CONF_BOOL; - conf->debug.edns0.f = FLAG_ADVANCED_SETTING; conf->debug.edns0.d.b = false; conf->debug.edns0.c = validate_stub; // Only type-based checking conf->debug.clients.k = "debug.clients"; conf->debug.clients.h = "Log various important client events such as change of interface (e.g., client switching from WiFi to wired or VPN connection), as well as extensive reporting about how clients were assigned to its groups."; conf->debug.clients.t = CONF_BOOL; - conf->debug.clients.f = FLAG_ADVANCED_SETTING; conf->debug.clients.d.b = false; conf->debug.clients.c = validate_stub; // Only type-based checking conf->debug.aliasclients.k = "debug.aliasclients"; conf->debug.aliasclients.h = "Log information related to alias-client processing."; conf->debug.aliasclients.t = CONF_BOOL; - conf->debug.aliasclients.f = FLAG_ADVANCED_SETTING; conf->debug.aliasclients.d.b = false; conf->debug.aliasclients.c = validate_stub; // Only type-based checking conf->debug.events.k = "debug.events"; conf->debug.events.h = "Log information regarding FTL's embedded event handling queue."; conf->debug.events.t = CONF_BOOL; - conf->debug.events.f = FLAG_ADVANCED_SETTING; conf->debug.events.d.b = false; conf->debug.events.c = validate_stub; // Only type-based checking conf->debug.helper.k = "debug.helper"; conf->debug.helper.h = "Log information about script helpers, e.g., due to dhcp-script."; conf->debug.helper.t = CONF_BOOL; - conf->debug.helper.f = FLAG_ADVANCED_SETTING; conf->debug.helper.d.b = false; conf->debug.helper.c = validate_stub; // Only type-based checking conf->debug.config.k = "debug.config"; conf->debug.config.h = "Print config parsing details"; conf->debug.config.t = CONF_BOOL; - conf->debug.config.f = FLAG_ADVANCED_SETTING; conf->debug.config.d.b = false; conf->debug.config.c = validate_stub; // Only type-based checking conf->debug.inotify.k = "debug.inotify"; conf->debug.inotify.h = "Debug monitoring of /etc/pihole filesystem events"; conf->debug.inotify.t = CONF_BOOL; - conf->debug.inotify.f = FLAG_ADVANCED_SETTING; conf->debug.inotify.d.b = false; conf->debug.inotify.c = validate_stub; // Only type-based checking conf->debug.webserver.k = "debug.webserver"; conf->debug.webserver.h = "Debug monitoring of the webserver (CivetWeb) events"; conf->debug.webserver.t = CONF_BOOL; - conf->debug.webserver.f = FLAG_ADVANCED_SETTING; conf->debug.webserver.d.b = false; conf->debug.webserver.c = validate_stub; // Only type-based checking conf->debug.extra.k = "debug.extra"; conf->debug.extra.h = "Temporary flag that may print additional information. This debug flag is meant to be used whenever needed for temporary investigations. The logged content may change without further notice at any time."; conf->debug.extra.t = CONF_BOOL; - conf->debug.extra.f = FLAG_ADVANCED_SETTING; conf->debug.extra.d.b = false; conf->debug.extra.c = validate_stub; // Only type-based checking conf->debug.reserved.k = "debug.reserved"; conf->debug.reserved.h = "Reserved debug flag"; conf->debug.reserved.t = CONF_BOOL; - conf->debug.reserved.f = FLAG_ADVANCED_SETTING; conf->debug.reserved.d.b = false; conf->debug.reserved.c = validate_stub; // Only type-based checking conf->debug.all.k = "debug.all"; conf->debug.all.h = "Set all debug flags at once. This is a convenience option to enable all debug flags at once. Note that this option is not persistent, setting it to true will enable all *remaining* debug flags but unsetting it will disable *all* debug flags."; conf->debug.all.t = CONF_ALL_DEBUG_BOOL; - conf->debug.all.f = FLAG_ADVANCED_SETTING; conf->debug.all.d.b = false; conf->debug.all.c = validate_stub; // Only type-based checking @@ -1641,7 +1585,6 @@ bool getLogFilePath(void) config.files.log.ftl.h = "The location of FTL's log file"; config.files.log.ftl.a = cJSON_CreateStringReference(""); config.files.log.ftl.t = CONF_STRING; - config.files.log.ftl.f = FLAG_ADVANCED_SETTING; config.files.log.ftl.d.s = (char*)"/var/log/pihole/FTL.log"; config.files.log.ftl.v.s = config.files.log.ftl.d.s; config.files.log.ftl.c = validate_filepath; diff --git a/src/config/config.h b/src/config/config.h index cba4cbdf..1656bcf8 100644 --- a/src/config/config.h +++ b/src/config/config.h @@ -95,12 +95,11 @@ enum conf_type { #define MAX_CONFIG_PATH_DEPTH 6 #define FLAG_RESTART_FTL (1 << 0) -#define FLAG_ADVANCED_SETTING (1 << 1) -#define FLAG_PSEUDO_ITEM (1 << 2) -#define FLAG_INVALIDATE_SESSIONS (1 << 3) -#define FLAG_WRITE_ONLY (1 << 4) -#define FLAG_ENV_VAR (1 << 5) -#define FLAG_CONF_IMPORTED (1 << 6) +#define FLAG_PSEUDO_ITEM (1 << 1) +#define FLAG_INVALIDATE_SESSIONS (1 << 2) +#define FLAG_WRITE_ONLY (1 << 3) +#define FLAG_ENV_VAR (1 << 4) +#define FLAG_CONF_IMPORTED (1 << 5) struct conf_item { const char *k; // item Key From 75b792f589ed0b26549679cd61a7d0ae8562dd4f Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 17 Mar 2024 06:21:24 +0100 Subject: [PATCH 032/339] Update embedded SQLite3 to 3.45.2 Signed-off-by: DL6ER --- src/database/shell.c | 93 ++++++++----- src/database/sqlite3.c | 295 +++++++++++++++++++++++++++++------------ src/database/sqlite3.h | 8 +- 3 files changed, 276 insertions(+), 120 deletions(-) diff --git a/src/database/shell.c b/src/database/shell.c index ead6aa9f..5550f010 100644 --- a/src/database/shell.c +++ b/src/database/shell.c @@ -582,6 +582,9 @@ zSkipValidUtf8(const char *z, int nAccept, long ccm); #ifndef HAVE_CONSOLE_IO_H # include "console_io.h" #endif +#if defined(_MSC_VER) +# pragma warning(disable : 4204) +#endif #ifndef SQLITE_CIO_NO_TRANSLATE # if (defined(_WIN32) || defined(WIN32)) && !SQLITE_OS_WINRT @@ -680,6 +683,10 @@ static short streamOfConsole(FILE *pf, /* out */ PerStreamTags *ppst){ # endif } +# ifndef ENABLE_VIRTUAL_TERMINAL_PROCESSING +# define ENABLE_VIRTUAL_TERMINAL_PROCESSING (0x4) +# endif + # if CIO_WIN_WC_XLATE /* Define console modes for use with the Windows Console API. */ # define SHELL_CONI_MODE \ @@ -1230,6 +1237,10 @@ SQLITE_INTERNAL_LINKAGE char* fGetsUtf8(char *cBuf, int ncMax, FILE *pfIn){ } #endif /* !defined(SQLITE_CIO_NO_TRANSLATE) */ +#if defined(_MSC_VER) +# pragma warning(default : 4204) +#endif + #undef SHELL_INVALID_FILE_PTR /************************* End ../ext/consio/console_io.c ********************/ @@ -20621,6 +20632,7 @@ static void exec_prepared_stmt_columnar( rc = sqlite3_step(pStmt); if( rc!=SQLITE_ROW ) return; nColumn = sqlite3_column_count(pStmt); + if( nColumn==0 ) goto columnar_end; nAlloc = nColumn*4; if( nAlloc<=0 ) nAlloc = 1; azData = sqlite3_malloc64( nAlloc*sizeof(char*) ); @@ -20706,7 +20718,6 @@ static void exec_prepared_stmt_columnar( if( n>p->actualWidth[j] ) p->actualWidth[j] = n; } if( seenInterrupt ) goto columnar_end; - if( nColumn==0 ) goto columnar_end; switch( p->cMode ){ case MODE_Column: { colSep = " "; @@ -25555,16 +25566,15 @@ static int do_meta_command(char *zLine, ShellState *p){ #ifndef SQLITE_SHELL_FIDDLE if( c=='i' && cli_strncmp(azArg[0], "import", n)==0 ){ char *zTable = 0; /* Insert data into this table */ - char *zSchema = 0; /* within this schema (may default to "main") */ + char *zSchema = 0; /* Schema of zTable */ char *zFile = 0; /* Name of file to extra content from */ sqlite3_stmt *pStmt = NULL; /* A statement */ int nCol; /* Number of columns in the table */ - int nByte; /* Number of bytes in an SQL string */ + i64 nByte; /* Number of bytes in an SQL string */ int i, j; /* Loop counters */ int needCommit; /* True to COMMIT or ROLLBACK at end */ int nSep; /* Number of bytes in p->colSeparator[] */ - char *zSql; /* An SQL statement */ - char *zFullTabName; /* Table name with schema if applicable */ + char *zSql = 0; /* An SQL statement */ ImportCtx sCtx; /* Reader context */ char *(SQLITE_CDECL *xRead)(ImportCtx*); /* Func to read one value */ int eVerbose = 0; /* Larger for more console output */ @@ -25698,24 +25708,14 @@ static int do_meta_command(char *zLine, ShellState *p){ while( (nSkip--)>0 ){ while( xRead(&sCtx) && sCtx.cTerm==sCtx.cColSep ){} } - if( zSchema!=0 ){ - zFullTabName = sqlite3_mprintf("\"%w\".\"%w\"", zSchema, zTable); - }else{ - zFullTabName = sqlite3_mprintf("\"%w\"", zTable); - } - zSql = sqlite3_mprintf("SELECT * FROM %s", zFullTabName); - if( zSql==0 || zFullTabName==0 ){ - import_cleanup(&sCtx); - shell_out_of_memory(); - } - nByte = strlen30(zSql); - rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); import_append_char(&sCtx, 0); /* To ensure sCtx.z is allocated */ - if( rc && sqlite3_strglob("no such table: *", sqlite3_errmsg(p->db))==0 ){ + if( sqlite3_table_column_metadata(p->db, zSchema, zTable,0,0,0,0,0,0) ){ + /* Table does not exist. Create it. */ sqlite3 *dbCols = 0; char *zRenames = 0; char *zColDefs; - zCreate = sqlite3_mprintf("CREATE TABLE %s", zFullTabName); + zCreate = sqlite3_mprintf("CREATE TABLE \"%w\".\"%w\"", + zSchema ? zSchema : "main", zTable); while( xRead(&sCtx) ){ zAutoColumn(sCtx.z, &dbCols, 0); if( sCtx.cTerm!=sCtx.cColSep ) break; @@ -25730,34 +25730,50 @@ static int do_meta_command(char *zLine, ShellState *p){ assert(dbCols==0); if( zColDefs==0 ){ eputf("%s: empty file\n", sCtx.zFile); - import_fail: - sqlite3_free(zCreate); - sqlite3_free(zSql); - sqlite3_free(zFullTabName); import_cleanup(&sCtx); rc = 1; goto meta_command_exit; } zCreate = sqlite3_mprintf("%z%z\n", zCreate, zColDefs); + if( zCreate==0 ){ + import_cleanup(&sCtx); + shell_out_of_memory(); + } if( eVerbose>=1 ){ oputf("%s\n", zCreate); } rc = sqlite3_exec(p->db, zCreate, 0, 0, 0); - if( rc ){ - eputf("%s failed:\n%s\n", zCreate, sqlite3_errmsg(p->db)); - goto import_fail; - } sqlite3_free(zCreate); zCreate = 0; - rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + if( rc ){ + eputf("%s failed:\n%s\n", zCreate, sqlite3_errmsg(p->db)); + import_cleanup(&sCtx); + rc = 1; + goto meta_command_exit; + } } + zSql = sqlite3_mprintf("SELECT count(*) FROM pragma_table_info(%Q,%Q);", + zTable, zSchema); + if( zSql==0 ){ + import_cleanup(&sCtx); + shell_out_of_memory(); + } + nByte = strlen(zSql); + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + zSql = 0; if( rc ){ if (pStmt) sqlite3_finalize(pStmt); eputf("Error: %s\n", sqlite3_errmsg(p->db)); - goto import_fail; + import_cleanup(&sCtx); + rc = 1; + goto meta_command_exit; + } + if( sqlite3_step(pStmt)==SQLITE_ROW ){ + nCol = sqlite3_column_int(pStmt, 0); + }else{ + nCol = 0; } - sqlite3_free(zSql); - nCol = sqlite3_column_count(pStmt); sqlite3_finalize(pStmt); pStmt = 0; if( nCol==0 ) return 0; /* no columns, no error */ @@ -25766,7 +25782,12 @@ static int do_meta_command(char *zLine, ShellState *p){ import_cleanup(&sCtx); shell_out_of_memory(); } - sqlite3_snprintf(nByte+20, zSql, "INSERT INTO %s VALUES(?", zFullTabName); + if( zSchema ){ + sqlite3_snprintf(nByte+20, zSql, "INSERT INTO \"%w\".\"%w\" VALUES(?", + zSchema, zTable); + }else{ + sqlite3_snprintf(nByte+20, zSql, "INSERT INTO \"%w\" VALUES(?", zTable); + } j = strlen30(zSql); for(i=1; idb, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + zSql = 0; if( rc ){ eputf("Error: %s\n", sqlite3_errmsg(p->db)); if (pStmt) sqlite3_finalize(pStmt); - goto import_fail; + import_cleanup(&sCtx); + rc = 1; + goto meta_command_exit; } - sqlite3_free(zSql); - sqlite3_free(zFullTabName); needCommit = sqlite3_get_autocommit(p->db); if( needCommit ) sqlite3_exec(p->db, "BEGIN", 0, 0, 0); do{ diff --git a/src/database/sqlite3.c b/src/database/sqlite3.c index a2718dcc..d6c4d244 100644 --- a/src/database/sqlite3.c +++ b/src/database/sqlite3.c @@ -1,6 +1,6 @@ /****************************************************************************** ** This file is an amalgamation of many separate C source files from SQLite -** version 3.45.1. By combining all the individual C code files into this +** version 3.45.2. By combining all the individual C code files into this ** single large file, the entire code can be compiled as a single translation ** unit. This allows many compilers to do optimizations that would not be ** possible if the files were compiled separately. Performance improvements @@ -18,7 +18,7 @@ ** separate file. This file contains only code for the core SQLite library. ** ** The content in this amalgamation comes from Fossil check-in -** e876e51a0ed5c5b3126f52e532044363a014. +** d8cd6d49b46a395b13955387d05e9e1a2a47. */ #define SQLITE_CORE 1 #define SQLITE_AMALGAMATION 1 @@ -459,9 +459,9 @@ extern "C" { ** [sqlite3_libversion_number()], [sqlite3_sourceid()], ** [sqlite_version()] and [sqlite_source_id()]. */ -#define SQLITE_VERSION "3.45.1" -#define SQLITE_VERSION_NUMBER 3045001 -#define SQLITE_SOURCE_ID "2024-01-30 16:01:20 e876e51a0ed5c5b3126f52e532044363a014bc594cfefa87ffb5b82257cc467a" +#define SQLITE_VERSION "3.45.2" +#define SQLITE_VERSION_NUMBER 3045002 +#define SQLITE_SOURCE_ID "2024-03-12 11:06:23 d8cd6d49b46a395b13955387d05e9e1a2a47e54fb99f3c9b59835bbefad6af77" /* ** CAPI3REF: Run-Time Library Version Numbers @@ -733,6 +733,8 @@ typedef int (*sqlite3_callback)(void*,int,char**, char**); ** the 1st parameter to sqlite3_exec() while sqlite3_exec() is running. **
  • The application must not modify the SQL statement text passed into ** the 2nd parameter of sqlite3_exec() while sqlite3_exec() is running. +**
  • The application must not dereference the arrays or string pointers +** passed as the 3rd and 4th callback parameters after it returns. ** */ SQLITE_API int sqlite3_exec( @@ -15097,6 +15099,7 @@ SQLITE_PRIVATE u32 sqlite3TreeTrace; ** 0x00010000 Beginning of DELETE/INSERT/UPDATE processing ** 0x00020000 Transform DISTINCT into GROUP BY ** 0x00040000 SELECT tree dump after all code has been generated +** 0x00080000 NOT NULL strength reduction */ /* @@ -19346,6 +19349,7 @@ struct NameContext { #define NC_InAggFunc 0x020000 /* True if analyzing arguments to an agg func */ #define NC_FromDDL 0x040000 /* SQL text comes from sqlite_schema */ #define NC_NoSelect 0x080000 /* Do not descend into sub-selects */ +#define NC_Where 0x100000 /* Processing WHERE clause of a SELECT */ #define NC_OrderAgg 0x8000000 /* Has an aggregate other than count/min/max */ /* @@ -19369,6 +19373,7 @@ struct Upsert { Expr *pUpsertWhere; /* WHERE clause for the ON CONFLICT UPDATE */ Upsert *pNextUpsert; /* Next ON CONFLICT clause in the list */ u8 isDoUpdate; /* True for DO UPDATE. False for DO NOTHING */ + u8 isDup; /* True if 2nd or later with same pUpsertIdx */ /* Above this point is the parse tree for the ON CONFLICT clauses. ** The next group of fields stores intermediate data. */ void *pToFree; /* Free memory when deleting the Upsert object */ @@ -21444,7 +21449,7 @@ SQLITE_PRIVATE With *sqlite3WithPush(Parse*, With*, u8); SQLITE_PRIVATE Upsert *sqlite3UpsertNew(sqlite3*,ExprList*,Expr*,ExprList*,Expr*,Upsert*); SQLITE_PRIVATE void sqlite3UpsertDelete(sqlite3*,Upsert*); SQLITE_PRIVATE Upsert *sqlite3UpsertDup(sqlite3*,Upsert*); -SQLITE_PRIVATE int sqlite3UpsertAnalyzeTarget(Parse*,SrcList*,Upsert*); +SQLITE_PRIVATE int sqlite3UpsertAnalyzeTarget(Parse*,SrcList*,Upsert*,Upsert*); SQLITE_PRIVATE void sqlite3UpsertDoUpdate(Parse*,Upsert*,Table*,Index*,int); SQLITE_PRIVATE Upsert *sqlite3UpsertOfIndex(Upsert*,Index*); SQLITE_PRIVATE int sqlite3UpsertNextIsIPK(Upsert*); @@ -31309,6 +31314,7 @@ SQLITE_API void sqlite3_str_vappendf( if( xtype==etFLOAT ){ iRound = -precision; }else if( xtype==etGENERIC ){ + if( precision==0 ) precision = 1; iRound = precision; }else{ iRound = precision+1; @@ -35199,6 +35205,9 @@ do_atof_calc: u64 s2; rr[0] = (double)s; s2 = (u64)rr[0]; +#if defined(_MSC_VER) && _MSC_VER<1700 + if( s2==0x8000000000000000LL ){ s2 = 2*(u64)(0.5*rr[0]); } +#endif rr[1] = s>=s2 ? (double)(s - s2) : -(double)(s2 - s); if( e>0 ){ while( e>=100 ){ @@ -35641,7 +35650,7 @@ SQLITE_PRIVATE void sqlite3FpDecode(FpDecode *p, double r, int iRound, int mxRou assert( p->n>0 ); assert( p->nzBuf) ); p->iDP = p->n + exp; - if( iRound<0 ){ + if( iRound<=0 ){ iRound = p->iDP - iRound; if( iRound==0 && p->zBuf[i+1]>='5' ){ iRound = 1; @@ -53262,6 +53271,14 @@ SQLITE_API unsigned char *sqlite3_serialize( pOut = 0; }else{ sz = sqlite3_column_int64(pStmt, 0)*szPage; + if( sz==0 ){ + sqlite3_reset(pStmt); + sqlite3_exec(db, "BEGIN IMMEDIATE; COMMIT;", 0, 0, 0); + rc = sqlite3_step(pStmt); + if( rc==SQLITE_ROW ){ + sz = sqlite3_column_int64(pStmt, 0)*szPage; + } + } if( piSize ) *piSize = sz; if( mFlags & SQLITE_SERIALIZE_NOCOPY ){ pOut = 0; @@ -77088,7 +77105,10 @@ static int fillInCell( n = nHeader + nPayload; testcase( n==3 ); testcase( n==4 ); - if( n<4 ) n = 4; + if( n<4 ){ + n = 4; + pPayload[nPayload] = 0; + } *pnSize = n; assert( nSrc<=nPayload ); testcase( nSrcpBt->nPreformatSize; - if( szNew<4 ) szNew = 4; + if( szNew<4 ){ + szNew = 4; + newCell[3] = 0; + } if( ISAUTOVACUUM(p->pBt) && szNew>pPage->maxLocal ){ CellInfo info; pPage->xParseCell(pPage, newCell, &info); @@ -88379,6 +88402,23 @@ static void serialGet( pMem->flags = IsNaN(x) ? MEM_Null : MEM_Real; } } +static int serialGet7( + const unsigned char *buf, /* Buffer to deserialize from */ + Mem *pMem /* Memory cell to write value into */ +){ + u64 x = FOUR_BYTE_UINT(buf); + u32 y = FOUR_BYTE_UINT(buf+4); + x = (x<<32) + y; + assert( sizeof(x)==8 && sizeof(pMem->u.r)==8 ); + swapMixedEndianFloat(x); + memcpy(&pMem->u.r, &x, sizeof(x)); + if( IsNaN(x) ){ + pMem->flags = MEM_Null; + return 1; + } + pMem->flags = MEM_Real; + return 0; +} SQLITE_PRIVATE void sqlite3VdbeSerialGet( const unsigned char *buf, /* Buffer to deserialize from */ u32 serial_type, /* Serial type to deserialize */ @@ -89058,7 +89098,7 @@ SQLITE_PRIVATE int sqlite3VdbeRecordCompareWithSkip( }else if( serial_type==0 ){ rc = -1; }else if( serial_type==7 ){ - sqlite3VdbeSerialGet(&aKey1[d1], serial_type, &mem1); + serialGet7(&aKey1[d1], &mem1); rc = -sqlite3IntFloatCompare(pRhs->u.i, mem1.u.r); }else{ i64 lhs = vdbeRecordDecodeInt(serial_type, &aKey1[d1]); @@ -89083,14 +89123,18 @@ SQLITE_PRIVATE int sqlite3VdbeRecordCompareWithSkip( }else if( serial_type==0 ){ rc = -1; }else{ - sqlite3VdbeSerialGet(&aKey1[d1], serial_type, &mem1); if( serial_type==7 ){ - if( mem1.u.ru.r ){ + if( serialGet7(&aKey1[d1], &mem1) ){ + rc = -1; /* mem1 is a NaN */ + }else if( mem1.u.ru.r ){ rc = -1; }else if( mem1.u.r>pRhs->u.r ){ rc = +1; + }else{ + assert( rc==0 ); } }else{ + sqlite3VdbeSerialGet(&aKey1[d1], serial_type, &mem1); rc = sqlite3IntFloatCompare(mem1.u.i, pRhs->u.r); } } @@ -89160,7 +89204,14 @@ SQLITE_PRIVATE int sqlite3VdbeRecordCompareWithSkip( /* RHS is null */ else{ serial_type = aKey1[idx1]; - rc = (serial_type!=0 && serial_type!=10); + if( serial_type==0 + || serial_type==10 + || (serial_type==7 && serialGet7(&aKey1[d1], &mem1)!=0) + ){ + assert( rc==0 ); + }else{ + rc = 1; + } } if( rc!=0 ){ @@ -94858,7 +94909,9 @@ case OP_Ge: { /* same as TK_GE, jump, in1, in3 */ } } }else if( affinity==SQLITE_AFF_TEXT && ((flags1 | flags3) & MEM_Str)!=0 ){ - if( (flags1 & MEM_Str)==0 && (flags1&(MEM_Int|MEM_Real|MEM_IntReal))!=0 ){ + if( (flags1 & MEM_Str)!=0 ){ + pIn1->flags &= ~(MEM_Int|MEM_Real|MEM_IntReal); + }else if( (flags1&(MEM_Int|MEM_Real|MEM_IntReal))!=0 ){ testcase( pIn1->flags & MEM_Int ); testcase( pIn1->flags & MEM_Real ); testcase( pIn1->flags & MEM_IntReal ); @@ -94867,7 +94920,9 @@ case OP_Ge: { /* same as TK_GE, jump, in1, in3 */ flags1 = (pIn1->flags & ~MEM_TypeMask) | (flags1 & MEM_TypeMask); if( NEVER(pIn1==pIn3) ) flags3 = flags1 | MEM_Str; } - if( (flags3 & MEM_Str)==0 && (flags3&(MEM_Int|MEM_Real|MEM_IntReal))!=0 ){ + if( (flags3 & MEM_Str)!=0 ){ + pIn3->flags &= ~(MEM_Int|MEM_Real|MEM_IntReal); + }else if( (flags3&(MEM_Int|MEM_Real|MEM_IntReal))!=0 ){ testcase( pIn3->flags & MEM_Int ); testcase( pIn3->flags & MEM_Real ); testcase( pIn3->flags & MEM_IntReal ); @@ -106212,6 +106267,8 @@ static void resolveAlias( assert( iCol>=0 && iColnExpr ); pOrig = pEList->a[iCol].pExpr; assert( pOrig!=0 ); + assert( !ExprHasProperty(pExpr, EP_Reduced|EP_TokenOnly) ); + if( pExpr->pAggInfo ) return; db = pParse->db; pDup = sqlite3ExprDup(db, pOrig, 0); if( db->mallocFailed ){ @@ -107097,6 +107154,19 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ ** resolved. This prevents "column" from being counted as having been ** referenced, which might prevent a SELECT from being erroneously ** marked as correlated. + ** + ** 2024-03-28: Beware of aggregates. A bare column of aggregated table + ** can still evaluate to NULL even though it is marked as NOT NULL. + ** Example: + ** + ** CREATE TABLE t1(a INT NOT NULL); + ** SELECT a, a IS NULL, a IS NOT NULL, count(*) FROM t1; + ** + ** The "a IS NULL" and "a IS NOT NULL" expressions cannot be optimized + ** here because at the time this case is hit, we do not yet know whether + ** or not t1 is being aggregated. We have to assume the worst and omit + ** the optimization. The only time it is safe to apply this optimization + ** is within the WHERE clause. */ case TK_NOTNULL: case TK_ISNULL: { @@ -107107,19 +107177,36 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ anRef[i] = p->nRef; } sqlite3WalkExpr(pWalker, pExpr->pLeft); - if( 0==sqlite3ExprCanBeNull(pExpr->pLeft) && !IN_RENAME_OBJECT ){ - testcase( ExprHasProperty(pExpr, EP_OuterON) ); - assert( !ExprHasProperty(pExpr, EP_IntValue) ); - pExpr->u.iValue = (pExpr->op==TK_NOTNULL); - pExpr->flags |= EP_IntValue; - pExpr->op = TK_INTEGER; - - for(i=0, p=pNC; p && ipNext, i++){ - p->nRef = anRef[i]; - } - sqlite3ExprDelete(pParse->db, pExpr->pLeft); - pExpr->pLeft = 0; + if( IN_RENAME_OBJECT ) return WRC_Prune; + if( sqlite3ExprCanBeNull(pExpr->pLeft) ){ + /* The expression can be NULL. So the optimization does not apply */ + return WRC_Prune; } + + for(i=0, p=pNC; p; p=p->pNext, i++){ + if( (p->ncFlags & NC_Where)==0 ){ + return WRC_Prune; /* Not in a WHERE clause. Unsafe to optimize. */ + } + } + testcase( ExprHasProperty(pExpr, EP_OuterON) ); + assert( !ExprHasProperty(pExpr, EP_IntValue) ); +#if TREETRACE_ENABLED + if( sqlite3TreeTrace & 0x80000 ){ + sqlite3DebugPrintf( + "NOT NULL strength reduction converts the following to %d:\n", + pExpr->op==TK_NOTNULL + ); + sqlite3ShowExpr(pExpr); + } +#endif /* TREETRACE_ENABLED */ + pExpr->u.iValue = (pExpr->op==TK_NOTNULL); + pExpr->flags |= EP_IntValue; + pExpr->op = TK_INTEGER; + for(i=0, p=pNC; p && ipNext, i++){ + p->nRef = anRef[i]; + } + sqlite3ExprDelete(pParse->db, pExpr->pLeft); + pExpr->pLeft = 0; return WRC_Prune; } @@ -108019,7 +108106,9 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ } if( sqlite3ResolveExprNames(&sNC, p->pHaving) ) return WRC_Abort; } + sNC.ncFlags |= NC_Where; if( sqlite3ResolveExprNames(&sNC, p->pWhere) ) return WRC_Abort; + sNC.ncFlags &= ~NC_Where; /* Resolve names in table-valued-function arguments */ for(i=0; ipSrc->nSrc; i++){ @@ -128947,13 +129036,13 @@ SQLITE_PRIVATE void sqlite3QuoteValue(StrAccum *pStr, sqlite3_value *pValue){ double r1, r2; const char *zVal; r1 = sqlite3_value_double(pValue); - sqlite3_str_appendf(pStr, "%!.15g", r1); + sqlite3_str_appendf(pStr, "%!0.15g", r1); zVal = sqlite3_str_value(pStr); if( zVal ){ sqlite3AtoF(zVal, &r2, pStr->nChar, SQLITE_UTF8); if( r1!=r2 ){ sqlite3_str_reset(pStr); - sqlite3_str_appendf(pStr, "%!.20e", r1); + sqlite3_str_appendf(pStr, "%!0.20e", r1); } } break; @@ -129255,7 +129344,7 @@ static void replaceFunc( } if( zPattern[0]==0 ){ assert( sqlite3_value_type(argv[1])!=SQLITE_NULL ); - sqlite3_result_value(context, argv[0]); + sqlite3_result_text(context, (const char*)zStr, nStr, SQLITE_TRANSIENT); return; } nPattern = sqlite3_value_bytes(argv[1]); @@ -133175,7 +133264,7 @@ SQLITE_PRIVATE void sqlite3Insert( pNx->iDataCur = iDataCur; pNx->iIdxCur = iIdxCur; if( pNx->pUpsertTarget ){ - if( sqlite3UpsertAnalyzeTarget(pParse, pTabList, pNx) ){ + if( sqlite3UpsertAnalyzeTarget(pParse, pTabList, pNx, pUpsert) ){ goto insert_cleanup; } } @@ -139474,31 +139563,7 @@ SQLITE_PRIVATE void sqlite3Pragma( int mxCol; /* Maximum non-virtual column number */ if( pObjTab && pObjTab!=pTab ) continue; - if( !IsOrdinaryTable(pTab) ){ -#ifndef SQLITE_OMIT_VIRTUALTABLE - sqlite3_vtab *pVTab; - int a1; - if( !IsVirtual(pTab) ) continue; - if( pTab->nCol<=0 ){ - const char *zMod = pTab->u.vtab.azArg[0]; - if( sqlite3HashFind(&db->aModule, zMod)==0 ) continue; - } - sqlite3ViewGetColumnNames(pParse, pTab); - if( pTab->u.vtab.p==0 ) continue; - pVTab = pTab->u.vtab.p->pVtab; - if( NEVER(pVTab==0) ) continue; - if( NEVER(pVTab->pModule==0) ) continue; - if( pVTab->pModule->iVersion<4 ) continue; - if( pVTab->pModule->xIntegrity==0 ) continue; - sqlite3VdbeAddOp3(v, OP_VCheck, i, 3, isQuick); - pTab->nTabRef++; - sqlite3VdbeAppendP4(v, pTab, P4_TABLEREF); - a1 = sqlite3VdbeAddOp1(v, OP_IsNull, 3); VdbeCoverage(v); - integrityCheckResultRow(v); - sqlite3VdbeJumpHere(v, a1); -#endif - continue; - } + if( !IsOrdinaryTable(pTab) ) continue; if( isQuick || HasRowid(pTab) ){ pPk = 0; r2 = 0; @@ -139633,6 +139698,7 @@ SQLITE_PRIVATE void sqlite3Pragma( ** is REAL, we have to load the actual data using OP_Column ** to reliably determine if the value is a NULL. */ sqlite3VdbeAddOp3(v, OP_Column, p1, p3, 3); + sqlite3ColumnDefault(v, pTab, j, 3); jmp3 = sqlite3VdbeAddOp2(v, OP_NotNull, 3, labelOk); VdbeCoverage(v); } @@ -139823,6 +139889,38 @@ SQLITE_PRIVATE void sqlite3Pragma( } } } + +#ifndef SQLITE_OMIT_VIRTUALTABLE + /* Second pass to invoke the xIntegrity method on all virtual + ** tables. + */ + for(x=sqliteHashFirst(pTbls); x; x=sqliteHashNext(x)){ + Table *pTab = sqliteHashData(x); + sqlite3_vtab *pVTab; + int a1; + if( pObjTab && pObjTab!=pTab ) continue; + if( IsOrdinaryTable(pTab) ) continue; + if( !IsVirtual(pTab) ) continue; + if( pTab->nCol<=0 ){ + const char *zMod = pTab->u.vtab.azArg[0]; + if( sqlite3HashFind(&db->aModule, zMod)==0 ) continue; + } + sqlite3ViewGetColumnNames(pParse, pTab); + if( pTab->u.vtab.p==0 ) continue; + pVTab = pTab->u.vtab.p->pVtab; + if( NEVER(pVTab==0) ) continue; + if( NEVER(pVTab->pModule==0) ) continue; + if( pVTab->pModule->iVersion<4 ) continue; + if( pVTab->pModule->xIntegrity==0 ) continue; + sqlite3VdbeAddOp3(v, OP_VCheck, i, 3, isQuick); + pTab->nTabRef++; + sqlite3VdbeAppendP4(v, pTab, P4_TABLEREF); + a1 = sqlite3VdbeAddOp1(v, OP_IsNull, 3); VdbeCoverage(v); + integrityCheckResultRow(v); + sqlite3VdbeJumpHere(v, a1); + continue; + } +#endif } { static const int iLn = VDBE_OFFSET_LINENO(2); @@ -153460,7 +153558,8 @@ SQLITE_PRIVATE Upsert *sqlite3UpsertNew( SQLITE_PRIVATE int sqlite3UpsertAnalyzeTarget( Parse *pParse, /* The parsing context */ SrcList *pTabList, /* Table into which we are inserting */ - Upsert *pUpsert /* The ON CONFLICT clauses */ + Upsert *pUpsert, /* The ON CONFLICT clauses */ + Upsert *pAll /* Complete list of all ON CONFLICT clauses */ ){ Table *pTab; /* That table into which we are inserting */ int rc; /* Result code */ @@ -153563,6 +153662,14 @@ SQLITE_PRIVATE int sqlite3UpsertAnalyzeTarget( continue; } pUpsert->pUpsertIdx = pIdx; + if( sqlite3UpsertOfIndex(pAll,pIdx)!=pUpsert ){ + /* Really this should be an error. The isDup ON CONFLICT clause will + ** never fire. But this problem was not discovered until three years + ** after multi-CONFLICT upsert was added, and so we silently ignore + ** the problem to prevent breaking applications that might actually + ** have redundant ON CONFLICT clauses. */ + pUpsert->isDup = 1; + } break; } if( pUpsert->pUpsertIdx==0 ){ @@ -153589,9 +153696,13 @@ SQLITE_PRIVATE int sqlite3UpsertNextIsIPK(Upsert *pUpsert){ Upsert *pNext; if( NEVER(pUpsert==0) ) return 0; pNext = pUpsert->pNextUpsert; - if( pNext==0 ) return 1; - if( pNext->pUpsertTarget==0 ) return 1; - if( pNext->pUpsertIdx==0 ) return 1; + while( 1 /*exit-by-return*/ ){ + if( pNext==0 ) return 1; + if( pNext->pUpsertTarget==0 ) return 1; + if( pNext->pUpsertIdx==0 ) return 1; + if( !pNext->isDup ) return 0; + pNext = pNext->pNextUpsert; + } return 0; } @@ -204783,6 +204894,7 @@ json_parse_restart: case '[': { /* Parse array */ iThis = pParse->nBlob; + assert( i<=(u32)pParse->nJson ); jsonBlobAppendNode(pParse, JSONB_ARRAY, pParse->nJson - i, 0); iStart = pParse->nBlob; if( pParse->oom ) return -1; @@ -205181,6 +205293,10 @@ static void jsonReturnStringAsBlob(JsonString *pStr){ JsonParse px; memset(&px, 0, sizeof(px)); jsonStringTerminate(pStr); + if( pStr->eErr ){ + sqlite3_result_error_nomem(pStr->pCtx); + return; + } px.zJson = pStr->zBuf; px.nJson = pStr->nUsed; px.db = sqlite3_context_db_handle(pStr->pCtx); @@ -206506,8 +206622,9 @@ rebuild_from_cache: } p->zJson = (char*)sqlite3_value_text(pArg); p->nJson = sqlite3_value_bytes(pArg); + if( db->mallocFailed ) goto json_pfa_oom; if( p->nJson==0 ) goto json_pfa_malformed; - if( NEVER(p->zJson==0) ) goto json_pfa_oom; + assert( p->zJson!=0 ); if( jsonConvertTextToBlob(p, (flgs & JSON_KEEPERROR) ? 0 : ctx) ){ if( flgs & JSON_KEEPERROR ){ p->nErr = 1; @@ -206673,10 +206790,10 @@ static void jsonDebugPrintBlob( if( sz==0 && x<=JSONB_FALSE ){ sqlite3_str_append(pOut, "\n", 1); }else{ - u32 i; + u32 j; sqlite3_str_appendall(pOut, ": \""); - for(i=iStart+n; iaBlob[i]; + for(j=iStart+n; jaBlob[j]; if( c<0x20 || c>=0x7f ) c = '.'; sqlite3_str_append(pOut, (char*)&c, 1); } @@ -208084,6 +208201,9 @@ static int jsonEachColumn( case JEACH_VALUE: { u32 i = jsonSkipLabel(p); jsonReturnFromBlob(&p->sParse, i, ctx, 1); + if( (p->sParse.aBlob[i] & 0x0f)>=JSONB_ARRAY ){ + sqlite3_result_subtype(ctx, JSON_SUBTYPE); + } break; } case JEACH_TYPE: { @@ -208130,9 +208250,9 @@ static int jsonEachColumn( case JEACH_JSON: { if( p->sParse.zJson==0 ){ sqlite3_result_blob(ctx, p->sParse.aBlob, p->sParse.nBlob, - SQLITE_STATIC); + SQLITE_TRANSIENT); }else{ - sqlite3_result_text(ctx, p->sParse.zJson, -1, SQLITE_STATIC); + sqlite3_result_text(ctx, p->sParse.zJson, -1, SQLITE_TRANSIENT); } break; } @@ -209158,11 +209278,9 @@ static RtreeNode *nodeNew(Rtree *pRtree, RtreeNode *pParent){ ** Clear the Rtree.pNodeBlob object */ static void nodeBlobReset(Rtree *pRtree){ - if( pRtree->pNodeBlob && pRtree->inWrTrans==0 && pRtree->nCursor==0 ){ - sqlite3_blob *pBlob = pRtree->pNodeBlob; - pRtree->pNodeBlob = 0; - sqlite3_blob_close(pBlob); - } + sqlite3_blob *pBlob = pRtree->pNodeBlob; + pRtree->pNodeBlob = 0; + sqlite3_blob_close(pBlob); } /* @@ -209206,7 +209324,6 @@ static int nodeAcquire( &pRtree->pNodeBlob); } if( rc ){ - nodeBlobReset(pRtree); *ppNode = 0; /* If unable to open an sqlite3_blob on the desired row, that can only ** be because the shadow tables hold erroneous data. */ @@ -209266,6 +209383,7 @@ static int nodeAcquire( } *ppNode = pNode; }else{ + nodeBlobReset(pRtree); if( pNode ){ pRtree->nNodeRef--; sqlite3_free(pNode); @@ -209410,6 +209528,7 @@ static void nodeGetCoord( int iCoord, /* Which coordinate to extract */ RtreeCoord *pCoord /* OUT: Space to write result to */ ){ + assert( iCellzData[12 + pRtree->nBytesPerCell*iCell + 4*iCoord], pCoord); } @@ -209599,7 +209718,9 @@ static int rtreeClose(sqlite3_vtab_cursor *cur){ sqlite3_finalize(pCsr->pReadAux); sqlite3_free(pCsr); pRtree->nCursor--; - nodeBlobReset(pRtree); + if( pRtree->nCursor==0 && pRtree->inWrTrans==0 ){ + nodeBlobReset(pRtree); + } return SQLITE_OK; } @@ -210184,7 +210305,11 @@ static int rtreeRowid(sqlite3_vtab_cursor *pVtabCursor, sqlite_int64 *pRowid){ int rc = SQLITE_OK; RtreeNode *pNode = rtreeNodeOfFirstSearchPoint(pCsr, &rc); if( rc==SQLITE_OK && ALWAYS(p) ){ - *pRowid = nodeGetRowid(RTREE_OF_CURSOR(pCsr), pNode, p->iCell); + if( p->iCell>=NCELL(pNode) ){ + rc = SQLITE_ABORT; + }else{ + *pRowid = nodeGetRowid(RTREE_OF_CURSOR(pCsr), pNode, p->iCell); + } } return rc; } @@ -210202,6 +210327,7 @@ static int rtreeColumn(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int i){ if( rc ) return rc; if( NEVER(p==0) ) return SQLITE_OK; + if( p->iCell>=NCELL(pNode) ) return SQLITE_ABORT; if( i==0 ){ sqlite3_result_int64(ctx, nodeGetRowid(pRtree, pNode, p->iCell)); }else if( i<=pRtree->nDim2 ){ @@ -211683,8 +211809,7 @@ constraint: */ static int rtreeBeginTransaction(sqlite3_vtab *pVtab){ Rtree *pRtree = (Rtree *)pVtab; - assert( pRtree->inWrTrans==0 ); - pRtree->inWrTrans++; + pRtree->inWrTrans = 1; return SQLITE_OK; } @@ -211698,6 +211823,9 @@ static int rtreeEndTransaction(sqlite3_vtab *pVtab){ nodeBlobReset(pRtree); return SQLITE_OK; } +static int rtreeRollback(sqlite3_vtab *pVtab){ + return rtreeEndTransaction(pVtab); +} /* ** The xRename method for rtree module virtual tables. @@ -211816,7 +211944,7 @@ static sqlite3_module rtreeModule = { rtreeBeginTransaction, /* xBegin - begin transaction */ rtreeEndTransaction, /* xSync - sync transaction */ rtreeEndTransaction, /* xCommit - commit transaction */ - rtreeEndTransaction, /* xRollback - rollback transaction */ + rtreeRollback, /* xRollback - rollback transaction */ 0, /* xFindFunction - function overloading */ rtreeRename, /* xRename - rename the table */ rtreeSavepoint, /* xSavepoint */ @@ -245375,23 +245503,26 @@ static void fts5IterSetOutputsTokendata(Fts5Iter *pIter){ static void fts5TokendataIterNext(Fts5Iter *pIter, int bFrom, i64 iFrom){ int ii; Fts5TokenDataIter *pT = pIter->pTokenDataIter; + Fts5Index *pIndex = pIter->pIndex; for(ii=0; iinIter; ii++){ Fts5Iter *p = pT->apIter[ii]; if( p->base.bEof==0 && (p->base.iRowid==pIter->base.iRowid || (bFrom && p->base.iRowidpIndex, p, bFrom, iFrom); + fts5MultiIterNext(pIndex, p, bFrom, iFrom); while( bFrom && p->base.bEof==0 && p->base.iRowidpIndex->rc==SQLITE_OK + && pIndex->rc==SQLITE_OK ){ - fts5MultiIterNext(p->pIndex, p, 0, 0); + fts5MultiIterNext(pIndex, p, 0, 0); } } } - fts5IterSetOutputsTokendata(pIter); + if( pIndex->rc==SQLITE_OK ){ + fts5IterSetOutputsTokendata(pIter); + } } /* @@ -250545,7 +250676,7 @@ static void fts5SourceIdFunc( ){ assert( nArg==0 ); UNUSED_PARAM2(nArg, apUnused); - sqlite3_result_text(pCtx, "fts5: 2024-01-30 16:01:20 e876e51a0ed5c5b3126f52e532044363a014bc594cfefa87ffb5b82257cc467a", -1, SQLITE_TRANSIENT); + sqlite3_result_text(pCtx, "fts5: 2024-03-12 11:06:23 d8cd6d49b46a395b13955387d05e9e1a2a47e54fb99f3c9b59835bbefad6af77", -1, SQLITE_TRANSIENT); } /* diff --git a/src/database/sqlite3.h b/src/database/sqlite3.h index 4fdfde00..c9fc77fb 100644 --- a/src/database/sqlite3.h +++ b/src/database/sqlite3.h @@ -146,9 +146,9 @@ extern "C" { ** [sqlite3_libversion_number()], [sqlite3_sourceid()], ** [sqlite_version()] and [sqlite_source_id()]. */ -#define SQLITE_VERSION "3.45.1" -#define SQLITE_VERSION_NUMBER 3045001 -#define SQLITE_SOURCE_ID "2024-01-30 16:01:20 e876e51a0ed5c5b3126f52e532044363a014bc594cfefa87ffb5b82257cc467a" +#define SQLITE_VERSION "3.45.2" +#define SQLITE_VERSION_NUMBER 3045002 +#define SQLITE_SOURCE_ID "2024-03-12 11:06:23 d8cd6d49b46a395b13955387d05e9e1a2a47e54fb99f3c9b59835bbefad6af77" /* ** CAPI3REF: Run-Time Library Version Numbers @@ -420,6 +420,8 @@ typedef int (*sqlite3_callback)(void*,int,char**, char**); ** the 1st parameter to sqlite3_exec() while sqlite3_exec() is running. **
  • The application must not modify the SQL statement text passed into ** the 2nd parameter of sqlite3_exec() while sqlite3_exec() is running. +**
  • The application must not dereference the arrays or string pointers +** passed as the 3rd and 4th callback parameters after it returns. ** */ SQLITE_API int sqlite3_exec( From 05867e20f26d2733a83f59b7873304594bdf7b99 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 24 Mar 2024 08:55:12 +0100 Subject: [PATCH 033/339] Ensure cJSON is used in a thread-safe manner and add CI tests ensuring this. Also ensure every JSON parsing is doing error checking and reduce some code duplication. No functional change. Signed-off-by: DL6ER --- src/api/auth.c | 16 ++------ src/api/config.c | 16 ++------ src/api/dns.c | 17 ++------- src/api/list.c | 38 ++++++------------- src/api/search.c | 18 +++------ src/api/teleporter.c | 10 ++++- src/config/cli.c | 5 ++- src/database/gravity-db.h | 6 +-- src/database/message-table.c | 2 + src/dnsmasq/config.h | 2 +- src/enums.h | 17 +++++++++ src/webserver/http-common.c | 71 ++++++++++++++++++++++++++++++++---- src/webserver/http-common.h | 20 ++-------- src/webserver/json_macros.h | 6 +-- test/test_suite.bats | 11 +++++- 15 files changed, 142 insertions(+), 113 deletions(-) diff --git a/src/api/auth.c b/src/api/auth.c index 714ca9d5..3d3ec73a 100644 --- a/src/api/auth.c +++ b/src/api/auth.c @@ -461,19 +461,9 @@ int api_auth(struct ftl_conn *api) if(api->method == HTTP_POST) { // Try to extract response from payload - if (api->payload.json == NULL) - { - if (api->payload.json_error == NULL) - return send_json_error(api, 400, - "bad_request", - "No request body data", - NULL); - else - return send_json_error(api, 400, - "bad_request", - "Invalid request body data (no valid JSON), error before hint", - api->payload.json_error); - } + const int ret = check_json_payload(api); + if(ret != 0) + return ret; // Check if password is available cJSON *json_password; diff --git a/src/api/config.c b/src/api/config.c index 3d3563e9..c998823b 100644 --- a/src/api/config.c +++ b/src/api/config.c @@ -656,19 +656,9 @@ static int api_config_get(struct ftl_conn *api) static int api_config_patch(struct ftl_conn *api) { // Is there a payload with valid JSON data? - if (api->payload.json == NULL) - { - if (api->payload.json_error == NULL) - return send_json_error(api, 400, - "bad_request", - "No request body data", - NULL); - else - return send_json_error(api, 400, - "bad_request", - "Invalid request body data (no valid JSON), error before hint", - api->payload.json_error); - } + const int ret = check_json_payload(api); + if(ret != 0) + return ret; // Is there a "config" object at the root of the received JSON payload? cJSON *conf = cJSON_GetObjectItem(api->payload.json, "config"); diff --git a/src/api/dns.c b/src/api/dns.c index 525c727e..10e90605 100644 --- a/src/api/dns.c +++ b/src/api/dns.c @@ -74,19 +74,10 @@ static int set_blocking(struct ftl_conn *api) NULL); } - if (api->payload.json == NULL) - { - if (api->payload.json_error == NULL) - return send_json_error(api, 400, - "bad_request", - "No request body data", - NULL); - else - return send_json_error(api, 400, - "bad_request", - "Invalid request body data (no valid JSON), error before hint", - api->payload.json_error); - } + // Check if the payload is valid JSON + const int ret = check_json_payload(api); + if(ret != 0) + return ret; cJSON *elem = cJSON_GetObjectItemCaseSensitive(api->payload.json, "blocking"); if (!cJSON_IsBool(elem)) diff --git a/src/api/list.c b/src/api/list.c index 56d4e009..e6fcc685 100644 --- a/src/api/list.c +++ b/src/api/list.c @@ -19,6 +19,8 @@ #include "database/network-table.h" // valid_domain() #include "tools/gravity-parseList.h" +// parse_groupIDs() +#include "webserver/http-common.h" #include static int api_list_read(struct ftl_conn *api, @@ -96,19 +98,13 @@ static int api_list_read(struct ftl_conn *api, { if(table.group_ids != NULL) { - // Black magic at work here: We build a JSON array from - // the group_concat result delivered from the database, - // parse it as valid array and append it as row to the - // data - const size_t buflen = strlen(table.group_ids)+3u; - char *group_ids_str = calloc(buflen, sizeof(char)); - group_ids_str[0] = '['; - strcpy(group_ids_str+1u , table.group_ids); - group_ids_str[buflen-2u] = ']'; - group_ids_str[buflen-1u] = '\0'; - cJSON * group_ids = cJSON_Parse(group_ids_str); - free(group_ids_str); - JSON_ADD_ITEM_TO_OBJECT(row, "groups", group_ids); + const int ret = parse_groupIDs(api, &table, row); + if(ret != 0) + { + JSON_DELETE(rows); + return ret; + } + } else { @@ -184,19 +180,9 @@ static int api_list_write(struct ftl_conn *api, tablerow row = { 0 }; // Check if valid JSON payload is available - if (api->payload.json == NULL) - { - if (api->payload.json_error == NULL) - return send_json_error(api, 400, - "bad_request", - "No request body data", - NULL); - else - return send_json_error(api, 400, - "bad_request", - "Invalid request body data (no valid JSON), error before hint", - api->payload.json_error); - } + const int json_ret = check_json_payload(api); + if(json_ret != 0) + return json_ret; bool spaces_allowed = false; bool allocated_json = false; diff --git a/src/api/search.c b/src/api/search.c index 7358a612..09098351 100644 --- a/src/api/search.c +++ b/src/api/search.c @@ -15,6 +15,8 @@ #include "database/gravity-db.h" // match_regex() #include "regex_r.h" +// parse_groupIDs() +#include "webserver/http-common.h" #include #define MAX_SEARCH_RESULTS 10000u @@ -77,19 +79,9 @@ static int search_table(struct ftl_conn *api, const char *item, if(table.group_ids != NULL) { - // Black magic at work here: We build a JSON array from - // the group_concat result delivered from the database, - // parse it as valid array and append it as row to the - // data - const size_t buflen = strlen(table.group_ids)+3u; - char *group_ids_str = calloc(buflen, sizeof(char)); - group_ids_str[0] = '['; - strcpy(group_ids_str+1u , table.group_ids); - group_ids_str[buflen-2u] = ']'; - group_ids_str[buflen-1u] = '\0'; - cJSON * group_ids = cJSON_Parse(group_ids_str); - free(group_ids_str); - JSON_ADD_ITEM_TO_OBJECT(row, "groups", group_ids); + const int ret = parse_groupIDs(api, &table, row); + if(ret != 0) + return ret; } else { diff --git a/src/api/teleporter.c b/src/api/teleporter.c index 12eb2ad1..a724d7d3 100644 --- a/src/api/teleporter.c +++ b/src/api/teleporter.c @@ -637,9 +637,17 @@ static int process_received_tar_gz(struct ftl_conn *api, struct upload_data *dat size_t fileSize = 0u; cJSON *json = NULL; const char *file = find_file_in_tar(archive, archive_size, teleporter_v5_files[i].filename, &fileSize); - if(file != NULL && fileSize > 0u && (json = cJSON_ParseWithLength(file, fileSize)) != NULL) + const char *json_error = NULL; + if(file != NULL && fileSize > 0u && (json = cJSON_ParseWithLengthOpts(file, fileSize, &json_error, false)) != NULL) + { if(import_json_table(json, &teleporter_v5_files[i])) JSON_COPY_STR_TO_ARRAY(imported_files, teleporter_v5_files[i].filename); + } + else if(json_error != NULL) + { + log_err("Unable to parse JSON file \"%s\", error at: %s", + teleporter_v5_files[i].filename, json_error); + } } // Temporarily write further files to to disk so we can import them on restart diff --git a/src/config/cli.c b/src/config/cli.c index d012d0d8..292f6955 100644 --- a/src/config/cli.c +++ b/src/config/cli.c @@ -342,10 +342,11 @@ static bool readStringValue(struct conf_item *conf_item, const char *value, stru } case CONF_JSON_STRING_ARRAY: { - cJSON *elem = cJSON_Parse(value); + const char *json_error = NULL; + cJSON *elem = cJSON_ParseWithOpts(value, &json_error, 0); if(elem == NULL) { - log_err("Config setting %s is invalid: not valid JSON, error before: %s", conf_item->k, cJSON_GetErrorPtr()); + log_err("Config setting %s is invalid: not valid JSON, error at: %s", conf_item->k, json_error); return false; } if(!cJSON_IsArray(elem)) diff --git a/src/database/gravity-db.h b/src/database/gravity-db.h index a77deb72..6ce75fd5 100644 --- a/src/database/gravity-db.h +++ b/src/database/gravity-db.h @@ -11,11 +11,9 @@ #define GRAVITY_H // clients data structure -#include "../datastructure.h" -// enum http_method -#include "../webserver/http-common.h" +#include "datastructure.h" // Definition of struct regexData -#include "../regex_r.h" +#include "regex_r.h" // Table row record, not all fields are used by all tables typedef struct { diff --git a/src/database/message-table.c b/src/database/message-table.c index 55c36a24..d7ca0a31 100644 --- a/src/database/message-table.c +++ b/src/database/message-table.c @@ -29,6 +29,8 @@ #include "files.h" // get_memdb() #include "database/query-table.h" +// escape_html() +#include "webserver/http-common.h" static const char *get_message_type_str(const enum message_type type) { diff --git a/src/dnsmasq/config.h b/src/dnsmasq/config.h index 144468a3..b04f964f 100644 --- a/src/dnsmasq/config.h +++ b/src/dnsmasq/config.h @@ -31,7 +31,7 @@ #define FORWARD_TEST 1000 /* try all servers every 1000 queries */ #define FORWARD_TIME 600 /* or 10 minutes */ #define UDP_TEST_TIME 60 /* How often to reset our idea of max packet size. */ -#define SERVERS_LOGGED 30 /* Only log this many servers when logging state */ +#define SERVERS_LOGGED 300 /* Only log this many servers when logging state */ #define LOCALS_LOGGED 8 /* Only log this many local addresses when logging state */ #define LEASE_RETRY 60 /* on error, retry writing leasefile after LEASE_RETRY seconds */ #define CACHESIZ 150 /* default cache size */ diff --git a/src/enums.h b/src/enums.h index bc80472b..dac8ff8d 100644 --- a/src/enums.h +++ b/src/enums.h @@ -323,4 +323,21 @@ enum cert_check { CERT_OKAY } __attribute__ ((packed)); +enum http_method { + HTTP_UNKNOWN = 0, + HTTP_GET = 1 << 0, + HTTP_POST = 1 << 1, + HTTP_PUT = 1 << 2, + HTTP_PATCH = 1 << 3, + HTTP_DELETE = 1 << 4, + HTTP_OPTIONS = 1 << 5, +}; + +enum api_flags { + API_FLAG_NONE = 0, + API_DOMAINS = 1 << 0, + API_PARSE_JSON = 1 << 1, + API_BATCHDELETE = 1 << 2, +}; + #endif // ENUMS_H diff --git a/src/webserver/http-common.c b/src/webserver/http-common.c index a8f9c193..dab68020 100644 --- a/src/webserver/http-common.c +++ b/src/webserver/http-common.c @@ -8,11 +8,11 @@ * This file is copyright under the latest version of the EUPL. * Please see LICENSE file for your rights under this license. */ -#include "../FTL.h" -#include "http-common.h" -#include "../config/config.h" -#include "../log.h" -#include "json_macros.h" +#include "FTL.h" +#include "webserver/http-common.h" +#include "config/config.h" +#include "log.h" +#include "webserver/json_macros.h" // UINT_MAX #include // HUGE_VAL @@ -515,8 +515,7 @@ void read_and_parse_payload(struct ftl_conn *api) api->payload.avail = true; // Try to parse possibly existing JSON payload - api->payload.json = cJSON_Parse(api->payload.raw); - api->payload.json_error = cJSON_GetErrorPtr(); + api->payload.json = cJSON_ParseWithOpts(api->payload.raw, &api->payload.json_error, 0); } // Escape a string to mask HTML special characters, the resulting string is @@ -569,3 +568,61 @@ char *__attribute__((malloc)) escape_html(const char *string) return escaped; } + +int check_json_payload(struct ftl_conn *api) +{ + if (api->payload.json == NULL) + { + if (api->payload.json_error == NULL) + return send_json_error(api, 400, + "bad_request", + "No request body data", + NULL); + else + return send_json_error(api, 400, + "bad_request", + "Invalid request body data (no valid JSON), error at hint", + api->payload.json_error); + } + + // All okay + return 0; +} + +// Black magic at work here: We build a JSON array from the group_concat result +// delivered from the database, parse it as valid array and append it as row to +// the data +int parse_groupIDs(struct ftl_conn *api, tablerow *table, cJSON *row) +{ + const size_t buflen = strlen(table->group_ids) + 3u; + char *group_ids_str = calloc(buflen, sizeof(char)); + if(group_ids_str == NULL) + { + return send_json_error(api, 500, // 500 Internal Server Error + "out_of_memory", + "Out of memory", + NULL); + } + group_ids_str[0] = '['; + strcpy(group_ids_str+1u , table->group_ids); + group_ids_str[buflen-2u] = ']'; + group_ids_str[buflen-1u] = '\0'; + const char *json_error = NULL; + cJSON *group_ids = cJSON_ParseWithOpts(group_ids_str, &json_error, false); + free(group_ids_str); + if(group_ids == NULL) + { + // Error parsing group_ids, substitute empty array + // Note: This should never happen as the database's aggregate + // function should always return a valid JSON array + log_err("Error parsing group_ids, error at: %s", json_error); + JSON_ADD_ITEM_TO_OBJECT(row, "groups", JSON_NEW_ARRAY()); + } + else + { + JSON_ADD_ITEM_TO_OBJECT(row, "groups", group_ids); + } + + // Success + return 0; +} diff --git a/src/webserver/http-common.h b/src/webserver/http-common.h index a710dba2..586af162 100644 --- a/src/webserver/http-common.h +++ b/src/webserver/http-common.h @@ -15,6 +15,8 @@ #include "webserver/cJSON/cJSON.h" // enum fifo_logs #include "enums.h" +// tablerow +#include "database/gravity-db.h" // strlen() #include @@ -23,22 +25,6 @@ // Maximum size of received and processed payload: 64 KB #define MAX_PAYLOAD_BYTES 64*1024 -enum http_method { - HTTP_UNKNOWN = 0, - HTTP_GET = 1 << 0, - HTTP_POST = 1 << 1, - HTTP_PUT = 1 << 2, - HTTP_PATCH = 1 << 3, - HTTP_DELETE = 1 << 4, - HTTP_OPTIONS = 1 << 5, -}; - -enum api_flags { - API_FLAG_NONE = 0, - API_DOMAINS = 1 << 0, - API_PARSE_JSON = 1 << 1, - API_BATCHDELETE = 1 << 2, -}; struct api_options { enum api_flags flags; @@ -109,5 +95,7 @@ enum http_method __attribute__((pure)) http_method(struct mg_connection *conn); const char* __attribute__((pure)) startsWith(const char *path, struct ftl_conn *api); void read_and_parse_payload(struct ftl_conn *api); char * __attribute__((malloc)) escape_html(const char *string); +int check_json_payload(struct ftl_conn *api); +int parse_groupIDs(struct ftl_conn *api, tablerow *table, cJSON *row); #endif // HTTP_H diff --git a/src/webserver/json_macros.h b/src/webserver/json_macros.h index f3966f08..90ff6b21 100644 --- a/src/webserver/json_macros.h +++ b/src/webserver/json_macros.h @@ -12,10 +12,10 @@ // logging routines #include "log.h" -#define JSON_NEW_OBJECT() cJSON_CreateObject(); -#define JSON_NEW_ARRAY() cJSON_CreateArray(); +#define JSON_NEW_OBJECT() cJSON_CreateObject() +#define JSON_NEW_ARRAY() cJSON_CreateArray() -#define JSON_ADD_ITEM_TO_ARRAY(array, item) cJSON_AddItemToArray(array, item); +#define JSON_ADD_ITEM_TO_ARRAY(array, item) cJSON_AddItemToArray(array, item) #define JSON_COPY_STR_TO_OBJECT(object, key, string)({ \ cJSON *string_item = NULL; \ diff --git a/test/test_suite.bats b/test/test_suite.bats index 64a4a9ee..d88ce70e 100644 --- a/test/test_suite.bats +++ b/test/test_suite.bats @@ -1368,6 +1368,15 @@ [[ ${lines[1]} == *"WARNING: - FTLCONF_dns_upstreams" ]] } +@test "cJSON_GetErrorPtr and cJSON_InitHooks are never used (for thread-safety reasons)" { + # cJSON_GetErrorPtr() is not thread-safe but can be replaces by cJSON_ParseWithOpts() + # cJSON_InitHooks() is only thread-safe if used before any other cJSON function in a thread + # We grep for the two functions recursively and exclude cJSON.{c,h} where they are defined + run bash -c 'grep -rE "(cJSON_GetErrorPtr)|(cJSON_InitHooks)" src/ | grep -vE "^src/webserver/cJSON/cJSON."' + printf "%s\n" "${lines[@]}" + [[ ${lines[0]} == "" ]] +} + @test "CLI complains about unknown config key and offers a suggestion" { run bash -c './pihole-FTL --config dbg.all' [[ ${lines[0]} == "Unknown config option dbg.all, did you mean:" ]] @@ -1472,7 +1481,7 @@ run bash -c './pihole-FTL --config dns.revServers "abc"' printf "%s\n" "${lines[@]}" - [[ ${lines[0]} == 'Config setting dns.revServers is invalid: not valid JSON, error before: abc' ]] + [[ ${lines[0]} == 'Config setting dns.revServers is invalid: not valid JSON, error at: abc' ]] [[ $status == 2 ]] } From 557d6a44af86f0ff80613cc6ecd054b9b717c60c Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 28 Mar 2024 09:46:17 +0100 Subject: [PATCH 034/339] Fix a left-over "whitelisted" instead of "allowed" message in debug mode Signed-off-by: DL6ER --- src/dnsmasq_interface.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c index 68cd310d..dc7ae777 100644 --- a/src/dnsmasq_interface.c +++ b/src/dnsmasq_interface.c @@ -1502,13 +1502,13 @@ static bool _FTL_check_blocking(int queryID, int domainID, int clientID, const c { // Explicitly mark as not blocked to skip the entire gravity/blacklist // chain when the same client asks for the same domain in the future. - // Store domain as whitelisted if this is the case + // Store domain as allowed if this is the case dns_cache->blocking_status = query->flags.allowed ? ALLOWED : NOT_BLOCKED; // Debug output // client is guaranteed to be non-NULL above log_debug(DEBUG_QUERIES, "DNS cache: %s/%s is %s (domainlist ID: %i)", getstr(client->ippos), - domainstr, query->flags.allowed ? "whitelisted" : "not blocked", dns_cache->list_id); + domainstr, query->flags.allowed ? "allowed" : "not blocked", dns_cache->list_id); } free(domainstr); From f94fe117550edac2599470d4fc746f8e56b59c08 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 28 Mar 2024 15:24:38 +0100 Subject: [PATCH 035/339] Mark query as allowed when atigravity matches to prevent further checks such as CNAME inspection. This ensures antigravity matches have similar effects than explicitly allowed domains. Signed-off-by: DL6ER --- src/dnsmasq_interface.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c index 68cd310d..c8240329 100644 --- a/src/dnsmasq_interface.c +++ b/src/dnsmasq_interface.c @@ -1154,6 +1154,11 @@ static bool check_domain_blocked(const char *domain, const int clientID, // ... dns_cache->list_id = -1 * (list_id + 2); + // Mark query as allowed to prevent further checks such as CNAME + // inspection. This ensures antigravity matches have similar effects + // than explicitly allowed domains. + query->flags.allowed = true; + return false; } From 549bc164ea6af104bdd3172baf31ccd506b99d0f Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 28 Mar 2024 18:55:46 +0100 Subject: [PATCH 036/339] Slightly simplify the CI tests Signed-off-by: DL6ER --- test/test_suite.bats | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_suite.bats b/test/test_suite.bats index 64a4a9ee..32516b6c 100644 --- a/test/test_suite.bats +++ b/test/test_suite.bats @@ -1524,7 +1524,7 @@ run bash -c './pihole-FTL --config dns.revServers "[\"true,1.1.1.1,def,ghi\"]"' printf "%s\n" "${lines[@]}" - [[ ${lines[0]} == 'New dnsmasq configuration is not valid ('*'Name does not resolve at line '*' of /etc/pihole/dnsmasq.conf.temp: "rev-server=1.1.1.1,def"), config remains unchanged' ]] + [[ ${lines[0]} == 'New dnsmasq configuration is not valid ('*'resolve at line '*' of /etc/pihole/dnsmasq.conf.temp: "rev-server=1.1.1.1,def"), config remains unchanged' ]] [[ $status == 3 ]] run bash -c './pihole-FTL --config webserver.api.excludeClients "[\".*\",\"$$$\",\"[[[\"]"' From daf5eb89348e44113a9ab3b6e2013efe93717218 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 28 Mar 2024 10:25:12 +0100 Subject: [PATCH 037/339] Remove two characters TLDs constraint in hostname validation. Empty labels are still forbidden. Signed-off-by: DL6ER --- src/tools/gravity-parseList.c | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/tools/gravity-parseList.c b/src/tools/gravity-parseList.c index de30f7ca..42a7c267 100644 --- a/src/tools/gravity-parseList.c +++ b/src/tools/gravity-parseList.c @@ -94,12 +94,6 @@ inline bool __attribute__((pure)) valid_domain(const char *domain, const size_t if(domain[last_dot + 1] == '-' || domain[len - 1] == '-') return false; - // TLD length check - // The last label must be at least 2 characters long - // (len-1) because we start counting from zero - if((len - 1) - last_dot < 2) - return false; - return true; } From 40864d5c38f1441ca99f27c25aad6fb40d13d17d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 30 Mar 2024 10:42:21 +0000 Subject: [PATCH 038/339] Bump the github_action-dependencies group with 1 update Bumps the github_action-dependencies group with 1 update: [eps1lon/actions-label-merge-conflict](https://github.com/eps1lon/actions-label-merge-conflict). Updates `eps1lon/actions-label-merge-conflict` from 2.1.0 to 3.0.0 - [Release notes](https://github.com/eps1lon/actions-label-merge-conflict/releases) - [Changelog](https://github.com/eps1lon/actions-label-merge-conflict/blob/main/CHANGELOG.md) - [Commits](https://github.com/eps1lon/actions-label-merge-conflict/compare/v2.1.0...v3.0.0) --- updated-dependencies: - dependency-name: eps1lon/actions-label-merge-conflict dependency-type: direct:production update-type: version-update:semver-major dependency-group: github_action-dependencies ... Signed-off-by: dependabot[bot] --- .github/workflows/merge-conflict.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/merge-conflict.yml b/.github/workflows/merge-conflict.yml index 43b59de4..86c2c4fd 100644 --- a/.github/workflows/merge-conflict.yml +++ b/.github/workflows/merge-conflict.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check if PRs are have merge conflicts - uses: eps1lon/actions-label-merge-conflict@v2.1.0 + uses: eps1lon/actions-label-merge-conflict@v3.0.0 with: dirtyLabel: "Merge conflicts" repoToken: "${{ secrets.GITHUB_TOKEN }}" From ed36a9a7d5bea9c866c43251f33d59871fc1c55b Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 30 Mar 2024 17:14:28 +0100 Subject: [PATCH 039/339] Simplify v5 gravity table import condition Signed-off-by: DL6ER --- src/api/teleporter.c | 65 +++++++++++++++++++++----------------------- 1 file changed, 31 insertions(+), 34 deletions(-) diff --git a/src/api/teleporter.c b/src/api/teleporter.c index 5bf313b3..8db8fb5e 100644 --- a/src/api/teleporter.c +++ b/src/api/teleporter.c @@ -670,43 +670,40 @@ static int process_received_tar_gz(struct ftl_conn *api, struct upload_data *dat // Check if the archive contains gravity tables cJSON *gravity = data->import != NULL ? cJSON_GetObjectItemCaseSensitive(data->import, "gravity") : NULL; - if(data->import == NULL || gravity != NULL) + for(size_t i = 0; i < sizeof(teleporter_v5_files) / sizeof(struct teleporter_files); i++) { - for(size_t i = 0; i < sizeof(teleporter_v5_files) / sizeof(struct teleporter_files); i++) + // - if import is NULL we import all files/tables + // - if import is non-NULL, but gravity is NULL we skip + // the import of gravity tables + // - if import is non-NULL, and gravity is non-NULL, we + // import the file/table if it is in the object, a + // boolean and true + if(data->import != NULL || gravity == NULL || !JSON_KEY_TRUE(gravity, teleporter_v5_files[i].table_name)) { - // - if import is NULL we import all files/tables - // - if import is non-NULL, but gravity is NULL we skip - // the import of gravity tables - // - if import is non-NULL, and gravity is non-NULL, we - // import the file/table if it is in the object, a - // boolean and true - if(data->import != NULL || gravity == NULL || !JSON_KEY_TRUE(gravity, teleporter_v5_files[i].table_name)) - { - log_info("Skipping import of \"%s\" as it was not requested for import", - teleporter_v5_files[i].filename); - continue; - } + log_info("Skipping import of \"%s\" as it was not requested for import", + teleporter_v5_files[i].filename); + continue; + } - // Import the JSON file - size_t fileSize = 0u; - cJSON *json = NULL; - const char *file = find_file_in_tar(archive, archive_size, teleporter_v5_files[i].filename, &fileSize); - const char *json_error = NULL; - if(file != NULL && fileSize > 0u && (json = cJSON_ParseWithLengthOpts(file, fileSize, &json_error, false)) != NULL) - { - if(import_json_table(json, &teleporter_v5_files[i])) - JSON_COPY_STR_TO_ARRAY(imported_files, teleporter_v5_files[i].filename); - } - else if(json_error != NULL) - { - log_err("Unable to parse JSON file \"%s\", error at: %s", - teleporter_v5_files[i].filename, json_error); - } - else - { - log_debug(DEBUG_CONFIG, "Unable to find file \"%s\" in TAR archive", - teleporter_v5_files[i].filename); - } + // Import the JSON file + size_t fileSize = 0u; + cJSON *json = NULL; + const char *file = find_file_in_tar(archive, archive_size, teleporter_v5_files[i].filename, &fileSize); + const char *json_error = NULL; + if(file != NULL && fileSize > 0u && (json = cJSON_ParseWithLengthOpts(file, fileSize, &json_error, false)) != NULL) + { + if(import_json_table(json, &teleporter_v5_files[i])) + JSON_COPY_STR_TO_ARRAY(imported_files, teleporter_v5_files[i].filename); + } + else if(json_error != NULL) + { + log_err("Unable to parse JSON file \"%s\", error at: %s", + teleporter_v5_files[i].filename, json_error); + } + else + { + log_debug(DEBUG_CONFIG, "Unable to find file \"%s\" in TAR archive", + teleporter_v5_files[i].filename); } } From 8875a0e29a18ec0602631f0c1ba8352c578f3774 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 30 Mar 2024 17:17:17 +0100 Subject: [PATCH 040/339] Add further debugging output if files are NOT imported Signed-off-by: DL6ER --- src/api/teleporter.c | 17 ++++++++++------- src/config/cli.c | 2 +- src/webserver/http-common.c | 2 +- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/src/api/teleporter.c b/src/api/teleporter.c index 8db8fb5e..f60266d5 100644 --- a/src/api/teleporter.c +++ b/src/api/teleporter.c @@ -150,10 +150,11 @@ static int field_get(const char *key, const char *value, size_t valuelen, void * else if(data->field.import) { // Try to parse the JSON data - cJSON *json = cJSON_ParseWithLength(value, valuelen); + const char *json_error = NULL; + cJSON *json = cJSON_ParseWithLengthOpts(value, valuelen, &json_error, false); if(json == NULL) { - log_err("Unable to parse JSON data in API request: %s", cJSON_GetErrorPtr()); + log_err("Unable to parse JSON data in API request, error at: %.20s", json_error); return MG_FORM_FIELD_HANDLE_ABORT; } @@ -680,8 +681,10 @@ static int process_received_tar_gz(struct ftl_conn *api, struct upload_data *dat // boolean and true if(data->import != NULL || gravity == NULL || !JSON_KEY_TRUE(gravity, teleporter_v5_files[i].table_name)) { - log_info("Skipping import of \"%s\" as it was not requested for import", - teleporter_v5_files[i].filename); + log_info("Skipping import of \"%s\" as it was not requested for import (JSON: %s, gravity: %s)", + teleporter_v5_files[i].filename, + data->import != NULL ? "yes" : "no", + gravity != NULL ? "yes" : "no"); continue; } @@ -697,13 +700,13 @@ static int process_received_tar_gz(struct ftl_conn *api, struct upload_data *dat } else if(json_error != NULL) { - log_err("Unable to parse JSON file \"%s\", error at: %s", - teleporter_v5_files[i].filename, json_error); + log_err("Unable to parse JSON file \"%s\", error at: %.20s", + teleporter_v5_files[i].filename, json_error); } else { log_debug(DEBUG_CONFIG, "Unable to find file \"%s\" in TAR archive", - teleporter_v5_files[i].filename); + teleporter_v5_files[i].filename); } } diff --git a/src/config/cli.c b/src/config/cli.c index 292f6955..98dd765b 100644 --- a/src/config/cli.c +++ b/src/config/cli.c @@ -346,7 +346,7 @@ static bool readStringValue(struct conf_item *conf_item, const char *value, stru cJSON *elem = cJSON_ParseWithOpts(value, &json_error, 0); if(elem == NULL) { - log_err("Config setting %s is invalid: not valid JSON, error at: %s", conf_item->k, json_error); + log_err("Config setting %s is invalid: not valid JSON, error at: %.20s", conf_item->k, json_error); return false; } if(!cJSON_IsArray(elem)) diff --git a/src/webserver/http-common.c b/src/webserver/http-common.c index 69503238..2b11057d 100644 --- a/src/webserver/http-common.c +++ b/src/webserver/http-common.c @@ -620,7 +620,7 @@ int parse_groupIDs(struct ftl_conn *api, tablerow *table, cJSON *row) // Error parsing group_ids, substitute empty array // Note: This should never happen as the database's aggregate // function should always return a valid JSON array - log_err("Error parsing group_ids, error at: %s", json_error); + log_err("Error parsing group_ids, error at: %.20s", json_error); JSON_ADD_ITEM_TO_OBJECT(row, "groups", JSON_NEW_ARRAY()); } else From 5075144bbee3193bb353fd4e84516b9b2066c70a Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 30 Mar 2024 20:00:52 +0100 Subject: [PATCH 041/339] Fix importing logic for v5 teleporter files Signed-off-by: DL6ER --- src/api/teleporter.c | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/src/api/teleporter.c b/src/api/teleporter.c index f60266d5..5e4a3b5a 100644 --- a/src/api/teleporter.c +++ b/src/api/teleporter.c @@ -673,19 +673,22 @@ static int process_received_tar_gz(struct ftl_conn *api, struct upload_data *dat cJSON *gravity = data->import != NULL ? cJSON_GetObjectItemCaseSensitive(data->import, "gravity") : NULL; for(size_t i = 0; i < sizeof(teleporter_v5_files) / sizeof(struct teleporter_files); i++) { - // - if import is NULL we import all files/tables - // - if import is non-NULL, but gravity is NULL we skip - // the import of gravity tables - // - if import is non-NULL, and gravity is non-NULL, we - // import the file/table if it is in the object, a - // boolean and true - if(data->import != NULL || gravity == NULL || !JSON_KEY_TRUE(gravity, teleporter_v5_files[i].table_name)) + // - if import is non-NULL we may skip some tables + if(data->import != NULL) { - log_info("Skipping import of \"%s\" as it was not requested for import (JSON: %s, gravity: %s)", - teleporter_v5_files[i].filename, - data->import != NULL ? "yes" : "no", - gravity != NULL ? "yes" : "no"); - continue; + // - if import is non-NULL, but gravity is NULL we skip + // the import of gravity tables altogether + // - if import is non-NULL, and gravity is non-NULL, we + // import the file/table if it is in the object, a + // boolean and true + if(gravity == NULL || !JSON_KEY_TRUE(gravity, teleporter_v5_files[i].table_name)) + { + log_info("Skipping import of \"%s\" as it was not requested for import (JSON: %s, gravity: %s)", + teleporter_v5_files[i].filename, + data->import != NULL ? "yes" : "no", + gravity != NULL ? "yes" : "no"); + continue; + } } // Import the JSON file From 27ff979ef8ec2a825e8b3338927d28b9a549bb13 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 30 Mar 2024 20:14:15 +0100 Subject: [PATCH 042/339] Reintroduce a workaround for docker on macOS accidentally removed in bd266d6589e95e889bdcbbe1fe4cc0db85e81651 Signed-off-by: DL6ER --- src/files.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/files.c b/src/files.c index d34bea09..afb32df9 100644 --- a/src/files.c +++ b/src/files.c @@ -269,11 +269,12 @@ unsigned int get_path_usage(const char *path, char buffer[64]) // If size is 0, we return 0% to avoid division by zero below if(size == 0) return 0; - // If used is larger than size, we return 100% - if(used > size) - return 100; + // Return percentage of used memory at this path (rounded down) - return (used*100)/(size + 1); + // If the used size is larger than the total size, this intentionally + // returns more than 100% so that the caller can handle this case + // (this can happen with docker on macOS) + return (used * 100) / size; } // Get the filesystem where the given path is located From 58ca959cf2277f39d2ba37e9b8d8ffe3c5812c60 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 30 Mar 2024 20:21:20 +0100 Subject: [PATCH 043/339] Add proper memory allocation checking in the message formatting subroutines Signed-off-by: DL6ER --- src/database/message-table.c | 141 ++++++++++++++++++++++++++--------- 1 file changed, 107 insertions(+), 34 deletions(-) diff --git a/src/database/message-table.c b/src/database/message-table.c index f6eaef8c..6b33ec3c 100644 --- a/src/database/message-table.c +++ b/src/database/message-table.c @@ -443,14 +443,22 @@ static void format_regex_message(char *plain, const int sizeof_plain, char *html char *escaped_regex = escape_html(regex); char *escaped_warning = escape_html(warning); + // Return early if memory allocation failed + if(escaped_regex == NULL || escaped_warning == NULL) + { + if(escaped_regex != NULL) + free(escaped_regex); + if(escaped_warning != NULL) + free(escaped_warning); + return; + } + if(snprintf(html, sizeof_html, "Encountered an error when processing regex %s filter with ID %d: 
    %s
    Error message: 
    %s
    ", dbindex, type, dbindex, escaped_regex, escaped_warning)) log_warn("format_regex_message(): Buffer too small to hold HTML message, warning truncated"); - if(escaped_regex != NULL) - free(escaped_regex); - if(escaped_warning != NULL) - free(escaped_warning); + free(escaped_regex); + free(escaped_warning); } static void format_subnet_message(char *plain, const int sizeof_plain, char *html, const int sizeof_html, const char *ip, const int matching_count, const char *names, const char *matching_ids, const char *chosen_match_text, const int chosen_match_id) @@ -469,17 +477,26 @@ static void format_subnet_message(char *plain, const int sizeof_plain, char *htm char *escaped_ids = escape_html(matching_ids); char *escaped_names = escape_html(names); + // Return early if memory allocation failed + if(escaped_ip == NULL || escaped_ids == NULL || escaped_names == NULL) + { + if(escaped_ip != NULL) + free(escaped_ip); + if(escaped_ids != NULL) + free(escaped_ids); + if(escaped_names != NULL) + free(escaped_names); + return; + } + if(snprintf(html, sizeof_html, "Client %s is managed by %i groups (IDs [%s]), all describing the same subnet:
    %s
    " "FTL chose the most recent entry (ID %i) to obtain the group configuration for this client.", escaped_ip, matching_count, escaped_ids, escaped_names, chosen_match_id) > sizeof_html) log_warn("format_subnet_message(): Buffer too small to hold HTML message, warning truncated"); - if(escaped_ip != NULL) - free(escaped_ip); - if(escaped_ids != NULL) - free(escaped_ids); - if(escaped_names != NULL) - free(escaped_names); + free(escaped_ip); + free(escaped_ids); + free(escaped_names); } static void format_hostname_message(char *plain, const int sizeof_plain, char *html, const int sizeof_html, const char *ip, const char *name, const int pos) @@ -519,14 +536,24 @@ static void format_hostname_message(char *plain, const int sizeof_plain, char *h char *escaped_ip = escape_html(ip); char *escaped_name = escape_html(namep); + // Return early if memory allocation failed + if(escaped_ip == NULL || escaped_name == NULL) + { + if(escaped_ip != NULL) + free(escaped_ip); + if(escaped_name != NULL) + free(escaped_name); + if(namep != NULL) + free(namep); + return; + } + if(snprintf(html, sizeof_html, "Host name of client %s => %s contains (at least) one invalid character (hex %02x) at position %i", escaped_ip, escaped_name, (unsigned char)name[pos], pos) > sizeof_html) log_warn("format_hostname_message(): Buffer too small to hold HTML message, warning truncated"); - if(escaped_ip != NULL) - free(escaped_ip); - if(escaped_name != NULL) - free(escaped_name); + free(escaped_ip); + free(escaped_name); if(namep != NULL) free(namep); } @@ -542,11 +569,14 @@ static void format_dnsmasq_config_message(char *plain, const int sizeof_plain, c char *escaped_message = escape_html(message); + // Return early if memory allocation failed + if(escaped_message == NULL) + return; + if(snprintf(html, sizeof_html, "FTL failed to start due to %s.", escaped_message) > sizeof_html) log_warn("format_dnsmasq_config_message(): Buffer too small to hold HTML message, warning truncated"); - if(escaped_message != NULL) - free(escaped_message); + free(escaped_message); } static void format_rate_limit_message(char *plain, const int sizeof_plain, char *html, const int sizeof_html, const char *clientIP, const unsigned int count, const unsigned int interval, const time_t turnaround) @@ -561,12 +591,15 @@ static void format_rate_limit_message(char *plain, const int sizeof_plain, char char *escaped_clientIP = escape_html(clientIP); + // Return early if memory allocation failed + if(escaped_clientIP == NULL) + return; + if(snprintf(html, sizeof_html, "Client %s has been rate-limited for at least %lu second%s (current limit: %u queries per %u seconds)", escaped_clientIP, (unsigned long int)turnaround, turnaround == 1 ? "" : "s", count, interval) > sizeof_html) log_warn("format_rate_limit_message(): Buffer too small to hold HTML message, warning truncated"); - if(escaped_clientIP != NULL) - free(escaped_clientIP); + free(escaped_clientIP); } static void format_dnsmasq_warn_message(char *plain, const int sizeof_plain, char *html, const int sizeof_html, const char *message) @@ -610,14 +643,22 @@ static void format_shmem_message(char *plain, const int sizeof_plain, char *html char *escaped_path = escape_html(path); char *escaped_msg = escape_html(msg); + // Return early if memory allocation failed + if(escaped_path == NULL || escaped_msg == NULL) + { + if(escaped_path != NULL) + free(escaped_path); + if(escaped_msg != NULL) + free(escaped_msg); + return; + } + if(snprintf(html, sizeof_html, "Shared memory shortage (%s) ahead: %d%% is used
    %s", escaped_path, shmem, escaped_msg) > sizeof_html) log_warn("log_resource_shortage(): Buffer too small to hold HTML message, warning truncated"); - if(escaped_path != NULL) - free(escaped_path); - if(escaped_msg != NULL) - free(escaped_msg); + free(escaped_path); + free(escaped_msg); } static void format_disk_message(char *plain, const int sizeof_plain, char *html, const int sizeof_html, @@ -634,10 +675,22 @@ static void format_disk_message(char *plain, const int sizeof_plain, char *html, char *escaped_path = escape_html(path); char *escaped_msg = escape_html(msg); + // Return early if memory allocation failed + if(escaped_path == NULL || escaped_msg == NULL) + { + if(escaped_path != NULL) + free(escaped_path); + if(escaped_msg != NULL) + free(escaped_msg); + return; + } if(snprintf(html, sizeof_html, "Disk shortage ahead: %d%% is used (%s) on partition containing the file %s", disk, escaped_msg, escaped_path) > sizeof_html) log_warn("format_disk_message(): Buffer too small to hold HTML message, warning truncated"); + + free(escaped_path); + free(escaped_msg); } static void format_disk_message_extended(char *plain, const int sizeof_plain, char *html, const int sizeof_html, @@ -655,16 +708,25 @@ static void format_disk_message_extended(char *plain, const int sizeof_plain, ch char *escaped_mnt_dir = escape_html(mnt_dir); char *escaped_msg = escape_html(msg); + // Return early if memory allocation failed + if(escaped_mnt_type == NULL || escaped_mnt_dir == NULL || escaped_msg == NULL) + { + if(escaped_mnt_type != NULL) + free(escaped_mnt_type); + if(escaped_mnt_dir != NULL) + free(escaped_mnt_dir); + if(escaped_msg != NULL) + free(escaped_msg); + return; + } + if(snprintf(html, sizeof_html, "Disk shortage ahead: %d%% is used (%s) on %s filesystem mounted at %s", disk, escaped_msg, escaped_mnt_type, escaped_mnt_dir) > sizeof_html) log_warn("format_disk_message_extended(): Buffer too small to hold HTML message, warning truncated"); - if(escaped_mnt_type != NULL) - free(escaped_mnt_type); - if(escaped_mnt_dir != NULL) - free(escaped_mnt_dir); - if(escaped_msg != NULL) - free(escaped_msg); + free(escaped_mnt_type); + free(escaped_mnt_dir); + free(escaped_msg); } static void format_inaccessible_adlist_message(char *plain, const int sizeof_plain, char *html, const int sizeof_html, @@ -680,12 +742,15 @@ static void format_inaccessible_adlist_message(char *plain, const int sizeof_pla char *escaped_address = escape_html(address); + // Return early if memory allocation failed + if(escaped_address == NULL) + return; + if(snprintf(html, sizeof_html, "List with ID %d (%s) was inaccessible during last gravity run", dbindex, dbindex, escaped_address) > sizeof_html) log_warn("format_inaccessible_adlist_message(): Buffer too small to hold HTML message, warning truncated"); - if(escaped_address != NULL) - free(escaped_address); + free(escaped_address); } static void format_certificate_domain_mismatch(char *plain, const int sizeof_plain, char *html, const int sizeof_html, @@ -701,13 +766,21 @@ static void format_certificate_domain_mismatch(char *plain, const int sizeof_pla char *escaped_certfile = escape_html(certfile); char *escaped_domain = escape_html(domain); + // Return early if memory allocation failed + if(escaped_certfile == NULL || escaped_domain == NULL) + { + if(escaped_certfile != NULL) + free(escaped_certfile); + if(escaped_domain != NULL) + free(escaped_domain); + return; + } + if(snprintf(html, sizeof_html, "SSL/TLS certificate %s does not match domain %s!", escaped_certfile, escaped_domain) > sizeof_html) log_warn("format_certificate_domain_mismatch(): Buffer too small to hold HTML message, warning truncated"); - if(escaped_certfile != NULL) - free(escaped_certfile); - if(escaped_domain != NULL) - free(escaped_domain); + free(escaped_certfile); + free(escaped_domain); } int count_messages(const bool filter_dnsmasq_warnings) From df1f70dd095ed63018b568b90964b4449f993b58 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 31 Mar 2024 12:21:21 +0200 Subject: [PATCH 044/339] Add further debug output concerning disk usage when debug.gc=true Signed-off-by: DL6ER --- src/database/message-table.c | 17 +++++++++++++++++ src/files.c | 17 +++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/src/database/message-table.c b/src/database/message-table.c index 6b33ec3c..895dffae 100644 --- a/src/database/message-table.c +++ b/src/database/message-table.c @@ -1206,6 +1206,23 @@ void log_resource_shortage(const double load, const int nprocs, const int shmem, // Get filesystem details for this path struct mntent *fsdetails = get_filesystem_details(path); + // Log filesystem details if in debug mode + if(config.debug.gc.v.b) + { + if(fsdetails != NULL) + { + log_debug(DEBUG_GC, "Disk details for path \"%s\":", path); + log_debug(DEBUG_GC, " Device or server for filesystem: %s", fsdetails->mnt_fsname); + log_debug(DEBUG_GC, " Directory mounted on: %s", fsdetails->mnt_dir); + log_debug(DEBUG_GC, " Type of filesystem: %s", fsdetails->mnt_type); + log_debug(DEBUG_GC, " Comma-separated options for fs: %s", fsdetails->mnt_opts); + log_debug(DEBUG_GC, " Dump frequency (in days): %d", fsdetails->mnt_freq); + log_debug(DEBUG_GC, " Pass number for `fsck': %d", fsdetails->mnt_passno); + } + else + log_debug(DEBUG_GC, "Failed to get filesystem details for path \"%s\"", path); + } + // Create plain message if(fsdetails != NULL) format_disk_message_extended(buf, sizeof(buf), NULL, 0, disk, msg, fsdetails->mnt_type, fsdetails->mnt_dir); diff --git a/src/files.c b/src/files.c index afb32df9..b5c3277c 100644 --- a/src/files.c +++ b/src/files.c @@ -252,6 +252,23 @@ unsigned int get_path_usage(const char *path, char buffer[64]) const unsigned long long free = (unsigned long long)f.f_bavail * f.f_bsize; const unsigned long long used = size - free; + // Print statvfs() results if in debug.gc mode + if(config.debug.gc.v.b) + { + log_debug(DEBUG_GC, "Statvfs() results for %s:", path); + log_debug(DEBUG_GC, " Block size: %lu", f.f_bsize); + log_debug(DEBUG_GC, " Fragment size: %lu", f.f_frsize); + log_debug(DEBUG_GC, " Total blocks: %lu", f.f_blocks); + log_debug(DEBUG_GC, " Free blocks: %lu", f.f_bfree); + log_debug(DEBUG_GC, " Available blocks: %lu", f.f_bavail); + log_debug(DEBUG_GC, " Total inodes: %lu", f.f_files); + log_debug(DEBUG_GC, " Free inodes: %lu", f.f_ffree); + log_debug(DEBUG_GC, " Available inodes: %lu", f.f_favail); + log_debug(DEBUG_GC, " Filesystem ID: %lu", f.f_fsid); + log_debug(DEBUG_GC, " Mount flags: %lu", f.f_flag); + log_debug(DEBUG_GC, " Maximum filename length: %lu", f.f_namemax); + } + // Create human-readable total size char prefix_size[2] = { 0 }; double formatted_size = 0.0; From 16c541e2330e8b2c185c148b335b59ef81096caa Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 31 Mar 2024 12:26:08 +0200 Subject: [PATCH 045/339] Show warning when in debug mode and stat() failed to get file system details Signed-off-by: DL6ER --- src/files.c | 40 ++++++++++++++++++++++++++++------------ 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/src/files.c b/src/files.c index b5c3277c..196f2e61 100644 --- a/src/files.c +++ b/src/files.c @@ -30,6 +30,9 @@ #include #include +// PRIu64 +#include + // chmod_file() changes the file mode bits of a given file (relative // to the directory file descriptor) according to mode. mode is an // octal number representing the bit pattern for the new mode bits @@ -258,12 +261,12 @@ unsigned int get_path_usage(const char *path, char buffer[64]) log_debug(DEBUG_GC, "Statvfs() results for %s:", path); log_debug(DEBUG_GC, " Block size: %lu", f.f_bsize); log_debug(DEBUG_GC, " Fragment size: %lu", f.f_frsize); - log_debug(DEBUG_GC, " Total blocks: %lu", f.f_blocks); - log_debug(DEBUG_GC, " Free blocks: %lu", f.f_bfree); - log_debug(DEBUG_GC, " Available blocks: %lu", f.f_bavail); - log_debug(DEBUG_GC, " Total inodes: %lu", f.f_files); - log_debug(DEBUG_GC, " Free inodes: %lu", f.f_ffree); - log_debug(DEBUG_GC, " Available inodes: %lu", f.f_favail); + log_debug(DEBUG_GC, " Total blocks: %"PRIu64, f.f_blocks); + log_debug(DEBUG_GC, " Free blocks: %"PRIu64, f.f_bfree); + log_debug(DEBUG_GC, " Available blocks: %"PRIu64, f.f_bavail); + log_debug(DEBUG_GC, " Total inodes: %"PRIu64, f.f_files); + log_debug(DEBUG_GC, " Free inodes: %"PRIu64, f.f_ffree); + log_debug(DEBUG_GC, " Available inodes: %"PRIu64, f.f_favail); log_debug(DEBUG_GC, " Filesystem ID: %lu", f.f_fsid); log_debug(DEBUG_GC, " Mount flags: %lu", f.f_flag); log_debug(DEBUG_GC, " Maximum filename length: %lu", f.f_namemax); @@ -297,27 +300,40 @@ unsigned int get_path_usage(const char *path, char buffer[64]) // Get the filesystem where the given path is located struct mntent *get_filesystem_details(const char *path) { - /* stat the file in question */ + // stat the file in question struct stat path_stat; stat(path, &path_stat); - /* iterate through the list of devices */ + // iterate through the list of devices FILE *file = setmntent("/proc/mounts", "r"); struct mntent *ent = NULL; + bool found = false; while(file != NULL && (ent = getmntent(file)) != NULL) { - /* stat the mount point */ + // stat the mount point struct stat dev_stat; - stat(ent->mnt_dir, &dev_stat); + if(stat(ent->mnt_dir, &dev_stat) < 0) + { + if(config.debug.gc.v.b) + { + log_warn("get_filesystem_details(): Failed to get stat for \"%s\": %s", + ent->mnt_dir, strerror(errno)); + } + continue; + } - /* check if our file and the mount point are on the same device */ + // check if our file and the mount point are on the same device if(dev_stat.st_dev == path_stat.st_dev) + { + found = true; break; + } } + // Close mount table file handle endmntent(file); - return ent; + return found ? ent : NULL; } // Credits: https://stackoverflow.com/a/55410469 From 90dda14276bdf6440704e10e5955a92dff9fd09a Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 1 Apr 2024 09:11:06 +0200 Subject: [PATCH 046/339] Use fragment size when computing filesystem sizes Signed-off-by: DL6ER --- src/files.c | 12 +++++++----- src/log.c | 7 +++---- src/log.h | 5 +++-- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/src/files.c b/src/files.c index 196f2e61..5e1f57c0 100644 --- a/src/files.c +++ b/src/files.c @@ -249,11 +249,13 @@ unsigned int get_path_usage(const char *path, char buffer[64]) return 0; } - // Explicitly cast the block counts to unsigned long long to avoid - // overflowing with drives larger than 4 GB on 32bit systems - const unsigned long long size = (unsigned long long)f.f_blocks * f.f_frsize; - const unsigned long long free = (unsigned long long)f.f_bavail * f.f_bsize; - const unsigned long long used = size - free; + // Explicitly cast the block counts to uint64_t to avoid overflowing + // with drives larger than 4 GB on 32bit systems. Multiply the block + // count with the fragment size to get the total size in bytes, see + // https://github.com/torvalds/linux/blob/39cd87c4eb2b893354f3b850f916353f2658ae6f/fs/nfs/super.c#L285-L291 + const uint64_t size = (uint64_t)f.f_blocks * f.f_frsize; + const uint64_t free = (uint64_t)f.f_bavail * f.f_frsize; + const uint64_t used = size - free; // Print statvfs() results if in debug.gc mode if(config.debug.gc.v.b) diff --git a/src/log.c b/src/log.c index 8eac8c70..e861c922 100644 --- a/src/log.c +++ b/src/log.c @@ -407,19 +407,18 @@ void FTL_log_helper(const unsigned char n, ...) free(arg); } -void format_memory_size(char prefix[2], const unsigned long long int bytes, - double * const formatted) +void format_memory_size(char prefix[2], const uint64_t bytes, double * const formatted) { unsigned int i; *formatted = bytes; // Determine exponent for human-readable display - for(i = 0; i < 7; i++) + const char prefixes[] = { '\0', 'K', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y', 'R', '?' }; + for(i = 0; i < sizeof(prefixes)/sizeof(*prefixes) - 1; i++) { if(*formatted <= 1e3) break; *formatted /= 1e3; } - const char prefixes[8] = { '\0', 'K', 'M', 'G', 'T', 'P', 'E', '?' }; // Chose matching SI prefix prefix[0] = prefixes[i]; prefix[1] = '\0'; diff --git a/src/log.h b/src/log.h index 59495921..215b0bec 100644 --- a/src/log.h +++ b/src/log.h @@ -15,6 +15,8 @@ // enums #include "enums.h" #include +// uint64_t +#include #define DEBUG_ANY 0 #define TIMESTR_SIZE 84 @@ -43,8 +45,7 @@ extern bool only_testing; void clear_debug_flags(void); void init_FTL_log(const char *name); void log_counter_info(void); -void format_memory_size(char prefix[2], unsigned long long int bytes, - double * const formatted); +void format_memory_size(char prefix[2], const uint64_t bytes, double * const formatted); void format_time(char buffer[42], unsigned long seconds, double milliseconds); unsigned int get_year(const time_t timein); const char *get_FTL_version(void); From 5b2dda886cc83c01bf0cd4caa70c94eec68295a5 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 1 Apr 2024 09:43:34 +0200 Subject: [PATCH 047/339] Store message in database as well Signed-off-by: DL6ER --- src/database/message-table.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/database/message-table.c b/src/database/message-table.c index 895dffae..53607050 100644 --- a/src/database/message-table.c +++ b/src/database/message-table.c @@ -1234,7 +1234,7 @@ void log_resource_shortage(const double load, const int nprocs, const int shmem, // Log to database const int rowid = fsdetails != NULL ? - add_message(DISK_MESSAGE_EXTENDED, path, 4, disk, fsdetails->mnt_type, fsdetails->mnt_dir) : + add_message(DISK_MESSAGE_EXTENDED, path, 4, disk, msg, fsdetails->mnt_type, fsdetails->mnt_dir) : add_message(DISK_MESSAGE, path, 2, disk, msg); if(rowid == -1) From d88e52d8c86506adebc16367a3bdfab6e8ad5f4b Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 1 Apr 2024 10:08:45 +0200 Subject: [PATCH 048/339] Improve diagnosis message adding subroutine to not require manually typed in number of arguments and do strict testing against the number of given arguments (instead of crashing if fewer are given and ignoring if more are given) Signed-off-by: DL6ER --- src/database/message-table.c | 107 +++++++++++++++++++++++++++++------ 1 file changed, 89 insertions(+), 18 deletions(-) diff --git a/src/database/message-table.c b/src/database/message-table.c index 53607050..4c70c61d 100644 --- a/src/database/message-table.c +++ b/src/database/message-table.c @@ -32,6 +32,41 @@ // escape_html() #include "webserver/http-common.h" +// Number of arguments in a variadic macro +// Credit: https://stackoverflow.com/a/35693080/2087442 +#define PP_NARG(...) \ + PP_NARG_(__VA_ARGS__,PP_RSEQ_N()) +#define PP_NARG_(...) \ + PP_128TH_ARG(__VA_ARGS__) +#define PP_128TH_ARG( \ + _1, _2, _3, _4, _5, _6, _7, _8, _9,_10, \ + _11,_12,_13,_14,_15,_16,_17,_18,_19,_20, \ + _21,_22,_23,_24,_25,_26,_27,_28,_29,_30, \ + _31,_32,_33,_34,_35,_36,_37,_38,_39,_40, \ + _41,_42,_43,_44,_45,_46,_47,_48,_49,_50, \ + _51,_52,_53,_54,_55,_56,_57,_58,_59,_60, \ + _61,_62,_63,_64,_65,_66,_67,_68,_69,_70, \ + _71,_72,_73,_74,_75,_76,_77,_78,_79,_80, \ + _81,_82,_83,_84,_85,_86,_87,_88,_89,_90, \ + _91,_92,_93,_94,_95,_96,_97,_98,_99,_100, \ + _101,_102,_103,_104,_105,_106,_107,_108,_109,_110, \ + _111,_112,_113,_114,_115,_116,_117,_118,_119,_120, \ + _121,_122,_123,_124,_125,_126,_127,N,...) N +#define PP_RSEQ_N() \ + 127,126,125,124,123,122,121,120, \ + 119,118,117,116,115,114,113,112,111,110, \ + 109,108,107,106,105,104,103,102,101,100, \ + 99,98,97,96,95,94,93,92,91,90, \ + 89,88,87,86,85,84,83,82,81,80, \ + 79,78,77,76,75,74,73,72,71,70, \ + 69,68,67,66,65,64,63,62,61,60, \ + 59,58,57,56,55,54,53,52,51,50, \ + 49,48,47,46,45,44,43,42,41,40, \ + 39,38,37,36,35,34,33,32,31,30, \ + 29,28,27,26,25,24,23,22,21,20, \ + 19,18,17,16,15,14,13,12,11,10, \ + 9,8,7,6,5,4,3,2,1,0 + static const char *get_message_type_str(const enum message_type type) { switch(type) @@ -226,20 +261,56 @@ bool flush_message_table(void) return true; } -static int add_message(const enum message_type type, - const char *message, const int count,...) +static int _add_message(const enum message_type type, + const char *message, const size_t count, ...); +#define add_message(type, message, ...) _add_message(type, message, PP_NARG(__VA_ARGS__), __VA_ARGS__) +#define add_message_no_args(type, message) _add_message(type, message, 0) + +static int _add_message(const enum message_type type, + const char *message, const size_t count,...) { int rowid = -1; // Return early if database is known to be broken if(FTLDBerror()) - return rowid; + return -1; + + // Check if message type is known + if(type >= MAX_MESSAGE) + { + log_err("add_message(type=%u, message=%s) - Invalid message type with %zu arguments", + type, message, count); + return -1; + } + + // Check if number of arguments is valid + // Total number of arguments + if(count > 5) + { + log_err("add_message(type=%u, message=%s) - Too many arguments (%zu), expected at most 5", + type, message, count); + return -1; + } + // No arguments check + if(count == 0 && message_blob_types[type][0] != SQLITE_NULL) + { + log_err("add_message(type=%u, message=%s) - Invalid number of arguments: No arguments passed for message type requiring arguments", + type, message); + return -1; + } + // Non-zero arguments check + else if(count > 1 && message_blob_types[type][count - 2] == SQLITE_NULL) + { + log_err("add_message(type=%u, message=%s) - Invalid number of arguments: Too many (%zu) arguments passed for this message type", + type, message, count); + return -1; + } sqlite3 *db; // Open database connection if((db = dbopen(false, false)) == NULL) { log_err("add_message() - Failed to open DB"); - return rowid; + return -1; } // Ensure there are no duplicates when adding messages @@ -317,7 +388,7 @@ static int add_message(const enum message_type type, va_list ap; va_start(ap, count); - for (int j = 0; j < count; j++) + for (size_t j = 0; j < count; j++) { const unsigned char datatype = message_blob_types[type][j]; switch (datatype) @@ -345,7 +416,7 @@ static int add_message(const enum message_type type, // Bind message to prepared statement if(rc != SQLITE_OK) { - log_err("add_message(type=%u, message=%s) - Failed to bind argument %d (type %u): %s", + log_err("add_message(type=%u, message=%s) - Failed to bind argument %zu (type %u): %s", type, message, 3 + j, datatype, sqlite3_errstr(rc)); sqlite3_reset(stmt); sqlite3_finalize(stmt); @@ -1072,7 +1143,7 @@ void logg_regex_warning(const char *type, const char *warning, const int dbindex return; // Add to database - const int rowid = add_message(REGEX_MESSAGE, warning, 3, type, regex, dbindex); + const int rowid = add_message(REGEX_MESSAGE, warning, type, regex, dbindex); if(rowid == -1) log_err("logg_regex_warning(): Failed to add message to database"); } @@ -1092,7 +1163,7 @@ void logg_subnet_warning(const char *ip, const int matching_count, const char *m log_warn("%s", buf); // Log to database - const int rowid = add_message(SUBNET_MESSAGE, ip, 5, matching_count, names, matching_ids, chosen_match_text, chosen_match_id); + const int rowid = add_message(SUBNET_MESSAGE, ip, matching_count, names, matching_ids, chosen_match_text, chosen_match_id); if(rowid == -1) log_err("logg_subnet_warning(): Failed to add message to database"); @@ -1114,7 +1185,7 @@ void logg_hostname_warning(const char *ip, const char *name, const unsigned int log_warn("%s", buf); // Log to database - const int rowid = add_message(HOSTNAME_MESSAGE, ip, 2, name, (const int)pos); + const int rowid = add_message(HOSTNAME_MESSAGE, ip, name, (const int)pos); if(rowid == -1) log_err("logg_hostname_warning(): Failed to add message to database"); @@ -1130,7 +1201,7 @@ void logg_fatal_dnsmasq_message(const char *message) log_crit("%s", buf); // Log to database - const int rowid = add_message(DNSMASQ_CONFIG_MESSAGE, message, 0); + const int rowid = add_message_no_args(DNSMASQ_CONFIG_MESSAGE, message); if(rowid == -1) log_err("logg_fatal_dnsmasq_message(): Failed to add message to database"); @@ -1148,7 +1219,7 @@ void logg_rate_limit_message(const char *clientIP, const unsigned int rate_limit log_info("%s", buf); // Log to database - const int rowid = add_message(RATE_LIMIT_MESSAGE, clientIP, 3, config.dns.rateLimit.count.v.ui, config.dns.rateLimit.interval.v.ui, turnaround); + const int rowid = add_message(RATE_LIMIT_MESSAGE, clientIP, config.dns.rateLimit.count.v.ui, config.dns.rateLimit.interval.v.ui, turnaround); if(rowid == -1) log_err("logg_rate_limit_message(): Failed to add message to database"); @@ -1164,7 +1235,7 @@ void logg_warn_dnsmasq_message(char *message) log_warn("%s", buf); // Log to database - const int rowid = add_message(DNSMASQ_WARN_MESSAGE, message, 0); + const int rowid = add_message_no_args(DNSMASQ_WARN_MESSAGE, message); if(rowid == -1) log_err("logg_warn_dnsmasq_message(): Failed to add message to database"); @@ -1183,7 +1254,7 @@ void log_resource_shortage(const double load, const int nprocs, const int shmem, log_warn("%s", buf); // Log to database - const int rowid = add_message(LOAD_MESSAGE, "excessive load", 2, load, nprocs); + const int rowid = add_message(LOAD_MESSAGE, "excessive load", load, nprocs); if(rowid == -1) log_err("log_resource_shortage(): Failed to add message to database"); @@ -1196,7 +1267,7 @@ void log_resource_shortage(const double load, const int nprocs, const int shmem, log_warn("%s", buf); // Log to database - const int rowid = add_message(SHMEM_MESSAGE, path, 2, shmem, msg); + const int rowid = add_message(SHMEM_MESSAGE, path, shmem, msg); if(rowid == -1) log_err("log_resource_shortage(): Failed to add message to database"); @@ -1234,8 +1305,8 @@ void log_resource_shortage(const double load, const int nprocs, const int shmem, // Log to database const int rowid = fsdetails != NULL ? - add_message(DISK_MESSAGE_EXTENDED, path, 4, disk, msg, fsdetails->mnt_type, fsdetails->mnt_dir) : - add_message(DISK_MESSAGE, path, 2, disk, msg); + add_message(DISK_MESSAGE_EXTENDED, path, disk, msg, fsdetails->mnt_type, fsdetails->mnt_dir) : + add_message(DISK_MESSAGE, path, disk, msg); if(rowid == -1) log_err("log_resource_shortage(): Failed to add message to database"); @@ -1252,7 +1323,7 @@ void logg_inaccessible_adlist(const int dbindex, const char *address) log_warn("%s", buf); // Log to database - const int rowid = add_message(INACCESSIBLE_ADLIST_MESSAGE, address, 1, dbindex); + const int rowid = add_message(INACCESSIBLE_ADLIST_MESSAGE, address, dbindex); if(rowid == -1) log_err("logg_inaccessible_adlist(): Failed to add message to database"); @@ -1268,7 +1339,7 @@ void log_certificate_domain_mismatch(const char *certfile, const char *domain) log_warn("%s", buf); // Log to database - const int rowid = add_message(CERTIFICATE_DOMAIN_MISMATCH_MESSAGE, certfile, 1, domain); + const int rowid = add_message(CERTIFICATE_DOMAIN_MISMATCH_MESSAGE, certfile, domain); if(rowid == -1) log_err("log_certificate_domain_mismatch(): Failed to add message to database"); From ed41584c92d1be3e3fbeff45fbdb8440c9878b58 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 24 Mar 2024 12:42:49 +0100 Subject: [PATCH 049/339] Add extra logging around network issues (EDE: network error) Signed-off-by: DL6ER --- src/dnsmasq/forward.c | 24 +++++++++++++++++++++++- src/dnsmasq_interface.c | 27 +++++++++++++++++++++++++-- src/dnsmasq_interface.h | 2 ++ 3 files changed, 50 insertions(+), 3 deletions(-) diff --git a/src/dnsmasq/forward.c b/src/dnsmasq/forward.c index 2176c231..f9316a08 100644 --- a/src/dnsmasq/forward.c +++ b/src/dnsmasq/forward.c @@ -105,7 +105,12 @@ int send_from(int fd, int nowild, char *packet, size_t len, #ifdef HAVE_LINUX_NETWORK /* If interface is still in DAD, EINVAL results - ignore that. */ if (errno != EINVAL) - my_syslog(LOG_ERR, _("failed to send packet: %s"), strerror(errno)); + { + my_syslog(LOG_ERR, _("failed to send packet: %s"), strerror(errno)); + /********** Pi-hole modification **********/ + FTL_connection_error("failed to send UDP reply", to); + /******************************************/ + } #endif return 0; } @@ -567,6 +572,12 @@ static int forward_query(int udpfd, union mysockaddr *udpaddr, break; forward->forwardall++; } + /**** Pi-hole modification ****/ + else + { + FTL_connection_error("failed to send UDP request", &srv->addr); + } + /******************************/ } if (++start == last) @@ -2087,12 +2098,19 @@ static ssize_t tcp_talk(int first, int last, int start, unsigned char *packet, data_sent = 1; else if (errno == ETIMEDOUT || errno == EHOSTUNREACH) timedout = 1; + /**** Pi-hole modification ****/ + if (errno != 0) + FTL_connection_error("failed to send TCP(fast-open) packet", &serv->addr); + /******************************/ #endif /* If fastopen failed due to lack of reply, then there's no point in trying again in non-FASTOPEN mode. */ if (timedout || (!data_sent && connect(serv->tcpfd, &serv->addr.sa, sa_len(&serv->addr)) == -1)) { + /**** Pi-hole modification ****/ + FTL_connection_error("failed to send TCP(connect) packet", &serv->addr); + /******************************/ close(serv->tcpfd); serv->tcpfd = -1; continue; @@ -2107,6 +2125,10 @@ static ssize_t tcp_talk(int first, int last, int start, unsigned char *packet, !read_write(serv->tcpfd, &c2, 1, 1) || !read_write(serv->tcpfd, payload, (rsize = (c1 << 8) | c2), 1)) { + /**** Pi-hole modification ****/ + FTL_connection_error("failed to send TCP(read_write) packet", &serv->addr); + /******************************/ + close(serv->tcpfd); serv->tcpfd = -1; /* We get data then EOF, reopen connection to same server, diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c index b8257208..686c3016 100644 --- a/src/dnsmasq_interface.c +++ b/src/dnsmasq_interface.c @@ -72,7 +72,7 @@ static void FTL_forwarded(const unsigned int flags, const char *name, const unio static void FTL_reply(const unsigned int flags, const char *name, const union all_addr *addr, const char* arg, const int id, const char* file, const int line); static void FTL_upstream_error(const union all_addr *addr, const unsigned int flags, const int id, const char* file, const int line); static void FTL_dnssec(const char *result, const union all_addr *addr, const int id, const char* file, const int line); -static void mysockaddr_extract_ip_port(union mysockaddr *server, char ip[ADDRSTRLEN+1], in_port_t *port); +static void mysockaddr_extract_ip_port(const union mysockaddr *server, char ip[ADDRSTRLEN+1], in_port_t *port); static void alladdr_extract_ip(union all_addr *addr, const sa_family_t family, char ip[ADDRSTRLEN+1]); static void check_pihole_PTR(char *domain); #define query_set_dnssec(query, dnssec) _query_set_dnssec(query, dnssec, __FILE__, __LINE__) @@ -1829,7 +1829,7 @@ static void alladdr_extract_ip(union all_addr *addr, const sa_family_t family, c inet_ntop(family, addr, ip, ADDRSTRLEN); } -static void mysockaddr_extract_ip_port(union mysockaddr *server, char ip[ADDRSTRLEN+1], in_port_t *port) +static void mysockaddr_extract_ip_port(const union mysockaddr *server, char ip[ADDRSTRLEN+1], in_port_t *port) { // Extract IP address inet_ntop(server->sa.sa_family, @@ -3507,4 +3507,27 @@ void get_dnsmasq_metrics_obj(cJSON *json) { for (unsigned int i = 0; i < __METRIC_MAX; i++) cJSON_AddNumberToObject(json, get_metric_name(i), daemon->metrics[i]); +} + +void FTL_connection_error(const char *reason, const union mysockaddr *addr) +{ + // Make a private copy of the error + const char *error = strerror(errno); + + if(config.debug.queries.v.b) + { + const int id = daemon->log_display_id; + + // Format the address into a string (if available) + in_port_t port = 0; + char ip[ADDRSTRLEN] = { 0 }; + if(addr != NULL) + mysockaddr_extract_ip_port(addr, ip, &port); + + // Log to FTL.log + log_debug(DEBUG_QUERIES, "Connection error: %s (%s) for %s#%u (ID %d)", reason, error, ip, port, id); + } + + // Log to pihole.log + my_syslog(LOG_ERR, "%s: %s", reason, error); } \ No newline at end of file diff --git a/src/dnsmasq_interface.h b/src/dnsmasq_interface.h index f58851dc..3335fc6a 100644 --- a/src/dnsmasq_interface.h +++ b/src/dnsmasq_interface.h @@ -48,6 +48,8 @@ void FTL_TCP_worker_terminating(bool finished); bool FTL_unlink_DHCP_lease(const char *ipaddr, const char **hint); +void FTL_connection_error(const char *reason, const union mysockaddr *addr); + // defined in src/dnsmasq/cache.c extern char *querystr(char *desc, unsigned short type); From 563b02ccc7a898694243e784ec9d8557e25cf6db Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 2 Apr 2024 13:43:43 +0200 Subject: [PATCH 050/339] Add new CONNECTION_ERROR message to the Pi-hole diagnosis system Signed-off-by: DL6ER --- src/database/message-table.c | 81 +++++++++++++++++++++++++++++++++++- src/database/message-table.h | 1 + src/dnsmasq_interface.c | 43 +++++++++++++------ src/enums.h | 1 + 4 files changed, 113 insertions(+), 13 deletions(-) diff --git a/src/database/message-table.c b/src/database/message-table.c index f6eaef8c..a8f00f7e 100644 --- a/src/database/message-table.c +++ b/src/database/message-table.c @@ -60,6 +60,8 @@ static const char *get_message_type_str(const enum message_type type) return "DISK_EXTENDED"; case CERTIFICATE_DOMAIN_MISMATCH_MESSAGE: return "CERTIFICATE_DOMAIN_MISMATCH"; + case CONNECTION_ERROR_MESSAGE: + return "CONNECTION_ERROR"; case MAX_MESSAGE: default: return "UNKNOWN"; @@ -92,6 +94,8 @@ static enum message_type get_message_type_from_string(const char *typestr) return DISK_MESSAGE_EXTENDED; else if (strcmp(typestr, "CERTIFICATE_DOMAIN_MISMATCH") == 0) return CERTIFICATE_DOMAIN_MISMATCH_MESSAGE; + else if (strcmp(typestr, "CONNECTION_ERROR") == 0) + return CONNECTION_ERROR_MESSAGE; else return MAX_MESSAGE; } @@ -183,6 +187,14 @@ static unsigned char message_blob_types[MAX_MESSAGE][5] = SQLITE_NULL, // not used SQLITE_NULL, // not used SQLITE_NULL // not used + }, + { + // CONNECTION_ERROR_MESSAGE: The message column contains the server address + SQLITE_TEXT, // reason + SQLITE_TEXT, // error message + SQLITE_NULL, // not used + SQLITE_NULL, // not used + SQLITE_NULL // not used } }; // Create message table in the database @@ -710,6 +722,40 @@ static void format_certificate_domain_mismatch(char *plain, const int sizeof_pla free(escaped_domain); } +static void format_connection_error(char *plain, const int sizeof_plain, char *html, const int sizeof_html, + const char *server, const char *reason, const char *error) +{ + if(snprintf(plain, sizeof_plain, "Connection error (%s): %s (%s)", server, reason, error) > sizeof_plain) + log_warn("format_connection_error(): Buffer too small to hold plain message, warning truncated"); + + // Return early if HTML text is not required + if(sizeof_html < 1 || html == NULL) + return; + + char *escaped_reason = escape_html(reason); + char *escaped_error = escape_html(error); + char *escaped_server = escape_html(server); + + // Return early if memory allocation failed + if(escaped_reason == NULL || escaped_error == NULL || escaped_server == NULL) + { + if(escaped_reason != NULL) + free(escaped_reason); + if(escaped_error != NULL) + free(escaped_error); + if(escaped_server != NULL) + free(escaped_server); + return; + } + + if(snprintf(html, sizeof_html, "Connection error (%s): %s (%s)", server, reason, error) > sizeof_html) + log_warn("format_connection_error(): Buffer too small to hold HTML message, warning truncated"); + + free(escaped_reason); + free(escaped_error); + free(escaped_server); +} + int count_messages(const bool filter_dnsmasq_warnings) { int count = 0; @@ -798,7 +844,7 @@ bool format_messages(cJSON *array) // Generate messages char plain[1024] = { 0 }, html[2048] = { 0 }; - const int mtype = get_message_type_from_string(mtypestr); + const enum message_type mtype = get_message_type_from_string(mtypestr); switch(mtype) { case REGEX_MESSAGE: @@ -944,6 +990,23 @@ bool format_messages(cJSON *array) break; } + + case CONNECTION_ERROR_MESSAGE: + { + const char *server = (const char*)sqlite3_column_text(stmt, 3); + const char *reason = (const char*)sqlite3_column_text(stmt, 4); + const char *error = (const char*)sqlite3_column_text(stmt, 5); + + format_connection_error(plain, sizeof(plain), html, sizeof(html), + server, reason, error); + + break; + } + + case MAX_MESSAGE: // Fall through + default: + log_warn("format_messages() - Unknown message type: %s", mtypestr); + break; } // Add the plain message @@ -1183,3 +1246,19 @@ void log_certificate_domain_mismatch(const char *certfile, const char *domain) if(rowid == -1) log_err("log_certificate_domain_mismatch(): Failed to add message to database"); } + +void log_connection_error(const char *server, const char *reason, const char *error) +{ + // Create message + char buf[2048]; + format_connection_error(buf, sizeof(buf), NULL, 0, server, reason, error); + + // Log to FTL.log + log_warn("%s", buf); + + // Log to database + const int rowid = add_message(CONNECTION_ERROR_MESSAGE, server, 2, reason, error); + + if(rowid == -1) + log_err("logg_connection_error(): Failed to add message to database"); +} diff --git a/src/database/message-table.h b/src/database/message-table.h index 14956bf5..d92bbe8d 100644 --- a/src/database/message-table.h +++ b/src/database/message-table.h @@ -29,5 +29,6 @@ void logg_warn_dnsmasq_message(char *message); void log_resource_shortage(const double load, const int nprocs, const int shmem, const int disk, const char *path, const char *msg); void logg_inaccessible_adlist(const int dbindex, const char *address); void log_certificate_domain_mismatch(const char *certfile, const char *domain); +void log_connection_error(const char *server, const char *reason, const char *error); #endif //MESSAGETABLE_H diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c index 686c3016..1691056a 100644 --- a/src/dnsmasq_interface.c +++ b/src/dnsmasq_interface.c @@ -3514,20 +3514,39 @@ void FTL_connection_error(const char *reason, const union mysockaddr *addr) // Make a private copy of the error const char *error = strerror(errno); - if(config.debug.queries.v.b) - { - const int id = daemon->log_display_id; + // Format the address into a string (if available) + in_port_t port = 0; + char ip[ADDRSTRLEN + 1] = { 0 }; + if(addr != NULL) + mysockaddr_extract_ip_port(addr, ip, &port); - // Format the address into a string (if available) - in_port_t port = 0; - char ip[ADDRSTRLEN] = { 0 }; - if(addr != NULL) - mysockaddr_extract_ip_port(addr, ip, &port); - - // Log to FTL.log - log_debug(DEBUG_QUERIES, "Connection error: %s (%s) for %s#%u (ID %d)", reason, error, ip, port, id); - } + // Log to FTL.log + const int id = daemon->log_display_id; + log_debug(DEBUG_QUERIES, "Connection error (%s#%u, ID %d): %s (%s)", ip, port, id, reason, error); // Log to pihole.log my_syslog(LOG_ERR, "%s: %s", reason, error); + + // Add to Pi-hole diagnostics but do not add messages more often than + // once every five seconds to avoid hammering the database with errors + // on continuously failing connections + static time_t last = 0; + if(time(NULL) - last > 5) + { + last = time(NULL); + char *server = NULL; + if(ip[0] != '\0') + { + const size_t len = strlen(ip) + 6; + server = calloc(len, sizeof(char)); + if(server != NULL) + { + snprintf(server, len, "%s#%u", ip, port); + server[len - 1] = '\0'; + } + } + log_connection_error(server, reason, error); + if(server != NULL) + free(server); + } } \ No newline at end of file diff --git a/src/enums.h b/src/enums.h index dac8ff8d..ab09498a 100644 --- a/src/enums.h +++ b/src/enums.h @@ -273,6 +273,7 @@ enum message_type { INACCESSIBLE_ADLIST_MESSAGE, DISK_MESSAGE_EXTENDED, CERTIFICATE_DOMAIN_MISMATCH_MESSAGE, + CONNECTION_ERROR_MESSAGE, MAX_MESSAGE, } __attribute__ ((packed)); From f2a7662e9504a7201693327da7f66cce7a2f08f7 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 2 Apr 2024 21:34:55 +0200 Subject: [PATCH 051/339] Be more verbose in which tables are imported during teleporter importing Signed-off-by: DL6ER --- src/api/docs/content/specs/teleporter.yaml | 4 ++- src/zip/teleporter.c | 38 +++++++++++++++++++--- test/api/libs/responseVerifyer.py | 4 ++- test/test_suite.bats | 12 +++++-- 4 files changed, 49 insertions(+), 9 deletions(-) diff --git a/src/api/docs/content/specs/teleporter.yaml b/src/api/docs/content/specs/teleporter.yaml index 5a52fbf0..cc25e4ca 100644 --- a/src/api/docs/content/specs/teleporter.yaml +++ b/src/api/docs/content/specs/teleporter.yaml @@ -150,4 +150,6 @@ components: value: processed: - etc/pihole/pihole.toml - - etc/pihole/gravity.db + - etc/pihole/gravity.db->group + - etc/pihole/gravity.db->adlist + - etc/pihole/gravity.db->adlist_by_group diff --git a/src/zip/teleporter.c b/src/zip/teleporter.c index 690b6f2a..26480ca2 100644 --- a/src/zip/teleporter.c +++ b/src/zip/teleporter.c @@ -584,6 +584,8 @@ const char *read_teleporter_zip(uint8_t *buffer, const size_t buflen, char * con file_stat.m_comment, (unsigned long)file_stat.m_time); // Process file + const char *import_tables[ArraySize(gravity_tables)] = { NULL }; + size_t num_tables = 0u; // Is this "etc/pihole/pihole.toml" ? if(strcmp(file_stat.m_filename, extract_files[0]) == 0) { @@ -637,8 +639,6 @@ const char *read_teleporter_zip(uint8_t *buffer, const size_t buflen, char * con continue; } - const char *import_tables[ArraySize(gravity_tables)] = { NULL }; - size_t num_tables = 0u; if(import == NULL) { // Import all tables @@ -658,11 +658,13 @@ const char *read_teleporter_zip(uint8_t *buffer, const size_t buflen, char * con continue; } - // Import selected tables + // Import selected tables from import.gravity object for(size_t j = 0; j < ArraySize(gravity_tables); j++) { - if(JSON_KEY_TRUE(import, gravity_tables[j])) + if(JSON_KEY_TRUE(import_gravity, gravity_tables[j])) import_tables[num_tables++] = gravity_tables[j]; + else + log_info("Ignoring table %s in %s (not in import list)", gravity_tables[j], file_stat.m_filename); } } @@ -676,10 +678,38 @@ const char *read_teleporter_zip(uint8_t *buffer, const size_t buflen, char * con return err; } log_debug(DEBUG_CONFIG, "Imported database: %s", file_stat.m_filename); + + // Add filename of processed files to JSON array + for(unsigned j = 0; j < num_tables; j++) + { + const size_t len = strlen(file_stat.m_filename) + 3 + strlen(import_tables[j]); + char *tablename = calloc(len, sizeof(char)); + if(tablename == NULL) + { + log_err("Failed to allocate memory for table name"); + free(ptr); + continue; + } + + // Create imported pseudo file name in the + // format "filename->table" and add it to the + // JSON array + snprintf(tablename, len, "%s->%s", file_stat.m_filename, import_tables[j]); + if(imported_files != NULL && !cJSON_AddItemToArray(imported_files, cJSON_CreateString(tablename))) + log_warn("Failed to add table %s to JSON array", tablename); + free(tablename); + } + + // Free allocated memory and skip to next file without + // adding it to the JSON array again below + free(ptr); + continue; } else { log_warn("Ignoring file %s in Teleporter archive", file_stat.m_filename); + + // Free allocated memory and skip to next file free(ptr); continue; } diff --git a/test/api/libs/responseVerifyer.py b/test/api/libs/responseVerifyer.py index ab6878b2..238ca24c 100644 --- a/test/api/libs/responseVerifyer.py +++ b/test/api/libs/responseVerifyer.py @@ -11,6 +11,7 @@ import io import ipaddress +import json import random import zipfile from libs.openAPI import openApi @@ -23,7 +24,7 @@ class ResponseVerifyer(): # Translate between OpenAPI and Python types YAML_TYPES = { "string": [str], "integer": [int], "number": [int, float], "boolean": [bool], "array": [list] } TELEPORTER_FILES_EXPORT = ["etc/pihole/gravity.db", "etc/pihole/pihole.toml", "etc/pihole/pihole-FTL.db", "etc/hosts"] - TELEPORTER_FILES_IMPORT = ['etc/pihole/pihole.toml', 'etc/pihole/dhcp.leases', 'etc/pihole/gravity.db'] + TELEPORTER_FILES_IMPORT = ['etc/pihole/pihole.toml', 'etc/pihole/dhcp.leases', 'etc/pihole/gravity.db->group', 'etc/pihole/gravity.db->adlist', 'etc/pihole/gravity.db->adlist_by_group', 'etc/pihole/gravity.db->domainlist', 'etc/pihole/gravity.db->domainlist_by_group', 'etc/pihole/gravity.db->client', 'etc/pihole/gravity.db->client_by_group' ] auth_method = "?" teleporter_archive = None @@ -229,6 +230,7 @@ class ResponseVerifyer(): for expected_file in self.TELEPORTER_FILES_IMPORT: if expected_file not in FTLresponse['files']: self.errors.append("File " + expected_file + " is missing in FTL response") + self.errors.append(json.dumps(FTLresponse['files'], indent=4)) return self.errors diff --git a/test/test_suite.bats b/test/test_suite.bats index 3c60f69a..26b65709 100644 --- a/test/test_suite.bats +++ b/test/test_suite.bats @@ -1788,9 +1788,15 @@ # [[ $status == 0 ]] run bash -c "./pihole-FTL --teleporter ${filename}" printf "%s\n" "${lines[@]}" - [[ "${lines[-3]}" == "Imported etc/pihole/pihole.toml" ]] - [[ "${lines[-2]}" == "Imported etc/pihole/dhcp.leases" ]] - [[ "${lines[-1]}" == "Imported etc/pihole/gravity.db" ]] + [[ "${lines[-9]}" == "Imported etc/pihole/pihole.toml" ]] + [[ "${lines[-8]}" == "Imported etc/pihole/dhcp.leases" ]] + [[ "${lines[-7]}" == "Imported etc/pihole/gravity.db->group" ]] + [[ "${lines[-6]}" == "Imported etc/pihole/gravity.db->adlist" ]] + [[ "${lines[-5]}" == "Imported etc/pihole/gravity.db->adlist_by_group" ]] + [[ "${lines[-4]}" == "Imported etc/pihole/gravity.db->domainlist" ]] + [[ "${lines[-3]}" == "Imported etc/pihole/gravity.db->domainlist_by_group" ]] + [[ "${lines[-2]}" == "Imported etc/pihole/gravity.db->client" ]] + [[ "${lines[-1]}" == "Imported etc/pihole/gravity.db->client_by_group" ]] [[ $status == 0 ]] run bash -c "rm ${filename}" } From c7ce5553828661f02373175d2a14d1416f50c497 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 16 Apr 2024 08:22:53 +0200 Subject: [PATCH 052/339] Update embedded SQLite3 to 3.45.3 Signed-off-by: DL6ER --- src/database/shell.c | 33 +++++-- src/database/sqlite3.c | 205 ++++++++++++++++++++++++++++++++--------- src/database/sqlite3.h | 23 ++++- 3 files changed, 210 insertions(+), 51 deletions(-) diff --git a/src/database/shell.c b/src/database/shell.c index 5550f010..3de2f5d0 100644 --- a/src/database/shell.c +++ b/src/database/shell.c @@ -14755,6 +14755,15 @@ static void dbdataValue( } } +/* This macro is a copy of the MX_CELL() macro in the SQLite core. Given +** a page-size, it returns the maximum number of cells that may be present +** on the page. */ +#define DBDATA_MX_CELL(pgsz) ((pgsz-8)/6) + +/* Maximum number of fields that may appear in a single record. This is +** the "hard-limit", according to comments in sqliteLimit.h. */ +#define DBDATA_MX_FIELD 32676 + /* ** Move an sqlite_dbdata or sqlite_dbptr cursor to the next entry. */ @@ -14783,6 +14792,9 @@ static int dbdataNext(sqlite3_vtab_cursor *pCursor){ assert( iOff+3+2<=pCsr->nPage ); pCsr->iCell = pTab->bPtr ? -2 : 0; pCsr->nCell = get_uint16(&pCsr->aPage[iOff+3]); + if( pCsr->nCell>DBDATA_MX_CELL(pCsr->nPage) ){ + pCsr->nCell = DBDATA_MX_CELL(pCsr->nPage); + } } if( pTab->bPtr ){ @@ -14827,19 +14839,19 @@ static int dbdataNext(sqlite3_vtab_cursor *pCursor){ if( pCsr->iCell>=pCsr->nCell ){ bNextPage = 1; }else{ + int iCellPtr = iOff + 8 + nPointer + pCsr->iCell*2; - iOff += 8 + nPointer + pCsr->iCell*2; - if( iOff>pCsr->nPage ){ + if( iCellPtr>pCsr->nPage ){ bNextPage = 1; }else{ - iOff = get_uint16(&pCsr->aPage[iOff]); + iOff = get_uint16(&pCsr->aPage[iCellPtr]); } /* For an interior node cell, skip past the child-page number */ iOff += nPointer; /* Load the "byte of payload including overflow" field */ - if( bNextPage || iOff>pCsr->nPage ){ + if( bNextPage || iOff>pCsr->nPage || iOff<=iCellPtr ){ bNextPage = 1; }else{ iOff += dbdataGetVarintU32(&pCsr->aPage[iOff], &nPayload); @@ -14922,7 +14934,9 @@ static int dbdataNext(sqlite3_vtab_cursor *pCursor){ pCsr->iField++; if( pCsr->iField>0 ){ sqlite3_int64 iType; - if( pCsr->pHdrPtr>&pCsr->pRec[pCsr->nRec] ){ + if( pCsr->pHdrPtr>=&pCsr->pRec[pCsr->nRec] + || pCsr->iField>=DBDATA_MX_FIELD + ){ bNextPage = 1; }else{ int szField = 0; @@ -16410,7 +16424,7 @@ static int recoverWriteSchema1(sqlite3_recover *p){ if( bTable && !bVirtual ){ if( SQLITE_ROW==sqlite3_step(pTblname) ){ const char *zTbl = (const char*)sqlite3_column_text(pTblname, 0); - recoverAddTable(p, zTbl, iRoot); + if( zTbl ) recoverAddTable(p, zTbl, iRoot); } recoverReset(p, pTblname); } @@ -28773,6 +28787,7 @@ static const char zOptions[] = " -newline SEP set output row separator. Default: '\\n'\n" " -nofollow refuse to open symbolic links to database files\n" " -nonce STRING set the safe-mode escape nonce\n" + " -no-rowid-in-view Disable rowid-in-view using sqlite3_config()\n" " -nullvalue TEXT set text string for NULL values. Default ''\n" " -pagecache SIZE N use N slots of SZ bytes each for page cache memory\n" " -pcachetrace trace all page cache operations\n" @@ -29063,6 +29078,10 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ stdin_is_interactive = 0; }else if( cli_strcmp(z,"-utf8")==0 ){ }else if( cli_strcmp(z,"-no-utf8")==0 ){ + }else if( cli_strcmp(z,"-no-rowid-in-view")==0 ){ + int val = 0; + sqlite3_config(SQLITE_CONFIG_ROWID_IN_VIEW, &val); + assert( val==0 ); }else if( cli_strcmp(z,"-heap")==0 ){ #if defined(SQLITE_ENABLE_MEMSYS3) || defined(SQLITE_ENABLE_MEMSYS5) const char *zSize; @@ -29338,6 +29357,8 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ /* already handled */ }else if( cli_strcmp(z,"-no-utf8")==0 ){ /* already handled */ + }else if( cli_strcmp(z,"-no-rowid-in-view")==0 ){ + /* already handled */ }else if( cli_strcmp(z,"-heap")==0 ){ i++; }else if( cli_strcmp(z,"-pagecache")==0 ){ diff --git a/src/database/sqlite3.c b/src/database/sqlite3.c index d6c4d244..0d22c717 100644 --- a/src/database/sqlite3.c +++ b/src/database/sqlite3.c @@ -1,6 +1,6 @@ /****************************************************************************** ** This file is an amalgamation of many separate C source files from SQLite -** version 3.45.2. By combining all the individual C code files into this +** version 3.45.3. By combining all the individual C code files into this ** single large file, the entire code can be compiled as a single translation ** unit. This allows many compilers to do optimizations that would not be ** possible if the files were compiled separately. Performance improvements @@ -18,7 +18,7 @@ ** separate file. This file contains only code for the core SQLite library. ** ** The content in this amalgamation comes from Fossil check-in -** d8cd6d49b46a395b13955387d05e9e1a2a47. +** 8653b758870e6ef0c98d46b3ace27849054a. */ #define SQLITE_CORE 1 #define SQLITE_AMALGAMATION 1 @@ -459,9 +459,9 @@ extern "C" { ** [sqlite3_libversion_number()], [sqlite3_sourceid()], ** [sqlite_version()] and [sqlite_source_id()]. */ -#define SQLITE_VERSION "3.45.2" -#define SQLITE_VERSION_NUMBER 3045002 -#define SQLITE_SOURCE_ID "2024-03-12 11:06:23 d8cd6d49b46a395b13955387d05e9e1a2a47e54fb99f3c9b59835bbefad6af77" +#define SQLITE_VERSION "3.45.3" +#define SQLITE_VERSION_NUMBER 3045003 +#define SQLITE_SOURCE_ID "2024-04-15 13:34:05 8653b758870e6ef0c98d46b3ace27849054af85da891eb121e9aaa537f1e8355" /* ** CAPI3REF: Run-Time Library Version Numbers @@ -2456,6 +2456,22 @@ struct sqlite3_mem_methods { ** configuration setting is never used, then the default maximum is determined ** by the [SQLITE_MEMDB_DEFAULT_MAXSIZE] compile-time option. If that ** compile-time option is not set, then the default maximum is 1073741824. +** +** [[SQLITE_CONFIG_ROWID_IN_VIEW]] +**
    SQLITE_CONFIG_ROWID_IN_VIEW +**
    The SQLITE_CONFIG_ROWID_IN_VIEW option enables or disables the ability +** for VIEWs to have a ROWID. The capability can only be enabled if SQLite is +** compiled with -DSQLITE_ALLOW_ROWID_IN_VIEW, in which case the capability +** defaults to on. This configuration option queries the current setting or +** changes the setting to off or on. The argument is a pointer to an integer. +** If that integer initially holds a value of 1, then the ability for VIEWs to +** have ROWIDs is activated. If the integer initially holds zero, then the +** ability is deactivated. Any other initial value for the integer leaves the +** setting unchanged. After changes, if any, the integer is written with +** a 1 or 0, if the ability for VIEWs to have ROWIDs is on or off. If SQLite +** is compiled without -DSQLITE_ALLOW_ROWID_IN_VIEW (which is the usual and +** recommended case) then the integer is always filled with zero, regardless +** if its initial value. ** */ #define SQLITE_CONFIG_SINGLETHREAD 1 /* nil */ @@ -2487,6 +2503,7 @@ struct sqlite3_mem_methods { #define SQLITE_CONFIG_SMALL_MALLOC 27 /* boolean */ #define SQLITE_CONFIG_SORTERREF_SIZE 28 /* int nByte */ #define SQLITE_CONFIG_MEMDB_MAXSIZE 29 /* sqlite3_int64 */ +#define SQLITE_CONFIG_ROWID_IN_VIEW 30 /* int* */ /* ** CAPI3REF: Database Connection Configuration Options @@ -18430,6 +18447,15 @@ struct Table { #define HasRowid(X) (((X)->tabFlags & TF_WithoutRowid)==0) #define VisibleRowid(X) (((X)->tabFlags & TF_NoVisibleRowid)==0) +/* Macro is true if the SQLITE_ALLOW_ROWID_IN_VIEW (mis-)feature is +** available. By default, this macro is false +*/ +#ifndef SQLITE_ALLOW_ROWID_IN_VIEW +# define ViewCanHaveRowid 0 +#else +# define ViewCanHaveRowid (sqlite3Config.mNoVisibleRowid==0) +#endif + /* ** Each foreign key constraint is an instance of the following structure. ** @@ -20144,6 +20170,11 @@ struct Sqlite3Config { #endif #ifndef SQLITE_UNTESTABLE int (*xTestCallback)(int); /* Invoked by sqlite3FaultSim() */ +#endif +#ifdef SQLITE_ALLOW_ROWID_IN_VIEW + u32 mNoVisibleRowid; /* TF_NoVisibleRowid if the ROWID_IN_VIEW + ** feature is disabled. 0 if rowids can + ** occur in views. */ #endif int bLocaltimeFault; /* True to fail localtime() calls */ int (*xAltLocaltime)(const void*,void*); /* Alternative localtime() routine */ @@ -20600,10 +20631,13 @@ SQLITE_PRIVATE void sqlite3MutexWarnOnContention(sqlite3_mutex*); # define EXP754 (((u64)0x7ff)<<52) # define MAN754 ((((u64)1)<<52)-1) # define IsNaN(X) (((X)&EXP754)==EXP754 && ((X)&MAN754)!=0) +# define IsOvfl(X) (((X)&EXP754)==EXP754) SQLITE_PRIVATE int sqlite3IsNaN(double); +SQLITE_PRIVATE int sqlite3IsOverflow(double); #else -# define IsNaN(X) 0 -# define sqlite3IsNaN(X) 0 +# define IsNaN(X) 0 +# define sqlite3IsNaN(X) 0 +# define sqlite3IsOVerflow(X) 0 #endif /* @@ -21839,6 +21873,9 @@ static const char * const sqlite3azCompileOpt[] = { "ALLOW_COVERING_INDEX_SCAN=" CTIMEOPT_VAL(SQLITE_ALLOW_COVERING_INDEX_SCAN), # endif #endif +#ifdef SQLITE_ALLOW_ROWID_IN_VIEW + "ALLOW_ROWID_IN_VIEW", +#endif #ifdef SQLITE_ALLOW_URI_AUTHORITY "ALLOW_URI_AUTHORITY", #endif @@ -22858,6 +22895,9 @@ SQLITE_PRIVATE SQLITE_WSD struct Sqlite3Config sqlite3Config = { #endif #ifndef SQLITE_UNTESTABLE 0, /* xTestCallback */ +#endif +#ifdef SQLITE_ALLOW_ROWID_IN_VIEW + 0, /* mNoVisibleRowid. 0 == allow rowid-in-view */ #endif 0, /* bLocaltimeFault */ 0, /* xAltLocaltime */ @@ -34646,6 +34686,19 @@ SQLITE_PRIVATE int sqlite3IsNaN(double x){ } #endif /* SQLITE_OMIT_FLOATING_POINT */ +#ifndef SQLITE_OMIT_FLOATING_POINT +/* +** Return true if the floating point value is NaN or +Inf or -Inf. +*/ +SQLITE_PRIVATE int sqlite3IsOverflow(double x){ + int rc; /* The value return */ + u64 y; + memcpy(&y,&x,sizeof(y)); + rc = IsOvfl(y); + return rc; +} +#endif /* SQLITE_OMIT_FLOATING_POINT */ + /* ** Compute a string length that is limited to what can be stored in ** lower 30 bits of a 32-bit signed integer. @@ -63802,7 +63855,7 @@ SQLITE_PRIVATE sqlite3_file *sqlite3PagerFile(Pager *pPager){ ** This will be either the rollback journal or the WAL file. */ SQLITE_PRIVATE sqlite3_file *sqlite3PagerJrnlFile(Pager *pPager){ -#if SQLITE_OMIT_WAL +#ifdef SQLITE_OMIT_WAL return pPager->jfd; #else return pPager->pWal ? sqlite3WalFile(pPager->pWal) : pPager->jfd; @@ -79619,7 +79672,7 @@ SQLITE_PRIVATE int sqlite3BtreeInsert( }else if( loc<0 && pPage->nCell>0 ){ assert( pPage->leaf ); idx = ++pCur->ix; - pCur->curFlags &= ~BTCF_ValidNKey; + pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl); }else{ assert( pPage->leaf ); } @@ -79649,7 +79702,7 @@ SQLITE_PRIVATE int sqlite3BtreeInsert( */ if( pPage->nOverflow ){ assert( rc==SQLITE_OK ); - pCur->curFlags &= ~(BTCF_ValidNKey); + pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl); rc = balance(pCur); /* Must make sure nOverflow is reset to zero even if the balance() @@ -106656,8 +106709,37 @@ static int lookupName( } } if( 0==cnt && VisibleRowid(pTab) ){ + /* pTab is a potential ROWID match. Keep track of it and match + ** the ROWID later if that seems appropriate. (Search for "cntTab" + ** to find related code.) Only allow a ROWID match if there is + ** a single ROWID match candidate. + */ +#ifdef SQLITE_ALLOW_ROWID_IN_VIEW + /* In SQLITE_ALLOW_ROWID_IN_VIEW mode, allow a ROWID match + ** if there is a single VIEW candidate or if there is a single + ** non-VIEW candidate plus multiple VIEW candidates. In other + ** words non-VIEW candidate terms take precedence over VIEWs. + */ + if( cntTab==0 + || (cntTab==1 + && ALWAYS(pMatch!=0) + && ALWAYS(pMatch->pTab!=0) + && (pMatch->pTab->tabFlags & TF_Ephemeral)!=0 + && (pTab->tabFlags & TF_Ephemeral)==0) + ){ + cntTab = 1; + pMatch = pItem; + }else{ + cntTab++; + } +#else + /* The (much more common) non-SQLITE_ALLOW_ROWID_IN_VIEW case is + ** simpler since we require exactly one candidate, which will + ** always be a non-VIEW + */ cntTab++; pMatch = pItem; +#endif } } if( pMatch ){ @@ -106783,13 +106865,13 @@ static int lookupName( ** Perhaps the name is a reference to the ROWID */ if( cnt==0 - && cntTab==1 + && cntTab>=1 && pMatch && (pNC->ncFlags & (NC_IdxExpr|NC_GenCol))==0 && sqlite3IsRowid(zCol) && ALWAYS(VisibleRowid(pMatch->pTab) || pMatch->fg.isNestedFrom) ){ - cnt = 1; + cnt = cntTab; if( pMatch->fg.isNestedFrom==0 ) pExpr->iColumn = -1; pExpr->affExpr = SQLITE_AFF_INTEGER; } @@ -108647,9 +108729,10 @@ SQLITE_PRIVATE Expr *sqlite3ExprSkipCollateAndLikely(Expr *pExpr){ assert( pExpr->x.pList->nExpr>0 ); assert( pExpr->op==TK_FUNCTION ); pExpr = pExpr->x.pList->a[0].pExpr; - }else{ - assert( pExpr->op==TK_COLLATE ); + }else if( pExpr->op==TK_COLLATE ){ pExpr = pExpr->pLeft; + }else{ + break; } } return pExpr; @@ -111168,9 +111251,12 @@ SQLITE_PRIVATE int sqlite3ExprCanBeNull(const Expr *p){ return 0; case TK_COLUMN: assert( ExprUseYTab(p) ); - return ExprHasProperty(p, EP_CanBeNull) || - NEVER(p->y.pTab==0) || /* Reference to column of index on expr */ - (p->iColumn>=0 + return ExprHasProperty(p, EP_CanBeNull) + || NEVER(p->y.pTab==0) /* Reference to column of index on expr */ +#ifdef SQLITE_ALLOW_ROWID_IN_VIEW + || (p->iColumn==XN_ROWID && IsView(p->y.pTab)) +#endif + || (p->iColumn>=0 && p->y.pTab->aCol!=0 /* Possible due to prior error */ && ALWAYS(p->iColumny.pTab->nCol) && p->y.pTab->aCol[p->iColumn].notNull==0); @@ -123661,9 +123747,12 @@ SQLITE_PRIVATE void sqlite3CreateView( ** on a view, even though views do not have rowids. The following flag ** setting fixes this problem. But the fix can be disabled by compiling ** with -DSQLITE_ALLOW_ROWID_IN_VIEW in case there are legacy apps that - ** depend upon the old buggy behavior. */ -#ifndef SQLITE_ALLOW_ROWID_IN_VIEW - p->tabFlags |= TF_NoVisibleRowid; + ** depend upon the old buggy behavior. The ability can also be toggled + ** using sqlite3_config(SQLITE_CONFIG_ROWID_IN_VIEW,...) */ +#ifdef SQLITE_ALLOW_ROWID_IN_VIEW + p->tabFlags |= sqlite3Config.mNoVisibleRowid; /* Optional. Allow by default */ +#else + p->tabFlags |= TF_NoVisibleRowid; /* Never allow rowid in view */ #endif sqlite3TwoPartName(pParse, pName1, pName2, &pName); @@ -129827,7 +129916,7 @@ static void sumFinalize(sqlite3_context *context){ if( p->approx ){ if( p->ovrfl ){ sqlite3_result_error(context,"integer overflow",-1); - }else if( !sqlite3IsNaN(p->rErr) ){ + }else if( !sqlite3IsOverflow(p->rErr) ){ sqlite3_result_double(context, p->rSum+p->rErr); }else{ sqlite3_result_double(context, p->rSum); @@ -129844,7 +129933,7 @@ static void avgFinalize(sqlite3_context *context){ double r; if( p->approx ){ r = p->rSum; - if( !sqlite3IsNaN(p->rErr) ) r += p->rErr; + if( !sqlite3IsOverflow(p->rErr) ) r += p->rErr; }else{ r = (double)(p->iSum); } @@ -129858,7 +129947,7 @@ static void totalFinalize(sqlite3_context *context){ if( p ){ if( p->approx ){ r = p->rSum; - if( !sqlite3IsNaN(p->rErr) ) r += p->rErr; + if( !sqlite3IsOverflow(p->rErr) ) r += p->rErr; }else{ r = (double)(p->iSum); } @@ -135156,7 +135245,10 @@ static int xferOptimization( } } #ifndef SQLITE_OMIT_CHECK - if( pDest->pCheck && sqlite3ExprListCompare(pSrc->pCheck,pDest->pCheck,-1) ){ + if( pDest->pCheck + && (db->mDbFlags & DBFLAG_Vacuum)==0 + && sqlite3ExprListCompare(pSrc->pCheck,pDest->pCheck,-1) + ){ return 0; /* Tables have different CHECK constraints. Ticket #2252 */ } #endif @@ -140557,7 +140649,11 @@ static int pragmaVtabBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ j = seen[0]-1; pIdxInfo->aConstraintUsage[j].argvIndex = 1; pIdxInfo->aConstraintUsage[j].omit = 1; - if( seen[1]==0 ) return SQLITE_OK; + if( seen[1]==0 ){ + pIdxInfo->estimatedCost = (double)1000; + pIdxInfo->estimatedRows = 1000; + return SQLITE_OK; + } pIdxInfo->estimatedCost = (double)20; pIdxInfo->estimatedRows = 20; j = seen[1]-1; @@ -143784,11 +143880,7 @@ static const char *columnTypeImpl( ** data for the result-set column of the sub-select. */ if( iColpEList->nExpr -#ifdef SQLITE_ALLOW_ROWID_IN_VIEW - && iCol>=0 -#else - && ALWAYS(iCol>=0) -#endif + && (!ViewCanHaveRowid || iCol>=0) ){ /* If iCol is less than zero, then the expression requests the ** rowid of the sub-select or view. This expression is legal (see @@ -146963,6 +147055,10 @@ static int pushDownWindowCheck(Parse *pParse, Select *pSubq, Expr *pExpr){ ** ** (11) The subquery is not a VALUES clause ** +** (12) The WHERE clause is not "rowid ISNULL" or the equivalent. This +** case only comes up if SQLite is compiled using +** SQLITE_ALLOW_ROWID_IN_VIEW. +** ** Return 0 if no changes are made and non-zero if one or more WHERE clause ** terms are duplicated into the subquery. */ @@ -147073,6 +147169,18 @@ static int pushDownWhereTerms( } #endif +#ifdef SQLITE_ALLOW_ROWID_IN_VIEW + if( ViewCanHaveRowid && (pWhere->op==TK_ISNULL || pWhere->op==TK_NOTNULL) ){ + Expr *pLeft = pWhere->pLeft; + if( ALWAYS(pLeft) + && pLeft->op==TK_COLUMN + && pLeft->iColumn < 0 + ){ + return 0; /* Restriction (12) */ + } + } +#endif + if( sqlite3ExprIsSingleTableConstraint(pWhere, pSrcList, iSrc) ){ nChng++; pSubq->selFlags |= SF_PushDown; @@ -147700,12 +147808,14 @@ SQLITE_PRIVATE int sqlite3ExpandSubquery(Parse *pParse, SrcItem *pFrom){ while( pSel->pPrior ){ pSel = pSel->pPrior; } sqlite3ColumnsFromExprList(pParse, pSel->pEList,&pTab->nCol,&pTab->aCol); pTab->iPKey = -1; + pTab->eTabType = TABTYP_VIEW; pTab->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) ); #ifndef SQLITE_ALLOW_ROWID_IN_VIEW /* The usual case - do not allow ROWID on a subquery */ pTab->tabFlags |= TF_Ephemeral | TF_NoVisibleRowid; #else - pTab->tabFlags |= TF_Ephemeral; /* Legacy compatibility mode */ + /* Legacy compatibility mode */ + pTab->tabFlags |= TF_Ephemeral | sqlite3Config.mNoVisibleRowid; #endif return pParse->nErr ? SQLITE_ERROR : SQLITE_OK; } @@ -147973,7 +148083,7 @@ static int selectExpander(Walker *pWalker, Select *p){ pNestedFrom = pFrom->pSelect->pEList; assert( pNestedFrom!=0 ); assert( pNestedFrom->nExpr==pTab->nCol ); - assert( VisibleRowid(pTab)==0 ); + assert( VisibleRowid(pTab)==0 || ViewCanHaveRowid ); }else{ if( zTName && sqlite3StrICmp(zTName, zTabName)!=0 ){ continue; @@ -148005,7 +148115,8 @@ static int selectExpander(Walker *pWalker, Select *p){ pUsing = 0; } - nAdd = pTab->nCol + (VisibleRowid(pTab) && (selFlags&SF_NestedFrom)); + nAdd = pTab->nCol; + if( VisibleRowid(pTab) && (selFlags & SF_NestedFrom)!=0 ) nAdd++; for(j=0; ja[pNew->nExpr-1]; assert( pX->zEName==0 ); if( (selFlags & SF_NestedFrom)!=0 && !IN_RENAME_OBJECT ){ - if( pNestedFrom ){ + if( pNestedFrom && (!ViewCanHaveRowid || jnExpr) ){ + assert( jnExpr ); pX->zEName = sqlite3DbStrDup(db, pNestedFrom->a[j].zEName); testcase( pX->zEName==0 ); }else{ @@ -153021,6 +153133,9 @@ SQLITE_PRIVATE void sqlite3Update( } } if( chngRowid==0 && pPk==0 ){ +#ifdef SQLITE_ALLOW_ROWID_IN_VIEW + if( isView ) sqlite3VdbeAddOp2(v, OP_Null, 0, regOldRowid); +#endif sqlite3VdbeAddOp2(v, OP_Copy, regOldRowid, regNewRowid); } } @@ -166730,16 +166845,10 @@ static SQLITE_NOINLINE void whereAddIndexedExpr( for(i=0; inColumn; i++){ Expr *pExpr; int j = pIdx->aiColumn[i]; - int bMaybeNullRow; if( j==XN_EXPR ){ pExpr = pIdx->aColExpr->a[i].pExpr; - testcase( pTabItem->fg.jointype & JT_LEFT ); - testcase( pTabItem->fg.jointype & JT_RIGHT ); - testcase( pTabItem->fg.jointype & JT_LTORJ ); - bMaybeNullRow = (pTabItem->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))!=0; }else if( j>=0 && (pTab->aCol[j].colFlags & COLFLAG_VIRTUAL)!=0 ){ pExpr = sqlite3ColumnExpr(pTab, &pTab->aCol[j]); - bMaybeNullRow = 0; }else{ continue; } @@ -166771,7 +166880,7 @@ static SQLITE_NOINLINE void whereAddIndexedExpr( p->iDataCur = pTabItem->iCursor; p->iIdxCur = iIdxCur; p->iIdxCol = i; - p->bMaybeNullRow = bMaybeNullRow; + p->bMaybeNullRow = (pTabItem->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))!=0; if( sqlite3IndexAffinityStr(pParse->db, pIdx) ){ p->aff = pIdx->zColAff[i]; } @@ -178976,6 +179085,18 @@ SQLITE_API int sqlite3_config(int op, ...){ } #endif /* SQLITE_OMIT_DESERIALIZE */ + case SQLITE_CONFIG_ROWID_IN_VIEW: { + int *pVal = va_arg(ap,int*); +#ifdef SQLITE_ALLOW_ROWID_IN_VIEW + if( 0==*pVal ) sqlite3GlobalConfig.mNoVisibleRowid = TF_NoVisibleRowid; + if( 1==*pVal ) sqlite3GlobalConfig.mNoVisibleRowid = 0; + *pVal = (sqlite3GlobalConfig.mNoVisibleRowid==0); +#else + *pVal = 0; +#endif + break; + } + default: { rc = SQLITE_ERROR; break; @@ -250676,7 +250797,7 @@ static void fts5SourceIdFunc( ){ assert( nArg==0 ); UNUSED_PARAM2(nArg, apUnused); - sqlite3_result_text(pCtx, "fts5: 2024-03-12 11:06:23 d8cd6d49b46a395b13955387d05e9e1a2a47e54fb99f3c9b59835bbefad6af77", -1, SQLITE_TRANSIENT); + sqlite3_result_text(pCtx, "fts5: 2024-04-15 13:34:05 8653b758870e6ef0c98d46b3ace27849054af85da891eb121e9aaa537f1e8355", -1, SQLITE_TRANSIENT); } /* diff --git a/src/database/sqlite3.h b/src/database/sqlite3.h index c9fc77fb..2618b37a 100644 --- a/src/database/sqlite3.h +++ b/src/database/sqlite3.h @@ -146,9 +146,9 @@ extern "C" { ** [sqlite3_libversion_number()], [sqlite3_sourceid()], ** [sqlite_version()] and [sqlite_source_id()]. */ -#define SQLITE_VERSION "3.45.2" -#define SQLITE_VERSION_NUMBER 3045002 -#define SQLITE_SOURCE_ID "2024-03-12 11:06:23 d8cd6d49b46a395b13955387d05e9e1a2a47e54fb99f3c9b59835bbefad6af77" +#define SQLITE_VERSION "3.45.3" +#define SQLITE_VERSION_NUMBER 3045003 +#define SQLITE_SOURCE_ID "2024-04-15 13:34:05 8653b758870e6ef0c98d46b3ace27849054af85da891eb121e9aaa537f1e8355" /* ** CAPI3REF: Run-Time Library Version Numbers @@ -2143,6 +2143,22 @@ struct sqlite3_mem_methods { ** configuration setting is never used, then the default maximum is determined ** by the [SQLITE_MEMDB_DEFAULT_MAXSIZE] compile-time option. If that ** compile-time option is not set, then the default maximum is 1073741824. +** +** [[SQLITE_CONFIG_ROWID_IN_VIEW]] +**
    SQLITE_CONFIG_ROWID_IN_VIEW +**
    The SQLITE_CONFIG_ROWID_IN_VIEW option enables or disables the ability +** for VIEWs to have a ROWID. The capability can only be enabled if SQLite is +** compiled with -DSQLITE_ALLOW_ROWID_IN_VIEW, in which case the capability +** defaults to on. This configuration option queries the current setting or +** changes the setting to off or on. The argument is a pointer to an integer. +** If that integer initially holds a value of 1, then the ability for VIEWs to +** have ROWIDs is activated. If the integer initially holds zero, then the +** ability is deactivated. Any other initial value for the integer leaves the +** setting unchanged. After changes, if any, the integer is written with +** a 1 or 0, if the ability for VIEWs to have ROWIDs is on or off. If SQLite +** is compiled without -DSQLITE_ALLOW_ROWID_IN_VIEW (which is the usual and +** recommended case) then the integer is always filled with zero, regardless +** if its initial value. ** */ #define SQLITE_CONFIG_SINGLETHREAD 1 /* nil */ @@ -2174,6 +2190,7 @@ struct sqlite3_mem_methods { #define SQLITE_CONFIG_SMALL_MALLOC 27 /* boolean */ #define SQLITE_CONFIG_SORTERREF_SIZE 28 /* int nByte */ #define SQLITE_CONFIG_MEMDB_MAXSIZE 29 /* sqlite3_int64 */ +#define SQLITE_CONFIG_ROWID_IN_VIEW 30 /* int* */ /* ** CAPI3REF: Database Connection Configuration Options From e29bcb0695336021440178e2470418a327826462 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 20 Apr 2024 09:39:39 +0200 Subject: [PATCH 053/339] Fix a crash resulting from a bad interaction between PRs #1928 and #1930 Signed-off-by: DL6ER --- src/database/message-table.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/database/message-table.c b/src/database/message-table.c index cf028fe1..f3cc4103 100644 --- a/src/database/message-table.c +++ b/src/database/message-table.c @@ -1418,7 +1418,7 @@ void log_connection_error(const char *server, const char *reason, const char *er log_warn("%s", buf); // Log to database - const int rowid = add_message(CONNECTION_ERROR_MESSAGE, server, 2, reason, error); + const int rowid = add_message(CONNECTION_ERROR_MESSAGE, server, reason, error); if(rowid == -1) log_err("logg_connection_error(): Failed to add message to database"); From 49a5a0c60cb7ce0409299136fad4b6da50d5101b Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 25 Apr 2024 20:27:53 +0200 Subject: [PATCH 054/339] Provide human-readable message about the session status when authenticating Signed-off-by: DL6ER --- src/api/api.c | 1 + src/api/auth.c | 31 ++++++++++++++++++++++------ src/api/docs/content/specs/auth.yaml | 13 +++++++++++- src/webserver/http-common.h | 1 + src/webserver/lua_web.c | 6 ++++-- test/test_suite.bats | 13 ++++++------ 6 files changed, 49 insertions(+), 16 deletions(-) diff --git a/src/api/api.c b/src/api/api.c index bb26c2a4..d99b2b3d 100644 --- a/src/api/api.c +++ b/src/api/api.c @@ -113,6 +113,7 @@ int api_handler(struct mg_connection *conn, void *ignored) http_method(conn), NULL, NULL, + NULL, API_AUTH_UNAUTHORIZED, double_time(), { false, NULL, NULL, NULL, 0u }, diff --git a/src/api/auth.c b/src/api/auth.c index 3d3ec73a..60c25000 100644 --- a/src/api/auth.c +++ b/src/api/auth.c @@ -82,6 +82,7 @@ int check_client_auth(struct ftl_conn *api, const bool is_api) // This may be allowed without authentication depending on the configuration if(!config.webserver.api.localAPIauth.v.b && is_local_api_user(api->request->remote_addr)) { + api->message = "no auth for local user"; add_request_info(api, NULL); return API_AUTH_LOCALHOST; } @@ -89,6 +90,7 @@ int check_client_auth(struct ftl_conn *api, const bool is_api) // When the pwhash is unset, authentication is disabled if(config.webserver.api.pwhash.v.s[0] == '\0') { + api->message = "no password set"; add_request_info(api, NULL); return API_AUTH_EMPTYPASS; } @@ -186,7 +188,8 @@ int check_client_auth(struct ftl_conn *api, const bool is_api) if(!sid_avail) { - log_debug(DEBUG_API, "API Authentication: FAIL (no SID provided)"); + api->message = "no SID provided"; + log_debug(DEBUG_API, "API Authentication: FAIL (%s)", api->message); return API_AUTH_UNAUTHORIZED; } @@ -212,21 +215,28 @@ int check_client_auth(struct ftl_conn *api, const bool is_api) } else { - log_debug(DEBUG_API, "API Authentication: FAIL (Cookie authentication without CSRF token)"); + api->message = "Cookie authentication without CSRF token"; + log_debug(DEBUG_API, "API Authentication: FAIL (%s)", api->message); return API_AUTH_UNAUTHORIZED; } } + bool expired = false; for(unsigned int i = 0; i < max_sessions; i++) { if(auth_data[i].used && - auth_data[i].valid_until >= now && strcmp(auth_data[i].sid, sid) == 0) { + // Check if session is known but expired + if(auth_data[i].valid_until < now) + expired = true; + + // Check CSRF if authentiating via cookie if(need_csrf && strcmp(auth_data[i].csrf, csrf) != 0) { - log_debug(DEBUG_API, "API Authentication: FAIL (CSRF token mismatch, received \"%s\", expected \"%s\")", - csrf, auth_data[i].csrf); + api->message = "CSRF token mismatch"; + log_debug(DEBUG_API, "API Authentication: FAIL (%s, received \"%s\", expected \"%s\")", + api->message, csrf, auth_data[i].csrf); return API_AUTH_UNAUTHORIZED; } user_id = i; @@ -266,12 +276,14 @@ int check_client_auth(struct ftl_conn *api, const bool is_api) } else { - log_debug(DEBUG_API, "API Authentication: FAIL (SID invalid/expired)"); + api->message = expired ? "session expired" : "session unknown"; + log_debug(DEBUG_API, "API Authentication: FAIL (%s)", api->message); return API_AUTH_UNAUTHORIZED; } api->user_id = user_id; + api->message = "correct password"; return user_id; } @@ -314,6 +326,7 @@ static int get_session_object(struct ftl_conn *api, cJSON *json, const int user_ JSON_ADD_BOOL_TO_OBJECT(session, "totp", strlen(config.webserver.api.totp_secret.v.s) > 0); JSON_ADD_NULL_TO_OBJECT(session, "sid"); JSON_ADD_NUMBER_TO_OBJECT(session, "validity", -1); + JSON_REF_STR_IN_OBJECT(session, "message", api->message); JSON_ADD_ITEM_TO_OBJECT(json, "session", session); return 0; } @@ -326,6 +339,7 @@ static int get_session_object(struct ftl_conn *api, cJSON *json, const int user_ JSON_REF_STR_IN_OBJECT(session, "sid", auth_data[user_id].sid); JSON_REF_STR_IN_OBJECT(session, "csrf", auth_data[user_id].csrf); JSON_ADD_NUMBER_TO_OBJECT(session, "validity", auth_data[user_id].valid_until - now); + JSON_REF_STR_IN_OBJECT(session, "message", api->message); JSON_ADD_ITEM_TO_OBJECT(json, "session", session); return 0; } @@ -335,6 +349,7 @@ static int get_session_object(struct ftl_conn *api, cJSON *json, const int user_ JSON_ADD_BOOL_TO_OBJECT(session, "totp", strlen(config.webserver.api.totp_secret.v.s) > 0); JSON_ADD_NULL_TO_OBJECT(session, "sid"); JSON_ADD_NUMBER_TO_OBJECT(session, "validity", -1); + JSON_REF_STR_IN_OBJECT(session, "message", api->message); JSON_ADD_ITEM_TO_OBJECT(json, "session", session); return 0; } @@ -632,6 +647,8 @@ int api_auth(struct ftl_conn *api) "API seats exceeded", "increase webserver.api.max_sessions"); } + + api->message = result == APPPASSWORD_CORRECT ? "app-password correct" : "password correct"; } else if(result == PASSWORD_RATE_LIMITED) { @@ -644,10 +661,12 @@ int api_auth(struct ftl_conn *api) else if(result == NO_PASSWORD_SET) { // No password set + api->message = "password incorrect"; log_debug(DEBUG_API, "API: Trying to auth with password but none set: '%s'", password); } else { + api->message = "password incorrect"; log_debug(DEBUG_API, "API: Password incorrect: '%s'", password); } diff --git a/src/api/docs/content/specs/auth.yaml b/src/api/docs/content/specs/auth.yaml index 8ea10a13..0a739957 100644 --- a/src/api/docs/content/specs/auth.yaml +++ b/src/api/docs/content/specs/auth.yaml @@ -280,6 +280,7 @@ components: - sid - csrf - validity + - message - totp properties: valid: @@ -299,6 +300,10 @@ components: validity: type: integer description: Remaining lifetime of this session unless refreshed (seconds) + message: + type: string + description: Human-readable message describing the session status + nullable: true password: type: object @@ -431,7 +436,7 @@ components: examples: auth_okay: - summary: Authentication valid + summary: Session valid value: session: valid: true @@ -439,6 +444,7 @@ components: sid: null csrf: null validity: 300 + message: null login_okay: summary: Login successful value: @@ -448,6 +454,7 @@ components: sid: "vFA+EP4MQ5JJvJg+3Q2Jnw=" csrf: "Ux87YTIiMOf/GKCefVIOMw=" validity: 300 + message: correct password no_login_required: summary: No login required for this client value: @@ -457,6 +464,7 @@ components: sid: null csrf: null validity: -1 + message: no auth for local user login_required: summary: Login required, 2FA disabled value: @@ -466,6 +474,7 @@ components: sid: null csrf: null validity: -1 + message: password incorrect login_required_2fa: summary: Login required, 2FA enabled value: @@ -475,6 +484,7 @@ components: sid: null csrf: null validity: -1 + message: password incorrect login_failed: summary: Login failed value: @@ -484,6 +494,7 @@ components: sid: null csrf: null validity: -1 + message: no SID provided errors: no_payload: summary: Bad request (no valid JSON payload) diff --git a/src/webserver/http-common.h b/src/webserver/http-common.h index a19d913e..edc979eb 100644 --- a/src/webserver/http-common.h +++ b/src/webserver/http-common.h @@ -37,6 +37,7 @@ struct ftl_conn { const enum http_method method; char *action_path; const char *item; + const char *message; int user_id; double now; struct { diff --git a/src/webserver/lua_web.c b/src/webserver/lua_web.c index 72b26a28..58e36004 100644 --- a/src/webserver/lua_web.c +++ b/src/webserver/lua_web.c @@ -162,8 +162,10 @@ int request_handler(struct mg_connection *conn, void *cbdata) free(target); // User is not authenticated, redirect to login page - log_web("Authentication required, redirecting to %slogin?target=%s", config.webserver.paths.webhome.v.s, encoded_target); - mg_printf(conn, "HTTP/1.1 302 Found\r\nLocation: %slogin?target=%s\r\n\r\n", config.webserver.paths.webhome.v.s, encoded_target); + log_web("Authentication required, redirecting to %slogin?target=%s", + config.webserver.paths.webhome.v.s, encoded_target); + mg_printf(conn, "HTTP/1.1 302 Found\r\nLocation: %slogin?target=%s\r\n\r\n", + config.webserver.paths.webhome.v.s, encoded_target); free(encoded_target); return 302; } diff --git a/test/test_suite.bats b/test/test_suite.bats index 26b65709..0d46ba84 100644 --- a/test/test_suite.bats +++ b/test/test_suite.bats @@ -1470,7 +1470,7 @@ @test "API authorization (without password): No login required" { run bash -c 'curl -s 127.0.0.1/api/auth' printf "%s\n" "${lines[@]}" - [[ ${lines[0]} == '{"session":{"valid":true,"totp":false,"sid":null,"validity":-1},"took":'*'}' ]] + [[ ${lines[0]} == '{"session":{"valid":true,"totp":false,"sid":null,"validity":-1,"message":"no password set"},"took":'*'}' ]] } @test "Config validation working on the CLI (type-based checking)" { @@ -1592,17 +1592,16 @@ @test "API authorization (with password): Incorrect password is rejected if password auth is enabled" { # Password: ABC - run bash -c 'curl -s -X POST 127.0.0.1/api/auth -d "{\"password\":\"XXX\"}" | jq .session.valid' + run bash -c 'curl -s -X POST 127.0.0.1/api/auth -d "{\"password\":\"XXX\"}"' printf "%s\n" "${lines[@]}" - [[ ${lines[0]} == "false" ]] + [[ ${lines[0]} == "{\"session\":{\"valid\":false,\"totp\":false,\"sid\":null,\"validity\":-1,\"message\":\"password incorrect\"},\"took\":"*"}" ]] } @test "API authorization (with password): Correct password is accepted" { - session="$(curl -s -X POST 127.0.0.1/api/auth -d "{\"password\":\"ABC\"}")" - printf "Session: %s\n" "${session}" - run jq .session.valid <<< "${session}" + # Password: ABC + run bash -c 'curl -s -X POST 127.0.0.1/api/auth -d "{\"password\":\"ABC\"}"' printf "%s\n" "${lines[@]}" - [[ ${lines[0]} == "true" ]] + [[ ${lines[0]} == "{\"session\":{\"valid\":true,\"totp\":false,\"sid\":\""*"\",\"csrf\":\""*"\",\"validity\":300,\"message\":\"password correct\"},\"took\":"*"}" ]] } @test "Test TLS/SSL server using self-signed certificate" { From 44377ad8939a2497f4bc4d84add9614320fb4f0c Mon Sep 17 00:00:00 2001 From: Dominik Date: Sat, 27 Apr 2024 06:38:39 +0200 Subject: [PATCH 055/339] Address review comments Co-authored-by: RD WebDesign Signed-off-by: Dominik --- src/api/auth.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/api/auth.c b/src/api/auth.c index 60c25000..88c8d10b 100644 --- a/src/api/auth.c +++ b/src/api/auth.c @@ -82,7 +82,7 @@ int check_client_auth(struct ftl_conn *api, const bool is_api) // This may be allowed without authentication depending on the configuration if(!config.webserver.api.localAPIauth.v.b && is_local_api_user(api->request->remote_addr)) { - api->message = "no auth for local user"; + api->message = "no auth required for local user"; add_request_info(api, NULL); return API_AUTH_LOCALHOST; } From 771db50d8f9681b46dc2d341b6c833d35f7fe3fe Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 27 Apr 2024 10:40:49 +0000 Subject: [PATCH 056/339] Bump actions/checkout Bumps the github_action-dependencies group with 1 update in the / directory: [actions/checkout](https://github.com/actions/checkout). Updates `actions/checkout` from 4.1.2 to 4.1.4 - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v4.1.2...v4.1.4) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-patch dependency-group: github_action-dependencies ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 6 +++--- .github/workflows/codespell.yml | 2 +- .github/workflows/openapi-validator.yml | 2 +- .github/workflows/stale.yml | 2 +- .github/workflows/sync-back-to-dev.yml | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 07dd5280..8f83db7c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -26,7 +26,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v4.1.2 + uses: actions/checkout@v4.1.4 - name: "Calculate required variables" id: variables @@ -70,7 +70,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v4.1.2 + uses: actions/checkout@v4.1.4 - name: Build and test and deploy FTL uses: ./.github/actions/build-and-test @@ -108,7 +108,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v4.1.2 + uses: actions/checkout@v4.1.4 - name: Build and test and deploy FTL uses: ./.github/actions/build-and-test diff --git a/.github/workflows/codespell.yml b/.github/workflows/codespell.yml index 2989b5b4..4a68c780 100644 --- a/.github/workflows/codespell.yml +++ b/.github/workflows/codespell.yml @@ -13,7 +13,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4.1.2 + uses: actions/checkout@v4.1.4 - name: Spell-Checking uses: codespell-project/actions-codespell@master diff --git a/.github/workflows/openapi-validator.yml b/.github/workflows/openapi-validator.yml index ccafd283..c6916733 100644 --- a/.github/workflows/openapi-validator.yml +++ b/.github/workflows/openapi-validator.yml @@ -12,7 +12,7 @@ jobs: steps: - name: Clone repository - uses: actions/checkout@v4.1.2 + uses: actions/checkout@v4.1.4 - name: Set Node.js version uses: actions/setup-node@v4 diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 87b75885..aed50c55 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -40,7 +40,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4.1.2 + uses: actions/checkout@v4.1.4 - name: Remove 'stale' label run: gh issue edit ${{ github.event.issue.number }} --remove-label ${{ env.stale_label }} env: diff --git a/.github/workflows/sync-back-to-dev.yml b/.github/workflows/sync-back-to-dev.yml index 36085247..29c3373b 100644 --- a/.github/workflows/sync-back-to-dev.yml +++ b/.github/workflows/sync-back-to-dev.yml @@ -11,7 +11,7 @@ jobs: name: Syncing branches steps: - name: Checkout - uses: actions/checkout@v4.1.2 + uses: actions/checkout@v4.1.4 - name: Opening pull request run: gh pr create -B development -H master --title 'Sync master back into development' --body 'Created by Github action' --label 'internal' env: From 01697669ac9c99ffc8faa3c55d36f2c4bc0de0e2 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 3 May 2024 20:05:00 +0200 Subject: [PATCH 057/339] API /clients: Add note that {client} needs to be URI-encoded (if specified) and add documentation of read-only optional {name0} field Signed-off-by: DL6ER --- src/api/docs/content/specs/clients.yaml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/api/docs/content/specs/clients.yaml b/src/api/docs/content/specs/clients.yaml index 72f0a471..687cb2cb 100644 --- a/src/api/docs/content/specs/clients.yaml +++ b/src/api/docs/content/specs/clients.yaml @@ -11,7 +11,7 @@ components: - "Client management" operationId: "get_clients" description: | - `{client}` is optional. Specifying it will result in only the requested client being returned. + `{client}` is optional. If it is specified, it will result in only the requested client being returned. This parameter needs to be URI-encoded. Valid combinations are: - `/api/clients` (all clients) @@ -42,7 +42,7 @@ components: - "Client management" operationId: "replace_client" description: | - Items may be updated by replacing them. `{client}` is required. + Items may be updated by replacing them. `{client}` is required and needs to be URI-encoded. Ensure to send all the required parameters (such as `comment` or `groups`) to ensure these properties are retained. The read-only fields `id` and `date_added` are preserved, `date_modified` is automatically updated on success. @@ -91,7 +91,7 @@ components: - "Client management" operationId: "delete_client" description: | - *Note:* There will be no content on success. + *Note:* There will be no content on success. `{client}` is required and needs to be URI-encoded. responses: '204': description: Item deleted @@ -383,6 +383,12 @@ components: type: integer readOnly: true example: 1611239099 + name: + description: hostname (only if available) + type: string + readOnly: true + nullable: true + example: localhost lists_processed: type: object properties: From 1611da221cec6281dc2746a574f2238bd59c4b7d Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 4 May 2024 10:05:58 +0200 Subject: [PATCH 058/339] Improve error logging when TCP connections are prematurely closed by remote server Signed-off-by: DL6ER --- src/dnsmasq_interface.c | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c index 1691056a..f816abb6 100644 --- a/src/dnsmasq_interface.c +++ b/src/dnsmasq_interface.c @@ -3512,7 +3512,19 @@ void get_dnsmasq_metrics_obj(cJSON *json) void FTL_connection_error(const char *reason, const union mysockaddr *addr) { // Make a private copy of the error - const char *error = strerror(errno); + const int errnum = errno; + const char *error = strerror(errnum); + + // Set log priority + int priority = LOG_ERR; + + // If this is a TCP connection error and errno == 0, this isn't a + // connection error but the remote side closed the connection + if(errnum == 0 && strstr(reason, "TCP(read_write)") != NULL) + { + error = "Connection prematurely closed by remote server"; + priority = LOG_INFO; + } // Format the address into a string (if available) in_port_t port = 0; @@ -3525,7 +3537,7 @@ void FTL_connection_error(const char *reason, const union mysockaddr *addr) log_debug(DEBUG_QUERIES, "Connection error (%s#%u, ID %d): %s (%s)", ip, port, id, reason, error); // Log to pihole.log - my_syslog(LOG_ERR, "%s: %s", reason, error); + my_syslog(priority, "%s: %s", reason, error); // Add to Pi-hole diagnostics but do not add messages more often than // once every five seconds to avoid hammering the database with errors From 230989ebbd3bf33b74c65596d025c72ebad2b41a Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 4 May 2024 10:11:35 +0200 Subject: [PATCH 059/339] Exit after fatal dnsmasq errors Signed-off-by: DL6ER --- src/dnsmasq/log.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/dnsmasq/log.c b/src/dnsmasq/log.c index 661f077f..356b4fcb 100644 --- a/src/dnsmasq/log.c +++ b/src/dnsmasq/log.c @@ -511,4 +511,6 @@ void die(char *message, char *arg1, int exit_code) /********** Pi-hole modification *************/ FTL_log_dnsmasq_fatal(message, arg1, errmess); /*********************************************/ + + exit(exit_code); } From 633b825f35b0e0d2cb569fdc8cbec55bbe93420c Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 5 May 2024 10:39:08 +0200 Subject: [PATCH 060/339] Add artifact attestation Signed-off-by: DL6ER --- .github/actions/build-and-test/action.yml | 5 +++++ .github/workflows/build.yml | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/.github/actions/build-and-test/action.yml b/.github/actions/build-and-test/action.yml index 731f382e..d45fc125 100644 --- a/.github/actions/build-and-test/action.yml +++ b/.github/actions/build-and-test/action.yml @@ -98,6 +98,11 @@ runs: with: name: ${{ inputs.artifact_name }} path: '${{ inputs.bin_name }}*' + - + name: Generate artifact attestation + uses: actions/attest-build-provenance@v1 + with: + subject-path: ${{ inputs.bin_name }} - name: Extract documentation files from container if: inputs.event_name != 'pull_request' && inputs.platform == 'linux/amd64' diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8f83db7c..38859edb 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,5 +1,10 @@ name: Build, Test, Deploy +permissions: + id-token: write + contents: read + attestations: write + on: push: branches: From 6ce1668d41d8d3082c6913a260bb5dbd720d7b48 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 10 May 2024 13:17:58 +0200 Subject: [PATCH 061/339] Change database permissions to -rw-r----- (640) Signed-off-by: DL6ER --- src/database/common.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/database/common.c b/src/database/common.c index 2161fd38..8d6756c2 100644 --- a/src/database/common.c +++ b/src/database/common.c @@ -284,9 +284,9 @@ void db_init(void) } } - // Explicitly set permissions to 0644 - // 644 = u+w u+r g+w g+r o+r - const mode_t mode = S_IWUSR | S_IRUSR | S_IWGRP | S_IRGRP| S_IROTH; + // Explicitly set permissions to 0640 + // 640 = u+w u+r g+r + const mode_t mode = S_IWUSR | S_IRUSR | S_IRGRP; chmod_file(config.files.database.v.s, mode); // Open database From 5cc5b377f18b78b3214ccb6d2154d6bac582315e Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 10 May 2024 13:22:45 +0200 Subject: [PATCH 062/339] Update tests, remove duplicated test Signed-off-by: DL6ER --- test/test_suite.bats | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/test/test_suite.bats b/test/test_suite.bats index 0d46ba84..94c04019 100644 --- a/test/test_suite.bats +++ b/test/test_suite.bats @@ -483,7 +483,7 @@ printf "%s\n" "${lines[@]}" # Depending on the shell (x86_64-musl is built on busybox) there can be one or multiple spaces between user and group [[ ${lines[0]} == *"pihole"?*"pihole"* ]] - [[ ${lines[0]} == "-rw-rw-r--"* ]] + [[ ${lines[0]} == "-rw-r-----"* ]] run bash -c 'file /etc/pihole/pihole-FTL.db' printf "%s\n" "${lines[@]}" [[ ${lines[0]} == "/etc/pihole/pihole-FTL.db: SQLite 3.x database"* ]] @@ -938,17 +938,6 @@ [[ "${api}" == "${domain_api}" ]] } -# x86_64-musl is built on busybox which has a slightly different -# variant of ls displaying three, instead of one, spaces between the -# user and group names. - -@test "Ownership and permissions of pihole-FTL.db correct" { - run bash -c 'ls -l /etc/pihole/pihole-FTL.db' - printf "%s\n" "${lines[@]}" - [[ ${lines[0]} == *"pihole pihole"* || ${lines[0]} == *"pihole pihole"* ]] - [[ ${lines[0]} == "-rw-rw-r--"* ]] -} - # "ldd" prints library dependencies and the used interpreter for a given program # # Dependencies on shared libraries are displayed like From 7de24a09e36d70ef62156fd6eef67c2a03ffffab Mon Sep 17 00:00:00 2001 From: Olliver Schinagl Date: Fri, 10 May 2024 13:27:55 +0200 Subject: [PATCH 063/339] FTL_lua: Properly guard readline Readline support should only be used when we know it is available. Lets properly use a ifdef guard like we do in `shell.c`. Signed-off-by: Olliver Schinagl --- src/lua/ftl_lua.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/lua/ftl_lua.c b/src/lua/ftl_lua.c index 5f448406..0948ed06 100644 --- a/src/lua/ftl_lua.c +++ b/src/lua/ftl_lua.c @@ -20,7 +20,9 @@ #include "../files.h" // get_web_theme_str #include "../datastructure.h" +#if HAVE_READLINE #include +#endif #include #include "scripts/scripts.h" From be3d4cd0f684348de7403b372fcc8dc10bcb4d78 Mon Sep 17 00:00:00 2001 From: Olliver Schinagl Date: Fri, 10 May 2024 13:28:49 +0200 Subject: [PATCH 064/339] FTL: Avoid hidden HAVE_READLINE define We should set `HAVE_READLINE` based on whether it was actually detected/found, not via a hidden fixed, as that contains compile errors when readline is actually not available. Signed-off-by: Olliver Schinagl --- src/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 185ba5dc..c254b3bc 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -29,7 +29,6 @@ set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}) # SQLITE_DQS=0: This setting disables the double-quoted string literal misfeature. # SQLITE_ENABLE_DBPAGE_VTAB: Enables the SQLITE_DBPAGE virtual table. Warning: writing to the SQLITE_DBPAGE virtual table can very easily cause unrecoverably database corruption. # SQLITE_TEMP_STORE=2: Store temporary tables in memory for reduced IO and higher performance (can be overwritten by the user at runtime). -# HAVE_READLINE: Enable readline support to allow easy editing, history and auto-completion # SQLITE_DEFAULT_CACHE_SIZE=-16384: Allow up to 16 MiB of cache to be used by SQLite3 (default is 2000 kiB) # SQLITE_DEFAULT_SYNCHRONOUS=1: Use normal synchronous mode (default is 2) # SQLITE_LIKE_DOESNT_MATCH_BLOBS: This option causes the LIKE operator to only match BLOB values against BLOB values and TEXT values against TEXT values. This compile-time option makes SQLite run more efficiently when processing queries that use the LIKE operator. @@ -37,7 +36,7 @@ set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}) # HAVE_FDATASYNC: This option causes SQLite to try to use the fdatasync() system call to sync the database file to disk when committing a transaction. Syncing using fdatasync() is faster than syncing using fsync() as fdatasync() does not wait for the file metadata to be written to disk. # SQLITE_DEFAULT_WORKER_THREADS=4: This option sets the default number of worker threads to use when doing parallel sorting and indexing. The default is 0 which means to use a single thread. The default for SQLITE_MAX_WORKER_THREADS is 8. # SQLITE_MAX_PREPARE_RETRY=200: This option sets the maximum number of automatic re-preparation attempts that can occur after encountering a schema change. This can be caused by running ANALYZE which is done periodically by FTL. -set(SQLITE_DEFINES "-DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_DEFAULT_MEMSTATUS=0 -DSQLITE_OMIT_DEPRECATED -DSQLITE_OMIT_PROGRESS_CALLBACK -DSQLITE_OMIT_SHARED_CACHE -DSQLITE_DEFAULT_FOREIGN_KEYS=1 -DSQLITE_DQS=0 -DSQLITE_ENABLE_DBPAGE_VTAB -DSQLITE_TEMP_STORE=2 -DHAVE_READLINE -DSQLITE_DEFAULT_CACHE_SIZE=16384 -DSQLITE_DEFAULT_SYNCHRONOUS=1 -DSQLITE_LIKE_DOESNT_MATCH_BLOBS -DHAVE_MALLOC_USABLE_SIZE -DHAVE_FDATASYNC -DSQLITE_DEFAULT_WORKER_THREADS=4 -DSQLITE_MAX_PREPARE_RETRY=200") +set(SQLITE_DEFINES "-DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_DEFAULT_MEMSTATUS=0 -DSQLITE_OMIT_DEPRECATED -DSQLITE_OMIT_PROGRESS_CALLBACK -DSQLITE_OMIT_SHARED_CACHE -DSQLITE_DEFAULT_FOREIGN_KEYS=1 -DSQLITE_DQS=0 -DSQLITE_ENABLE_DBPAGE_VTAB -DSQLITE_TEMP_STORE=2 -DSQLITE_DEFAULT_CACHE_SIZE=16384 -DSQLITE_DEFAULT_SYNCHRONOUS=1 -DSQLITE_LIKE_DOESNT_MATCH_BLOBS -DHAVE_MALLOC_USABLE_SIZE -DHAVE_FDATASYNC -DSQLITE_DEFAULT_WORKER_THREADS=4 -DSQLITE_MAX_PREPARE_RETRY=200") # Code hardening and debugging improvements # -fstack-protector-strong: The program will be resistant to having its stack overflowed @@ -312,6 +311,7 @@ if(LIBREADLINE AND LIBHISTORY AND LIBTERMCAP) target_compile_definitions(FTL PRIVATE LUA_USE_READLINE) target_compile_definitions(pihole-FTL PRIVATE LUA_USE_READLINE) target_link_libraries(pihole-FTL ${LIBREADLINE} ${LIBHISTORY} ${LIBTERMCAP}) + set(HAVE_READLINE TRUE) else() message(STATUS "Building FTL with readline support: NO") endif() From ede734051baf7c4d9ed85f34a3339bbbcc2df672 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 11 May 2024 10:54:00 +0000 Subject: [PATCH 065/339] Bump the github_action-dependencies group with 2 updates Bumps the github_action-dependencies group with 2 updates: [actions/checkout](https://github.com/actions/checkout) and [eps1lon/actions-label-merge-conflict](https://github.com/eps1lon/actions-label-merge-conflict). Updates `actions/checkout` from 4.1.4 to 4.1.5 - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v4.1.4...v4.1.5) Updates `eps1lon/actions-label-merge-conflict` from 3.0.0 to 3.0.1 - [Release notes](https://github.com/eps1lon/actions-label-merge-conflict/releases) - [Changelog](https://github.com/eps1lon/actions-label-merge-conflict/blob/main/CHANGELOG.md) - [Commits](https://github.com/eps1lon/actions-label-merge-conflict/compare/v3.0.0...v3.0.1) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-patch dependency-group: github_action-dependencies - dependency-name: eps1lon/actions-label-merge-conflict dependency-type: direct:production update-type: version-update:semver-patch dependency-group: github_action-dependencies ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 6 +++--- .github/workflows/codespell.yml | 2 +- .github/workflows/merge-conflict.yml | 2 +- .github/workflows/openapi-validator.yml | 2 +- .github/workflows/stale.yml | 2 +- .github/workflows/sync-back-to-dev.yml | 2 +- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8f83db7c..e09f83f7 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -26,7 +26,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v4.1.4 + uses: actions/checkout@v4.1.5 - name: "Calculate required variables" id: variables @@ -70,7 +70,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v4.1.4 + uses: actions/checkout@v4.1.5 - name: Build and test and deploy FTL uses: ./.github/actions/build-and-test @@ -108,7 +108,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v4.1.4 + uses: actions/checkout@v4.1.5 - name: Build and test and deploy FTL uses: ./.github/actions/build-and-test diff --git a/.github/workflows/codespell.yml b/.github/workflows/codespell.yml index 4a68c780..47b5d2e8 100644 --- a/.github/workflows/codespell.yml +++ b/.github/workflows/codespell.yml @@ -13,7 +13,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4.1.4 + uses: actions/checkout@v4.1.5 - name: Spell-Checking uses: codespell-project/actions-codespell@master diff --git a/.github/workflows/merge-conflict.yml b/.github/workflows/merge-conflict.yml index 86c2c4fd..c2d3444f 100644 --- a/.github/workflows/merge-conflict.yml +++ b/.github/workflows/merge-conflict.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check if PRs are have merge conflicts - uses: eps1lon/actions-label-merge-conflict@v3.0.0 + uses: eps1lon/actions-label-merge-conflict@v3.0.1 with: dirtyLabel: "Merge conflicts" repoToken: "${{ secrets.GITHUB_TOKEN }}" diff --git a/.github/workflows/openapi-validator.yml b/.github/workflows/openapi-validator.yml index c6916733..2809c389 100644 --- a/.github/workflows/openapi-validator.yml +++ b/.github/workflows/openapi-validator.yml @@ -12,7 +12,7 @@ jobs: steps: - name: Clone repository - uses: actions/checkout@v4.1.4 + uses: actions/checkout@v4.1.5 - name: Set Node.js version uses: actions/setup-node@v4 diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index aed50c55..61147230 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -40,7 +40,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4.1.4 + uses: actions/checkout@v4.1.5 - name: Remove 'stale' label run: gh issue edit ${{ github.event.issue.number }} --remove-label ${{ env.stale_label }} env: diff --git a/.github/workflows/sync-back-to-dev.yml b/.github/workflows/sync-back-to-dev.yml index 29c3373b..c13bacd7 100644 --- a/.github/workflows/sync-back-to-dev.yml +++ b/.github/workflows/sync-back-to-dev.yml @@ -11,7 +11,7 @@ jobs: name: Syncing branches steps: - name: Checkout - uses: actions/checkout@v4.1.4 + uses: actions/checkout@v4.1.5 - name: Opening pull request run: gh pr create -B development -H master --title 'Sync master back into development' --body 'Created by Github action' --label 'internal' env: From b83fcadf6c321bcac00bcc0d3cc80adb64562d7a Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 12 May 2024 21:01:24 +0200 Subject: [PATCH 066/339] Remove all-in build option - we aren't using it in Pi-hole v6.0 Signed-off-by: DL6ER --- src/CMakeLists.txt | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 185ba5dc..72c99c27 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -316,24 +316,6 @@ else() message(STATUS "Building FTL with readline support: NO") endif() -# Do we want to compile an all-in FTL version? -if(DEFINED ENV{CI_ARCH}) - if($ENV{CI_ARCH} STREQUAL "x86_64_full") - add_definitions(-DDNSMASQ_ALL_OPTS) - set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${CMAKE_CURRENT_SOURCE_DIR}) - find_package(DBus REQUIRED) - # Use results of find_package() call. - include_directories(${DBUS_INCLUDE_DIRS}) - target_link_libraries(pihole-FTL ${DBUS_LIBRARIES}) - find_library(LIBMNL mnl) - find_library(LIBNFTNL nftnl) - find_library(LIBNFTABLES nftables) - find_library(LIBNFNETLINK nfnetlink) - find_library(LIBNETFILTER_CONNTRACK netfilter_conntrack) - target_link_libraries(pihole-FTL ${LIBMNL} ${LIBNFTABLES} ${LIBNFTNL} ${LIBNFNETLINK} ${LIBNETFILTER_CONNTRACK}) - endif() -endif() - if(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT) set(CMAKE_INSTALL_PREFIX "/usr" CACHE PATH "..." FORCE) endif() From fccbe2bee4c14c4d1b0a6f19d6a91c02bec76922 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 12 May 2024 21:08:39 +0200 Subject: [PATCH 067/339] Add missing #include Signed-off-by: DL6ER --- src/api/teleporter.c | 2 ++ src/files.c | 3 +++ 2 files changed, 5 insertions(+) diff --git a/src/api/teleporter.c b/src/api/teleporter.c index 5e4a3b5a..986f6bc4 100644 --- a/src/api/teleporter.c +++ b/src/api/teleporter.c @@ -25,6 +25,8 @@ #include "database/common.h" // MAX_ROTATIONS #include "files.h" +//basename() +#include #define MAXFILESIZE (50u*1024*1024) diff --git a/src/files.c b/src/files.c index 5e1f57c0..cb5799fd 100644 --- a/src/files.c +++ b/src/files.c @@ -33,6 +33,9 @@ // PRIu64 #include +//basename() +#include + // chmod_file() changes the file mode bits of a given file (relative // to the directory file descriptor) according to mode. mode is an // octal number representing the bit pattern for the new mode bits From 91b1ced3ab1e2b5ad02268262b35d7238a5b3dc5 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 12 May 2024 21:10:54 +0200 Subject: [PATCH 068/339] Fix use-after-free warning Signed-off-by: DL6ER --- src/tre-regex/xmalloc.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/tre-regex/xmalloc.c b/src/tre-regex/xmalloc.c index 3459d2d9..afe1bd14 100644 --- a/src/tre-regex/xmalloc.c +++ b/src/tre-regex/xmalloc.c @@ -340,6 +340,7 @@ xrealloc_impl(void *ptr, size_t new_size, const char *file, int line, new_ptr = realloc(ptr, new_size); if (new_ptr != NULL) { + ptr = NULL; hash_table_del(xmalloc_table, ptr); hash_table_add(xmalloc_table, new_ptr, (int)new_size, file, line, func); } From 55258150c1e35e19b35d6d39153ba6e4a0cb92bb Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 12 May 2024 21:40:50 +0200 Subject: [PATCH 069/339] Allocate memory for basename() Signed-off-by: DL6ER --- src/api/teleporter.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/api/teleporter.c b/src/api/teleporter.c index 986f6bc4..68c870ae 100644 --- a/src/api/teleporter.c +++ b/src/api/teleporter.c @@ -797,8 +797,8 @@ static int process_received_tar_gz(struct ftl_conn *api, struct upload_data *dat // restore on restart for(unsigned int i = MAX_ROTATIONS; i > 0; i--) { - const char *fname = GLOBALTOMLPATH; - const char *filename = basename(fname); + char *fname = strdup(GLOBALTOMLPATH); + char *filename = basename(fname); // extra 6 bytes is enough space for up to 999 rotations ("/", ".", "\0", "999") const size_t buflen = strlen(filename) + strlen(BACKUP_DIR) + 6; char *path = calloc(buflen, sizeof(char)); @@ -807,6 +807,8 @@ static int process_received_tar_gz(struct ftl_conn *api, struct upload_data *dat // Remove file (if it exists) if(remove(path) != 0 && errno != ENOENT) log_err("Unable to remove file \"%s\": %s", path, strerror(errno)); + + free(fname); } // Free allocated memory From d4a89f3b3e64048c2823607508ecf24be2eb1dce Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 13 May 2024 16:55:12 +0100 Subject: [PATCH 070/339] Update bundled cJSON from 1.7.17 -> 1.7.18 released earlier today Signed-off-by: DL6ER --- src/webserver/cJSON/cJSON.c | 20 +++++++++++++++++--- src/webserver/cJSON/cJSON.h | 2 +- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/src/webserver/cJSON/cJSON.c b/src/webserver/cJSON/cJSON.c index 4e4979e9..61483d90 100644 --- a/src/webserver/cJSON/cJSON.c +++ b/src/webserver/cJSON/cJSON.c @@ -117,7 +117,7 @@ CJSON_PUBLIC(double) cJSON_GetNumberValue(const cJSON * const item) } /* This is a safeguard to prevent copy-pasters from using incompatible C and header files */ -#if (CJSON_VERSION_MAJOR != 1) || (CJSON_VERSION_MINOR != 7) || (CJSON_VERSION_PATCH != 17) +#if (CJSON_VERSION_MAJOR != 1) || (CJSON_VERSION_MINOR != 7) || (CJSON_VERSION_PATCH != 18) #error cJSON.h and cJSON.c have different versions. Make sure that both have the same. #endif @@ -263,10 +263,12 @@ CJSON_PUBLIC(void) cJSON_Delete(cJSON *item) if (!(item->type & cJSON_IsReference) && (item->valuestring != NULL)) { global_hooks.deallocate(item->valuestring); + item->valuestring = NULL; } if (!(item->type & cJSON_StringIsConst) && (item->string != NULL)) { global_hooks.deallocate(item->string); + item->string = NULL; } global_hooks.deallocate(item); item = next; @@ -397,6 +399,7 @@ CJSON_PUBLIC(double) cJSON_SetNumberHelper(cJSON *object, double number) return object->valuedouble = number; } +/* Note: when passing a NULL valuestring, cJSON_SetValuestring treats this as an error and return NULL */ CJSON_PUBLIC(char*) cJSON_SetValuestring(cJSON *object, const char *valuestring) { char *copy = NULL; @@ -405,8 +408,8 @@ CJSON_PUBLIC(char*) cJSON_SetValuestring(cJSON *object, const char *valuestring) { return NULL; } - /* return NULL if the object is corrupted */ - if (object->valuestring == NULL) + /* return NULL if the object is corrupted or valuestring is NULL */ + if (object->valuestring == NULL || valuestring == NULL) { return NULL; } @@ -893,6 +896,7 @@ fail: if (output != NULL) { input_buffer->hooks.deallocate(output); + output = NULL; } if (input_pointer != NULL) @@ -1235,6 +1239,7 @@ static unsigned char *print(const cJSON * const item, cJSON_bool format, const i /* free the buffer */ hooks->deallocate(buffer->buffer); + buffer->buffer = NULL; } return printed; @@ -1243,11 +1248,13 @@ fail: if (buffer->buffer != NULL) { hooks->deallocate(buffer->buffer); + buffer->buffer = NULL; } if (printed != NULL) { hooks->deallocate(printed); + printed = NULL; } return NULL; @@ -1288,6 +1295,7 @@ CJSON_PUBLIC(char *) cJSON_PrintBuffered(const cJSON *item, int prebuffer, cJSON if (!print_value(item, &p)) { global_hooks.deallocate(p.buffer); + p.buffer = NULL; return NULL; } @@ -1659,6 +1667,11 @@ static cJSON_bool parse_object(cJSON * const item, parse_buffer * const input_bu current_item = new_item; } + if (cannot_access_at_index(input_buffer, 1)) + { + goto fail; /* nothing comes after the comma */ + } + /* parse the name of the child */ input_buffer->offset++; buffer_skip_whitespace(input_buffer); @@ -3126,4 +3139,5 @@ CJSON_PUBLIC(void *) cJSON_malloc(size_t size) CJSON_PUBLIC(void) cJSON_free(void *object) { global_hooks.deallocate(object); + object = NULL; } diff --git a/src/webserver/cJSON/cJSON.h b/src/webserver/cJSON/cJSON.h index 218cc9ea..88cf0bcf 100644 --- a/src/webserver/cJSON/cJSON.h +++ b/src/webserver/cJSON/cJSON.h @@ -81,7 +81,7 @@ then using the CJSON_API_VISIBILITY flag to "export" the same symbols the way CJ /* project version */ #define CJSON_VERSION_MAJOR 1 #define CJSON_VERSION_MINOR 7 -#define CJSON_VERSION_PATCH 17 +#define CJSON_VERSION_PATCH 18 #include From 688a551a2418aa6886fee1754f36cf7777950c6c Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 14 May 2024 20:17:05 +0200 Subject: [PATCH 071/339] Improve query storing algorithm to better cope with bursts of queries of arbitrary size and frequency Signed-off-by: DL6ER --- src/FTL.h | 13 +-- src/database/query-table.c | 167 +++++++++++++++++++------------------ 2 files changed, 94 insertions(+), 86 deletions(-) diff --git a/src/FTL.h b/src/FTL.h index 9edd2034..5cd84f60 100644 --- a/src/FTL.h +++ b/src/FTL.h @@ -124,12 +124,13 @@ // Default: 180 [seconds] #define DELAY_UPTIME 180 -// DB_QUERY_MAX_ITER defines how many queries we check periodically for updates to be added -// to the in-memory database. This value may need to be increased on *very* busy systems. -// However, there is an algorithm in place that tries to ensure we are not missing queries -// on systems with > 100 queries per second -// Default: 100 (per second) -#define DB_QUERY_MAX_ITER 100 +// REPLY_TIMEOUT defines until how far back in the history of queries we are +// checking for changed/updated queries. This value should not be set too high +// to avoid unecessary spinning in the updating loop of the queries running +// every second. The value should be set to a value that is high enough to +// catch all queries that are still in the process of being resolved. +// Default: 30 [seconds] +#define REPLY_TIMEOUT 30 // Special exit code used to signal that FTL wants to restart #define RESTART_FTL_CODE 22 diff --git a/src/database/query-table.c b/src/database/query-table.c index f66d5152..83c6a182 100644 --- a/src/database/query-table.c +++ b/src/database/query-table.c @@ -27,6 +27,16 @@ static double new_last_timestamp = 0; static unsigned int new_total = 0, new_blocked = 0; static unsigned long last_mem_db_idx = 0, last_disk_db_idx = 0; static unsigned int mem_db_num = 0, disk_db_num = 0; +static sqlite3_stmt *query_stmt = NULL; +static sqlite3_stmt *domain_stmt = NULL; +static sqlite3_stmt *client_stmt = NULL; +static sqlite3_stmt *forward_stmt = NULL; +static sqlite3_stmt *addinfo_stmt = NULL; +static sqlite3_stmt **stmts[] = { &query_stmt, + &domain_stmt, + &client_stmt, + &forward_stmt, + &addinfo_stmt }; // Return the maximum ID of the in-memory database unsigned long __attribute__((pure)) get_max_db_idx(void) @@ -155,6 +165,58 @@ bool init_memory_database(void) } } + // Prepare insertion/replace statements + rc = sqlite3_prepare_v3(_memdb, "REPLACE INTO query_storage VALUES "\ + "(?1," \ + "?2," \ + "?3," \ + "?4," \ + "(SELECT id FROM domain_by_id WHERE domain = ?5)," \ + "(SELECT id FROM client_by_id WHERE ip = ?6 AND name = ?7)," \ + "(SELECT id FROM forward_by_id WHERE forward = ?8)," \ + "(SELECT id FROM addinfo_by_id WHERE type = ?9 AND content = ?10)," + "?11," \ + "?12," \ + "?13," \ + "?14)", -1, SQLITE_PREPARE_PERSISTENT, &query_stmt, NULL); + if( rc != SQLITE_OK ) + { + log_err("queries_to_database(query_storage) - SQL error step: %s", sqlite3_errstr(rc)); + return false; + } + + rc = sqlite3_prepare_v3(_memdb, "INSERT OR IGNORE INTO domain_by_id (domain) VALUES (?)", + -1, SQLITE_PREPARE_PERSISTENT, &domain_stmt, NULL); + if( rc != SQLITE_OK ) + { + log_err("queries_to_database(domain_by_id) - SQL error step: %s", sqlite3_errstr(rc)); + return false; + } + + rc = sqlite3_prepare_v3(_memdb, "INSERT OR IGNORE INTO client_by_id (ip,name) VALUES (?,?)", + -1, SQLITE_PREPARE_PERSISTENT, &client_stmt, NULL); + if( rc != SQLITE_OK ) + { + log_err("queries_to_database(client_by_id) - SQL error step: %s", sqlite3_errstr(rc)); + return false; + } + + rc = sqlite3_prepare_v3(_memdb, "INSERT OR IGNORE INTO forward_by_id (forward) VALUES (?)", + -1, SQLITE_PREPARE_PERSISTENT, &forward_stmt, NULL); + if( rc != SQLITE_OK ) + { + log_err("queries_to_database(forward_by_id) - SQL error step: %s", sqlite3_errstr(rc)); + return false; + } + + rc = sqlite3_prepare_v3(_memdb, "INSERT OR IGNORE INTO addinfo_by_id (type,content) VALUES (?,?)", + -1, SQLITE_PREPARE_PERSISTENT, &addinfo_stmt, NULL); + if( rc != SQLITE_OK ) + { + log_err("queries_to_database(addinfo_by_id) - SQL error step: %s", sqlite3_errstr(rc)); + return false; + } + // Everything went well return true; } @@ -166,6 +228,15 @@ void close_memory_database(void) if(_memdb == NULL) return; + // Finalize all statements + for(unsigned int i = 0; i < ArraySize(stmts); i++) + { + if(*stmts[i] == NULL) + continue; + sqlite3_finalize(*stmts[i]); + *stmts[i] = NULL; + } + // Detach disk database if(!detach_database(_memdb, NULL, "disk")) log_err("close_memory_database(): Failed to detach disk database"); @@ -1232,6 +1303,7 @@ void DB_read_queries(void) log_info(" %d queries parsed...", counters->queries); } + // Release shared memory unlock_shm(); if( rc != SQLITE_DONE ) @@ -1279,16 +1351,6 @@ bool queries_to_database(void) int rc; unsigned int added = 0, updated = 0; sqlite3_int64 idx = 0; - sqlite3_stmt *query_stmt = NULL; - sqlite3_stmt *domain_stmt = NULL; - sqlite3_stmt *client_stmt = NULL; - sqlite3_stmt *forward_stmt = NULL; - sqlite3_stmt *addinfo_stmt = NULL; - sqlite3_stmt **stmts[] = { &query_stmt, - &domain_stmt, - &client_stmt, - &forward_stmt, - &addinfo_stmt }; // Skip, we never store nor count queries recorded while have been in // maximum privacy mode in the database @@ -1303,82 +1365,33 @@ bool queries_to_database(void) return true; } - // Start preparing query - sqlite3 *memdb = get_memdb(); - rc = sqlite3_prepare_v3(memdb, "REPLACE INTO query_storage VALUES "\ - "(?1," \ - "?2," \ - "?3," \ - "?4," \ - "(SELECT id FROM domain_by_id WHERE domain = ?5)," \ - "(SELECT id FROM client_by_id WHERE ip = ?6 AND name = ?7)," \ - "(SELECT id FROM forward_by_id WHERE forward = ?8)," \ - "(SELECT id FROM addinfo_by_id WHERE type = ?9 AND content = ?10)," - "?11," \ - "?12," \ - "?13," \ - "?14)", -1, SQLITE_PREPARE_PERSISTENT, &query_stmt, NULL); - if( rc != SQLITE_OK ) - { - log_err("queries_to_database(query_storage) - SQL error step: %s", sqlite3_errstr(rc)); - return false; - } - - rc = sqlite3_prepare_v3(memdb, "INSERT OR IGNORE INTO domain_by_id (domain) VALUES (?)", - -1, SQLITE_PREPARE_PERSISTENT, &domain_stmt, NULL); - if( rc != SQLITE_OK ) - { - log_err("queries_to_database(domain_by_id) - SQL error step: %s", sqlite3_errstr(rc)); - return false; - } - - rc = sqlite3_prepare_v3(memdb, "INSERT OR IGNORE INTO client_by_id (ip,name) VALUES (?,?)", - -1, SQLITE_PREPARE_PERSISTENT, &client_stmt, NULL); - if( rc != SQLITE_OK ) - { - log_err("queries_to_database(client_by_id) - SQL error step: %s", sqlite3_errstr(rc)); - return false; - } - - rc = sqlite3_prepare_v3(memdb, "INSERT OR IGNORE INTO forward_by_id (forward) VALUES (?)", - -1, SQLITE_PREPARE_PERSISTENT, &forward_stmt, NULL); - if( rc != SQLITE_OK ) - { - log_err("queries_to_database(forward_by_id) - SQL error step: %s", sqlite3_errstr(rc)); - return false; - } - - rc = sqlite3_prepare_v3(memdb, "INSERT OR IGNORE INTO addinfo_by_id (type,content) VALUES (?,?)", - -1, SQLITE_PREPARE_PERSISTENT, &addinfo_stmt, NULL); - if( rc != SQLITE_OK ) - { - log_err("queries_to_database(addinfo_by_id) - SQL error step: %s", sqlite3_errstr(rc)); - return false; - } - - // Loop over recent queries and store new or changed ones in the in-memory database - const unsigned int min_iter = counters->queries - 1; - unsigned int max_iter = min_iter > DB_QUERY_MAX_ITER ? min_iter - DB_QUERY_MAX_ITER : 0; - for(unsigned int queryID = min_iter; queryID > max_iter; queryID--) + // Loop over recent queries and store new or changed ones in the + // in-memory database + // The upper bound is the last query in the array, the lower bound is + // indirectly given by the first query older than 30 seconds - we do not + // expect replies to still arrive after 30 seconds - they are anyway + // useless as the client will have already timed out tis particular + // query and retried or failed + const double limit_timestamp = double_time() - REPLY_TIMEOUT; + for(unsigned int queryID = counters->queries - 1; queryID > 0; queryID--) { // Get query pointer queriesData *query = getQuery(queryID, true); if(query == NULL) { // Encountered memory error, skip query - log_err("Memory error in queries_to_database()"); + log_err("Memory error in queries_to_database() when trying to access query %u", queryID); break; } + // Skip too old queries (see note above the loop) + if(query->timestamp < limit_timestamp) + break; + // Skip queries which have not changed since the last iteration if(!query->flags.database.changed) continue; - // Update max_iter in case we have changes queries very close to - // the end of the iteration interval - if(min_iter - max_iter < 10) - max_iter = max_iter > DB_QUERY_MAX_ITER ? max_iter - DB_QUERY_MAX_ITER : 0; - // Explicitly set ID to match what is in the on-disk database if(query->db > -1) { @@ -1599,14 +1612,8 @@ bool queries_to_database(void) query->flags.database.changed = false; } - // Finalize all statements - for(unsigned int i = 0; i < ArraySize(stmts); i++) - { - sqlite3_finalize(*stmts[i]); - *stmts[i] = NULL; - } - // Update number of queries in in-memory database + sqlite3 *memdb = get_memdb(); mem_db_num = get_number_of_queries_in_DB(memdb, "query_storage"); if(config.debug.database.v.b && updated + added > 0) From 30cade643ffbaf6b7f8abb0be79ebfd0048b8aca Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 14 May 2024 20:19:06 +0200 Subject: [PATCH 072/339] Use new constant also as back-off factor for exporting into the database Signed-off-by: DL6ER --- src/FTL.h | 2 +- src/database/query-table.c | 24 +++++++++++++----------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/src/FTL.h b/src/FTL.h index 5cd84f60..69b88b5d 100644 --- a/src/FTL.h +++ b/src/FTL.h @@ -126,7 +126,7 @@ // REPLY_TIMEOUT defines until how far back in the history of queries we are // checking for changed/updated queries. This value should not be set too high -// to avoid unecessary spinning in the updating loop of the queries running +// to avoid unnecessary spinning in the updating loop of the queries running // every second. The value should be set to a value that is high enough to // catch all queries that are still in the process of being resolved. // Default: 30 [seconds] diff --git a/src/database/query-table.c b/src/database/query-table.c index 83c6a182..d76c49bc 100644 --- a/src/database/query-table.c +++ b/src/database/query-table.c @@ -165,7 +165,7 @@ bool init_memory_database(void) } } - // Prepare insertion/replace statements + // Prepare persistent insertion/replace statements rc = sqlite3_prepare_v3(_memdb, "REPLACE INTO query_storage VALUES "\ "(?1," \ "?2," \ @@ -181,7 +181,7 @@ bool init_memory_database(void) "?14)", -1, SQLITE_PREPARE_PERSISTENT, &query_stmt, NULL); if( rc != SQLITE_OK ) { - log_err("queries_to_database(query_storage) - SQL error step: %s", sqlite3_errstr(rc)); + log_err("init_memory_database(query_storage) - SQL error step: %s", sqlite3_errstr(rc)); return false; } @@ -189,7 +189,7 @@ bool init_memory_database(void) -1, SQLITE_PREPARE_PERSISTENT, &domain_stmt, NULL); if( rc != SQLITE_OK ) { - log_err("queries_to_database(domain_by_id) - SQL error step: %s", sqlite3_errstr(rc)); + log_err("init_memory_database(domain_by_id) - SQL error step: %s", sqlite3_errstr(rc)); return false; } @@ -197,7 +197,7 @@ bool init_memory_database(void) -1, SQLITE_PREPARE_PERSISTENT, &client_stmt, NULL); if( rc != SQLITE_OK ) { - log_err("queries_to_database(client_by_id) - SQL error step: %s", sqlite3_errstr(rc)); + log_err("init_memory_database(client_by_id) - SQL error step: %s", sqlite3_errstr(rc)); return false; } @@ -205,7 +205,7 @@ bool init_memory_database(void) -1, SQLITE_PREPARE_PERSISTENT, &forward_stmt, NULL); if( rc != SQLITE_OK ) { - log_err("queries_to_database(forward_by_id) - SQL error step: %s", sqlite3_errstr(rc)); + log_err("init_memory_database(forward_by_id) - SQL error step: %s", sqlite3_errstr(rc)); return false; } @@ -213,7 +213,7 @@ bool init_memory_database(void) -1, SQLITE_PREPARE_PERSISTENT, &addinfo_stmt, NULL); if( rc != SQLITE_OK ) { - log_err("queries_to_database(addinfo_by_id) - SQL error step: %s", sqlite3_errstr(rc)); + log_err("init_memory_database(addinfo_by_id) - SQL error step: %s", sqlite3_errstr(rc)); return false; } @@ -575,10 +575,14 @@ bool import_queries_from_disk(void) // Export in-memory queries to disk - either due to periodic dumping (final = // false) or because of a shutdown (final = true) +// When final is false, we only export queries that are older than REPLY_TIMEOUT +// seconds. This is to give queries some time to complete before they are +// exported to disk. When final is true, we export all queries (nothing is going +// to be added to the in-memory database anymore). bool export_queries_to_disk(bool final) { bool okay = false; - const double time = double_time() - (final ? 0.0 : 30.0); + const double time = double_time() - (final ? 0.0 : REPLY_TIMEOUT); const char *querystr = "INSERT INTO disk.query_storage SELECT * FROM query_storage WHERE id > ? AND timestamp < ?"; log_debug(DEBUG_DATABASE, "Storing queries on disk WHERE id > %lu (max is %lu) and timestamp < %f", @@ -741,8 +745,7 @@ bool delete_old_queries_from_db(const bool use_memdb, const double mintime) mintime, sqlite3_errstr(rc)); // Update number of queries in in-memory database - sqlite3 *memdb = get_memdb(); - const int new_num = get_number_of_queries_in_DB(memdb, "query_storage"); + const int new_num = get_number_of_queries_in_DB(NULL, "query_storage"); log_debug(DEBUG_GC, "delete_old_queries_from_db(): Deleted %i (%u) queries, new number of queries in memory: %i", sqlite3_changes(db), (mem_db_num - new_num), new_num); mem_db_num = new_num; @@ -1613,8 +1616,7 @@ bool queries_to_database(void) } // Update number of queries in in-memory database - sqlite3 *memdb = get_memdb(); - mem_db_num = get_number_of_queries_in_DB(memdb, "query_storage"); + mem_db_num = get_number_of_queries_in_DB(NULL, "query_storage"); if(config.debug.database.v.b && updated + added > 0) { From 32de390b002b06ada79489449f2dbe8d37b66761 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 15 May 2024 20:41:52 +0200 Subject: [PATCH 073/339] Add Clang compiler support, tested with Clang 14.0.0 (Ubuntu 22.04.3 LTS), Clang 16.0.2 (Alpine 3.18), and Clang 17.0.6 (Alping Edge) Signed-off-by: DL6ER --- build.sh | 7 +++++++ src/CMakeLists.txt | 39 +++++++++++++++++++++++++++--------- src/api/auth.h | 2 +- src/api/config.c | 4 ++-- src/api/dhcp.c | 2 +- src/api/queries.c | 4 ++++ src/api/stats.c | 2 -- src/api/stats_database.c | 2 -- src/config/cli.h | 2 +- src/config/legacy_reader.c | 29 ++++++++++++++++----------- src/config/password.c | 5 +---- src/config/validator.c | 12 ----------- src/database/CMakeLists.txt | 4 ++++ src/database/common.c | 2 ++ src/database/common.h | 2 +- src/database/query-table.c | 2 +- src/database/sqlite3-ext.c | 2 +- src/dnsmasq/CMakeLists.txt | 6 +++++- src/dnsmasq_interface.c | 13 +++++++----- src/edns0.c | 2 +- src/enums.h | 1 + src/events.h | 2 +- src/log.c | 7 ++++--- src/log.h | 17 ++++++++-------- src/lua/CMakeLists.txt | 4 +++- src/lua/ftl_lua.c | 2 ++ src/lua/ftl_lua.h | 2 +- src/procps.h | 2 +- src/resolve.c | 6 +++--- src/shmem.c | 10 ++++++++- src/struct_size.h | 2 +- src/syscalls/accept.c | 2 +- src/syscalls/recv.c | 2 +- src/syscalls/recvfrom.c | 2 +- src/syscalls/select.c | 2 +- src/syscalls/sendto.c | 2 +- src/syscalls/strdup.c | 2 +- src/syscalls/syscalls.h | 16 +++++++-------- src/syscalls/write.c | 2 +- src/tools/arp-scan.c | 4 ++-- src/tools/dhcp-discover.c | 19 ++++++++---------- src/tre-regex/CMakeLists.txt | 4 +++- src/webserver/lua_web.h | 2 +- src/webserver/webserver.c | 3 ++- src/webserver/webserver.h | 2 +- src/zip/gzip.c | 16 +++++++++++---- src/zip/miniz/CMakeLists.txt | 1 + src/zip/miniz/miniz.h | 2 +- src/zip/tar.c | 2 +- src/zip/tar.h | 2 +- 50 files changed, 168 insertions(+), 117 deletions(-) diff --git a/build.sh b/build.sh index b83062a4..b92062d1 100755 --- a/build.sh +++ b/build.sh @@ -23,6 +23,7 @@ do "-C" | "CLEAN" ) clean=1 && nobuild=1;; "-i" | "install" ) install=1;; "-t" | "test" ) test=1;; + "clang" ) clang=1;; "ci" ) builddir="cmake_ci/";; esac done @@ -60,6 +61,12 @@ for scriptname in src/lua/scripts/*.lua; do fi done +# Set compiler to clang if requested +if [[ -n "${clang}" ]]; then + export CC=clang + export CXX=clang++ +fi + # Configure build, pass CMake CACHE entries if present # Wrap multiple options in "" as first argument to ./build.sh: # ./build.sh "-DA=1 -DB=2" install diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 185ba5dc..9e3c35bb 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -53,8 +53,10 @@ set(SQLITE_DEFINES "-DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_DEFAULT_MEMSTATUS=0 -D # -Wl,-z,now: Disable lazy binding # -Wl,-z,relro: Read-only segments after relocation # -fno-common: Emit globals without explicit initializer from `.bss` to `.data`. This causes GCC to reject multiple definitions of global variables. This is the new default from GCC-10 on. -set(HARDENING_FLAGS "-fstack-protector-strong -Wp,-D_FORTIFY_SOURCE=2 -Wl,-z,relro,-z,now -fexceptions -funwind-tables -fasynchronous-unwind-tables -Wl,-z,defs -Wl,-z,now -Wl,-z,relro -fno-common") -set(DEBUG_FLAGS "-rdynamic -fno-omit-frame-pointer") +if (CMAKE_C_COMPILER_ID STREQUAL "GNU") + set(HARDENING_FLAGS "-fstack-protector-strong -Wp,-D_FORTIFY_SOURCE=2 -Wl,-z,relro,-z,now -fexceptions -funwind-tables -fasynchronous-unwind-tables -Wl,-z,defs -Wl,-z,now -Wl,-z,relro -fno-common") + set(DEBUG_FLAGS "-rdynamic -fno-omit-frame-pointer") +endif() # -Wall: This enables all the warnings about constructions that some users consider questionable, and that are easy to avoid (or modify to prevent the warning), even in conjunction with macros. This also enables some language-specific warnings described in C++ Dialect Options and Objective-C and Objective-C++ Dialect Options. # -Wextra: This enables some extra warning flags that are not enabled by -Wall. @@ -155,11 +157,25 @@ else() set(EXTRAWARN_GCC13 "") endif() -set(EXTRAWARN "${EXTRAWARN_GCC6} \ - ${EXTRAWARN_GCC7} \ - ${EXTRAWARN_GCC8} \ - ${EXTRAWARN_GCC12} \ - ${EXTRAWARN_GCC13}") +# Set extrawarn flags if CC is GCC +if (CMAKE_C_COMPILER_ID STREQUAL "GNU") + set(EXTRAWARN "${EXTRAWARN_GCC6} \ + ${EXTRAWARN_GCC7} \ + ${EXTRAWARN_GCC8} \ + ${EXTRAWARN_GCC12} \ + ${EXTRAWARN_GCC13}") +elseif (CMAKE_C_COMPILER_ID STREQUAL "Clang") + set(EXTRAWARN "-Wnewline-eof \ + -Wno-dangling-else \ + -Wno-gnu-zero-variadic-macro-arguments \ + -Wno-gnu-variable-sized-type-not-at-end \ + -Wno-declaration-after-statement \ + -Wno-reserved-identifier \ + -Wno-reserved-macro-identifier") +else() + message(WARNING "Unknown compiler, not setting warnings flags") + set(EXTRAWARN "") +endif() # Remove extra spaces from EXTRAWARN string(REGEX REPLACE " +" " " EXTRAWARN "${EXTRAWARN}") @@ -185,11 +201,14 @@ else() message(STATUS "Compiling dynamically linked executable") endif() # -pie -fPIE: (Dynamic) position independent executable -set(HARDENING_FLAGS "${HARDENING_FLAGS} -pie -fPIE") + +if (CMAKE_C_COMPILER_ID STREQUAL "GNU") + set(HARDENING_FLAGS "${HARDENING_FLAGS} -pie -fPIE") +endif() # -FILE_OFFSET_BITS=64: used by stat(). Avoids problems with files > 2 GB on 32bit machines # We define HAVE_POLL_H as this is needed for the musl builds to succeed -set(CMAKE_C_FLAGS "-pipe ${WARN_FLAGS} -D_FILE_OFFSET_BITS=64 ${HARDENING_FLAGS} ${DEBUG_FLAGS} ${CMAKE_C_FLAGS} -DHAVE_POLL_H ${SQLITE_DEFINES}") +set(CMAKE_C_FLAGS "-std=c99 -pipe ${WARN_FLAGS} -D_FILE_OFFSET_BITS=64 ${HARDENING_FLAGS} ${DEBUG_FLAGS} ${CMAKE_C_FLAGS} -DHAVE_POLL_H ${SQLITE_DEFINES}") set(CMAKE_C_FLAGS_DEBUG "-O0 -g3") set(CMAKE_C_FLAGS_RELEASE "-O3 -DNDEBUG") @@ -279,7 +298,7 @@ add_executable(pihole-FTL if(STATIC) set_target_properties(pihole-FTL PROPERTIES LINK_SEARCH_START_STATIC ON) set_target_properties(pihole-FTL PROPERTIES LINK_SEARCH_END_STATIC ON) - target_link_libraries(pihole-FTL -static-libgcc -static -pie) + target_link_libraries(pihole-FTL -static-libgcc -static) else() find_library(LIBMATH m) target_link_libraries(pihole-FTL ${LIBMATH}) diff --git a/src/api/auth.h b/src/api/auth.h index 53663026..5028b6c8 100644 --- a/src/api/auth.h +++ b/src/api/auth.h @@ -60,4 +60,4 @@ struct session { char csrf[SID_SIZE]; }; -#endif // AUTH_H \ No newline at end of file +#endif // AUTH_H diff --git a/src/api/config.c b/src/api/config.c index c998823b..f413e502 100644 --- a/src/api/config.c +++ b/src/api/config.c @@ -238,7 +238,7 @@ static const char *getJSONvalue(struct conf_item *conf_item, cJSON *elem, struct // 1. Check it is a number // 2. Check the number is within the allowed range for the given data type if(!cJSON_IsNumber(elem) || - elem->valuedouble < LONG_MIN || elem->valuedouble > LONG_MAX) + elem->valuedouble < (double)LONG_MIN || elem->valuedouble > (double)LONG_MAX) return "not of type long"; // Set item conf_item->v.l = elem->valuedouble; @@ -250,7 +250,7 @@ static const char *getJSONvalue(struct conf_item *conf_item, cJSON *elem, struct // 1. Check it is a number // 2. Check the number is within the allowed range for the given data type if(!cJSON_IsNumber(elem) || - elem->valuedouble < 0 || elem->valuedouble > ULONG_MAX) + elem->valuedouble < 0 || elem->valuedouble > (double)ULONG_MAX) return "not of type unsigned long"; // Set item conf_item->v.ul = elem->valuedouble; diff --git a/src/api/dhcp.c b/src/api/dhcp.c index 16898ca9..4e9e67af 100644 --- a/src/api/dhcp.c +++ b/src/api/dhcp.c @@ -110,4 +110,4 @@ int api_dhcp_leases_DELETE(struct ftl_conn *api) // - 404 Not Found (if no lease was found) cJSON *json = JSON_NEW_OBJECT(); JSON_SEND_OBJECT_CODE(json, found ? 204 : 404); -} \ No newline at end of file +} diff --git a/src/api/queries.c b/src/api/queries.c index 82735726..eb16529f 100644 --- a/src/api/queries.c +++ b/src/api/queries.c @@ -432,10 +432,14 @@ int api_queries(struct ftl_conn *api) // Encoded URI string: %5B = [ and %5D = ] if(GET_VAR(sort_col_id, sort_col, api->request->query_string) > 0) + { log_debug(DEBUG_API, "Sorting by column %s (%s)", sort_col, sort_dir); + } else + { log_warn("Sorting by column %d (%s) requested, but column name not found", sort_column, sort_dir); + } } // Column searching? diff --git a/src/api/stats.c b/src/api/stats.c index 86de44c5..b272e768 100644 --- a/src/api/stats.c +++ b/src/api/stats.c @@ -458,7 +458,6 @@ int api_stats_top_clients(struct ftl_conn *api) int api_stats_upstreams(struct ftl_conn *api) { - unsigned int totalcount = 0; const int upstreams = counters->upstreams; int *temparray = calloc(2*upstreams, sizeof(int)); if(temparray == NULL) @@ -480,7 +479,6 @@ int api_stats_upstreams(struct ftl_conn *api) temparray[2*added_upstreams + 0] = upstreamID; temparray[2*added_upstreams + 1] = upstream->count; - totalcount += upstream->count; added_upstreams++; } diff --git a/src/api/stats_database.c b/src/api/stats_database.c index b4309f07..5a30b105 100644 --- a/src/api/stats_database.c +++ b/src/api/stats_database.c @@ -514,13 +514,11 @@ int api_history_database_clients(struct ftl_conn *api) // Loop over clients and accumulate results cJSON *clients = JSON_NEW_OBJECT(); - unsigned int num_clients = 0; while((rc = sqlite3_step(stmt)) == SQLITE_ROW) { cJSON *item = JSON_NEW_OBJECT(); JSON_COPY_STR_TO_OBJECT(item, "name", sqlite3_column_text(stmt, 2)); JSON_ADD_ITEM_TO_OBJECT(clients, (const char*)sqlite3_column_text(stmt, 1), item); - num_clients++; } sqlite3_finalize(stmt); diff --git a/src/config/cli.h b/src/config/cli.h index 4cf4cc31..c9398bcf 100644 --- a/src/config/cli.h +++ b/src/config/cli.h @@ -13,4 +13,4 @@ int set_config_from_CLI(const char *key, const char *value); int get_config_from_CLI(const char *key, const bool quiet); -#endif //CONFIG_CLI_H \ No newline at end of file +#endif //CONFIG_CLI_H diff --git a/src/config/legacy_reader.c b/src/config/legacy_reader.c index 857bcc2b..06173fea 100644 --- a/src/config/legacy_reader.c +++ b/src/config/legacy_reader.c @@ -77,9 +77,12 @@ bool getLogFilePathLegacy(struct config *conf, FILE *fp) strerror(errno), errno); exit(EXIT_FAILURE); } + + fclose(fp); + return true; } // Use sscanf() to obtain filename from config file parameter only if buffer != NULL - else if(sscanf(buffer, "%127ms", &val_buffer) == 0) + else if((val_buffer = calloc(128, sizeof(char))) == NULL || sscanf(buffer, "%127s", val_buffer) == 0) { // Free previously allocated memory (if any) if(conf->files.log.ftl.t == CONF_STRING_ALLOCATED) @@ -91,7 +94,8 @@ bool getLogFilePathLegacy(struct config *conf, FILE *fp) log_info("Using syslog facility"); } - if(val_buffer) + // Set string if memory allocation was successful and a value was read + if(val_buffer != NULL && strlen(val_buffer) > 0) { // Free previously allocated memory (if any) if(conf->files.log.ftl.t == CONF_STRING_ALLOCATED) @@ -589,35 +593,36 @@ const char *readFTLlegacy(struct config *conf) return path; } -static char* getPath(FILE* fp, const char *option, char *ptr) +static char *getPath(FILE* fp, const char *option, char *path_default) { // This subroutine is used to read paths from pihole-FTL.conf - // fp: File ptr to opened and readable config file - // option: Option string ("key") to try to read - // ptr: Location where read (or default) parameter is stored + // fp: File path to opened and readable config file + // option: Option string ("key") to try to read + // path_default: Location where read (or default) parameter is stored char *buffer = parseFTLconf(fp, option); errno = 0; // Use sscanf() to obtain filename from config file parameter only if buffer != NULL - if(buffer == NULL || sscanf(buffer, "%127ms", &ptr) != 1) + char *val_ptr = calloc(128, sizeof(char)); + if(buffer == NULL || sscanf(buffer, "%127s", val_ptr) != 1) { // Use standard path if no custom path was obtained from the config file - return ptr; + return path_default; } // Test if memory allocation was successful - if(ptr == NULL) + if(val_ptr == NULL) { log_crit("Allocating memory for %s failed (%s, %i). Exiting.", option, strerror(errno), errno); exit(EXIT_FAILURE); } - else if(strlen(ptr) == 0) + else if(strlen(val_ptr) == 0) { log_info(" %s: Empty path is not possible, using default", option); } - return ptr; + return val_ptr; } static char *parseFTLconf(FILE *fp, const char * key) @@ -703,7 +708,7 @@ void releaseConfigMemory(void) void init_config_mutex(void) { // Initialize the lock attributes - pthread_mutexattr_t lock_attr = {}; + pthread_mutexattr_t lock_attr; pthread_mutexattr_init(&lock_attr); // Initialize the lock diff --git a/src/config/password.c b/src/config/password.c index d97df6d2..c1231ed7 100644 --- a/src/config/password.c +++ b/src/config/password.c @@ -315,10 +315,7 @@ char * __attribute__((malloc)) create_password(const char *password) enum password_result verify_login(const char *password) { enum password_result pw = verify_password(password, config.webserver.api.pwhash.v.s, true); - if(pw == PASSWORD_CORRECT) - log_debug(DEBUG_API, "Password correct"); - else - log_debug(DEBUG_API, "Password incorrect"); + log_debug(DEBUG_API, pw == PASSWORD_CORRECT ? "Password correct" : "Password incorrect"); // Check if an application password is set and if it matches if(pw == PASSWORD_INCORRECT && diff --git a/src/config/validator.c b/src/config/validator.c index 543057d7..06b8affd 100644 --- a/src/config/validator.c +++ b/src/config/validator.c @@ -123,12 +123,6 @@ bool validate_dns_cnames(union conf_value *val, const char *key, char err[VALIDA return false; } - // Count the number of elements in the string - unsigned int elements = 1; - for(unsigned int j = 0; j < strlen(item->valuestring); j++) - if(item->valuestring[j] == ',') - elements++; - // Check if it's in the form ",[,][,]" // is optional and may be repeated char *str = strdup(item->valuestring); @@ -398,12 +392,6 @@ bool validate_dns_revServers(union conf_value *val, const char *key, char err[VA return false; } - // Count the number of elements in the string - unsigned int elements = 1; - for(unsigned int j = 0; j < strlen(item->valuestring); j++) - if(item->valuestring[j] == ',') - elements++; - // Check if it's in the form ",[/],[#]," // Mandatory elements are: , , , and // Optional elements are: [/] and [#] diff --git a/src/database/CMakeLists.txt b/src/database/CMakeLists.txt index 3a16bf9b..b0a4597d 100644 --- a/src/database/CMakeLists.txt +++ b/src/database/CMakeLists.txt @@ -20,6 +20,10 @@ set(sqlite3_sources add_library(sqlite3 OBJECT ${sqlite3_sources}) target_compile_options(sqlite3 PRIVATE -Wno-implicit-fallthrough -Wno-cast-function-type -Wno-sign-compare) +if (CMAKE_C_COMPILER_ID STREQUAL "Clang") + target_compile_options(sqlite3 PRIVATE "-Wno-null-pointer-subtraction") +endif() + set(database_sources common.c common.h diff --git a/src/database/common.c b/src/database/common.c index 8d6756c2..f209a50f 100644 --- a/src/database/common.c +++ b/src/database/common.c @@ -253,11 +253,13 @@ void SQLite3LogCallback(void *pArg, int iErrCode, const char *zMsg) if(iErrCode == SQLITE_WARNING) log_warn("SQLite3: %s (%d)", zMsg, iErrCode); else if(iErrCode == SQLITE_NOTICE || iErrCode == SQLITE_SCHEMA) + { // SQLITE_SCHEMA is returned when the database schema has changed // This is not necessarily an error, as sqlite3_step() will re-prepare // the statement and try again. If it cannot, it will return an error // and this will be handled over there. log_debug(DEBUG_ANY, "SQLite3: %s (%d)", zMsg, iErrCode); + } else log_err("SQLite3: %s (%d)", zMsg, iErrCode); } diff --git a/src/database/common.h b/src/database/common.h index d2369185..5dd0c875 100644 --- a/src/database/common.h +++ b/src/database/common.h @@ -36,7 +36,7 @@ bool db_set_FTL_property(sqlite3* db, const enum ftl_table_props ID, const int v bool db_set_FTL_property_double(sqlite3* db, const enum ftl_table_props ID, const double value); /// Execute a formatted SQL query and get the return code -int dbquery(sqlite3* db, const char *format, ...) __attribute__ ((format (gnu_printf, 2, 3)));; +int dbquery(sqlite3* db, const char *format, ...) __attribute__ ((format (printf, 2, 3)));; #define dbopen(readonly, create) _dbopen(readonly, create, __FUNCTION__, __LINE__, __FILE__) sqlite3 *_dbopen(const bool readonly, const bool create, const char *func, const int line, const char *file) __attribute__((warn_unused_result)); diff --git a/src/database/query-table.c b/src/database/query-table.c index f66d5152..40949bc2 100644 --- a/src/database/query-table.c +++ b/src/database/query-table.c @@ -1046,7 +1046,7 @@ void DB_read_queries(void) (buffer = (const char *)sqlite3_column_text(stmt, 6)) != NULL) { // Get IP address and port of upstream destination - char serv_addr[INET6_ADDRSTRLEN] = { 0 }; + char serv_addr[INET6_ADDRSTRLEN + 1] = { 0 }; unsigned int serv_port = 53; // We limit the number of bytes written into the serv_addr buffer // to prevent buffer overflows. If there is no port available in diff --git a/src/database/sqlite3-ext.c b/src/database/sqlite3-ext.c index f52ef029..e3e28498 100644 --- a/src/database/sqlite3-ext.c +++ b/src/database/sqlite3-ext.c @@ -215,4 +215,4 @@ int sqlite3_pihole_extensions_init(sqlite3 *db, const char **pzErrMsg, const str } return rc; -} \ No newline at end of file +} diff --git a/src/dnsmasq/CMakeLists.txt b/src/dnsmasq/CMakeLists.txt index 2497bc72..9caab62f 100644 --- a/src/dnsmasq/CMakeLists.txt +++ b/src/dnsmasq/CMakeLists.txt @@ -65,5 +65,9 @@ set(sources add_library(dnsmasq OBJECT ${sources}) target_compile_definitions(dnsmasq PRIVATE VERSION=\"${DNSMASQ_VERSION}\") target_compile_definitions(dnsmasq PRIVATE CONFFILE=\"/etc/pihole/dnsmasq.conf\") -target_compile_options(dnsmasq PRIVATE -Wno-maybe-uninitialized) +if (CMAKE_C_COMPILER_ID STREQUAL "GNU") + target_compile_options(dnsmasq PRIVATE -Wno-maybe-uninitialized) +elseif (CMAKE_C_COMPILER_ID STREQUAL "Clang") + target_compile_options(dnsmasq PRIVATE -Wno-gnu-variable-sized-type-not-at-end -Wno-sign-compare -Wno-deprecated-non-prototype) +endif() target_include_directories(dnsmasq PRIVATE ${PROJECT_SOURCE_DIR}/src ${PROJECT_SOURCE_DIR}/src/lua) diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c index f816abb6..d17991bf 100644 --- a/src/dnsmasq_interface.c +++ b/src/dnsmasq_interface.c @@ -198,10 +198,7 @@ size_t _FTL_make_answer(struct dns_header *header, char *limit, const size_t len return 0; // Debug logging - if(*ede != EDE_UNSET) - log_debug(DEBUG_QUERIES, "Preparing reply for \"%s\", EDE: %s (%d)", name, edestr(*ede), *ede); - else - log_debug(DEBUG_QUERIES, "Preparing reply for \"%s\", EDE: N/A", name); + log_debug(DEBUG_QUERIES, "Preparing reply for \"%s\", EDE: %s (%d)", name, *ede != EDE_UNSET ? edestr(*ede) : "N/A", *ede); // Get question type int qtype, flags = 0; @@ -844,13 +841,17 @@ bool _FTL_new_query(const unsigned int flags, const char *name, if(config.debug.arp.v.b) { if(client->hwlen == 6) + { log_debug(DEBUG_ARP, "find_mac(\"%s\") returned hardware address " "%02X:%02X:%02X:%02X:%02X:%02X", clientIP, client->hwaddr[0], client->hwaddr[1], client->hwaddr[2], client->hwaddr[3], client->hwaddr[4], client->hwaddr[5]); + } else + { log_debug(DEBUG_ARP, "find_mac(\"%s\") returned %i bytes of data", clientIP, client->hwlen); + } } } @@ -1997,10 +1998,12 @@ static void FTL_reply(const unsigned int flags, const char *name, const union al dispname = "."; if(cached || last_server.sa.sa_family == 0) + { // Log cache or upstream reply from unknown source log_debug(DEBUG_QUERIES, "**** got %s%s reply: %s is %s (ID %i, %s:%i)", stale ? "stale ": "", cached ? "cache" : "upstream", dispname, answer, id, file, line); + } else { char ip[ADDRSTRLEN+1] = { 0 }; @@ -3561,4 +3564,4 @@ void FTL_connection_error(const char *reason, const union mysockaddr *addr) if(server != NULL) free(server); } -} \ No newline at end of file +} diff --git a/src/edns0.c b/src/edns0.c index a310a3d9..8f27a184 100644 --- a/src/edns0.c +++ b/src/edns0.c @@ -419,4 +419,4 @@ void FTL_parse_pseudoheaders(unsigned char *pheader, const size_t plen) p += optlen; } } -} \ No newline at end of file +} diff --git a/src/enums.h b/src/enums.h index ab09498a..09769a9c 100644 --- a/src/enums.h +++ b/src/enums.h @@ -134,6 +134,7 @@ enum domain_client_status { } __attribute__ ((packed)); enum debug_flag { + DEBUG_NONE = 0, DEBUG_DATABASE = 1, DEBUG_NETWORKING, DEBUG_LOCKS, diff --git a/src/events.h b/src/events.h index ea5b7f27..3ae0a70d 100644 --- a/src/events.h +++ b/src/events.h @@ -19,4 +19,4 @@ void _set_event(const enum events event, int line, const char *function, const c #define get_and_clear_event(event) _get_and_clear_event(event, __LINE__, __FUNCTION__, __FILE__) bool _get_and_clear_event(const enum events event, int line, const char *function, const char *file); -#endif // EVENTS_H \ No newline at end of file +#endif // EVENTS_H diff --git a/src/log.c b/src/log.c index e861c922..f57004a2 100644 --- a/src/log.c +++ b/src/log.c @@ -219,12 +219,13 @@ const char *debugstr(const enum debug_flag flag) return "DEBUG_RESERVED"; case DEBUG_MAX: return "DEBUG_MAX"; + case DEBUG_NONE: // fall through default: return "DEBUG_ANY"; } } -void __attribute__ ((format (gnu_printf, 3, 4))) _FTL_log(const int priority, const enum debug_flag flag, const char *format, ...) +void __attribute__ ((format (printf, 3, 4))) _FTL_log(const int priority, const enum debug_flag flag, const char *format, ...) { char timestring[TIMESTR_SIZE] = ""; va_list args; @@ -321,7 +322,7 @@ void __attribute__ ((format (gnu_printf, 3, 4))) _FTL_log(const int priority, co } } -void __attribute__ ((format (gnu_printf, 1, 2))) log_web(const char *format, ...) +void __attribute__ ((format (printf, 1, 2))) log_web(const char *format, ...) { char timestring[TIMESTR_SIZE] = ""; const time_t now = time(NULL); @@ -362,7 +363,7 @@ void __attribute__ ((format (gnu_printf, 1, 2))) log_web(const char *format, ... } // Log helper activity (may be script or lua) -void FTL_log_helper(const unsigned char n, ...) +void FTL_log_helper(const unsigned int n, ...) { // Only log helper debug messages if enabled if(!(config.debug.helper.v.b)) diff --git a/src/log.h b/src/log.h index 215b0bec..d794191b 100644 --- a/src/log.h +++ b/src/log.h @@ -53,7 +53,7 @@ void log_FTL_version(bool crashreport); double double_time(void); void get_timestr(char timestring[TIMESTR_SIZE], const time_t timein, const bool millis, const bool uri_compatible); const char *debugstr(const enum debug_flag flag) __attribute__((const)); -void log_web(const char *format, ...) __attribute__ ((format (gnu_printf, 1, 2))); +void log_web(const char *format, ...) __attribute__ ((format (printf, 1, 2))); const char *get_ordinal_suffix(unsigned int number) __attribute__ ((const)); void print_FTL_version(void); unsigned int countchar(const char *str, const char c) __attribute__ ((pure)); @@ -66,14 +66,13 @@ void dnsmasq_diagnosis_warning(char *message); #define log_warn(format, ...) _FTL_log(LOG_WARNING, 0, format, ## __VA_ARGS__) #define log_notice(format, ...) _FTL_log(LOG_NOTICE, 0, format, ## __VA_ARGS__) #define log_info(format, ...) _FTL_log(LOG_INFO, 0, format, ## __VA_ARGS__) -#define log_debug(flag, format, ...)({ \ +#define log_debug(flag, format, ...) \ if(flag > -1 && flag < DEBUG_MAX && debug_flags[flag]) \ - _FTL_log(LOG_DEBUG, flag, format, ## __VA_ARGS__); \ -}) -void _FTL_log(const int priority, const enum debug_flag flag, const char *format, ...) __attribute__ ((format (gnu_printf, 3, 4))); -void FTL_log_dnsmasq_fatal(const char *format, ...) __attribute__ ((format (gnu_printf, 1, 2))); + _FTL_log(LOG_DEBUG, flag, format, ## __VA_ARGS__) +void _FTL_log(const int priority, const enum debug_flag flag, const char *format, ...) __attribute__ ((format (printf, 3, 4))); +void FTL_log_dnsmasq_fatal(const char *format, ...) __attribute__ ((format (printf, 1, 2))); void log_ctrl(bool vlog, bool vstdout); -void FTL_log_helper(const unsigned char n, ...); +void FTL_log_helper(const unsigned int n, ...); int binbuf_to_escaped_C_literal(const char *src_buf, size_t src_sz, char *dst_str, size_t dst_sz); @@ -84,7 +83,7 @@ int blocked_queries(void) __attribute__ ((pure)); const char *short_path(const char *full_path) __attribute__ ((pure)); // How long is each line in the FIFO buffer allowed to be? -#define MAX_MSG_FIFO 256u +#define MAX_MSG_FIFO 260u // How many messages do we keep in memory (FIFO message buffer)? // This number multiplied by MAX_MSG_FIFO (see above) gives the total buffer size @@ -97,9 +96,9 @@ bool flush_dnsmasq_log(void); typedef struct { struct { + char message[LOG_SIZE][MAX_MSG_FIFO]; unsigned int next_id; double timestamp[LOG_SIZE]; - char message[LOG_SIZE][MAX_MSG_FIFO]; const char *prio[LOG_SIZE]; } logs[FIFO_MAX]; } fifologData; diff --git a/src/lua/CMakeLists.txt b/src/lua/CMakeLists.txt index 908b5a7e..e319393e 100644 --- a/src/lua/CMakeLists.txt +++ b/src/lua/CMakeLists.txt @@ -65,7 +65,9 @@ set(sources ) add_library(lua OBJECT ${sources}) -target_compile_options(lua PRIVATE -Wno-maybe-uninitialized -Wno-unused-variable -Wno-unused-value) +if (CMAKE_C_COMPILER_ID STREQUAL "GNU") + target_compile_options(lua PRIVATE -Wno-maybe-uninitialized -Wno-unused-variable -Wno-unused-value) +endif() # LUA_USE_POSIX: ensures recommended POSIX functions are used instead of # (partially obsoleted) standard C functions diff --git a/src/lua/ftl_lua.c b/src/lua/ftl_lua.c index 5f448406..0948ed06 100644 --- a/src/lua/ftl_lua.c +++ b/src/lua/ftl_lua.c @@ -20,7 +20,9 @@ #include "../files.h" // get_web_theme_str #include "../datastructure.h" +#if HAVE_READLINE #include +#endif #include #include "scripts/scripts.h" diff --git a/src/lua/ftl_lua.h b/src/lua/ftl_lua.h index 29c4cb5f..d986498a 100644 --- a/src/lua/ftl_lua.h +++ b/src/lua/ftl_lua.h @@ -26,4 +26,4 @@ extern int dolibrary (lua_State *L, char *name); void print_embedded_scripts(void); void ftl_lua_init(lua_State *L); -#endif //FTL_LUA_H \ No newline at end of file +#endif //FTL_LUA_H diff --git a/src/procps.h b/src/procps.h index 986d79bb..e707ed6b 100644 --- a/src/procps.h +++ b/src/procps.h @@ -43,4 +43,4 @@ bool read_self_memory_status(struct statm_t *result); bool getProcessMemory(struct proc_mem *mem, const unsigned long total_memory); bool parse_proc_meminfo(struct proc_meminfo *mem); -#endif // PROCPS_H \ No newline at end of file +#endif // PROCPS_H diff --git a/src/resolve.c b/src/resolve.c index b083b0cc..f83c197b 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -66,7 +66,7 @@ struct DNS_HEADER uint16_t auth_count; // number of authority entries uint16_t add_count; // number of resource entries } __attribute__((packed)); -static_assert(sizeof(struct DNS_HEADER) == 12); +static_assert(sizeof(struct DNS_HEADER) == 12, "DNS_HEADER size mismatch"); // Constant sized fields of query structure struct QUESTION @@ -74,7 +74,7 @@ struct QUESTION uint16_t qtype; uint16_t qclass; }; -static_assert(sizeof(struct QUESTION) == 4); +static_assert(sizeof(struct QUESTION) == 4, "QUESTION size mismatch"); // Constant sized fields of the resource record structure struct R_DATA @@ -84,7 +84,7 @@ struct R_DATA uint32_t ttl; // RFC 1035 defines the TTL field as "positive values of a signed 32bit number" uint16_t data_len; } __attribute__((packed)); -static_assert(sizeof(struct R_DATA) == 10); +static_assert(sizeof(struct R_DATA) == 10, "R_DATA size mismatch"); _Pragma("GCC diagnostic pop") // Pointers to resource record contents diff --git a/src/shmem.c b/src/shmem.c index 1d267146..44f72eb8 100644 --- a/src/shmem.c +++ b/src/shmem.c @@ -286,7 +286,7 @@ const char *_getstr(const size_t pos, const char *func, const int line, const ch // Create a mutex for shared memory static void create_mutex(pthread_mutex_t *lock) { log_debug(DEBUG_SHMEM, "Creating SHM mutex lock"); - pthread_mutexattr_t lock_attr = {}; + pthread_mutexattr_t lock_attr; // Initialize the lock attributes pthread_mutexattr_init(&lock_attr); @@ -740,11 +740,15 @@ static bool realloc_shm(SharedMemory *sharedMemory, const size_t size1, const si // Log output if(resize) + { log_debug(DEBUG_SHMEM, "Resizing \"%s\" from %zu to (%zu * %zu) == %zu (%s)", sharedMemory->name, sharedMemory->size, size1, size2, size, df); + } else + { log_debug(DEBUG_SHMEM, "Remapping \"%s\" from %zu to (%zu * %zu) == %zu", sharedMemory->name, sharedMemory->size, size1, size2, size); + } if(config.misc.check.shmem.v.ui > 0 && percentage > config.misc.check.shmem.v.ui) log_resource_shortage(-1.0, 0, percentage, -1, SHMEM_PATH, df); @@ -801,11 +805,15 @@ static bool realloc_shm(SharedMemory *sharedMemory, const size_t size1, const si used_shmem += (size - sharedMemory->size); if(sharedMemory->ptr == new_ptr) + { log_debug(DEBUG_SHMEM, "SHMEM pointer not updated: %p (%zu %zu)", sharedMemory->ptr, sharedMemory->size, size); + } else + { log_debug(DEBUG_SHMEM, "SHMEM pointer updated: %p -> %p (%zu %zu)", sharedMemory->ptr, new_ptr, sharedMemory->size, size); + } sharedMemory->ptr = new_ptr; sharedMemory->size = size; diff --git a/src/struct_size.h b/src/struct_size.h index 94c692c7..53bfe12b 100644 --- a/src/struct_size.h +++ b/src/struct_size.h @@ -15,4 +15,4 @@ int check_one_struct(const char *struct_name, const size_t found_size, const size_t size64, const size_t size32); -#endif // STRUCT_SIZE_HEADER \ No newline at end of file +#endif // STRUCT_SIZE_HEADER diff --git a/src/syscalls/accept.c b/src/syscalls/accept.c index 710a7f3d..60edbf62 100644 --- a/src/syscalls/accept.c +++ b/src/syscalls/accept.c @@ -39,4 +39,4 @@ int FTLaccept(int sockfd, struct sockaddr *addr, socklen_t *addrlen, const char errno = _errno; return ret; -} \ No newline at end of file +} diff --git a/src/syscalls/recv.c b/src/syscalls/recv.c index c0e77ecf..bdae2b25 100644 --- a/src/syscalls/recv.c +++ b/src/syscalls/recv.c @@ -41,4 +41,4 @@ ssize_t FTLrecv(int sockfd, void *buf, size_t len, int flags, const char *file, errno = _errno; return ret; -} \ No newline at end of file +} diff --git a/src/syscalls/recvfrom.c b/src/syscalls/recvfrom.c index d40dfadf..b78f8bf5 100644 --- a/src/syscalls/recvfrom.c +++ b/src/syscalls/recvfrom.c @@ -45,4 +45,4 @@ ssize_t FTLrecvfrom(int sockfd, void *buf, size_t len, int flags, struct sockadd errno = _errno; return ret; -} \ No newline at end of file +} diff --git a/src/syscalls/select.c b/src/syscalls/select.c index 5eb07c9a..b6889fb6 100644 --- a/src/syscalls/select.c +++ b/src/syscalls/select.c @@ -41,4 +41,4 @@ int FTLselect(int nfds, fd_set *readfds, fd_set *writefds, fd_set *exceptfds, st errno = _errno; return ret; -} \ No newline at end of file +} diff --git a/src/syscalls/sendto.c b/src/syscalls/sendto.c index 4e7b4a8e..b0cf0141 100644 --- a/src/syscalls/sendto.c +++ b/src/syscalls/sendto.c @@ -43,4 +43,4 @@ ssize_t FTLsendto(int sockfd, void *buf, size_t len, int flags, const struct soc errno = _errno; return ret; -} \ No newline at end of file +} diff --git a/src/syscalls/strdup.c b/src/syscalls/strdup.c index f83dde47..bd2d7b41 100644 --- a/src/syscalls/strdup.c +++ b/src/syscalls/strdup.c @@ -35,4 +35,4 @@ char* __attribute__((malloc)) FTLstrdup(const char *src, const char *file, const dest[len] = '\0'; return dest; -} \ No newline at end of file +} diff --git a/src/syscalls/syscalls.h b/src/syscalls/syscalls.h index 3d77ebe7..77e7725c 100644 --- a/src/syscalls/syscalls.h +++ b/src/syscalls/syscalls.h @@ -21,17 +21,17 @@ int FTLfallocate(const int fd, const off_t offset, const off_t len, const char * // Interrupt-safe printing routines // printf() is derived from fprintf(stdout, ...) // vprintf() is derived from vfprintf(stdout, ...) -int FTLfprintf(FILE *stream, const char*file, const char *func, const int line, const char *format, ...) __attribute__ ((format (gnu_printf, 5, 6))); -int FTLvfprintf(FILE *stream, const char*file, const char *func, const int line, const char *format, va_list args) __attribute__ ((format (gnu_printf, 5, 0))); +int FTLfprintf(FILE *stream, const char*file, const char *func, const int line, const char *format, ...) __attribute__ ((format (printf, 5, 6))); +int FTLvfprintf(FILE *stream, const char*file, const char *func, const int line, const char *format, va_list args) __attribute__ ((format (printf, 5, 0))); -int FTLsprintf(const char *file, const char *func, const int line, char *__restrict__ buffer, const char *format, ...) __attribute__ ((format (gnu_printf, 5, 6))); -int FTLvsprintf(const char *file, const char *func, const int line, char *__restrict__ buffer, const char *format, va_list args) __attribute__ ((format (gnu_printf, 5, 0))); +int FTLsprintf(const char *file, const char *func, const int line, char *__restrict__ buffer, const char *format, ...) __attribute__ ((format (printf, 5, 6))); +int FTLvsprintf(const char *file, const char *func, const int line, char *__restrict__ buffer, const char *format, va_list args) __attribute__ ((format (printf, 5, 0))); -int FTLasprintf(const char *file, const char *func, const int line, char **buffer, const char *format, ...) __attribute__ ((format (gnu_printf, 5, 6))); -int FTLvasprintf(const char *file, const char *func, const int line, char **buffer, const char *format, va_list args) __attribute__ ((format (gnu_printf, 5, 0))); +int FTLasprintf(const char *file, const char *func, const int line, char **buffer, const char *format, ...) __attribute__ ((format (printf, 5, 6))); +int FTLvasprintf(const char *file, const char *func, const int line, char **buffer, const char *format, va_list args) __attribute__ ((format (printf, 5, 0))); -int FTLsnprintf(const char *file, const char *func, const int line, char *__restrict__ buffer, const size_t maxlen, const char *format, ...) __attribute__ ((format (gnu_printf, 6, 7))); -int FTLvsnprintf(const char *file, const char *func, const int line, char *__restrict__ buffer, const size_t maxlen, const char *format, va_list args) __attribute__ ((format (gnu_printf, 6, 0))); +int FTLsnprintf(const char *file, const char *func, const int line, char *__restrict__ buffer, const size_t maxlen, const char *format, ...) __attribute__ ((format (printf, 6, 7))); +int FTLvsnprintf(const char *file, const char *func, const int line, char *__restrict__ buffer, const size_t maxlen, const char *format, va_list args) __attribute__ ((format (printf, 6, 0))); // Interrupt-safe socket routines ssize_t FTLwrite(int fd, const void *buf, size_t total, const char *file, const char *func, const int line); diff --git a/src/syscalls/write.c b/src/syscalls/write.c index df62adda..e145007f 100644 --- a/src/syscalls/write.c +++ b/src/syscalls/write.c @@ -50,4 +50,4 @@ ssize_t FTLwrite(int fd, const void *buf, size_t total, const char *file, const // Return number of written bytes return written; -} \ No newline at end of file +} diff --git a/src/tools/arp-scan.c b/src/tools/arp-scan.c index 680a90a3..923625a3 100644 --- a/src/tools/arp-scan.c +++ b/src/tools/arp-scan.c @@ -381,7 +381,7 @@ static void *arp_scan_iface(void *args) thread_data->dst_cidr = netmask_to_cidr(&thread_data->mask.sin_addr); // Get interface index - const int ifindex = if_nametoindex(iface); + const int ifindex = (int)if_nametoindex(iface); // Scan only interfaces with CIDR >= 24 if(thread_data->dst_cidr < 24 && !thread_data->scan_all) @@ -701,7 +701,7 @@ int run_arp_scan(const bool scan_all, const bool extreme_mode) { // Calculate progress (total number of scans / total number of addresses) // We add 1 to total_scans to avoid division by zero - const unsigned int new_progress = 100 * num_scans / (total_scans + 1); + const unsigned int new_progress = 100 * (unsigned int)(num_scans / (total_scans + 1)); if(new_progress > progress) { // Print progress diff --git a/src/tools/dhcp-discover.c b/src/tools/dhcp-discover.c index 85e994b4..c68a74f9 100644 --- a/src/tools/dhcp-discover.c +++ b/src/tools/dhcp-discover.c @@ -54,15 +54,12 @@ // we scan for DHCP activity. #define MAXTHREADS 32 -// Probe DHCP servers responding to the broadcast address -#define PROBE_BCAST - // Should we generate test data for DHCP option 249? //#define TEST_OPT_249 // Global lock used by all threads static pthread_mutex_t lock; -static void __attribute__((format(gnu_printf, 1, 2))) printf_locked(const char *format, ...) +static void __attribute__((format(printf, 1, 2))) printf_locked(const char *format, ...) { va_list args; va_start(args, format); @@ -179,7 +176,7 @@ struct dhcp_packet_data unsigned char chaddr [MAX_DHCP_CHADDR_LENGTH]; // hardware address of this machine char sname [MAX_DHCP_SNAME_LENGTH]; // name of DHCP server char file [MAX_DHCP_FILE_LENGTH]; // boot file name (used for diskless booting?) - char options[MAX_DHCP_OPTIONS_LENGTH]; // options + unsigned char options[MAX_DHCP_OPTIONS_LENGTH]; // options }; // sends a DHCPDISCOVER message to the specified in an attempt to find DHCP servers @@ -219,7 +216,7 @@ static bool send_dhcp_discover(const int sock, const uint32_t xid, const char *i discover_packet.options[6] = 1; // DHCP message type code for DHCPDISCOVER // Place end option at the end of the options - discover_packet.options[7] = 255; + discover_packet.options[7] = (char)255; // Send the DHCPDISCOVER packet to the specified address struct sockaddr_in target = { 0 }; @@ -236,7 +233,7 @@ static bool send_dhcp_discover(const int sock, const uint32_t xid, const char *i printf_locked("DHCDISCOVER giaddr: %s\n", inet_ntoa(discover_packet.giaddr)); #endif // send the DHCPDISCOVER packet - const int bytes = sendto(sock, (char *)&discover_packet, sizeof(discover_packet), 0, (struct sockaddr *)&target, sizeof(target)); + const ssize_t bytes = sendto(sock, (char *)&discover_packet, sizeof(discover_packet), 0, (struct sockaddr *)&target, sizeof(target)); if(bytes < 0) { // strerror() returns "Required key not available" for ENOKEY @@ -250,7 +247,7 @@ static bool send_dhcp_discover(const int sock, const uint32_t xid, const char *i } #ifdef DEBUG - printf_locked("Sent %d bytes\n", bytes); + printf_locked("Sent %zu bytes\n", (size_t)bytes); #endif return true; } @@ -340,7 +337,7 @@ static void print_dhcp_offer(struct in_addr source, struct dhcp_packet_data *off // possible "(empty)" const size_t bufsiz = 4*optlen + 9; char *buffer = calloc(bufsiz, sizeof(char)); - binbuf_to_escaped_C_literal(&offer_packet->options[x], optlen, buffer, bufsiz); + binbuf_to_escaped_C_literal((char*)&offer_packet->options[x], optlen, buffer, bufsiz); printf("%s: \"%s\"\n", opttab[i].name, buffer); free(buffer); } @@ -428,7 +425,7 @@ static void print_dhcp_offer(struct in_addr source, struct dhcp_packet_data *off // chars per control character plus room for // possible "(empty)" char *buffer = calloc(4*optlen + 9, sizeof(char)); - binbuf_to_escaped_C_literal(&offer_packet->options[x], optlen, buffer, sizeof(buffer)); + binbuf_to_escaped_C_literal((char*)&offer_packet->options[x], optlen, buffer, sizeof(buffer)); printf("wpad-server: \"%s\"\n", buffer); free(buffer); } @@ -730,7 +727,7 @@ int run_dhcp_discover(void) pthread_attr_init(&attr); // Create processing/printfing lock - pthread_mutexattr_t lock_attr = {}; + pthread_mutexattr_t lock_attr; // Initialize the lock attributes pthread_mutexattr_init(&lock_attr); // Initialize the lock diff --git a/src/tre-regex/CMakeLists.txt b/src/tre-regex/CMakeLists.txt index f4a8ba96..cdf8ec2e 100644 --- a/src/tre-regex/CMakeLists.txt +++ b/src/tre-regex/CMakeLists.txt @@ -27,4 +27,6 @@ set(sources ) add_library(tre-regex OBJECT ${sources}) -target_compile_options(tre-regex PRIVATE -Wno-maybe-uninitialized -Wno-unused-value -Wno-empty-body) +if (CMAKE_C_COMPILER_ID STREQUAL "GNU") + target_compile_options(tre-regex PRIVATE -Wno-maybe-uninitialized -Wno-unused-value -Wno-empty-body) +endif() diff --git a/src/webserver/lua_web.h b/src/webserver/lua_web.h index 478a28a0..5c0b2fe6 100644 --- a/src/webserver/lua_web.h +++ b/src/webserver/lua_web.h @@ -18,4 +18,4 @@ void free_lua(void); void init_lua(const struct mg_connection *conn, void *L, unsigned context_flags); int request_handler(struct mg_connection *conn, void *cbdata); -#endif // LUA_WEB_H \ No newline at end of file +#endif // LUA_WEB_H diff --git a/src/webserver/webserver.c b/src/webserver/webserver.c index 5485046f..ec6c9c7a 100644 --- a/src/webserver/webserver.c +++ b/src/webserver/webserver.c @@ -469,7 +469,8 @@ void http_init(void) } // Configure logging handlers - struct mg_callbacks callbacks = { NULL }; + struct mg_callbacks callbacks; + memset(&callbacks, 0, sizeof(callbacks)); callbacks.log_message = log_http_message; callbacks.log_access = log_http_access; callbacks.init_lua = init_lua; diff --git a/src/webserver/webserver.h b/src/webserver/webserver.h index d87001f4..4fff052a 100644 --- a/src/webserver/webserver.h +++ b/src/webserver/webserver.h @@ -18,4 +18,4 @@ void http_terminate(void); in_port_t get_https_port(void) __attribute__((pure)); unsigned short get_api_string(char **buf, const bool domain); -#endif // WEBSERVER_H \ No newline at end of file +#endif // WEBSERVER_H diff --git a/src/zip/gzip.c b/src/zip/gzip.c index 74aed94c..aa31d86a 100644 --- a/src/zip/gzip.c +++ b/src/zip/gzip.c @@ -92,7 +92,7 @@ static bool deflate_buffer(const unsigned char *buffer_uncompressed, const mz_ul // ITU-T V.42.) // isize: This contains the size of the original (uncompressed) input // data modulo 2^32 (little endian). - const uint32_t crc = mz_crc32(MZ_CRC32_INIT, buffer_uncompressed, size_uncompressed); + const uint32_t crc = (uint32_t)mz_crc32(MZ_CRC32_INIT, buffer_uncompressed, size_uncompressed); memcpy(*buffer_compressed + *size_compressed, &crc, sizeof(crc)); *size_compressed += sizeof(crc); const uint32_t isize = htole32(size_uncompressed); @@ -313,7 +313,15 @@ bool inflate_file(const char *infilename, const char *outfilename, bool verbose) // Get file size fseek(infile, 0, SEEK_END); - const mz_ulong size_compressed = ftell(infile); + const long sc = ftell(infile); + if(sc < 0) + { + log_warn("Failed to get file size of %s", infilename); + fclose(infile); + fclose(outfile); + return false; + } + const mz_ulong size_compressed = (mz_ulong)sc; fseek(infile, 0, SEEK_SET); // Read file into memory @@ -398,7 +406,7 @@ bool deflate_file(const char *infilename, const char *outfilename, bool verbose) // Get file size fseek(infile, 0, SEEK_END); - const mz_ulong size_uncompressed = ftell(infile); + const long size_uncompressed = ftell(infile); fseek(infile, 0, SEEK_SET); // Read file into memory @@ -410,7 +418,7 @@ bool deflate_file(const char *infilename, const char *outfilename, bool verbose) fclose(outfile); return false; } - if(fread(buffer_uncompressed, 1, size_uncompressed, infile) != size_uncompressed) + if(fread(buffer_uncompressed, 1, size_uncompressed, infile) != (size_t)size_uncompressed) { log_warn("Failed to read %lu bytes from %s", (unsigned long)size_uncompressed, infilename); fclose(infile); diff --git a/src/zip/miniz/CMakeLists.txt b/src/zip/miniz/CMakeLists.txt index c6e7de96..40803374 100644 --- a/src/zip/miniz/CMakeLists.txt +++ b/src/zip/miniz/CMakeLists.txt @@ -15,4 +15,5 @@ set(sources add_library(miniz OBJECT ${sources}) target_compile_options(miniz PRIVATE) +target_compile_options(miniz PRIVATE "-Wno-padded") target_include_directories(miniz PRIVATE ${PROJECT_SOURCE_DIR}/src) diff --git a/src/zip/miniz/miniz.h b/src/zip/miniz/miniz.h index d6a354bf..35c740c7 100644 --- a/src/zip/miniz/miniz.h +++ b/src/zip/miniz/miniz.h @@ -1419,4 +1419,4 @@ MINIZ_EXPORT void *mz_zip_extract_archive_file_to_heap_v2(const char *pZip_filen } #endif -#endif /* MINIZ_NO_ARCHIVE_APIS */ \ No newline at end of file +#endif /* MINIZ_NO_ARCHIVE_APIS */ diff --git a/src/zip/tar.c b/src/zip/tar.c index 5e497622..fa65cd6d 100644 --- a/src/zip/tar.c +++ b/src/zip/tar.c @@ -125,4 +125,4 @@ cJSON * __attribute__((nonnull (1))) list_files_in_tar(const uint8_t *tarData, c } while (p + newOffset + TAR_BLOCK_SIZE <= tarSize); return files; -} \ No newline at end of file +} diff --git a/src/zip/tar.h b/src/zip/tar.h index 11f5e200..a8f30769 100644 --- a/src/zip/tar.h +++ b/src/zip/tar.h @@ -16,4 +16,4 @@ const char *find_file_in_tar(const uint8_t *tar, const size_t tarSize, const char *fileName, size_t *fileSize) __attribute__((nonnull (1,3,4))); cJSON *list_files_in_tar(const uint8_t *tarData, const size_t tarSize) __attribute__((nonnull (1))); -#endif // TAR_H \ No newline at end of file +#endif // TAR_H From ecd2e8198bf259e5e0e0b8b27a55205fe8e02906 Mon Sep 17 00:00:00 2001 From: Jack'lul Date: Fri, 17 May 2024 18:03:57 +0200 Subject: [PATCH 074/339] Fix error message mentioning wrong file Signed-off-by: Jack'lul --- src/zip/teleporter.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/zip/teleporter.c b/src/zip/teleporter.c index 26480ca2..5eb02317 100644 --- a/src/zip/teleporter.c +++ b/src/zip/teleporter.c @@ -181,7 +181,7 @@ const char *generate_teleporter_zip(mz_zip_archive *zip, char filename[128], voi if(file_exists(file_path) && !mz_zip_writer_add_file(zip, file_path+1, file_path, file_comment, (uint16_t)strlen(file_comment), MZ_BEST_COMPRESSION)) { mz_zip_writer_end(zip); - return "Failed to add /etc/hosts to heap ZIP archive!"; + return "Failed to add /etc/pihole/dhcp.leases to heap ZIP archive!"; } const char *directory = "/etc/dnsmasq.d"; From 98407e3a58b09fb6993cee6ba682659689d1981c Mon Sep 17 00:00:00 2001 From: Erik Karlsson Date: Sat, 18 May 2024 09:13:54 +0200 Subject: [PATCH 075/339] Update DNS records after pruning DHCP leases Not doing so can result in a use after free since the name for DHCP derived DNS records is represented as a pointer into the DHCP lease table. Update will only happen when necessary since lease_update_dns tests internally on dns_dirty and the force argument is zero. Signed-off-by: Erik Karlsson Signed-off-by: DL6ER --- src/dnsmasq/dnsmasq.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/dnsmasq/dnsmasq.c b/src/dnsmasq/dnsmasq.c index 98ad525a..7990ed61 100644 --- a/src/dnsmasq/dnsmasq.c +++ b/src/dnsmasq/dnsmasq.c @@ -1534,6 +1534,7 @@ static void async_event(int pipe, time_t now) { lease_prune(NULL, now); lease_update_file(now); + lease_update_dns(0); } #ifdef HAVE_DHCP6 else if (daemon->doing_ra) From ebc195af698a48bd0e5cc07e242a5a78d04bb85b Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 18 May 2024 09:14:27 +0200 Subject: [PATCH 076/339] Update custom dnsmasq version Signed-off-by: DL6ER --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index a5c421ad..e116d4dc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -12,6 +12,6 @@ cmake_minimum_required(VERSION 2.8.12) project(PIHOLE_FTL C) -set(DNSMASQ_VERSION pi-hole-v2.90+1) +set(DNSMASQ_VERSION pi-hole-v2.90+2) add_subdirectory(src) From f8949103262f85c5c1e046aab513a75737a5c398 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 18 May 2024 10:59:40 +0200 Subject: [PATCH 077/339] Remove explicit static instructions - binaries compiled on alpine will balways e linked statically Signed-off-by: DL6ER --- src/CMakeLists.txt | 3 +-- src/lua/CMakeLists.txt | 2 +- src/lua/ftl_lua.c | 16 ++++++++++------ 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 8ab5494f..bb2ca546 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -308,9 +308,8 @@ find_library(LIBHISTORY NAMES libhistory${CMAKE_STATIC_LIBRARY_SUFFIX} history) find_library(LIBTERMCAP NAMES libtermcap${CMAKE_STATIC_LIBRARY_SUFFIX} termcap) if(LIBREADLINE AND LIBHISTORY AND LIBTERMCAP) message(STATUS "Building FTL with readline support: YES") - target_compile_definitions(FTL PRIVATE LUA_USE_READLINE) - target_compile_definitions(pihole-FTL PRIVATE LUA_USE_READLINE) target_link_libraries(pihole-FTL ${LIBREADLINE} ${LIBHISTORY} ${LIBTERMCAP}) + add_compile_definitions(HAVE_READLINE) set(HAVE_READLINE TRUE) else() message(STATUS "Building FTL with readline support: NO") diff --git a/src/lua/CMakeLists.txt b/src/lua/CMakeLists.txt index 908b5a7e..79a0305b 100644 --- a/src/lua/CMakeLists.txt +++ b/src/lua/CMakeLists.txt @@ -78,7 +78,7 @@ if(LUA_DL STREQUAL "true") target_compile_definitions(lua PRIVATE LUA_USE_DLOPEN) endif() -if(LIBREADLINE AND LIBHISTORY AND LIBTERMCAP) +if(HAVE_READLINE) message(STATUS "Embedded LUA will use readline for history: YES") target_compile_definitions(lua PRIVATE LUA_USE_READLINE) else() diff --git a/src/lua/ftl_lua.c b/src/lua/ftl_lua.c index 0948ed06..96ce412b 100644 --- a/src/lua/ftl_lua.c +++ b/src/lua/ftl_lua.c @@ -8,20 +8,24 @@ * This file is copyright under the latest version of the EUPL. * Please see LICENSE file for your rights under this license. */ -#include "../FTL.h" #include "ftl_lua.h" + +#include "FTL.h" // struct luaL_Reg #include "lauxlib.h" // get_FTL_version() -#include "../log.h" +#include "log.h" // config struct -#include "../config/config.h" +#include "config/config.h" // file_exists -#include "../files.h" +#include "files.h" // get_web_theme_str -#include "../datastructure.h" +#include "datastructure.h" + #if HAVE_READLINE -#include +# include +# include + #endif #include #include "scripts/scripts.h" From 9fbd40e05fe5e93fce370d1bcba86c5123e12128 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 18 May 2024 11:10:47 +0200 Subject: [PATCH 078/339] Avoid ambigious else Signed-off-by: DL6ER --- src/database/common.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/database/common.c b/src/database/common.c index 8d6756c2..f209a50f 100644 --- a/src/database/common.c +++ b/src/database/common.c @@ -253,11 +253,13 @@ void SQLite3LogCallback(void *pArg, int iErrCode, const char *zMsg) if(iErrCode == SQLITE_WARNING) log_warn("SQLite3: %s (%d)", zMsg, iErrCode); else if(iErrCode == SQLITE_NOTICE || iErrCode == SQLITE_SCHEMA) + { // SQLITE_SCHEMA is returned when the database schema has changed // This is not necessarily an error, as sqlite3_step() will re-prepare // the statement and try again. If it cannot, it will return an error // and this will be handled over there. log_debug(DEBUG_ANY, "SQLite3: %s (%d)", zMsg, iErrCode); + } else log_err("SQLite3: %s (%d)", zMsg, iErrCode); } From 313cb2353f1114e5c87540644b75be7e26d8f52b Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 18 May 2024 11:39:23 +0200 Subject: [PATCH 079/339] Update C standard to C17 Signed-off-by: DL6ER --- CMakeLists.txt | 6 +++++- src/CMakeLists.txt | 9 +++++---- src/api/theme.c | 2 +- src/config/inotify.c | 2 +- src/database/CMakeLists.txt | 2 +- src/resolve.c | 23 +++++++++++++++++++---- src/tre-regex/tre-config.h | 8 ++++---- src/zip/gzip.c | 6 ++++-- 8 files changed, 40 insertions(+), 18 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index a5c421ad..67650963 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -8,7 +8,11 @@ # This file is copyright under the latest version of the EUPL. # Please see LICENSE file for your rights under this license. -cmake_minimum_required(VERSION 2.8.12) +# C17 supports requires minimum CMake version 3.21 +# GCC 8.1.0 +# LLVM Clang 7.0.0 +cmake_minimum_required(VERSION 3.21) +set(CMAKE_C_STANDARD 17) project(PIHOLE_FTL C) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 9e3c35bb..878b3971 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -8,8 +8,6 @@ # This file is copyright under the latest version of the EUPL. # Please see LICENSE file for your rights under this license. -set(CMAKE_C_STANDARD 11) - # Default to a release with debug info build if (NOT EXISTS ${CMAKE_BINARY_DIR}/CMakeCache.txt) if (NOT CMAKE_BUILD_TYPE) @@ -165,13 +163,16 @@ if (CMAKE_C_COMPILER_ID STREQUAL "GNU") ${EXTRAWARN_GCC12} \ ${EXTRAWARN_GCC13}") elseif (CMAKE_C_COMPILER_ID STREQUAL "Clang") - set(EXTRAWARN "-Wnewline-eof \ + set(EXTRAWARN " + -Werror \ + -Wnewline-eof \ -Wno-dangling-else \ -Wno-gnu-zero-variadic-macro-arguments \ -Wno-gnu-variable-sized-type-not-at-end \ -Wno-declaration-after-statement \ -Wno-reserved-identifier \ - -Wno-reserved-macro-identifier") + -Wno-reserved-macro-identifier \ + -Wl,--fatal-warnings") else() message(WARNING "Unknown compiler, not setting warnings flags") set(EXTRAWARN "") diff --git a/src/api/theme.c b/src/api/theme.c index 0b4cf4dc..42529f01 100644 --- a/src/api/theme.c +++ b/src/api/theme.c @@ -11,7 +11,7 @@ // NULL #include // strcasecmp() -#include +#include #include "theme.h" diff --git a/src/config/inotify.c b/src/config/inotify.c index 17767350..dfb3837c 100644 --- a/src/config/inotify.c +++ b/src/config/inotify.c @@ -12,7 +12,7 @@ #include "log.h" #include // NAME_MAX -#include +#include #define WATCHDIR "/etc/pihole" diff --git a/src/database/CMakeLists.txt b/src/database/CMakeLists.txt index b0a4597d..fda25a1a 100644 --- a/src/database/CMakeLists.txt +++ b/src/database/CMakeLists.txt @@ -18,7 +18,7 @@ set(sqlite3_sources ) add_library(sqlite3 OBJECT ${sqlite3_sources}) -target_compile_options(sqlite3 PRIVATE -Wno-implicit-fallthrough -Wno-cast-function-type -Wno-sign-compare) +target_compile_options(sqlite3 PRIVATE -Wno-implicit-fallthrough -Wno-cast-function-type -Wno-sign-compare -Wno-implicit-function-declaration -Wno-int-conversion) if (CMAKE_C_COMPILER_ID STREQUAL "Clang") target_compile_options(sqlite3 PRIVATE "-Wno-null-pointer-subtraction") diff --git a/src/resolve.c b/src/resolve.c index f83c197b..91b5dce2 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -40,7 +40,7 @@ static unsigned char *name_fromDNS(unsigned char *reader, unsigned char *buffer, // Avoid "error: packed attribute causes inefficient alignment for ..." on ARM32 // builds due to the use of __attribute__((packed)) in the following structs -// Their correct size is ensured for each by static_assert() below +// Their correct size is ensured for each by check_struct_sizes() below _Pragma("GCC diagnostic push") _Pragma("GCC diagnostic ignored \"-Wattributes\"") @@ -66,7 +66,6 @@ struct DNS_HEADER uint16_t auth_count; // number of authority entries uint16_t add_count; // number of resource entries } __attribute__((packed)); -static_assert(sizeof(struct DNS_HEADER) == 12, "DNS_HEADER size mismatch"); // Constant sized fields of query structure struct QUESTION @@ -74,7 +73,6 @@ struct QUESTION uint16_t qtype; uint16_t qclass; }; -static_assert(sizeof(struct QUESTION) == 4, "QUESTION size mismatch"); // Constant sized fields of the resource record structure struct R_DATA @@ -84,9 +82,18 @@ struct R_DATA uint32_t ttl; // RFC 1035 defines the TTL field as "positive values of a signed 32bit number" uint16_t data_len; } __attribute__((packed)); -static_assert(sizeof(struct R_DATA) == 10, "R_DATA size mismatch"); _Pragma("GCC diagnostic pop") +static bool check_struct_sizes(void) +{ + // Check sizes of structs + assert(sizeof(struct DNS_HEADER) == 12); + assert(sizeof(struct QUESTION) == 4); + assert(sizeof(struct R_DATA) == 10); + + return true; +} + // Pointers to resource record contents struct RES_RECORD { @@ -812,6 +819,14 @@ void *DNSclient_thread(void *val) thread_running[DNSclient] = true; prctl(PR_SET_NAME, thread_names[DNSclient], 0, 0, 0); + // Test struct sizes + if(!check_struct_sizes()) + { + log_err("Struct sizes do not match expected sizes, aborting resolver thread"); + thread_running[DNSclient] = false; + return NULL; + } + // Initial delay until we first try to resolve anything thread_sleepms(DNSclient, 2000); diff --git a/src/tre-regex/tre-config.h b/src/tre-regex/tre-config.h index c93e539c..fdcac795 100644 --- a/src/tre-regex/tre-config.h +++ b/src/tre-regex/tre-config.h @@ -10,17 +10,17 @@ /* #undef C_ALLOCA */ /* Define to 1 if you have `alloca', as a function or macro. */ -#define HAVE_ALLOCA 1 +#define HAVE_ALLOCA 0 /* Define to 1 if you have and it should be used (not on Ultrix). */ -#define HAVE_ALLOCA_H 1 +#define HAVE_ALLOCA_H 0 /* Define if the GNU gettext() function is already present or preinstalled. */ /* #define HAVE_GETTEXT 1 */ /* Define to 1 if you have the `isascii' function. */ -#define HAVE_ISASCII 1 +//#define HAVE_ISASCII 1 /* Define to 1 if you have the `isblank' function. */ #define HAVE_ISBLANK 1 @@ -72,7 +72,7 @@ /* Define if you want TRE to use alloca() instead of malloc() when allocating memory needed for regexec operations. */ -#define TRE_USE_ALLOCA 1 +// #define TRE_USE_ALLOCA 1 /* Define to include the system regex.h from TRE regex.h */ /* #undef TRE_USE_SYSTEM_REGEX_H */ diff --git a/src/zip/gzip.c b/src/zip/gzip.c index aa31d86a..4ec97c74 100644 --- a/src/zip/gzip.c +++ b/src/zip/gzip.c @@ -8,14 +8,16 @@ * This file is copyright under the latest version of the EUPL. * Please see LICENSE file for your rights under this license. */ +#include "gzip.h" +#include "log.h" + #include #include #include #include // le32toh and friends +#define __USE_MISC #include -#include "gzip.h" -#include "log.h" static int mz_uncompress2_raw(unsigned char *pDest, mz_ulong *pDest_len, const unsigned char *pSource, mz_ulong *pSource_len); From 6fed81a91659039c331a528c5174deb3c2c9caba Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 18 May 2024 10:36:18 +0000 Subject: [PATCH 080/339] Bump actions/checkout in the github_action-dependencies group Bumps the github_action-dependencies group with 1 update: [actions/checkout](https://github.com/actions/checkout). Updates `actions/checkout` from 4.1.5 to 4.1.6 - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v4.1.5...v4.1.6) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-patch dependency-group: github_action-dependencies ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 6 +++--- .github/workflows/codespell.yml | 2 +- .github/workflows/openapi-validator.yml | 2 +- .github/workflows/stale.yml | 2 +- .github/workflows/sync-back-to-dev.yml | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ed5553f8..a723888f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -31,7 +31,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v4.1.5 + uses: actions/checkout@v4.1.6 - name: "Calculate required variables" id: variables @@ -75,7 +75,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v4.1.5 + uses: actions/checkout@v4.1.6 - name: Build and test and deploy FTL uses: ./.github/actions/build-and-test @@ -113,7 +113,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v4.1.5 + uses: actions/checkout@v4.1.6 - name: Build and test and deploy FTL uses: ./.github/actions/build-and-test diff --git a/.github/workflows/codespell.yml b/.github/workflows/codespell.yml index 47b5d2e8..1fda2c3f 100644 --- a/.github/workflows/codespell.yml +++ b/.github/workflows/codespell.yml @@ -13,7 +13,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4.1.5 + uses: actions/checkout@v4.1.6 - name: Spell-Checking uses: codespell-project/actions-codespell@master diff --git a/.github/workflows/openapi-validator.yml b/.github/workflows/openapi-validator.yml index 2809c389..0070f8fe 100644 --- a/.github/workflows/openapi-validator.yml +++ b/.github/workflows/openapi-validator.yml @@ -12,7 +12,7 @@ jobs: steps: - name: Clone repository - uses: actions/checkout@v4.1.5 + uses: actions/checkout@v4.1.6 - name: Set Node.js version uses: actions/setup-node@v4 diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 61147230..c2699158 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -40,7 +40,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4.1.5 + uses: actions/checkout@v4.1.6 - name: Remove 'stale' label run: gh issue edit ${{ github.event.issue.number }} --remove-label ${{ env.stale_label }} env: diff --git a/.github/workflows/sync-back-to-dev.yml b/.github/workflows/sync-back-to-dev.yml index c13bacd7..0592cf35 100644 --- a/.github/workflows/sync-back-to-dev.yml +++ b/.github/workflows/sync-back-to-dev.yml @@ -11,7 +11,7 @@ jobs: name: Syncing branches steps: - name: Checkout - uses: actions/checkout@v4.1.5 + uses: actions/checkout@v4.1.6 - name: Opening pull request run: gh pr create -B development -H master --title 'Sync master back into development' --body 'Created by Github action' --label 'internal' env: From e286d4b1479edf584fd3dae3781bb2c850d17661 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 18 May 2024 16:33:14 +0200 Subject: [PATCH 081/339] Properly guard X.509 functions to allow compiling FTL without mbedTLS library being available, also improve how definitions are done in src/CMakeLists.txt and reduce instruction-duplication on the way Signed-off-by: DL6ER --- src/CMakeLists.txt | 50 ++++++++++++++++----------- src/args.c | 5 +-- src/lua/CMakeLists.txt | 7 ---- src/lua/ftl_lua.c | 2 +- src/webserver/civetweb/CMakeLists.txt | 11 ------ src/webserver/webserver.c | 4 +-- src/webserver/x509.c | 26 +++++++++++--- src/webserver/x509.h | 6 ++-- 8 files changed, 60 insertions(+), 51 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index c254b3bc..c040c688 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -249,15 +249,15 @@ add_custom_target( COMMAND ${CMAKE_COMMAND} -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} -P ${CMAKE_CURRENT_SOURCE_DIR}/gen_version.cmake WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) -add_library(FTL OBJECT ${sources}) -target_compile_options(FTL PRIVATE ${EXTRAWARN}) -target_compile_definitions(FTL PRIVATE DNSMASQ_VERSION=\"${DNSMASQ_VERSION}\") -target_include_directories(FTL PRIVATE ${PROJECT_SOURCE_DIR}/src) -add_dependencies(FTL gen_version) +add_library(core OBJECT ${sources}) +target_compile_options(core PRIVATE ${EXTRAWARN}) +target_compile_definitions(core PRIVATE DNSMASQ_VERSION=\"${DNSMASQ_VERSION}\") +target_include_directories(core PRIVATE ${PROJECT_SOURCE_DIR}/src) +add_dependencies(core gen_version) add_executable(pihole-FTL - $ + $ $ $ $ @@ -303,15 +303,26 @@ if(LUA_DL STREQUAL "true") target_link_libraries(pihole-FTL ${LIBDL}) endif() +add_subdirectory(api) +add_subdirectory(webserver) +add_subdirectory(zip) +add_subdirectory(database) +add_subdirectory(dnsmasq) +add_subdirectory(lua) +add_subdirectory(lua/scripts) +add_subdirectory(tre-regex) +add_subdirectory(syscalls) +add_subdirectory(config) +add_subdirectory(tools) + find_library(LIBREADLINE NAMES libreadline${CMAKE_STATIC_LIBRARY_SUFFIX} readline) find_library(LIBHISTORY NAMES libhistory${CMAKE_STATIC_LIBRARY_SUFFIX} history) find_library(LIBTERMCAP NAMES libtermcap${CMAKE_STATIC_LIBRARY_SUFFIX} termcap) if(LIBREADLINE AND LIBHISTORY AND LIBTERMCAP) message(STATUS "Building FTL with readline support: YES") - target_compile_definitions(FTL PRIVATE LUA_USE_READLINE) - target_compile_definitions(pihole-FTL PRIVATE LUA_USE_READLINE) + target_compile_definitions(lua PRIVATE LUA_USE_READLINE) + target_compile_definitions(sqlite3 PRIVATE HAVE_READLINE) target_link_libraries(pihole-FTL ${LIBREADLINE} ${LIBHISTORY} ${LIBTERMCAP}) - set(HAVE_READLINE TRUE) else() message(STATUS "Building FTL with readline support: NO") endif() @@ -342,9 +353,17 @@ find_library(LIBMBEDCRYPTO NAMES lmbedcrypto${CMAKE_STATIC_LIBRARY_SUFFIX} mbedc find_library(LIBMBEDX509 NAMES lmbedx509${CMAKE_STATIC_LIBRARY_SUFFIX} mbedx509) find_library(LIBMBEDTLS NAMES lmbedtls${CMAKE_STATIC_LIBRARY_SUFFIX} mbedtls) if(LIBMBEDCRYPTO AND LIBMBEDX509 AND LIBMBEDTLS) + # Enable TLS support in civetweb if mbedTLS is available + message(STATUS "Building FTL with TLS support: YES") + target_compile_definitions(core PRIVATE HAVE_MBEDTLS) + target_compile_definitions(civetweb PRIVATE USE_MBEDTLS) + target_compile_definitions(webserver PRIVATE HAVE_MBEDTLS) # Link against the mbedTLS libraries, the order is important (!) - target_compile_definitions(FTL PRIVATE HAVE_MBEDTLS) target_link_libraries(pihole-FTL ${LIBMBEDTLS} ${LIBMBEDX509} ${LIBMBEDCRYPTO}) +else() + # Disable TLS support in civetweb if mbedTLS is not available + message(STATUS "Building FTL with TLS support: NO") + target_compile_definitions(civetweb PRIVATE NO_SSL) endif() find_program(SETCAP setcap) @@ -353,14 +372,3 @@ install(TARGETS pihole-FTL PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE) install(CODE "execute_process(COMMAND ${SETCAP} CAP_NET_BIND_SERVICE,CAP_NET_RAW,CAP_NET_ADMIN,CAP_SYS_NICE,CAP_CHOWN+eip \$ENV{DESTDIR}\${CMAKE_INSTALL_PREFIX}/bin/pihole-FTL)") -add_subdirectory(api) -add_subdirectory(webserver) -add_subdirectory(zip) -add_subdirectory(database) -add_subdirectory(dnsmasq) -add_subdirectory(lua) -add_subdirectory(lua/scripts) -add_subdirectory(tre-regex) -add_subdirectory(syscalls) -add_subdirectory(config) -add_subdirectory(tools) diff --git a/src/args.c b/src/args.c index 29b24e2f..e4bbd0cd 100644 --- a/src/args.c +++ b/src/args.c @@ -750,11 +750,12 @@ void parse_args(int argc, char* argv[]) printf("\n"); printf("****************************** %s%sCivetWeb%s *****************************\n", yellow, bold, normal); -#ifdef MBEDTLS_VERSION_STRING_FULL +#ifdef HAVE_MBEDTLS printf("Version: %s%s%s%s with %smbed TLS %s%s"MBEDTLS_VERSION_STRING"%s\n", green, bold, mg_version(), normal, yellow, green, bold, normal); #else - printf("Version: %s%s%s%s\n", green, bold, mg_version(), normal); + printf("Version: %s%s%s%s%s without %smbed TLS%s\n", + green, bold, mg_version(), normal, red, yellow, normal); #endif printf("Features: "); if(mg_check_feature(MG_FEATURES_FILES)) diff --git a/src/lua/CMakeLists.txt b/src/lua/CMakeLists.txt index 908b5a7e..7abcb4af 100644 --- a/src/lua/CMakeLists.txt +++ b/src/lua/CMakeLists.txt @@ -78,11 +78,4 @@ if(LUA_DL STREQUAL "true") target_compile_definitions(lua PRIVATE LUA_USE_DLOPEN) endif() -if(LIBREADLINE AND LIBHISTORY AND LIBTERMCAP) - message(STATUS "Embedded LUA will use readline for history: YES") - target_compile_definitions(lua PRIVATE LUA_USE_READLINE) -else() - message(STATUS "Embedded LUA will use readline for history: NO") -endif() - target_include_directories(lua PRIVATE ${PROJECT_SOURCE_DIR}/src ${PROJECT_SOURCE_DIR}/src/lua) diff --git a/src/lua/ftl_lua.c b/src/lua/ftl_lua.c index 0948ed06..c5753327 100644 --- a/src/lua/ftl_lua.c +++ b/src/lua/ftl_lua.c @@ -20,7 +20,7 @@ #include "../files.h" // get_web_theme_str #include "../datastructure.h" -#if HAVE_READLINE +#if LUA_USE_READLINE #include #endif #include diff --git a/src/webserver/civetweb/CMakeLists.txt b/src/webserver/civetweb/CMakeLists.txt index 6a6cf922..943619d5 100644 --- a/src/webserver/civetweb/CMakeLists.txt +++ b/src/webserver/civetweb/CMakeLists.txt @@ -32,16 +32,5 @@ target_compile_definitions(civetweb PRIVATE NO_CGI USE_LUA TIMER_RESOLUTION=1000) -if(LIBMBEDCRYPTO AND LIBMBEDX509 AND LIBMBEDTLS) - # Enable TLS support in civetweb if mbedTLS is available - message(STATUS "Building FTL with TLS support: YES") - target_compile_definitions(civetweb PRIVATE USE_MBEDTLS) - target_compile_definitions(webserver PRIVATE HAVE_TLS) -else() - # Disable TLS support in civetweb if mbedTLS is not available - message(STATUS "Building FTL with TLS support: NO") - target_compile_definitions(civetweb PRIVATE NO_SSL) -endif() - include_directories(${PROJECT_SOURCE_DIR}/src/lua /usr/local/include) target_include_directories(civetweb PRIVATE ${PROJECT_SOURCE_DIR}/src) diff --git a/src/webserver/webserver.c b/src/webserver/webserver.c index 5485046f..f0541e6e 100644 --- a/src/webserver/webserver.c +++ b/src/webserver/webserver.c @@ -353,7 +353,7 @@ void http_init(void) MG_FEATURES_IPV6 | MG_FEATURES_CACHE; -#ifdef HAVE_TLS +#ifdef HAVE_MBEDTLS features |= MG_FEATURES_TLS; #endif @@ -419,7 +419,7 @@ void http_init(void) // from the end of the array. unsigned int next_option = ArraySize(options) - 6; -#ifdef HAVE_TLS +#ifdef HAVE_MBEDTLS // Add TLS options if configured if(config.webserver.tls.cert.v.s != NULL && strlen(config.webserver.tls.cert.v.s) > 0) diff --git a/src/webserver/x509.c b/src/webserver/x509.c index 7c2a3d82..597ce7b1 100644 --- a/src/webserver/x509.c +++ b/src/webserver/x509.c @@ -11,11 +11,11 @@ #include "FTL.h" #include "log.h" #include "x509.h" -#include -#include -#include -#include -#include + +#ifdef HAVE_MBEDTLS +# include +# include +# include #define RSA_KEY_SIZE 4096 #define BUFFER_SIZE 16000 @@ -621,3 +621,19 @@ end: return CERT_OKAY; } + +#else + +bool generate_certificate(const char* certfile, bool rsa, const char *domain) +{ + log_err("FTL was not compiled with mbedtls support"); + return false; +} + +enum cert_check read_certificate(const char* certfile, const char *domain, const bool private_key) +{ + log_err("FTL was not compiled with mbedtls support"); + return CERT_FILE_NOT_FOUND; +} + +#endif diff --git a/src/webserver/x509.h b/src/webserver/x509.h index e59ee1a7..1c6f4af6 100644 --- a/src/webserver/x509.h +++ b/src/webserver/x509.h @@ -10,8 +10,10 @@ #ifndef X509_H #define X509_H -#include -#include +#ifdef HAVE_MBEDTLS +# include +# include +#endif #include "enums.h" From 4b2cb9882509b44e98dae8a40c033fe4684422d8 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 18 May 2024 16:46:47 +0200 Subject: [PATCH 082/339] Enforce minimum version of mbedTLS (3.5.0) Signed-off-by: DL6ER --- src/webserver/x509.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/webserver/x509.c b/src/webserver/x509.c index 597ce7b1..d8706022 100644 --- a/src/webserver/x509.c +++ b/src/webserver/x509.c @@ -17,6 +17,12 @@ # include # include +// We enforce at least mbedTLS v3.5.0 if we use it +#if MBEDTLS_VERSION_NUMBER < 0x03050000 +# error "mbedTLS version 3.5.0 or later is required" +#endif + + #define RSA_KEY_SIZE 4096 #define BUFFER_SIZE 16000 From e3346ff641a9d0a09b46576a308dca4f275233f5 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 18 May 2024 17:29:24 +0200 Subject: [PATCH 083/339] Ensure we also change ownership of the WAL database files Signed-off-by: DL6ER --- src/dnsmasq_interface.c | 38 +++++++++++++++++++++++++++++++++++--- 1 file changed, 35 insertions(+), 3 deletions(-) diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c index f816abb6..dd337c5b 100644 --- a/src/dnsmasq_interface.c +++ b/src/dnsmasq_interface.c @@ -2937,13 +2937,45 @@ void FTL_fork_and_bind_sockets(struct passwd *ent_pw, bool dnsmasq_start) if(ent_pw != NULL && ent_pw->pw_uid != 0) { log_info("FTL is going to drop from root to user %s (UID %u)", - ent_pw->pw_name, ent_pw->pw_uid); + ent_pw->pw_name, ent_pw->pw_uid); if(chown(config.files.log.ftl.v.s, ent_pw->pw_uid, ent_pw->pw_gid) == -1) + { log_warn("Setting ownership (%u:%u) of %s failed: %s (%i)", - ent_pw->pw_uid, ent_pw->pw_gid, config.files.log.ftl.v.s, strerror(errno), errno); + ent_pw->pw_uid, ent_pw->pw_gid, config.files.log.ftl.v.s, strerror(errno), errno); + } + if(chown(config.files.database.v.s, ent_pw->pw_uid, ent_pw->pw_gid) == -1) + { log_warn("Setting ownership (%u:%u) of %s failed: %s (%i)", - ent_pw->pw_uid, ent_pw->pw_gid, config.files.database.v.s, strerror(errno), errno); + ent_pw->pw_uid, ent_pw->pw_gid, config.files.database.v.s, strerror(errno), errno); + + // Check if WAL files are present and change + // their ownership, too + char *walname = calloc(strlen(config.files.database.v.s) + 5, sizeof(char)); + if(walname != NULL) + { + strcpy(walname, config.files.database.v.s); + strcat(walname, "-wal"); + if(chown(walname, ent_pw->pw_uid, ent_pw->pw_gid) == -1) + { + log_warn("Setting ownership (%u:%u) of %s failed: %s (%i)", + ent_pw->pw_uid, ent_pw->pw_gid, walname, strerror(errno), errno); + } + free(walname); + } + char *shmname = calloc(strlen(config.files.database.v.s) + 5, sizeof(char)); + if(shmname != NULL) + { + strcpy(shmname, config.files.database.v.s); + strcat(shmname, "-shm"); + if(chown(shmname, ent_pw->pw_uid, ent_pw->pw_gid) == -1) + { + log_warn("Setting ownership (%u:%u) of %s failed: %s (%i)", + ent_pw->pw_uid, ent_pw->pw_gid, shmname, strerror(errno), errno); + } + free(shmname); + } + } chown_all_shmem(ent_pw); } else From d57941790111e789767310102118d6c7636f54b1 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 19 May 2024 07:27:06 +0200 Subject: [PATCH 084/339] Ensure target lua_scripts is built before target ftl_lua depending on it Signed-off-by: DL6ER --- src/lua/scripts/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lua/scripts/CMakeLists.txt b/src/lua/scripts/CMakeLists.txt index 9afd1afa..0f7aa19d 100644 --- a/src/lua/scripts/CMakeLists.txt +++ b/src/lua/scripts/CMakeLists.txt @@ -27,8 +27,8 @@ foreach(INPUT_FILE ${COMPILED_RESOURCES}) list(APPEND COMPILED_RESOURCES ${OUTPUT_FILE}) endforeach() -# Ensure target lua_scripts is build before target lua -add_dependencies(lua lua_scripts) +# Ensure target lua_scripts is build before target ftl_lua depending on it +add_dependencies(ftl_lua lua_scripts) add_library(lua_scripts OBJECT ${sources}) target_compile_options(lua_scripts PRIVATE ${EXTRAWARN}) From 67fdf2915b04bf45c3888d4b4dff0444e2699f43 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 19 May 2024 08:56:14 +0200 Subject: [PATCH 085/339] Fix conditional for readline inclusion in LUA code Signed-off-by: DL6ER --- src/lua/ftl_lua.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lua/ftl_lua.c b/src/lua/ftl_lua.c index f649ff10..c5a40066 100644 --- a/src/lua/ftl_lua.c +++ b/src/lua/ftl_lua.c @@ -27,7 +27,7 @@ // prototype for luaopen_pihole() #include "lualib.h" -#if LUA_USE_READLINE +#if defined(LUA_USE_READLINE) # include #endif #include From 25ab7b726c76a9d5151eb2e03a56a85fd3469c19 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 19 May 2024 09:05:44 +0200 Subject: [PATCH 086/339] Print GLIBC version in pihole-FTL -vv and not during compile time as "#pragma message" is not supported by older clang versions Signed-off-by: DL6ER --- src/args.c | 7 ++++++- src/main.c | 6 ------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/args.c b/src/args.c index e4bbd0cd..4333fc4b 100644 --- a/src/args.c +++ b/src/args.c @@ -715,7 +715,12 @@ void parse_args(int argc, char* argv[]) printf("Branch: " GIT_BRANCH "\n"); printf("Commit: " GIT_HASH " (" GIT_DATE ")\n"); printf("Architecture: " FTL_ARCH "\n"); - printf("Compiler: " FTL_CC "\n\n"); + printf("Compiler: " FTL_CC "\n"); +#if defined(__GLIBC__) && defined(__GLIBC_MINOR__) + printf("GLIBC version: %d.%d\n\n", __GLIBC__, __GLIBC_MINOR__); +#else + printf("GLIBC version: -\n\n"); +#endif // Print dnsmasq version and compile time options print_dnsmasq_version(yellow, green, bold, normal); diff --git a/src/main.c b/src/main.c index 3d18c74e..61e7b405 100644 --- a/src/main.c +++ b/src/main.c @@ -28,12 +28,6 @@ // export_queries_to_disk() #include "database/query-table.h" -#if defined(__GLIBC__) && defined(__GLIBC_MINOR__) -#pragma message "Minimum GLIBC version: " xstr(__GLIBC__) "." xstr(__GLIBC_MINOR__) -#else -#pragma message "Minimum GLIBC version: unknown, assuming this is a MUSL build" -#endif - char *username; bool needGC = false; bool needDBGC = false; From f88d96a884312d00979982457d66646f8ca54a95 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 19 May 2024 09:14:46 +0200 Subject: [PATCH 087/339] Skip attestation and deployment steps for fork-based PRs having no access to secrets Signed-off-by: DL6ER --- .github/actions/build-and-test/action.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/actions/build-and-test/action.yml b/.github/actions/build-and-test/action.yml index d45fc125..189d9437 100644 --- a/.github/actions/build-and-test/action.yml +++ b/.github/actions/build-and-test/action.yml @@ -101,6 +101,9 @@ runs: - name: Generate artifact attestation uses: actions/attest-build-provenance@v1 + # Skip attestation if ACTIONS_ID_TOKEN_REQUEST_URL env variable is not + # available (e.g., PR originating from a fork) + if: ${{ env.ACTIONS_ID_TOKEN_REQUEST_URL != '' }} with: subject-path: ${{ inputs.bin_name }} - @@ -118,7 +121,11 @@ runs: path: 'api-docs.tar.gz' - name: Deploy - if: inputs.event_name != 'pull_request' + # Skip deployment step if: + # - this is a triggered by a PR event (we only push on commit to branch + # events) + # - no SSH key is provided (this is a PR from a fork) + if: inputs.event_name != 'pull_request' && ${{ inputs.SSH_KEY != '' }} uses: ./.github/actions/deploy with: pattern: ${{ inputs.bin_name }}-binary From bfbe309b42373cf69c34046b641e969ec17c89c4 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 20 May 2024 10:24:27 +0200 Subject: [PATCH 088/339] Update build containers to ftl-build v2.5.1 Signed-off-by: DL6ER --- .devcontainer/devcontainer.json | 2 +- .github/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 652e4ece..d58d4db8 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,6 +1,6 @@ { "name": "FTL x86_64 Build Env", - "image": "ghcr.io/pi-hole/ftl-build:v2.5", + "image": "ghcr.io/pi-hole/ftl-build:v2.5.1", "runArgs": [ "--cap-add=SYS_PTRACE", "--security-opt", "seccomp=unconfined" ], "customizations": { "vscode": { diff --git a/.github/Dockerfile b/.github/Dockerfile index 2731b8a2..9ca5a7f3 100644 --- a/.github/Dockerfile +++ b/.github/Dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/pi-hole/ftl-build:v2.5 AS builder +FROM ghcr.io/pi-hole/ftl-build:v2.5.1 AS builder WORKDIR /app From c957f7820901d67f832f69bc3b387b9ac8bdc8a7 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 20 May 2024 10:31:06 +0200 Subject: [PATCH 089/339] Add Visual Studio Code CMake configuration to get ccompile-time definition detection working Signed-off-by: DL6ER --- .gitignore | 5 ++--- .vscode/c_cpp_properties.json | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+), 3 deletions(-) create mode 100644 .vscode/c_cpp_properties.json diff --git a/.gitignore b/.gitignore index 075dad8f..a3c7c317 100644 --- a/.gitignore +++ b/.gitignore @@ -14,9 +14,8 @@ version~ # IDE files .idea/ *.sw* -/.vscode -.vscode/ -/.vscode/ +.vscode/* +!.vscode/c_cpp_properties.json /build/ # __pycache__ files (API tests) diff --git a/.vscode/c_cpp_properties.json b/.vscode/c_cpp_properties.json new file mode 100644 index 00000000..901ccbde --- /dev/null +++ b/.vscode/c_cpp_properties.json @@ -0,0 +1,18 @@ +{ + "configurations": [ + { + "name": "Linux", + "includePath": [ + "${workspaceFolder}/src/**" + ], + "compileCommands": "${workspaceFolder}/build/compile_commands.json", + "defines": [], + "compilerPath": "/usr/bin/gcc", + "cStandard": "gnu17", + "cppStandard": "gnu++17", + "intelliSenseMode": "linux-gcc-x64", + "configurationProvider": "ms-vscode.cmake-tools" + } + ], + "version": 4 +} \ No newline at end of file From 8c5da713f8f7e3250deb904b5758b49b6d11c5b2 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 20 May 2024 11:05:15 +0200 Subject: [PATCH 090/339] Add clang build Signed-off-by: DL6ER --- .github/Dockerfile | 4 +++- .github/actions/build-and-test/action.yml | 4 ++++ .github/workflows/build.yml | 7 +++++++ 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/.github/Dockerfile b/.github/Dockerfile index 9ca5a7f3..7dcbabb4 100644 --- a/.github/Dockerfile +++ b/.github/Dockerfile @@ -10,12 +10,14 @@ ARG GIT_BRANCH="test" ENV GIT_BRANCH ${GIT_BRANCH} ARG GIT_TAG="test" ENV GIT_TAG ${GIT_TAG} +ARG BUILD_OPTS="" +ENV BUILD_OPTS ${BUILD_OPTS} # Build FTL # Remove possible old build files RUN rm -rf cmake && \ # Build FTL - bash build.sh "-DSTATIC=${STATIC}" && \ + bash build.sh "-DSTATIC=${STATIC}" ${BUILD_OPTS} && \ # Run binary architecture tests bash test/arch_test.sh && \ # Run full test suite diff --git a/.github/actions/build-and-test/action.yml b/.github/actions/build-and-test/action.yml index d45fc125..a9c85ea8 100644 --- a/.github/actions/build-and-test/action.yml +++ b/.github/actions/build-and-test/action.yml @@ -5,6 +5,9 @@ inputs: platform: required: true description: The platform to build for + build_opts: + required: true + description: Any extra build opts to use git_branch: required: true description: The branch to build from @@ -76,6 +79,7 @@ runs: "CI_ARCH=${{ inputs.platform }}" "GIT_BRANCH=${{ inputs.git_branch }}" "GIT_TAG=${{ inputs.git_tag }}" + "BUILD_OPTS=${{ inputs.build_opts }}" - name: List files in current directory shell: bash diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a723888f..370d88ef 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -64,10 +64,16 @@ jobs: include: - platform: linux/amd64 bin_name: pihole-FTL-amd64 + build_opts: "" + - platform: linux/amd64 + bin_name: pihole-FTL-amd64-clang + build_opts: clang - platform: linux/386 bin_name: pihole-FTL-386 + build_opts: "" - platform: linux/riscv64 bin_name: pihole-FTL-riscv64 + build_opts: "" env: CI_ARCH: ${{ matrix.platform }} GIT_BRANCH: ${{ needs.smoke-tests.outputs.GIT_BRANCH }} @@ -82,6 +88,7 @@ jobs: with: platform: ${{ matrix.platform }} bin_name: ${{ matrix.bin_name }} + build_opts: ${{ matrix.build_opts }} artifact_name: ${{ matrix.bin_name }}-binary target_dir: ${{ needs.smoke-tests.outputs.OUTPUT_DIR }} git_branch: ${{ needs.smoke-tests.outputs.GIT_BRANCH }} From 212d0f02a9e09a7c329cfaa09fc3912249c868d0 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 20 May 2024 11:39:44 +0200 Subject: [PATCH 091/339] Use ftl-build:nightly for devcontainer Signed-off-by: DL6ER --- .devcontainer/devcontainer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index d58d4db8..bb8c5892 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,6 +1,6 @@ { "name": "FTL x86_64 Build Env", - "image": "ghcr.io/pi-hole/ftl-build:v2.5.1", + "image": "ghcr.io/pi-hole/ftl-build:nightly", "runArgs": [ "--cap-add=SYS_PTRACE", "--security-opt", "seccomp=unconfined" ], "customizations": { "vscode": { From 4a70b5989c917ffad96bbc81d43e836f7c3ce6bf Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 20 May 2024 20:46:18 +0200 Subject: [PATCH 092/339] Use new-clang ftl-build containers Signed-off-by: DL6ER --- .github/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/Dockerfile b/.github/Dockerfile index 7dcbabb4..141bfc6b 100644 --- a/.github/Dockerfile +++ b/.github/Dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/pi-hole/ftl-build:v2.5.1 AS builder +FROM ghcr.io/pi-hole/ftl-build:new-clang AS builder WORKDIR /app From 1a72da36169b34a3b7032933c08b699a68ee34cb Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 20 May 2024 22:10:46 +0200 Subject: [PATCH 093/339] Do not build static exectuable in clang test to avoid LTO linking issue Signed-off-by: DL6ER --- build.sh | 1 + src/CMakeLists.txt | 24 +++++++++++++----------- src/dnsmasq/CMakeLists.txt | 2 +- src/zip/miniz/CMakeLists.txt | 3 +-- 4 files changed, 16 insertions(+), 14 deletions(-) diff --git a/build.sh b/build.sh index b92062d1..9809fdd5 100755 --- a/build.sh +++ b/build.sh @@ -65,6 +65,7 @@ done if [[ -n "${clang}" ]]; then export CC=clang export CXX=clang++ + export STATIC="false" fi # Configure build, pass CMake CACHE entries if present diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index fa12747f..4fb40e3d 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -298,22 +298,24 @@ if(STATIC) set_target_properties(pihole-FTL PROPERTIES LINK_SEARCH_START_STATIC ON) set_target_properties(pihole-FTL PROPERTIES LINK_SEARCH_END_STATIC ON) target_link_libraries(pihole-FTL -static-libgcc -static) + set(LIBRARY_SUFFIX "${CMAKE_STATIC_LIBRARY_SUFFIX}") else() find_library(LIBMATH m) target_link_libraries(pihole-FTL ${LIBMATH}) + set(LIBRARY_SUFFIX "") endif() set(CMAKE_THREAD_PREFER_PTHREAD TRUE) set(THREADS_PREFER_PTHREAD_FLAG TRUE) find_package(Threads REQUIRED) # for DNSSEC we need the nettle (+ hogweed) crypto and the gmp math libraries -find_library(LIBHOGWEED NAMES libhogweed${CMAKE_STATIC_LIBRARY_SUFFIX} hogweed HINTS /usr/local/lib64) -find_library(LIBGMP NAMES libgmp${CMAKE_STATIC_LIBRARY_SUFFIX} gmp) -find_library(LIBNETTLE NAMES libnettle${CMAKE_STATIC_LIBRARY_SUFFIX} nettle HINTS /usr/local/lib64) +find_library(LIBHOGWEED NAMES libhogweed${LIBRARY_SUFFIX} hogweed HINTS /usr/local/lib64) +find_library(LIBGMP NAMES libgmp${LIBRARY_SUFFIX} gmp) +find_library(LIBNETTLE NAMES libnettle${LIBRARY_SUFFIX} nettle HINTS /usr/local/lib64) # for IDN2 we need the idn2 library which in turn depends on the unistring library -find_library(LIBIDN2 NAMES libidn2${CMAKE_STATIC_LIBRARY_SUFFIX} idn2) -find_library(LIBUNISTRING NAMES libunistring${CMAKE_STATIC_LIBRARY_SUFFIX} unistring) +find_library(LIBIDN2 NAMES libidn2${LIBRARY_SUFFIX} idn2) +find_library(LIBUNISTRING NAMES libunistring${LIBRARY_SUFFIX} unistring) target_link_libraries(pihole-FTL rt Threads::Threads ${LIBHOGWEED} ${LIBGMP} ${LIBNETTLE} ${LIBIDN2} ${LIBUNISTRING}) @@ -334,9 +336,9 @@ add_subdirectory(syscalls) add_subdirectory(config) add_subdirectory(tools) -find_library(LIBREADLINE NAMES libreadline${CMAKE_STATIC_LIBRARY_SUFFIX} readline) -find_library(LIBHISTORY NAMES libhistory${CMAKE_STATIC_LIBRARY_SUFFIX} history) -find_library(LIBTERMCAP NAMES libtermcap${CMAKE_STATIC_LIBRARY_SUFFIX} termcap) +find_library(LIBREADLINE NAMES libreadline${LIBRARY_SUFFIX} readline) +find_library(LIBHISTORY NAMES libhistory${LIBRARY_SUFFIX} history) +find_library(LIBTERMCAP NAMES libtermcap${LIBRARY_SUFFIX} termcap) if(LIBREADLINE AND LIBHISTORY AND LIBTERMCAP) message(STATUS "Building FTL with readline support: YES") target_compile_definitions(lua PRIVATE LUA_USE_READLINE) @@ -350,9 +352,9 @@ if(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT) set(CMAKE_INSTALL_PREFIX "/usr" CACHE PATH "..." FORCE) endif() -find_library(LIBMBEDCRYPTO NAMES lmbedcrypto${CMAKE_STATIC_LIBRARY_SUFFIX} mbedcrypto) -find_library(LIBMBEDX509 NAMES lmbedx509${CMAKE_STATIC_LIBRARY_SUFFIX} mbedx509) -find_library(LIBMBEDTLS NAMES lmbedtls${CMAKE_STATIC_LIBRARY_SUFFIX} mbedtls) +find_library(LIBMBEDCRYPTO NAMES lmbedcrypto${LIBRARY_SUFFIX} mbedcrypto) +find_library(LIBMBEDX509 NAMES lmbedx509${LIBRARY_SUFFIX} mbedx509) +find_library(LIBMBEDTLS NAMES lmbedtls${LIBRARY_SUFFIX} mbedtls) if(LIBMBEDCRYPTO AND LIBMBEDX509 AND LIBMBEDTLS) # Enable TLS support in civetweb if mbedTLS is available message(STATUS "Building FTL with TLS support: YES") diff --git a/src/dnsmasq/CMakeLists.txt b/src/dnsmasq/CMakeLists.txt index 9caab62f..927ed572 100644 --- a/src/dnsmasq/CMakeLists.txt +++ b/src/dnsmasq/CMakeLists.txt @@ -66,7 +66,7 @@ add_library(dnsmasq OBJECT ${sources}) target_compile_definitions(dnsmasq PRIVATE VERSION=\"${DNSMASQ_VERSION}\") target_compile_definitions(dnsmasq PRIVATE CONFFILE=\"/etc/pihole/dnsmasq.conf\") if (CMAKE_C_COMPILER_ID STREQUAL "GNU") - target_compile_options(dnsmasq PRIVATE -Wno-maybe-uninitialized) + target_compile_options(dnsmasq PRIVATE -Wno-maybe-uninitialized -Wno-sign-compare) elseif (CMAKE_C_COMPILER_ID STREQUAL "Clang") target_compile_options(dnsmasq PRIVATE -Wno-gnu-variable-sized-type-not-at-end -Wno-sign-compare -Wno-deprecated-non-prototype) endif() diff --git a/src/zip/miniz/CMakeLists.txt b/src/zip/miniz/CMakeLists.txt index 40803374..5047aea3 100644 --- a/src/zip/miniz/CMakeLists.txt +++ b/src/zip/miniz/CMakeLists.txt @@ -14,6 +14,5 @@ set(sources ) add_library(miniz OBJECT ${sources}) -target_compile_options(miniz PRIVATE) -target_compile_options(miniz PRIVATE "-Wno-padded") +target_compile_options(miniz PRIVATE -Wno-padded -Wno-type-limits) target_include_directories(miniz PRIVATE ${PROJECT_SOURCE_DIR}/src) From e10d2a52e1dacd3345b40837c27c9604a5596b6e Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 21 May 2024 19:58:19 +0200 Subject: [PATCH 094/339] Clang-built binaries are expected to be dynamic Signed-off-by: DL6ER --- .devcontainer/devcontainer.json | 2 +- .github/Dockerfile | 8 ++------ build.sh | 3 ++- test/arch_test.sh | 14 ++++++++++---- 4 files changed, 15 insertions(+), 12 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index bb8c5892..3fe6ed7f 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,6 +1,6 @@ { "name": "FTL x86_64 Build Env", - "image": "ghcr.io/pi-hole/ftl-build:nightly", + "image": "ghcr.io/pi-hole/ftl-build:new-clang", "runArgs": [ "--cap-add=SYS_PTRACE", "--security-opt", "seccomp=unconfined" ], "customizations": { "vscode": { diff --git a/.github/Dockerfile b/.github/Dockerfile index 141bfc6b..1cbde40e 100644 --- a/.github/Dockerfile +++ b/.github/Dockerfile @@ -16,12 +16,8 @@ ENV BUILD_OPTS ${BUILD_OPTS} # Build FTL # Remove possible old build files RUN rm -rf cmake && \ -# Build FTL - bash build.sh "-DSTATIC=${STATIC}" ${BUILD_OPTS} && \ -# Run binary architecture tests - bash test/arch_test.sh && \ -# Run full test suite - bash test/run.sh && \ +# Build and test FTL + bash build.sh "-DSTATIC=${STATIC}" test ${BUILD_OPTS} && \ # Move FTL binary to root directory cd / &&\ mv /app/pihole-FTL . && \ diff --git a/build.sh b/build.sh index 9809fdd5..2df0f8f1 100755 --- a/build.sh +++ b/build.sh @@ -96,5 +96,6 @@ fi # If we are asked to run tests, we do this here if [[ -n "${test}" ]]; then cd .. - ./test/run.sh + bash test/arch_test.sh + bash test/run.sh fi diff --git a/test/arch_test.sh b/test/arch_test.sh index 01ca0f4e..ff4bc003 100644 --- a/test/arch_test.sh +++ b/test/arch_test.sh @@ -95,10 +95,16 @@ check_minimum_glibc_version() { if [[ "${CI_ARCH}" == "linux/amd64" ]]; then - check_machine "ELF64" "Advanced Micro Devices X86-64" - check_static # Binary should not rely on any dynamic interpreter - check_libs "" # No dependency on any shared library is intended - check_file "ELF 64-bit LSB executable, x86-64, version 1 (SYSV), statically linked, with debug_info, not stripped" + if [[ "${STATIC}" == "true" ]]; then + check_machine "ELF64" "Advanced Micro Devices X86-64" + check_static # Binary should not rely on any dynamic interpreter + check_libs "" # No dependency on any shared library is intended + check_file "ELF 64-bit LSB executable, x86-64, version 1 (SYSV), statically linked, with debug_info, not stripped" +else + check_machine "ELF64" "Advanced Micro Devices X86-64" + check_libs "[libgmp.so.10] [libidn2.so.0] [libc.musl-x86_64.so.1]" + check_file "ELF 64-bit LSB pie executable, x86-64, version 1 (SYSV), dynamically linked, interpreter /lib/ld-musl-x86_64.so.1, with debug_info, not stripped" + fi elif [[ "${CI_ARCH}" == "linux/386" ]]; then From be9fe875f6f415cdb022179eb35db36568e68874 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 20 May 2024 19:33:25 +0200 Subject: [PATCH 095/339] Always try to chown auxiliary database files, not only when chown of the database files itself failed Signed-off-by: DL6ER --- .github/.codespellignore | 1 + src/dnsmasq_interface.c | 64 ++++++++++++++++++++++------------------ 2 files changed, 37 insertions(+), 28 deletions(-) diff --git a/.github/.codespellignore b/.github/.codespellignore index cdccd1cd..645d300f 100644 --- a/.github/.codespellignore +++ b/.github/.codespellignore @@ -7,3 +7,4 @@ requestor requestors punycode bitap +mmapped diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c index dd337c5b..4bef6109 100644 --- a/src/dnsmasq_interface.c +++ b/src/dnsmasq_interface.c @@ -2933,50 +2933,58 @@ void FTL_fork_and_bind_sockets(struct passwd *ent_pw, bool dnsmasq_start) if(getuid() == 0) { // Only print this and change ownership of shmem objects when - // we're actually dropping root (user/group my be set to root) + // we're actually dropping root (user/group may be set to root) if(ent_pw != NULL && ent_pw->pw_uid != 0) { log_info("FTL is going to drop from root to user %s (UID %u)", ent_pw->pw_name, ent_pw->pw_uid); + + // Change ownership of shared memory objects + chown_all_shmem(ent_pw); + + // Configured FTL log file if(chown(config.files.log.ftl.v.s, ent_pw->pw_uid, ent_pw->pw_gid) == -1) { log_warn("Setting ownership (%u:%u) of %s failed: %s (%i)", ent_pw->pw_uid, ent_pw->pw_gid, config.files.log.ftl.v.s, strerror(errno), errno); } + // Configured FTL database file if(chown(config.files.database.v.s, ent_pw->pw_uid, ent_pw->pw_gid) == -1) { log_warn("Setting ownership (%u:%u) of %s failed: %s (%i)", ent_pw->pw_uid, ent_pw->pw_gid, config.files.database.v.s, strerror(errno), errno); - // Check if WAL files are present and change - // their ownership, too - char *walname = calloc(strlen(config.files.database.v.s) + 5, sizeof(char)); - if(walname != NULL) - { - strcpy(walname, config.files.database.v.s); - strcat(walname, "-wal"); - if(chown(walname, ent_pw->pw_uid, ent_pw->pw_gid) == -1) - { - log_warn("Setting ownership (%u:%u) of %s failed: %s (%i)", - ent_pw->pw_uid, ent_pw->pw_gid, walname, strerror(errno), errno); - } - free(walname); - } - char *shmname = calloc(strlen(config.files.database.v.s) + 5, sizeof(char)); - if(shmname != NULL) - { - strcpy(shmname, config.files.database.v.s); - strcat(shmname, "-shm"); - if(chown(shmname, ent_pw->pw_uid, ent_pw->pw_gid) == -1) - { - log_warn("Setting ownership (%u:%u) of %s failed: %s (%i)", - ent_pw->pw_uid, ent_pw->pw_gid, shmname, strerror(errno), errno); - } - free(shmname); - } } - chown_all_shmem(ent_pw); + + // Check if auxiliary files exist and change ownership + char *extrafile = calloc(strlen(config.files.database.v.s) + 5, sizeof(char)); + if(extrafile == NULL) + { + log_err("Memory allocation failed. Skipping some file ownership checks."); + return; + } + + // Check -wal file (write-ahead log) + strcpy(extrafile, config.files.database.v.s); + strcat(extrafile, "-wal"); + if(file_exists(extrafile) && chown(extrafile, ent_pw->pw_uid, ent_pw->pw_gid) == -1) + { + log_warn("Setting ownership (%u:%u) of %s failed: %s (%i)", + ent_pw->pw_uid, ent_pw->pw_gid, extrafile, strerror(errno), errno); + } + + // Check -shm file (mmapped shared memory) + strcpy(extrafile, config.files.database.v.s); + strcat(extrafile, "-shm"); + if(file_exists(extrafile) && chown(extrafile, ent_pw->pw_uid, ent_pw->pw_gid) == -1) + { + log_warn("Setting ownership (%u:%u) of %s failed: %s (%i)", + ent_pw->pw_uid, ent_pw->pw_gid, extrafile, strerror(errno), errno); + } + + // Free allocated memory + free(extrafile); } else { From 62413141ff2bdff2c973ad2ea3be5ea5c9cf5bbb Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 21 May 2024 20:05:07 +0200 Subject: [PATCH 096/339] Only one builder is allowed to push the API documentation Signed-off-by: DL6ER --- .github/actions/build-and-test/action.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/build-and-test/action.yml b/.github/actions/build-and-test/action.yml index a9c85ea8..8170a570 100644 --- a/.github/actions/build-and-test/action.yml +++ b/.github/actions/build-and-test/action.yml @@ -109,13 +109,13 @@ runs: subject-path: ${{ inputs.bin_name }} - name: Extract documentation files from container - if: inputs.event_name != 'pull_request' && inputs.platform == 'linux/amd64' + if: inputs.event_name != 'pull_request' && inputs.platform == 'linux/amd64' && inputs.build_opts == '' shell: bash run: | tar -xf build.tar api-docs.tar.gz - name: Upload documentation artifacts for deployoment - if: inputs.event_name != 'pull_request' && inputs.platform == 'linux/amd64' + if: inputs.event_name != 'pull_request' && inputs.platform == 'linux/amd64' && inputs.build_opts == '' uses: actions/upload-artifact@v4.3.1 with: name: pihole-api-docs From 782178c6c1ec74f6acea74c93ea6e63d93311e48 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 22 May 2024 20:37:55 +0200 Subject: [PATCH 097/339] Run tests with attached debugger Signed-off-by: DL6ER --- test/run.sh | 8 ++++++++ test/test_suite.bats | 13 ------------- 2 files changed, 8 insertions(+), 13 deletions(-) diff --git a/test/run.sh b/test/run.sh index 5f34e9d6..24976551 100755 --- a/test/run.sh +++ b/test/run.sh @@ -73,6 +73,9 @@ export FTLCONF_misc_nice="-11" export FTLCONF_dns_upstrrr="-11" export FTLCONF_debug_api="not_a_bool" +# Prepare gdb session +echo "handle SIGHUP nostop SIGPIPE nostop SIGTERM nostop SIG32 nostop SIG33 nostop SIG34 nostop SIG35 nostop SIG41 nostop" > /root/.gdbinit + # Start FTL if ! su pihole -s /bin/sh -c /home/pihole/pihole-FTL; then echo "pihole-FTL failed to start" @@ -89,6 +92,10 @@ fi # Give FTL some time for startup preparations sleep 2 +# Attach debugger and immediately continue running the binary +# In case a non-ignored signal occurs (a crash), create a full backtrace +gdb -p $(cat /run/pihole-FTL.pid) --ex continue --ex "bt full" & + # Print versions of pihole-FTL echo -n "FTL version (DNS): " dig TXT CHAOS version.FTL @127.0.0.1 +short @@ -130,6 +137,7 @@ if [[ $RET != 0 ]]; then fi # Kill pihole-FTL after having completed tests +# This will also shut down the debugger kill "$(pidof pihole-FTL)" # Restore umask diff --git a/test/test_suite.bats b/test/test_suite.bats index 94c04019..351c85fa 100644 --- a/test/test_suite.bats +++ b/test/test_suite.bats @@ -972,19 +972,6 @@ [[ "${STATIC}" == "true" && "${lines[@]}" != *"interpreter"* ]] } -@test "Architecture is correctly reported on startup" { - run bash -c 'grep "Compiled for" /var/log/pihole/FTL.log' - printf "Output: %s\n\$CI_ARCH: %s\nuname -m: %s\n" "${lines[@]:-not set}" "${CI_ARCH:-not set}" "$(uname -m)" - [[ ${lines[0]} == *"Compiled for ${CI_ARCH:-$(uname -m)}"* ]] -} - -@test "Building machine (CI) is reported on startup" { - [[ ${CI_ARCH} != "" ]] && compiled_str="on CI" || compiled_str="locally" && export compiled_str - run bash -c 'grep "Compiled for" /var/log/pihole/FTL.log' - printf "Output: %s\n\$CI_ARCH: %s\n" "${lines[@]:-not set}" "${CI_ARCH:-not set}" - [[ ${lines[0]} == *"(compiled ${compiled_str})"* ]] -} - @test "Compiler version is correctly reported on startup" { compiler_version="$(${CC} --version | head -n1)" && export compiler_version run bash -c 'grep "Compiled for" /var/log/pihole/FTL.log' From 3afd9f0d41300b722379da000ab085e0bd551d94 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 23 May 2024 20:53:09 +0200 Subject: [PATCH 098/339] Update embedded SQLite3 to 3.46.0 Signed-off-by: DL6ER --- src/database/shell.c | 1679 +++++++- src/database/sqlite3.c | 8471 ++++++++++++++++++++++++---------------- src/database/sqlite3.h | 97 +- 3 files changed, 6727 insertions(+), 3520 deletions(-) diff --git a/src/database/shell.c b/src/database/shell.c index 3de2f5d0..fc6bf5e6 100644 --- a/src/database/shell.c +++ b/src/database/shell.c @@ -1263,6 +1263,9 @@ SQLITE_INTERNAL_LINKAGE char* fGetsUtf8(char *cBuf, int ncMax, FILE *pfIn){ * setOutputStream(FILE *pf) * This is normally the stream that CLI normal output goes to. * For the stand-alone CLI, it is stdout with no .output redirect. + * + * The ?putz(z) forms are required for the Fiddle builds for string literal + * output, in aid of enforcing format string to argument correspondence. */ # define sputz(s,z) fPutsUtf8(z,s) # define sputf fPrintfUtf8 @@ -1274,12 +1277,18 @@ SQLITE_INTERNAL_LINKAGE char* fGetsUtf8(char *cBuf, int ncMax, FILE *pfIn){ #else /* For Fiddle, all console handling and emit redirection is omitted. */ -# define sputz(fp,z) fputs(z,fp) -# define sputf(fp,fmt, ...) fprintf(fp,fmt,__VA_ARGS__) -# define oputz(z) fputs(z,stdout) +/* These next 3 macros are for emitting formatted output. When complaints + * from the WASM build are issued for non-formatted output, (when a mere + * string literal is to be emitted, the ?putz(z) forms should be used. + * (This permits compile-time checking of format string / argument mismatch.) + */ # define oputf(fmt, ...) printf(fmt,__VA_ARGS__) -# define eputz(z) fputs(z,stderr) # define eputf(fmt, ...) fprintf(stderr,fmt,__VA_ARGS__) +# define sputf(fp,fmt, ...) fprintf(fp,fmt,__VA_ARGS__) +/* These next 3 macros are for emitting simple string literals. */ +# define oputz(z) fputs(z,stdout) +# define eputz(z) fputs(z,stderr) +# define sputz(fp,z) fputs(z,fp) # define oputb(buf,na) fwrite(buf,1,na,stdout) #endif @@ -5724,16 +5733,20 @@ SQLITE_EXTENSION_INIT1 ** index is ix. The 0th member is given by smBase. The sequence members ** progress per ix increment by smStep. */ -static sqlite3_int64 genSeqMember(sqlite3_int64 smBase, - sqlite3_int64 smStep, - sqlite3_uint64 ix){ - if( ix>=(sqlite3_uint64)LLONG_MAX ){ +static sqlite3_int64 genSeqMember( + sqlite3_int64 smBase, + sqlite3_int64 smStep, + sqlite3_uint64 ix +){ + static const sqlite3_uint64 mxI64 = + ((sqlite3_uint64)0x7fffffff)<<32 | 0xffffffff; + if( ix>=mxI64 ){ /* Get ix into signed i64 range. */ - ix -= (sqlite3_uint64)LLONG_MAX; + ix -= mxI64; /* With 2's complement ALU, this next can be 1 step, but is split into * 2 for UBSAN's satisfaction (and hypothetical 1's complement ALUs.) */ - smBase += (LLONG_MAX/2) * smStep; - smBase += (LLONG_MAX - LLONG_MAX/2) * smStep; + smBase += (mxI64/2) * smStep; + smBase += (mxI64 - mxI64/2) * smStep; } /* Under UBSAN (or on 1's complement machines), must do this last term * in steps to avoid the dreaded (and harmless) signed multiply overlow. */ @@ -5993,13 +6006,13 @@ static int seriesEof(sqlite3_vtab_cursor *cur){ ** parameter. (idxStr is not used in this implementation.) idxNum ** is a bitmask showing which constraints are available: ** -** 1: start=VALUE -** 2: stop=VALUE -** 4: step=VALUE -** -** Also, if bit 8 is set, that means that the series should be output -** in descending order rather than in ascending order. If bit 16 is -** set, then output must appear in ascending order. +** 0x01: start=VALUE +** 0x02: stop=VALUE +** 0x04: step=VALUE +** 0x08: descending order +** 0x10: ascending order +** 0x20: LIMIT VALUE +** 0x40: OFFSET VALUE ** ** This routine should initialize the cursor and position it so that it ** is pointing at the first row, or pointing off the end of the table @@ -6013,26 +6026,44 @@ static int seriesFilter( series_cursor *pCur = (series_cursor *)pVtabCursor; int i = 0; (void)idxStrUnused; - if( idxNum & 1 ){ + if( idxNum & 0x01 ){ pCur->ss.iBase = sqlite3_value_int64(argv[i++]); }else{ pCur->ss.iBase = 0; } - if( idxNum & 2 ){ + if( idxNum & 0x02 ){ pCur->ss.iTerm = sqlite3_value_int64(argv[i++]); }else{ pCur->ss.iTerm = 0xffffffff; } - if( idxNum & 4 ){ + if( idxNum & 0x04 ){ pCur->ss.iStep = sqlite3_value_int64(argv[i++]); if( pCur->ss.iStep==0 ){ pCur->ss.iStep = 1; }else if( pCur->ss.iStep<0 ){ - if( (idxNum & 16)==0 ) idxNum |= 8; + if( (idxNum & 0x10)==0 ) idxNum |= 0x08; } }else{ pCur->ss.iStep = 1; } + if( idxNum & 0x20 ){ + sqlite3_int64 iLimit = sqlite3_value_int64(argv[i++]); + sqlite3_int64 iTerm; + if( idxNum & 0x40 ){ + sqlite3_int64 iOffset = sqlite3_value_int64(argv[i++]); + if( iOffset>0 ){ + pCur->ss.iBase += pCur->ss.iStep*iOffset; + } + } + if( iLimit>=0 ){ + iTerm = pCur->ss.iBase + (iLimit - 1)*pCur->ss.iStep; + if( pCur->ss.iStep<0 ){ + if( iTerm>pCur->ss.iTerm ) pCur->ss.iTerm = iTerm; + }else{ + if( iTermss.iTerm ) pCur->ss.iTerm = iTerm; + } + } + } for(i=0; iss.isReversing = pCur->ss.iStep > 0; }else{ pCur->ss.isReversing = pCur->ss.iStep < 0; @@ -6063,10 +6094,13 @@ static int seriesFilter( ** ** The query plan is represented by bits in idxNum: ** -** (1) start = $value -- constraint exists -** (2) stop = $value -- constraint exists -** (4) step = $value -- constraint exists -** (8) output in descending order +** 0x01 start = $value -- constraint exists +** 0x02 stop = $value -- constraint exists +** 0x04 step = $value -- constraint exists +** 0x08 output is in descending order +** 0x10 output is in ascending order +** 0x20 LIMIT $value -- constraint exists +** 0x40 OFFSET $value -- constraint exists */ static int seriesBestIndex( sqlite3_vtab *pVTab, @@ -6074,10 +6108,12 @@ static int seriesBestIndex( ){ int i, j; /* Loop over constraints */ int idxNum = 0; /* The query plan bitmask */ +#ifndef ZERO_ARGUMENT_GENERATE_SERIES int bStartSeen = 0; /* EQ constraint seen on the START column */ +#endif int unusableMask = 0; /* Mask of unusable constraints */ int nArg = 0; /* Number of arguments that seriesFilter() expects */ - int aIdx[3]; /* Constraints on start, stop, and step */ + int aIdx[5]; /* Constraints on start, stop, step, LIMIT, OFFSET */ const struct sqlite3_index_constraint *pConstraint; /* This implementation assumes that the start, stop, and step columns @@ -6085,28 +6121,54 @@ static int seriesBestIndex( assert( SERIES_COLUMN_STOP == SERIES_COLUMN_START+1 ); assert( SERIES_COLUMN_STEP == SERIES_COLUMN_START+2 ); - aIdx[0] = aIdx[1] = aIdx[2] = -1; + aIdx[0] = aIdx[1] = aIdx[2] = aIdx[3] = aIdx[4] = -1; pConstraint = pIdxInfo->aConstraint; for(i=0; inConstraint; i++, pConstraint++){ int iCol; /* 0 for start, 1 for stop, 2 for step */ int iMask; /* bitmask for those column */ + int op = pConstraint->op; + if( op>=SQLITE_INDEX_CONSTRAINT_LIMIT + && op<=SQLITE_INDEX_CONSTRAINT_OFFSET + ){ + if( pConstraint->usable==0 ){ + /* do nothing */ + }else if( op==SQLITE_INDEX_CONSTRAINT_LIMIT ){ + aIdx[3] = i; + idxNum |= 0x20; + }else{ + assert( op==SQLITE_INDEX_CONSTRAINT_OFFSET ); + aIdx[4] = i; + idxNum |= 0x40; + } + continue; + } if( pConstraint->iColumniColumn - SERIES_COLUMN_START; assert( iCol>=0 && iCol<=2 ); iMask = 1 << iCol; - if( iCol==0 ) bStartSeen = 1; +#ifndef ZERO_ARGUMENT_GENERATE_SERIES + if( iCol==0 && op==SQLITE_INDEX_CONSTRAINT_EQ ){ + bStartSeen = 1; + } +#endif if( pConstraint->usable==0 ){ unusableMask |= iMask; continue; - }else if( pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ ){ + }else if( op==SQLITE_INDEX_CONSTRAINT_EQ ){ idxNum |= iMask; aIdx[iCol] = i; } } - for(i=0; i<3; i++){ + if( aIdx[3]==0 ){ + /* Ignore OFFSET if LIMIT is omitted */ + idxNum &= ~0x60; + aIdx[4] = 0; + } + for(i=0; i<5; i++){ if( (j = aIdx[i])>=0 ){ pIdxInfo->aConstraintUsage[j].argvIndex = ++nArg; - pIdxInfo->aConstraintUsage[j].omit = !SQLITE_SERIES_CONSTRAINT_VERIFY; + pIdxInfo->aConstraintUsage[j].omit = + !SQLITE_SERIES_CONSTRAINT_VERIFY || i>=3; } } /* The current generate_column() implementation requires at least one @@ -6127,19 +6189,22 @@ static int seriesBestIndex( ** this plan is unusable */ return SQLITE_CONSTRAINT; } - if( (idxNum & 3)==3 ){ + if( (idxNum & 0x03)==0x03 ){ /* Both start= and stop= boundaries are available. This is the ** the preferred case */ pIdxInfo->estimatedCost = (double)(2 - ((idxNum&4)!=0)); pIdxInfo->estimatedRows = 1000; if( pIdxInfo->nOrderBy>=1 && pIdxInfo->aOrderBy[0].iColumn==0 ){ if( pIdxInfo->aOrderBy[0].desc ){ - idxNum |= 8; + idxNum |= 0x08; }else{ - idxNum |= 16; + idxNum |= 0x10; } pIdxInfo->orderByConsumed = 1; } + }else if( (idxNum & 0x21)==0x21 ){ + /* We have start= and LIMIT */ + pIdxInfo->estimatedRows = 2500; }else{ /* If either boundary is missing, we have to generate a huge span ** of numbers. Make this case very expensive so that the query @@ -7466,7 +7531,9 @@ static int writeFile( #if !defined(_WIN32) && !defined(WIN32) if( S_ISLNK(mode) ){ const char *zTo = (const char*)sqlite3_value_text(pData); - if( zTo==0 || symlink(zTo, zFile)<0 ) return 1; + if( zTo==0 ) return 1; + unlink(zFile); + if( symlink(zTo, zFile)<0 ) return 1; }else #endif { @@ -7552,13 +7619,19 @@ static int writeFile( return 1; } #else - /* Legacy unix */ - struct timeval times[2]; - times[0].tv_usec = times[1].tv_usec = 0; - times[0].tv_sec = time(0); - times[1].tv_sec = mtime; - if( utimes(zFile, times) ){ - return 1; + /* Legacy unix. + ** + ** Do not use utimes() on a symbolic link - it sees through the link and + ** modifies the timestamps on the target. Or fails if the target does + ** not exist. */ + if( 0==S_ISLNK(mode) ){ + struct timeval times[2]; + times[0].tv_usec = times[1].tv_usec = 0; + times[0].tv_sec = time(0); + times[1].tv_sec = mtime; + if( utimes(zFile, times) ){ + return 1; + } } #endif } @@ -11630,7 +11703,7 @@ static void sqlarUncompressFunc( sqlite3_value **argv ){ uLong nData; - uLongf sz; + sqlite3_int64 sz; assert( argc==2 ); sz = sqlite3_value_int(argv[1]); @@ -11638,14 +11711,15 @@ static void sqlarUncompressFunc( if( sz<=0 || sz==(nData = sqlite3_value_bytes(argv[0])) ){ sqlite3_result_value(context, argv[0]); }else{ + uLongf szf = sz; const Bytef *pData= sqlite3_value_blob(argv[0]); Bytef *pOut = sqlite3_malloc(sz); if( pOut==0 ){ sqlite3_result_error_nomem(context); - }else if( Z_OK!=uncompress(pOut, &sz, pData, nData) ){ + }else if( Z_OK!=uncompress(pOut, &szf, pData, nData) ){ sqlite3_result_error(context, "error in uncompress()", -1); }else{ - sqlite3_result_blob(context, pOut, sz, SQLITE_TRANSIENT); + sqlite3_result_blob(context, pOut, szf, SQLITE_TRANSIENT); } sqlite3_free(pOut); } @@ -13797,7 +13871,7 @@ sqlite3expert *sqlite3_expert_new(sqlite3 *db, char **pzErrmsg){ sqlite3_stmt *pSql = 0; rc = idxPrintfPrepareStmt(pNew->db, &pSql, pzErrmsg, "SELECT sql FROM sqlite_schema WHERE name NOT LIKE 'sqlite_%%'" - " AND sql NOT LIKE 'CREATE VIRTUAL %%'" + " AND sql NOT LIKE 'CREATE VIRTUAL %%' ORDER BY rowid" ); while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSql) ){ const char *zSql = (const char*)sqlite3_column_text(pSql, 0); @@ -13999,6 +14073,1124 @@ void sqlite3_expert_destroy(sqlite3expert *p){ /************************* End ../ext/expert/sqlite3expert.c ********************/ +/************************* Begin ../ext/intck/sqlite3intck.h ******************/ +/* +** 2024-02-08 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +*/ + +/* +** Incremental Integrity-Check Extension +** ------------------------------------- +** +** This module contains code to check whether or not an SQLite database +** is well-formed or corrupt. This is the same task as performed by SQLite's +** built-in "PRAGMA integrity_check" command. This module differs from +** "PRAGMA integrity_check" in that: +** +** + It is less thorough - this module does not detect certain types +** of corruption that are detected by the PRAGMA command. However, +** it does detect all kinds of corruption that are likely to cause +** errors in SQLite applications. +** +** + It is slower. Sometimes up to three times slower. +** +** + It allows integrity-check operations to be split into multiple +** transactions, so that the database does not need to be read-locked +** for the duration of the integrity-check. +** +** One way to use the API to run integrity-check on the "main" database +** of handle db is: +** +** int rc = SQLITE_OK; +** sqlite3_intck *p = 0; +** +** sqlite3_intck_open(db, "main", &p); +** while( SQLITE_OK==sqlite3_intck_step(p) ){ +** const char *zMsg = sqlite3_intck_message(p); +** if( zMsg ) printf("corruption: %s\n", zMsg); +** } +** rc = sqlite3_intck_error(p, &zErr); +** if( rc!=SQLITE_OK ){ +** printf("error occured (rc=%d), (errmsg=%s)\n", rc, zErr); +** } +** sqlite3_intck_close(p); +** +** Usually, the sqlite3_intck object opens a read transaction within the +** first call to sqlite3_intck_step() and holds it open until the +** integrity-check is complete. However, if sqlite3_intck_unlock() is +** called, the read transaction is ended and a new read transaction opened +** by the subsequent call to sqlite3_intck_step(). +*/ + +#ifndef _SQLITE_INTCK_H +#define _SQLITE_INTCK_H + +/* #include "sqlite3.h" */ + +#ifdef __cplusplus +extern "C" { +#endif + +/* +** An ongoing incremental integrity-check operation is represented by an +** opaque pointer of the following type. +*/ +typedef struct sqlite3_intck sqlite3_intck; + +/* +** Open a new incremental integrity-check object. If successful, populate +** output variable (*ppOut) with the new object handle and return SQLITE_OK. +** Or, if an error occurs, set (*ppOut) to NULL and return an SQLite error +** code (e.g. SQLITE_NOMEM). +** +** The integrity-check will be conducted on database zDb (which must be "main", +** "temp", or the name of an attached database) of database handle db. Once +** this function has been called successfully, the caller should not use +** database handle db until the integrity-check object has been destroyed +** using sqlite3_intck_close(). +*/ +int sqlite3_intck_open( + sqlite3 *db, /* Database handle */ + const char *zDb, /* Database name ("main", "temp" etc.) */ + sqlite3_intck **ppOut /* OUT: New sqlite3_intck handle */ +); + +/* +** Close and release all resources associated with a handle opened by an +** earlier call to sqlite3_intck_open(). The results of using an +** integrity-check handle after it has been passed to this function are +** undefined. +*/ +void sqlite3_intck_close(sqlite3_intck *pCk); + +/* +** Do the next step of the integrity-check operation specified by the handle +** passed as the only argument. This function returns SQLITE_DONE if the +** integrity-check operation is finished, or an SQLite error code if +** an error occurs, or SQLITE_OK if no error occurs but the integrity-check +** is not finished. It is not considered an error if database corruption +** is encountered. +** +** Following a successful call to sqlite3_intck_step() (one that returns +** SQLITE_OK), sqlite3_intck_message() returns a non-NULL value if +** corruption was detected in the db. +** +** If an error occurs and a value other than SQLITE_OK or SQLITE_DONE is +** returned, then the integrity-check handle is placed in an error state. +** In this state all subsequent calls to sqlite3_intck_step() or +** sqlite3_intck_unlock() will immediately return the same error. The +** sqlite3_intck_error() method may be used to obtain an English language +** error message in this case. +*/ +int sqlite3_intck_step(sqlite3_intck *pCk); + +/* +** If the previous call to sqlite3_intck_step() encountered corruption +** within the database, then this function returns a pointer to a buffer +** containing a nul-terminated string describing the corruption in +** English. If the previous call to sqlite3_intck_step() did not encounter +** corruption, or if there was no previous call, this function returns +** NULL. +*/ +const char *sqlite3_intck_message(sqlite3_intck *pCk); + +/* +** Close any read-transaction opened by an earlier call to +** sqlite3_intck_step(). Any subsequent call to sqlite3_intck_step() will +** open a new transaction. Return SQLITE_OK if successful, or an SQLite error +** code otherwise. +** +** If an error occurs, then the integrity-check handle is placed in an error +** state. In this state all subsequent calls to sqlite3_intck_step() or +** sqlite3_intck_unlock() will immediately return the same error. The +** sqlite3_intck_error() method may be used to obtain an English language +** error message in this case. +*/ +int sqlite3_intck_unlock(sqlite3_intck *pCk); + +/* +** If an error has occurred in an earlier call to sqlite3_intck_step() +** or sqlite3_intck_unlock(), then this method returns the associated +** SQLite error code. Additionally, if pzErr is not NULL, then (*pzErr) +** may be set to point to a nul-terminated string containing an English +** language error message. Or, if no error message is available, to +** NULL. +** +** If no error has occurred within sqlite3_intck_step() or +** sqlite_intck_unlock() calls on the handle passed as the first argument, +** then SQLITE_OK is returned and (*pzErr) set to NULL. +*/ +int sqlite3_intck_error(sqlite3_intck *pCk, const char **pzErr); + +/* +** This API is used for testing only. It returns the full-text of an SQL +** statement used to test object zObj, which may be a table or index. +** The returned buffer is valid until the next call to either this function +** or sqlite3_intck_close() on the same sqlite3_intck handle. +*/ +const char *sqlite3_intck_test_sql(sqlite3_intck *pCk, const char *zObj); + + +#ifdef __cplusplus +} /* end of the 'extern "C"' block */ +#endif + +#endif /* ifndef _SQLITE_INTCK_H */ + +/************************* End ../ext/intck/sqlite3intck.h ********************/ +/************************* Begin ../ext/intck/sqlite3intck.c ******************/ +/* +** 2024-02-08 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +*/ + +/* #include "sqlite3intck.h" */ +#include +#include + +#include +#include + +/* +** nKeyVal: +** The number of values that make up the 'key' for the current pCheck +** statement. +** +** rc: +** Error code returned by most recent sqlite3_intck_step() or +** sqlite3_intck_unlock() call. This is set to SQLITE_DONE when +** the integrity-check operation is finished. +** +** zErr: +** If the object has entered the error state, this is the error message. +** Is freed using sqlite3_free() when the object is deleted. +** +** zTestSql: +** The value returned by the most recent call to sqlite3_intck_testsql(). +** Each call to testsql() frees the previous zTestSql value (using +** sqlite3_free()) and replaces it with the new value it will return. +*/ +struct sqlite3_intck { + sqlite3 *db; + const char *zDb; /* Copy of zDb parameter to _open() */ + char *zObj; /* Current object. Or NULL. */ + + sqlite3_stmt *pCheck; /* Current check statement */ + char *zKey; + int nKeyVal; + + char *zMessage; + int bCorruptSchema; + + int rc; /* Error code */ + char *zErr; /* Error message */ + char *zTestSql; /* Returned by sqlite3_intck_test_sql() */ +}; + + +/* +** Some error has occurred while using database p->db. Save the error message +** and error code currently held by the database handle in p->rc and p->zErr. +*/ +static void intckSaveErrmsg(sqlite3_intck *p){ + p->rc = sqlite3_errcode(p->db); + sqlite3_free(p->zErr); + p->zErr = sqlite3_mprintf("%s", sqlite3_errmsg(p->db)); +} + +/* +** If the handle passed as the first argument is already in the error state, +** then this function is a no-op (returns NULL immediately). Otherwise, if an +** error occurs within this function, it leaves an error in said handle. +** +** Otherwise, this function attempts to prepare SQL statement zSql and +** return the resulting statement handle to the user. +*/ +static sqlite3_stmt *intckPrepare(sqlite3_intck *p, const char *zSql){ + sqlite3_stmt *pRet = 0; + if( p->rc==SQLITE_OK ){ + p->rc = sqlite3_prepare_v2(p->db, zSql, -1, &pRet, 0); + if( p->rc!=SQLITE_OK ){ + intckSaveErrmsg(p); + assert( pRet==0 ); + } + } + return pRet; +} + +/* +** If the handle passed as the first argument is already in the error state, +** then this function is a no-op (returns NULL immediately). Otherwise, if an +** error occurs within this function, it leaves an error in said handle. +** +** Otherwise, this function treats argument zFmt as a printf() style format +** string. It formats it according to the trailing arguments and then +** attempts to prepare the results and return the resulting prepared +** statement. +*/ +static sqlite3_stmt *intckPrepareFmt(sqlite3_intck *p, const char *zFmt, ...){ + sqlite3_stmt *pRet = 0; + va_list ap; + char *zSql = 0; + va_start(ap, zFmt); + zSql = sqlite3_vmprintf(zFmt, ap); + if( p->rc==SQLITE_OK && zSql==0 ){ + p->rc = SQLITE_NOMEM; + } + pRet = intckPrepare(p, zSql); + sqlite3_free(zSql); + va_end(ap); + return pRet; +} + +/* +** Finalize SQL statement pStmt. If an error occurs and the handle passed +** as the first argument does not already contain an error, store the +** error in the handle. +*/ +static void intckFinalize(sqlite3_intck *p, sqlite3_stmt *pStmt){ + int rc = sqlite3_finalize(pStmt); + if( p->rc==SQLITE_OK && rc!=SQLITE_OK ){ + intckSaveErrmsg(p); + } +} + +/* +** If there is already an error in handle p, return it. Otherwise, call +** sqlite3_step() on the statement handle and return that value. +*/ +static int intckStep(sqlite3_intck *p, sqlite3_stmt *pStmt){ + if( p->rc ) return p->rc; + return sqlite3_step(pStmt); +} + +/* +** Execute SQL statement zSql. There is no way to obtain any results +** returned by the statement. This function uses the sqlite3_intck error +** code convention. +*/ +static void intckExec(sqlite3_intck *p, const char *zSql){ + sqlite3_stmt *pStmt = 0; + pStmt = intckPrepare(p, zSql); + intckStep(p, pStmt); + intckFinalize(p, pStmt); +} + +/* +** A wrapper around sqlite3_mprintf() that uses the sqlite3_intck error +** code convention. +*/ +static char *intckMprintf(sqlite3_intck *p, const char *zFmt, ...){ + va_list ap; + char *zRet = 0; + va_start(ap, zFmt); + zRet = sqlite3_vmprintf(zFmt, ap); + if( p->rc==SQLITE_OK ){ + if( zRet==0 ){ + p->rc = SQLITE_NOMEM; + } + }else{ + sqlite3_free(zRet); + zRet = 0; + } + return zRet; +} + +/* +** This is used by sqlite3_intck_unlock() to save the vector key value +** required to restart the current pCheck query as a nul-terminated string +** in p->zKey. +*/ +static void intckSaveKey(sqlite3_intck *p){ + int ii; + char *zSql = 0; + sqlite3_stmt *pStmt = 0; + sqlite3_stmt *pXinfo = 0; + const char *zDir = 0; + + assert( p->pCheck ); + assert( p->zKey==0 ); + + pXinfo = intckPrepareFmt(p, + "SELECT group_concat(desc, '') FROM %Q.sqlite_schema s, " + "pragma_index_xinfo(%Q, %Q) " + "WHERE s.type='index' AND s.name=%Q", + p->zDb, p->zObj, p->zDb, p->zObj + ); + if( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pXinfo) ){ + zDir = (const char*)sqlite3_column_text(pXinfo, 0); + } + + if( zDir==0 ){ + /* Object is a table, not an index. This is the easy case,as there are + ** no DESC columns or NULL values in a primary key. */ + const char *zSep = "SELECT '(' || "; + for(ii=0; iinKeyVal; ii++){ + zSql = intckMprintf(p, "%z%squote(?)", zSql, zSep); + zSep = " || ', ' || "; + } + zSql = intckMprintf(p, "%z || ')'", zSql); + }else{ + + /* Object is an index. */ + assert( p->nKeyVal>1 ); + for(ii=p->nKeyVal; ii>0; ii--){ + int bLastIsDesc = zDir[ii-1]=='1'; + int bLastIsNull = sqlite3_column_type(p->pCheck, ii)==SQLITE_NULL; + const char *zLast = sqlite3_column_name(p->pCheck, ii); + char *zLhs = 0; + char *zRhs = 0; + char *zWhere = 0; + + if( bLastIsNull ){ + if( bLastIsDesc ) continue; + zWhere = intckMprintf(p, "'%s IS NOT NULL'", zLast); + }else{ + const char *zOp = bLastIsDesc ? "<" : ">"; + zWhere = intckMprintf(p, "'%s %s ' || quote(?%d)", zLast, zOp, ii); + } + + if( ii>1 ){ + const char *zLhsSep = ""; + const char *zRhsSep = ""; + int jj; + for(jj=0; jjpCheck,jj+1); + zLhs = intckMprintf(p, "%z%s%s", zLhs, zLhsSep, zAlias); + zRhs = intckMprintf(p, "%z%squote(?%d)", zRhs, zRhsSep, jj+1); + zLhsSep = ","; + zRhsSep = " || ',' || "; + } + + zWhere = intckMprintf(p, + "'(%z) IS (' || %z || ') AND ' || %z", + zLhs, zRhs, zWhere); + } + zWhere = intckMprintf(p, "'WHERE ' || %z", zWhere); + + zSql = intckMprintf(p, "%z%s(quote( %z ) )", + zSql, + (zSql==0 ? "VALUES" : ",\n "), + zWhere + ); + } + zSql = intckMprintf(p, + "WITH wc(q) AS (\n%z\n)" + "SELECT 'VALUES' || group_concat('(' || q || ')', ',\n ') FROM wc" + , zSql + ); + } + + pStmt = intckPrepare(p, zSql); + if( p->rc==SQLITE_OK ){ + for(ii=0; iinKeyVal; ii++){ + sqlite3_bind_value(pStmt, ii+1, sqlite3_column_value(p->pCheck, ii+1)); + } + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + p->zKey = intckMprintf(p,"%s",(const char*)sqlite3_column_text(pStmt, 0)); + } + intckFinalize(p, pStmt); + } + + sqlite3_free(zSql); + intckFinalize(p, pXinfo); +} + +/* +** Find the next database object (table or index) to check. If successful, +** set sqlite3_intck.zObj to point to a nul-terminated buffer containing +** the object's name before returning. +*/ +static void intckFindObject(sqlite3_intck *p){ + sqlite3_stmt *pStmt = 0; + char *zPrev = p->zObj; + p->zObj = 0; + + assert( p->rc==SQLITE_OK ); + assert( p->pCheck==0 ); + + pStmt = intckPrepareFmt(p, + "WITH tables(table_name) AS (" + " SELECT name" + " FROM %Q.sqlite_schema WHERE (type='table' OR type='index') AND rootpage" + " UNION ALL " + " SELECT 'sqlite_schema'" + ")" + "SELECT table_name FROM tables " + "WHERE ?1 IS NULL OR table_name%s?1 " + "ORDER BY 1" + , p->zDb, (p->zKey ? ">=" : ">") + ); + + if( p->rc==SQLITE_OK ){ + sqlite3_bind_text(pStmt, 1, zPrev, -1, SQLITE_TRANSIENT); + if( sqlite3_step(pStmt)==SQLITE_ROW ){ + p->zObj = intckMprintf(p,"%s",(const char*)sqlite3_column_text(pStmt, 0)); + } + } + intckFinalize(p, pStmt); + + /* If this is a new object, ensure the previous key value is cleared. */ + if( sqlite3_stricmp(p->zObj, zPrev) ){ + sqlite3_free(p->zKey); + p->zKey = 0; + } + + sqlite3_free(zPrev); +} + +/* +** Return the size in bytes of the first token in nul-terminated buffer z. +** For the purposes of this call, a token is either: +** +** * a quoted SQL string, +* * a contiguous series of ascii alphabet characters, or +* * any other single byte. +*/ +static int intckGetToken(const char *z){ + char c = z[0]; + int iRet = 1; + if( c=='\'' || c=='"' || c=='`' ){ + while( 1 ){ + if( z[iRet]==c ){ + iRet++; + if( z[iRet]!=c ) break; + } + iRet++; + } + } + else if( c=='[' ){ + while( z[iRet++]!=']' && z[iRet] ); + } + else if( (c>='A' && c<='Z') || (c>='a' && c<='z') ){ + while( (z[iRet]>='A' && z[iRet]<='Z') || (z[iRet]>='a' && z[iRet]<='z') ){ + iRet++; + } + } + + return iRet; +} + +/* +** Return true if argument c is an ascii whitespace character. +*/ +static int intckIsSpace(char c){ + return (c==' ' || c=='\t' || c=='\n' || c=='\r'); +} + +/* +** Argument z points to the text of a CREATE INDEX statement. This function +** identifies the part of the text that contains either the index WHERE +** clause (if iCol<0) or the iCol'th column of the index. +** +** If (iCol<0), the identified fragment does not include the "WHERE" keyword, +** only the expression that follows it. If (iCol>=0) then the identified +** fragment does not include any trailing sort-order keywords - "ASC" or +** "DESC". +** +** If the CREATE INDEX statement does not contain the requested field or +** clause, NULL is returned and (*pnByte) is set to 0. Otherwise, a pointer to +** the identified fragment is returned and output parameter (*pnByte) set +** to its size in bytes. +*/ +static const char *intckParseCreateIndex(const char *z, int iCol, int *pnByte){ + int iOff = 0; + int iThisCol = 0; + int iStart = 0; + int nOpen = 0; + + const char *zRet = 0; + int nRet = 0; + + int iEndOfCol = 0; + + /* Skip forward until the first "(" token */ + while( z[iOff]!='(' ){ + iOff += intckGetToken(&z[iOff]); + if( z[iOff]=='\0' ) return 0; + } + assert( z[iOff]=='(' ); + + nOpen = 1; + iOff++; + iStart = iOff; + while( z[iOff] ){ + const char *zToken = &z[iOff]; + int nToken = 0; + + /* Check if this is the end of the current column - either a "," or ")" + ** when nOpen==1. */ + if( nOpen==1 ){ + if( z[iOff]==',' || z[iOff]==')' ){ + if( iCol==iThisCol ){ + int iEnd = iEndOfCol ? iEndOfCol : iOff; + nRet = (iEnd - iStart); + zRet = &z[iStart]; + break; + } + iStart = iOff+1; + while( intckIsSpace(z[iStart]) ) iStart++; + iThisCol++; + } + if( z[iOff]==')' ) break; + } + if( z[iOff]=='(' ) nOpen++; + if( z[iOff]==')' ) nOpen--; + nToken = intckGetToken(zToken); + + if( (nToken==3 && 0==sqlite3_strnicmp(zToken, "ASC", nToken)) + || (nToken==4 && 0==sqlite3_strnicmp(zToken, "DESC", nToken)) + ){ + iEndOfCol = iOff; + }else if( 0==intckIsSpace(zToken[0]) ){ + iEndOfCol = 0; + } + + iOff += nToken; + } + + /* iStart is now the byte offset of 1 byte passed the final ')' in the + ** CREATE INDEX statement. Try to find a WHERE clause to return. */ + while( zRet==0 && z[iOff] ){ + int n = intckGetToken(&z[iOff]); + if( n==5 && 0==sqlite3_strnicmp(&z[iOff], "where", 5) ){ + zRet = &z[iOff+5]; + nRet = (int)strlen(zRet); + } + iOff += n; + } + + /* Trim any whitespace from the start and end of the returned string. */ + if( zRet ){ + while( intckIsSpace(zRet[0]) ){ + nRet--; + zRet++; + } + while( nRet>0 && intckIsSpace(zRet[nRet-1]) ) nRet--; + } + + *pnByte = nRet; + return zRet; +} + +/* +** User-defined SQL function wrapper for intckParseCreateIndex(): +** +** SELECT parse_create_index(, ); +*/ +static void intckParseCreateIndexFunc( + sqlite3_context *pCtx, + int nVal, + sqlite3_value **apVal +){ + const char *zSql = (const char*)sqlite3_value_text(apVal[0]); + int idx = sqlite3_value_int(apVal[1]); + const char *zRes = 0; + int nRes = 0; + + assert( nVal==2 ); + if( zSql ){ + zRes = intckParseCreateIndex(zSql, idx, &nRes); + } + sqlite3_result_text(pCtx, zRes, nRes, SQLITE_TRANSIENT); +} + +/* +** Return true if sqlite3_intck.db has automatic indexes enabled, false +** otherwise. +*/ +static int intckGetAutoIndex(sqlite3_intck *p){ + int bRet = 0; + sqlite3_stmt *pStmt = 0; + pStmt = intckPrepare(p, "PRAGMA automatic_index"); + if( SQLITE_ROW==intckStep(p, pStmt) ){ + bRet = sqlite3_column_int(pStmt, 0); + } + intckFinalize(p, pStmt); + return bRet; +} + +/* +** Return true if zObj is an index, or false otherwise. +*/ +static int intckIsIndex(sqlite3_intck *p, const char *zObj){ + int bRet = 0; + sqlite3_stmt *pStmt = 0; + pStmt = intckPrepareFmt(p, + "SELECT 1 FROM %Q.sqlite_schema WHERE name=%Q AND type='index'", + p->zDb, zObj + ); + if( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + bRet = 1; + } + intckFinalize(p, pStmt); + return bRet; +} + +/* +** Return a pointer to a nul-terminated buffer containing the SQL statement +** used to check database object zObj (a table or index) for corruption. +** If parameter zPrev is not NULL, then it must be a string containing the +** vector key required to restart the check where it left off last time. +** If pnKeyVal is not NULL, then (*pnKeyVal) is set to the number of +** columns in the vector key value for the specified object. +** +** This function uses the sqlite3_intck error code convention. +*/ +static char *intckCheckObjectSql( + sqlite3_intck *p, /* Integrity check object */ + const char *zObj, /* Object (table or index) to scan */ + const char *zPrev, /* Restart key vector, if any */ + int *pnKeyVal /* OUT: Number of key-values for this scan */ +){ + char *zRet = 0; + sqlite3_stmt *pStmt = 0; + int bAutoIndex = 0; + int bIsIndex = 0; + + const char *zCommon = + /* Relation without_rowid also contains just one row. Column "b" is + ** set to true if the table being examined is a WITHOUT ROWID table, + ** or false otherwise. */ + ", without_rowid(b) AS (" + " SELECT EXISTS (" + " SELECT 1 FROM tabname, pragma_index_list(tab, db) AS l" + " WHERE origin='pk' " + " AND NOT EXISTS (SELECT 1 FROM sqlite_schema WHERE name=l.name)" + " )" + ")" + "" + /* Table idx_cols contains 1 row for each column in each index on the + ** table being checked. Columns are: + ** + ** idx_name: Name of the index. + ** idx_ispk: True if this index is the PK of a WITHOUT ROWID table. + ** col_name: Name of indexed column, or NULL for index on expression. + ** col_expr: Indexed expression, including COLLATE clause. + ** col_alias: Alias used for column in 'intck_wrapper' table. + */ + ", idx_cols(idx_name, idx_ispk, col_name, col_expr, col_alias) AS (" + " SELECT l.name, (l.origin=='pk' AND w.b), i.name, COALESCE((" + " SELECT parse_create_index(sql, i.seqno) FROM " + " sqlite_schema WHERE name = l.name" + " ), format('\"%w\"', i.name) || ' COLLATE ' || quote(i.coll))," + " 'c' || row_number() OVER ()" + " FROM " + " tabname t," + " without_rowid w," + " pragma_index_list(t.tab, t.db) l," + " pragma_index_xinfo(l.name) i" + " WHERE i.key" + " UNION ALL" + " SELECT '', 1, '_rowid_', '_rowid_', 'r1' FROM without_rowid WHERE b=0" + ")" + "" + "" + /* + ** For a PK declared as "PRIMARY KEY(a, b) ... WITHOUT ROWID", where + ** the intck_wrapper aliases of "a" and "b" are "c1" and "c2": + ** + ** o_pk: "o.c1, o.c2" + ** i_pk: "i.'a', i.'b'" + ** ... + ** n_pk: 2 + */ + ", tabpk(db, tab, idx, o_pk, i_pk, q_pk, eq_pk, ps_pk, pk_pk, n_pk) AS (" + " WITH pkfields(f, a) AS (" + " SELECT i.col_name, i.col_alias FROM idx_cols i WHERE i.idx_ispk" + " )" + " SELECT t.db, t.tab, t.idx, " + " group_concat(a, ', '), " + " group_concat('i.'||quote(f), ', '), " + " group_concat('quote(o.'||a||')', ' || '','' || '), " + " format('(%s)==(%s)'," + " group_concat('o.'||a, ', '), " + " group_concat(format('\"%w\"', f), ', ')" + " )," + " group_concat('%s', ',')," + " group_concat('quote('||a||')', ', '), " + " count(*)" + " FROM tabname t, pkfields" + ")" + "" + ", idx(name, match_expr, partial, partial_alias, idx_ps, idx_idx) AS (" + " SELECT idx_name," + " format('(%s,%s) IS (%s,%s)', " + " group_concat(i.col_expr, ', '), i_pk," + " group_concat('o.'||i.col_alias, ', '), o_pk" + " ), " + " parse_create_index(" + " (SELECT sql FROM sqlite_schema WHERE name=idx_name), -1" + " )," + " 'cond' || row_number() OVER ()" + " , group_concat('%s', ',')" + " , group_concat('quote('||i.col_alias||')', ', ')" + " FROM tabpk t, " + " without_rowid w," + " idx_cols i" + " WHERE i.idx_ispk==0 " + " GROUP BY idx_name" + ")" + "" + ", wrapper_with(s) AS (" + " SELECT 'intck_wrapper AS (\n SELECT\n ' || (" + " WITH f(a, b) AS (" + " SELECT col_expr, col_alias FROM idx_cols" + " UNION ALL " + " SELECT partial, partial_alias FROM idx WHERE partial IS NOT NULL" + " )" + " SELECT group_concat(format('%s AS %s', a, b), ',\n ') FROM f" + " )" + " || format('\n FROM %Q.%Q ', t.db, t.tab)" + /* If the object being checked is a table, append "NOT INDEXED". + ** Otherwise, append "INDEXED BY ", and then, if the index + ** is a partial index " WHERE ". */ + " || CASE WHEN t.idx IS NULL THEN " + " 'NOT INDEXED'" + " ELSE" + " format('INDEXED BY %Q%s', t.idx, ' WHERE '||i.partial)" + " END" + " || '\n)'" + " FROM tabname t LEFT JOIN idx i ON (i.name=t.idx)" + ")" + "" + ; + + bAutoIndex = intckGetAutoIndex(p); + if( bAutoIndex ) intckExec(p, "PRAGMA automatic_index = 0"); + + bIsIndex = intckIsIndex(p, zObj); + if( bIsIndex ){ + pStmt = intckPrepareFmt(p, + /* Table idxname contains a single row. The first column, "db", contains + ** the name of the db containing the table (e.g. "main") and the second, + ** "tab", the name of the table itself. */ + "WITH tabname(db, tab, idx) AS (" + " SELECT %Q, (SELECT tbl_name FROM %Q.sqlite_schema WHERE name=%Q), %Q " + ")" + "" + ", whereclause(w_c) AS (%s)" + "" + "%s" /* zCommon */ + "" + ", case_statement(c) AS (" + " SELECT " + " 'CASE WHEN (' || group_concat(col_alias, ', ') || ', 1) IS (\n' " + " || ' SELECT ' || group_concat(col_expr, ', ') || ', 1 FROM '" + " || format('%%Q.%%Q NOT INDEXED WHERE %%s\n', t.db, t.tab, p.eq_pk)" + " || ' )\n THEN NULL\n '" + " || 'ELSE format(''surplus entry ('" + " || group_concat('%%s', ',') || ',' || p.ps_pk" + " || ') in index ' || t.idx || ''', ' " + " || group_concat('quote('||i.col_alias||')', ', ') || ', ' || p.pk_pk" + " || ')'" + " || '\n END AS error_message'" + " FROM tabname t, tabpk p, idx_cols i WHERE i.idx_name=t.idx" + ")" + "" + ", thiskey(k, n) AS (" + " SELECT group_concat(i.col_alias, ', ') || ', ' || p.o_pk, " + " count(*) + p.n_pk " + " FROM tabpk p, idx_cols i WHERE i.idx_name=p.idx" + ")" + "" + ", main_select(m, n) AS (" + " SELECT format(" + " 'WITH %%s\n' ||" + " ', idx_checker AS (\n' ||" + " ' SELECT %%s,\n' ||" + " ' %%s\n' || " + " ' FROM intck_wrapper AS o\n' ||" + " ')\n'," + " ww.s, c, t.k" + " ), t.n" + " FROM case_statement, wrapper_with ww, thiskey t" + ")" + + "SELECT m || " + " group_concat('SELECT * FROM idx_checker ' || w_c, ' UNION ALL '), n" + " FROM " + "main_select, whereclause " + , p->zDb, p->zDb, zObj, zObj + , zPrev ? zPrev : "VALUES('')", zCommon + ); + }else{ + pStmt = intckPrepareFmt(p, + /* Table tabname contains a single row. The first column, "db", contains + ** the name of the db containing the table (e.g. "main") and the second, + ** "tab", the name of the table itself. */ + "WITH tabname(db, tab, idx, prev) AS (SELECT %Q, %Q, NULL, %Q)" + "" + "%s" /* zCommon */ + + /* expr(e) contains one row for each index on table zObj. Value e + ** is set to an expression that evaluates to NULL if the required + ** entry is present in the index, or an error message otherwise. */ + ", expr(e, p) AS (" + " SELECT format('CASE WHEN EXISTS \n" + " (SELECT 1 FROM %%Q.%%Q AS i INDEXED BY %%Q WHERE %%s%%s)\n" + " THEN NULL\n" + " ELSE format(''entry (%%s,%%s) missing from index %%s'', %%s, %%s)\n" + " END\n'" + " , t.db, t.tab, i.name, i.match_expr, ' AND (' || partial || ')'," + " i.idx_ps, t.ps_pk, i.name, i.idx_idx, t.pk_pk)," + " CASE WHEN partial IS NULL THEN NULL ELSE i.partial_alias END" + " FROM tabpk t, idx i" + ")" + + ", numbered(ii, cond, e) AS (" + " SELECT 0, 'n.ii=0', 'NULL'" + " UNION ALL " + " SELECT row_number() OVER ()," + " '(n.ii='||row_number() OVER ()||COALESCE(' AND '||p||')', ')'), e" + " FROM expr" + ")" + + ", counter_with(w) AS (" + " SELECT 'WITH intck_counter(ii) AS (\n ' || " + " group_concat('SELECT '||ii, ' UNION ALL\n ') " + " || '\n)' FROM numbered" + ")" + "" + ", case_statement(c) AS (" + " SELECT 'CASE ' || " + " group_concat(format('\n WHEN %%s THEN (%%s)', cond, e), '') ||" + " '\nEND AS error_message'" + " FROM numbered" + ")" + "" + + /* This table contains a single row consisting of a single value - + ** the text of an SQL expression that may be used by the main SQL + ** statement to output an SQL literal that can be used to resume + ** the scan if it is suspended. e.g. for a rowid table, an expression + ** like: + ** + ** format('(%d,%d)', _rowid_, n.ii) + */ + ", thiskey(k, n) AS (" + " SELECT o_pk || ', ii', n_pk+1 FROM tabpk" + ")" + "" + ", whereclause(w_c) AS (" + " SELECT CASE WHEN prev!='' THEN " + " '\nWHERE (' || o_pk ||', n.ii) > ' || prev" + " ELSE ''" + " END" + " FROM tabpk, tabname" + ")" + "" + ", main_select(m, n) AS (" + " SELECT format(" + " '%%s, %%s\nSELECT %%s,\n%%s\nFROM intck_wrapper AS o" + ", intck_counter AS n%%s\nORDER BY %%s', " + " w, ww.s, c, thiskey.k, whereclause.w_c, t.o_pk" + " ), thiskey.n" + " FROM case_statement, tabpk t, counter_with, " + " wrapper_with ww, thiskey, whereclause" + ")" + + "SELECT m, n FROM main_select", + p->zDb, zObj, zPrev, zCommon + ); + } + + while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + zRet = intckMprintf(p, "%s", (const char*)sqlite3_column_text(pStmt, 0)); + if( pnKeyVal ){ + *pnKeyVal = sqlite3_column_int(pStmt, 1); + } + } + intckFinalize(p, pStmt); + + if( bAutoIndex ) intckExec(p, "PRAGMA automatic_index = 1"); + return zRet; +} + +/* +** Open a new integrity-check object. +*/ +int sqlite3_intck_open( + sqlite3 *db, /* Database handle to operate on */ + const char *zDbArg, /* "main", "temp" etc. */ + sqlite3_intck **ppOut /* OUT: New integrity-check handle */ +){ + sqlite3_intck *pNew = 0; + int rc = SQLITE_OK; + const char *zDb = zDbArg ? zDbArg : "main"; + int nDb = (int)strlen(zDb); + + pNew = (sqlite3_intck*)sqlite3_malloc(sizeof(*pNew) + nDb + 1); + if( pNew==0 ){ + rc = SQLITE_NOMEM; + }else{ + memset(pNew, 0, sizeof(*pNew)); + pNew->db = db; + pNew->zDb = (const char*)&pNew[1]; + memcpy(&pNew[1], zDb, nDb+1); + rc = sqlite3_create_function(db, "parse_create_index", + 2, SQLITE_UTF8, 0, intckParseCreateIndexFunc, 0, 0 + ); + if( rc!=SQLITE_OK ){ + sqlite3_intck_close(pNew); + pNew = 0; + } + } + + *ppOut = pNew; + return rc; +} + +/* +** Free the integrity-check object. +*/ +void sqlite3_intck_close(sqlite3_intck *p){ + if( p ){ + sqlite3_finalize(p->pCheck); + sqlite3_create_function( + p->db, "parse_create_index", 1, SQLITE_UTF8, 0, 0, 0, 0 + ); + sqlite3_free(p->zObj); + sqlite3_free(p->zKey); + sqlite3_free(p->zTestSql); + sqlite3_free(p->zErr); + sqlite3_free(p->zMessage); + sqlite3_free(p); + } +} + +/* +** Step the integrity-check object. +*/ +int sqlite3_intck_step(sqlite3_intck *p){ + if( p->rc==SQLITE_OK ){ + + if( p->zMessage ){ + sqlite3_free(p->zMessage); + p->zMessage = 0; + } + + if( p->bCorruptSchema ){ + p->rc = SQLITE_DONE; + }else + if( p->pCheck==0 ){ + intckFindObject(p); + if( p->rc==SQLITE_OK ){ + if( p->zObj ){ + char *zSql = 0; + zSql = intckCheckObjectSql(p, p->zObj, p->zKey, &p->nKeyVal); + p->pCheck = intckPrepare(p, zSql); + sqlite3_free(zSql); + sqlite3_free(p->zKey); + p->zKey = 0; + }else{ + p->rc = SQLITE_DONE; + } + }else if( p->rc==SQLITE_CORRUPT ){ + p->rc = SQLITE_OK; + p->zMessage = intckMprintf(p, "%s", + "corruption found while reading database schema" + ); + p->bCorruptSchema = 1; + } + } + + if( p->pCheck ){ + assert( p->rc==SQLITE_OK ); + if( sqlite3_step(p->pCheck)==SQLITE_ROW ){ + /* Normal case, do nothing. */ + }else{ + intckFinalize(p, p->pCheck); + p->pCheck = 0; + p->nKeyVal = 0; + if( p->rc==SQLITE_CORRUPT ){ + p->rc = SQLITE_OK; + p->zMessage = intckMprintf(p, + "corruption found while scanning database object %s", p->zObj + ); + } + } + } + } + + return p->rc; +} + +/* +** Return a message describing the corruption encountered by the most recent +** call to sqlite3_intck_step(), or NULL if no corruption was encountered. +*/ +const char *sqlite3_intck_message(sqlite3_intck *p){ + assert( p->pCheck==0 || p->zMessage==0 ); + if( p->zMessage ){ + return p->zMessage; + } + if( p->pCheck ){ + return (const char*)sqlite3_column_text(p->pCheck, 0); + } + return 0; +} + +/* +** Return the error code and message. +*/ +int sqlite3_intck_error(sqlite3_intck *p, const char **pzErr){ + if( pzErr ) *pzErr = p->zErr; + return (p->rc==SQLITE_DONE ? SQLITE_OK : p->rc); +} + +/* +** Close any read transaction the integrity-check object is holding open +** on the database. +*/ +int sqlite3_intck_unlock(sqlite3_intck *p){ + if( p->rc==SQLITE_OK && p->pCheck ){ + assert( p->zKey==0 && p->nKeyVal>0 ); + intckSaveKey(p); + intckFinalize(p, p->pCheck); + p->pCheck = 0; + } + return p->rc; +} + +/* +** Return the SQL statement used to check object zObj. Or, if zObj is +** NULL, the current SQL statement. +*/ +const char *sqlite3_intck_test_sql(sqlite3_intck *p, const char *zObj){ + sqlite3_free(p->zTestSql); + if( zObj ){ + p->zTestSql = intckCheckObjectSql(p, zObj, 0, 0); + }else{ + if( p->zObj ){ + p->zTestSql = intckCheckObjectSql(p, p->zObj, p->zKey, 0); + }else{ + sqlite3_free(p->zTestSql); + p->zTestSql = 0; + } + } + return p->zTestSql; +} + +/************************* End ../ext/intck/sqlite3intck.c ********************/ + #if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_DBPAGE_VTAB) #define SQLITE_SHELL_HAVE_RECOVER 1 #else @@ -14349,6 +15541,15 @@ int sqlite3_recover_finish(sqlite3_recover*); typedef struct DbdataTable DbdataTable; typedef struct DbdataCursor DbdataCursor; +typedef struct DbdataBuffer DbdataBuffer; + +/* +** Buffer type. +*/ +struct DbdataBuffer { + u8 *aBuf; + sqlite3_int64 nBuf; +}; /* Cursor object */ struct DbdataCursor { @@ -14365,7 +15566,7 @@ struct DbdataCursor { sqlite3_int64 iRowid; /* Only for the sqlite_dbdata table */ - u8 *pRec; /* Buffer containing current record */ + DbdataBuffer rec; sqlite3_int64 nRec; /* Size of pRec[] in bytes */ sqlite3_int64 nHdr; /* Size of header in bytes */ int iField; /* Current field number */ @@ -14410,6 +15611,31 @@ struct DbdataTable { " schema TEXT HIDDEN" \ ")" +/* +** Ensure the buffer passed as the first argument is at least nMin bytes +** in size. If an error occurs while attempting to resize the buffer, +** SQLITE_NOMEM is returned. Otherwise, SQLITE_OK. +*/ +static int dbdataBufferSize(DbdataBuffer *pBuf, sqlite3_int64 nMin){ + if( nMin>pBuf->nBuf ){ + sqlite3_int64 nNew = nMin+16384; + u8 *aNew = (u8*)sqlite3_realloc64(pBuf->aBuf, nNew); + + if( aNew==0 ) return SQLITE_NOMEM; + pBuf->aBuf = aNew; + pBuf->nBuf = nNew; + } + return SQLITE_OK; +} + +/* +** Release the allocation managed by buffer pBuf. +*/ +static void dbdataBufferFree(DbdataBuffer *pBuf){ + sqlite3_free(pBuf->aBuf); + memset(pBuf, 0, sizeof(*pBuf)); +} + /* ** Connect to an sqlite_dbdata (pAux==0) or sqlite_dbptr (pAux!=0) virtual ** table. @@ -14550,9 +15776,9 @@ static void dbdataResetCursor(DbdataCursor *pCsr){ pCsr->iField = 0; pCsr->bOnePage = 0; sqlite3_free(pCsr->aPage); - sqlite3_free(pCsr->pRec); - pCsr->pRec = 0; + dbdataBufferFree(&pCsr->rec); pCsr->aPage = 0; + pCsr->nRec = 0; } /* @@ -14694,62 +15920,74 @@ static void dbdataValue( u8 *pData, sqlite3_int64 nData ){ - if( eType>=0 && dbdataValueBytes(eType)<=nData ){ - switch( eType ){ - case 0: - case 10: - case 11: - sqlite3_result_null(pCtx); - break; - - case 8: - sqlite3_result_int(pCtx, 0); - break; - case 9: - sqlite3_result_int(pCtx, 1); - break; - - case 1: case 2: case 3: case 4: case 5: case 6: case 7: { - sqlite3_uint64 v = (signed char)pData[0]; - pData++; - switch( eType ){ - case 7: - case 6: v = (v<<16) + (pData[0]<<8) + pData[1]; pData += 2; - case 5: v = (v<<16) + (pData[0]<<8) + pData[1]; pData += 2; - case 4: v = (v<<8) + pData[0]; pData++; - case 3: v = (v<<8) + pData[0]; pData++; - case 2: v = (v<<8) + pData[0]; pData++; - } - - if( eType==7 ){ - double r; - memcpy(&r, &v, sizeof(r)); - sqlite3_result_double(pCtx, r); - }else{ - sqlite3_result_int64(pCtx, (sqlite3_int64)v); - } - break; - } - - default: { - int n = ((eType-12) / 2); - if( eType % 2 ){ - switch( enc ){ -#ifndef SQLITE_OMIT_UTF16 - case SQLITE_UTF16BE: - sqlite3_result_text16be(pCtx, (void*)pData, n, SQLITE_TRANSIENT); - break; - case SQLITE_UTF16LE: - sqlite3_result_text16le(pCtx, (void*)pData, n, SQLITE_TRANSIENT); - break; -#endif - default: - sqlite3_result_text(pCtx, (char*)pData, n, SQLITE_TRANSIENT); - break; + if( eType>=0 ){ + if( dbdataValueBytes(eType)<=nData ){ + switch( eType ){ + case 0: + case 10: + case 11: + sqlite3_result_null(pCtx); + break; + + case 8: + sqlite3_result_int(pCtx, 0); + break; + case 9: + sqlite3_result_int(pCtx, 1); + break; + + case 1: case 2: case 3: case 4: case 5: case 6: case 7: { + sqlite3_uint64 v = (signed char)pData[0]; + pData++; + switch( eType ){ + case 7: + case 6: v = (v<<16) + (pData[0]<<8) + pData[1]; pData += 2; + case 5: v = (v<<16) + (pData[0]<<8) + pData[1]; pData += 2; + case 4: v = (v<<8) + pData[0]; pData++; + case 3: v = (v<<8) + pData[0]; pData++; + case 2: v = (v<<8) + pData[0]; pData++; } - }else{ - sqlite3_result_blob(pCtx, pData, n, SQLITE_TRANSIENT); + + if( eType==7 ){ + double r; + memcpy(&r, &v, sizeof(r)); + sqlite3_result_double(pCtx, r); + }else{ + sqlite3_result_int64(pCtx, (sqlite3_int64)v); + } + break; } + + default: { + int n = ((eType-12) / 2); + if( eType % 2 ){ + switch( enc ){ + #ifndef SQLITE_OMIT_UTF16 + case SQLITE_UTF16BE: + sqlite3_result_text16be(pCtx, (void*)pData, n, SQLITE_TRANSIENT); + break; + case SQLITE_UTF16LE: + sqlite3_result_text16le(pCtx, (void*)pData, n, SQLITE_TRANSIENT); + break; + #endif + default: + sqlite3_result_text(pCtx, (char*)pData, n, SQLITE_TRANSIENT); + break; + } + }else{ + sqlite3_result_blob(pCtx, pData, n, SQLITE_TRANSIENT); + } + } + } + }else{ + if( eType==7 ){ + sqlite3_result_double(pCtx, 0.0); + }else if( eType<7 ){ + sqlite3_result_int(pCtx, 0); + }else if( eType%2 ){ + sqlite3_result_text(pCtx, "", 0, SQLITE_STATIC); + }else{ + sqlite3_result_blob(pCtx, "", 0, SQLITE_STATIC); } } } @@ -14812,7 +16050,8 @@ static int dbdataNext(sqlite3_vtab_cursor *pCursor){ } }else{ /* If there is no record loaded, load it now. */ - if( pCsr->pRec==0 ){ + assert( pCsr->rec.aBuf!=0 || pCsr->nRec==0 ); + if( pCsr->nRec==0 ){ int bHasRowid = 0; int nPointer = 0; sqlite3_int64 nPayload = 0; @@ -14856,6 +16095,7 @@ static int dbdataNext(sqlite3_vtab_cursor *pCursor){ }else{ iOff += dbdataGetVarintU32(&pCsr->aPage[iOff], &nPayload); if( nPayload>0x7fffff00 ) nPayload &= 0x3fff; + if( nPayload==0 ) nPayload = 1; } /* If this is a leaf intkey cell, load the rowid */ @@ -14890,13 +16130,12 @@ static int dbdataNext(sqlite3_vtab_cursor *pCursor){ /* Allocate space for payload. And a bit more to catch small buffer ** overruns caused by attempting to read a varint or similar from ** near the end of a corrupt record. */ - pCsr->pRec = (u8*)sqlite3_malloc64(nPayload+DBDATA_PADDING_BYTES); - if( pCsr->pRec==0 ) return SQLITE_NOMEM; - memset(pCsr->pRec, 0, nPayload+DBDATA_PADDING_BYTES); - pCsr->nRec = nPayload; + rc = dbdataBufferSize(&pCsr->rec, nPayload+DBDATA_PADDING_BYTES); + if( rc!=SQLITE_OK ) return rc; + assert( nPayload!=0 ); /* Load the nLocal bytes of payload */ - memcpy(pCsr->pRec, &pCsr->aPage[iOff], nLocal); + memcpy(pCsr->rec.aBuf, &pCsr->aPage[iOff], nLocal); iOff += nLocal; /* Load content from overflow pages */ @@ -14914,19 +16153,22 @@ static int dbdataNext(sqlite3_vtab_cursor *pCursor){ nCopy = U-4; if( nCopy>nRem ) nCopy = nRem; - memcpy(&pCsr->pRec[nPayload-nRem], &aOvfl[4], nCopy); + memcpy(&pCsr->rec.aBuf[nPayload-nRem], &aOvfl[4], nCopy); nRem -= nCopy; pgnoOvfl = get_uint32(aOvfl); sqlite3_free(aOvfl); } + nPayload -= nRem; } + memset(&pCsr->rec.aBuf[nPayload], 0, DBDATA_PADDING_BYTES); + pCsr->nRec = nPayload; - iHdr = dbdataGetVarintU32(pCsr->pRec, &nHdr); + iHdr = dbdataGetVarintU32(pCsr->rec.aBuf, &nHdr); if( nHdr>nPayload ) nHdr = 0; pCsr->nHdr = nHdr; - pCsr->pHdrPtr = &pCsr->pRec[iHdr]; - pCsr->pPtr = &pCsr->pRec[pCsr->nHdr]; + pCsr->pHdrPtr = &pCsr->rec.aBuf[iHdr]; + pCsr->pPtr = &pCsr->rec.aBuf[pCsr->nHdr]; pCsr->iField = (bHasRowid ? -1 : 0); } } @@ -14934,7 +16176,7 @@ static int dbdataNext(sqlite3_vtab_cursor *pCursor){ pCsr->iField++; if( pCsr->iField>0 ){ sqlite3_int64 iType; - if( pCsr->pHdrPtr>=&pCsr->pRec[pCsr->nRec] + if( pCsr->pHdrPtr>=&pCsr->rec.aBuf[pCsr->nRec] || pCsr->iField>=DBDATA_MX_FIELD ){ bNextPage = 1; @@ -14942,8 +16184,8 @@ static int dbdataNext(sqlite3_vtab_cursor *pCursor){ int szField = 0; pCsr->pHdrPtr += dbdataGetVarintU32(pCsr->pHdrPtr, &iType); szField = dbdataValueBytes(iType); - if( (pCsr->nRec - (pCsr->pPtr - pCsr->pRec))pPtr = &pCsr->pRec[pCsr->nRec]; + if( (pCsr->nRec - (pCsr->pPtr - pCsr->rec.aBuf))pPtr = &pCsr->rec.aBuf[pCsr->nRec]; }else{ pCsr->pPtr += szField; } @@ -14953,20 +16195,18 @@ static int dbdataNext(sqlite3_vtab_cursor *pCursor){ if( bNextPage ){ sqlite3_free(pCsr->aPage); - sqlite3_free(pCsr->pRec); pCsr->aPage = 0; - pCsr->pRec = 0; + pCsr->nRec = 0; if( pCsr->bOnePage ) return SQLITE_OK; pCsr->iPgno++; }else{ - if( pCsr->iField<0 || pCsr->pHdrPtr<&pCsr->pRec[pCsr->nHdr] ){ + if( pCsr->iField<0 || pCsr->pHdrPtr<&pCsr->rec.aBuf[pCsr->nHdr] ){ return SQLITE_OK; } /* Advance to the next cell. The next iteration of the loop will load ** the record and so on. */ - sqlite3_free(pCsr->pRec); - pCsr->pRec = 0; + pCsr->nRec = 0; pCsr->iCell++; } } @@ -15156,12 +16396,12 @@ static int dbdataColumn( case DBDATA_COLUMN_VALUE: { if( pCsr->iField<0 ){ sqlite3_result_int64(ctx, pCsr->iIntkey); - }else if( &pCsr->pRec[pCsr->nRec] >= pCsr->pPtr ){ + }else if( &pCsr->rec.aBuf[pCsr->nRec] >= pCsr->pPtr ){ sqlite3_int64 iType; dbdataGetVarintU32(pCsr->pHdrPtr, &iType); dbdataValue( ctx, pCsr->enc, iType, pCsr->pPtr, - &pCsr->pRec[pCsr->nRec] - pCsr->pPtr + &pCsr->rec.aBuf[pCsr->nRec] - pCsr->pPtr ); } break; @@ -21614,6 +22854,7 @@ static const char *(azHelp[]) = { ".indexes ?TABLE? Show names of indexes", " If TABLE is specified, only show indexes for", " tables matching TABLE using the LIKE operator.", + ".intck ?STEPS_PER_UNLOCK? Run an incremental integrity check on the db", #ifdef SQLITE_ENABLE_IOTRACE ",iotrace FILE Enable I/O diagnostic logging to FILE", #endif @@ -24523,6 +25764,40 @@ static int recoverDatabaseCmd(ShellState *pState, int nArg, char **azArg){ } #endif /* SQLITE_SHELL_HAVE_RECOVER */ +/* +** Implementation of ".intck STEPS_PER_UNLOCK" command. +*/ +static int intckDatabaseCmd(ShellState *pState, i64 nStepPerUnlock){ + sqlite3_intck *p = 0; + int rc = SQLITE_OK; + + rc = sqlite3_intck_open(pState->db, "main", &p); + if( rc==SQLITE_OK ){ + i64 nStep = 0; + i64 nError = 0; + const char *zErr = 0; + while( SQLITE_OK==sqlite3_intck_step(p) ){ + const char *zMsg = sqlite3_intck_message(p); + if( zMsg ){ + oputf("%s\n", zMsg); + nError++; + } + nStep++; + if( nStepPerUnlock && (nStep % nStepPerUnlock)==0 ){ + sqlite3_intck_unlock(p); + } + } + rc = sqlite3_intck_error(p, &zErr); + if( zErr ){ + eputf("%s\n", zErr); + } + sqlite3_intck_close(p); + + oputf("%lld steps, %lld errors\n", nStep, nError); + } + + return rc; +} /* * zAutoColumn(zCol, &db, ?) => Maybe init db, add column zCol to it. @@ -24766,6 +26041,45 @@ static int outputDumpWarning(ShellState *p, const char *zLike){ return rc; } +/* +** Fault-Simulator state and logic. +*/ +static struct { + int iId; /* ID that triggers a simulated fault. -1 means "any" */ + int iErr; /* The error code to return on a fault */ + int iCnt; /* Trigger the fault only if iCnt is already zero */ + int iInterval; /* Reset iCnt to this value after each fault */ + int eVerbose; /* When to print output */ + int nHit; /* Number of hits seen so far */ + int nRepeat; /* Turn off after this many hits. 0 for never */ + int nSkip; /* Skip this many before first fault */ +} faultsim_state = {-1, 0, 0, 0, 0, 0, 0, 0}; + +/* +** This is the fault-sim callback +*/ +static int faultsim_callback(int iArg){ + if( faultsim_state.iId>0 && faultsim_state.iId!=iArg ){ + return SQLITE_OK; + } + if( faultsim_state.iCnt ){ + if( faultsim_state.iCnt>0 ) faultsim_state.iCnt--; + if( faultsim_state.eVerbose>=2 ){ + oputf("FAULT-SIM id=%d no-fault (cnt=%d)\n", iArg, faultsim_state.iCnt); + } + return SQLITE_OK; + } + if( faultsim_state.eVerbose>=1 ){ + oputf("FAULT-SIM id=%d returns %d\n", iArg, faultsim_state.iErr); + } + faultsim_state.iCnt = faultsim_state.iInterval; + faultsim_state.nHit++; + if( faultsim_state.nRepeat>0 && faultsim_state.nRepeat<=faultsim_state.nHit ){ + faultsim_state.iCnt = -1; + } + return faultsim_state.iErr; +} + /* ** If an input line begins with "." then invoke this routine to ** process that line. @@ -25257,7 +26571,8 @@ static int do_meta_command(char *zLine, ShellState *p){ zSql = sqlite3_mprintf( "SELECT sql FROM sqlite_schema AS o " "WHERE (%s) AND sql NOT NULL" - " AND type IN ('index','trigger','view')", + " AND type IN ('index','trigger','view') " + "ORDER BY type COLLATE NOCASE DESC", zLike ); run_table_dump_query(p, zSql); @@ -25992,6 +27307,21 @@ static int do_meta_command(char *zLine, ShellState *p){ }else #endif /* !defined(SQLITE_OMIT_TEST_CONTROL) */ + if( c=='i' && cli_strncmp(azArg[0], "intck", n)==0 ){ + i64 iArg = 0; + if( nArg==2 ){ + iArg = integerValue(azArg[1]); + if( iArg==0 ) iArg = -1; + } + if( (nArg!=1 && nArg!=2) || iArg<0 ){ + eputf("%s","Usage: .intck STEPS_PER_UNLOCK\n"); + rc = 1; + goto meta_command_exit; + } + open_db(p, 0); + rc = intckDatabaseCmd(p, iArg); + }else + #ifdef SQLITE_ENABLE_IOTRACE if( c=='i' && cli_strncmp(azArg[0], "iotrace", n)==0 ){ SQLITE_API extern void (SQLITE_CDECL *sqlite3IoTrace)(const char*, ...); @@ -27665,7 +28995,7 @@ static int do_meta_command(char *zLine, ShellState *p){ /*{"bitvec_test", SQLITE_TESTCTRL_BITVEC_TEST, 1, "" },*/ {"byteorder", SQLITE_TESTCTRL_BYTEORDER, 0, "" }, {"extra_schema_checks",SQLITE_TESTCTRL_EXTRA_SCHEMA_CHECKS,0,"BOOLEAN" }, - /*{"fault_install", SQLITE_TESTCTRL_FAULT_INSTALL, 1,"" },*/ + {"fault_install", SQLITE_TESTCTRL_FAULT_INSTALL, 1,"args..." }, {"fk_no_action", SQLITE_TESTCTRL_FK_NO_ACTION, 0, "BOOLEAN" }, {"imposter", SQLITE_TESTCTRL_IMPOSTER,1,"SCHEMA ON/OFF ROOTPAGE"}, {"internal_functions", SQLITE_TESTCTRL_INTERNAL_FUNCTIONS,0,"" }, @@ -27898,6 +29228,76 @@ static int do_meta_command(char *zLine, ShellState *p){ } sqlite3_test_control(testctrl, &rc2); break; + case SQLITE_TESTCTRL_FAULT_INSTALL: { + int kk; + int bShowHelp = nArg<=2; + isOk = 3; + for(kk=2; kk0 ) faultsim_state.eVerbose--; + }else if( cli_strcmp(z,"-id")==0 && kk+1=0 ){ @@ -28820,7 +30220,7 @@ static void usage(int showDetail){ }else{ eputz("Use the -help option for additional information\n"); } - exit(1); + exit(0); } /* @@ -29521,6 +30921,11 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ #ifndef SQLITE_SHELL_FIDDLE /* In WASM mode we have to leave the db state in place so that ** client code can "push" SQL into it after this call returns. */ +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( data.expert.pExpert ){ + expertFinish(&data, 1, 0); + } +#endif free(azCmd); set_table_name(&data, 0); if( data.db ){ @@ -29587,7 +30992,7 @@ sqlite3_vfs * fiddle_db_vfs(const char *zDbName){ /* Only for emcc experimentation purposes. */ sqlite3 * fiddle_db_arg(sqlite3 *arg){ - printf("fiddle_db_arg(%p)\n", (const void*)arg); + oputf("fiddle_db_arg(%p)\n", (const void*)arg); return arg; } @@ -29613,12 +31018,22 @@ const char * fiddle_db_filename(const char * zDbName){ /* ** Completely wipes out the contents of the currently-opened database -** but leaves its storage intact for reuse. +** but leaves its storage intact for reuse. If any transactions are +** active, they are forcibly rolled back. */ void fiddle_reset_db(void){ if( globalDb ){ - int rc = sqlite3_db_config(globalDb, SQLITE_DBCONFIG_RESET_DATABASE, 1, 0); - if( 0==rc ) rc = sqlite3_exec(globalDb, "VACUUM", 0, 0, 0); + int rc; + while( sqlite3_txn_state(globalDb,0)>0 ){ + /* + ** Resolve problem reported in + ** https://sqlite.org/forum/forumpost/0b41a25d65 + */ + oputz("Rolling back in-progress transaction.\n"); + sqlite3_exec(globalDb,"ROLLBACK", 0, 0, 0); + } + rc = sqlite3_db_config(globalDb, SQLITE_DBCONFIG_RESET_DATABASE, 1, 0); + if( 0==rc ) sqlite3_exec(globalDb, "VACUUM", 0, 0, 0); sqlite3_db_config(globalDb, SQLITE_DBCONFIG_RESET_DATABASE, 0, 0); } } diff --git a/src/database/sqlite3.c b/src/database/sqlite3.c index 0d22c717..4458f270 100644 --- a/src/database/sqlite3.c +++ b/src/database/sqlite3.c @@ -1,6 +1,6 @@ /****************************************************************************** ** This file is an amalgamation of many separate C source files from SQLite -** version 3.45.3. By combining all the individual C code files into this +** version 3.46.0. By combining all the individual C code files into this ** single large file, the entire code can be compiled as a single translation ** unit. This allows many compilers to do optimizations that would not be ** possible if the files were compiled separately. Performance improvements @@ -18,7 +18,7 @@ ** separate file. This file contains only code for the core SQLite library. ** ** The content in this amalgamation comes from Fossil check-in -** 8653b758870e6ef0c98d46b3ace27849054a. +** 96c92aba00c8375bc32fafcdf12429c58bd8. */ #define SQLITE_CORE 1 #define SQLITE_AMALGAMATION 1 @@ -459,9 +459,9 @@ extern "C" { ** [sqlite3_libversion_number()], [sqlite3_sourceid()], ** [sqlite_version()] and [sqlite_source_id()]. */ -#define SQLITE_VERSION "3.45.3" -#define SQLITE_VERSION_NUMBER 3045003 -#define SQLITE_SOURCE_ID "2024-04-15 13:34:05 8653b758870e6ef0c98d46b3ace27849054af85da891eb121e9aaa537f1e8355" +#define SQLITE_VERSION "3.46.0" +#define SQLITE_VERSION_NUMBER 3046000 +#define SQLITE_SOURCE_ID "2024-05-23 13:25:27 96c92aba00c8375bc32fafcdf12429c58bd8aabfcadab6683e35bbb9cdebf19e" /* ** CAPI3REF: Run-Time Library Version Numbers @@ -1077,11 +1077,11 @@ struct sqlite3_file { ** ** xLock() upgrades the database file lock. In other words, xLock() moves the ** database file lock in the direction NONE toward EXCLUSIVE. The argument to -** xLock() is always on of SHARED, RESERVED, PENDING, or EXCLUSIVE, never +** xLock() is always one of SHARED, RESERVED, PENDING, or EXCLUSIVE, never ** SQLITE_LOCK_NONE. If the database file lock is already at or above the ** requested lock, then the call to xLock() is a no-op. ** xUnlock() downgrades the database file lock to either SHARED or NONE. -* If the lock is already at or below the requested lock state, then the call +** If the lock is already at or below the requested lock state, then the call ** to xUnlock() is a no-op. ** The xCheckReservedLock() method checks whether any database connection, ** either in this process or in some other process, is holding a RESERVED, @@ -3618,8 +3618,8 @@ SQLITE_API int sqlite3_set_authorizer( #define SQLITE_RECURSIVE 33 /* NULL NULL */ /* -** CAPI3REF: Tracing And Profiling Functions -** METHOD: sqlite3 +** CAPI3REF: Deprecated Tracing And Profiling Functions +** DEPRECATED ** ** These routines are deprecated. Use the [sqlite3_trace_v2()] interface ** instead of the routines described here. @@ -7200,6 +7200,12 @@ SQLITE_API int sqlite3_autovacuum_pages( ** The exceptions defined in this paragraph might change in a future ** release of SQLite. ** +** Whether the update hook is invoked before or after the +** corresponding change is currently unspecified and may differ +** depending on the type of change. Do not rely on the order of the +** hook call with regards to the final result of the operation which +** triggers the hook. +** ** The update hook implementation must not do anything that will modify ** the database connection that invoked the update hook. Any actions ** to modify the database connection must be deferred until after the @@ -8670,7 +8676,7 @@ SQLITE_API int sqlite3_test_control(int op, ...); ** The sqlite3_keyword_count() interface returns the number of distinct ** keywords understood by SQLite. ** -** The sqlite3_keyword_name(N,Z,L) interface finds the N-th keyword and +** The sqlite3_keyword_name(N,Z,L) interface finds the 0-based N-th keyword and ** makes *Z point to that keyword expressed as UTF8 and writes the number ** of bytes in the keyword into *L. The string that *Z points to is not ** zero-terminated. The sqlite3_keyword_name(N,Z,L) routine returns @@ -10249,24 +10255,45 @@ SQLITE_API const char *sqlite3_vtab_collation(sqlite3_index_info*,int); **

  • ** ^(If the sqlite3_vtab_distinct() interface returns 2, that means ** that the query planner does not need the rows returned in any particular -** order, as long as rows with the same values in all "aOrderBy" columns -** are adjacent.)^ ^(Furthermore, only a single row for each particular -** combination of values in the columns identified by the "aOrderBy" field -** needs to be returned.)^ ^It is always ok for two or more rows with the same -** values in all "aOrderBy" columns to be returned, as long as all such rows -** are adjacent. ^The virtual table may, if it chooses, omit extra rows -** that have the same value for all columns identified by "aOrderBy". -** ^However omitting the extra rows is optional. +** order, as long as rows with the same values in all columns identified +** by "aOrderBy" are adjacent.)^ ^(Furthermore, when two or more rows +** contain the same values for all columns identified by "colUsed", all but +** one such row may optionally be omitted from the result.)^ +** The virtual table is not required to omit rows that are duplicates +** over the "colUsed" columns, but if the virtual table can do that without +** too much extra effort, it could potentially help the query to run faster. ** This mode is used for a DISTINCT query. **

  • -** ^(If the sqlite3_vtab_distinct() interface returns 3, that means -** that the query planner needs only distinct rows but it does need the -** rows to be sorted.)^ ^The virtual table implementation is free to omit -** rows that are identical in all aOrderBy columns, if it wants to, but -** it is not required to omit any rows. This mode is used for queries +** ^(If the sqlite3_vtab_distinct() interface returns 3, that means the +** virtual table must return rows in the order defined by "aOrderBy" as +** if the sqlite3_vtab_distinct() interface had returned 0. However if +** two or more rows in the result have the same values for all columns +** identified by "colUsed", then all but one such row may optionally be +** omitted.)^ Like when the return value is 2, the virtual table +** is not required to omit rows that are duplicates over the "colUsed" +** columns, but if the virtual table can do that without +** too much extra effort, it could potentially help the query to run faster. +** This mode is used for queries ** that have both DISTINCT and ORDER BY clauses. ** ** +**

    The following table summarizes the conditions under which the +** virtual table is allowed to set the "orderByConsumed" flag based on +** the value returned by sqlite3_vtab_distinct(). This table is a +** restatement of the previous four paragraphs: +** +** +** +**
    sqlite3_vtab_distinct() return value +** Rows are returned in aOrderBy order +** Rows with the same value in all aOrderBy columns are adjacent +** Duplicates over all colUsed columns may be omitted +**
    0yesyesno +**
    1noyesno +**
    2noyesyes +**
    3yesyesyes +**
    +** ** ^For the purposes of comparing virtual table output values to see if the ** values are same value for sorting purposes, two NULL values are considered ** to be the same. In other words, the comparison operator is "IS" @@ -12311,6 +12338,30 @@ SQLITE_API int sqlite3changegroup_schema(sqlite3_changegroup*, sqlite3*, const c */ SQLITE_API int sqlite3changegroup_add(sqlite3_changegroup*, int nData, void *pData); +/* +** CAPI3REF: Add A Single Change To A Changegroup +** METHOD: sqlite3_changegroup +** +** This function adds the single change currently indicated by the iterator +** passed as the second argument to the changegroup object. The rules for +** adding the change are just as described for [sqlite3changegroup_add()]. +** +** If the change is successfully added to the changegroup, SQLITE_OK is +** returned. Otherwise, an SQLite error code is returned. +** +** The iterator must point to a valid entry when this function is called. +** If it does not, SQLITE_ERROR is returned and no change is added to the +** changegroup. Additionally, the iterator must not have been opened with +** the SQLITE_CHANGESETAPPLY_INVERT flag. In this case SQLITE_ERROR is also +** returned. +*/ +SQLITE_API int sqlite3changegroup_add_change( + sqlite3_changegroup*, + sqlite3_changeset_iter* +); + + + /* ** CAPI3REF: Obtain A Composite Changeset From A Changegroup ** METHOD: sqlite3_changegroup @@ -13115,8 +13166,8 @@ struct Fts5PhraseIter { ** EXTENSION API FUNCTIONS ** ** xUserData(pFts): -** Return a copy of the context pointer the extension function was -** registered with. +** Return a copy of the pUserData pointer passed to the xCreateFunction() +** API when the extension function was registered. ** ** xColumnTotalSize(pFts, iCol, pnToken): ** If parameter iCol is less than zero, set output variable *pnToken @@ -14314,6 +14365,8 @@ struct fts5_api { # define SQLITE_OMIT_ALTERTABLE #endif +#define SQLITE_DIGIT_SEPARATOR '_' + /* ** Return true (non-zero) if the input is an integer that is too large ** to fit in 32-bits. This macro is used inside of various testcase() @@ -14606,8 +14659,8 @@ SQLITE_PRIVATE void sqlite3HashClear(Hash*); #define TK_TRUEFALSE 170 #define TK_ISNOT 171 #define TK_FUNCTION 172 -#define TK_UMINUS 173 -#define TK_UPLUS 174 +#define TK_UPLUS 173 +#define TK_UMINUS 174 #define TK_TRUTH 175 #define TK_REGISTER 176 #define TK_VECTOR 177 @@ -14616,8 +14669,9 @@ SQLITE_PRIVATE void sqlite3HashClear(Hash*); #define TK_ASTERISK 180 #define TK_SPAN 181 #define TK_ERROR 182 -#define TK_SPACE 183 -#define TK_ILLEGAL 184 +#define TK_QNUMBER 183 +#define TK_SPACE 184 +#define TK_ILLEGAL 185 /************** End of parse.h ***********************************************/ /************** Continuing where we left off in sqliteInt.h ******************/ @@ -14879,7 +14933,7 @@ typedef INT16_TYPE LogEst; # define SQLITE_PTRSIZE __SIZEOF_POINTER__ # elif defined(i386) || defined(__i386__) || defined(_M_IX86) || \ defined(_M_ARM) || defined(__arm__) || defined(__x86) || \ - (defined(__APPLE__) && defined(__POWERPC__)) || \ + (defined(__APPLE__) && defined(__ppc__)) || \ (defined(__TOS_AIX__) && !defined(__64BIT__)) # define SQLITE_PTRSIZE 4 # else @@ -15147,7 +15201,7 @@ SQLITE_PRIVATE u32 sqlite3WhereTrace; ** 0x00000010 Display sqlite3_index_info xBestIndex calls ** 0x00000020 Range an equality scan metrics ** 0x00000040 IN operator decisions -** 0x00000080 WhereLoop cost adjustements +** 0x00000080 WhereLoop cost adjustments ** 0x00000100 ** 0x00000200 Covering index decisions ** 0x00000400 OR optimization @@ -16296,6 +16350,7 @@ SQLITE_PRIVATE int sqlite3BtreeIntegrityCheck( sqlite3 *db, /* Database connection that is running the check */ Btree *p, /* The btree to be checked */ Pgno *aRoot, /* An array of root pages numbers for individual trees */ + sqlite3_value *aCnt, /* OUT: entry counts for each btree in aRoot[] */ int nRoot, /* Number of entries in aRoot[] */ int mxErr, /* Stop reporting errors after this many */ int *pnErr, /* OUT: Write number of errors seen to this variable */ @@ -16566,12 +16621,12 @@ typedef struct VdbeOpList VdbeOpList; #define OP_Vacuum 5 #define OP_VFilter 6 /* jump, synopsis: iplan=r[P3] zplan='P4' */ #define OP_VUpdate 7 /* synopsis: data=r[P3@P2] */ -#define OP_Init 8 /* jump, synopsis: Start at P2 */ +#define OP_Init 8 /* jump0, synopsis: Start at P2 */ #define OP_Goto 9 /* jump */ #define OP_Gosub 10 /* jump */ -#define OP_InitCoroutine 11 /* jump */ -#define OP_Yield 12 /* jump */ -#define OP_MustBeInt 13 /* jump */ +#define OP_InitCoroutine 11 /* jump0 */ +#define OP_Yield 12 /* jump0 */ +#define OP_MustBeInt 13 /* jump0 */ #define OP_Jump 14 /* jump */ #define OP_Once 15 /* jump */ #define OP_If 16 /* jump */ @@ -16579,22 +16634,22 @@ typedef struct VdbeOpList VdbeOpList; #define OP_IsType 18 /* jump, synopsis: if typeof(P1.P3) in P5 goto P2 */ #define OP_Not 19 /* same as TK_NOT, synopsis: r[P2]= !r[P1] */ #define OP_IfNullRow 20 /* jump, synopsis: if P1.nullRow then r[P3]=NULL, goto P2 */ -#define OP_SeekLT 21 /* jump, synopsis: key=r[P3@P4] */ -#define OP_SeekLE 22 /* jump, synopsis: key=r[P3@P4] */ -#define OP_SeekGE 23 /* jump, synopsis: key=r[P3@P4] */ -#define OP_SeekGT 24 /* jump, synopsis: key=r[P3@P4] */ +#define OP_SeekLT 21 /* jump0, synopsis: key=r[P3@P4] */ +#define OP_SeekLE 22 /* jump0, synopsis: key=r[P3@P4] */ +#define OP_SeekGE 23 /* jump0, synopsis: key=r[P3@P4] */ +#define OP_SeekGT 24 /* jump0, synopsis: key=r[P3@P4] */ #define OP_IfNotOpen 25 /* jump, synopsis: if( !csr[P1] ) goto P2 */ #define OP_IfNoHope 26 /* jump, synopsis: key=r[P3@P4] */ #define OP_NoConflict 27 /* jump, synopsis: key=r[P3@P4] */ #define OP_NotFound 28 /* jump, synopsis: key=r[P3@P4] */ #define OP_Found 29 /* jump, synopsis: key=r[P3@P4] */ -#define OP_SeekRowid 30 /* jump, synopsis: intkey=r[P3] */ +#define OP_SeekRowid 30 /* jump0, synopsis: intkey=r[P3] */ #define OP_NotExists 31 /* jump, synopsis: intkey=r[P3] */ -#define OP_Last 32 /* jump */ -#define OP_IfSmaller 33 /* jump */ +#define OP_Last 32 /* jump0 */ +#define OP_IfSizeBetween 33 /* jump */ #define OP_SorterSort 34 /* jump */ #define OP_Sort 35 /* jump */ -#define OP_Rewind 36 /* jump */ +#define OP_Rewind 36 /* jump0 */ #define OP_SorterNext 37 /* jump */ #define OP_Prev 38 /* jump */ #define OP_Next 39 /* jump */ @@ -16606,7 +16661,7 @@ typedef struct VdbeOpList VdbeOpList; #define OP_IdxGE 45 /* jump, synopsis: key=r[P3@P4] */ #define OP_RowSetRead 46 /* jump, synopsis: r[P3]=rowset(P1) */ #define OP_RowSetTest 47 /* jump, synopsis: if r[P3] in rowset(P1) goto P2 */ -#define OP_Program 48 /* jump */ +#define OP_Program 48 /* jump0 */ #define OP_FkIfZero 49 /* jump, synopsis: if fkctr[P1]==0 goto P2 */ #define OP_IsNull 50 /* jump, same as TK_ISNULL, synopsis: if r[P1]==NULL goto P2 */ #define OP_NotNull 51 /* jump, same as TK_NOTNULL, synopsis: if r[P1]!=NULL goto P2 */ @@ -16636,7 +16691,7 @@ typedef struct VdbeOpList VdbeOpList; #define OP_Null 75 /* synopsis: r[P2..P3]=NULL */ #define OP_SoftNull 76 /* synopsis: r[P1]=NULL */ #define OP_Blob 77 /* synopsis: r[P2]=P4 (len=P1) */ -#define OP_Variable 78 /* synopsis: r[P2]=parameter(P1,P4) */ +#define OP_Variable 78 /* synopsis: r[P2]=parameter(P1) */ #define OP_Move 79 /* synopsis: r[P2@P3]=r[P1@P3] */ #define OP_Copy 80 /* synopsis: r[P2@P3+1]=r[P1@P3+1] */ #define OP_SCopy 81 /* synopsis: r[P2]=r[P1] */ @@ -16760,14 +16815,15 @@ typedef struct VdbeOpList VdbeOpList; #define OPFLG_OUT2 0x10 /* out2: P2 is an output */ #define OPFLG_OUT3 0x20 /* out3: P3 is an output */ #define OPFLG_NCYCLE 0x40 /* ncycle:Cycles count against P1 */ +#define OPFLG_JUMP0 0x80 /* jump0: P2 might be zero */ #define OPFLG_INITIALIZER {\ /* 0 */ 0x00, 0x00, 0x00, 0x00, 0x10, 0x00, 0x41, 0x00,\ -/* 8 */ 0x01, 0x01, 0x01, 0x01, 0x03, 0x03, 0x01, 0x01,\ -/* 16 */ 0x03, 0x03, 0x01, 0x12, 0x01, 0x49, 0x49, 0x49,\ -/* 24 */ 0x49, 0x01, 0x49, 0x49, 0x49, 0x49, 0x49, 0x49,\ -/* 32 */ 0x41, 0x01, 0x41, 0x41, 0x41, 0x01, 0x41, 0x41,\ +/* 8 */ 0x81, 0x01, 0x01, 0x81, 0x83, 0x83, 0x01, 0x01,\ +/* 16 */ 0x03, 0x03, 0x01, 0x12, 0x01, 0xc9, 0xc9, 0xc9,\ +/* 24 */ 0xc9, 0x01, 0x49, 0x49, 0x49, 0x49, 0xc9, 0x49,\ +/* 32 */ 0xc1, 0x01, 0x41, 0x41, 0xc1, 0x01, 0x41, 0x41,\ /* 40 */ 0x41, 0x41, 0x41, 0x26, 0x26, 0x41, 0x23, 0x0b,\ -/* 48 */ 0x01, 0x01, 0x03, 0x03, 0x0b, 0x0b, 0x0b, 0x0b,\ +/* 48 */ 0x81, 0x01, 0x03, 0x03, 0x0b, 0x0b, 0x0b, 0x0b,\ /* 56 */ 0x0b, 0x0b, 0x01, 0x03, 0x03, 0x03, 0x01, 0x41,\ /* 64 */ 0x01, 0x00, 0x00, 0x02, 0x02, 0x08, 0x00, 0x10,\ /* 72 */ 0x10, 0x10, 0x00, 0x10, 0x00, 0x10, 0x10, 0x00,\ @@ -16927,6 +16983,8 @@ SQLITE_PRIVATE RecordCompare sqlite3VdbeFindCompare(UnpackedRecord*); SQLITE_PRIVATE void sqlite3VdbeLinkSubProgram(Vdbe *, SubProgram *); SQLITE_PRIVATE int sqlite3VdbeHasSubProgram(Vdbe*); +SQLITE_PRIVATE void sqlite3MemSetArrayInt64(sqlite3_value *aMem, int iIdx, i64 val); + SQLITE_PRIVATE int sqlite3NotPureFunc(sqlite3_context*); #ifdef SQLITE_ENABLE_BYTECODE_VTAB SQLITE_PRIVATE int sqlite3VdbeBytecodeVtabInit(sqlite3*); @@ -17514,6 +17572,10 @@ struct FuncDefHash { }; #define SQLITE_FUNC_HASH(C,L) (((C)+(L))%SQLITE_FUNC_HASH_SZ) +#if defined(SQLITE_USER_AUTHENTICATION) +# warning "The SQLITE_USER_AUTHENTICATION extension is deprecated. \ + See ext/userauth/user-auth.txt for details." +#endif #ifdef SQLITE_USER_AUTHENTICATION /* ** Information held in the "sqlite3" database connection object and used @@ -17817,7 +17879,7 @@ struct sqlite3 { #define SQLITE_CursorHints 0x00000400 /* Add OP_CursorHint opcodes */ #define SQLITE_Stat4 0x00000800 /* Use STAT4 data */ /* TH3 expects this value ^^^^^^^^^^ to be 0x0000800. Don't change it */ -#define SQLITE_PushDown 0x00001000 /* The push-down optimization */ +#define SQLITE_PushDown 0x00001000 /* WHERE-clause push-down opt */ #define SQLITE_SimplifyJoin 0x00002000 /* Convert LEFT JOIN to JOIN */ #define SQLITE_SkipScan 0x00004000 /* Skip-scans */ #define SQLITE_PropagateConst 0x00008000 /* The constant propagation opt */ @@ -18390,8 +18452,7 @@ struct Table { #define TF_HasStored 0x00000040 /* Has one or more STORED columns */ #define TF_HasGenerated 0x00000060 /* Combo: HasVirtual + HasStored */ #define TF_WithoutRowid 0x00000080 /* No rowid. PRIMARY KEY is the key */ -#define TF_StatsUsed 0x00000100 /* Query planner decisions affected by - ** Index.aiRowLogEst[] values */ +#define TF_MaybeReanalyze 0x00000100 /* Maybe run ANALYZE on this table */ #define TF_NoVisibleRowid 0x00000200 /* No user-visible "rowid" column */ #define TF_OOOHidden 0x00000400 /* Out-of-Order hidden columns */ #define TF_HasNotNull 0x00000800 /* Contains NOT NULL constraints */ @@ -19191,10 +19252,12 @@ struct IdList { ** ** Union member validity: ** -** u1.zIndexedBy fg.isIndexedBy && !fg.isTabFunc -** u1.pFuncArg fg.isTabFunc && !fg.isIndexedBy -** u2.pIBIndex fg.isIndexedBy && !fg.isCte -** u2.pCteUse fg.isCte && !fg.isIndexedBy +** u1.zIndexedBy fg.isIndexedBy && !fg.isTabFunc +** u1.pFuncArg fg.isTabFunc && !fg.isIndexedBy +** u1.nRow !fg.isTabFunc && !fg.isIndexedBy +** +** u2.pIBIndex fg.isIndexedBy && !fg.isCte +** u2.pCteUse fg.isCte && !fg.isIndexedBy */ struct SrcItem { Schema *pSchema; /* Schema to which this item is fixed */ @@ -19222,6 +19285,7 @@ struct SrcItem { unsigned isOn :1; /* u3.pOn was once valid and non-NULL */ unsigned isSynthUsing :1; /* u3.pUsing is synthesized from NATURAL */ unsigned isNestedFrom :1; /* pSelect is a SF_NestedFrom subquery */ + unsigned rowidUsed :1; /* The ROWID of this table is referenced */ } fg; int iCursor; /* The VDBE cursor number used to access this table */ union { @@ -19232,6 +19296,7 @@ struct SrcItem { union { char *zIndexedBy; /* Identifier from "INDEXED BY " clause */ ExprList *pFuncArg; /* Arguments to table-valued-function */ + u32 nRow; /* Number of rows in a VALUES clause */ } u1; union { Index *pIBIndex; /* Index structure corresponding to u1.zIndexedBy */ @@ -19489,11 +19554,12 @@ struct Select { #define SF_View 0x0200000 /* SELECT statement is a view */ #define SF_NoopOrderBy 0x0400000 /* ORDER BY is ignored for this query */ #define SF_UFSrcCheck 0x0800000 /* Check pSrc as required by UPDATE...FROM */ -#define SF_PushDown 0x1000000 /* SELECT has be modified by push-down opt */ +#define SF_PushDown 0x1000000 /* Modified by WHERE-clause push-down opt */ #define SF_MultiPart 0x2000000 /* Has multiple incompatible PARTITIONs */ #define SF_CopyCte 0x4000000 /* SELECT statement is a copy of a CTE */ #define SF_OrderByReqd 0x8000000 /* The ORDER BY clause may not be omitted */ #define SF_UpdateFrom 0x10000000 /* Query originates with UPDATE FROM */ +#define SF_Correlated 0x20000000 /* True if references the outer context */ /* True if S exists and has SF_NestedFrom */ #define IsNestedFrom(S) ((S)!=0 && ((S)->selFlags&SF_NestedFrom)!=0) @@ -19733,6 +19799,7 @@ struct Parse { u8 disableLookaside; /* Number of times lookaside has been disabled */ u8 prepFlags; /* SQLITE_PREPARE_* flags */ u8 withinRJSubrtn; /* Nesting level for RIGHT JOIN body subroutines */ + u8 bHasWith; /* True if statement contains WITH */ #if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST) u8 earlyCleanup; /* OOM inside sqlite3ParserAddCleanup() */ #endif @@ -20412,6 +20479,9 @@ struct Window { ** due to the SQLITE_SUBTYPE flag */ }; +SQLITE_PRIVATE Select *sqlite3MultiValues(Parse *pParse, Select *pLeft, ExprList *pRow); +SQLITE_PRIVATE void sqlite3MultiValuesEnd(Parse *pParse, Select *pVal); + #ifndef SQLITE_OMIT_WINDOWFUNC SQLITE_PRIVATE void sqlite3WindowDelete(sqlite3*, Window*); SQLITE_PRIVATE void sqlite3WindowUnlinkFromSelect(Window*); @@ -20729,6 +20799,7 @@ SQLITE_PRIVATE int sqlite3ErrorToParser(sqlite3*,int); SQLITE_PRIVATE void sqlite3Dequote(char*); SQLITE_PRIVATE void sqlite3DequoteExpr(Expr*); SQLITE_PRIVATE void sqlite3DequoteToken(Token*); +SQLITE_PRIVATE void sqlite3DequoteNumber(Parse*, Expr*); SQLITE_PRIVATE void sqlite3TokenInit(Token*,char*); SQLITE_PRIVATE int sqlite3KeywordCode(const unsigned char*, int); SQLITE_PRIVATE int sqlite3RunParser(Parse*, const char*); @@ -20759,7 +20830,7 @@ SQLITE_PRIVATE void sqlite3ExprFunctionUsable(Parse*,const Expr*,const FuncDef*) SQLITE_PRIVATE void sqlite3ExprAssignVarNumber(Parse*, Expr*, u32); SQLITE_PRIVATE void sqlite3ExprDelete(sqlite3*, Expr*); SQLITE_PRIVATE void sqlite3ExprDeleteGeneric(sqlite3*,void*); -SQLITE_PRIVATE void sqlite3ExprDeferredDelete(Parse*, Expr*); +SQLITE_PRIVATE int sqlite3ExprDeferredDelete(Parse*, Expr*); SQLITE_PRIVATE void sqlite3ExprUnmapAndDelete(Parse*, Expr*); SQLITE_PRIVATE ExprList *sqlite3ExprListAppend(Parse*,ExprList*,Expr*); SQLITE_PRIVATE ExprList *sqlite3ExprListAppendVector(Parse*,ExprList*,IdList*,Expr*); @@ -20982,12 +21053,10 @@ SQLITE_PRIVATE void sqlite3LeaveMutexAndCloseZombie(sqlite3*); SQLITE_PRIVATE u32 sqlite3IsTrueOrFalse(const char*); SQLITE_PRIVATE int sqlite3ExprIdToTrueFalse(Expr*); SQLITE_PRIVATE int sqlite3ExprTruthValue(const Expr*); -SQLITE_PRIVATE int sqlite3ExprIsConstant(Expr*); -SQLITE_PRIVATE int sqlite3ExprIsConstantNotJoin(Expr*); +SQLITE_PRIVATE int sqlite3ExprIsConstant(Parse*,Expr*); SQLITE_PRIVATE int sqlite3ExprIsConstantOrFunction(Expr*, u8); SQLITE_PRIVATE int sqlite3ExprIsConstantOrGroupBy(Parse*, Expr*, ExprList*); -SQLITE_PRIVATE int sqlite3ExprIsTableConstant(Expr*,int); -SQLITE_PRIVATE int sqlite3ExprIsSingleTableConstraint(Expr*,const SrcList*,int); +SQLITE_PRIVATE int sqlite3ExprIsSingleTableConstraint(Expr*,const SrcList*,int,int); #ifdef SQLITE_ENABLE_CURSOR_HINTS SQLITE_PRIVATE int sqlite3ExprContainsSubquery(Expr*); #endif @@ -21172,7 +21241,9 @@ SQLITE_PRIVATE void sqlite3ErrorWithMsg(sqlite3*, int, const char*,...); SQLITE_PRIVATE void sqlite3Error(sqlite3*,int); SQLITE_PRIVATE void sqlite3ErrorClear(sqlite3*); SQLITE_PRIVATE void sqlite3SystemError(sqlite3*,int); +#if !defined(SQLITE_OMIT_BLOB_LITERAL) SQLITE_PRIVATE void *sqlite3HexToBlob(sqlite3*, const char *z, int n); +#endif SQLITE_PRIVATE u8 sqlite3HexToInt(int h); SQLITE_PRIVATE int sqlite3TwoPartName(Parse *, Token *, Token *, Token **); @@ -24219,13 +24290,14 @@ struct DateTime { int tz; /* Timezone offset in minutes */ double s; /* Seconds */ char validJD; /* True (1) if iJD is valid */ - char rawS; /* Raw numeric value stored in s */ char validYMD; /* True (1) if Y,M,D are valid */ char validHMS; /* True (1) if h,m,s are valid */ - char validTZ; /* True (1) if tz is valid */ - char tzSet; /* Timezone was set explicitly */ - char isError; /* An overflow has occurred */ - char useSubsec; /* Display subsecond precision */ + char nFloor; /* Days to implement "floor" */ + unsigned rawS : 1; /* Raw numeric value stored in s */ + unsigned isError : 1; /* An overflow has occurred */ + unsigned useSubsec : 1; /* Display subsecond precision */ + unsigned isUtc : 1; /* Time is known to be UTC */ + unsigned isLocal : 1; /* Time is known to be localtime */ }; @@ -24323,6 +24395,8 @@ static int parseTimezone(const char *zDate, DateTime *p){ sgn = +1; }else if( c=='Z' || c=='z' ){ zDate++; + p->isLocal = 0; + p->isUtc = 1; goto zulu_time; }else{ return c!=0; @@ -24335,7 +24409,6 @@ static int parseTimezone(const char *zDate, DateTime *p){ p->tz = sgn*(nMn + nHr*60); zulu_time: while( sqlite3Isspace(*zDate) ){ zDate++; } - p->tzSet = 1; return *zDate!=0; } @@ -24379,7 +24452,6 @@ static int parseHhMmSs(const char *zDate, DateTime *p){ p->m = m; p->s = s + ms; if( parseTimezone(zDate, p) ) return 1; - p->validTZ = (p->tz!=0)?1:0; return 0; } @@ -24426,15 +24498,40 @@ static void computeJD(DateTime *p){ p->validJD = 1; if( p->validHMS ){ p->iJD += p->h*3600000 + p->m*60000 + (sqlite3_int64)(p->s*1000 + 0.5); - if( p->validTZ ){ + if( p->tz ){ p->iJD -= p->tz*60000; p->validYMD = 0; p->validHMS = 0; - p->validTZ = 0; + p->tz = 0; + p->isUtc = 1; + p->isLocal = 0; } } } +/* +** Given the YYYY-MM-DD information current in p, determine if there +** is day-of-month overflow and set nFloor to the number of days that +** would need to be subtracted from the date in order to bring the +** date back to the end of the month. +*/ +static void computeFloor(DateTime *p){ + assert( p->validYMD || p->isError ); + assert( p->D>=0 && p->D<=31 ); + assert( p->M>=0 && p->M<=12 ); + if( p->D<=28 ){ + p->nFloor = 0; + }else if( (1<M) & 0x15aa ){ + p->nFloor = 0; + }else if( p->M!=2 ){ + p->nFloor = (p->D==31); + }else if( p->Y%4!=0 || (p->Y%100==0 && p->Y%400!=0) ){ + p->nFloor = p->D - 28; + }else{ + p->nFloor = p->D - 29; + } +} + /* ** Parse dates of the form ** @@ -24473,12 +24570,16 @@ static int parseYyyyMmDd(const char *zDate, DateTime *p){ p->Y = neg ? -Y : Y; p->M = M; p->D = D; - if( p->validTZ ){ + computeFloor(p); + if( p->tz ){ computeJD(p); } return 0; } + +static void clearYMD_HMS_TZ(DateTime *p); /* Forward declaration */ + /* ** Set the time to the current time reported by the VFS. ** @@ -24488,6 +24589,9 @@ static int setDateTimeToCurrent(sqlite3_context *context, DateTime *p){ p->iJD = sqlite3StmtCurrentTime(context); if( p->iJD>0 ){ p->validJD = 1; + p->isUtc = 1; + p->isLocal = 0; + clearYMD_HMS_TZ(p); return 0; }else{ return 1; @@ -24626,7 +24730,7 @@ static void computeYMD_HMS(DateTime *p){ static void clearYMD_HMS_TZ(DateTime *p){ p->validYMD = 0; p->validHMS = 0; - p->validTZ = 0; + p->tz = 0; } #ifndef SQLITE_OMIT_LOCALTIME @@ -24758,7 +24862,7 @@ static int toLocaltime( p->validHMS = 1; p->validJD = 0; p->rawS = 0; - p->validTZ = 0; + p->tz = 0; p->isError = 0; return SQLITE_OK; } @@ -24778,12 +24882,12 @@ static const struct { float rLimit; /* Maximum NNN value for this transform */ float rXform; /* Constant used for this transform */ } aXformType[] = { - { 6, "second", 4.6427e+14, 1.0 }, - { 6, "minute", 7.7379e+12, 60.0 }, - { 4, "hour", 1.2897e+11, 3600.0 }, - { 3, "day", 5373485.0, 86400.0 }, - { 5, "month", 176546.0, 2592000.0 }, - { 4, "year", 14713.0, 31536000.0 }, + /* 0 */ { 6, "second", 4.6427e+14, 1.0 }, + /* 1 */ { 6, "minute", 7.7379e+12, 60.0 }, + /* 2 */ { 4, "hour", 1.2897e+11, 3600.0 }, + /* 3 */ { 3, "day", 5373485.0, 86400.0 }, + /* 4 */ { 5, "month", 176546.0, 30.0*86400.0 }, + /* 5 */ { 4, "year", 14713.0, 365.0*86400.0 }, }; /* @@ -24815,14 +24919,20 @@ static void autoAdjustDate(DateTime *p){ ** NNN.NNNN seconds ** NNN months ** NNN years +** +/-YYYY-MM-DD HH:MM:SS.SSS +** ceiling +** floor ** start of month ** start of year ** start of week ** start of day ** weekday N ** unixepoch +** auto ** localtime ** utc +** subsec +** subsecond ** ** Return 0 on success and 1 if there is any kind of error. If the error ** is in a system call (i.e. localtime()), then an error message is written @@ -24853,6 +24963,37 @@ static int parseModifier( } break; } + case 'c': { + /* + ** ceiling + ** + ** Resolve day-of-month overflow by rolling forward into the next + ** month. As this is the default action, this modifier is really + ** a no-op that is only included for symmetry. See "floor". + */ + if( sqlite3_stricmp(z, "ceiling")==0 ){ + computeJD(p); + clearYMD_HMS_TZ(p); + rc = 0; + p->nFloor = 0; + } + break; + } + case 'f': { + /* + ** floor + ** + ** Resolve day-of-month overflow by rolling back to the end of the + ** previous month. + */ + if( sqlite3_stricmp(z, "floor")==0 ){ + computeJD(p); + p->iJD -= p->nFloor*86400000; + clearYMD_HMS_TZ(p); + rc = 0; + } + break; + } case 'j': { /* ** julianday @@ -24879,7 +25020,9 @@ static int parseModifier( ** show local time. */ if( sqlite3_stricmp(z, "localtime")==0 && sqlite3NotPureFunc(pCtx) ){ - rc = toLocaltime(p, pCtx); + rc = p->isLocal ? SQLITE_OK : toLocaltime(p, pCtx); + p->isUtc = 0; + p->isLocal = 1; } break; } @@ -24904,7 +25047,7 @@ static int parseModifier( } #ifndef SQLITE_OMIT_LOCALTIME else if( sqlite3_stricmp(z, "utc")==0 && sqlite3NotPureFunc(pCtx) ){ - if( p->tzSet==0 ){ + if( p->isUtc==0 ){ i64 iOrigJD; /* Original localtime */ i64 iGuess; /* Guess at the corresponding utc time */ int cnt = 0; /* Safety to prevent infinite loop */ @@ -24927,7 +25070,8 @@ static int parseModifier( memset(p, 0, sizeof(*p)); p->iJD = iGuess; p->validJD = 1; - p->tzSet = 1; + p->isUtc = 1; + p->isLocal = 0; } rc = SQLITE_OK; } @@ -24947,7 +25091,7 @@ static int parseModifier( && r>=0.0 && r<7.0 && (n=(int)r)==r ){ sqlite3_int64 Z; computeYMD_HMS(p); - p->validTZ = 0; + p->tz = 0; p->validJD = 0; computeJD(p); Z = ((p->iJD + 129600000)/86400000) % 7; @@ -24987,7 +25131,7 @@ static int parseModifier( p->h = p->m = 0; p->s = 0.0; p->rawS = 0; - p->validTZ = 0; + p->tz = 0; p->validJD = 0; if( sqlite3_stricmp(z,"month")==0 ){ p->D = 1; @@ -25058,6 +25202,7 @@ static int parseModifier( x = p->M>0 ? (p->M-1)/12 : (p->M-12)/12; p->Y += x; p->M -= x*12; + computeFloor(p); computeJD(p); p->validHMS = 0; p->validYMD = 0; @@ -25104,11 +25249,12 @@ static int parseModifier( z += n; while( sqlite3Isspace(*z) ) z++; n = sqlite3Strlen30(z); - if( n>10 || n<3 ) break; + if( n<3 || n>10 ) break; if( sqlite3UpperToLower[(u8)z[n-1]]=='s' ) n--; computeJD(p); assert( rc==1 ); rRounder = r<0 ? -0.5 : +0.5; + p->nFloor = 0; for(i=0; iM += (int)r; x = p->M>0 ? (p->M-1)/12 : (p->M-12)/12; p->Y += x; p->M -= x*12; + computeFloor(p); p->validJD = 0; r -= (int)r; break; } case 5: { /* Special processing to add years */ int y = (int)r; - assert( strcmp(aXformType[i].zName,"year")==0 ); + assert( strcmp(aXformType[5].zName,"year")==0 ); computeYMD_HMS(p); + assert( p->M>=0 && p->M<=12 ); p->Y += y; + computeFloor(p); p->validJD = 0; r -= (int)r; break; @@ -25384,22 +25533,83 @@ static void dateFunc( } } +/* +** Compute the number of days after the most recent January 1. +** +** In other words, compute the zero-based day number for the +** current year: +** +** Jan01 = 0, Jan02 = 1, ..., Jan31 = 30, Feb01 = 31, ... +** Dec31 = 364 or 365. +*/ +static int daysAfterJan01(DateTime *pDate){ + DateTime jan01 = *pDate; + assert( jan01.validYMD ); + assert( jan01.validHMS ); + assert( pDate->validJD ); + jan01.validJD = 0; + jan01.M = 1; + jan01.D = 1; + computeJD(&jan01); + return (int)((pDate->iJD-jan01.iJD+43200000)/86400000); +} + +/* +** Return the number of days after the most recent Monday. +** +** In other words, return the day of the week according +** to this code: +** +** 0=Monday, 1=Tuesday, 2=Wednesday, ..., 6=Sunday. +*/ +static int daysAfterMonday(DateTime *pDate){ + assert( pDate->validJD ); + return (int)((pDate->iJD+43200000)/86400000) % 7; +} + +/* +** Return the number of days after the most recent Sunday. +** +** In other words, return the day of the week according +** to this code: +** +** 0=Sunday, 1=Monday, 2=Tues, ..., 6=Saturday +*/ +static int daysAfterSunday(DateTime *pDate){ + assert( pDate->validJD ); + return (int)((pDate->iJD+129600000)/86400000) % 7; +} + /* ** strftime( FORMAT, TIMESTRING, MOD, MOD, ...) ** ** Return a string described by FORMAT. Conversions as follows: ** -** %d day of month +** %d day of month 01-31 +** %e day of month 1-31 ** %f ** fractional seconds SS.SSS +** %F ISO date. YYYY-MM-DD +** %G ISO year corresponding to %V 0000-9999. +** %g 2-digit ISO year corresponding to %V 00-99 ** %H hour 00-24 -** %j day of year 000-366 +** %k hour 0-24 (leading zero converted to space) +** %I hour 01-12 +** %j day of year 001-366 ** %J ** julian day number +** %l hour 1-12 (leading zero converted to space) ** %m month 01-12 ** %M minute 00-59 +** %p "am" or "pm" +** %P "AM" or "PM" +** %R time as HH:MM ** %s seconds since 1970-01-01 ** %S seconds 00-59 -** %w day of week 0-6 Sunday==0 -** %W week of year 00-53 +** %T time as HH:MM:SS +** %u day of week 1-7 Monday==1, Sunday==7 +** %w day of week 0-6 Sunday==0, Monday==1 +** %U week of year 00-53 (First Sunday is start of week 01) +** %V week of year 01-53 (First week containing Thursday is week 01) +** %W week of year 00-53 (First Monday is start of week 01) ** %Y year 0000-9999 ** %% % */ @@ -25436,7 +25646,7 @@ static void strftimeFunc( sqlite3_str_appendf(&sRes, cf=='d' ? "%02d" : "%2d", x.D); break; } - case 'f': { + case 'f': { /* Fractional seconds. (Non-standard) */ double s = x.s; if( s>59.999 ) s = 59.999; sqlite3_str_appendf(&sRes, "%06.3f", s); @@ -25446,6 +25656,21 @@ static void strftimeFunc( sqlite3_str_appendf(&sRes, "%04d-%02d-%02d", x.Y, x.M, x.D); break; } + case 'G': /* Fall thru */ + case 'g': { + DateTime y = x; + assert( y.validJD ); + /* Move y so that it is the Thursday in the same week as x */ + y.iJD += (3 - daysAfterMonday(&x))*86400000; + y.validYMD = 0; + computeYMD(&y); + if( cf=='g' ){ + sqlite3_str_appendf(&sRes, "%02d", y.Y%100); + }else{ + sqlite3_str_appendf(&sRes, "%04d", y.Y); + } + break; + } case 'H': case 'k': { sqlite3_str_appendf(&sRes, cf=='H' ? "%02d" : "%2d", x.h); @@ -25459,25 +25684,11 @@ static void strftimeFunc( sqlite3_str_appendf(&sRes, cf=='I' ? "%02d" : "%2d", h); break; } - case 'W': /* Fall thru */ - case 'j': { - int nDay; /* Number of days since 1st day of year */ - DateTime y = x; - y.validJD = 0; - y.M = 1; - y.D = 1; - computeJD(&y); - nDay = (int)((x.iJD-y.iJD+43200000)/86400000); - if( cf=='W' ){ - int wd; /* 0=Monday, 1=Tuesday, ... 6=Sunday */ - wd = (int)(((x.iJD+43200000)/86400000)%7); - sqlite3_str_appendf(&sRes,"%02d",(nDay+7-wd)/7); - }else{ - sqlite3_str_appendf(&sRes,"%03d",nDay+1); - } + case 'j': { /* Day of year. Jan01==1, Jan02==2, and so forth */ + sqlite3_str_appendf(&sRes,"%03d",daysAfterJan01(&x)+1); break; } - case 'J': { + case 'J': { /* Julian day number. (Non-standard) */ sqlite3_str_appendf(&sRes,"%.16g",x.iJD/86400000.0); break; } @@ -25520,13 +25731,33 @@ static void strftimeFunc( sqlite3_str_appendf(&sRes,"%02d:%02d:%02d", x.h, x.m, (int)x.s); break; } - case 'u': /* Fall thru */ - case 'w': { - char c = (char)(((x.iJD+129600000)/86400000) % 7) + '0'; + case 'u': /* Day of week. 1 to 7. Monday==1, Sunday==7 */ + case 'w': { /* Day of week. 0 to 6. Sunday==0, Monday==1 */ + char c = (char)daysAfterSunday(&x) + '0'; if( c=='0' && cf=='u' ) c = '7'; sqlite3_str_appendchar(&sRes, 1, c); break; } + case 'U': { /* Week num. 00-53. First Sun of the year is week 01 */ + sqlite3_str_appendf(&sRes,"%02d", + (daysAfterJan01(&x)-daysAfterSunday(&x)+7)/7); + break; + } + case 'V': { /* Week num. 01-53. First week with a Thur is week 01 */ + DateTime y = x; + /* Adjust y so that is the Thursday in the same week as x */ + assert( y.validJD ); + y.iJD += (3 - daysAfterMonday(&x))*86400000; + y.validYMD = 0; + computeYMD(&y); + sqlite3_str_appendf(&sRes,"%02d", daysAfterJan01(&y)/7+1); + break; + } + case 'W': { /* Week num. 00-53. First Mon of the year is week 01 */ + sqlite3_str_appendf(&sRes,"%02d", + (daysAfterJan01(&x)-daysAfterMonday(&x)+7)/7); + break; + } case 'Y': { sqlite3_str_appendf(&sRes,"%04d",x.Y); break; @@ -25673,9 +25904,7 @@ static void timediffFunc( d1.iJD = d2.iJD - d1.iJD; d1.iJD += (u64)1486995408 * (u64)100000; } - d1.validYMD = 0; - d1.validHMS = 0; - d1.validTZ = 0; + clearYMD_HMS_TZ(&d1); computeYMD_HMS(&d1); sqlite3StrAccumInit(&sRes, 0, 0, 0, 100); sqlite3_str_appendf(&sRes, "%c%04d-%02d-%02d %02d:%02d:%06.3f", @@ -25744,6 +25973,36 @@ static void currentTimeFunc( } #endif +#if !defined(SQLITE_OMIT_DATETIME_FUNCS) && defined(SQLITE_DEBUG) +/* +** datedebug(...) +** +** This routine returns JSON that describes the internal DateTime object. +** Used for debugging and testing only. Subject to change. +*/ +static void datedebugFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + DateTime x; + if( isDate(context, argc, argv, &x)==0 ){ + char *zJson; + zJson = sqlite3_mprintf( + "{iJD:%lld,Y:%d,M:%d,D:%d,h:%d,m:%d,tz:%d," + "s:%.3f,validJD:%d,validYMS:%d,validHMS:%d," + "nFloor:%d,rawS:%d,isError:%d,useSubsec:%d," + "isUtc:%d,isLocal:%d}", + x.iJD, x.Y, x.M, x.D, x.h, x.m, x.tz, + x.s, x.validJD, x.validYMD, x.validHMS, + x.nFloor, x.rawS, x.isError, x.useSubsec, + x.isUtc, x.isLocal); + sqlite3_result_text(context, zJson, -1, sqlite3_free); + } +} +#endif /* !SQLITE_OMIT_DATETIME_FUNCS && SQLITE_DEBUG */ + + /* ** This function registered all of the above C functions as SQL ** functions. This should be the only routine in this file with @@ -25759,6 +26018,9 @@ SQLITE_PRIVATE void sqlite3RegisterDateTimeFunctions(void){ PURE_DATE(datetime, -1, 0, 0, datetimeFunc ), PURE_DATE(strftime, -1, 0, 0, strftimeFunc ), PURE_DATE(timediff, 2, 0, 0, timediffFunc ), +#ifdef SQLITE_DEBUG + PURE_DATE(datedebug, -1, 0, 0, datedebugFunc ), +#endif DFUNCTION(current_time, 0, 0, 0, ctimeFunc ), DFUNCTION(current_timestamp, 0, 0, 0, ctimestampFunc), DFUNCTION(current_date, 0, 0, 0, cdateFunc ), @@ -30174,6 +30436,24 @@ static void sqlite3MallocAlarm(int nByte){ sqlite3_mutex_enter(mem0.mutex); } +#ifdef SQLITE_DEBUG +/* +** This routine is called whenever an out-of-memory condition is seen, +** It's only purpose to to serve as a breakpoint for gdb or similar +** code debuggers when working on out-of-memory conditions, for example +** caused by PRAGMA hard_heap_limit=N. +*/ +static SQLITE_NOINLINE void test_oom_breakpoint(u64 n){ + static u64 nOomFault = 0; + nOomFault += n; + /* The assert() is never reached in a human lifetime. It is here mostly + ** to prevent code optimizers from optimizing out this function. */ + assert( (nOomFault>>32) < 0xffffffff ); +} +#else +# define test_oom_breakpoint(X) /* No-op for production builds */ +#endif + /* ** Do a memory allocation with statistics and alarms. Assume the ** lock is already held. @@ -30200,6 +30480,7 @@ static void mallocWithAlarm(int n, void **pp){ if( mem0.hardLimit ){ nUsed = sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED); if( nUsed >= mem0.hardLimit - nFull ){ + test_oom_breakpoint(1); *pp = 0; return; } @@ -30488,6 +30769,7 @@ SQLITE_PRIVATE void *sqlite3Realloc(void *pOld, u64 nBytes){ sqlite3MallocAlarm(nDiff); if( mem0.hardLimit>0 && nUsed >= mem0.hardLimit - nDiff ){ sqlite3_mutex_leave(mem0.mutex); + test_oom_breakpoint(1); return 0; } } @@ -31390,13 +31672,14 @@ SQLITE_API void sqlite3_str_vappendf( } exp = s.iDP-1; - if( xtype==etGENERIC && precision>0 ) precision--; /* ** If the field type is etGENERIC, then convert to either etEXP ** or etFLOAT, as appropriate. */ if( xtype==etGENERIC ){ + assert( precision>0 ); + precision--; flag_rtz = !flag_alternateform; if( exp<-4 || exp>precision ){ xtype = etEXP; @@ -31712,9 +31995,13 @@ SQLITE_API void sqlite3_str_vappendf( sqlite3_str_appendall(pAccum, pItem->zAlias); }else{ Select *pSel = pItem->pSelect; - assert( pSel!=0 ); + assert( pSel!=0 ); /* Because of tag-20240424-1 */ if( pSel->selFlags & SF_NestedFrom ){ sqlite3_str_appendf(pAccum, "(join-%u)", pSel->selId); + }else if( pSel->selFlags & SF_MultiValue ){ + assert( !pItem->fg.isTabFunc && !pItem->fg.isIndexedBy ); + sqlite3_str_appendf(pAccum, "%u-ROW VALUES CLAUSE", + pItem->u1.nRow); }else{ sqlite3_str_appendf(pAccum, "(subquery-%u)", pSel->selId); } @@ -32491,8 +32778,10 @@ SQLITE_PRIVATE void sqlite3TreeViewSrcList(TreeView *pView, const SrcList *pSrc) x.printfFlags |= SQLITE_PRINTF_INTERNAL; sqlite3_str_appendf(&x, "{%d:*} %!S", pItem->iCursor, pItem); if( pItem->pTab ){ - sqlite3_str_appendf(&x, " tab=%Q nCol=%d ptr=%p used=%llx", - pItem->pTab->zName, pItem->pTab->nCol, pItem->pTab, pItem->colUsed); + sqlite3_str_appendf(&x, " tab=%Q nCol=%d ptr=%p used=%llx%s", + pItem->pTab->zName, pItem->pTab->nCol, pItem->pTab, + pItem->colUsed, + pItem->fg.rowidUsed ? "+rowid" : ""); } if( (pItem->fg.jointype & (JT_LEFT|JT_RIGHT))==(JT_LEFT|JT_RIGHT) ){ sqlite3_str_appendf(&x, " FULL-OUTER-JOIN"); @@ -32532,12 +32821,14 @@ SQLITE_PRIVATE void sqlite3TreeViewSrcList(TreeView *pView, const SrcList *pSrc) sqlite3TreeViewIdList(pView, pItem->u3.pUsing, (--n)>0, "USING"); } if( pItem->pSelect ){ + sqlite3TreeViewPush(&pView, i+1nSrc); if( pItem->pTab ){ Table *pTab = pItem->pTab; sqlite3TreeViewColumnList(pView, pTab->aCol, pTab->nCol, 1); } assert( (int)pItem->fg.isNestedFrom == IsNestedFrom(pItem->pSelect) ); sqlite3TreeViewSelect(pView, pItem->pSelect, (--n)>0); + sqlite3TreeViewPop(&pView); } if( pItem->fg.isTabFunc ){ sqlite3TreeViewExprList(pView, pItem->u1.pFuncArg, 0, "func-args:"); @@ -32641,7 +32932,7 @@ SQLITE_PRIVATE void sqlite3TreeViewSelect(TreeView *pView, const Select *p, u8 m sqlite3TreeViewItem(pView, "LIMIT", (n--)>0); sqlite3TreeViewExpr(pView, p->pLimit->pLeft, p->pLimit->pRight!=0); if( p->pLimit->pRight ){ - sqlite3TreeViewItem(pView, "OFFSET", (n--)>0); + sqlite3TreeViewItem(pView, "OFFSET", 0); sqlite3TreeViewExpr(pView, p->pLimit->pRight, 0); sqlite3TreeViewPop(&pView); } @@ -34942,6 +35233,44 @@ SQLITE_PRIVATE void sqlite3DequoteExpr(Expr *p){ sqlite3Dequote(p->u.zToken); } +/* +** Expression p is a QNUMBER (quoted number). Dequote the value in p->u.zToken +** and set the type to INTEGER or FLOAT. "Quoted" integers or floats are those +** that contain '_' characters that must be removed before further processing. +*/ +SQLITE_PRIVATE void sqlite3DequoteNumber(Parse *pParse, Expr *p){ + assert( p!=0 || pParse->db->mallocFailed ); + if( p ){ + const char *pIn = p->u.zToken; + char *pOut = p->u.zToken; + int bHex = (pIn[0]=='0' && (pIn[1]=='x' || pIn[1]=='X')); + int iValue; + assert( p->op==TK_QNUMBER ); + p->op = TK_INTEGER; + do { + if( *pIn!=SQLITE_DIGIT_SEPARATOR ){ + *pOut++ = *pIn; + if( *pIn=='e' || *pIn=='E' || *pIn=='.' ) p->op = TK_FLOAT; + }else{ + if( (bHex==0 && (!sqlite3Isdigit(pIn[-1]) || !sqlite3Isdigit(pIn[1]))) + || (bHex==1 && (!sqlite3Isxdigit(pIn[-1]) || !sqlite3Isxdigit(pIn[1]))) + ){ + sqlite3ErrorMsg(pParse, "unrecognized token: \"%s\"", p->u.zToken); + } + } + }while( *pIn++ ); + if( bHex ) p->op = TK_INTEGER; + + /* tag-20240227-a: If after dequoting, the number is an integer that + ** fits in 32 bits, then it must be converted into EP_IntValue. Other + ** parts of the code expect this. See also tag-20240227-b. */ + if( p->op==TK_INTEGER && sqlite3GetInt32(p->u.zToken, &iValue) ){ + p->u.iValue = iValue; + p->flags |= EP_IntValue; + } + } +} + /* ** If the input token p is quoted, try to adjust the token to remove ** the quotes. This is not always possible: @@ -36881,7 +37210,7 @@ SQLITE_PRIVATE const char *sqlite3OpcodeName(int i){ /* 30 */ "SeekRowid" OpHelp("intkey=r[P3]"), /* 31 */ "NotExists" OpHelp("intkey=r[P3]"), /* 32 */ "Last" OpHelp(""), - /* 33 */ "IfSmaller" OpHelp(""), + /* 33 */ "IfSizeBetween" OpHelp(""), /* 34 */ "SorterSort" OpHelp(""), /* 35 */ "Sort" OpHelp(""), /* 36 */ "Rewind" OpHelp(""), @@ -36926,7 +37255,7 @@ SQLITE_PRIVATE const char *sqlite3OpcodeName(int i){ /* 75 */ "Null" OpHelp("r[P2..P3]=NULL"), /* 76 */ "SoftNull" OpHelp("r[P1]=NULL"), /* 77 */ "Blob" OpHelp("r[P2]=P4 (len=P1)"), - /* 78 */ "Variable" OpHelp("r[P2]=parameter(P1,P4)"), + /* 78 */ "Variable" OpHelp("r[P2]=parameter(P1)"), /* 79 */ "Move" OpHelp("r[P2@P3]=r[P1@P3]"), /* 80 */ "Copy" OpHelp("r[P2@P3+1]=r[P1@P3+1]"), /* 81 */ "SCopy" OpHelp("r[P2]=r[P1]"), @@ -39324,8 +39653,12 @@ static int unixLogErrorAtLine( ** available, the error message will often be an empty string. Not a ** huge problem. Incorrectly concluding that the GNU version is available ** could lead to a segfault though. + ** + ** Forum post 3f13857fa4062301 reports that the Android SDK may use + ** int-type return, depending on its version. */ -#if defined(STRERROR_R_CHAR_P) || defined(__USE_GNU) +#if (defined(STRERROR_R_CHAR_P) || defined(__USE_GNU)) \ + && !defined(ANDROID) && !defined(__ANDROID__) zErr = # endif strerror_r(iErrno, aErr, sizeof(aErr)-1); @@ -44423,12 +44756,19 @@ static int unixOpen( rc = SQLITE_READONLY_DIRECTORY; }else if( errno!=EISDIR && isReadWrite ){ /* Failed to open the file for read/write access. Try read-only. */ + UnixUnusedFd *pReadonly = 0; flags &= ~(SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE); openFlags &= ~(O_RDWR|O_CREAT); flags |= SQLITE_OPEN_READONLY; openFlags |= O_RDONLY; isReadonly = 1; - fd = robust_open(zName, openFlags, openMode); + pReadonly = findReusableFd(zName, flags); + if( pReadonly ){ + fd = pReadonly->fd; + sqlite3_free(pReadonly); + }else{ + fd = robust_open(zName, openFlags, openMode); + } } } if( fd<0 ){ @@ -69879,6 +70219,7 @@ struct IntegrityCk { StrAccum errMsg; /* Accumulate the error message text here */ u32 *heap; /* Min-heap used for analyzing cell coverage */ sqlite3 *db; /* Database connection running the check */ + i64 nRow; /* Number of rows visited in current tree */ }; /* @@ -70353,8 +70694,47 @@ int corruptPageError(int lineno, MemPage *p){ # define SQLITE_CORRUPT_PAGE(pMemPage) SQLITE_CORRUPT_PGNO(pMemPage->pgno) #endif +/* Default value for SHARED_LOCK_TRACE macro if shared-cache is disabled +** or if the lock tracking is disabled. This is always the value for +** release builds. +*/ +#define SHARED_LOCK_TRACE(X,MSG,TAB,TYPE) /*no-op*/ + #ifndef SQLITE_OMIT_SHARED_CACHE +#if 0 +/* ^---- Change to 1 and recompile to enable shared-lock tracing +** for debugging purposes. +** +** Print all shared-cache locks on a BtShared. Debugging use only. +*/ +static void sharedLockTrace( + BtShared *pBt, + const char *zMsg, + int iRoot, + int eLockType +){ + BtLock *pLock; + if( iRoot>0 ){ + printf("%s-%p %u%s:", zMsg, pBt, iRoot, eLockType==READ_LOCK?"R":"W"); + }else{ + printf("%s-%p:", zMsg, pBt); + } + for(pLock=pBt->pLock; pLock; pLock=pLock->pNext){ + printf(" %p/%u%s", pLock->pBtree, pLock->iTable, + pLock->eLock==READ_LOCK ? "R" : "W"); + while( pLock->pNext && pLock->pBtree==pLock->pNext->pBtree ){ + pLock = pLock->pNext; + printf(",%u%s", pLock->iTable, pLock->eLock==READ_LOCK ? "R" : "W"); + } + } + printf("\n"); + fflush(stdout); +} +#undef SHARED_LOCK_TRACE +#define SHARED_LOCK_TRACE(X,MSG,TAB,TYPE) sharedLockTrace(X,MSG,TAB,TYPE) +#endif /* Shared-lock tracing */ + #ifdef SQLITE_DEBUG /* **** This function is only used as part of an assert() statement. *** @@ -70431,6 +70811,8 @@ static int hasSharedCacheTableLock( iTab = iRoot; } + SHARED_LOCK_TRACE(pBtree->pBt,"hasLock",iRoot,eLockType); + /* Search for the required lock. Either a write-lock on root-page iTab, a ** write-lock on the schema table, or (if the client is reading) a ** read-lock on iTab will suffice. Return 1 if any of these are found. */ @@ -70564,6 +70946,8 @@ static int setSharedCacheTableLock(Btree *p, Pgno iTable, u8 eLock){ BtLock *pLock = 0; BtLock *pIter; + SHARED_LOCK_TRACE(pBt,"setLock", iTable, eLock); + assert( sqlite3BtreeHoldsMutex(p) ); assert( eLock==READ_LOCK || eLock==WRITE_LOCK ); assert( p->db!=0 ); @@ -70631,6 +71015,8 @@ static void clearAllSharedCacheTableLocks(Btree *p){ assert( p->sharable || 0==*ppIter ); assert( p->inTrans>0 ); + SHARED_LOCK_TRACE(pBt, "clearAllLocks", 0, 0); + while( *ppIter ){ BtLock *pLock = *ppIter; assert( (pBt->btsFlags & BTS_EXCLUSIVE)==0 || pBt->pWriter==pLock->pBtree ); @@ -70669,6 +71055,9 @@ static void clearAllSharedCacheTableLocks(Btree *p){ */ static void downgradeAllSharedCacheTableLocks(Btree *p){ BtShared *pBt = p->pBt; + + SHARED_LOCK_TRACE(pBt, "downgradeLocks", 0, 0); + if( pBt->pWriter==p ){ BtLock *pLock; pBt->pWriter = 0; @@ -75282,9 +75671,12 @@ static int accessPayload( if( pCur->aOverflow==0 || nOvfl*(int)sizeof(Pgno) > sqlite3MallocSize(pCur->aOverflow) ){ - Pgno *aNew = (Pgno*)sqlite3Realloc( - pCur->aOverflow, nOvfl*2*sizeof(Pgno) - ); + Pgno *aNew; + if( sqlite3FaultSim(413) ){ + aNew = 0; + }else{ + aNew = (Pgno*)sqlite3Realloc(pCur->aOverflow, nOvfl*2*sizeof(Pgno)); + } if( aNew==0 ){ return SQLITE_NOMEM_BKPT; }else{ @@ -75294,6 +75686,12 @@ static int accessPayload( memset(pCur->aOverflow, 0, nOvfl*sizeof(Pgno)); pCur->curFlags |= BTCF_ValidOvfl; }else{ + /* Sanity check the validity of the overflow page cache */ + assert( pCur->aOverflow[0]==nextPage + || pCur->aOverflow[0]==0 + || CORRUPT_DB ); + assert( pCur->aOverflow[0]!=0 || pCur->aOverflow[offset/ovflSize]==0 ); + /* If the overflow page-list cache has been allocated and the ** entry for the first required overflow page is valid, skip ** directly to it. @@ -75775,6 +76173,23 @@ SQLITE_PRIVATE int sqlite3BtreeFirst(BtCursor *pCur, int *pRes){ return rc; } +#ifdef SQLITE_DEBUG +/* The cursors is CURSOR_VALID and has BTCF_AtLast set. Verify that +** this flags are true for a consistent database. +** +** This routine is is called from within assert() statements only. +** It is an internal verification routine and does not appear in production +** builds. +*/ +static int cursorIsAtLastEntry(BtCursor *pCur){ + int ii; + for(ii=0; iiiPage; ii++){ + if( pCur->aiIdx[ii]!=pCur->apPage[ii]->nCell ) return 0; + } + return pCur->ix==pCur->pPage->nCell-1 && pCur->pPage->leaf!=0; +} +#endif + /* Move the cursor to the last entry in the table. Return SQLITE_OK ** on success. Set *pRes to 0 if the cursor actually points to something ** or set *pRes to 1 if the table is empty. @@ -75803,18 +76218,7 @@ SQLITE_PRIVATE int sqlite3BtreeLast(BtCursor *pCur, int *pRes){ /* If the cursor already points to the last entry, this is a no-op. */ if( CURSOR_VALID==pCur->eState && (pCur->curFlags & BTCF_AtLast)!=0 ){ -#ifdef SQLITE_DEBUG - /* This block serves to assert() that the cursor really does point - ** to the last entry in the b-tree. */ - int ii; - for(ii=0; iiiPage; ii++){ - assert( pCur->aiIdx[ii]==pCur->apPage[ii]->nCell ); - } - assert( pCur->ix==pCur->pPage->nCell-1 || CORRUPT_DB ); - testcase( pCur->ix!=pCur->pPage->nCell-1 ); - /* ^-- dbsqlfuzz b92b72e4de80b5140c30ab71372ca719b8feb618 */ - assert( pCur->pPage->leaf ); -#endif + assert( cursorIsAtLastEntry(pCur) || CORRUPT_DB ); *pRes = 0; return SQLITE_OK; } @@ -75867,6 +76271,7 @@ SQLITE_PRIVATE int sqlite3BtreeTableMoveto( } if( pCur->info.nKeycurFlags & BTCF_AtLast)!=0 ){ + assert( cursorIsAtLastEntry(pCur) || CORRUPT_DB ); *pRes = -1; return SQLITE_OK; } @@ -76333,10 +76738,10 @@ SQLITE_PRIVATE i64 sqlite3BtreeRowCountEst(BtCursor *pCur){ assert( cursorOwnsBtShared(pCur) ); assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); - /* Currently this interface is only called by the OP_IfSmaller - ** opcode, and it that case the cursor will always be valid and - ** will always point to a leaf node. */ - if( NEVER(pCur->eState!=CURSOR_VALID) ) return -1; + /* Currently this interface is only called by the OP_IfSizeBetween + ** opcode and the OP_Count opcode with P3=1. In either case, + ** the cursor will always be valid unless the btree is empty. */ + if( pCur->eState!=CURSOR_VALID ) return 0; if( NEVER(pCur->pPage->leaf==0) ) return -1; n = pCur->pPage->nCell; @@ -78467,7 +78872,7 @@ static int balance_nonroot( ** table-interior, index-leaf, or index-interior). */ if( pOld->aData[0]!=apOld[0]->aData[0] ){ - rc = SQLITE_CORRUPT_BKPT; + rc = SQLITE_CORRUPT_PAGE(pOld); goto balance_cleanup; } @@ -78491,7 +78896,7 @@ static int balance_nonroot( memset(&b.szCell[b.nCell], 0, sizeof(b.szCell[0])*(limit+pOld->nOverflow)); if( pOld->nOverflow>0 ){ if( NEVER(limitaiOvfl[0]) ){ - rc = SQLITE_CORRUPT_BKPT; + rc = SQLITE_CORRUPT_PAGE(pOld); goto balance_cleanup; } limit = pOld->aiOvfl[0]; @@ -79134,7 +79539,7 @@ static int anotherValidCursor(BtCursor *pCur){ && pOther->eState==CURSOR_VALID && pOther->pPage==pCur->pPage ){ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_PAGE(pCur->pPage); } } return SQLITE_OK; @@ -79194,7 +79599,7 @@ static int balance(BtCursor *pCur){ /* The page being written is not a root page, and there is currently ** more than one reference to it. This only happens if the page is one ** of its own ancestor pages. Corruption. */ - rc = SQLITE_CORRUPT_BKPT; + rc = SQLITE_CORRUPT_PAGE(pPage); }else{ MemPage * const pParent = pCur->apPage[iPage-1]; int const iIdx = pCur->aiIdx[iPage-1]; @@ -79358,7 +79763,7 @@ static SQLITE_NOINLINE int btreeOverwriteOverflowCell( rc = btreeGetPage(pBt, ovflPgno, &pPage, 0); if( rc ) return rc; if( sqlite3PagerPageRefcount(pPage->pDbPage)!=1 || pPage->isInit ){ - rc = SQLITE_CORRUPT_BKPT; + rc = SQLITE_CORRUPT_PAGE(pPage); }else{ if( iOffset+ovflPageSize<(u32)nTotal ){ ovflPgno = get4byte(pPage->aData); @@ -79386,7 +79791,7 @@ static int btreeOverwriteCell(BtCursor *pCur, const BtreePayload *pX){ if( pCur->info.pPayload + pCur->info.nLocal > pPage->aDataEnd || pCur->info.pPayload < pPage->aData + pPage->cellOffset ){ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_PAGE(pPage); } if( pCur->info.nLocal==nTotal ){ /* The entire cell is local */ @@ -79467,7 +79872,7 @@ SQLITE_PRIVATE int sqlite3BtreeInsert( ** Which can only happen if the SQLITE_NoSchemaError flag was set when ** the schema was loaded. This cannot be asserted though, as a user might ** set the flag, load the schema, and then unset the flag. */ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_PGNO(pCur->pgnoRoot); } } @@ -79590,7 +79995,7 @@ SQLITE_PRIVATE int sqlite3BtreeInsert( if( pPage->nFree<0 ){ if( NEVER(pCur->eState>CURSOR_INVALID) ){ /* ^^^^^--- due to the moveToRoot() call above */ - rc = SQLITE_CORRUPT_BKPT; + rc = SQLITE_CORRUPT_PAGE(pPage); }else{ rc = btreeComputeFreeSpace(pPage); } @@ -79632,7 +80037,7 @@ SQLITE_PRIVATE int sqlite3BtreeInsert( CellInfo info; assert( idx>=0 ); if( idx>=pPage->nCell ){ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_PAGE(pPage); } rc = sqlite3PagerWrite(pPage->pDbPage); if( rc ){ @@ -79659,10 +80064,10 @@ SQLITE_PRIVATE int sqlite3BtreeInsert( ** necessary to add the PTRMAP_OVERFLOW1 pointer-map entry. */ assert( rc==SQLITE_OK ); /* clearCell never fails when nLocal==nPayload */ if( oldCell < pPage->aData+pPage->hdrOffset+10 ){ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_PAGE(pPage); } if( oldCell+szNew > pPage->aDataEnd ){ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_PAGE(pPage); } memcpy(oldCell, newCell, szNew); return SQLITE_OK; @@ -79764,7 +80169,7 @@ SQLITE_PRIVATE int sqlite3BtreeTransferRow(BtCursor *pDest, BtCursor *pSrc, i64 nIn = pSrc->info.nLocal; aIn = pSrc->info.pPayload; if( aIn+nIn>pSrc->pPage->aDataEnd ){ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_PAGE(pSrc->pPage); } nRem = pSrc->info.nPayload; if( nIn==nRem && nInpPage->maxLocal ){ @@ -79789,7 +80194,7 @@ SQLITE_PRIVATE int sqlite3BtreeTransferRow(BtCursor *pDest, BtCursor *pSrc, i64 if( nRem>nIn ){ if( aIn+nIn+4>pSrc->pPage->aDataEnd ){ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_PAGE(pSrc->pPage); } ovflIn = get4byte(&pSrc->info.pPayload[nIn]); } @@ -79885,7 +80290,7 @@ SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor *pCur, u8 flags){ assert( rc!=SQLITE_OK || CORRUPT_DB || pCur->eState==CURSOR_VALID ); if( rc || pCur->eState!=CURSOR_VALID ) return rc; }else{ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_PGNO(pCur->pgnoRoot); } } assert( pCur->eState==CURSOR_VALID ); @@ -79894,14 +80299,14 @@ SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor *pCur, u8 flags){ iCellIdx = pCur->ix; pPage = pCur->pPage; if( pPage->nCell<=iCellIdx ){ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_PAGE(pPage); } pCell = findCell(pPage, iCellIdx); if( pPage->nFree<0 && btreeComputeFreeSpace(pPage) ){ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_PAGE(pPage); } if( pCell<&pPage->aCellIdx[pPage->nCell] ){ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_PAGE(pPage); } /* If the BTREE_SAVEPOSITION bit is on, then the cursor position must @@ -79992,7 +80397,7 @@ SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor *pCur, u8 flags){ n = pCur->pPage->pgno; } pCell = findCell(pLeaf, pLeaf->nCell-1); - if( pCell<&pLeaf->aData[4] ) return SQLITE_CORRUPT_BKPT; + if( pCell<&pLeaf->aData[4] ) return SQLITE_CORRUPT_PAGE(pLeaf); nCell = pLeaf->xCellSize(pLeaf, pCell); assert( MX_CELL_SIZE(pBt) >= nCell ); pTmp = pBt->pTmpSpace; @@ -80108,7 +80513,7 @@ static int btreeCreateTable(Btree *p, Pgno *piTable, int createTabFlags){ */ sqlite3BtreeGetMeta(p, BTREE_LARGEST_ROOT_PAGE, &pgnoRoot); if( pgnoRoot>btreePagecount(pBt) ){ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_PGNO(pgnoRoot); } pgnoRoot++; @@ -80156,7 +80561,7 @@ static int btreeCreateTable(Btree *p, Pgno *piTable, int createTabFlags){ } rc = ptrmapGet(pBt, pgnoRoot, &eType, &iPtrPage); if( eType==PTRMAP_ROOTPAGE || eType==PTRMAP_FREEPAGE ){ - rc = SQLITE_CORRUPT_BKPT; + rc = SQLITE_CORRUPT_PGNO(pgnoRoot); } if( rc!=SQLITE_OK ){ releasePage(pRoot); @@ -80246,14 +80651,14 @@ static int clearDatabasePage( assert( sqlite3_mutex_held(pBt->mutex) ); if( pgno>btreePagecount(pBt) ){ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_PGNO(pgno); } rc = getAndInitPage(pBt, pgno, &pPage, 0); if( rc ) return rc; if( (pBt->openFlags & BTREE_SINGLE)==0 && sqlite3PagerPageRefcount(pPage->pDbPage) != (1 + (pgno==1)) ){ - rc = SQLITE_CORRUPT_BKPT; + rc = SQLITE_CORRUPT_PAGE(pPage); goto cleardatabasepage_out; } hdr = pPage->hdrOffset; @@ -80357,7 +80762,7 @@ static int btreeDropTable(Btree *p, Pgno iTable, int *piMoved){ assert( p->inTrans==TRANS_WRITE ); assert( iTable>=2 ); if( iTable>btreePagecount(pBt) ){ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_PGNO(iTable); } rc = sqlite3BtreeClearTable(p, iTable, 0); @@ -80951,6 +81356,9 @@ static int checkTreePage( ** number of cells on the page. */ nCell = get2byte(&data[hdr+3]); assert( pPage->nCell==nCell ); + if( pPage->leaf || pPage->intKey==0 ){ + pCheck->nRow += nCell; + } /* EVIDENCE-OF: R-23882-45353 The cell pointer array of a b-tree page ** immediately follows the b-tree page header. */ @@ -81062,6 +81470,7 @@ static int checkTreePage( btreeHeapInsert(heap, (pc<<16)|(pc+size-1)); } } + assert( heap!=0 ); /* Add the freeblocks to the min-heap ** ** EVIDENCE-OF: R-20690-50594 The second field of the b-tree page header @@ -81161,6 +81570,7 @@ SQLITE_PRIVATE int sqlite3BtreeIntegrityCheck( sqlite3 *db, /* Database connection that is running the check */ Btree *p, /* The btree to be checked */ Pgno *aRoot, /* An array of root pages numbers for individual trees */ + Mem *aCnt, /* Memory cells to write counts for each tree to */ int nRoot, /* Number of entries in aRoot[] */ int mxErr, /* Stop reporting errors after this many */ int *pnErr, /* OUT: Write number of errors seen to this variable */ @@ -81174,7 +81584,9 @@ SQLITE_PRIVATE int sqlite3BtreeIntegrityCheck( int bPartial = 0; /* True if not checking all btrees */ int bCkFreelist = 1; /* True to scan the freelist */ VVA_ONLY( int nRef ); + assert( nRoot>0 ); + assert( aCnt!=0 ); /* aRoot[0]==0 means this is a partial check */ if( aRoot[0]==0 ){ @@ -81247,15 +81659,18 @@ SQLITE_PRIVATE int sqlite3BtreeIntegrityCheck( testcase( pBt->db->flags & SQLITE_CellSizeCk ); pBt->db->flags &= ~(u64)SQLITE_CellSizeCk; for(i=0; (int)iautoVacuum && aRoot[i]>1 && !bPartial ){ - checkPtrmap(&sCheck, aRoot[i], PTRMAP_ROOTPAGE, 0); - } + if( pBt->autoVacuum && aRoot[i]>1 && !bPartial ){ + checkPtrmap(&sCheck, aRoot[i], PTRMAP_ROOTPAGE, 0); + } #endif - sCheck.v0 = aRoot[i]; - checkTreePage(&sCheck, aRoot[i], ¬Used, LARGEST_INT64); + sCheck.v0 = aRoot[i]; + checkTreePage(&sCheck, aRoot[i], ¬Used, LARGEST_INT64); + } + sqlite3MemSetArrayInt64(aCnt, i, sCheck.nRow); } pBt->db->flags = savedDbFlags; @@ -83310,6 +83725,13 @@ SQLITE_PRIVATE void sqlite3VdbeMemSetInt64(Mem *pMem, i64 val){ } } +/* +** Set the iIdx'th entry of array aMem[] to contain integer value val. +*/ +SQLITE_PRIVATE void sqlite3MemSetArrayInt64(sqlite3_value *aMem, int iIdx, i64 val){ + sqlite3VdbeMemSetInt64(&aMem[iIdx], val); +} + /* A no-op destructor */ SQLITE_PRIVATE void sqlite3NoopDestructor(void *p){ UNUSED_PARAMETER(p); } @@ -83998,14 +84420,20 @@ static int valueFromExpr( } /* Handle negative integers in a single step. This is needed in the - ** case when the value is -9223372036854775808. - */ - if( op==TK_UMINUS - && (pExpr->pLeft->op==TK_INTEGER || pExpr->pLeft->op==TK_FLOAT) ){ - pExpr = pExpr->pLeft; - op = pExpr->op; - negInt = -1; - zNeg = "-"; + ** case when the value is -9223372036854775808. Except - do not do this + ** for hexadecimal literals. */ + if( op==TK_UMINUS ){ + Expr *pLeft = pExpr->pLeft; + if( (pLeft->op==TK_INTEGER || pLeft->op==TK_FLOAT) ){ + if( ExprHasProperty(pLeft, EP_IntValue) + || pLeft->u.zToken[0]!='0' || (pLeft->u.zToken[1] & ~0x20)!='X' + ){ + pExpr = pLeft; + op = pExpr->op; + negInt = -1; + zNeg = "-"; + } + } } if( op==TK_STRING || op==TK_FLOAT || op==TK_INTEGER ){ @@ -84014,12 +84442,26 @@ static int valueFromExpr( if( ExprHasProperty(pExpr, EP_IntValue) ){ sqlite3VdbeMemSetInt64(pVal, (i64)pExpr->u.iValue*negInt); }else{ - zVal = sqlite3MPrintf(db, "%s%s", zNeg, pExpr->u.zToken); - if( zVal==0 ) goto no_mem; - sqlite3ValueSetStr(pVal, -1, zVal, SQLITE_UTF8, SQLITE_DYNAMIC); + i64 iVal; + if( op==TK_INTEGER && 0==sqlite3DecOrHexToI64(pExpr->u.zToken, &iVal) ){ + sqlite3VdbeMemSetInt64(pVal, iVal*negInt); + }else{ + zVal = sqlite3MPrintf(db, "%s%s", zNeg, pExpr->u.zToken); + if( zVal==0 ) goto no_mem; + sqlite3ValueSetStr(pVal, -1, zVal, SQLITE_UTF8, SQLITE_DYNAMIC); + } } - if( (op==TK_INTEGER || op==TK_FLOAT ) && affinity==SQLITE_AFF_BLOB ){ - sqlite3ValueApplyAffinity(pVal, SQLITE_AFF_NUMERIC, SQLITE_UTF8); + if( affinity==SQLITE_AFF_BLOB ){ + if( op==TK_FLOAT ){ + assert( pVal && pVal->z && pVal->flags==(MEM_Str|MEM_Term) ); + sqlite3AtoF(pVal->z, &pVal->u.r, pVal->n, SQLITE_UTF8); + pVal->flags = MEM_Real; + }else if( op==TK_INTEGER ){ + /* This case is required by -9223372036854775808 and other strings + ** that look like integers but cannot be handled by the + ** sqlite3DecOrHexToI64() call above. */ + sqlite3ValueApplyAffinity(pVal, SQLITE_AFF_NUMERIC, SQLITE_UTF8); + } }else{ sqlite3ValueApplyAffinity(pVal, affinity, SQLITE_UTF8); } @@ -84289,17 +84731,17 @@ SQLITE_PRIVATE int sqlite3Stat4Column( sqlite3_value **ppVal /* OUT: Extracted value */ ){ u32 t = 0; /* a column type code */ - int nHdr; /* Size of the header in the record */ - int iHdr; /* Next unread header byte */ - int iField; /* Next unread data byte */ - int szField = 0; /* Size of the current data field */ + u32 nHdr; /* Size of the header in the record */ + u32 iHdr; /* Next unread header byte */ + i64 iField; /* Next unread data byte */ + u32 szField = 0; /* Size of the current data field */ int i; /* Column index */ u8 *a = (u8*)pRec; /* Typecast byte array */ Mem *pMem = *ppVal; /* Write result into this Mem object */ assert( iCol>0 ); iHdr = getVarint32(a, nHdr); - if( nHdr>nRec || iHdr>=nHdr ) return SQLITE_CORRUPT_BKPT; + if( nHdr>(u32)nRec || iHdr>=nHdr ) return SQLITE_CORRUPT_BKPT; iField = nHdr; for(i=0; i<=iCol; i++){ iHdr += getVarint32(&a[iHdr], t); @@ -85334,6 +85776,15 @@ static void resolveP2Values(Vdbe *p, int *pMaxFuncArgs){ assert( aLabel!=0 ); /* True because of tag-20230419-1 */ pOp->p2 = aLabel[ADDR(pOp->p2)]; } + + /* OPFLG_JUMP opcodes never have P2==0, though OPFLG_JUMP0 opcodes + ** might */ + assert( pOp->p2>0 + || (sqlite3OpcodeProperty[pOp->opcode] & OPFLG_JUMP0)!=0 ); + + /* Jumps never go off the end of the bytecode array */ + assert( pOp->p2nOp + || (sqlite3OpcodeProperty[pOp->opcode] & OPFLG_JUMP)==0 ); break; } } @@ -87741,7 +88192,7 @@ SQLITE_PRIVATE int sqlite3VdbeHalt(Vdbe *p){ /* Check for immediate foreign key violations. */ if( p->rc==SQLITE_OK || (p->errorAction==OE_Fail && !isSpecialError) ){ - sqlite3VdbeCheckFk(p, 0); + (void)sqlite3VdbeCheckFk(p, 0); } /* If the auto-commit flag is set and this is the only active writer @@ -88911,17 +89362,15 @@ SQLITE_PRIVATE int sqlite3IntFloatCompare(i64 i, double r){ return (xr); }else{ i64 y; - double s; if( r<-9223372036854775808.0 ) return +1; if( r>=9223372036854775808.0 ) return -1; y = (i64)r; if( iy ) return +1; - s = (double)i; - testcase( doubleLt(s,r) ); - testcase( doubleLt(r,s) ); - testcase( doubleEq(r,s) ); - return (sr); + testcase( doubleLt(((double)i),r) ); + testcase( doubleLt(r,((double)i)) ); + testcase( doubleEq(r,((double)i)) ); + return (((double)i)r); } } @@ -92329,7 +92778,6 @@ SQLITE_API int sqlite3_stmt_scanstatus_v2( } if( flags & SQLITE_SCANSTAT_COMPLEX ){ idx = iScan; - pScan = &p->aScan[idx]; }else{ /* If the COMPLEX flag is clear, then this function must ignore any ** ScanStatus structures with ScanStatus.addrLoop set to 0. */ @@ -92342,6 +92790,8 @@ SQLITE_API int sqlite3_stmt_scanstatus_v2( } } if( idx>=p->nScan ) return 1; + assert( pScan==0 || pScan==&p->aScan[idx] ); + pScan = &p->aScan[idx]; switch( iScanStatusOp ){ case SQLITE_SCANSTAT_NLOOP: { @@ -93790,7 +94240,7 @@ case OP_Return: { /* in1 */ ** ** See also: EndCoroutine */ -case OP_InitCoroutine: { /* jump */ +case OP_InitCoroutine: { /* jump0 */ assert( pOp->p1>0 && pOp->p1<=(p->nMem+1 - p->nCursor) ); assert( pOp->p2>=0 && pOp->p2nOp ); assert( pOp->p3>=0 && pOp->p3nOp ); @@ -93813,7 +94263,9 @@ jump_to_p2: ** ** The instruction at the address in register P1 is a Yield. ** Jump to the P2 parameter of that Yield. -** After the jump, register P1 becomes undefined. +** After the jump, the value register P1 is left with a value +** such that subsequent OP_Yields go back to the this same +** OP_EndCoroutine instruction. ** ** See also: InitCoroutine */ @@ -93825,8 +94277,8 @@ case OP_EndCoroutine: { /* in1 */ pCaller = &aOp[pIn1->u.i]; assert( pCaller->opcode==OP_Yield ); assert( pCaller->p2>=0 && pCaller->p2nOp ); + pIn1->u.i = (int)(pOp - p->aOp) - 1; pOp = &aOp[pCaller->p2 - 1]; - pIn1->flags = MEM_Undefined; break; } @@ -93843,7 +94295,7 @@ case OP_EndCoroutine: { /* in1 */ ** ** See also: InitCoroutine */ -case OP_Yield: { /* in1, jump */ +case OP_Yield: { /* in1, jump0 */ int pcDest; pIn1 = &aMem[pOp->p1]; assert( VdbeMemDynamic(pIn1)==0 ); @@ -94173,19 +94625,15 @@ case OP_Blob: { /* out2 */ break; } -/* Opcode: Variable P1 P2 * P4 * -** Synopsis: r[P2]=parameter(P1,P4) +/* Opcode: Variable P1 P2 * * * +** Synopsis: r[P2]=parameter(P1) ** ** Transfer the values of bound parameter P1 into register P2 -** -** If the parameter is named, then its name appears in P4. -** The P4 value is used by sqlite3_bind_parameter_name(). */ case OP_Variable: { /* out2 */ Mem *pVar; /* Value being transferred */ assert( pOp->p1>0 && pOp->p1<=p->nVar ); - assert( pOp->p4.z==0 || pOp->p4.z==sqlite3VListNumToName(p->pVList,pOp->p1) ); pVar = &p->aVar[pOp->p1 - 1]; if( sqlite3VdbeMemTooBig(pVar) ){ goto too_big; @@ -94706,7 +95154,7 @@ case OP_AddImm: { /* in1 */ ** without data loss, then jump immediately to P2, or if P2==0 ** raise an SQLITE_MISMATCH exception. */ -case OP_MustBeInt: { /* jump, in1 */ +case OP_MustBeInt: { /* jump0, in1 */ pIn1 = &aMem[pOp->p1]; if( (pIn1->flags & MEM_Int)==0 ){ applyAffinity(pIn1, SQLITE_AFF_NUMERIC, encoding); @@ -94747,7 +95195,7 @@ case OP_RealAffinity: { /* in1 */ } #endif -#ifndef SQLITE_OMIT_CAST +#if !defined(SQLITE_OMIT_CAST) && !defined(SQLITE_OMIT_ANALYZE) /* Opcode: Cast P1 P2 * * * ** Synopsis: affinity(r[P1]) ** @@ -96319,11 +96767,16 @@ case OP_MakeRecord: { switch( len ){ default: zPayload[7] = (u8)(v&0xff); v >>= 8; zPayload[6] = (u8)(v&0xff); v >>= 8; + /* no break */ deliberate_fall_through case 6: zPayload[5] = (u8)(v&0xff); v >>= 8; zPayload[4] = (u8)(v&0xff); v >>= 8; + /* no break */ deliberate_fall_through case 4: zPayload[3] = (u8)(v&0xff); v >>= 8; + /* no break */ deliberate_fall_through case 3: zPayload[2] = (u8)(v&0xff); v >>= 8; + /* no break */ deliberate_fall_through case 2: zPayload[1] = (u8)(v&0xff); v >>= 8; + /* no break */ deliberate_fall_through case 1: zPayload[0] = (u8)(v&0xff); } zPayload += len; @@ -97242,7 +97695,8 @@ case OP_SequenceTest: { ** is the only cursor opcode that works with a pseudo-table. ** ** P3 is the number of fields in the records that will be stored by -** the pseudo-table. +** the pseudo-table. If P2 is 0 or negative then the pseudo-cursor +** will return NULL for every column. */ case OP_OpenPseudo: { VdbeCursor *pCx; @@ -97385,10 +97839,10 @@ case OP_ColumnsUsed: { ** ** See also: Found, NotFound, SeekGt, SeekGe, SeekLt */ -case OP_SeekLT: /* jump, in3, group, ncycle */ -case OP_SeekLE: /* jump, in3, group, ncycle */ -case OP_SeekGE: /* jump, in3, group, ncycle */ -case OP_SeekGT: { /* jump, in3, group, ncycle */ +case OP_SeekLT: /* jump0, in3, group, ncycle */ +case OP_SeekLE: /* jump0, in3, group, ncycle */ +case OP_SeekGE: /* jump0, in3, group, ncycle */ +case OP_SeekGT: { /* jump0, in3, group, ncycle */ int res; /* Comparison result */ int oc; /* Opcode */ VdbeCursor *pC; /* The cursor to seek */ @@ -98055,7 +98509,7 @@ case OP_Found: { /* jump, in3, ncycle */ ** ** See also: Found, NotFound, NoConflict, SeekRowid */ -case OP_SeekRowid: { /* jump, in3, ncycle */ +case OP_SeekRowid: { /* jump0, in3, ncycle */ VdbeCursor *pC; BtCursor *pCrsr; int res; @@ -98814,7 +99268,7 @@ case OP_NullRow: { ** configured to use Prev, not Next. */ case OP_SeekEnd: /* ncycle */ -case OP_Last: { /* jump, ncycle */ +case OP_Last: { /* jump0, ncycle */ VdbeCursor *pC; BtCursor *pCrsr; int res; @@ -98848,28 +99302,38 @@ case OP_Last: { /* jump, ncycle */ break; } -/* Opcode: IfSmaller P1 P2 P3 * * +/* Opcode: IfSizeBetween P1 P2 P3 P4 * ** -** Estimate the number of rows in the table P1. Jump to P2 if that -** estimate is less than approximately 2**(0.1*P3). +** Let N be the approximate number of rows in the table or index +** with cursor P1 and let X be 10*log2(N) if N is positive or -1 +** if N is zero. +** +** Jump to P2 if X is in between P3 and P4, inclusive. */ -case OP_IfSmaller: { /* jump */ +case OP_IfSizeBetween: { /* jump */ VdbeCursor *pC; BtCursor *pCrsr; int res; i64 sz; assert( pOp->p1>=0 && pOp->p1nCursor ); + assert( pOp->p4type==P4_INT32 ); + assert( pOp->p3>=-1 && pOp->p3<=640*2 ); + assert( pOp->p4.i>=-1 && pOp->p4.i<=640*2 ); pC = p->apCsr[pOp->p1]; assert( pC!=0 ); pCrsr = pC->uc.pCursor; assert( pCrsr ); rc = sqlite3BtreeFirst(pCrsr, &res); if( rc ) goto abort_due_to_error; - if( res==0 ){ + if( res!=0 ){ + sz = -1; /* -Infinity encoding */ + }else{ sz = sqlite3BtreeRowCountEst(pCrsr); - if( ALWAYS(sz>=0) && sqlite3LogEst((u64)sz)p3 ) res = 1; + assert( sz>0 ); + sz = sqlite3LogEst((u64)sz); } + res = sz>=pOp->p3 && sz<=pOp->p4.i; VdbeBranchTaken(res!=0,2); if( res ) goto jump_to_p2; break; @@ -98922,7 +99386,7 @@ case OP_Sort: { /* jump ncycle */ ** from the beginning toward the end. In other words, the cursor is ** configured to use Next, not Prev. */ -case OP_Rewind: { /* jump, ncycle */ +case OP_Rewind: { /* jump0, ncycle */ VdbeCursor *pC; BtCursor *pCrsr; int res; @@ -99569,11 +100033,18 @@ case OP_CreateBtree: { /* out2 */ break; } -/* Opcode: SqlExec * * * P4 * +/* Opcode: SqlExec P1 P2 * P4 * ** ** Run the SQL statement or statements specified in the P4 string. -** Disable Auth and Trace callbacks while those statements are running if -** P1 is true. +** +** The P1 parameter is a bitmask of options: +** +** 0x0001 Disable Auth and Trace callbacks while the statements +** in P4 are running. +** +** 0x0002 Set db->nAnalysisLimit to P2 while the statements in +** P4 are running. +** */ case OP_SqlExec: { char *zErr; @@ -99581,6 +100052,7 @@ case OP_SqlExec: { sqlite3_xauth xAuth; #endif u8 mTrace; + int savedAnalysisLimit; sqlite3VdbeIncrWriteCounter(p, 0); db->nSqlExec++; @@ -99589,18 +100061,23 @@ case OP_SqlExec: { xAuth = db->xAuth; #endif mTrace = db->mTrace; - if( pOp->p1 ){ + savedAnalysisLimit = db->nAnalysisLimit; + if( pOp->p1 & 0x0001 ){ #ifndef SQLITE_OMIT_AUTHORIZATION db->xAuth = 0; #endif db->mTrace = 0; } + if( pOp->p1 & 0x0002 ){ + db->nAnalysisLimit = pOp->p2; + } rc = sqlite3_exec(db, pOp->p4.z, 0, 0, &zErr); db->nSqlExec--; #ifndef SQLITE_OMIT_AUTHORIZATION db->xAuth = xAuth; #endif db->mTrace = mTrace; + db->nAnalysisLimit = savedAnalysisLimit; if( zErr || rc ){ sqlite3VdbeError(p, "%s", zErr); sqlite3_free(zErr); @@ -99752,11 +100229,11 @@ case OP_DropTrigger: { /* Opcode: IntegrityCk P1 P2 P3 P4 P5 ** ** Do an analysis of the currently open database. Store in -** register P1 the text of an error message describing any problems. -** If no problems are found, store a NULL in register P1. +** register (P1+1) the text of an error message describing any problems. +** If no problems are found, store a NULL in register (P1+1). ** -** The register P3 contains one less than the maximum number of allowed errors. -** At most reg(P3) errors will be reported. +** The register (P1) contains one less than the maximum number of allowed +** errors. At most reg(P1) errors will be reported. ** In other words, the analysis stops as soon as reg(P1) errors are ** seen. Reg(P1) is updated with the number of errors remaining. ** @@ -99776,19 +100253,21 @@ case OP_IntegrityCk: { Mem *pnErr; /* Register keeping track of errors remaining */ assert( p->bIsReader ); + assert( pOp->p4type==P4_INTARRAY ); nRoot = pOp->p2; aRoot = pOp->p4.ai; assert( nRoot>0 ); + assert( aRoot!=0 ); assert( aRoot[0]==(Pgno)nRoot ); - assert( pOp->p3>0 && pOp->p3<=(p->nMem+1 - p->nCursor) ); - pnErr = &aMem[pOp->p3]; + assert( pOp->p1>0 && (pOp->p1+1)<=(p->nMem+1 - p->nCursor) ); + pnErr = &aMem[pOp->p1]; assert( (pnErr->flags & MEM_Int)!=0 ); assert( (pnErr->flags & (MEM_Str|MEM_Blob))==0 ); - pIn1 = &aMem[pOp->p1]; + pIn1 = &aMem[pOp->p1+1]; assert( pOp->p5nDb ); assert( DbMaskTest(p->btreeMask, pOp->p5) ); - rc = sqlite3BtreeIntegrityCheck(db, db->aDb[pOp->p5].pBt, &aRoot[1], nRoot, - (int)pnErr->u.i+1, &nErr, &z); + rc = sqlite3BtreeIntegrityCheck(db, db->aDb[pOp->p5].pBt, &aRoot[1], + &aMem[pOp->p3], nRoot, (int)pnErr->u.i+1, &nErr, &z); sqlite3VdbeMemSetNull(pIn1); if( nErr==0 ){ assert( z==0 ); @@ -99915,7 +100394,9 @@ case OP_RowSetTest: { /* jump, in1, in3 */ ** P1 contains the address of the memory cell that contains the first memory ** cell in an array of values used as arguments to the sub-program. P2 ** contains the address to jump to if the sub-program throws an IGNORE -** exception using the RAISE() function. Register P3 contains the address +** exception using the RAISE() function. P2 might be zero, if there is +** no possibility that an IGNORE exception will be raised. +** Register P3 contains the address ** of a memory cell in this (the parent) VM that is used to allocate the ** memory required by the sub-vdbe at runtime. ** @@ -99923,7 +100404,7 @@ case OP_RowSetTest: { /* jump, in1, in3 */ ** ** If P5 is non-zero, then recursive program invocation is enabled. */ -case OP_Program: { /* jump */ +case OP_Program: { /* jump0 */ int nMem; /* Number of memory registers for sub-program */ int nByte; /* Bytes of runtime space required for sub-program */ Mem *pRt; /* Register to allocate runtime space */ @@ -101472,7 +101953,7 @@ case OP_Filter: { /* jump */ ** error is encountered. */ case OP_Trace: -case OP_Init: { /* jump */ +case OP_Init: { /* jump0 */ int i; #ifndef SQLITE_OMIT_TRACE char *zTrace; @@ -105373,10 +105854,10 @@ static int bytecodevtabColumn( #ifdef SQLITE_ENABLE_STMT_SCANSTATUS case 9: /* nexec */ - sqlite3_result_int(ctx, pOp->nExec); + sqlite3_result_int64(ctx, pOp->nExec); break; case 10: /* ncycle */ - sqlite3_result_int(ctx, pOp->nCycle); + sqlite3_result_int64(ctx, pOp->nCycle); break; #else case 9: /* nexec */ @@ -106520,7 +107001,7 @@ static int lookupName( Parse *pParse, /* The parsing context */ const char *zDb, /* Name of the database containing table, or NULL */ const char *zTab, /* Name of table containing column, or NULL */ - const char *zCol, /* Name of the column. */ + const Expr *pRight, /* Name of the column. */ NameContext *pNC, /* The name context used to resolve the name */ Expr *pExpr /* Make this EXPR node point to the selected column */ ){ @@ -106537,6 +107018,7 @@ static int lookupName( Table *pTab = 0; /* Table holding the row */ Column *pCol; /* A column of pTab */ ExprList *pFJMatch = 0; /* Matches for FULL JOIN .. USING */ + const char *zCol = pRight->u.zToken; assert( pNC ); /* the name context cannot be NULL. */ assert( zCol ); /* The Z in X.Y.Z cannot be NULL */ @@ -106768,7 +107250,8 @@ static int lookupName( if( pParse->bReturning ){ if( (pNC->ncFlags & NC_UBaseReg)!=0 && ALWAYS(zTab==0 - || sqlite3StrICmp(zTab,pParse->pTriggerTab->zName)==0) + || sqlite3StrICmp(zTab,pParse->pTriggerTab->zName)==0 + || isValidSchemaTableName(zTab, pParse->pTriggerTab, 0)) ){ pExpr->iTable = op!=TK_DELETE; pTab = pParse->pTriggerTab; @@ -106872,6 +107355,11 @@ static int lookupName( && ALWAYS(VisibleRowid(pMatch->pTab) || pMatch->fg.isNestedFrom) ){ cnt = cntTab; +#if SQLITE_ALLOW_ROWID_IN_VIEW+0==2 + if( pMatch->pTab!=0 && IsView(pMatch->pTab) ){ + eNewExprOp = TK_NULL; + } +#endif if( pMatch->fg.isNestedFrom==0 ) pExpr->iColumn = -1; pExpr->affExpr = SQLITE_AFF_INTEGER; } @@ -107025,6 +107513,10 @@ static int lookupName( sqlite3ErrorMsg(pParse, "%s: %s.%s.%s", zErr, zDb, zTab, zCol); }else if( zTab ){ sqlite3ErrorMsg(pParse, "%s: %s.%s", zErr, zTab, zCol); + }else if( cnt==0 && ExprHasProperty(pRight,EP_DblQuoted) ){ + sqlite3ErrorMsg(pParse, "%s: \"%s\" - should this be a" + " string literal in single-quotes?", + zErr, zCol); }else{ sqlite3ErrorMsg(pParse, "%s: %s", zErr, zCol); } @@ -107058,8 +107550,12 @@ static int lookupName( ** If a generated column is referenced, set bits for every column ** of the table. */ - if( pExpr->iColumn>=0 && cnt==1 && pMatch!=0 ){ - pMatch->colUsed |= sqlite3ExprColUsed(pExpr); + if( pMatch ){ + if( pExpr->iColumn>=0 ){ + pMatch->colUsed |= sqlite3ExprColUsed(pExpr); + }else{ + pMatch->fg.rowidUsed = 1; + } } pExpr->op = eNewExprOp; @@ -107302,7 +107798,6 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ */ case TK_ID: case TK_DOT: { - const char *zColumn; const char *zTable; const char *zDb; Expr *pRight; @@ -107311,7 +107806,7 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ zDb = 0; zTable = 0; assert( !ExprHasProperty(pExpr, EP_IntValue) ); - zColumn = pExpr->u.zToken; + pRight = pExpr; }else{ Expr *pLeft = pExpr->pLeft; testcase( pNC->ncFlags & NC_IdxExpr ); @@ -107330,14 +107825,13 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ } assert( ExprUseUToken(pLeft) && ExprUseUToken(pRight) ); zTable = pLeft->u.zToken; - zColumn = pRight->u.zToken; assert( ExprUseYTab(pExpr) ); if( IN_RENAME_OBJECT ){ sqlite3RenameTokenRemap(pParse, (void*)pExpr, (void*)pRight); sqlite3RenameTokenRemap(pParse, (void*)&pExpr->y.pTab, (void*)pLeft); } } - return lookupName(pParse, zDb, zTable, zColumn, pNC, pExpr); + return lookupName(pParse, zDb, zTable, pRight, pNC, pExpr); } /* Resolve function names @@ -107513,11 +108007,9 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ #endif } } -#ifndef SQLITE_OMIT_WINDOWFUNC - else if( ExprHasProperty(pExpr, EP_WinFunc) ){ + else if( ExprHasProperty(pExpr, EP_WinFunc) || pExpr->pLeft ){ is_agg = 1; } -#endif sqlite3WalkExprList(pWalker, pList); if( is_agg ){ if( pExpr->pLeft ){ @@ -107587,6 +108079,7 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ testcase( pNC->ncFlags & NC_PartIdx ); testcase( pNC->ncFlags & NC_IdxExpr ); testcase( pNC->ncFlags & NC_GenCol ); + assert( pExpr->x.pSelect ); if( pNC->ncFlags & NC_SelfRef ){ notValidImpl(pParse, pNC, "subqueries", pExpr, pExpr); }else{ @@ -107595,6 +108088,7 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ assert( pNC->nRef>=nRef ); if( nRef!=pNC->nRef ){ ExprSetProperty(pExpr, EP_VarSelect); + pExpr->x.pSelect->selFlags |= SF_Correlated; } pNC->ncFlags |= NC_Subquery; } @@ -108120,6 +108614,7 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ if( pOuterNC ) pOuterNC->nNestedSelect++; for(i=0; ipSrc->nSrc; i++){ SrcItem *pItem = &p->pSrc->a[i]; + assert( pItem->zName!=0 || pItem->pSelect!=0 );/* Test of tag-20240424-1*/ if( pItem->pSelect && (pItem->pSelect->selFlags & SF_Resolved)==0 ){ int nRef = pOuterNC ? pOuterNC->nRef : 0; const char *zSavedContext = pParse->zAuthContext; @@ -109426,11 +109921,12 @@ SQLITE_PRIVATE void sqlite3ExprSetErrorOffset(Expr *pExpr, int iOfst){ ** appear to be quoted. If the quotes were of the form "..." (double-quotes) ** then the EP_DblQuoted flag is set on the expression node. ** -** Special case: If op==TK_INTEGER and pToken points to a string that -** can be translated into a 32-bit integer, then the token is not -** stored in u.zToken. Instead, the integer values is written -** into u.iValue and the EP_IntValue flag is set. No extra storage +** Special case (tag-20240227-a): If op==TK_INTEGER and pToken points to +** a string that can be translated into a 32-bit integer, then the token is +** not stored in u.zToken. Instead, the integer values is written +** into u.iValue and the EP_IntValue flag is set. No extra storage ** is allocated to hold the integer text and the dequote flag is ignored. +** See also tag-20240227-b. */ SQLITE_PRIVATE Expr *sqlite3ExprAlloc( sqlite3 *db, /* Handle for sqlite3DbMallocRawNN() */ @@ -109446,7 +109942,7 @@ SQLITE_PRIVATE Expr *sqlite3ExprAlloc( if( pToken ){ if( op!=TK_INTEGER || pToken->z==0 || sqlite3GetInt32(pToken->z, &iValue)==0 ){ - nExtra = pToken->n+1; + nExtra = pToken->n+1; /* tag-20240227-a */ assert( iValue>=0 ); } } @@ -109878,6 +110374,7 @@ SQLITE_PRIVATE void sqlite3ExprAssignVarNumber(Parse *pParse, Expr *pExpr, u32 n static SQLITE_NOINLINE void sqlite3ExprDeleteNN(sqlite3 *db, Expr *p){ assert( p!=0 ); assert( db!=0 ); +exprDeleteRestart: assert( !ExprUseUValue(p) || p->u.iValue>=0 ); assert( !ExprUseYWin(p) || !ExprUseYSub(p) ); assert( !ExprUseYWin(p) || p->y.pWin!=0 || db->mallocFailed ); @@ -109893,7 +110390,6 @@ static SQLITE_NOINLINE void sqlite3ExprDeleteNN(sqlite3 *db, Expr *p){ if( !ExprHasProperty(p, (EP_TokenOnly|EP_Leaf)) ){ /* The Expr.x union is never used at the same time as Expr.pRight */ assert( (ExprUseXList(p) && p->x.pList==0) || p->pRight==0 ); - if( p->pLeft && p->op!=TK_SELECT_COLUMN ) sqlite3ExprDeleteNN(db, p->pLeft); if( p->pRight ){ assert( !ExprHasProperty(p, EP_WinFunc) ); sqlite3ExprDeleteNN(db, p->pRight); @@ -109908,6 +110404,19 @@ static SQLITE_NOINLINE void sqlite3ExprDeleteNN(sqlite3 *db, Expr *p){ } #endif } + if( p->pLeft && p->op!=TK_SELECT_COLUMN ){ + Expr *pLeft = p->pLeft; + if( !ExprHasProperty(p, EP_Static) + && !ExprHasProperty(pLeft, EP_Static) + ){ + /* Avoid unnecessary recursion on unary operators */ + sqlite3DbNNFreeNN(db, p); + p = pLeft; + goto exprDeleteRestart; + }else{ + sqlite3ExprDeleteNN(db, pLeft); + } + } } if( !ExprHasProperty(p, EP_Static) ){ sqlite3DbNNFreeNN(db, p); @@ -109940,11 +110449,11 @@ SQLITE_PRIVATE void sqlite3ClearOnOrUsing(sqlite3 *db, OnOrUsing *p){ ** ** The pExpr might be deleted immediately on an OOM error. ** -** The deferred delete is (currently) implemented by adding the -** pExpr to the pParse->pConstExpr list with a register number of 0. +** Return 0 if the delete was successfully deferred. Return non-zero +** if the delete happened immediately because of an OOM. */ -SQLITE_PRIVATE void sqlite3ExprDeferredDelete(Parse *pParse, Expr *pExpr){ - sqlite3ParserAddCleanup(pParse, sqlite3ExprDeleteGeneric, pExpr); +SQLITE_PRIVATE int sqlite3ExprDeferredDelete(Parse *pParse, Expr *pExpr){ + return 0==sqlite3ParserAddCleanup(pParse, sqlite3ExprDeleteGeneric, pExpr); } /* Invoke sqlite3RenameExprUnmap() and sqlite3ExprDelete() on the @@ -110380,17 +110889,19 @@ SQLITE_PRIVATE SrcList *sqlite3SrcListDup(sqlite3 *db, const SrcList *p, int fla pNewItem->iCursor = pOldItem->iCursor; pNewItem->addrFillSub = pOldItem->addrFillSub; pNewItem->regReturn = pOldItem->regReturn; + pNewItem->regResult = pOldItem->regResult; if( pNewItem->fg.isIndexedBy ){ pNewItem->u1.zIndexedBy = sqlite3DbStrDup(db, pOldItem->u1.zIndexedBy); + }else if( pNewItem->fg.isTabFunc ){ + pNewItem->u1.pFuncArg = + sqlite3ExprListDup(db, pOldItem->u1.pFuncArg, flags); + }else{ + pNewItem->u1.nRow = pOldItem->u1.nRow; } pNewItem->u2 = pOldItem->u2; if( pNewItem->fg.isCte ){ pNewItem->u2.pCteUse->nUse++; } - if( pNewItem->fg.isTabFunc ){ - pNewItem->u1.pFuncArg = - sqlite3ExprListDup(db, pOldItem->u1.pFuncArg, flags); - } pTab = pNewItem->pTab = pOldItem->pTab; if( pTab ){ pTab->nTabRef++; @@ -110856,6 +111367,54 @@ SQLITE_PRIVATE Expr *sqlite3ExprSimplifiedAndOr(Expr *pExpr){ return pExpr; } +/* +** pExpr is a TK_FUNCTION node. Try to determine whether or not the +** function is a constant function. A function is constant if all of +** the following are true: +** +** (1) It is a scalar function (not an aggregate or window function) +** (2) It has either the SQLITE_FUNC_CONSTANT or SQLITE_FUNC_SLOCHNG +** property. +** (3) All of its arguments are constants +** +** This routine sets pWalker->eCode to 0 if pExpr is not a constant. +** It makes no changes to pWalker->eCode if pExpr is constant. In +** every case, it returns WRC_Abort. +** +** Called as a service subroutine from exprNodeIsConstant(). +*/ +static SQLITE_NOINLINE int exprNodeIsConstantFunction( + Walker *pWalker, + Expr *pExpr +){ + int n; /* Number of arguments */ + ExprList *pList; /* List of arguments */ + FuncDef *pDef; /* The function */ + sqlite3 *db; /* The database */ + + assert( pExpr->op==TK_FUNCTION ); + if( ExprHasProperty(pExpr, EP_TokenOnly) + || (pList = pExpr->x.pList)==0 + ){; + n = 0; + }else{ + n = pList->nExpr; + sqlite3WalkExprList(pWalker, pList); + if( pWalker->eCode==0 ) return WRC_Abort; + } + db = pWalker->pParse->db; + pDef = sqlite3FindFunction(db, pExpr->u.zToken, n, ENC(db), 0); + if( pDef==0 + || pDef->xFinalize!=0 + || (pDef->funcFlags & (SQLITE_FUNC_CONSTANT|SQLITE_FUNC_SLOCHNG))==0 + || ExprHasProperty(pExpr, EP_WinFunc) + ){ + pWalker->eCode = 0; + return WRC_Abort; + } + return WRC_Prune; +} + /* ** These routines are Walker callbacks used to check expressions to @@ -110884,6 +111443,7 @@ SQLITE_PRIVATE Expr *sqlite3ExprSimplifiedAndOr(Expr *pExpr){ ** malformed schema error. */ static int exprNodeIsConstant(Walker *pWalker, Expr *pExpr){ + assert( pWalker->eCode>0 ); /* If pWalker->eCode is 2 then any term of the expression that comes from ** the ON or USING clauses of an outer join disqualifies the expression @@ -110903,6 +111463,8 @@ static int exprNodeIsConstant(Walker *pWalker, Expr *pExpr){ ){ if( pWalker->eCode==5 ) ExprSetProperty(pExpr, EP_FromDDL); return WRC_Continue; + }else if( pWalker->pParse ){ + return exprNodeIsConstantFunction(pWalker, pExpr); }else{ pWalker->eCode = 0; return WRC_Abort; @@ -110931,9 +111493,11 @@ static int exprNodeIsConstant(Walker *pWalker, Expr *pExpr){ case TK_IF_NULL_ROW: case TK_REGISTER: case TK_DOT: + case TK_RAISE: testcase( pExpr->op==TK_REGISTER ); testcase( pExpr->op==TK_IF_NULL_ROW ); testcase( pExpr->op==TK_DOT ); + testcase( pExpr->op==TK_RAISE ); pWalker->eCode = 0; return WRC_Abort; case TK_VARIABLE: @@ -110955,15 +111519,15 @@ static int exprNodeIsConstant(Walker *pWalker, Expr *pExpr){ return WRC_Continue; } } -static int exprIsConst(Expr *p, int initFlag, int iCur){ +static int exprIsConst(Parse *pParse, Expr *p, int initFlag){ Walker w; w.eCode = initFlag; + w.pParse = pParse; w.xExprCallback = exprNodeIsConstant; w.xSelectCallback = sqlite3SelectWalkFail; #ifdef SQLITE_DEBUG w.xSelectCallback2 = sqlite3SelectWalkAssert2; #endif - w.u.iCur = iCur; sqlite3WalkExpr(&w, p); return w.eCode; } @@ -110975,9 +111539,15 @@ static int exprIsConst(Expr *p, int initFlag, int iCur){ ** For the purposes of this function, a double-quoted string (ex: "abc") ** is considered a variable but a single-quoted string (ex: 'abc') is ** a constant. +** +** The pParse parameter may be NULL. But if it is NULL, there is no way +** to determine if function calls are constant or not, and hence all +** function calls will be considered to be non-constant. If pParse is +** not NULL, then a function call might be constant, depending on the +** function and on its parameters. */ -SQLITE_PRIVATE int sqlite3ExprIsConstant(Expr *p){ - return exprIsConst(p, 1, 0); +SQLITE_PRIVATE int sqlite3ExprIsConstant(Parse *pParse, Expr *p){ + return exprIsConst(pParse, p, 1); } /* @@ -110993,8 +111563,24 @@ SQLITE_PRIVATE int sqlite3ExprIsConstant(Expr *p){ ** can be added to the pParse->pConstExpr list and evaluated once when ** the prepared statement starts up. See sqlite3ExprCodeRunJustOnce(). */ -SQLITE_PRIVATE int sqlite3ExprIsConstantNotJoin(Expr *p){ - return exprIsConst(p, 2, 0); +static int sqlite3ExprIsConstantNotJoin(Parse *pParse, Expr *p){ + return exprIsConst(pParse, p, 2); +} + +/* +** This routine examines sub-SELECT statements as an expression is being +** walked as part of sqlite3ExprIsTableConstant(). Sub-SELECTs are considered +** constant as long as they are uncorrelated - meaning that they do not +** contain any terms from outer contexts. +*/ +static int exprSelectWalkTableConstant(Walker *pWalker, Select *pSelect){ + assert( pSelect!=0 ); + assert( pWalker->eCode==3 || pWalker->eCode==0 ); + if( (pSelect->selFlags & SF_Correlated)!=0 ){ + pWalker->eCode = 0; + return WRC_Abort; + } + return WRC_Prune; } /* @@ -111002,9 +111588,26 @@ SQLITE_PRIVATE int sqlite3ExprIsConstantNotJoin(Expr *p){ ** for any single row of the table with cursor iCur. In other words, the ** expression must not refer to any non-deterministic function nor any ** table other than iCur. +** +** Consider uncorrelated subqueries to be constants if the bAllowSubq +** parameter is true. */ -SQLITE_PRIVATE int sqlite3ExprIsTableConstant(Expr *p, int iCur){ - return exprIsConst(p, 3, iCur); +static int sqlite3ExprIsTableConstant(Expr *p, int iCur, int bAllowSubq){ + Walker w; + w.eCode = 3; + w.pParse = 0; + w.xExprCallback = exprNodeIsConstant; + if( bAllowSubq ){ + w.xSelectCallback = exprSelectWalkTableConstant; + }else{ + w.xSelectCallback = sqlite3SelectWalkFail; +#ifdef SQLITE_DEBUG + w.xSelectCallback2 = sqlite3SelectWalkAssert2; +#endif + } + w.u.iCur = iCur; + sqlite3WalkExpr(&w, p); + return w.eCode; } /* @@ -111022,7 +111625,10 @@ SQLITE_PRIVATE int sqlite3ExprIsTableConstant(Expr *p, int iCur){ ** ** (1) pExpr cannot refer to any table other than pSrc->iCursor. ** -** (2) pExpr cannot use subqueries or non-deterministic functions. +** (2a) pExpr cannot use subqueries unless the bAllowSubq parameter is +** true and the subquery is non-correlated +** +** (2b) pExpr cannot use non-deterministic functions. ** ** (3) pSrc cannot be part of the left operand for a RIGHT JOIN. ** (Is there some way to relax this constraint?) @@ -111051,7 +111657,8 @@ SQLITE_PRIVATE int sqlite3ExprIsTableConstant(Expr *p, int iCur){ SQLITE_PRIVATE int sqlite3ExprIsSingleTableConstraint( Expr *pExpr, /* The constraint */ const SrcList *pSrcList, /* Complete FROM clause */ - int iSrc /* Which element of pSrcList to use */ + int iSrc, /* Which element of pSrcList to use */ + int bAllowSubq /* Allow non-correlated subqueries */ ){ const SrcItem *pSrc = &pSrcList->a[iSrc]; if( pSrc->fg.jointype & JT_LTORJ ){ @@ -111076,7 +111683,8 @@ SQLITE_PRIVATE int sqlite3ExprIsSingleTableConstraint( } } } - return sqlite3ExprIsTableConstant(pExpr, pSrc->iCursor); /* rules (1), (2) */ + /* Rules (1), (2a), and (2b) handled by the following: */ + return sqlite3ExprIsTableConstant(pExpr, pSrc->iCursor, bAllowSubq); } @@ -111161,7 +111769,7 @@ SQLITE_PRIVATE int sqlite3ExprIsConstantOrGroupBy(Parse *pParse, Expr *p, ExprLi */ SQLITE_PRIVATE int sqlite3ExprIsConstantOrFunction(Expr *p, u8 isInit){ assert( isInit==0 || isInit==1 ); - return exprIsConst(p, 4+isInit, 0); + return exprIsConst(0, p, 4+isInit); } #ifdef SQLITE_ENABLE_CURSOR_HINTS @@ -111409,13 +112017,13 @@ static void sqlite3SetHasNullFlag(Vdbe *v, int iCur, int regHasNull){ ** The argument is an IN operator with a list (not a subquery) on the ** right-hand side. Return TRUE if that list is constant. */ -static int sqlite3InRhsIsConstant(Expr *pIn){ +static int sqlite3InRhsIsConstant(Parse *pParse, Expr *pIn){ Expr *pLHS; int res; assert( !ExprHasProperty(pIn, EP_xIsSelect) ); pLHS = pIn->pLeft; pIn->pLeft = 0; - res = sqlite3ExprIsConstant(pIn); + res = sqlite3ExprIsConstant(pParse, pIn); pIn->pLeft = pLHS; return res; } @@ -111684,7 +112292,7 @@ SQLITE_PRIVATE int sqlite3FindInIndex( if( eType==0 && (inFlags & IN_INDEX_NOOP_OK) && ExprUseXList(pX) - && (!sqlite3InRhsIsConstant(pX) || pX->x.pList->nExpr<=2) + && (!sqlite3InRhsIsConstant(pParse,pX) || pX->x.pList->nExpr<=2) ){ pParse->nTab--; /* Back out the allocation of the unused cursor */ iTab = -1; /* Cursor is not allocated */ @@ -111967,7 +112575,7 @@ SQLITE_PRIVATE void sqlite3CodeRhsOfIN( ** this code only executes once. Because for a non-constant ** expression we need to rerun this code each time. */ - if( addrOnce && !sqlite3ExprIsConstant(pE2) ){ + if( addrOnce && !sqlite3ExprIsConstant(pParse, pE2) ){ sqlite3VdbeChangeToNoop(v, addrOnce-1); sqlite3VdbeChangeToNoop(v, addrOnce); ExprClearProperty(pExpr, EP_Subrtn); @@ -113131,12 +113739,6 @@ expr_code_doover: assert( pExpr->u.zToken!=0 ); assert( pExpr->u.zToken[0]!=0 ); sqlite3VdbeAddOp2(v, OP_Variable, pExpr->iColumn, target); - if( pExpr->u.zToken[1]!=0 ){ - const char *z = sqlite3VListNumToName(pParse->pVList, pExpr->iColumn); - assert( pExpr->u.zToken[0]=='?' || (z && !strcmp(pExpr->u.zToken, z)) ); - pParse->pVList[0] = 0; /* Indicate VList may no longer be enlarged */ - sqlite3VdbeAppendP4(v, (char*)z, P4_STATIC); - } return target; } case TK_REGISTER: { @@ -113310,7 +113912,9 @@ expr_code_doover: } #endif - if( ConstFactorOk(pParse) && sqlite3ExprIsConstantNotJoin(pExpr) ){ + if( ConstFactorOk(pParse) + && sqlite3ExprIsConstantNotJoin(pParse,pExpr) + ){ /* SQL functions can be expensive. So try to avoid running them ** multiple times if we know they always give the same result */ return sqlite3ExprCodeRunJustOnce(pParse, pExpr, -1); @@ -113341,7 +113945,7 @@ expr_code_doover: } for(i=0; ia[i].pExpr) ){ + if( i<32 && sqlite3ExprIsConstant(pParse, pFarg->a[i].pExpr) ){ testcase( i==31 ); constMask |= MASKBIT32(i); } @@ -113483,8 +114087,9 @@ expr_code_doover: if( !ExprHasProperty(pExpr, EP_Collate) ){ /* A TK_COLLATE Expr node without the EP_Collate tag is a so-called ** "SOFT-COLLATE" that is added to constraints that are pushed down - ** from outer queries into sub-queries by the push-down optimization. - ** Clear subtypes as subtypes may not cross a subquery boundary. + ** from outer queries into sub-queries by the WHERE-clause push-down + ** optimization. Clear subtypes as subtypes may not cross a subquery + ** boundary. */ assert( pExpr->pLeft ); sqlite3ExprCode(pParse, pExpr->pLeft, target); @@ -113808,7 +114413,7 @@ SQLITE_PRIVATE int sqlite3ExprCodeTemp(Parse *pParse, Expr *pExpr, int *pReg){ if( ConstFactorOk(pParse) && ALWAYS(pExpr!=0) && pExpr->op!=TK_REGISTER - && sqlite3ExprIsConstantNotJoin(pExpr) + && sqlite3ExprIsConstantNotJoin(pParse, pExpr) ){ *pReg = 0; r2 = sqlite3ExprCodeRunJustOnce(pParse, pExpr, -1); @@ -113872,7 +114477,7 @@ SQLITE_PRIVATE void sqlite3ExprCodeCopy(Parse *pParse, Expr *pExpr, int target){ ** might choose to code the expression at initialization time. */ SQLITE_PRIVATE void sqlite3ExprCodeFactorable(Parse *pParse, Expr *pExpr, int target){ - if( pParse->okConstFactor && sqlite3ExprIsConstantNotJoin(pExpr) ){ + if( pParse->okConstFactor && sqlite3ExprIsConstantNotJoin(pParse,pExpr) ){ sqlite3ExprCodeRunJustOnce(pParse, pExpr, target); }else{ sqlite3ExprCodeCopy(pParse, pExpr, target); @@ -113931,7 +114536,7 @@ SQLITE_PRIVATE int sqlite3ExprCodeExprList( sqlite3VdbeAddOp2(v, copyOp, j+srcReg-1, target+i); } }else if( (flags & SQLITE_ECEL_FACTOR)!=0 - && sqlite3ExprIsConstantNotJoin(pExpr) + && sqlite3ExprIsConstantNotJoin(pParse,pExpr) ){ sqlite3ExprCodeRunJustOnce(pParse, pExpr, target+i); }else{ @@ -115082,9 +115687,8 @@ static int agginfoPersistExprCb(Walker *pWalker, Expr *pExpr){ && pAggInfo->aCol[iAgg].pCExpr==pExpr ){ pExpr = sqlite3ExprDup(db, pExpr, 0); - if( pExpr ){ + if( pExpr && !sqlite3ExprDeferredDelete(pParse, pExpr) ){ pAggInfo->aCol[iAgg].pCExpr = pExpr; - sqlite3ExprDeferredDelete(pParse, pExpr); } } }else{ @@ -115093,9 +115697,8 @@ static int agginfoPersistExprCb(Walker *pWalker, Expr *pExpr){ && pAggInfo->aFunc[iAgg].pFExpr==pExpr ){ pExpr = sqlite3ExprDup(db, pExpr, 0); - if( pExpr ){ + if( pExpr && !sqlite3ExprDeferredDelete(pParse, pExpr) ){ pAggInfo->aFunc[iAgg].pFExpr = pExpr; - sqlite3ExprDeferredDelete(pParse, pExpr); } } } @@ -117796,7 +118399,12 @@ SQLITE_PRIVATE void sqlite3AlterDropColumn(Parse *pParse, SrcList *pSrc, const T if( i==pTab->iPKey ){ sqlite3VdbeAddOp2(v, OP_Null, 0, regOut); }else{ + char aff = pTab->aCol[i].affinity; + if( aff==SQLITE_AFF_REAL ){ + pTab->aCol[i].affinity = SQLITE_AFF_NUMERIC; + } sqlite3ExprCodeGetColumnOfTable(v, pTab, iCur, i, regOut); + pTab->aCol[i].affinity = aff; } nField++; } @@ -118715,7 +119323,7 @@ static void statGet( if( iVal==2 && p->nRow*10 <= nDistinct*11 ) iVal = 1; sqlite3_str_appendf(&sStat, " %llu", iVal); #ifdef SQLITE_ENABLE_STAT4 - assert( p->current.anEq[i] ); + assert( p->current.anEq[i] || p->nRow==0 ); #endif } sqlite3ResultStrAccum(context, &sStat); @@ -118900,7 +119508,7 @@ static void analyzeOneTable( for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ int nCol; /* Number of columns in pIdx. "N" */ - int addrRewind; /* Address of "OP_Rewind iIdxCur" */ + int addrGotoEnd; /* Address of "OP_Rewind iIdxCur" */ int addrNextRow; /* Address of "next_row:" */ const char *zIdxName; /* Name of the index */ int nColTest; /* Number of columns to test for changes */ @@ -118924,9 +119532,14 @@ static void analyzeOneTable( /* ** Pseudo-code for loop that calls stat_push(): ** - ** Rewind csr - ** if eof(csr) goto end_of_scan; ** regChng = 0 + ** Rewind csr + ** if eof(csr){ + ** stat_init() with count = 0; + ** goto end_of_scan; + ** } + ** count() + ** stat_init() ** goto chng_addr_0; ** ** next_row: @@ -118965,41 +119578,36 @@ static void analyzeOneTable( sqlite3VdbeSetP4KeyInfo(pParse, pIdx); VdbeComment((v, "%s", pIdx->zName)); - /* Invoke the stat_init() function. The arguments are: + /* Implementation of the following: ** + ** regChng = 0 + ** Rewind csr + ** if eof(csr){ + ** stat_init() with count = 0; + ** goto end_of_scan; + ** } + ** count() + ** stat_init() + ** goto chng_addr_0; + */ + assert( regTemp2==regStat+4 ); + sqlite3VdbeAddOp2(v, OP_Integer, db->nAnalysisLimit, regTemp2); + + /* Arguments to stat_init(): ** (1) the number of columns in the index including the rowid ** (or for a WITHOUT ROWID table, the number of PK columns), ** (2) the number of columns in the key without the rowid/pk - ** (3) estimated number of rows in the index, - */ + ** (3) estimated number of rows in the index. */ sqlite3VdbeAddOp2(v, OP_Integer, nCol, regStat+1); assert( regRowid==regStat+2 ); sqlite3VdbeAddOp2(v, OP_Integer, pIdx->nKeyCol, regRowid); -#ifdef SQLITE_ENABLE_STAT4 - if( OptimizationEnabled(db, SQLITE_Stat4) ){ - sqlite3VdbeAddOp2(v, OP_Count, iIdxCur, regTemp); - addrRewind = sqlite3VdbeAddOp1(v, OP_Rewind, iIdxCur); - VdbeCoverage(v); - }else -#endif - { - addrRewind = sqlite3VdbeAddOp1(v, OP_Rewind, iIdxCur); - VdbeCoverage(v); - sqlite3VdbeAddOp3(v, OP_Count, iIdxCur, regTemp, 1); - } - assert( regTemp2==regStat+4 ); - sqlite3VdbeAddOp2(v, OP_Integer, db->nAnalysisLimit, regTemp2); + sqlite3VdbeAddOp3(v, OP_Count, iIdxCur, regTemp, + OptimizationDisabled(db, SQLITE_Stat4)); sqlite3VdbeAddFunctionCall(pParse, 0, regStat+1, regStat, 4, &statInitFuncdef, 0); + addrGotoEnd = sqlite3VdbeAddOp1(v, OP_Rewind, iIdxCur); + VdbeCoverage(v); - /* Implementation of the following: - ** - ** Rewind csr - ** if eof(csr) goto end_of_scan; - ** regChng = 0 - ** goto next_push_0; - ** - */ sqlite3VdbeAddOp2(v, OP_Integer, 0, regChng); addrNextRow = sqlite3VdbeCurrentAddr(v); @@ -119106,6 +119714,12 @@ static void analyzeOneTable( } /* Add the entry to the stat1 table. */ + if( pIdx->pPartIdxWhere ){ + /* Partial indexes might get a zero-entry in sqlite_stat1. But + ** an empty table is omitted from sqlite_stat1. */ + sqlite3VdbeJumpHere(v, addrGotoEnd); + addrGotoEnd = 0; + } callStatGet(pParse, regStat, STAT_GET_STAT1, regStat1); assert( "BBB"[0]==SQLITE_AFF_TEXT ); sqlite3VdbeAddOp4(v, OP_MakeRecord, regTabname, 3, regTemp, "BBB", 0); @@ -119129,6 +119743,13 @@ static void analyzeOneTable( int addrIsNull; u8 seekOp = HasRowid(pTab) ? OP_NotExists : OP_NotFound; + /* No STAT4 data is generated if the number of rows is zero */ + if( addrGotoEnd==0 ){ + sqlite3VdbeAddOp2(v, OP_Cast, regStat1, SQLITE_AFF_INTEGER); + addrGotoEnd = sqlite3VdbeAddOp1(v, OP_IfNot, regStat1); + VdbeCoverage(v); + } + if( doOnce ){ int mxCol = nCol; Index *pX; @@ -119181,7 +119802,7 @@ static void analyzeOneTable( #endif /* SQLITE_ENABLE_STAT4 */ /* End of analysis */ - sqlite3VdbeJumpHere(v, addrRewind); + if( addrGotoEnd ) sqlite3VdbeJumpHere(v, addrGotoEnd); } @@ -120930,7 +121551,7 @@ SQLITE_PRIVATE void sqlite3FinishCoding(Parse *pParse){ } sqlite3VdbeAddOp0(v, OP_Halt); -#if SQLITE_USER_AUTHENTICATION +#if SQLITE_USER_AUTHENTICATION && !defined(SQLITE_OMIT_SHARED_CACHE) if( pParse->nTableLock>0 && db->init.busy==0 ){ sqlite3UserAuthInit(db); if( db->auth.authLevelrc = SQLITE_ERROR; pParse->nErr++; return; } + iCsr = pParse->nTab++; regYield = ++pParse->nMem; regRec = ++pParse->nMem; regRowid = ++pParse->nMem; - assert(pParse->nTab==1); sqlite3MayAbort(pParse); - sqlite3VdbeAddOp3(v, OP_OpenWrite, 1, pParse->regRoot, iDb); + sqlite3VdbeAddOp3(v, OP_OpenWrite, iCsr, pParse->regRoot, iDb); sqlite3VdbeChangeP5(v, OPFLAG_P2ISREG); - pParse->nTab = 2; addrTop = sqlite3VdbeCurrentAddr(v) + 1; sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, addrTop); if( pParse->nErr ) return; @@ -123603,11 +124224,11 @@ SQLITE_PRIVATE void sqlite3EndTable( VdbeCoverage(v); sqlite3VdbeAddOp3(v, OP_MakeRecord, dest.iSdst, dest.nSdst, regRec); sqlite3TableAffinity(v, p, 0); - sqlite3VdbeAddOp2(v, OP_NewRowid, 1, regRowid); - sqlite3VdbeAddOp3(v, OP_Insert, 1, regRec, regRowid); + sqlite3VdbeAddOp2(v, OP_NewRowid, iCsr, regRowid); + sqlite3VdbeAddOp3(v, OP_Insert, iCsr, regRec, regRowid); sqlite3VdbeGoto(v, addrInsLoop); sqlite3VdbeJumpHere(v, addrInsLoop); - sqlite3VdbeAddOp1(v, OP_Close, 1); + sqlite3VdbeAddOp1(v, OP_Close, iCsr); } /* Compute the complete text of the CREATE statement */ @@ -123664,13 +124285,10 @@ SQLITE_PRIVATE void sqlite3EndTable( /* Test for cycles in generated columns and illegal expressions ** in CHECK constraints and in DEFAULT clauses. */ if( p->tabFlags & TF_HasGenerated ){ - sqlite3VdbeAddOp4(v, OP_SqlExec, 1, 0, 0, + sqlite3VdbeAddOp4(v, OP_SqlExec, 0x0001, 0, 0, sqlite3MPrintf(db, "SELECT*FROM\"%w\".\"%w\"", db->aDb[iDb].zDbSName, p->zName), P4_DYNAMIC); } - sqlite3VdbeAddOp4(v, OP_SqlExec, 1, 0, 0, - sqlite3MPrintf(db, "PRAGMA \"%w\".integrity_check(%Q)", - db->aDb[iDb].zDbSName, p->zName), P4_DYNAMIC); } /* Add the table to the in-memory representation of the database. @@ -132844,6 +133462,195 @@ SQLITE_PRIVATE void sqlite3AutoincrementEnd(Parse *pParse){ # define autoIncStep(A,B,C) #endif /* SQLITE_OMIT_AUTOINCREMENT */ +/* +** If argument pVal is a Select object returned by an sqlite3MultiValues() +** that was able to use the co-routine optimization, finish coding the +** co-routine. +*/ +SQLITE_PRIVATE void sqlite3MultiValuesEnd(Parse *pParse, Select *pVal){ + if( ALWAYS(pVal) && pVal->pSrc->nSrc>0 ){ + SrcItem *pItem = &pVal->pSrc->a[0]; + sqlite3VdbeEndCoroutine(pParse->pVdbe, pItem->regReturn); + sqlite3VdbeJumpHere(pParse->pVdbe, pItem->addrFillSub - 1); + } +} + +/* +** Return true if all expressions in the expression-list passed as the +** only argument are constant. +*/ +static int exprListIsConstant(Parse *pParse, ExprList *pRow){ + int ii; + for(ii=0; iinExpr; ii++){ + if( 0==sqlite3ExprIsConstant(pParse, pRow->a[ii].pExpr) ) return 0; + } + return 1; +} + +/* +** Return true if all expressions in the expression-list passed as the +** only argument are both constant and have no affinity. +*/ +static int exprListIsNoAffinity(Parse *pParse, ExprList *pRow){ + int ii; + if( exprListIsConstant(pParse,pRow)==0 ) return 0; + for(ii=0; iinExpr; ii++){ + Expr *pExpr = pRow->a[ii].pExpr; + assert( pExpr->op!=TK_RAISE ); + assert( pExpr->affExpr==0 ); + if( 0!=sqlite3ExprAffinity(pExpr) ) return 0; + } + return 1; + +} + +/* +** This function is called by the parser for the second and subsequent +** rows of a multi-row VALUES clause. Argument pLeft is the part of +** the VALUES clause already parsed, argument pRow is the vector of values +** for the new row. The Select object returned represents the complete +** VALUES clause, including the new row. +** +** There are two ways in which this may be achieved - by incremental +** coding of a co-routine (the "co-routine" method) or by returning a +** Select object equivalent to the following (the "UNION ALL" method): +** +** "pLeft UNION ALL SELECT pRow" +** +** If the VALUES clause contains a lot of rows, this compound Select +** object may consume a lot of memory. +** +** When the co-routine method is used, each row that will be returned +** by the VALUES clause is coded into part of a co-routine as it is +** passed to this function. The returned Select object is equivalent to: +** +** SELECT * FROM ( +** Select object to read co-routine +** ) +** +** The co-routine method is used in most cases. Exceptions are: +** +** a) If the current statement has a WITH clause. This is to avoid +** statements like: +** +** WITH cte AS ( VALUES('x'), ('y') ... ) +** SELECT * FROM cte AS a, cte AS b; +** +** This will not work, as the co-routine uses a hard-coded register +** for its OP_Yield instructions, and so it is not possible for two +** cursors to iterate through it concurrently. +** +** b) The schema is currently being parsed (i.e. the VALUES clause is part +** of a schema item like a VIEW or TRIGGER). In this case there is no VM +** being generated when parsing is taking place, and so generating +** a co-routine is not possible. +** +** c) There are non-constant expressions in the VALUES clause (e.g. +** the VALUES clause is part of a correlated sub-query). +** +** d) One or more of the values in the first row of the VALUES clause +** has an affinity (i.e. is a CAST expression). This causes problems +** because the complex rules SQLite uses (see function +** sqlite3SubqueryColumnTypes() in select.c) to determine the effective +** affinity of such a column for all rows require access to all values in +** the column simultaneously. +*/ +SQLITE_PRIVATE Select *sqlite3MultiValues(Parse *pParse, Select *pLeft, ExprList *pRow){ + + if( pParse->bHasWith /* condition (a) above */ + || pParse->db->init.busy /* condition (b) above */ + || exprListIsConstant(pParse,pRow)==0 /* condition (c) above */ + || (pLeft->pSrc->nSrc==0 && + exprListIsNoAffinity(pParse,pLeft->pEList)==0) /* condition (d) above */ + || IN_SPECIAL_PARSE + ){ + /* The co-routine method cannot be used. Fall back to UNION ALL. */ + Select *pSelect = 0; + int f = SF_Values | SF_MultiValue; + if( pLeft->pSrc->nSrc ){ + sqlite3MultiValuesEnd(pParse, pLeft); + f = SF_Values; + }else if( pLeft->pPrior ){ + /* In this case set the SF_MultiValue flag only if it was set on pLeft */ + f = (f & pLeft->selFlags); + } + pSelect = sqlite3SelectNew(pParse, pRow, 0, 0, 0, 0, 0, f, 0); + pLeft->selFlags &= ~SF_MultiValue; + if( pSelect ){ + pSelect->op = TK_ALL; + pSelect->pPrior = pLeft; + pLeft = pSelect; + } + }else{ + SrcItem *p = 0; /* SrcItem that reads from co-routine */ + + if( pLeft->pSrc->nSrc==0 ){ + /* Co-routine has not yet been started and the special Select object + ** that accesses the co-routine has not yet been created. This block + ** does both those things. */ + Vdbe *v = sqlite3GetVdbe(pParse); + Select *pRet = sqlite3SelectNew(pParse, 0, 0, 0, 0, 0, 0, 0, 0); + + /* Ensure the database schema has been read. This is to ensure we have + ** the correct text encoding. */ + if( (pParse->db->mDbFlags & DBFLAG_SchemaKnownOk)==0 ){ + sqlite3ReadSchema(pParse); + } + + if( pRet ){ + SelectDest dest; + pRet->pSrc->nSrc = 1; + pRet->pPrior = pLeft->pPrior; + pRet->op = pLeft->op; + pLeft->pPrior = 0; + pLeft->op = TK_SELECT; + assert( pLeft->pNext==0 ); + assert( pRet->pNext==0 ); + p = &pRet->pSrc->a[0]; + p->pSelect = pLeft; + p->fg.viaCoroutine = 1; + p->addrFillSub = sqlite3VdbeCurrentAddr(v) + 1; + p->regReturn = ++pParse->nMem; + p->iCursor = -1; + p->u1.nRow = 2; + sqlite3VdbeAddOp3(v,OP_InitCoroutine,p->regReturn,0,p->addrFillSub); + sqlite3SelectDestInit(&dest, SRT_Coroutine, p->regReturn); + + /* Allocate registers for the output of the co-routine. Do so so + ** that there are two unused registers immediately before those + ** used by the co-routine. This allows the code in sqlite3Insert() + ** to use these registers directly, instead of copying the output + ** of the co-routine to a separate array for processing. */ + dest.iSdst = pParse->nMem + 3; + dest.nSdst = pLeft->pEList->nExpr; + pParse->nMem += 2 + dest.nSdst; + + pLeft->selFlags |= SF_MultiValue; + sqlite3Select(pParse, pLeft, &dest); + p->regResult = dest.iSdst; + assert( pParse->nErr || dest.iSdst>0 ); + pLeft = pRet; + } + }else{ + p = &pLeft->pSrc->a[0]; + assert( !p->fg.isTabFunc && !p->fg.isIndexedBy ); + p->u1.nRow++; + } + + if( pParse->nErr==0 ){ + assert( p!=0 ); + if( p->pSelect->pEList->nExpr!=pRow->nExpr ){ + sqlite3SelectWrongNumTermsError(pParse, p->pSelect); + }else{ + sqlite3ExprCodeExprList(pParse, pRow, p->regResult, 0, 0); + sqlite3VdbeAddOp1(pParse->pVdbe, OP_Yield, p->regReturn); + } + } + sqlite3ExprListDelete(pParse->db, pRow); + } + + return pLeft; +} /* Forward declaration */ static int xferOptimization( @@ -133180,25 +133987,40 @@ SQLITE_PRIVATE void sqlite3Insert( if( pSelect ){ /* Data is coming from a SELECT or from a multi-row VALUES clause. ** Generate a co-routine to run the SELECT. */ - int regYield; /* Register holding co-routine entry-point */ - int addrTop; /* Top of the co-routine */ int rc; /* Result code */ - regYield = ++pParse->nMem; - addrTop = sqlite3VdbeCurrentAddr(v) + 1; - sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, addrTop); - sqlite3SelectDestInit(&dest, SRT_Coroutine, regYield); - dest.iSdst = bIdListInOrder ? regData : 0; - dest.nSdst = pTab->nCol; - rc = sqlite3Select(pParse, pSelect, &dest); - regFromSelect = dest.iSdst; - assert( db->pParse==pParse ); - if( rc || pParse->nErr ) goto insert_cleanup; - assert( db->mallocFailed==0 ); - sqlite3VdbeEndCoroutine(v, regYield); - sqlite3VdbeJumpHere(v, addrTop - 1); /* label B: */ - assert( pSelect->pEList ); - nColumn = pSelect->pEList->nExpr; + if( pSelect->pSrc->nSrc==1 + && pSelect->pSrc->a[0].fg.viaCoroutine + && pSelect->pPrior==0 + ){ + SrcItem *pItem = &pSelect->pSrc->a[0]; + dest.iSDParm = pItem->regReturn; + regFromSelect = pItem->regResult; + nColumn = pItem->pSelect->pEList->nExpr; + ExplainQueryPlan((pParse, 0, "SCAN %S", pItem)); + if( bIdListInOrder && nColumn==pTab->nCol ){ + regData = regFromSelect; + regRowid = regData - 1; + regIns = regRowid - (IsVirtual(pTab) ? 1 : 0); + } + }else{ + int addrTop; /* Top of the co-routine */ + int regYield = ++pParse->nMem; + addrTop = sqlite3VdbeCurrentAddr(v) + 1; + sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, addrTop); + sqlite3SelectDestInit(&dest, SRT_Coroutine, regYield); + dest.iSdst = bIdListInOrder ? regData : 0; + dest.nSdst = pTab->nCol; + rc = sqlite3Select(pParse, pSelect, &dest); + regFromSelect = dest.iSdst; + assert( db->pParse==pParse ); + if( rc || pParse->nErr ) goto insert_cleanup; + assert( db->mallocFailed==0 ); + sqlite3VdbeEndCoroutine(v, regYield); + sqlite3VdbeJumpHere(v, addrTop - 1); /* label B: */ + assert( pSelect->pEList ); + nColumn = pSelect->pEList->nExpr; + } /* Set useTempTable to TRUE if the result of the SELECT statement ** should be written into a temporary table (template 4). Set to @@ -137923,6 +138745,34 @@ static const PragmaName aPragmaName[] = { /************** End of pragma.h **********************************************/ /************** Continuing where we left off in pragma.c *********************/ +/* +** When the 0x10 bit of PRAGMA optimize is set, any ANALYZE commands +** will be run with an analysis_limit set to the lessor of the value of +** the following macro or to the actual analysis_limit if it is non-zero, +** in order to prevent PRAGMA optimize from running for too long. +** +** The value of 2000 is chosen emperically so that the worst-case run-time +** for PRAGMA optimize does not exceed 100 milliseconds against a variety +** of test databases on a RaspberryPI-4 compiled using -Os and without +** -DSQLITE_DEBUG. Of course, your mileage may vary. For the purpose of +** this paragraph, "worst-case" means that ANALYZE ends up being +** run on every table in the database. The worst case typically only +** happens if PRAGMA optimize is run on a database file for which ANALYZE +** has not been previously run and the 0x10000 flag is included so that +** all tables are analyzed. The usual case for PRAGMA optimize is that +** no ANALYZE commands will be run at all, or if any ANALYZE happens it +** will be against a single table, so that expected timing for PRAGMA +** optimize on a PI-4 is more like 1 millisecond or less with the 0x10000 +** flag or less than 100 microseconds without the 0x10000 flag. +** +** An analysis limit of 2000 is almost always sufficient for the query +** planner to fully characterize an index. The additional accuracy from +** a larger analysis is not usually helpful. +*/ +#ifndef SQLITE_DEFAULT_OPTIMIZE_LIMIT +# define SQLITE_DEFAULT_OPTIMIZE_LIMIT 2000 +#endif + /* ** Interpret the given string as a safety level. Return 0 for OFF, ** 1 for ON or NORMAL, 2 for FULL, and 3 for EXTRA. Return 1 for an empty or @@ -139568,7 +140418,7 @@ SQLITE_PRIVATE void sqlite3Pragma( /* Set the maximum error count */ mxErr = SQLITE_INTEGRITY_CHECK_ERROR_MAX; if( zRight ){ - if( sqlite3GetInt32(zRight, &mxErr) ){ + if( sqlite3GetInt32(pValue->z, &mxErr) ){ if( mxErr<=0 ){ mxErr = SQLITE_INTEGRITY_CHECK_ERROR_MAX; } @@ -139585,7 +140435,6 @@ SQLITE_PRIVATE void sqlite3Pragma( Hash *pTbls; /* Set of all tables in the schema */ int *aRoot; /* Array of root page numbers of all btrees */ int cnt = 0; /* Number of entries in aRoot[] */ - int mxIdx = 0; /* Maximum number of indexes for any table */ if( OMIT_TEMPDB && i==1 ) continue; if( iDb>=0 && i!=iDb ) continue; @@ -139607,7 +140456,6 @@ SQLITE_PRIVATE void sqlite3Pragma( if( pObjTab && pObjTab!=pTab ) continue; if( HasRowid(pTab) ) cnt++; for(nIdx=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, nIdx++){ cnt++; } - if( nIdx>mxIdx ) mxIdx = nIdx; } if( cnt==0 ) continue; if( pObjTab ) cnt++; @@ -139627,11 +140475,11 @@ SQLITE_PRIVATE void sqlite3Pragma( aRoot[0] = cnt; /* Make sure sufficient number of registers have been allocated */ - sqlite3TouchRegister(pParse, 8+mxIdx); + sqlite3TouchRegister(pParse, 8+cnt); sqlite3ClearTempRegCache(pParse); /* Do the b-tree integrity checks */ - sqlite3VdbeAddOp4(v, OP_IntegrityCk, 2, cnt, 1, (char*)aRoot,P4_INTARRAY); + sqlite3VdbeAddOp4(v, OP_IntegrityCk, 1, cnt, 8, (char*)aRoot,P4_INTARRAY); sqlite3VdbeChangeP5(v, (u8)i); addr = sqlite3VdbeAddOp1(v, OP_IsNull, 2); VdbeCoverage(v); sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, @@ -139641,6 +140489,36 @@ SQLITE_PRIVATE void sqlite3Pragma( integrityCheckResultRow(v); sqlite3VdbeJumpHere(v, addr); + /* Check that the indexes all have the right number of rows */ + cnt = pObjTab ? 1 : 0; + sqlite3VdbeLoadString(v, 2, "wrong # of entries in index "); + for(x=sqliteHashFirst(pTbls); x; x=sqliteHashNext(x)){ + int iTab = 0; + Table *pTab = sqliteHashData(x); + Index *pIdx; + if( pObjTab && pObjTab!=pTab ) continue; + if( HasRowid(pTab) ){ + iTab = cnt++; + }else{ + iTab = cnt; + for(pIdx=pTab->pIndex; ALWAYS(pIdx); pIdx=pIdx->pNext){ + if( IsPrimaryKeyIndex(pIdx) ) break; + iTab++; + } + } + for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + if( pIdx->pPartIdxWhere==0 ){ + addr = sqlite3VdbeAddOp3(v, OP_Eq, 8+cnt, 0, 8+iTab); + VdbeCoverageNeverNull(v); + sqlite3VdbeLoadString(v, 4, pIdx->zName); + sqlite3VdbeAddOp3(v, OP_Concat, 4, 2, 3); + integrityCheckResultRow(v); + sqlite3VdbeJumpHere(v, addr); + } + cnt++; + } + } + /* Make sure all the indices are constructed correctly. */ for(x=sqliteHashFirst(pTbls); x; x=sqliteHashNext(x)){ @@ -139964,21 +140842,9 @@ SQLITE_PRIVATE void sqlite3Pragma( } sqlite3VdbeAddOp2(v, OP_Next, iDataCur, loopTop); VdbeCoverage(v); sqlite3VdbeJumpHere(v, loopTop-1); - if( !isQuick ){ - sqlite3VdbeLoadString(v, 2, "wrong # of entries in index "); - for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){ - if( pPk==pIdx ) continue; - sqlite3VdbeAddOp2(v, OP_Count, iIdxCur+j, 3); - addr = sqlite3VdbeAddOp3(v, OP_Eq, 8+j, 0, 3); VdbeCoverage(v); - sqlite3VdbeChangeP5(v, SQLITE_NOTNULL); - sqlite3VdbeLoadString(v, 4, pIdx->zName); - sqlite3VdbeAddOp3(v, OP_Concat, 4, 2, 3); - integrityCheckResultRow(v); - sqlite3VdbeJumpHere(v, addr); - } - if( pPk ){ - sqlite3ReleaseTempRange(pParse, r2, pPk->nKeyCol); - } + if( pPk ){ + assert( !isQuick ); + sqlite3ReleaseTempRange(pParse, r2, pPk->nKeyCol); } } @@ -140276,44 +141142,63 @@ SQLITE_PRIVATE void sqlite3Pragma( ** ** The optional argument is a bitmask of optimizations to perform: ** - ** 0x0001 Debugging mode. Do not actually perform any optimizations - ** but instead return one line of text for each optimization - ** that would have been done. Off by default. + ** 0x00001 Debugging mode. Do not actually perform any optimizations + ** but instead return one line of text for each optimization + ** that would have been done. Off by default. ** - ** 0x0002 Run ANALYZE on tables that might benefit. On by default. - ** See below for additional information. + ** 0x00002 Run ANALYZE on tables that might benefit. On by default. + ** See below for additional information. ** - ** 0x0004 (Not yet implemented) Record usage and performance - ** information from the current session in the - ** database file so that it will be available to "optimize" - ** pragmas run by future database connections. + ** 0x00010 Run all ANALYZE operations using an analysis_limit that + ** is the lessor of the current analysis_limit and the + ** SQLITE_DEFAULT_OPTIMIZE_LIMIT compile-time option. + ** The default value of SQLITE_DEFAULT_OPTIMIZE_LIMIT is + ** currently (2024-02-19) set to 2000, which is such that + ** the worst case run-time for PRAGMA optimize on a 100MB + ** database will usually be less than 100 milliseconds on + ** a RaspberryPI-4 class machine. On by default. ** - ** 0x0008 (Not yet implemented) Create indexes that might have - ** been helpful to recent queries + ** 0x10000 Look at tables to see if they need to be reanalyzed + ** due to growth or shrinkage even if they have not been + ** queried during the current connection. Off by default. ** - ** The default MASK is and always shall be 0xfffe. 0xfffe means perform all - ** of the optimizations listed above except Debug Mode, including new - ** optimizations that have not yet been invented. If new optimizations are - ** ever added that should be off by default, those off-by-default - ** optimizations will have bitmasks of 0x10000 or larger. + ** The default MASK is and always shall be 0x0fffe. In the current + ** implementation, the default mask only covers the 0x00002 optimization, + ** though additional optimizations that are covered by 0x0fffe might be + ** added in the future. Optimizations that are off by default and must + ** be explicitly requested have masks of 0x10000 or greater. ** ** DETERMINATION OF WHEN TO RUN ANALYZE ** ** In the current implementation, a table is analyzed if only if all of ** the following are true: ** - ** (1) MASK bit 0x02 is set. + ** (1) MASK bit 0x00002 is set. ** - ** (2) The query planner used sqlite_stat1-style statistics for one or - ** more indexes of the table at some point during the lifetime of - ** the current connection. + ** (2) The table is an ordinary table, not a virtual table or view. ** - ** (3) One or more indexes of the table are currently unanalyzed OR - ** the number of rows in the table has increased by 25 times or more - ** since the last time ANALYZE was run. + ** (3) The table name does not begin with "sqlite_". + ** + ** (4) One or more of the following is true: + ** (4a) The 0x10000 MASK bit is set. + ** (4b) One or more indexes on the table lacks an entry + ** in the sqlite_stat1 table. + ** (4c) The query planner used sqlite_stat1-style statistics for one + ** or more indexes of the table at some point during the lifetime + ** of the current connection. + ** + ** (5) One or more of the following is true: + ** (5a) One or more indexes on the table lacks an entry + ** in the sqlite_stat1 table. (Same as 4a) + ** (5b) The number of rows in the table has increased or decreased by + ** 10-fold. In other words, the current size of the table is + ** 10 times larger than the size in sqlite_stat1 or else the + ** current size is less than 1/10th the size in sqlite_stat1. ** ** The rules for when tables are analyzed are likely to change in - ** future releases. + ** future releases. Future versions of SQLite might accept a string + ** literal argument to this pragma that contains a mnemonic description + ** of the options rather than a bitmap. */ case PragTyp_OPTIMIZE: { int iDbLast; /* Loop termination point for the schema loop */ @@ -140325,6 +141210,10 @@ SQLITE_PRIVATE void sqlite3Pragma( LogEst szThreshold; /* Size threshold above which reanalysis needed */ char *zSubSql; /* SQL statement for the OP_SqlExec opcode */ u32 opMask; /* Mask of operations to perform */ + int nLimit; /* Analysis limit to use */ + int nCheck = 0; /* Number of tables to be optimized */ + int nBtree = 0; /* Number of btrees to scan */ + int nIndex; /* Number of indexes on the current table */ if( zRight ){ opMask = (u32)sqlite3Atoi(zRight); @@ -140332,6 +141221,14 @@ SQLITE_PRIVATE void sqlite3Pragma( }else{ opMask = 0xfffe; } + if( (opMask & 0x10)==0 ){ + nLimit = 0; + }else if( db->nAnalysisLimit>0 + && db->nAnalysisLimitnTab++; for(iDbLast = zDb?iDb:db->nDb-1; iDb<=iDbLast; iDb++){ if( iDb==1 ) continue; @@ -140340,23 +141237,61 @@ SQLITE_PRIVATE void sqlite3Pragma( for(k=sqliteHashFirst(&pSchema->tblHash); k; k=sqliteHashNext(k)){ pTab = (Table*)sqliteHashData(k); - /* If table pTab has not been used in a way that would benefit from - ** having analysis statistics during the current session, then skip it. - ** This also has the effect of skipping virtual tables and views */ - if( (pTab->tabFlags & TF_StatsUsed)==0 ) continue; + /* This only works for ordinary tables */ + if( !IsOrdinaryTable(pTab) ) continue; - /* Reanalyze if the table is 25 times larger than the last analysis */ - szThreshold = pTab->nRowLogEst + 46; assert( sqlite3LogEst(25)==46 ); + /* Do not scan system tables */ + if( 0==sqlite3StrNICmp(pTab->zName, "sqlite_", 7) ) continue; + + /* Find the size of the table as last recorded in sqlite_stat1. + ** If any index is unanalyzed, then the threshold is -1 to + ** indicate a new, unanalyzed index + */ + szThreshold = pTab->nRowLogEst; + nIndex = 0; for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + nIndex++; if( !pIdx->hasStat1 ){ - szThreshold = 0; /* Always analyze if any index lacks statistics */ - break; + szThreshold = -1; /* Always analyze if any index lacks statistics */ } } - if( szThreshold ){ - sqlite3OpenTable(pParse, iTabCur, iDb, pTab, OP_OpenRead); - sqlite3VdbeAddOp3(v, OP_IfSmaller, iTabCur, - sqlite3VdbeCurrentAddr(v)+2+(opMask&1), szThreshold); + + /* If table pTab has not been used in a way that would benefit from + ** having analysis statistics during the current session, then skip it, + ** unless the 0x10000 MASK bit is set. */ + if( (pTab->tabFlags & TF_MaybeReanalyze)!=0 ){ + /* Check for size change if stat1 has been used for a query */ + }else if( opMask & 0x10000 ){ + /* Check for size change if 0x10000 is set */ + }else if( pTab->pIndex!=0 && szThreshold<0 ){ + /* Do analysis if unanalyzed indexes exists */ + }else{ + /* Otherwise, we can skip this table */ + continue; + } + + nCheck++; + if( nCheck==2 ){ + /* If ANALYZE might be invoked two or more times, hold a write + ** transaction for efficiency */ + sqlite3BeginWriteOperation(pParse, 0, iDb); + } + nBtree += nIndex+1; + + /* Reanalyze if the table is 10 times larger or smaller than + ** the last analysis. Unconditional reanalysis if there are + ** unanalyzed indexes. */ + sqlite3OpenTable(pParse, iTabCur, iDb, pTab, OP_OpenRead); + if( szThreshold>=0 ){ + const LogEst iRange = 33; /* 10x size change */ + sqlite3VdbeAddOp4Int(v, OP_IfSizeBetween, iTabCur, + sqlite3VdbeCurrentAddr(v)+2+(opMask&1), + szThreshold>=iRange ? szThreshold-iRange : -1, + szThreshold+iRange); + VdbeCoverage(v); + }else{ + sqlite3VdbeAddOp2(v, OP_Rewind, iTabCur, + sqlite3VdbeCurrentAddr(v)+2+(opMask&1)); VdbeCoverage(v); } zSubSql = sqlite3MPrintf(db, "ANALYZE \"%w\".\"%w\"", @@ -140366,11 +141301,27 @@ SQLITE_PRIVATE void sqlite3Pragma( sqlite3VdbeAddOp4(v, OP_String8, 0, r1, 0, zSubSql, P4_DYNAMIC); sqlite3VdbeAddOp2(v, OP_ResultRow, r1, 1); }else{ - sqlite3VdbeAddOp4(v, OP_SqlExec, 0, 0, 0, zSubSql, P4_DYNAMIC); + sqlite3VdbeAddOp4(v, OP_SqlExec, nLimit ? 0x02 : 00, nLimit, 0, + zSubSql, P4_DYNAMIC); } } } sqlite3VdbeAddOp0(v, OP_Expire); + + /* In a schema with a large number of tables and indexes, scale back + ** the analysis_limit to avoid excess run-time in the worst case. + */ + if( !db->mallocFailed && nLimit>0 && nBtree>100 ){ + int iAddr, iEnd; + VdbeOp *aOp; + nLimit = 100*nLimit/nBtree; + if( nLimit<100 ) nLimit = 100; + aOp = sqlite3VdbeGetOp(v, 0); + iEnd = sqlite3VdbeCurrentAddr(v); + for(iAddr=0; iAddrnConstraint; i++, pConstraint++){ - if( pConstraint->usable==0 ) continue; - if( pConstraint->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue; if( pConstraint->iColumn < pTab->iHidden ) continue; + if( pConstraint->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue; + if( pConstraint->usable==0 ) return SQLITE_CONSTRAINT; j = pConstraint->iColumn - pTab->iHidden; assert( j < 2 ); seen[j] = i+1; @@ -140649,16 +141600,13 @@ static int pragmaVtabBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ j = seen[0]-1; pIdxInfo->aConstraintUsage[j].argvIndex = 1; pIdxInfo->aConstraintUsage[j].omit = 1; - if( seen[1]==0 ){ - pIdxInfo->estimatedCost = (double)1000; - pIdxInfo->estimatedRows = 1000; - return SQLITE_OK; - } pIdxInfo->estimatedCost = (double)20; pIdxInfo->estimatedRows = 20; - j = seen[1]-1; - pIdxInfo->aConstraintUsage[j].argvIndex = 2; - pIdxInfo->aConstraintUsage[j].omit = 1; + if( seen[1] ){ + j = seen[1]-1; + pIdxInfo->aConstraintUsage[j].argvIndex = 2; + pIdxInfo->aConstraintUsage[j].omit = 1; + } return SQLITE_OK; } @@ -140678,6 +141626,7 @@ static void pragmaVtabCursorClear(PragmaVtabCursor *pCsr){ int i; sqlite3_finalize(pCsr->pPragma); pCsr->pPragma = 0; + pCsr->iRowid = 0; for(i=0; iazArg); i++){ sqlite3_free(pCsr->azArg[i]); pCsr->azArg[i] = 0; @@ -141478,7 +142427,13 @@ SQLITE_PRIVATE void *sqlite3ParserAddCleanup( void (*xCleanup)(sqlite3*,void*), /* The cleanup routine */ void *pPtr /* Pointer to object to be cleaned up */ ){ - ParseCleanup *pCleanup = sqlite3DbMallocRaw(pParse->db, sizeof(*pCleanup)); + ParseCleanup *pCleanup; + if( sqlite3FaultSim(300) ){ + pCleanup = 0; + sqlite3OomFault(pParse->db); + }else{ + pCleanup = sqlite3DbMallocRaw(pParse->db, sizeof(*pCleanup)); + } if( pCleanup ){ pCleanup->pNext = pParse->pCleanup; pParse->pCleanup = pCleanup; @@ -143600,9 +144555,16 @@ static void generateSortTail( int addrExplain; /* Address of OP_Explain instruction */ #endif - ExplainQueryPlan2(addrExplain, (pParse, 0, - "USE TEMP B-TREE FOR %sORDER BY", pSort->nOBSat>0?"RIGHT PART OF ":"") - ); + nKey = pOrderBy->nExpr - pSort->nOBSat; + if( pSort->nOBSat==0 || nKey==1 ){ + ExplainQueryPlan2(addrExplain, (pParse, 0, + "USE TEMP B-TREE FOR %sORDER BY", pSort->nOBSat?"LAST TERM OF ":"" + )); + }else{ + ExplainQueryPlan2(addrExplain, (pParse, 0, + "USE TEMP B-TREE FOR LAST %d TERMS OF ORDER BY", nKey + )); + } sqlite3VdbeScanStatusRange(v, addrExplain,pSort->addrPush,pSort->addrPushEnd); sqlite3VdbeScanStatusCounters(v, addrExplain, addrExplain, pSort->addrPush); @@ -143640,7 +144602,6 @@ static void generateSortTail( regRow = sqlite3GetTempRange(pParse, nColumn); } } - nKey = pOrderBy->nExpr - pSort->nOBSat; if( pSort->sortFlags & SORTFLAG_UseSorter ){ int regSortOut = ++pParse->nMem; iSortTab = pParse->nTab++; @@ -144245,8 +145206,7 @@ SQLITE_PRIVATE void sqlite3SubqueryColumnTypes( NameContext sNC; assert( pSelect!=0 ); - testcase( (pSelect->selFlags & SF_Resolved)==0 ); - assert( (pSelect->selFlags & SF_Resolved)!=0 || IN_RENAME_OBJECT ); + assert( (pSelect->selFlags & SF_Resolved)!=0 ); assert( pTab->nCol==pSelect->pEList->nExpr || pParse->nErr>0 ); assert( aff==SQLITE_AFF_NONE || aff==SQLITE_AFF_BLOB ); if( db->mallocFailed || IN_RENAME_OBJECT ) return; @@ -144257,17 +145217,22 @@ SQLITE_PRIVATE void sqlite3SubqueryColumnTypes( for(i=0, pCol=pTab->aCol; inCol; i++, pCol++){ const char *zType; i64 n; + int m = 0; + Select *pS2 = pSelect; pTab->tabFlags |= (pCol->colFlags & COLFLAG_NOINSERT); p = a[i].pExpr; /* pCol->szEst = ... // Column size est for SELECT tables never used */ pCol->affinity = sqlite3ExprAffinity(p); + while( pCol->affinity<=SQLITE_AFF_NONE && pS2->pNext!=0 ){ + m |= sqlite3ExprDataType(pS2->pEList->a[i].pExpr); + pS2 = pS2->pNext; + pCol->affinity = sqlite3ExprAffinity(pS2->pEList->a[i].pExpr); + } if( pCol->affinity<=SQLITE_AFF_NONE ){ pCol->affinity = aff; } - if( pCol->affinity>=SQLITE_AFF_TEXT && pSelect->pNext ){ - int m = 0; - Select *pS2; - for(m=0, pS2=pSelect->pNext; pS2; pS2=pS2->pNext){ + if( pCol->affinity>=SQLITE_AFF_TEXT && (pS2->pNext || pS2!=pSelect) ){ + for(pS2=pS2->pNext; pS2; pS2=pS2->pNext){ m |= sqlite3ExprDataType(pS2->pEList->a[i].pExpr); } if( pCol->affinity==SQLITE_AFF_TEXT && (m&0x01)!=0 ){ @@ -144297,12 +145262,12 @@ SQLITE_PRIVATE void sqlite3SubqueryColumnTypes( } } if( zType ){ - i64 m = sqlite3Strlen30(zType); + const i64 k = sqlite3Strlen30(zType); n = sqlite3Strlen30(pCol->zCnName); - pCol->zCnName = sqlite3DbReallocOrFree(db, pCol->zCnName, n+m+2); + pCol->zCnName = sqlite3DbReallocOrFree(db, pCol->zCnName, n+k+2); pCol->colFlags &= ~(COLFLAG_HASTYPE|COLFLAG_HASCOLL); if( pCol->zCnName ){ - memcpy(&pCol->zCnName[n+1], zType, m+1); + memcpy(&pCol->zCnName[n+1], zType, k+1); pCol->colFlags |= COLFLAG_HASTYPE; } } @@ -146699,7 +147664,7 @@ static void constInsert( ){ int i; assert( pColumn->op==TK_COLUMN ); - assert( sqlite3ExprIsConstant(pValue) ); + assert( sqlite3ExprIsConstant(pConst->pParse, pValue) ); if( ExprHasProperty(pColumn, EP_FixedCol) ) return; if( sqlite3ExprAffinity(pValue)!=0 ) return; @@ -146757,10 +147722,10 @@ static void findConstInWhere(WhereConst *pConst, Expr *pExpr){ pLeft = pExpr->pLeft; assert( pRight!=0 ); assert( pLeft!=0 ); - if( pRight->op==TK_COLUMN && sqlite3ExprIsConstant(pLeft) ){ + if( pRight->op==TK_COLUMN && sqlite3ExprIsConstant(pConst->pParse, pLeft) ){ constInsert(pConst,pRight,pLeft,pExpr); } - if( pLeft->op==TK_COLUMN && sqlite3ExprIsConstant(pRight) ){ + if( pLeft->op==TK_COLUMN && sqlite3ExprIsConstant(pConst->pParse, pRight) ){ constInsert(pConst,pLeft,pRight,pExpr); } } @@ -146981,6 +147946,18 @@ static int pushDownWindowCheck(Parse *pParse, Select *pSubq, Expr *pExpr){ ** The hope is that the terms added to the inner query will make it more ** efficient. ** +** NAME AMBIGUITY +** +** This optimization is called the "WHERE-clause push-down optimization". +** +** Do not confuse this optimization with another unrelated optimization +** with a similar name: The "MySQL push-down optimization" causes WHERE +** clause terms that can be evaluated using only the index and without +** reference to the table are run first, so that if they are false, +** unnecessary table seeks are avoided. +** +** RULES +** ** Do not attempt this optimization if: ** ** (1) (** This restriction was removed on 2017-09-29. We used to @@ -147046,10 +148023,10 @@ static int pushDownWindowCheck(Parse *pParse, Select *pSubq, Expr *pExpr){ ** (9c) There is a RIGHT JOIN (or FULL JOIN) in between the ON/USING ** clause and the subquery. ** -** Without this restriction, the push-down optimization might move -** the ON/USING filter expression from the left side of a RIGHT JOIN -** over to the right side, which leads to incorrect answers. See -** also restriction (6) in sqlite3ExprIsSingleTableConstraint(). +** Without this restriction, the WHERE-clause push-down optimization +** might move the ON/USING filter expression from the left side of a +** RIGHT JOIN over to the right side, which leads to incorrect answers. +** See also restriction (6) in sqlite3ExprIsSingleTableConstraint(). ** ** (10) The inner query is not the right-hand table of a RIGHT JOIN. ** @@ -147181,7 +148158,7 @@ static int pushDownWhereTerms( } #endif - if( sqlite3ExprIsSingleTableConstraint(pWhere, pSrcList, iSrc) ){ + if( sqlite3ExprIsSingleTableConstraint(pWhere, pSrcList, iSrc, 1) ){ nChng++; pSubq->selFlags |= SF_PushDown; while( pSubq ){ @@ -148316,8 +149293,7 @@ static void selectAddSubqueryTypeInfo(Walker *pWalker, Select *p){ if( p->selFlags & SF_HasTypeInfo ) return; p->selFlags |= SF_HasTypeInfo; pParse = pWalker->pParse; - testcase( (p->selFlags & SF_Resolved)==0 ); - assert( (p->selFlags & SF_Resolved) || IN_RENAME_OBJECT ); + assert( (p->selFlags & SF_Resolved) ); pTabList = p->pSrc; for(i=0, pFrom=pTabList->a; inSrc; i++, pFrom++){ Table *pTab = pFrom->pTab; @@ -148387,6 +149363,8 @@ SQLITE_PRIVATE void sqlite3SelectPrep( */ static void printAggInfo(AggInfo *pAggInfo){ int ii; + sqlite3DebugPrintf("AggInfo %d/%p:\n", + pAggInfo->selId, pAggInfo); for(ii=0; iinColumn; ii++){ struct AggInfo_col *pCol = &pAggInfo->aCol[ii]; sqlite3DebugPrintf( @@ -149577,7 +150555,7 @@ SQLITE_PRIVATE int sqlite3Select( /* Generate code for all sub-queries in the FROM clause */ pSub = pItem->pSelect; - if( pSub==0 ) continue; + if( pSub==0 || pItem->addrFillSub!=0 ) continue; /* The code for a subquery should only be generated once. */ assert( pItem->addrFillSub==0 ); @@ -149608,7 +150586,7 @@ SQLITE_PRIVATE int sqlite3Select( #endif assert( pItem->pSelect && (pItem->pSelect->selFlags & SF_PushDown)!=0 ); }else{ - TREETRACE(0x4000,pParse,p,("Push-down not possible\n")); + TREETRACE(0x4000,pParse,p,("WHERE-lcause push-down not possible\n")); } /* Convert unused result columns of the subquery into simple NULL @@ -150489,6 +151467,12 @@ select_end: sqlite3ExprListDelete(db, pMinMaxOrderBy); #ifdef SQLITE_DEBUG if( pAggInfo && !db->mallocFailed ){ +#if TREETRACE_ENABLED + if( sqlite3TreeTrace & 0x20 ){ + TREETRACE(0x20,pParse,p,("Finished with AggInfo\n")); + printAggInfo(pAggInfo); + } +#endif for(i=0; inColumn; i++){ Expr *pExpr = pAggInfo->aCol[i].pCExpr; if( pExpr==0 ) continue; @@ -151670,6 +152654,72 @@ static ExprList *sqlite3ExpandReturning( return pNew; } +/* If the Expr node is a subquery or an EXISTS operator or an IN operator that +** uses a subquery, and if the subquery is SF_Correlated, then mark the +** expression as EP_VarSelect. +*/ +static int sqlite3ReturningSubqueryVarSelect(Walker *NotUsed, Expr *pExpr){ + UNUSED_PARAMETER(NotUsed); + if( ExprUseXSelect(pExpr) + && (pExpr->x.pSelect->selFlags & SF_Correlated)!=0 + ){ + testcase( ExprHasProperty(pExpr, EP_VarSelect) ); + ExprSetProperty(pExpr, EP_VarSelect); + } + return WRC_Continue; +} + + +/* +** If the SELECT references the table pWalker->u.pTab, then do two things: +** +** (1) Mark the SELECT as as SF_Correlated. +** (2) Set pWalker->eCode to non-zero so that the caller will know +** that (1) has happened. +*/ +static int sqlite3ReturningSubqueryCorrelated(Walker *pWalker, Select *pSelect){ + int i; + SrcList *pSrc; + assert( pSelect!=0 ); + pSrc = pSelect->pSrc; + assert( pSrc!=0 ); + for(i=0; inSrc; i++){ + if( pSrc->a[i].pTab==pWalker->u.pTab ){ + testcase( pSelect->selFlags & SF_Correlated ); + pSelect->selFlags |= SF_Correlated; + pWalker->eCode = 1; + break; + } + } + return WRC_Continue; +} + +/* +** Scan the expression list that is the argument to RETURNING looking +** for subqueries that depend on the table which is being modified in the +** statement that is hosting the RETURNING clause (pTab). Mark all such +** subqueries as SF_Correlated. If the subqueries are part of an +** expression, mark the expression as EP_VarSelect. +** +** https://sqlite.org/forum/forumpost/2c83569ce8945d39 +*/ +static void sqlite3ProcessReturningSubqueries( + ExprList *pEList, + Table *pTab +){ + Walker w; + memset(&w, 0, sizeof(w)); + w.xExprCallback = sqlite3ExprWalkNoop; + w.xSelectCallback = sqlite3ReturningSubqueryCorrelated; + w.u.pTab = pTab; + sqlite3WalkExprList(&w, pEList); + if( w.eCode ){ + w.xExprCallback = sqlite3ReturningSubqueryVarSelect; + w.xSelectCallback = sqlite3SelectWalkNoop; + sqlite3WalkExprList(&w, pEList); + } +} + /* ** Generate code for the RETURNING trigger. Unlike other triggers ** that invoke a subprogram in the bytecode, the code for RETURNING @@ -151706,6 +152756,7 @@ static void codeReturningTrigger( sSelect.pSrc = &sFrom; sFrom.nSrc = 1; sFrom.a[0].pTab = pTab; + sFrom.a[0].zName = pTab->zName; /* tag-20240424-1 */ sFrom.a[0].iCursor = -1; sqlite3SelectPrep(pParse, &sSelect, 0); if( pParse->nErr==0 ){ @@ -151732,6 +152783,7 @@ static void codeReturningTrigger( int i; int nCol = pNew->nExpr; int reg = pParse->nMem+1; + sqlite3ProcessReturningSubqueries(pNew, pTab); pParse->nMem += nCol+2; pReturning->iRetReg = reg; for(i=0; ipVtabCtx = &sCtx; pTab->nTabRef++; rc = xConstruct(db, pMod->pAux, nArg, azArg, &pVTable->pVtab, &zErr); + assert( pTab!=0 ); + assert( pTab->nTabRef>1 || rc!=SQLITE_OK ); sqlite3DeleteTable(db, pTab); db->pVtabCtx = sCtx.pPrior; if( rc==SQLITE_NOMEM ) sqlite3OomFault(db); @@ -154964,7 +156018,7 @@ static int vtabCallConstructor( pVTable->nRef = 1; if( sCtx.bDeclared==0 ){ const char *zFormat = "vtable constructor did not declare schema: %s"; - *pzErr = sqlite3MPrintf(db, zFormat, pTab->zName); + *pzErr = sqlite3MPrintf(db, zFormat, zModuleName); sqlite3VtabUnlock(pVTable); rc = SQLITE_ERROR; }else{ @@ -155142,12 +156196,30 @@ SQLITE_API int sqlite3_declare_vtab(sqlite3 *db, const char *zCreateTable){ Table *pTab; Parse sParse; int initBusy; + int i; + const unsigned char *z; + static const u8 aKeyword[] = { TK_CREATE, TK_TABLE, 0 }; #ifdef SQLITE_ENABLE_API_ARMOR if( !sqlite3SafetyCheckOk(db) || zCreateTable==0 ){ return SQLITE_MISUSE_BKPT; } #endif + + /* Verify that the first two keywords in the CREATE TABLE statement + ** really are "CREATE" and "TABLE". If this is not the case, then + ** sqlite3_declare_vtab() is being misused. + */ + z = (const unsigned char*)zCreateTable; + for(i=0; aKeyword[i]; i++){ + int tokenType = 0; + do{ z += sqlite3GetToken(z, &tokenType); }while( tokenType==TK_SPACE ); + if( tokenType!=aKeyword[i] ){ + sqlite3ErrorWithMsg(db, SQLITE_ERROR, "syntax error"); + return SQLITE_ERROR; + } + } + sqlite3_mutex_enter(db->mutex); pCtx = db->pVtabCtx; if( !pCtx || pCtx->bDeclared ){ @@ -155155,6 +156227,7 @@ SQLITE_API int sqlite3_declare_vtab(sqlite3 *db, const char *zCreateTable){ sqlite3_mutex_leave(db->mutex); return SQLITE_MISUSE_BKPT; } + pTab = pCtx->pTab; assert( IsVirtual(pTab) ); @@ -155168,11 +156241,10 @@ SQLITE_API int sqlite3_declare_vtab(sqlite3 *db, const char *zCreateTable){ initBusy = db->init.busy; db->init.busy = 0; sParse.nQueryLoop = 1; - if( SQLITE_OK==sqlite3RunParser(&sParse, zCreateTable) - && ALWAYS(sParse.pNewTable!=0) - && ALWAYS(!db->mallocFailed) - && IsOrdinaryTable(sParse.pNewTable) - ){ + if( SQLITE_OK==sqlite3RunParser(&sParse, zCreateTable) ){ + assert( sParse.pNewTable!=0 ); + assert( !db->mallocFailed ); + assert( IsOrdinaryTable(sParse.pNewTable) ); assert( sParse.zErrMsg==0 ); if( !pTab->aCol ){ Table *pNew = sParse.pNewTable; @@ -157667,6 +158739,27 @@ static SQLITE_NOINLINE void filterPullDown( } } +/* +** Loop pLoop is a WHERE_INDEXED level that uses at least one IN(...) +** operator. Return true if level pLoop is guaranteed to visit only one +** row for each key generated for the index. +*/ +static int whereLoopIsOneRow(WhereLoop *pLoop){ + if( pLoop->u.btree.pIndex->onError + && pLoop->nSkip==0 + && pLoop->u.btree.nEq==pLoop->u.btree.pIndex->nKeyCol + ){ + int ii; + for(ii=0; iiu.btree.nEq; ii++){ + if( pLoop->aLTerm[ii]->eOperator & (WO_IS|WO_ISNULL) ){ + return 0; + } + } + return 1; + } + return 0; +} + /* ** Generate code for the start of the iLevel-th loop in the WHERE clause ** implementation described by pWInfo. @@ -157745,7 +158838,7 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( if( pLevel->iFrom>0 && (pTabItem[0].fg.jointype & JT_LEFT)!=0 ){ pLevel->iLeftJoin = ++pParse->nMem; sqlite3VdbeAddOp2(v, OP_Integer, 0, pLevel->iLeftJoin); - VdbeComment((v, "init LEFT JOIN no-match flag")); + VdbeComment((v, "init LEFT JOIN match flag")); } /* Compute a safe address to jump to if we discover that the table for @@ -158414,7 +159507,9 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( } /* Record the instruction used to terminate the loop. */ - if( pLoop->wsFlags & WHERE_ONEROW ){ + if( (pLoop->wsFlags & WHERE_ONEROW) + || (pLevel->u.in.nIn && regBignull==0 && whereLoopIsOneRow(pLoop)) + ){ pLevel->op = OP_Noop; }else if( bRev ){ pLevel->op = OP_Prev; @@ -158804,6 +159899,12 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( ** iLoop==3: Code all remaining expressions. ** ** An effort is made to skip unnecessary iterations of the loop. + ** + ** This optimization of causing simple query restrictions to occur before + ** more complex one is call the "push-down" optimization in MySQL. Here + ** in SQLite, the name is "MySQL push-down", since there is also another + ** totally unrelated optimization called "WHERE-clause push-down". + ** Sometimes the qualifier is omitted, resulting in an ambiguity, so beware. */ iLoop = (pIdx ? 1 : 2); do{ @@ -159054,7 +160155,16 @@ SQLITE_PRIVATE SQLITE_NOINLINE void sqlite3WhereRightJoinLoop( pRJ->regReturn); for(k=0; ka[k].pWLoop->iTab == pWInfo->a[k].iFrom ); + pRight = &pWInfo->pTabList->a[pWInfo->a[k].iFrom]; mAll |= pWInfo->a[k].pWLoop->maskSelf; + if( pRight->fg.viaCoroutine ){ + sqlite3VdbeAddOp3( + v, OP_Null, 0, pRight->regResult, + pRight->regResult + pRight->pSelect->pEList->nExpr-1 + ); + } sqlite3VdbeAddOp1(v, OP_NullRow, pWInfo->a[k].iTabCur); iIdxCur = pWInfo->a[k].iIdxCur; if( iIdxCur ){ @@ -160111,7 +161221,7 @@ static SQLITE_NOINLINE int exprMightBeIndexed2( if( pIdx->aiColumn[i]!=XN_EXPR ) continue; assert( pIdx->bHasExpr ); if( sqlite3ExprCompareSkip(pExpr,pIdx->aColExpr->a[i].pExpr,iCur)==0 - && pExpr->op!=TK_STRING + && !sqlite3ExprIsConstant(0,pIdx->aColExpr->a[i].pExpr) ){ aiCurCol[0] = iCur; aiCurCol[1] = XN_EXPR; @@ -160760,6 +161870,7 @@ SQLITE_PRIVATE void SQLITE_NOINLINE sqlite3WhereAddLimit(WhereClause *pWC, Selec continue; } if( pWC->a[ii].leftCursor!=iCsr ) return; + if( pWC->a[ii].prereqRight!=0 ) return; } /* Check condition (5). Return early if it is not met. */ @@ -160774,12 +161885,14 @@ SQLITE_PRIVATE void SQLITE_NOINLINE sqlite3WhereAddLimit(WhereClause *pWC, Selec /* All conditions are met. Add the terms to the where-clause object. */ assert( p->pLimit->op==TK_LIMIT ); - whereAddLimitExpr(pWC, p->iLimit, p->pLimit->pLeft, - iCsr, SQLITE_INDEX_CONSTRAINT_LIMIT); - if( p->iOffset>0 ){ + if( p->iOffset!=0 && (p->selFlags & SF_Compound)==0 ){ whereAddLimitExpr(pWC, p->iOffset, p->pLimit->pRight, iCsr, SQLITE_INDEX_CONSTRAINT_OFFSET); } + if( p->iOffset==0 || (p->selFlags & SF_Compound)==0 ){ + whereAddLimitExpr(pWC, p->iLimit, p->pLimit->pLeft, + iCsr, SQLITE_INDEX_CONSTRAINT_LIMIT); + } } } @@ -161297,6 +162410,42 @@ static Expr *whereRightSubexprIsColumn(Expr *p){ return 0; } +/* +** Term pTerm is guaranteed to be a WO_IN term. It may be a component term +** of a vector IN expression of the form "(x, y, ...) IN (SELECT ...)". +** This function checks to see if the term is compatible with an index +** column with affinity idxaff (one of the SQLITE_AFF_XYZ values). If so, +** it returns a pointer to the name of the collation sequence (e.g. "BINARY" +** or "NOCASE") used by the comparison in pTerm. If it is not compatible +** with affinity idxaff, NULL is returned. +*/ +static SQLITE_NOINLINE const char *indexInAffinityOk( + Parse *pParse, + WhereTerm *pTerm, + u8 idxaff +){ + Expr *pX = pTerm->pExpr; + Expr inexpr; + + assert( pTerm->eOperator & WO_IN ); + + if( sqlite3ExprIsVector(pX->pLeft) ){ + int iField = pTerm->u.x.iField - 1; + inexpr.flags = 0; + inexpr.op = TK_EQ; + inexpr.pLeft = pX->pLeft->x.pList->a[iField].pExpr; + assert( ExprUseXSelect(pX) ); + inexpr.pRight = pX->x.pSelect->pEList->a[iField].pExpr; + pX = &inexpr; + } + + if( sqlite3IndexAffinityOk(pX, idxaff) ){ + CollSeq *pRet = sqlite3ExprCompareCollSeq(pParse, pX); + return pRet ? pRet->zName : sqlite3StrBINARY; + } + return 0; +} + /* ** Advance to the next WhereTerm that matches according to the criteria ** established when the pScan object was initialized by whereScanInit(). @@ -161347,16 +162496,24 @@ static WhereTerm *whereScanNext(WhereScan *pScan){ if( (pTerm->eOperator & pScan->opMask)!=0 ){ /* Verify the affinity and collating sequence match */ if( pScan->zCollName && (pTerm->eOperator & WO_ISNULL)==0 ){ - CollSeq *pColl; + const char *zCollName; Parse *pParse = pWC->pWInfo->pParse; pX = pTerm->pExpr; - if( !sqlite3IndexAffinityOk(pX, pScan->idxaff) ){ - continue; + + if( (pTerm->eOperator & WO_IN) ){ + zCollName = indexInAffinityOk(pParse, pTerm, pScan->idxaff); + if( !zCollName ) continue; + }else{ + CollSeq *pColl; + if( !sqlite3IndexAffinityOk(pX, pScan->idxaff) ){ + continue; + } + assert(pX->pLeft); + pColl = sqlite3ExprCompareCollSeq(pParse, pX); + zCollName = pColl ? pColl->zName : sqlite3StrBINARY; } - assert(pX->pLeft); - pColl = sqlite3ExprCompareCollSeq(pParse, pX); - if( pColl==0 ) pColl = pParse->db->pDfltColl; - if( sqlite3StrICmp(pColl->zName, pScan->zCollName) ){ + + if( sqlite3StrICmp(zCollName, pScan->zCollName) ){ continue; } } @@ -161708,9 +162865,13 @@ static void translateColumnToCopy( ** are no-ops. */ #if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(WHERETRACE_ENABLED) -static void whereTraceIndexInfoInputs(sqlite3_index_info *p){ +static void whereTraceIndexInfoInputs( + sqlite3_index_info *p, /* The IndexInfo object */ + Table *pTab /* The TABLE that is the virtual table */ +){ int i; if( (sqlite3WhereTrace & 0x10)==0 ) return; + sqlite3DebugPrintf("sqlite3_index_info inputs for %s:\n", pTab->zName); for(i=0; inConstraint; i++){ sqlite3DebugPrintf( " constraint[%d]: col=%d termid=%d op=%d usabled=%d collseq=%s\n", @@ -161728,9 +162889,13 @@ static void whereTraceIndexInfoInputs(sqlite3_index_info *p){ p->aOrderBy[i].desc); } } -static void whereTraceIndexInfoOutputs(sqlite3_index_info *p){ +static void whereTraceIndexInfoOutputs( + sqlite3_index_info *p, /* The IndexInfo object */ + Table *pTab /* The TABLE that is the virtual table */ +){ int i; if( (sqlite3WhereTrace & 0x10)==0 ) return; + sqlite3DebugPrintf("sqlite3_index_info outputs for %s:\n", pTab->zName); for(i=0; inConstraint; i++){ sqlite3DebugPrintf(" usage[%d]: argvIdx=%d omit=%d\n", i, @@ -161744,8 +162909,8 @@ static void whereTraceIndexInfoOutputs(sqlite3_index_info *p){ sqlite3DebugPrintf(" estimatedRows=%lld\n", p->estimatedRows); } #else -#define whereTraceIndexInfoInputs(A) -#define whereTraceIndexInfoOutputs(A) +#define whereTraceIndexInfoInputs(A,B) +#define whereTraceIndexInfoOutputs(A,B) #endif /* @@ -161929,7 +163094,7 @@ static SQLITE_NOINLINE void constructAutomaticIndex( ** WHERE clause (or the ON clause of a LEFT join) that constrain which ** rows of the target table (pSrc) that can be used. */ if( (pTerm->wtFlags & TERM_VIRTUAL)==0 - && sqlite3ExprIsSingleTableConstraint(pExpr, pTabList, pLevel->iFrom) + && sqlite3ExprIsSingleTableConstraint(pExpr, pTabList, pLevel->iFrom, 0) ){ pPartial = sqlite3ExprAnd(pParse, pPartial, sqlite3ExprDup(pParse->db, pExpr, 0)); @@ -161971,7 +163136,7 @@ static SQLITE_NOINLINE void constructAutomaticIndex( ** if they go out of sync. */ if( IsView(pTable) ){ - extraCols = ALLBITS; + extraCols = ALLBITS & ~idxCols; }else{ extraCols = pSrc->colUsed & (~idxCols | MASKBIT(BMS-1)); } @@ -162198,7 +163363,7 @@ static SQLITE_NOINLINE void sqlite3ConstructBloomFilter( for(pTerm=pWInfo->sWC.a; pTermpExpr; if( (pTerm->wtFlags & TERM_VIRTUAL)==0 - && sqlite3ExprIsSingleTableConstraint(pExpr, pTabList, iSrc) + && sqlite3ExprIsSingleTableConstraint(pExpr, pTabList, iSrc, 0) ){ sqlite3ExprIfFalse(pParse, pTerm->pExpr, addrCont, SQLITE_JUMPIFNULL); } @@ -162324,7 +163489,7 @@ static sqlite3_index_info *allocateIndexInfo( Expr *pE2; /* Skip over constant terms in the ORDER BY clause */ - if( sqlite3ExprIsConstant(pExpr) ){ + if( sqlite3ExprIsConstant(0, pExpr) ){ continue; } @@ -162359,7 +163524,7 @@ static sqlite3_index_info *allocateIndexInfo( } if( i==n ){ nOrderBy = n; - if( (pWInfo->wctrlFlags & WHERE_DISTINCTBY) ){ + if( (pWInfo->wctrlFlags & WHERE_DISTINCTBY) && !pSrc->fg.rowidUsed ){ eDistinct = 2 + ((pWInfo->wctrlFlags & WHERE_SORTBYGROUP)!=0); }else if( pWInfo->wctrlFlags & WHERE_GROUPBY ){ eDistinct = 1; @@ -162436,7 +163601,7 @@ static sqlite3_index_info *allocateIndexInfo( pIdxInfo->nConstraint = j; for(i=j=0; ia[i].pExpr; - if( sqlite3ExprIsConstant(pExpr) ) continue; + if( sqlite3ExprIsConstant(0, pExpr) ) continue; assert( pExpr->op==TK_COLUMN || (pExpr->op==TK_COLLATE && pExpr->pLeft->op==TK_COLUMN && pExpr->iColumn==pExpr->pLeft->iColumn) ); @@ -162488,11 +163653,11 @@ static int vtabBestIndex(Parse *pParse, Table *pTab, sqlite3_index_info *p){ sqlite3_vtab *pVtab = sqlite3GetVTable(pParse->db, pTab)->pVtab; int rc; - whereTraceIndexInfoInputs(p); + whereTraceIndexInfoInputs(p, pTab); pParse->db->nSchemaLock++; rc = pVtab->pModule->xBestIndex(pVtab, p); pParse->db->nSchemaLock--; - whereTraceIndexInfoOutputs(p); + whereTraceIndexInfoOutputs(p, pTab); if( rc!=SQLITE_OK && rc!=SQLITE_CONSTRAINT ){ if( rc==SQLITE_NOMEM ){ @@ -163970,7 +165135,9 @@ static int whereLoopAddBtreeIndex( } if( pProbe->bUnordered || pProbe->bLowQual ){ if( pProbe->bUnordered ) opMask &= ~(WO_GT|WO_GE|WO_LT|WO_LE); - if( pProbe->bLowQual ) opMask &= ~(WO_EQ|WO_IN|WO_IS); + if( pProbe->bLowQual && pSrc->fg.isIndexedBy==0 ){ + opMask &= ~(WO_EQ|WO_IN|WO_IS); + } } assert( pNew->u.btree.nEqnColumn ); @@ -164237,10 +165404,13 @@ static int whereLoopAddBtreeIndex( } } - /* Set rCostIdx to the cost of visiting selected rows in index. Add - ** it to pNew->rRun, which is currently set to the cost of the index - ** seek only. Then, if this is a non-covering index, add the cost of - ** visiting the rows in the main table. */ + /* Set rCostIdx to the estimated cost of visiting selected rows in the + ** index. The estimate is the sum of two values: + ** 1. The cost of doing one search-by-key to find the first matching + ** entry + ** 2. Stepping forward in the index pNew->nOut times to find all + ** additional matching entries. + */ assert( pSrc->pTab->szTabRow>0 ); if( pProbe->idxType==SQLITE_IDXTYPE_IPK ){ /* The pProbe->szIdxRow is low for an IPK table since the interior @@ -164251,7 +165421,15 @@ static int whereLoopAddBtreeIndex( }else{ rCostIdx = pNew->nOut + 1 + (15*pProbe->szIdxRow)/pSrc->pTab->szTabRow; } - pNew->rRun = sqlite3LogEstAdd(rLogSize, rCostIdx); + rCostIdx = sqlite3LogEstAdd(rLogSize, rCostIdx); + + /* Estimate the cost of running the loop. If all data is coming + ** from the index, then this is just the cost of doing the index + ** lookup and scan. But if some data is coming out of the main table, + ** we also have to add in the cost of doing pNew->nOut searches to + ** locate the row in the main table that corresponds to the index entry. + */ + pNew->rRun = rCostIdx; if( (pNew->wsFlags & (WHERE_IDX_ONLY|WHERE_IPK|WHERE_EXPRIDX))==0 ){ pNew->rRun = sqlite3LogEstAdd(pNew->rRun, pNew->nOut + 16); } @@ -164357,7 +165535,9 @@ static int indexMightHelpWithOrderBy( for(ii=0; iinExpr; ii++){ Expr *pExpr = sqlite3ExprSkipCollateAndLikely(pOB->a[ii].pExpr); if( NEVER(pExpr==0) ) continue; - if( pExpr->op==TK_COLUMN && pExpr->iTable==iCursor ){ + if( (pExpr->op==TK_COLUMN || pExpr->op==TK_AGG_COLUMN) + && pExpr->iTable==iCursor + ){ if( pExpr->iColumn<0 ) return 1; for(jj=0; jjnKeyCol; jj++){ if( pExpr->iColumn==pIndex->aiColumn[jj] ) return 1; @@ -164614,7 +165794,7 @@ static void wherePartIdxExpr( u8 aff; if( pLeft->op!=TK_COLUMN ) return; - if( !sqlite3ExprIsConstant(pRight) ) return; + if( !sqlite3ExprIsConstant(0, pRight) ) return; if( !sqlite3IsBinary(sqlite3ExprCompareCollSeq(pParse, pPart)) ) return; if( pLeft->iColumn<0 ) return; aff = pIdx->pTable->aCol[pLeft->iColumn].affinity; @@ -164963,7 +166143,7 @@ static int whereLoopAddBtree( ** unique index is used (making the index functionally non-unique) ** then the sqlite_stat1 data becomes important for scoring the ** plan */ - pTab->tabFlags |= TF_StatsUsed; + pTab->tabFlags |= TF_MaybeReanalyze; } #ifdef SQLITE_ENABLE_STAT4 sqlite3Stat4ProbeFree(pBuilder->pRec); @@ -164985,6 +166165,21 @@ static int isLimitTerm(WhereTerm *pTerm){ && pTerm->eMatchOp<=SQLITE_INDEX_CONSTRAINT_OFFSET; } +/* +** Return true if the first nCons constraints in the pUsage array are +** marked as in-use (have argvIndex>0). False otherwise. +*/ +static int allConstraintsUsed( + struct sqlite3_index_constraint_usage *aUsage, + int nCons +){ + int ii; + for(ii=0; iipNew->iTab. This @@ -165125,13 +166320,20 @@ static int whereLoopAddVirtualOne( *pbIn = 1; assert( (mExclude & WO_IN)==0 ); } + /* Unless pbRetryLimit is non-NULL, there should be no LIMIT/OFFSET + ** terms. And if there are any, they should follow all other terms. */ assert( pbRetryLimit || !isLimitTerm(pTerm) ); - if( isLimitTerm(pTerm) && *pbIn ){ + assert( !isLimitTerm(pTerm) || i>=nConstraint-2 ); + assert( !isLimitTerm(pTerm) || i==nConstraint-1 || isLimitTerm(pTerm+1) ); + + if( isLimitTerm(pTerm) && (*pbIn || !allConstraintsUsed(pUsage, i)) ){ /* If there is an IN(...) term handled as an == (separate call to ** xFilter for each value on the RHS of the IN) and a LIMIT or - ** OFFSET term handled as well, the plan is unusable. Set output - ** variable *pbRetryLimit to true to tell the caller to retry with - ** LIMIT and OFFSET disabled. */ + ** OFFSET term handled as well, the plan is unusable. Similarly, + ** if there is a LIMIT/OFFSET and there are other unused terms, + ** the plan cannot be used. In these cases set variable *pbRetryLimit + ** to true to tell the caller to retry with LIMIT and OFFSET + ** disabled. */ if( pIdxInfo->needToFreeIdxStr ){ sqlite3_free(pIdxInfo->idxStr); pIdxInfo->idxStr = 0; @@ -165988,7 +167190,7 @@ static i8 wherePathSatisfiesOrderBy( if( MASKBIT(i) & obSat ) continue; p = pOrderBy->a[i].pExpr; mTerm = sqlite3WhereExprUsage(&pWInfo->sMaskSet,p); - if( mTerm==0 && !sqlite3ExprIsConstant(p) ) continue; + if( mTerm==0 && !sqlite3ExprIsConstant(0,p) ) continue; if( (mTerm&~orderDistinctMask)==0 ){ obSat |= MASKBIT(i); } @@ -166457,10 +167659,9 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ if( pFrom->isOrdered==pWInfo->pOrderBy->nExpr ){ pWInfo->eDistinct = WHERE_DISTINCT_ORDERED; } - if( pWInfo->pSelect->pOrderBy - && pWInfo->nOBSat > pWInfo->pSelect->pOrderBy->nExpr ){ - pWInfo->nOBSat = pWInfo->pSelect->pOrderBy->nExpr; - } + /* vvv--- See check-in [12ad822d9b827777] on 2023-03-16 ---vvv */ + assert( pWInfo->pSelect->pOrderBy==0 + || pWInfo->nOBSat <= pWInfo->pSelect->pOrderBy->nExpr ); }else{ pWInfo->revMask = pFrom->revLoop; if( pWInfo->nOBSat<=0 ){ @@ -166503,7 +167704,6 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ } } - pWInfo->nRowOut = pFrom->nRow; /* Free temporary memory and return success */ @@ -166511,6 +167711,83 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ return SQLITE_OK; } +/* +** This routine implements a heuristic designed to improve query planning. +** This routine is called in between the first and second call to +** wherePathSolver(). Hence the name "Interstage" "Heuristic". +** +** The first call to wherePathSolver() (hereafter just "solver()") computes +** the best path without regard to the order of the outputs. The second call +** to the solver() builds upon the first call to try to find an alternative +** path that satisfies the ORDER BY clause. +** +** This routine looks at the results of the first solver() run, and for +** every FROM clause term in the resulting query plan that uses an equality +** constraint against an index, disable other WhereLoops for that same +** FROM clause term that would try to do a full-table scan. This prevents +** an index search from being converted into a full-table scan in order to +** satisfy an ORDER BY clause, since even though we might get slightly better +** performance using the full-scan without sorting if the output size +** estimates are very precise, we might also get severe performance +** degradation using the full-scan if the output size estimate is too large. +** It is better to err on the side of caution. +** +** Except, if the first solver() call generated a full-table scan in an outer +** loop then stop this analysis at the first full-scan, since the second +** solver() run might try to swap that full-scan for another in order to +** get the output into the correct order. In other words, we allow a +** rewrite like this: +** +** First Solver() Second Solver() +** |-- SCAN t1 |-- SCAN t2 +** |-- SEARCH t2 `-- SEARCH t1 +** `-- SORT USING B-TREE +** +** The purpose of this routine is to disallow rewrites such as: +** +** First Solver() Second Solver() +** |-- SEARCH t1 |-- SCAN t2 <--- bad! +** |-- SEARCH t2 `-- SEARCH t1 +** `-- SORT USING B-TREE +** +** See test cases in test/whereN.test for the real-world query that +** originally provoked this heuristic. +*/ +static SQLITE_NOINLINE void whereInterstageHeuristic(WhereInfo *pWInfo){ + int i; +#ifdef WHERETRACE_ENABLED + int once = 0; +#endif + for(i=0; inLevel; i++){ + WhereLoop *p = pWInfo->a[i].pWLoop; + if( p==0 ) break; + if( (p->wsFlags & WHERE_VIRTUALTABLE)!=0 ) continue; + if( (p->wsFlags & (WHERE_COLUMN_EQ|WHERE_COLUMN_NULL|WHERE_COLUMN_IN))!=0 ){ + u8 iTab = p->iTab; + WhereLoop *pLoop; + for(pLoop=pWInfo->pLoops; pLoop; pLoop=pLoop->pNextLoop){ + if( pLoop->iTab!=iTab ) continue; + if( (pLoop->wsFlags & (WHERE_CONSTRAINT|WHERE_AUTO_INDEX))!=0 ){ + /* Auto-index and index-constrained loops allowed to remain */ + continue; + } +#ifdef WHERETRACE_ENABLED + if( sqlite3WhereTrace & 0x80 ){ + if( once==0 ){ + sqlite3DebugPrintf("Loops disabled by interstage heuristic:\n"); + once = 1; + } + sqlite3WhereLoopPrint(pLoop, &pWInfo->sWC); + } +#endif /* WHERETRACE_ENABLED */ + pLoop->prereq = ALLBITS; /* Prevent 2nd solver() from using this one */ + } + }else{ + break; + } + } +} + /* ** Most queries use only a single table (they are not joins) and have ** simple == constraints against indexed fields. This routine attempts @@ -166799,7 +168076,7 @@ static SQLITE_NOINLINE void whereCheckIfBloomFilterIsUseful( SrcItem *pItem = &pWInfo->pTabList->a[pLoop->iTab]; Table *pTab = pItem->pTab; if( (pTab->tabFlags & TF_HasStat1)==0 ) break; - pTab->tabFlags |= TF_StatsUsed; + pTab->tabFlags |= TF_MaybeReanalyze; if( i>=1 && (pLoop->wsFlags & reqFlags)==reqFlags /* vvvvvv--- Always the case if WHERE_COLUMN_EQ is defined */ @@ -166820,6 +168097,58 @@ static SQLITE_NOINLINE void whereCheckIfBloomFilterIsUseful( } } +/* +** Expression Node callback for sqlite3ExprCanReturnSubtype(). +** +** Only a function call is able to return a subtype. So if the node +** is not a function call, return WRC_Prune immediately. +** +** A function call is able to return a subtype if it has the +** SQLITE_RESULT_SUBTYPE property. +** +** Assume that every function is able to pass-through a subtype from +** one of its argument (using sqlite3_result_value()). Most functions +** are not this way, but we don't have a mechanism to distinguish those +** that are from those that are not, so assume they all work this way. +** That means that if one of its arguments is another function and that +** other function is able to return a subtype, then this function is +** able to return a subtype. +*/ +static int exprNodeCanReturnSubtype(Walker *pWalker, Expr *pExpr){ + int n; + FuncDef *pDef; + sqlite3 *db; + if( pExpr->op!=TK_FUNCTION ){ + return WRC_Prune; + } + assert( ExprUseXList(pExpr) ); + db = pWalker->pParse->db; + n = pExpr->x.pList ? pExpr->x.pList->nExpr : 0; + pDef = sqlite3FindFunction(db, pExpr->u.zToken, n, ENC(db), 0); + if( pDef==0 || (pDef->funcFlags & SQLITE_RESULT_SUBTYPE)!=0 ){ + pWalker->eCode = 1; + return WRC_Prune; + } + return WRC_Continue; +} + +/* +** Return TRUE if expression pExpr is able to return a subtype. +** +** A TRUE return does not guarantee that a subtype will be returned. +** It only indicates that a subtype return is possible. False positives +** are acceptable as they only disable an optimization. False negatives, +** on the other hand, can lead to incorrect answers. +*/ +static int sqlite3ExprCanReturnSubtype(Parse *pParse, Expr *pExpr){ + Walker w; + memset(&w, 0, sizeof(w)); + w.pParse = pParse; + w.xExprCallback = exprNodeCanReturnSubtype; + sqlite3WalkExpr(&w, pExpr); + return w.eCode; +} + /* ** The index pIdx is used by a query and contains one or more expressions. ** In other words pIdx is an index on an expression. iIdxCur is the cursor @@ -166852,20 +168181,12 @@ static SQLITE_NOINLINE void whereAddIndexedExpr( }else{ continue; } - if( sqlite3ExprIsConstant(pExpr) ) continue; - if( pExpr->op==TK_FUNCTION ){ + if( sqlite3ExprIsConstant(0,pExpr) ) continue; + if( pExpr->op==TK_FUNCTION && sqlite3ExprCanReturnSubtype(pParse,pExpr) ){ /* Functions that might set a subtype should not be replaced by the ** value taken from an expression index since the index omits the ** subtype. https://sqlite.org/forum/forumpost/68d284c86b082c3e */ - int n; - FuncDef *pDef; - sqlite3 *db = pParse->db; - assert( ExprUseXList(pExpr) ); - n = pExpr->x.pList ? pExpr->x.pList->nExpr : 0; - pDef = sqlite3FindFunction(db, pExpr->u.zToken, n, ENC(db), 0); - if( pDef==0 || (pDef->funcFlags & SQLITE_RESULT_SUBTYPE)!=0 ){ - continue; - } + continue; } p = sqlite3DbMallocRaw(pParse->db, sizeof(IndexedExpr)); if( p==0 ) break; @@ -167130,7 +168451,11 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( ){ pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE; } - ExplainQueryPlan((pParse, 0, "SCAN CONSTANT ROW")); + if( ALWAYS(pWInfo->pSelect) + && (pWInfo->pSelect->selFlags & SF_MultiValue)==0 + ){ + ExplainQueryPlan((pParse, 0, "SCAN CONSTANT ROW")); + } }else{ /* Assign a bit from the bitmask to every term in the FROM clause. ** @@ -167283,6 +168608,7 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( wherePathSolver(pWInfo, 0); if( db->mallocFailed ) goto whereBeginError; if( pWInfo->pOrderBy ){ + whereInterstageHeuristic(pWInfo); wherePathSolver(pWInfo, pWInfo->nRowOut+1); if( db->mallocFailed ) goto whereBeginError; } @@ -167832,7 +169158,15 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ addr = sqlite3VdbeAddOp1(v, OP_IfPos, pLevel->iLeftJoin); VdbeCoverage(v); assert( (ws & WHERE_IDX_ONLY)==0 || (ws & WHERE_INDEXED)!=0 ); if( (ws & WHERE_IDX_ONLY)==0 ){ - assert( pLevel->iTabCur==pTabList->a[pLevel->iFrom].iCursor ); + SrcItem *pSrc = &pTabList->a[pLevel->iFrom]; + assert( pLevel->iTabCur==pSrc->iCursor ); + if( pSrc->fg.viaCoroutine ){ + int m, n; + n = pSrc->regResult; + assert( pSrc->pTab!=0 ); + m = pSrc->pTab->nCol; + sqlite3VdbeAddOp3(v, OP_Null, 0, n, n+m-1); + } sqlite3VdbeAddOp1(v, OP_NullRow, pLevel->iTabCur); } if( (ws & WHERE_INDEXED) @@ -167882,6 +169216,7 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ */ if( pTabItem->fg.viaCoroutine ){ testcase( pParse->db->mallocFailed ); + assert( pTabItem->regResult>=0 ); translateColumnToCopy(pParse, pLevel->addrBody, pLevel->iTabCur, pTabItem->regResult, 0); continue; @@ -169186,7 +170521,7 @@ SQLITE_PRIVATE void sqlite3WindowListDelete(sqlite3 *db, Window *p){ ** variable values in the expression tree. */ static Expr *sqlite3WindowOffsetExpr(Parse *pParse, Expr *pExpr){ - if( 0==sqlite3ExprIsConstant(pExpr) ){ + if( 0==sqlite3ExprIsConstant(0,pExpr) ){ if( IN_RENAME_OBJECT ) sqlite3RenameExprUnmap(pParse, pExpr); sqlite3ExprDelete(pParse->db, pExpr); pExpr = sqlite3ExprAlloc(pParse->db, TK_NULL, 0, 0); @@ -171278,6 +172613,14 @@ static void updateDeleteLimitError( return pSelect; } + /* Memory allocator for parser stack resizing. This is a thin wrapper around + ** sqlite3_realloc() that includes a call to sqlite3FaultSim() to facilitate + ** testing. + */ + static void *parserStackRealloc(void *pOld, sqlite3_uint64 newSize){ + return sqlite3FaultSim(700) ? 0 : sqlite3_realloc(pOld, newSize); + } + /* Construct a new Expr object from a single token */ static Expr *tokenExpr(Parse *pParse, int op, Token t){ @@ -171527,8 +172870,8 @@ static void updateDeleteLimitError( #define TK_TRUEFALSE 170 #define TK_ISNOT 171 #define TK_FUNCTION 172 -#define TK_UMINUS 173 -#define TK_UPLUS 174 +#define TK_UPLUS 173 +#define TK_UMINUS 174 #define TK_TRUTH 175 #define TK_REGISTER 176 #define TK_VECTOR 177 @@ -171537,8 +172880,9 @@ static void updateDeleteLimitError( #define TK_ASTERISK 180 #define TK_SPAN 181 #define TK_ERROR 182 -#define TK_SPACE 183 -#define TK_ILLEGAL 184 +#define TK_QNUMBER 183 +#define TK_SPACE 184 +#define TK_ILLEGAL 185 #endif /**************** End token definitions ***************************************/ @@ -171579,6 +172923,9 @@ static void updateDeleteLimitError( ** sqlite3ParserARG_STORE Code to store %extra_argument into yypParser ** sqlite3ParserARG_FETCH Code to extract %extra_argument from yypParser ** sqlite3ParserCTX_* As sqlite3ParserARG_ except for %extra_context +** YYREALLOC Name of the realloc() function to use +** YYFREE Name of the free() function to use +** YYDYNSTACK True if stack space should be extended on heap ** YYERRORSYMBOL is the code number of the error symbol. If not ** defined, then do no error processing. ** YYNSTATE the combined number of states. @@ -171592,37 +172939,39 @@ static void updateDeleteLimitError( ** YY_NO_ACTION The yy_action[] code for no-op ** YY_MIN_REDUCE Minimum value for reduce actions ** YY_MAX_REDUCE Maximum value for reduce actions +** YY_MIN_DSTRCTR Minimum symbol value that has a destructor +** YY_MAX_DSTRCTR Maximum symbol value that has a destructor */ #ifndef INTERFACE # define INTERFACE 1 #endif /************* Begin control #defines *****************************************/ #define YYCODETYPE unsigned short int -#define YYNOCODE 319 +#define YYNOCODE 322 #define YYACTIONTYPE unsigned short int #define YYWILDCARD 101 #define sqlite3ParserTOKENTYPE Token typedef union { int yyinit; sqlite3ParserTOKENTYPE yy0; - TriggerStep* yy33; - Window* yy41; - Select* yy47; - SrcList* yy131; - struct TrigEvent yy180; - struct {int value; int mask;} yy231; - IdList* yy254; - u32 yy285; - ExprList* yy322; - Cte* yy385; - int yy394; - Upsert* yy444; - u8 yy516; - With* yy521; - const char* yy522; - Expr* yy528; - OnOrUsing yy561; - struct FrameBound yy595; + ExprList* yy14; + With* yy59; + Cte* yy67; + Upsert* yy122; + IdList* yy132; + int yy144; + const char* yy168; + SrcList* yy203; + Window* yy211; + OnOrUsing yy269; + struct TrigEvent yy286; + struct {int value; int mask;} yy383; + u32 yy391; + TriggerStep* yy427; + Expr* yy454; + u8 yy462; + struct FrameBound yy509; + Select* yy555; } YYMINORTYPE; #ifndef YYSTACKDEPTH #define YYSTACKDEPTH 100 @@ -171632,24 +172981,29 @@ typedef union { #define sqlite3ParserARG_PARAM #define sqlite3ParserARG_FETCH #define sqlite3ParserARG_STORE +#define YYREALLOC parserStackRealloc +#define YYFREE sqlite3_free +#define YYDYNSTACK 1 #define sqlite3ParserCTX_SDECL Parse *pParse; #define sqlite3ParserCTX_PDECL ,Parse *pParse #define sqlite3ParserCTX_PARAM ,pParse #define sqlite3ParserCTX_FETCH Parse *pParse=yypParser->pParse; #define sqlite3ParserCTX_STORE yypParser->pParse=pParse; #define YYFALLBACK 1 -#define YYNSTATE 579 -#define YYNRULE 405 -#define YYNRULE_WITH_ACTION 340 -#define YYNTOKEN 185 -#define YY_MAX_SHIFT 578 -#define YY_MIN_SHIFTREDUCE 838 -#define YY_MAX_SHIFTREDUCE 1242 -#define YY_ERROR_ACTION 1243 -#define YY_ACCEPT_ACTION 1244 -#define YY_NO_ACTION 1245 -#define YY_MIN_REDUCE 1246 -#define YY_MAX_REDUCE 1650 +#define YYNSTATE 583 +#define YYNRULE 409 +#define YYNRULE_WITH_ACTION 344 +#define YYNTOKEN 186 +#define YY_MAX_SHIFT 582 +#define YY_MIN_SHIFTREDUCE 845 +#define YY_MAX_SHIFTREDUCE 1253 +#define YY_ERROR_ACTION 1254 +#define YY_ACCEPT_ACTION 1255 +#define YY_NO_ACTION 1256 +#define YY_MIN_REDUCE 1257 +#define YY_MAX_REDUCE 1665 +#define YY_MIN_DSTRCTR 205 +#define YY_MAX_DSTRCTR 319 /************* End control #defines *******************************************/ #define YY_NLOOKAHEAD ((int)(sizeof(yy_lookahead)/sizeof(yy_lookahead[0]))) @@ -171665,6 +173019,22 @@ typedef union { # define yytestcase(X) #endif +/* Macro to determine if stack space has the ability to grow using +** heap memory. +*/ +#if YYSTACKDEPTH<=0 || YYDYNSTACK +# define YYGROWABLESTACK 1 +#else +# define YYGROWABLESTACK 0 +#endif + +/* Guarantee a minimum number of initial stack slots. +*/ +#if YYSTACKDEPTH<=0 +# undef YYSTACKDEPTH +# define YYSTACKDEPTH 2 /* Need a minimum stack size */ +#endif + /* Next are the tables used to determine what action to take based on the ** current state and lookahead token. These tables are used to implement @@ -171716,619 +173086,630 @@ typedef union { ** yy_default[] Default action for each state. ** *********** Begin parsing tables **********************************************/ -#define YY_ACTTAB_COUNT (2100) +#define YY_ACTTAB_COUNT (2142) static const YYACTIONTYPE yy_action[] = { - /* 0 */ 572, 210, 572, 119, 116, 231, 572, 119, 116, 231, - /* 10 */ 572, 1317, 379, 1296, 410, 566, 566, 566, 572, 411, - /* 20 */ 380, 1317, 1279, 42, 42, 42, 42, 210, 1529, 72, - /* 30 */ 72, 974, 421, 42, 42, 495, 305, 281, 305, 975, - /* 40 */ 399, 72, 72, 126, 127, 81, 1217, 1217, 1054, 1057, - /* 50 */ 1044, 1044, 124, 124, 125, 125, 125, 125, 480, 411, - /* 60 */ 1244, 1, 1, 578, 2, 1248, 554, 119, 116, 231, - /* 70 */ 319, 484, 147, 484, 528, 119, 116, 231, 533, 1330, - /* 80 */ 419, 527, 143, 126, 127, 81, 1217, 1217, 1054, 1057, - /* 90 */ 1044, 1044, 124, 124, 125, 125, 125, 125, 119, 116, - /* 100 */ 231, 329, 123, 123, 123, 123, 122, 122, 121, 121, - /* 110 */ 121, 120, 117, 448, 286, 286, 286, 286, 446, 446, - /* 120 */ 446, 1568, 378, 1570, 1193, 377, 1164, 569, 1164, 569, - /* 130 */ 411, 1568, 541, 261, 228, 448, 102, 146, 453, 318, - /* 140 */ 563, 242, 123, 123, 123, 123, 122, 122, 121, 121, - /* 150 */ 121, 120, 117, 448, 126, 127, 81, 1217, 1217, 1054, - /* 160 */ 1057, 1044, 1044, 124, 124, 125, 125, 125, 125, 143, - /* 170 */ 296, 1193, 341, 452, 121, 121, 121, 120, 117, 448, - /* 180 */ 128, 1193, 1194, 1193, 149, 445, 444, 572, 120, 117, - /* 190 */ 448, 125, 125, 125, 125, 118, 123, 123, 123, 123, - /* 200 */ 122, 122, 121, 121, 121, 120, 117, 448, 458, 114, - /* 210 */ 13, 13, 550, 123, 123, 123, 123, 122, 122, 121, - /* 220 */ 121, 121, 120, 117, 448, 424, 318, 563, 1193, 1194, - /* 230 */ 1193, 150, 1225, 411, 1225, 125, 125, 125, 125, 123, - /* 240 */ 123, 123, 123, 122, 122, 121, 121, 121, 120, 117, - /* 250 */ 448, 469, 344, 1041, 1041, 1055, 1058, 126, 127, 81, - /* 260 */ 1217, 1217, 1054, 1057, 1044, 1044, 124, 124, 125, 125, - /* 270 */ 125, 125, 1282, 526, 224, 1193, 572, 411, 226, 519, - /* 280 */ 177, 83, 84, 123, 123, 123, 123, 122, 122, 121, - /* 290 */ 121, 121, 120, 117, 448, 1010, 16, 16, 1193, 134, - /* 300 */ 134, 126, 127, 81, 1217, 1217, 1054, 1057, 1044, 1044, - /* 310 */ 124, 124, 125, 125, 125, 125, 123, 123, 123, 123, - /* 320 */ 122, 122, 121, 121, 121, 120, 117, 448, 1045, 550, - /* 330 */ 1193, 375, 1193, 1194, 1193, 254, 1438, 401, 508, 505, - /* 340 */ 504, 112, 564, 570, 4, 929, 929, 435, 503, 342, - /* 350 */ 464, 330, 362, 396, 1238, 1193, 1194, 1193, 567, 572, - /* 360 */ 123, 123, 123, 123, 122, 122, 121, 121, 121, 120, - /* 370 */ 117, 448, 286, 286, 371, 1581, 1607, 445, 444, 155, - /* 380 */ 411, 449, 72, 72, 1289, 569, 1222, 1193, 1194, 1193, - /* 390 */ 86, 1224, 273, 561, 547, 520, 520, 572, 99, 1223, - /* 400 */ 6, 1281, 476, 143, 126, 127, 81, 1217, 1217, 1054, - /* 410 */ 1057, 1044, 1044, 124, 124, 125, 125, 125, 125, 554, - /* 420 */ 13, 13, 1031, 511, 1225, 1193, 1225, 553, 110, 110, - /* 430 */ 224, 572, 1239, 177, 572, 429, 111, 199, 449, 573, - /* 440 */ 449, 432, 1555, 1019, 327, 555, 1193, 272, 289, 370, - /* 450 */ 514, 365, 513, 259, 72, 72, 547, 72, 72, 361, - /* 460 */ 318, 563, 1613, 123, 123, 123, 123, 122, 122, 121, - /* 470 */ 121, 121, 120, 117, 448, 1019, 1019, 1021, 1022, 28, - /* 480 */ 286, 286, 1193, 1194, 1193, 1159, 572, 1612, 411, 904, - /* 490 */ 192, 554, 358, 569, 554, 940, 537, 521, 1159, 437, - /* 500 */ 415, 1159, 556, 1193, 1194, 1193, 572, 548, 548, 52, - /* 510 */ 52, 216, 126, 127, 81, 1217, 1217, 1054, 1057, 1044, - /* 520 */ 1044, 124, 124, 125, 125, 125, 125, 1193, 478, 136, - /* 530 */ 136, 411, 286, 286, 1493, 509, 122, 122, 121, 121, - /* 540 */ 121, 120, 117, 448, 1010, 569, 522, 219, 545, 545, - /* 550 */ 318, 563, 143, 6, 536, 126, 127, 81, 1217, 1217, - /* 560 */ 1054, 1057, 1044, 1044, 124, 124, 125, 125, 125, 125, - /* 570 */ 1557, 123, 123, 123, 123, 122, 122, 121, 121, 121, - /* 580 */ 120, 117, 448, 489, 1193, 1194, 1193, 486, 283, 1270, - /* 590 */ 960, 254, 1193, 375, 508, 505, 504, 1193, 342, 574, - /* 600 */ 1193, 574, 411, 294, 503, 960, 879, 193, 484, 318, - /* 610 */ 563, 386, 292, 382, 123, 123, 123, 123, 122, 122, - /* 620 */ 121, 121, 121, 120, 117, 448, 126, 127, 81, 1217, - /* 630 */ 1217, 1054, 1057, 1044, 1044, 124, 124, 125, 125, 125, - /* 640 */ 125, 411, 396, 1139, 1193, 872, 101, 286, 286, 1193, - /* 650 */ 1194, 1193, 375, 1096, 1193, 1194, 1193, 1193, 1194, 1193, - /* 660 */ 569, 459, 33, 375, 235, 126, 127, 81, 1217, 1217, - /* 670 */ 1054, 1057, 1044, 1044, 124, 124, 125, 125, 125, 125, - /* 680 */ 1437, 962, 572, 230, 961, 123, 123, 123, 123, 122, - /* 690 */ 122, 121, 121, 121, 120, 117, 448, 1159, 230, 1193, - /* 700 */ 158, 1193, 1194, 1193, 1556, 13, 13, 303, 960, 1233, - /* 710 */ 1159, 154, 411, 1159, 375, 1584, 1177, 5, 371, 1581, - /* 720 */ 431, 1239, 3, 960, 123, 123, 123, 123, 122, 122, - /* 730 */ 121, 121, 121, 120, 117, 448, 126, 127, 81, 1217, - /* 740 */ 1217, 1054, 1057, 1044, 1044, 124, 124, 125, 125, 125, - /* 750 */ 125, 411, 210, 571, 1193, 1032, 1193, 1194, 1193, 1193, - /* 760 */ 390, 855, 156, 1555, 376, 404, 1101, 1101, 492, 572, - /* 770 */ 469, 344, 1322, 1322, 1555, 126, 127, 81, 1217, 1217, - /* 780 */ 1054, 1057, 1044, 1044, 124, 124, 125, 125, 125, 125, - /* 790 */ 130, 572, 13, 13, 532, 123, 123, 123, 123, 122, - /* 800 */ 122, 121, 121, 121, 120, 117, 448, 304, 572, 457, - /* 810 */ 229, 1193, 1194, 1193, 13, 13, 1193, 1194, 1193, 1300, - /* 820 */ 467, 1270, 411, 1320, 1320, 1555, 1015, 457, 456, 436, - /* 830 */ 301, 72, 72, 1268, 123, 123, 123, 123, 122, 122, - /* 840 */ 121, 121, 121, 120, 117, 448, 126, 127, 81, 1217, - /* 850 */ 1217, 1054, 1057, 1044, 1044, 124, 124, 125, 125, 125, - /* 860 */ 125, 411, 384, 1076, 1159, 286, 286, 421, 314, 280, - /* 870 */ 280, 287, 287, 461, 408, 407, 1539, 1159, 569, 572, - /* 880 */ 1159, 1196, 569, 409, 569, 126, 127, 81, 1217, 1217, - /* 890 */ 1054, 1057, 1044, 1044, 124, 124, 125, 125, 125, 125, - /* 900 */ 457, 1485, 13, 13, 1541, 123, 123, 123, 123, 122, - /* 910 */ 122, 121, 121, 121, 120, 117, 448, 202, 572, 462, - /* 920 */ 1587, 578, 2, 1248, 843, 844, 845, 1563, 319, 409, - /* 930 */ 147, 6, 411, 257, 256, 255, 208, 1330, 9, 1196, - /* 940 */ 264, 72, 72, 1436, 123, 123, 123, 123, 122, 122, - /* 950 */ 121, 121, 121, 120, 117, 448, 126, 127, 81, 1217, - /* 960 */ 1217, 1054, 1057, 1044, 1044, 124, 124, 125, 125, 125, - /* 970 */ 125, 572, 286, 286, 572, 1213, 411, 577, 315, 1248, - /* 980 */ 421, 371, 1581, 356, 319, 569, 147, 495, 529, 1644, - /* 990 */ 397, 935, 495, 1330, 71, 71, 934, 72, 72, 242, - /* 1000 */ 1328, 105, 81, 1217, 1217, 1054, 1057, 1044, 1044, 124, - /* 1010 */ 124, 125, 125, 125, 125, 123, 123, 123, 123, 122, - /* 1020 */ 122, 121, 121, 121, 120, 117, 448, 1117, 286, 286, - /* 1030 */ 1422, 452, 1528, 1213, 443, 286, 286, 1492, 1355, 313, - /* 1040 */ 478, 569, 1118, 454, 351, 495, 354, 1266, 569, 209, - /* 1050 */ 572, 418, 179, 572, 1031, 242, 385, 1119, 523, 123, - /* 1060 */ 123, 123, 123, 122, 122, 121, 121, 121, 120, 117, - /* 1070 */ 448, 1020, 108, 72, 72, 1019, 13, 13, 915, 572, - /* 1080 */ 1498, 572, 286, 286, 98, 530, 1537, 452, 916, 1334, - /* 1090 */ 1329, 203, 411, 286, 286, 569, 152, 211, 1498, 1500, - /* 1100 */ 426, 569, 56, 56, 57, 57, 569, 1019, 1019, 1021, - /* 1110 */ 447, 572, 411, 531, 12, 297, 126, 127, 81, 1217, - /* 1120 */ 1217, 1054, 1057, 1044, 1044, 124, 124, 125, 125, 125, - /* 1130 */ 125, 572, 411, 867, 15, 15, 126, 127, 81, 1217, - /* 1140 */ 1217, 1054, 1057, 1044, 1044, 124, 124, 125, 125, 125, - /* 1150 */ 125, 373, 529, 264, 44, 44, 126, 115, 81, 1217, - /* 1160 */ 1217, 1054, 1057, 1044, 1044, 124, 124, 125, 125, 125, - /* 1170 */ 125, 1498, 478, 1271, 417, 123, 123, 123, 123, 122, - /* 1180 */ 122, 121, 121, 121, 120, 117, 448, 205, 1213, 495, - /* 1190 */ 430, 867, 468, 322, 495, 123, 123, 123, 123, 122, - /* 1200 */ 122, 121, 121, 121, 120, 117, 448, 572, 557, 1140, - /* 1210 */ 1642, 1422, 1642, 543, 572, 123, 123, 123, 123, 122, - /* 1220 */ 122, 121, 121, 121, 120, 117, 448, 572, 1422, 572, - /* 1230 */ 13, 13, 542, 323, 1325, 411, 334, 58, 58, 349, - /* 1240 */ 1422, 1170, 326, 286, 286, 549, 1213, 300, 895, 530, - /* 1250 */ 45, 45, 59, 59, 1140, 1643, 569, 1643, 565, 417, - /* 1260 */ 127, 81, 1217, 1217, 1054, 1057, 1044, 1044, 124, 124, - /* 1270 */ 125, 125, 125, 125, 1367, 373, 500, 290, 1193, 512, - /* 1280 */ 1366, 427, 394, 394, 393, 275, 391, 896, 1138, 852, - /* 1290 */ 478, 258, 1422, 1170, 463, 1159, 12, 331, 428, 333, - /* 1300 */ 1117, 460, 236, 258, 325, 460, 544, 1544, 1159, 1098, - /* 1310 */ 491, 1159, 324, 1098, 440, 1118, 335, 516, 123, 123, - /* 1320 */ 123, 123, 122, 122, 121, 121, 121, 120, 117, 448, - /* 1330 */ 1119, 318, 563, 1138, 572, 1193, 1194, 1193, 112, 564, - /* 1340 */ 201, 4, 238, 433, 935, 490, 285, 228, 1517, 934, - /* 1350 */ 170, 560, 572, 142, 1516, 567, 572, 60, 60, 572, - /* 1360 */ 416, 572, 441, 572, 535, 302, 875, 8, 487, 572, - /* 1370 */ 237, 572, 416, 572, 485, 61, 61, 572, 449, 62, - /* 1380 */ 62, 332, 63, 63, 46, 46, 47, 47, 361, 572, - /* 1390 */ 561, 572, 48, 48, 50, 50, 51, 51, 572, 295, - /* 1400 */ 64, 64, 482, 295, 539, 412, 471, 1031, 572, 538, - /* 1410 */ 318, 563, 65, 65, 66, 66, 409, 475, 572, 1031, - /* 1420 */ 572, 14, 14, 875, 1020, 110, 110, 409, 1019, 572, - /* 1430 */ 474, 67, 67, 111, 455, 449, 573, 449, 98, 317, - /* 1440 */ 1019, 132, 132, 133, 133, 572, 1561, 572, 974, 409, - /* 1450 */ 6, 1562, 68, 68, 1560, 6, 975, 572, 6, 1559, - /* 1460 */ 1019, 1019, 1021, 6, 346, 218, 101, 531, 53, 53, - /* 1470 */ 69, 69, 1019, 1019, 1021, 1022, 28, 1586, 1181, 451, - /* 1480 */ 70, 70, 290, 87, 215, 31, 1363, 394, 394, 393, - /* 1490 */ 275, 391, 350, 109, 852, 107, 572, 112, 564, 483, - /* 1500 */ 4, 1212, 572, 239, 153, 572, 39, 236, 1299, 325, - /* 1510 */ 112, 564, 1298, 4, 567, 572, 32, 324, 572, 54, - /* 1520 */ 54, 572, 1135, 353, 398, 165, 165, 567, 166, 166, - /* 1530 */ 572, 291, 355, 572, 17, 357, 572, 449, 77, 77, - /* 1540 */ 1313, 55, 55, 1297, 73, 73, 572, 238, 470, 561, - /* 1550 */ 449, 472, 364, 135, 135, 170, 74, 74, 142, 163, - /* 1560 */ 163, 374, 561, 539, 572, 321, 572, 886, 540, 137, - /* 1570 */ 137, 339, 1353, 422, 298, 237, 539, 572, 1031, 572, - /* 1580 */ 340, 538, 101, 369, 110, 110, 162, 131, 131, 164, - /* 1590 */ 164, 1031, 111, 368, 449, 573, 449, 110, 110, 1019, - /* 1600 */ 157, 157, 141, 141, 572, 111, 572, 449, 573, 449, - /* 1610 */ 412, 288, 1019, 572, 882, 318, 563, 572, 219, 572, - /* 1620 */ 241, 1012, 477, 263, 263, 894, 893, 140, 140, 138, - /* 1630 */ 138, 1019, 1019, 1021, 1022, 28, 139, 139, 525, 455, - /* 1640 */ 76, 76, 78, 78, 1019, 1019, 1021, 1022, 28, 1181, - /* 1650 */ 451, 572, 1083, 290, 112, 564, 1575, 4, 394, 394, - /* 1660 */ 393, 275, 391, 572, 1023, 852, 572, 479, 345, 263, - /* 1670 */ 101, 567, 882, 1376, 75, 75, 1421, 501, 236, 260, - /* 1680 */ 325, 112, 564, 359, 4, 101, 43, 43, 324, 49, - /* 1690 */ 49, 901, 902, 161, 449, 101, 977, 978, 567, 1079, - /* 1700 */ 1349, 260, 965, 932, 263, 114, 561, 1095, 517, 1095, - /* 1710 */ 1083, 1094, 865, 1094, 151, 933, 1144, 114, 238, 1361, - /* 1720 */ 558, 449, 1023, 559, 1426, 1278, 170, 1269, 1257, 142, - /* 1730 */ 1601, 1256, 1258, 561, 1594, 1031, 496, 278, 213, 1346, - /* 1740 */ 310, 110, 110, 939, 311, 312, 237, 11, 234, 111, - /* 1750 */ 221, 449, 573, 449, 293, 395, 1019, 1408, 337, 1403, - /* 1760 */ 1396, 338, 1031, 299, 343, 1413, 1412, 481, 110, 110, - /* 1770 */ 506, 402, 225, 1296, 206, 367, 111, 1358, 449, 573, - /* 1780 */ 449, 412, 1359, 1019, 1489, 1488, 318, 563, 1019, 1019, - /* 1790 */ 1021, 1022, 28, 562, 207, 220, 80, 564, 389, 4, - /* 1800 */ 1597, 1357, 552, 1356, 1233, 181, 267, 232, 1536, 1534, - /* 1810 */ 455, 1230, 420, 567, 82, 1019, 1019, 1021, 1022, 28, - /* 1820 */ 86, 217, 85, 1494, 190, 175, 183, 465, 185, 466, - /* 1830 */ 36, 1409, 186, 187, 188, 499, 449, 244, 37, 99, - /* 1840 */ 400, 1415, 1414, 488, 1417, 194, 473, 403, 561, 1483, - /* 1850 */ 248, 92, 1505, 494, 198, 279, 112, 564, 250, 4, - /* 1860 */ 348, 497, 405, 352, 1259, 251, 252, 515, 1316, 434, - /* 1870 */ 1315, 1314, 94, 567, 1307, 886, 1306, 1031, 226, 406, - /* 1880 */ 1611, 1610, 438, 110, 110, 1580, 1286, 524, 439, 308, - /* 1890 */ 266, 111, 1285, 449, 573, 449, 449, 309, 1019, 366, - /* 1900 */ 1284, 1609, 265, 1566, 1565, 442, 372, 1381, 561, 129, - /* 1910 */ 550, 1380, 10, 1470, 383, 106, 316, 551, 100, 35, - /* 1920 */ 534, 575, 212, 1339, 381, 387, 1187, 1338, 274, 276, - /* 1930 */ 1019, 1019, 1021, 1022, 28, 277, 413, 1031, 576, 1254, - /* 1940 */ 388, 1521, 1249, 110, 110, 167, 1522, 168, 148, 1520, - /* 1950 */ 1519, 111, 306, 449, 573, 449, 222, 223, 1019, 839, - /* 1960 */ 169, 79, 450, 214, 414, 233, 320, 145, 1093, 1091, - /* 1970 */ 328, 182, 171, 1212, 918, 184, 240, 336, 243, 1107, - /* 1980 */ 189, 172, 173, 423, 425, 88, 180, 191, 89, 90, - /* 1990 */ 1019, 1019, 1021, 1022, 28, 91, 174, 1110, 245, 1106, - /* 2000 */ 246, 159, 18, 247, 347, 1099, 263, 195, 1227, 493, - /* 2010 */ 249, 196, 38, 854, 498, 368, 253, 360, 897, 197, - /* 2020 */ 502, 93, 19, 20, 507, 884, 363, 510, 95, 307, - /* 2030 */ 160, 96, 518, 97, 1175, 1060, 1146, 40, 21, 227, - /* 2040 */ 176, 1145, 282, 284, 969, 200, 963, 114, 262, 1165, - /* 2050 */ 22, 23, 24, 1161, 1169, 25, 1163, 1150, 34, 26, - /* 2060 */ 1168, 546, 27, 204, 101, 103, 104, 1074, 7, 1061, - /* 2070 */ 1059, 1063, 1116, 1064, 1115, 268, 269, 29, 41, 270, - /* 2080 */ 1024, 866, 113, 30, 568, 392, 1183, 144, 178, 1182, - /* 2090 */ 271, 928, 1245, 1245, 1245, 1245, 1245, 1245, 1245, 1602, + /* 0 */ 576, 128, 125, 232, 1622, 549, 576, 1290, 1281, 576, + /* 10 */ 328, 576, 1300, 212, 576, 128, 125, 232, 578, 412, + /* 20 */ 578, 391, 1542, 51, 51, 523, 405, 1293, 529, 51, + /* 30 */ 51, 983, 51, 51, 81, 81, 1107, 61, 61, 984, + /* 40 */ 1107, 1292, 380, 135, 136, 90, 1228, 1228, 1063, 1066, + /* 50 */ 1053, 1053, 133, 133, 134, 134, 134, 134, 1577, 412, + /* 60 */ 287, 287, 7, 287, 287, 422, 1050, 1050, 1064, 1067, + /* 70 */ 289, 556, 492, 573, 524, 561, 573, 497, 561, 482, + /* 80 */ 530, 262, 229, 135, 136, 90, 1228, 1228, 1063, 1066, + /* 90 */ 1053, 1053, 133, 133, 134, 134, 134, 134, 128, 125, + /* 100 */ 232, 1506, 132, 132, 132, 132, 131, 131, 130, 130, + /* 110 */ 130, 129, 126, 450, 1204, 1255, 1, 1, 582, 2, + /* 120 */ 1259, 1571, 420, 1582, 379, 320, 1174, 153, 1174, 1584, + /* 130 */ 412, 378, 1582, 543, 1341, 330, 111, 570, 570, 570, + /* 140 */ 293, 1054, 132, 132, 132, 132, 131, 131, 130, 130, + /* 150 */ 130, 129, 126, 450, 135, 136, 90, 1228, 1228, 1063, + /* 160 */ 1066, 1053, 1053, 133, 133, 134, 134, 134, 134, 287, + /* 170 */ 287, 1204, 1205, 1204, 255, 287, 287, 510, 507, 506, + /* 180 */ 137, 455, 573, 212, 561, 447, 446, 505, 573, 1616, + /* 190 */ 561, 134, 134, 134, 134, 127, 400, 243, 132, 132, + /* 200 */ 132, 132, 131, 131, 130, 130, 130, 129, 126, 450, + /* 210 */ 282, 471, 345, 132, 132, 132, 132, 131, 131, 130, + /* 220 */ 130, 130, 129, 126, 450, 574, 155, 936, 936, 454, + /* 230 */ 227, 521, 1236, 412, 1236, 134, 134, 134, 134, 132, + /* 240 */ 132, 132, 132, 131, 131, 130, 130, 130, 129, 126, + /* 250 */ 450, 130, 130, 130, 129, 126, 450, 135, 136, 90, + /* 260 */ 1228, 1228, 1063, 1066, 1053, 1053, 133, 133, 134, 134, + /* 270 */ 134, 134, 128, 125, 232, 450, 576, 412, 397, 1249, + /* 280 */ 180, 92, 93, 132, 132, 132, 132, 131, 131, 130, + /* 290 */ 130, 130, 129, 126, 450, 381, 387, 1204, 383, 81, + /* 300 */ 81, 135, 136, 90, 1228, 1228, 1063, 1066, 1053, 1053, + /* 310 */ 133, 133, 134, 134, 134, 134, 132, 132, 132, 132, + /* 320 */ 131, 131, 130, 130, 130, 129, 126, 450, 131, 131, + /* 330 */ 130, 130, 130, 129, 126, 450, 556, 1204, 302, 319, + /* 340 */ 567, 121, 568, 480, 4, 555, 1149, 1657, 1628, 1657, + /* 350 */ 45, 128, 125, 232, 1204, 1205, 1204, 1250, 571, 1169, + /* 360 */ 132, 132, 132, 132, 131, 131, 130, 130, 130, 129, + /* 370 */ 126, 450, 1169, 287, 287, 1169, 1019, 576, 422, 1019, + /* 380 */ 412, 451, 1602, 582, 2, 1259, 573, 44, 561, 95, + /* 390 */ 320, 110, 153, 565, 1204, 1205, 1204, 522, 522, 1341, + /* 400 */ 81, 81, 7, 44, 135, 136, 90, 1228, 1228, 1063, + /* 410 */ 1066, 1053, 1053, 133, 133, 134, 134, 134, 134, 295, + /* 420 */ 1149, 1658, 1040, 1658, 1204, 1147, 319, 567, 119, 119, + /* 430 */ 343, 466, 331, 343, 287, 287, 120, 556, 451, 577, + /* 440 */ 451, 1169, 1169, 1028, 319, 567, 438, 573, 210, 561, + /* 450 */ 1339, 1451, 546, 531, 1169, 1169, 1598, 1169, 1169, 416, + /* 460 */ 319, 567, 243, 132, 132, 132, 132, 131, 131, 130, + /* 470 */ 130, 130, 129, 126, 450, 1028, 1028, 1030, 1031, 35, + /* 480 */ 44, 1204, 1205, 1204, 472, 287, 287, 1328, 412, 1307, + /* 490 */ 372, 1595, 359, 225, 454, 1204, 195, 1328, 573, 1147, + /* 500 */ 561, 1333, 1333, 274, 576, 1188, 576, 340, 46, 196, + /* 510 */ 537, 217, 135, 136, 90, 1228, 1228, 1063, 1066, 1053, + /* 520 */ 1053, 133, 133, 134, 134, 134, 134, 19, 19, 19, + /* 530 */ 19, 412, 581, 1204, 1259, 511, 1204, 319, 567, 320, + /* 540 */ 944, 153, 425, 491, 430, 943, 1204, 488, 1341, 1450, + /* 550 */ 532, 1277, 1204, 1205, 1204, 135, 136, 90, 1228, 1228, + /* 560 */ 1063, 1066, 1053, 1053, 133, 133, 134, 134, 134, 134, + /* 570 */ 575, 132, 132, 132, 132, 131, 131, 130, 130, 130, + /* 580 */ 129, 126, 450, 287, 287, 528, 287, 287, 372, 1595, + /* 590 */ 1204, 1205, 1204, 1204, 1205, 1204, 573, 486, 561, 573, + /* 600 */ 889, 561, 412, 1204, 1205, 1204, 886, 40, 22, 22, + /* 610 */ 220, 243, 525, 1449, 132, 132, 132, 132, 131, 131, + /* 620 */ 130, 130, 130, 129, 126, 450, 135, 136, 90, 1228, + /* 630 */ 1228, 1063, 1066, 1053, 1053, 133, 133, 134, 134, 134, + /* 640 */ 134, 412, 180, 454, 1204, 879, 255, 287, 287, 510, + /* 650 */ 507, 506, 372, 1595, 1568, 1331, 1331, 576, 889, 505, + /* 660 */ 573, 44, 561, 559, 1207, 135, 136, 90, 1228, 1228, + /* 670 */ 1063, 1066, 1053, 1053, 133, 133, 134, 134, 134, 134, + /* 680 */ 81, 81, 422, 576, 377, 132, 132, 132, 132, 131, + /* 690 */ 131, 130, 130, 130, 129, 126, 450, 297, 287, 287, + /* 700 */ 460, 1204, 1205, 1204, 1204, 534, 19, 19, 448, 448, + /* 710 */ 448, 573, 412, 561, 230, 436, 1187, 535, 319, 567, + /* 720 */ 363, 432, 1207, 1435, 132, 132, 132, 132, 131, 131, + /* 730 */ 130, 130, 130, 129, 126, 450, 135, 136, 90, 1228, + /* 740 */ 1228, 1063, 1066, 1053, 1053, 133, 133, 134, 134, 134, + /* 750 */ 134, 412, 211, 949, 1169, 1041, 1110, 1110, 494, 547, + /* 760 */ 547, 1204, 1205, 1204, 7, 539, 1570, 1169, 376, 576, + /* 770 */ 1169, 5, 1204, 486, 3, 135, 136, 90, 1228, 1228, + /* 780 */ 1063, 1066, 1053, 1053, 133, 133, 134, 134, 134, 134, + /* 790 */ 576, 513, 19, 19, 427, 132, 132, 132, 132, 131, + /* 800 */ 131, 130, 130, 130, 129, 126, 450, 305, 1204, 433, + /* 810 */ 225, 1204, 385, 19, 19, 273, 290, 371, 516, 366, + /* 820 */ 515, 260, 412, 538, 1568, 549, 1024, 362, 437, 1204, + /* 830 */ 1205, 1204, 902, 1552, 132, 132, 132, 132, 131, 131, + /* 840 */ 130, 130, 130, 129, 126, 450, 135, 136, 90, 1228, + /* 850 */ 1228, 1063, 1066, 1053, 1053, 133, 133, 134, 134, 134, + /* 860 */ 134, 412, 1435, 514, 1281, 1204, 1205, 1204, 1204, 1205, + /* 870 */ 1204, 903, 48, 342, 1568, 1568, 1279, 1627, 1568, 911, + /* 880 */ 576, 129, 126, 450, 110, 135, 136, 90, 1228, 1228, + /* 890 */ 1063, 1066, 1053, 1053, 133, 133, 134, 134, 134, 134, + /* 900 */ 265, 576, 459, 19, 19, 132, 132, 132, 132, 131, + /* 910 */ 131, 130, 130, 130, 129, 126, 450, 1345, 204, 576, + /* 920 */ 459, 458, 50, 47, 19, 19, 49, 434, 1105, 573, + /* 930 */ 497, 561, 412, 428, 108, 1224, 1569, 1554, 376, 205, + /* 940 */ 550, 550, 81, 81, 132, 132, 132, 132, 131, 131, + /* 950 */ 130, 130, 130, 129, 126, 450, 135, 136, 90, 1228, + /* 960 */ 1228, 1063, 1066, 1053, 1053, 133, 133, 134, 134, 134, + /* 970 */ 134, 480, 576, 1204, 576, 1541, 412, 1435, 969, 315, + /* 980 */ 1659, 398, 284, 497, 969, 893, 1569, 1569, 376, 376, + /* 990 */ 1569, 461, 376, 1224, 459, 80, 80, 81, 81, 497, + /* 1000 */ 374, 114, 90, 1228, 1228, 1063, 1066, 1053, 1053, 133, + /* 1010 */ 133, 134, 134, 134, 134, 132, 132, 132, 132, 131, + /* 1020 */ 131, 130, 130, 130, 129, 126, 450, 1204, 1505, 576, + /* 1030 */ 1204, 1205, 1204, 1366, 316, 486, 281, 281, 497, 431, + /* 1040 */ 557, 288, 288, 402, 1340, 471, 345, 298, 429, 573, + /* 1050 */ 576, 561, 81, 81, 573, 374, 561, 971, 386, 132, + /* 1060 */ 132, 132, 132, 131, 131, 130, 130, 130, 129, 126, + /* 1070 */ 450, 231, 117, 81, 81, 287, 287, 231, 287, 287, + /* 1080 */ 576, 1511, 576, 1336, 1204, 1205, 1204, 139, 573, 556, + /* 1090 */ 561, 573, 412, 561, 441, 456, 969, 213, 558, 1511, + /* 1100 */ 1513, 1550, 969, 143, 143, 145, 145, 1368, 314, 478, + /* 1110 */ 444, 970, 412, 850, 851, 852, 135, 136, 90, 1228, + /* 1120 */ 1228, 1063, 1066, 1053, 1053, 133, 133, 134, 134, 134, + /* 1130 */ 134, 357, 412, 397, 1148, 304, 135, 136, 90, 1228, + /* 1140 */ 1228, 1063, 1066, 1053, 1053, 133, 133, 134, 134, 134, + /* 1150 */ 134, 1575, 323, 6, 862, 7, 135, 124, 90, 1228, + /* 1160 */ 1228, 1063, 1066, 1053, 1053, 133, 133, 134, 134, 134, + /* 1170 */ 134, 409, 408, 1511, 212, 132, 132, 132, 132, 131, + /* 1180 */ 131, 130, 130, 130, 129, 126, 450, 411, 118, 1204, + /* 1190 */ 116, 10, 352, 265, 355, 132, 132, 132, 132, 131, + /* 1200 */ 131, 130, 130, 130, 129, 126, 450, 576, 324, 306, + /* 1210 */ 576, 306, 1250, 469, 158, 132, 132, 132, 132, 131, + /* 1220 */ 131, 130, 130, 130, 129, 126, 450, 207, 1224, 1126, + /* 1230 */ 65, 65, 470, 66, 66, 412, 447, 446, 882, 531, + /* 1240 */ 335, 258, 257, 256, 1127, 1233, 1204, 1205, 1204, 327, + /* 1250 */ 1235, 874, 159, 576, 16, 480, 1085, 1040, 1234, 1128, + /* 1260 */ 136, 90, 1228, 1228, 1063, 1066, 1053, 1053, 133, 133, + /* 1270 */ 134, 134, 134, 134, 1029, 576, 81, 81, 1028, 1040, + /* 1280 */ 922, 576, 463, 1236, 576, 1236, 1224, 502, 107, 1435, + /* 1290 */ 923, 6, 576, 410, 1498, 882, 1029, 480, 21, 21, + /* 1300 */ 1028, 332, 1380, 334, 53, 53, 497, 81, 81, 874, + /* 1310 */ 1028, 1028, 1030, 445, 259, 19, 19, 533, 132, 132, + /* 1320 */ 132, 132, 131, 131, 130, 130, 130, 129, 126, 450, + /* 1330 */ 551, 301, 1028, 1028, 1030, 107, 532, 545, 121, 568, + /* 1340 */ 1188, 4, 1126, 1576, 449, 576, 462, 7, 1282, 418, + /* 1350 */ 462, 350, 1435, 576, 518, 571, 544, 1127, 121, 568, + /* 1360 */ 442, 4, 1188, 464, 533, 1180, 1223, 9, 67, 67, + /* 1370 */ 487, 576, 1128, 303, 410, 571, 54, 54, 451, 576, + /* 1380 */ 123, 944, 576, 417, 576, 333, 943, 1379, 576, 236, + /* 1390 */ 565, 576, 1574, 564, 68, 68, 7, 576, 451, 362, + /* 1400 */ 419, 182, 69, 69, 541, 70, 70, 71, 71, 540, + /* 1410 */ 565, 72, 72, 484, 55, 55, 473, 1180, 296, 1040, + /* 1420 */ 56, 56, 296, 493, 541, 119, 119, 410, 1573, 542, + /* 1430 */ 569, 418, 7, 120, 1244, 451, 577, 451, 465, 1040, + /* 1440 */ 1028, 576, 1557, 552, 476, 119, 119, 527, 259, 121, + /* 1450 */ 568, 240, 4, 120, 576, 451, 577, 451, 576, 477, + /* 1460 */ 1028, 576, 156, 576, 57, 57, 571, 576, 286, 229, + /* 1470 */ 410, 336, 1028, 1028, 1030, 1031, 35, 59, 59, 219, + /* 1480 */ 983, 60, 60, 220, 73, 73, 74, 74, 984, 451, + /* 1490 */ 75, 75, 1028, 1028, 1030, 1031, 35, 96, 216, 291, + /* 1500 */ 552, 565, 1188, 318, 395, 395, 394, 276, 392, 576, + /* 1510 */ 485, 859, 474, 1311, 410, 541, 576, 417, 1530, 1144, + /* 1520 */ 540, 399, 1188, 292, 237, 1153, 326, 38, 23, 576, + /* 1530 */ 1040, 576, 20, 20, 325, 299, 119, 119, 164, 76, + /* 1540 */ 76, 1529, 121, 568, 120, 4, 451, 577, 451, 203, + /* 1550 */ 576, 1028, 141, 141, 142, 142, 576, 322, 39, 571, + /* 1560 */ 341, 1021, 110, 264, 239, 901, 900, 423, 242, 908, + /* 1570 */ 909, 370, 173, 77, 77, 43, 479, 1310, 264, 62, + /* 1580 */ 62, 369, 451, 1028, 1028, 1030, 1031, 35, 1601, 1192, + /* 1590 */ 453, 1092, 238, 291, 565, 163, 1309, 110, 395, 395, + /* 1600 */ 394, 276, 392, 986, 987, 859, 481, 346, 264, 110, + /* 1610 */ 1032, 489, 576, 1188, 503, 1088, 261, 261, 237, 576, + /* 1620 */ 326, 121, 568, 1040, 4, 347, 1376, 413, 325, 119, + /* 1630 */ 119, 948, 319, 567, 351, 78, 78, 120, 571, 451, + /* 1640 */ 577, 451, 79, 79, 1028, 354, 356, 576, 360, 1092, + /* 1650 */ 110, 576, 974, 942, 264, 123, 457, 358, 239, 576, + /* 1660 */ 519, 451, 939, 1104, 123, 1104, 173, 576, 1032, 43, + /* 1670 */ 63, 63, 1324, 565, 168, 168, 1028, 1028, 1030, 1031, + /* 1680 */ 35, 576, 169, 169, 1308, 872, 238, 157, 1589, 576, + /* 1690 */ 86, 86, 365, 89, 568, 375, 4, 1103, 941, 1103, + /* 1700 */ 123, 576, 1040, 1389, 64, 64, 1188, 1434, 119, 119, + /* 1710 */ 571, 576, 82, 82, 563, 576, 120, 165, 451, 577, + /* 1720 */ 451, 413, 1362, 1028, 144, 144, 319, 567, 576, 1374, + /* 1730 */ 562, 498, 279, 451, 83, 83, 1439, 576, 166, 166, + /* 1740 */ 576, 1289, 554, 576, 1280, 565, 576, 12, 576, 1268, + /* 1750 */ 457, 146, 146, 1267, 576, 1028, 1028, 1030, 1031, 35, + /* 1760 */ 140, 140, 1269, 167, 167, 1609, 160, 160, 1359, 150, + /* 1770 */ 150, 149, 149, 311, 1040, 576, 312, 147, 147, 313, + /* 1780 */ 119, 119, 222, 235, 576, 1188, 396, 576, 120, 576, + /* 1790 */ 451, 577, 451, 1192, 453, 1028, 508, 291, 148, 148, + /* 1800 */ 1421, 1612, 395, 395, 394, 276, 392, 85, 85, 859, + /* 1810 */ 87, 87, 84, 84, 553, 576, 294, 576, 1426, 338, + /* 1820 */ 339, 1425, 237, 300, 326, 1416, 1409, 1028, 1028, 1030, + /* 1830 */ 1031, 35, 325, 344, 403, 483, 226, 1307, 52, 52, + /* 1840 */ 58, 58, 368, 1371, 1502, 566, 1501, 121, 568, 221, + /* 1850 */ 4, 208, 268, 209, 390, 1244, 1549, 1188, 1372, 1370, + /* 1860 */ 1369, 1547, 239, 184, 571, 233, 421, 1241, 95, 218, + /* 1870 */ 173, 1507, 193, 43, 91, 94, 178, 186, 467, 188, + /* 1880 */ 468, 1422, 13, 189, 190, 191, 501, 451, 245, 108, + /* 1890 */ 238, 401, 1428, 1427, 1430, 475, 404, 1496, 197, 565, + /* 1900 */ 14, 490, 249, 101, 1518, 496, 349, 280, 251, 201, + /* 1910 */ 353, 499, 252, 406, 1270, 253, 517, 1327, 1326, 435, + /* 1920 */ 1325, 1318, 103, 893, 1296, 413, 227, 407, 1040, 1626, + /* 1930 */ 319, 567, 1625, 1297, 119, 119, 439, 367, 1317, 1295, + /* 1940 */ 1624, 526, 120, 440, 451, 577, 451, 1594, 309, 1028, + /* 1950 */ 310, 373, 266, 267, 457, 1580, 1579, 443, 138, 1394, + /* 1960 */ 552, 1393, 11, 1483, 384, 115, 317, 1350, 109, 536, + /* 1970 */ 42, 579, 382, 214, 1349, 388, 1198, 389, 275, 277, + /* 1980 */ 278, 1028, 1028, 1030, 1031, 35, 580, 1265, 414, 1260, + /* 1990 */ 170, 415, 183, 1534, 1535, 1533, 171, 154, 307, 1532, + /* 2000 */ 846, 223, 224, 88, 452, 215, 172, 321, 234, 1102, + /* 2010 */ 152, 1188, 1100, 329, 185, 174, 1223, 925, 187, 241, + /* 2020 */ 337, 244, 1116, 192, 175, 176, 424, 426, 97, 194, + /* 2030 */ 98, 99, 100, 177, 1119, 1115, 246, 247, 161, 24, + /* 2040 */ 248, 348, 1238, 264, 1108, 250, 495, 199, 198, 15, + /* 2050 */ 861, 500, 369, 254, 504, 509, 512, 200, 102, 25, + /* 2060 */ 179, 361, 26, 364, 104, 891, 308, 162, 105, 904, + /* 2070 */ 520, 106, 1185, 1069, 1155, 17, 228, 27, 1154, 283, + /* 2080 */ 285, 263, 978, 202, 972, 123, 28, 1175, 29, 30, + /* 2090 */ 1179, 1171, 31, 1173, 1160, 41, 32, 206, 548, 33, + /* 2100 */ 110, 1178, 1083, 8, 112, 1070, 113, 1068, 1072, 34, + /* 2110 */ 1073, 560, 1125, 269, 1124, 270, 36, 18, 1194, 1033, + /* 2120 */ 873, 151, 122, 37, 393, 271, 272, 572, 181, 1193, + /* 2130 */ 1256, 1256, 1256, 935, 1256, 1256, 1256, 1256, 1256, 1256, + /* 2140 */ 1256, 1617, }; static const YYCODETYPE yy_lookahead[] = { - /* 0 */ 193, 193, 193, 274, 275, 276, 193, 274, 275, 276, - /* 10 */ 193, 223, 219, 225, 206, 210, 211, 212, 193, 19, - /* 20 */ 219, 233, 216, 216, 217, 216, 217, 193, 295, 216, - /* 30 */ 217, 31, 193, 216, 217, 193, 228, 213, 230, 39, - /* 40 */ 206, 216, 217, 43, 44, 45, 46, 47, 48, 49, - /* 50 */ 50, 51, 52, 53, 54, 55, 56, 57, 193, 19, - /* 60 */ 185, 186, 187, 188, 189, 190, 253, 274, 275, 276, - /* 70 */ 195, 193, 197, 193, 261, 274, 275, 276, 253, 204, - /* 80 */ 238, 204, 81, 43, 44, 45, 46, 47, 48, 49, - /* 90 */ 50, 51, 52, 53, 54, 55, 56, 57, 274, 275, - /* 100 */ 276, 262, 102, 103, 104, 105, 106, 107, 108, 109, - /* 110 */ 110, 111, 112, 113, 239, 240, 239, 240, 210, 211, - /* 120 */ 212, 314, 315, 314, 59, 316, 86, 252, 88, 252, - /* 130 */ 19, 314, 315, 256, 257, 113, 25, 72, 296, 138, - /* 140 */ 139, 266, 102, 103, 104, 105, 106, 107, 108, 109, + /* 0 */ 194, 276, 277, 278, 216, 194, 194, 217, 194, 194, + /* 10 */ 194, 194, 224, 194, 194, 276, 277, 278, 204, 19, + /* 20 */ 206, 202, 297, 217, 218, 205, 207, 217, 205, 217, + /* 30 */ 218, 31, 217, 218, 217, 218, 29, 217, 218, 39, + /* 40 */ 33, 217, 220, 43, 44, 45, 46, 47, 48, 49, + /* 50 */ 50, 51, 52, 53, 54, 55, 56, 57, 312, 19, + /* 60 */ 240, 241, 316, 240, 241, 194, 46, 47, 48, 49, + /* 70 */ 22, 254, 65, 253, 254, 255, 253, 194, 255, 194, + /* 80 */ 263, 258, 259, 43, 44, 45, 46, 47, 48, 49, + /* 90 */ 50, 51, 52, 53, 54, 55, 56, 57, 276, 277, + /* 100 */ 278, 285, 102, 103, 104, 105, 106, 107, 108, 109, + /* 110 */ 110, 111, 112, 113, 59, 186, 187, 188, 189, 190, + /* 120 */ 191, 310, 239, 317, 318, 196, 86, 198, 88, 317, + /* 130 */ 19, 319, 317, 318, 205, 264, 25, 211, 212, 213, + /* 140 */ 205, 121, 102, 103, 104, 105, 106, 107, 108, 109, /* 150 */ 110, 111, 112, 113, 43, 44, 45, 46, 47, 48, - /* 160 */ 49, 50, 51, 52, 53, 54, 55, 56, 57, 81, - /* 170 */ 292, 59, 292, 298, 108, 109, 110, 111, 112, 113, - /* 180 */ 69, 116, 117, 118, 72, 106, 107, 193, 111, 112, - /* 190 */ 113, 54, 55, 56, 57, 58, 102, 103, 104, 105, - /* 200 */ 106, 107, 108, 109, 110, 111, 112, 113, 120, 25, - /* 210 */ 216, 217, 145, 102, 103, 104, 105, 106, 107, 108, - /* 220 */ 109, 110, 111, 112, 113, 231, 138, 139, 116, 117, - /* 230 */ 118, 164, 153, 19, 155, 54, 55, 56, 57, 102, + /* 160 */ 49, 50, 51, 52, 53, 54, 55, 56, 57, 240, + /* 170 */ 241, 116, 117, 118, 119, 240, 241, 122, 123, 124, + /* 180 */ 69, 298, 253, 194, 255, 106, 107, 132, 253, 141, + /* 190 */ 255, 54, 55, 56, 57, 58, 207, 268, 102, 103, + /* 200 */ 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, + /* 210 */ 214, 128, 129, 102, 103, 104, 105, 106, 107, 108, + /* 220 */ 109, 110, 111, 112, 113, 134, 25, 136, 137, 300, + /* 230 */ 165, 166, 153, 19, 155, 54, 55, 56, 57, 102, /* 240 */ 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, - /* 250 */ 113, 128, 129, 46, 47, 48, 49, 43, 44, 45, + /* 250 */ 113, 108, 109, 110, 111, 112, 113, 43, 44, 45, /* 260 */ 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, - /* 270 */ 56, 57, 216, 193, 25, 59, 193, 19, 165, 166, - /* 280 */ 193, 67, 24, 102, 103, 104, 105, 106, 107, 108, - /* 290 */ 109, 110, 111, 112, 113, 73, 216, 217, 59, 216, - /* 300 */ 217, 43, 44, 45, 46, 47, 48, 49, 50, 51, + /* 270 */ 56, 57, 276, 277, 278, 113, 194, 19, 22, 23, + /* 280 */ 194, 67, 24, 102, 103, 104, 105, 106, 107, 108, + /* 290 */ 109, 110, 111, 112, 113, 220, 250, 59, 252, 217, + /* 300 */ 218, 43, 44, 45, 46, 47, 48, 49, 50, 51, /* 310 */ 52, 53, 54, 55, 56, 57, 102, 103, 104, 105, - /* 320 */ 106, 107, 108, 109, 110, 111, 112, 113, 121, 145, - /* 330 */ 59, 193, 116, 117, 118, 119, 273, 204, 122, 123, - /* 340 */ 124, 19, 20, 134, 22, 136, 137, 19, 132, 127, - /* 350 */ 128, 129, 24, 22, 23, 116, 117, 118, 36, 193, + /* 320 */ 106, 107, 108, 109, 110, 111, 112, 113, 106, 107, + /* 330 */ 108, 109, 110, 111, 112, 113, 254, 59, 205, 138, + /* 340 */ 139, 19, 20, 194, 22, 263, 22, 23, 231, 25, + /* 350 */ 72, 276, 277, 278, 116, 117, 118, 101, 36, 76, /* 360 */ 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, - /* 370 */ 112, 113, 239, 240, 311, 312, 215, 106, 107, 241, - /* 380 */ 19, 59, 216, 217, 223, 252, 115, 116, 117, 118, - /* 390 */ 151, 120, 26, 71, 193, 308, 309, 193, 149, 128, - /* 400 */ 313, 216, 269, 81, 43, 44, 45, 46, 47, 48, - /* 410 */ 49, 50, 51, 52, 53, 54, 55, 56, 57, 253, - /* 420 */ 216, 217, 100, 95, 153, 59, 155, 261, 106, 107, - /* 430 */ 25, 193, 101, 193, 193, 231, 114, 25, 116, 117, - /* 440 */ 118, 113, 304, 121, 193, 204, 59, 119, 120, 121, - /* 450 */ 122, 123, 124, 125, 216, 217, 193, 216, 217, 131, - /* 460 */ 138, 139, 230, 102, 103, 104, 105, 106, 107, 108, + /* 370 */ 112, 113, 89, 240, 241, 92, 73, 194, 194, 73, + /* 380 */ 19, 59, 188, 189, 190, 191, 253, 81, 255, 151, + /* 390 */ 196, 25, 198, 71, 116, 117, 118, 311, 312, 205, + /* 400 */ 217, 218, 316, 81, 43, 44, 45, 46, 47, 48, + /* 410 */ 49, 50, 51, 52, 53, 54, 55, 56, 57, 270, + /* 420 */ 22, 23, 100, 25, 59, 101, 138, 139, 106, 107, + /* 430 */ 127, 128, 129, 127, 240, 241, 114, 254, 116, 117, + /* 440 */ 118, 76, 76, 121, 138, 139, 263, 253, 264, 255, + /* 450 */ 205, 275, 87, 19, 89, 89, 194, 92, 92, 199, + /* 460 */ 138, 139, 268, 102, 103, 104, 105, 106, 107, 108, /* 470 */ 109, 110, 111, 112, 113, 153, 154, 155, 156, 157, - /* 480 */ 239, 240, 116, 117, 118, 76, 193, 23, 19, 25, - /* 490 */ 22, 253, 23, 252, 253, 108, 87, 204, 89, 261, - /* 500 */ 198, 92, 261, 116, 117, 118, 193, 306, 307, 216, - /* 510 */ 217, 150, 43, 44, 45, 46, 47, 48, 49, 50, - /* 520 */ 51, 52, 53, 54, 55, 56, 57, 59, 193, 216, - /* 530 */ 217, 19, 239, 240, 283, 23, 106, 107, 108, 109, - /* 540 */ 110, 111, 112, 113, 73, 252, 253, 142, 308, 309, - /* 550 */ 138, 139, 81, 313, 145, 43, 44, 45, 46, 47, + /* 480 */ 81, 116, 117, 118, 129, 240, 241, 224, 19, 226, + /* 490 */ 314, 315, 23, 25, 300, 59, 22, 234, 253, 101, + /* 500 */ 255, 236, 237, 26, 194, 183, 194, 152, 72, 22, + /* 510 */ 145, 150, 43, 44, 45, 46, 47, 48, 49, 50, + /* 520 */ 51, 52, 53, 54, 55, 56, 57, 217, 218, 217, + /* 530 */ 218, 19, 189, 59, 191, 23, 59, 138, 139, 196, + /* 540 */ 135, 198, 232, 283, 232, 140, 59, 287, 205, 275, + /* 550 */ 116, 205, 116, 117, 118, 43, 44, 45, 46, 47, /* 560 */ 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, - /* 570 */ 307, 102, 103, 104, 105, 106, 107, 108, 109, 110, - /* 580 */ 111, 112, 113, 281, 116, 117, 118, 285, 23, 193, - /* 590 */ 25, 119, 59, 193, 122, 123, 124, 59, 127, 203, - /* 600 */ 59, 205, 19, 268, 132, 25, 23, 22, 193, 138, - /* 610 */ 139, 249, 204, 251, 102, 103, 104, 105, 106, 107, + /* 570 */ 194, 102, 103, 104, 105, 106, 107, 108, 109, 110, + /* 580 */ 111, 112, 113, 240, 241, 194, 240, 241, 314, 315, + /* 590 */ 116, 117, 118, 116, 117, 118, 253, 194, 255, 253, + /* 600 */ 59, 255, 19, 116, 117, 118, 23, 22, 217, 218, + /* 610 */ 142, 268, 205, 275, 102, 103, 104, 105, 106, 107, /* 620 */ 108, 109, 110, 111, 112, 113, 43, 44, 45, 46, /* 630 */ 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, - /* 640 */ 57, 19, 22, 23, 59, 23, 25, 239, 240, 116, - /* 650 */ 117, 118, 193, 11, 116, 117, 118, 116, 117, 118, - /* 660 */ 252, 269, 22, 193, 15, 43, 44, 45, 46, 47, + /* 640 */ 57, 19, 194, 300, 59, 23, 119, 240, 241, 122, + /* 650 */ 123, 124, 314, 315, 194, 236, 237, 194, 117, 132, + /* 660 */ 253, 81, 255, 205, 59, 43, 44, 45, 46, 47, /* 670 */ 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, - /* 680 */ 273, 143, 193, 118, 143, 102, 103, 104, 105, 106, - /* 690 */ 107, 108, 109, 110, 111, 112, 113, 76, 118, 59, - /* 700 */ 241, 116, 117, 118, 304, 216, 217, 292, 143, 60, - /* 710 */ 89, 241, 19, 92, 193, 193, 23, 22, 311, 312, - /* 720 */ 231, 101, 22, 143, 102, 103, 104, 105, 106, 107, + /* 680 */ 217, 218, 194, 194, 194, 102, 103, 104, 105, 106, + /* 690 */ 107, 108, 109, 110, 111, 112, 113, 294, 240, 241, + /* 700 */ 120, 116, 117, 118, 59, 194, 217, 218, 211, 212, + /* 710 */ 213, 253, 19, 255, 194, 19, 23, 254, 138, 139, + /* 720 */ 24, 232, 117, 194, 102, 103, 104, 105, 106, 107, /* 730 */ 108, 109, 110, 111, 112, 113, 43, 44, 45, 46, /* 740 */ 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, - /* 750 */ 57, 19, 193, 193, 59, 23, 116, 117, 118, 59, - /* 760 */ 201, 21, 241, 304, 193, 206, 127, 128, 129, 193, - /* 770 */ 128, 129, 235, 236, 304, 43, 44, 45, 46, 47, + /* 750 */ 57, 19, 264, 108, 76, 23, 127, 128, 129, 311, + /* 760 */ 312, 116, 117, 118, 316, 87, 306, 89, 308, 194, + /* 770 */ 92, 22, 59, 194, 22, 43, 44, 45, 46, 47, /* 780 */ 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, - /* 790 */ 22, 193, 216, 217, 193, 102, 103, 104, 105, 106, - /* 800 */ 107, 108, 109, 110, 111, 112, 113, 231, 193, 193, - /* 810 */ 193, 116, 117, 118, 216, 217, 116, 117, 118, 226, - /* 820 */ 80, 193, 19, 235, 236, 304, 23, 211, 212, 231, - /* 830 */ 204, 216, 217, 205, 102, 103, 104, 105, 106, 107, + /* 790 */ 194, 95, 217, 218, 265, 102, 103, 104, 105, 106, + /* 800 */ 107, 108, 109, 110, 111, 112, 113, 232, 59, 113, + /* 810 */ 25, 59, 194, 217, 218, 119, 120, 121, 122, 123, + /* 820 */ 124, 125, 19, 145, 194, 194, 23, 131, 232, 116, + /* 830 */ 117, 118, 35, 194, 102, 103, 104, 105, 106, 107, /* 840 */ 108, 109, 110, 111, 112, 113, 43, 44, 45, 46, /* 850 */ 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, - /* 860 */ 57, 19, 193, 123, 76, 239, 240, 193, 253, 239, - /* 870 */ 240, 239, 240, 244, 106, 107, 193, 89, 252, 193, - /* 880 */ 92, 59, 252, 254, 252, 43, 44, 45, 46, 47, + /* 860 */ 57, 19, 194, 66, 194, 116, 117, 118, 116, 117, + /* 870 */ 118, 74, 242, 294, 194, 194, 206, 23, 194, 25, + /* 880 */ 194, 111, 112, 113, 25, 43, 44, 45, 46, 47, /* 890 */ 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, - /* 900 */ 284, 161, 216, 217, 193, 102, 103, 104, 105, 106, - /* 910 */ 107, 108, 109, 110, 111, 112, 113, 231, 193, 244, - /* 920 */ 187, 188, 189, 190, 7, 8, 9, 309, 195, 254, - /* 930 */ 197, 313, 19, 127, 128, 129, 262, 204, 22, 117, - /* 940 */ 24, 216, 217, 273, 102, 103, 104, 105, 106, 107, + /* 900 */ 24, 194, 194, 217, 218, 102, 103, 104, 105, 106, + /* 910 */ 107, 108, 109, 110, 111, 112, 113, 241, 232, 194, + /* 920 */ 212, 213, 242, 242, 217, 218, 242, 130, 11, 253, + /* 930 */ 194, 255, 19, 265, 149, 59, 306, 194, 308, 232, + /* 940 */ 309, 310, 217, 218, 102, 103, 104, 105, 106, 107, /* 950 */ 108, 109, 110, 111, 112, 113, 43, 44, 45, 46, /* 960 */ 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, - /* 970 */ 57, 193, 239, 240, 193, 59, 19, 188, 253, 190, - /* 980 */ 193, 311, 312, 16, 195, 252, 197, 193, 19, 301, - /* 990 */ 302, 135, 193, 204, 216, 217, 140, 216, 217, 266, - /* 1000 */ 204, 159, 45, 46, 47, 48, 49, 50, 51, 52, + /* 970 */ 57, 194, 194, 59, 194, 239, 19, 194, 25, 254, + /* 980 */ 303, 304, 23, 194, 25, 126, 306, 306, 308, 308, + /* 990 */ 306, 271, 308, 117, 286, 217, 218, 217, 218, 194, + /* 1000 */ 194, 159, 45, 46, 47, 48, 49, 50, 51, 52, /* 1010 */ 53, 54, 55, 56, 57, 102, 103, 104, 105, 106, - /* 1020 */ 107, 108, 109, 110, 111, 112, 113, 12, 239, 240, - /* 1030 */ 193, 298, 238, 117, 253, 239, 240, 238, 259, 260, - /* 1040 */ 193, 252, 27, 193, 77, 193, 79, 204, 252, 262, - /* 1050 */ 193, 299, 300, 193, 100, 266, 278, 42, 204, 102, + /* 1020 */ 107, 108, 109, 110, 111, 112, 113, 59, 239, 194, + /* 1030 */ 116, 117, 118, 260, 254, 194, 240, 241, 194, 233, + /* 1040 */ 205, 240, 241, 205, 239, 128, 129, 270, 265, 253, + /* 1050 */ 194, 255, 217, 218, 253, 194, 255, 143, 280, 102, /* 1060 */ 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, - /* 1070 */ 113, 117, 159, 216, 217, 121, 216, 217, 63, 193, - /* 1080 */ 193, 193, 239, 240, 115, 116, 193, 298, 73, 240, - /* 1090 */ 238, 231, 19, 239, 240, 252, 22, 24, 211, 212, - /* 1100 */ 263, 252, 216, 217, 216, 217, 252, 153, 154, 155, - /* 1110 */ 253, 193, 19, 144, 213, 268, 43, 44, 45, 46, + /* 1070 */ 113, 118, 159, 217, 218, 240, 241, 118, 240, 241, + /* 1080 */ 194, 194, 194, 239, 116, 117, 118, 22, 253, 254, + /* 1090 */ 255, 253, 19, 255, 233, 194, 143, 24, 263, 212, + /* 1100 */ 213, 194, 143, 217, 218, 217, 218, 261, 262, 271, + /* 1110 */ 254, 143, 19, 7, 8, 9, 43, 44, 45, 46, /* 1120 */ 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, - /* 1130 */ 57, 193, 19, 59, 216, 217, 43, 44, 45, 46, + /* 1130 */ 57, 16, 19, 22, 23, 294, 43, 44, 45, 46, /* 1140 */ 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, - /* 1150 */ 57, 193, 19, 24, 216, 217, 43, 44, 45, 46, + /* 1150 */ 57, 312, 194, 214, 21, 316, 43, 44, 45, 46, /* 1160 */ 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, - /* 1170 */ 57, 284, 193, 208, 209, 102, 103, 104, 105, 106, - /* 1180 */ 107, 108, 109, 110, 111, 112, 113, 286, 59, 193, - /* 1190 */ 232, 117, 291, 193, 193, 102, 103, 104, 105, 106, - /* 1200 */ 107, 108, 109, 110, 111, 112, 113, 193, 204, 22, - /* 1210 */ 23, 193, 25, 66, 193, 102, 103, 104, 105, 106, - /* 1220 */ 107, 108, 109, 110, 111, 112, 113, 193, 193, 193, - /* 1230 */ 216, 217, 85, 193, 238, 19, 16, 216, 217, 238, - /* 1240 */ 193, 94, 193, 239, 240, 231, 117, 268, 35, 116, - /* 1250 */ 216, 217, 216, 217, 22, 23, 252, 25, 208, 209, + /* 1170 */ 57, 106, 107, 286, 194, 102, 103, 104, 105, 106, + /* 1180 */ 107, 108, 109, 110, 111, 112, 113, 207, 158, 59, + /* 1190 */ 160, 22, 77, 24, 79, 102, 103, 104, 105, 106, + /* 1200 */ 107, 108, 109, 110, 111, 112, 113, 194, 194, 229, + /* 1210 */ 194, 231, 101, 80, 22, 102, 103, 104, 105, 106, + /* 1220 */ 107, 108, 109, 110, 111, 112, 113, 288, 59, 12, + /* 1230 */ 217, 218, 293, 217, 218, 19, 106, 107, 59, 19, + /* 1240 */ 16, 127, 128, 129, 27, 115, 116, 117, 118, 194, + /* 1250 */ 120, 59, 22, 194, 24, 194, 123, 100, 128, 42, /* 1260 */ 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, - /* 1270 */ 54, 55, 56, 57, 193, 193, 19, 5, 59, 66, - /* 1280 */ 193, 263, 10, 11, 12, 13, 14, 74, 101, 17, - /* 1290 */ 193, 46, 193, 146, 193, 76, 213, 77, 263, 79, - /* 1300 */ 12, 260, 30, 46, 32, 264, 87, 193, 89, 29, - /* 1310 */ 263, 92, 40, 33, 232, 27, 193, 108, 102, 103, + /* 1270 */ 54, 55, 56, 57, 117, 194, 217, 218, 121, 100, + /* 1280 */ 63, 194, 245, 153, 194, 155, 117, 19, 115, 194, + /* 1290 */ 73, 214, 194, 256, 161, 116, 117, 194, 217, 218, + /* 1300 */ 121, 77, 194, 79, 217, 218, 194, 217, 218, 117, + /* 1310 */ 153, 154, 155, 254, 46, 217, 218, 144, 102, 103, /* 1320 */ 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, - /* 1330 */ 42, 138, 139, 101, 193, 116, 117, 118, 19, 20, - /* 1340 */ 255, 22, 70, 130, 135, 65, 256, 257, 193, 140, - /* 1350 */ 78, 63, 193, 81, 193, 36, 193, 216, 217, 193, - /* 1360 */ 115, 193, 263, 193, 145, 268, 59, 48, 193, 193, - /* 1370 */ 98, 193, 115, 193, 291, 216, 217, 193, 59, 216, - /* 1380 */ 217, 161, 216, 217, 216, 217, 216, 217, 131, 193, - /* 1390 */ 71, 193, 216, 217, 216, 217, 216, 217, 193, 260, - /* 1400 */ 216, 217, 19, 264, 85, 133, 244, 100, 193, 90, - /* 1410 */ 138, 139, 216, 217, 216, 217, 254, 244, 193, 100, - /* 1420 */ 193, 216, 217, 116, 117, 106, 107, 254, 121, 193, - /* 1430 */ 115, 216, 217, 114, 162, 116, 117, 118, 115, 244, - /* 1440 */ 121, 216, 217, 216, 217, 193, 309, 193, 31, 254, - /* 1450 */ 313, 309, 216, 217, 309, 313, 39, 193, 313, 309, - /* 1460 */ 153, 154, 155, 313, 193, 150, 25, 144, 216, 217, - /* 1470 */ 216, 217, 153, 154, 155, 156, 157, 0, 1, 2, - /* 1480 */ 216, 217, 5, 149, 150, 22, 193, 10, 11, 12, - /* 1490 */ 13, 14, 193, 158, 17, 160, 193, 19, 20, 116, - /* 1500 */ 22, 25, 193, 24, 22, 193, 24, 30, 226, 32, - /* 1510 */ 19, 20, 226, 22, 36, 193, 53, 40, 193, 216, - /* 1520 */ 217, 193, 23, 193, 25, 216, 217, 36, 216, 217, - /* 1530 */ 193, 99, 193, 193, 22, 193, 193, 59, 216, 217, - /* 1540 */ 193, 216, 217, 193, 216, 217, 193, 70, 129, 71, - /* 1550 */ 59, 129, 193, 216, 217, 78, 216, 217, 81, 216, - /* 1560 */ 217, 193, 71, 85, 193, 133, 193, 126, 90, 216, - /* 1570 */ 217, 152, 258, 61, 152, 98, 85, 193, 100, 193, - /* 1580 */ 23, 90, 25, 121, 106, 107, 23, 216, 217, 216, - /* 1590 */ 217, 100, 114, 131, 116, 117, 118, 106, 107, 121, - /* 1600 */ 216, 217, 216, 217, 193, 114, 193, 116, 117, 118, - /* 1610 */ 133, 22, 121, 193, 59, 138, 139, 193, 142, 193, - /* 1620 */ 141, 23, 23, 25, 25, 120, 121, 216, 217, 216, - /* 1630 */ 217, 153, 154, 155, 156, 157, 216, 217, 19, 162, - /* 1640 */ 216, 217, 216, 217, 153, 154, 155, 156, 157, 1, - /* 1650 */ 2, 193, 59, 5, 19, 20, 318, 22, 10, 11, - /* 1660 */ 12, 13, 14, 193, 59, 17, 193, 23, 23, 25, - /* 1670 */ 25, 36, 117, 193, 216, 217, 193, 23, 30, 25, - /* 1680 */ 32, 19, 20, 23, 22, 25, 216, 217, 40, 216, - /* 1690 */ 217, 7, 8, 23, 59, 25, 83, 84, 36, 23, - /* 1700 */ 193, 25, 23, 23, 25, 25, 71, 153, 145, 155, - /* 1710 */ 117, 153, 23, 155, 25, 23, 97, 25, 70, 193, - /* 1720 */ 193, 59, 117, 236, 193, 193, 78, 193, 193, 81, - /* 1730 */ 141, 193, 193, 71, 193, 100, 288, 287, 242, 255, - /* 1740 */ 255, 106, 107, 108, 255, 255, 98, 243, 297, 114, - /* 1750 */ 214, 116, 117, 118, 245, 191, 121, 271, 293, 267, - /* 1760 */ 267, 246, 100, 246, 245, 271, 271, 293, 106, 107, - /* 1770 */ 220, 271, 229, 225, 249, 219, 114, 259, 116, 117, - /* 1780 */ 118, 133, 259, 121, 219, 219, 138, 139, 153, 154, - /* 1790 */ 155, 156, 157, 280, 249, 243, 19, 20, 245, 22, - /* 1800 */ 196, 259, 140, 259, 60, 297, 141, 297, 200, 200, - /* 1810 */ 162, 38, 200, 36, 294, 153, 154, 155, 156, 157, - /* 1820 */ 151, 150, 294, 283, 22, 43, 234, 18, 237, 200, - /* 1830 */ 270, 272, 237, 237, 237, 18, 59, 199, 270, 149, - /* 1840 */ 246, 272, 272, 200, 234, 234, 246, 246, 71, 246, - /* 1850 */ 199, 158, 290, 62, 22, 200, 19, 20, 199, 22, - /* 1860 */ 289, 221, 221, 200, 200, 199, 199, 115, 218, 64, - /* 1870 */ 218, 218, 22, 36, 227, 126, 227, 100, 165, 221, - /* 1880 */ 224, 224, 24, 106, 107, 312, 218, 305, 113, 282, - /* 1890 */ 91, 114, 220, 116, 117, 118, 59, 282, 121, 218, - /* 1900 */ 218, 218, 200, 317, 317, 82, 221, 265, 71, 148, - /* 1910 */ 145, 265, 22, 277, 200, 158, 279, 140, 147, 25, - /* 1920 */ 146, 202, 248, 250, 249, 247, 13, 250, 194, 194, - /* 1930 */ 153, 154, 155, 156, 157, 6, 303, 100, 192, 192, - /* 1940 */ 246, 213, 192, 106, 107, 207, 213, 207, 222, 213, - /* 1950 */ 213, 114, 222, 116, 117, 118, 214, 214, 121, 4, - /* 1960 */ 207, 213, 3, 22, 303, 15, 163, 16, 23, 23, - /* 1970 */ 139, 151, 130, 25, 20, 142, 24, 16, 144, 1, - /* 1980 */ 142, 130, 130, 61, 37, 53, 300, 151, 53, 53, - /* 1990 */ 153, 154, 155, 156, 157, 53, 130, 116, 34, 1, - /* 2000 */ 141, 5, 22, 115, 161, 68, 25, 68, 75, 41, - /* 2010 */ 141, 115, 24, 20, 19, 131, 125, 23, 28, 22, - /* 2020 */ 67, 22, 22, 22, 67, 59, 24, 96, 22, 67, - /* 2030 */ 23, 149, 22, 25, 23, 23, 23, 22, 34, 141, - /* 2040 */ 37, 97, 23, 23, 116, 22, 143, 25, 34, 75, - /* 2050 */ 34, 34, 34, 88, 75, 34, 86, 23, 22, 34, - /* 2060 */ 93, 24, 34, 25, 25, 142, 142, 23, 44, 23, - /* 2070 */ 23, 23, 23, 11, 23, 25, 22, 22, 22, 141, - /* 2080 */ 23, 23, 22, 22, 25, 15, 1, 23, 25, 1, - /* 2090 */ 141, 135, 319, 319, 319, 319, 319, 319, 319, 141, - /* 2100 */ 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, - /* 2110 */ 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, - /* 2120 */ 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, - /* 2130 */ 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, - /* 2140 */ 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, - /* 2150 */ 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, - /* 2160 */ 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, - /* 2170 */ 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, - /* 2180 */ 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, - /* 2190 */ 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, - /* 2200 */ 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, - /* 2210 */ 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, - /* 2220 */ 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, - /* 2230 */ 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, - /* 2240 */ 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, - /* 2250 */ 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, - /* 2260 */ 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, - /* 2270 */ 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, - /* 2280 */ 319, 319, 319, 319, 319, + /* 1330 */ 232, 270, 153, 154, 155, 115, 116, 66, 19, 20, + /* 1340 */ 183, 22, 12, 312, 254, 194, 262, 316, 209, 210, + /* 1350 */ 266, 239, 194, 194, 108, 36, 85, 27, 19, 20, + /* 1360 */ 265, 22, 183, 245, 144, 94, 25, 48, 217, 218, + /* 1370 */ 293, 194, 42, 270, 256, 36, 217, 218, 59, 194, + /* 1380 */ 25, 135, 194, 115, 194, 161, 140, 194, 194, 15, + /* 1390 */ 71, 194, 312, 63, 217, 218, 316, 194, 59, 131, + /* 1400 */ 301, 302, 217, 218, 85, 217, 218, 217, 218, 90, + /* 1410 */ 71, 217, 218, 19, 217, 218, 245, 146, 262, 100, + /* 1420 */ 217, 218, 266, 265, 85, 106, 107, 256, 312, 90, + /* 1430 */ 209, 210, 316, 114, 60, 116, 117, 118, 194, 100, + /* 1440 */ 121, 194, 194, 145, 115, 106, 107, 19, 46, 19, + /* 1450 */ 20, 24, 22, 114, 194, 116, 117, 118, 194, 245, + /* 1460 */ 121, 194, 164, 194, 217, 218, 36, 194, 258, 259, + /* 1470 */ 256, 194, 153, 154, 155, 156, 157, 217, 218, 150, + /* 1480 */ 31, 217, 218, 142, 217, 218, 217, 218, 39, 59, + /* 1490 */ 217, 218, 153, 154, 155, 156, 157, 149, 150, 5, + /* 1500 */ 145, 71, 183, 245, 10, 11, 12, 13, 14, 194, + /* 1510 */ 116, 17, 129, 227, 256, 85, 194, 115, 194, 23, + /* 1520 */ 90, 25, 183, 99, 30, 97, 32, 22, 22, 194, + /* 1530 */ 100, 194, 217, 218, 40, 152, 106, 107, 23, 217, + /* 1540 */ 218, 194, 19, 20, 114, 22, 116, 117, 118, 257, + /* 1550 */ 194, 121, 217, 218, 217, 218, 194, 133, 53, 36, + /* 1560 */ 23, 23, 25, 25, 70, 120, 121, 61, 141, 7, + /* 1570 */ 8, 121, 78, 217, 218, 81, 23, 227, 25, 217, + /* 1580 */ 218, 131, 59, 153, 154, 155, 156, 157, 0, 1, + /* 1590 */ 2, 59, 98, 5, 71, 23, 227, 25, 10, 11, + /* 1600 */ 12, 13, 14, 83, 84, 17, 23, 23, 25, 25, + /* 1610 */ 59, 194, 194, 183, 23, 23, 25, 25, 30, 194, + /* 1620 */ 32, 19, 20, 100, 22, 194, 194, 133, 40, 106, + /* 1630 */ 107, 108, 138, 139, 194, 217, 218, 114, 36, 116, + /* 1640 */ 117, 118, 217, 218, 121, 194, 194, 194, 23, 117, + /* 1650 */ 25, 194, 23, 23, 25, 25, 162, 194, 70, 194, + /* 1660 */ 145, 59, 23, 153, 25, 155, 78, 194, 117, 81, + /* 1670 */ 217, 218, 194, 71, 217, 218, 153, 154, 155, 156, + /* 1680 */ 157, 194, 217, 218, 194, 23, 98, 25, 321, 194, + /* 1690 */ 217, 218, 194, 19, 20, 194, 22, 153, 23, 155, + /* 1700 */ 25, 194, 100, 194, 217, 218, 183, 194, 106, 107, + /* 1710 */ 36, 194, 217, 218, 237, 194, 114, 243, 116, 117, + /* 1720 */ 118, 133, 194, 121, 217, 218, 138, 139, 194, 194, + /* 1730 */ 194, 290, 289, 59, 217, 218, 194, 194, 217, 218, + /* 1740 */ 194, 194, 140, 194, 194, 71, 194, 244, 194, 194, + /* 1750 */ 162, 217, 218, 194, 194, 153, 154, 155, 156, 157, + /* 1760 */ 217, 218, 194, 217, 218, 194, 217, 218, 257, 217, + /* 1770 */ 218, 217, 218, 257, 100, 194, 257, 217, 218, 257, + /* 1780 */ 106, 107, 215, 299, 194, 183, 192, 194, 114, 194, + /* 1790 */ 116, 117, 118, 1, 2, 121, 221, 5, 217, 218, + /* 1800 */ 273, 197, 10, 11, 12, 13, 14, 217, 218, 17, + /* 1810 */ 217, 218, 217, 218, 140, 194, 246, 194, 273, 295, + /* 1820 */ 247, 273, 30, 247, 32, 269, 269, 153, 154, 155, + /* 1830 */ 156, 157, 40, 246, 273, 295, 230, 226, 217, 218, + /* 1840 */ 217, 218, 220, 261, 220, 282, 220, 19, 20, 244, + /* 1850 */ 22, 250, 141, 250, 246, 60, 201, 183, 261, 261, + /* 1860 */ 261, 201, 70, 299, 36, 299, 201, 38, 151, 150, + /* 1870 */ 78, 285, 22, 81, 296, 296, 43, 235, 18, 238, + /* 1880 */ 201, 274, 272, 238, 238, 238, 18, 59, 200, 149, + /* 1890 */ 98, 247, 274, 274, 235, 247, 247, 247, 235, 71, + /* 1900 */ 272, 201, 200, 158, 292, 62, 291, 201, 200, 22, + /* 1910 */ 201, 222, 200, 222, 201, 200, 115, 219, 219, 64, + /* 1920 */ 219, 228, 22, 126, 221, 133, 165, 222, 100, 225, + /* 1930 */ 138, 139, 225, 219, 106, 107, 24, 219, 228, 219, + /* 1940 */ 219, 307, 114, 113, 116, 117, 118, 315, 284, 121, + /* 1950 */ 284, 222, 201, 91, 162, 320, 320, 82, 148, 267, + /* 1960 */ 145, 267, 22, 279, 201, 158, 281, 251, 147, 146, + /* 1970 */ 25, 203, 250, 249, 251, 248, 13, 247, 195, 195, + /* 1980 */ 6, 153, 154, 155, 156, 157, 193, 193, 305, 193, + /* 1990 */ 208, 305, 302, 214, 214, 214, 208, 223, 223, 214, + /* 2000 */ 4, 215, 215, 214, 3, 22, 208, 163, 15, 23, + /* 2010 */ 16, 183, 23, 139, 151, 130, 25, 20, 142, 24, + /* 2020 */ 16, 144, 1, 142, 130, 130, 61, 37, 53, 151, + /* 2030 */ 53, 53, 53, 130, 116, 1, 34, 141, 5, 22, + /* 2040 */ 115, 161, 75, 25, 68, 141, 41, 115, 68, 24, + /* 2050 */ 20, 19, 131, 125, 67, 67, 96, 22, 22, 22, + /* 2060 */ 37, 23, 22, 24, 22, 59, 67, 23, 149, 28, + /* 2070 */ 22, 25, 23, 23, 23, 22, 141, 34, 97, 23, + /* 2080 */ 23, 34, 116, 22, 143, 25, 34, 75, 34, 34, + /* 2090 */ 75, 88, 34, 86, 23, 22, 34, 25, 24, 34, + /* 2100 */ 25, 93, 23, 44, 142, 23, 142, 23, 23, 22, + /* 2110 */ 11, 25, 23, 25, 23, 22, 22, 22, 1, 23, + /* 2120 */ 23, 23, 22, 22, 15, 141, 141, 25, 25, 1, + /* 2130 */ 322, 322, 322, 135, 322, 322, 322, 322, 322, 322, + /* 2140 */ 322, 141, 322, 322, 322, 322, 322, 322, 322, 322, + /* 2150 */ 322, 322, 322, 322, 322, 322, 322, 322, 322, 322, + /* 2160 */ 322, 322, 322, 322, 322, 322, 322, 322, 322, 322, + /* 2170 */ 322, 322, 322, 322, 322, 322, 322, 322, 322, 322, + /* 2180 */ 322, 322, 322, 322, 322, 322, 322, 322, 322, 322, + /* 2190 */ 322, 322, 322, 322, 322, 322, 322, 322, 322, 322, + /* 2200 */ 322, 322, 322, 322, 322, 322, 322, 322, 322, 322, + /* 2210 */ 322, 322, 322, 322, 322, 322, 322, 322, 322, 322, + /* 2220 */ 322, 322, 322, 322, 322, 322, 322, 322, 322, 322, + /* 2230 */ 322, 322, 322, 322, 322, 322, 322, 322, 322, 322, + /* 2240 */ 322, 322, 322, 322, 322, 322, 322, 322, 322, 322, + /* 2250 */ 322, 322, 322, 322, 322, 322, 322, 322, 322, 322, + /* 2260 */ 322, 322, 322, 322, 322, 322, 322, 322, 322, 322, + /* 2270 */ 322, 322, 322, 322, 322, 322, 322, 322, 322, 322, + /* 2280 */ 322, 322, 322, 322, 322, 322, 322, 322, 322, 322, + /* 2290 */ 322, 322, 322, 322, 322, 322, 322, 322, 322, 322, + /* 2300 */ 322, 322, 322, 322, 322, 322, 322, 322, 322, 322, + /* 2310 */ 322, 322, 322, 322, 322, 322, 322, 322, 322, 322, + /* 2320 */ 322, 322, 322, 322, 322, 322, 322, 322, }; -#define YY_SHIFT_COUNT (578) +#define YY_SHIFT_COUNT (582) #define YY_SHIFT_MIN (0) -#define YY_SHIFT_MAX (2088) +#define YY_SHIFT_MAX (2128) static const unsigned short int yy_shift_ofst[] = { - /* 0 */ 1648, 1477, 1272, 322, 322, 1, 1319, 1478, 1491, 1837, - /* 10 */ 1837, 1837, 471, 0, 0, 214, 1093, 1837, 1837, 1837, - /* 20 */ 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, - /* 30 */ 1837, 271, 271, 1219, 1219, 216, 88, 1, 1, 1, - /* 40 */ 1, 1, 40, 111, 258, 361, 469, 512, 583, 622, - /* 50 */ 693, 732, 803, 842, 913, 1073, 1093, 1093, 1093, 1093, - /* 60 */ 1093, 1093, 1093, 1093, 1093, 1093, 1093, 1093, 1093, 1093, - /* 70 */ 1093, 1093, 1093, 1093, 1113, 1093, 1216, 957, 957, 1635, - /* 80 */ 1662, 1777, 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, - /* 90 */ 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, - /* 100 */ 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, - /* 110 */ 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, - /* 120 */ 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, - /* 130 */ 1837, 137, 181, 181, 181, 181, 181, 181, 181, 94, - /* 140 */ 430, 66, 65, 112, 366, 533, 533, 740, 1257, 533, - /* 150 */ 533, 79, 79, 533, 412, 412, 412, 77, 412, 123, - /* 160 */ 113, 113, 113, 22, 22, 2100, 2100, 328, 328, 328, - /* 170 */ 239, 468, 468, 468, 468, 1015, 1015, 409, 366, 1187, - /* 180 */ 1232, 533, 533, 533, 533, 533, 533, 533, 533, 533, - /* 190 */ 533, 533, 533, 533, 533, 533, 533, 533, 533, 533, - /* 200 */ 533, 969, 621, 621, 533, 642, 788, 788, 1133, 1133, - /* 210 */ 822, 822, 67, 1193, 2100, 2100, 2100, 2100, 2100, 2100, - /* 220 */ 2100, 1307, 954, 954, 585, 472, 640, 387, 695, 538, - /* 230 */ 541, 700, 533, 533, 533, 533, 533, 533, 533, 533, - /* 240 */ 533, 533, 222, 533, 533, 533, 533, 533, 533, 533, - /* 250 */ 533, 533, 533, 533, 533, 1213, 1213, 1213, 533, 533, - /* 260 */ 533, 565, 533, 533, 533, 916, 1147, 533, 533, 1288, - /* 270 */ 533, 533, 533, 533, 533, 533, 533, 533, 639, 1280, - /* 280 */ 209, 1129, 1129, 1129, 1129, 580, 209, 209, 1209, 768, - /* 290 */ 917, 649, 1315, 1334, 405, 1334, 1383, 249, 1315, 1315, - /* 300 */ 249, 1315, 405, 1383, 1441, 464, 1245, 1417, 1417, 1417, - /* 310 */ 1323, 1323, 1323, 1323, 184, 184, 1335, 1476, 856, 1482, - /* 320 */ 1744, 1744, 1665, 1665, 1773, 1773, 1665, 1669, 1671, 1802, - /* 330 */ 1782, 1809, 1809, 1809, 1809, 1665, 1817, 1690, 1671, 1671, - /* 340 */ 1690, 1802, 1782, 1690, 1782, 1690, 1665, 1817, 1693, 1791, - /* 350 */ 1665, 1817, 1832, 1665, 1817, 1665, 1817, 1832, 1752, 1752, - /* 360 */ 1752, 1805, 1850, 1850, 1832, 1752, 1749, 1752, 1805, 1752, - /* 370 */ 1752, 1713, 1858, 1775, 1775, 1832, 1665, 1799, 1799, 1823, - /* 380 */ 1823, 1761, 1765, 1890, 1665, 1757, 1761, 1771, 1774, 1690, - /* 390 */ 1894, 1913, 1913, 1929, 1929, 1929, 2100, 2100, 2100, 2100, - /* 400 */ 2100, 2100, 2100, 2100, 2100, 2100, 2100, 2100, 2100, 2100, - /* 410 */ 2100, 207, 1220, 331, 620, 967, 806, 1074, 1499, 1432, - /* 420 */ 1463, 1479, 1419, 1422, 1557, 1512, 1598, 1599, 1644, 1645, - /* 430 */ 1654, 1660, 1555, 1505, 1684, 1462, 1670, 1563, 1619, 1593, - /* 440 */ 1676, 1679, 1613, 1680, 1554, 1558, 1689, 1692, 1605, 1589, - /* 450 */ 1955, 1959, 1941, 1803, 1950, 1951, 1945, 1946, 1831, 1820, - /* 460 */ 1842, 1948, 1948, 1952, 1833, 1954, 1834, 1961, 1978, 1838, - /* 470 */ 1851, 1948, 1852, 1922, 1947, 1948, 1836, 1932, 1935, 1936, - /* 480 */ 1942, 1866, 1881, 1964, 1859, 1998, 1996, 1980, 1888, 1843, - /* 490 */ 1937, 1981, 1939, 1933, 1968, 1869, 1896, 1988, 1993, 1995, - /* 500 */ 1884, 1891, 1997, 1953, 1999, 2000, 1994, 2001, 1957, 1966, - /* 510 */ 2002, 1931, 1990, 2006, 1962, 2003, 2007, 2004, 1882, 2010, - /* 520 */ 2011, 2012, 2008, 2013, 2015, 1944, 1898, 2019, 2020, 1928, - /* 530 */ 2014, 2023, 1903, 2022, 2016, 2017, 2018, 2021, 1965, 1974, - /* 540 */ 1970, 2024, 1979, 1967, 2025, 2034, 2036, 2037, 2038, 2039, - /* 550 */ 2028, 1923, 1924, 2044, 2022, 2046, 2047, 2048, 2049, 2050, - /* 560 */ 2051, 2054, 2062, 2055, 2056, 2057, 2058, 2060, 2061, 2059, - /* 570 */ 1956, 1938, 1949, 1958, 2063, 2064, 2070, 2085, 2088, + /* 0 */ 1792, 1588, 1494, 322, 322, 399, 306, 1319, 1339, 1430, + /* 10 */ 1828, 1828, 1828, 580, 399, 399, 399, 399, 399, 0, + /* 20 */ 0, 214, 1093, 1828, 1828, 1828, 1828, 1828, 1828, 1828, + /* 30 */ 1828, 1828, 1828, 1828, 1828, 1828, 1828, 1828, 1130, 1130, + /* 40 */ 365, 365, 55, 278, 436, 713, 713, 201, 201, 201, + /* 50 */ 201, 40, 111, 258, 361, 469, 512, 583, 622, 693, + /* 60 */ 732, 803, 842, 913, 1073, 1093, 1093, 1093, 1093, 1093, + /* 70 */ 1093, 1093, 1093, 1093, 1093, 1093, 1093, 1093, 1093, 1093, + /* 80 */ 1093, 1093, 1093, 1113, 1093, 1216, 957, 957, 1523, 1602, + /* 90 */ 1674, 1828, 1828, 1828, 1828, 1828, 1828, 1828, 1828, 1828, + /* 100 */ 1828, 1828, 1828, 1828, 1828, 1828, 1828, 1828, 1828, 1828, + /* 110 */ 1828, 1828, 1828, 1828, 1828, 1828, 1828, 1828, 1828, 1828, + /* 120 */ 1828, 1828, 1828, 1828, 1828, 1828, 1828, 1828, 1828, 1828, + /* 130 */ 1828, 1828, 1828, 1828, 1828, 1828, 1828, 1828, 1828, 1828, + /* 140 */ 137, 181, 181, 181, 181, 181, 181, 181, 96, 222, + /* 150 */ 143, 477, 713, 1133, 1268, 713, 713, 79, 79, 713, + /* 160 */ 770, 83, 65, 65, 65, 288, 162, 162, 2142, 2142, + /* 170 */ 696, 696, 696, 238, 474, 474, 474, 474, 1217, 1217, + /* 180 */ 678, 477, 324, 398, 713, 713, 713, 713, 713, 713, + /* 190 */ 713, 713, 713, 713, 713, 713, 713, 713, 713, 713, + /* 200 */ 713, 713, 713, 1220, 366, 366, 713, 917, 283, 283, + /* 210 */ 434, 434, 605, 605, 1298, 2142, 2142, 2142, 2142, 2142, + /* 220 */ 2142, 2142, 1179, 1157, 1157, 487, 527, 585, 645, 749, + /* 230 */ 914, 968, 752, 713, 713, 713, 713, 713, 713, 713, + /* 240 */ 713, 713, 713, 303, 713, 713, 713, 713, 713, 713, + /* 250 */ 713, 713, 713, 713, 713, 713, 797, 797, 797, 713, + /* 260 */ 713, 713, 959, 713, 713, 713, 1169, 1271, 713, 713, + /* 270 */ 1330, 713, 713, 713, 713, 713, 713, 713, 713, 629, + /* 280 */ 7, 91, 876, 876, 876, 876, 953, 91, 91, 1246, + /* 290 */ 1065, 1106, 1374, 1329, 1348, 468, 1348, 1394, 785, 1329, + /* 300 */ 1329, 785, 1329, 468, 1394, 859, 854, 1402, 1449, 1449, + /* 310 */ 1449, 1173, 1173, 1173, 1173, 1355, 1355, 1030, 1341, 405, + /* 320 */ 1230, 1795, 1795, 1711, 1711, 1829, 1829, 1711, 1717, 1719, + /* 330 */ 1850, 1833, 1860, 1860, 1860, 1860, 1711, 1868, 1740, 1719, + /* 340 */ 1719, 1740, 1850, 1833, 1740, 1833, 1740, 1711, 1868, 1745, + /* 350 */ 1843, 1711, 1868, 1887, 1711, 1868, 1711, 1868, 1887, 1801, + /* 360 */ 1801, 1801, 1855, 1900, 1900, 1887, 1801, 1797, 1801, 1855, + /* 370 */ 1801, 1801, 1761, 1912, 1830, 1830, 1887, 1711, 1862, 1862, + /* 380 */ 1875, 1875, 1810, 1815, 1940, 1711, 1807, 1810, 1821, 1823, + /* 390 */ 1740, 1945, 1963, 1963, 1974, 1974, 1974, 2142, 2142, 2142, + /* 400 */ 2142, 2142, 2142, 2142, 2142, 2142, 2142, 2142, 2142, 2142, + /* 410 */ 2142, 2142, 20, 1224, 256, 1111, 1115, 1114, 1192, 1496, + /* 420 */ 1424, 1505, 1427, 355, 1383, 1537, 1506, 1538, 1553, 1583, + /* 430 */ 1584, 1591, 1625, 541, 1445, 1562, 1450, 1572, 1515, 1428, + /* 440 */ 1532, 1592, 1629, 1520, 1630, 1639, 1510, 1544, 1662, 1675, + /* 450 */ 1551, 48, 1996, 2001, 1983, 1844, 1993, 1994, 1986, 1989, + /* 460 */ 1874, 1863, 1885, 1991, 1991, 1995, 1876, 1997, 1877, 2004, + /* 470 */ 2021, 1881, 1894, 1991, 1895, 1965, 1990, 1991, 1878, 1975, + /* 480 */ 1977, 1978, 1979, 1903, 1918, 2002, 1896, 2034, 2033, 2017, + /* 490 */ 1925, 1880, 1976, 2018, 1980, 1967, 2005, 1904, 1932, 2025, + /* 500 */ 2030, 2032, 1921, 1928, 2035, 1987, 2036, 2037, 2038, 2040, + /* 510 */ 1988, 2006, 2039, 1960, 2041, 2042, 1999, 2023, 2044, 2043, + /* 520 */ 1919, 2048, 2049, 2050, 2046, 2051, 2053, 1981, 1935, 2056, + /* 530 */ 2057, 1966, 2047, 2061, 1941, 2060, 2052, 2054, 2055, 2058, + /* 540 */ 2003, 2012, 2007, 2059, 2015, 2008, 2062, 2071, 2073, 2074, + /* 550 */ 2072, 2075, 2065, 1962, 1964, 2079, 2060, 2082, 2084, 2085, + /* 560 */ 2087, 2086, 2089, 2088, 2091, 2093, 2099, 2094, 2095, 2096, + /* 570 */ 2097, 2100, 2101, 2102, 1998, 1984, 1985, 2000, 2103, 2098, + /* 580 */ 2109, 2117, 2128, }; -#define YY_REDUCE_COUNT (410) -#define YY_REDUCE_MIN (-271) -#define YY_REDUCE_MAX (1753) +#define YY_REDUCE_COUNT (411) +#define YY_REDUCE_MIN (-275) +#define YY_REDUCE_MAX (1798) static const short yy_reduce_ofst[] = { - /* 0 */ -125, 733, 789, 241, 293, -123, -193, -191, -183, -187, - /* 10 */ 166, 238, 133, -207, -199, -267, -176, -6, 204, 489, - /* 20 */ 576, 598, -175, 686, 860, 615, 725, 1014, 778, 781, - /* 30 */ 857, 616, 887, 87, 240, -192, 408, 626, 796, 843, - /* 40 */ 854, 1004, -271, -271, -271, -271, -271, -271, -271, -271, - /* 50 */ -271, -271, -271, -271, -271, -271, -271, -271, -271, -271, - /* 60 */ -271, -271, -271, -271, -271, -271, -271, -271, -271, -271, - /* 70 */ -271, -271, -271, -271, -271, -271, -271, -271, -271, 80, - /* 80 */ 83, 313, 886, 888, 918, 938, 1021, 1034, 1036, 1141, - /* 90 */ 1159, 1163, 1166, 1168, 1170, 1176, 1178, 1180, 1184, 1196, - /* 100 */ 1198, 1205, 1215, 1225, 1227, 1236, 1252, 1254, 1264, 1303, - /* 110 */ 1309, 1312, 1322, 1325, 1328, 1337, 1340, 1343, 1353, 1371, - /* 120 */ 1373, 1384, 1386, 1411, 1413, 1420, 1424, 1426, 1458, 1470, - /* 130 */ 1473, -271, -271, -271, -271, -271, -271, -271, -271, -271, - /* 140 */ -271, -271, 138, 459, 396, -158, 470, 302, -212, 521, - /* 150 */ 201, -195, -92, 559, 630, 632, 630, -271, 632, 901, - /* 160 */ 63, 407, 670, -271, -271, -271, -271, 161, 161, 161, - /* 170 */ 251, 335, 847, 979, 1097, 537, 588, 618, 628, 688, - /* 180 */ 688, -166, -161, 674, 787, 794, 799, 852, 996, -122, - /* 190 */ 837, -120, 1018, 1035, 415, 1047, 1001, 958, 1082, 400, - /* 200 */ 1099, 779, 1137, 1142, 263, 1083, 1145, 1150, 1041, 1139, - /* 210 */ 965, 1050, 362, 849, 752, 629, 675, 1162, 1173, 1090, - /* 220 */ 1195, -194, 56, 185, -135, 232, 522, 560, 571, 601, - /* 230 */ 617, 669, 683, 711, 850, 893, 1000, 1040, 1049, 1081, - /* 240 */ 1087, 1101, 392, 1114, 1123, 1155, 1161, 1175, 1271, 1293, - /* 250 */ 1299, 1330, 1339, 1342, 1347, 593, 1282, 1286, 1350, 1359, - /* 260 */ 1368, 1314, 1480, 1483, 1507, 1085, 1338, 1526, 1527, 1487, - /* 270 */ 1531, 560, 1532, 1534, 1535, 1538, 1539, 1541, 1448, 1450, - /* 280 */ 1496, 1484, 1485, 1489, 1490, 1314, 1496, 1496, 1504, 1536, - /* 290 */ 1564, 1451, 1486, 1492, 1509, 1493, 1465, 1515, 1494, 1495, - /* 300 */ 1517, 1500, 1519, 1474, 1550, 1543, 1548, 1556, 1565, 1566, - /* 310 */ 1518, 1523, 1542, 1544, 1525, 1545, 1513, 1553, 1552, 1604, - /* 320 */ 1508, 1510, 1608, 1609, 1520, 1528, 1612, 1540, 1559, 1560, - /* 330 */ 1592, 1591, 1595, 1596, 1597, 1629, 1638, 1594, 1569, 1570, - /* 340 */ 1600, 1568, 1610, 1601, 1611, 1603, 1643, 1651, 1562, 1571, - /* 350 */ 1655, 1659, 1640, 1663, 1666, 1664, 1667, 1641, 1650, 1652, - /* 360 */ 1653, 1647, 1656, 1657, 1658, 1668, 1672, 1681, 1649, 1682, - /* 370 */ 1683, 1573, 1582, 1607, 1615, 1685, 1702, 1586, 1587, 1642, - /* 380 */ 1646, 1673, 1675, 1636, 1714, 1637, 1677, 1674, 1678, 1694, - /* 390 */ 1719, 1734, 1735, 1746, 1747, 1750, 1633, 1661, 1686, 1738, - /* 400 */ 1728, 1733, 1736, 1737, 1740, 1726, 1730, 1742, 1743, 1748, - /* 410 */ 1753, + /* 0 */ -71, 194, 343, 835, -180, -177, 838, -194, -188, -185, + /* 10 */ -183, 82, 183, -65, 133, 245, 346, 407, 458, -178, + /* 20 */ 75, -275, -4, 310, 312, 489, 575, 596, 463, 686, + /* 30 */ 707, 725, 780, 1098, 856, 778, 1059, 1090, 708, 887, + /* 40 */ 86, 448, 980, 630, 680, 681, 684, 796, 801, 796, + /* 50 */ 801, -261, -261, -261, -261, -261, -261, -261, -261, -261, + /* 60 */ -261, -261, -261, -261, -261, -261, -261, -261, -261, -261, + /* 70 */ -261, -261, -261, -261, -261, -261, -261, -261, -261, -261, + /* 80 */ -261, -261, -261, -261, -261, -261, -261, -261, 391, 886, + /* 90 */ 888, 1013, 1016, 1081, 1087, 1151, 1159, 1177, 1185, 1188, + /* 100 */ 1190, 1194, 1197, 1203, 1247, 1260, 1264, 1267, 1269, 1273, + /* 110 */ 1315, 1322, 1335, 1337, 1356, 1362, 1418, 1425, 1453, 1457, + /* 120 */ 1465, 1473, 1487, 1495, 1507, 1517, 1521, 1534, 1543, 1546, + /* 130 */ 1549, 1552, 1554, 1560, 1581, 1590, 1593, 1595, 1621, 1623, + /* 140 */ -261, -261, -261, -261, -261, -261, -261, -261, -261, -261, + /* 150 */ -261, -186, -117, 260, 263, 460, 631, -74, 497, -181, + /* 160 */ -261, 939, 176, 274, 338, 676, -261, -261, -261, -261, + /* 170 */ -212, -212, -212, -184, 149, 777, 1061, 1103, 265, 419, + /* 180 */ -254, 670, 677, 677, -11, -129, 184, 488, 736, 789, + /* 190 */ 805, 844, 403, 529, 579, 668, 783, 841, 1158, 1112, + /* 200 */ 806, 861, 1095, 846, 839, 1031, -189, 1077, 1080, 1116, + /* 210 */ 1084, 1156, 1139, 1221, 46, 1099, 1037, 1118, 1171, 1214, + /* 220 */ 1210, 1258, -210, -190, -176, -115, 117, 262, 376, 490, + /* 230 */ 511, 520, 618, 639, 743, 901, 907, 958, 1014, 1055, + /* 240 */ 1108, 1193, 1244, 720, 1248, 1277, 1324, 1347, 1417, 1431, + /* 250 */ 1432, 1440, 1451, 1452, 1463, 1478, 1286, 1350, 1369, 1490, + /* 260 */ 1498, 1501, 773, 1509, 1513, 1528, 1292, 1367, 1535, 1536, + /* 270 */ 1477, 1542, 376, 1547, 1550, 1555, 1559, 1568, 1571, 1441, + /* 280 */ 1443, 1474, 1511, 1516, 1519, 1522, 773, 1474, 1474, 1503, + /* 290 */ 1567, 1594, 1484, 1527, 1556, 1570, 1557, 1524, 1573, 1545, + /* 300 */ 1548, 1576, 1561, 1587, 1540, 1575, 1606, 1611, 1622, 1624, + /* 310 */ 1626, 1582, 1597, 1598, 1599, 1601, 1603, 1563, 1608, 1605, + /* 320 */ 1604, 1564, 1566, 1655, 1660, 1578, 1579, 1665, 1586, 1607, + /* 330 */ 1610, 1642, 1641, 1645, 1646, 1647, 1679, 1688, 1644, 1618, + /* 340 */ 1619, 1648, 1628, 1659, 1649, 1663, 1650, 1700, 1702, 1612, + /* 350 */ 1615, 1706, 1708, 1689, 1709, 1712, 1713, 1715, 1691, 1698, + /* 360 */ 1699, 1701, 1693, 1704, 1707, 1705, 1714, 1703, 1718, 1710, + /* 370 */ 1720, 1721, 1632, 1634, 1664, 1666, 1729, 1751, 1635, 1636, + /* 380 */ 1692, 1694, 1716, 1722, 1684, 1763, 1685, 1723, 1724, 1727, + /* 390 */ 1730, 1768, 1783, 1784, 1793, 1794, 1796, 1683, 1686, 1690, + /* 400 */ 1782, 1779, 1780, 1781, 1785, 1788, 1774, 1775, 1786, 1787, + /* 410 */ 1789, 1798, }; static const YYACTIONTYPE yy_default[] = { - /* 0 */ 1648, 1648, 1648, 1478, 1243, 1354, 1243, 1243, 1243, 1478, - /* 10 */ 1478, 1478, 1243, 1384, 1384, 1531, 1276, 1243, 1243, 1243, - /* 20 */ 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1477, 1243, - /* 30 */ 1243, 1243, 1243, 1564, 1564, 1243, 1243, 1243, 1243, 1243, - /* 40 */ 1243, 1243, 1243, 1393, 1243, 1400, 1243, 1243, 1243, 1243, - /* 50 */ 1243, 1479, 1480, 1243, 1243, 1243, 1530, 1532, 1495, 1407, - /* 60 */ 1406, 1405, 1404, 1513, 1372, 1398, 1391, 1395, 1474, 1475, - /* 70 */ 1473, 1626, 1480, 1479, 1243, 1394, 1442, 1458, 1441, 1243, - /* 80 */ 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, - /* 90 */ 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, - /* 100 */ 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, - /* 110 */ 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, - /* 120 */ 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, - /* 130 */ 1243, 1450, 1457, 1456, 1455, 1464, 1454, 1451, 1444, 1443, - /* 140 */ 1445, 1446, 1243, 1243, 1267, 1243, 1243, 1264, 1318, 1243, - /* 150 */ 1243, 1243, 1243, 1243, 1550, 1549, 1243, 1447, 1243, 1276, - /* 160 */ 1435, 1434, 1433, 1461, 1448, 1460, 1459, 1538, 1600, 1599, - /* 170 */ 1496, 1243, 1243, 1243, 1243, 1243, 1243, 1564, 1243, 1243, - /* 180 */ 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, - /* 190 */ 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, - /* 200 */ 1243, 1374, 1564, 1564, 1243, 1276, 1564, 1564, 1375, 1375, - /* 210 */ 1272, 1272, 1378, 1243, 1545, 1345, 1345, 1345, 1345, 1354, - /* 220 */ 1345, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, - /* 230 */ 1243, 1243, 1243, 1243, 1243, 1243, 1535, 1533, 1243, 1243, - /* 240 */ 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, - /* 250 */ 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, - /* 260 */ 1243, 1243, 1243, 1243, 1243, 1350, 1243, 1243, 1243, 1243, - /* 270 */ 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1593, 1243, 1508, - /* 280 */ 1332, 1350, 1350, 1350, 1350, 1352, 1333, 1331, 1344, 1277, - /* 290 */ 1250, 1640, 1410, 1399, 1351, 1399, 1637, 1397, 1410, 1410, - /* 300 */ 1397, 1410, 1351, 1637, 1293, 1615, 1288, 1384, 1384, 1384, - /* 310 */ 1374, 1374, 1374, 1374, 1378, 1378, 1476, 1351, 1344, 1243, - /* 320 */ 1640, 1640, 1360, 1360, 1639, 1639, 1360, 1496, 1623, 1419, - /* 330 */ 1321, 1327, 1327, 1327, 1327, 1360, 1261, 1397, 1623, 1623, - /* 340 */ 1397, 1419, 1321, 1397, 1321, 1397, 1360, 1261, 1512, 1634, - /* 350 */ 1360, 1261, 1486, 1360, 1261, 1360, 1261, 1486, 1319, 1319, - /* 360 */ 1319, 1308, 1243, 1243, 1486, 1319, 1293, 1319, 1308, 1319, - /* 370 */ 1319, 1582, 1243, 1490, 1490, 1486, 1360, 1574, 1574, 1387, - /* 380 */ 1387, 1392, 1378, 1481, 1360, 1243, 1392, 1390, 1388, 1397, - /* 390 */ 1311, 1596, 1596, 1592, 1592, 1592, 1645, 1645, 1545, 1608, - /* 400 */ 1276, 1276, 1276, 1276, 1608, 1295, 1295, 1277, 1277, 1276, - /* 410 */ 1608, 1243, 1243, 1243, 1243, 1243, 1243, 1603, 1243, 1540, - /* 420 */ 1497, 1364, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, - /* 430 */ 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1551, 1243, - /* 440 */ 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1424, - /* 450 */ 1243, 1246, 1542, 1243, 1243, 1243, 1243, 1243, 1243, 1243, - /* 460 */ 1243, 1401, 1402, 1365, 1243, 1243, 1243, 1243, 1243, 1243, - /* 470 */ 1243, 1416, 1243, 1243, 1243, 1411, 1243, 1243, 1243, 1243, - /* 480 */ 1243, 1243, 1243, 1243, 1636, 1243, 1243, 1243, 1243, 1243, - /* 490 */ 1243, 1511, 1510, 1243, 1243, 1362, 1243, 1243, 1243, 1243, - /* 500 */ 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1291, - /* 510 */ 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, - /* 520 */ 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, - /* 530 */ 1243, 1243, 1243, 1389, 1243, 1243, 1243, 1243, 1243, 1243, - /* 540 */ 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1579, 1379, - /* 550 */ 1243, 1243, 1243, 1243, 1627, 1243, 1243, 1243, 1243, 1243, - /* 560 */ 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1619, - /* 570 */ 1335, 1425, 1243, 1428, 1265, 1243, 1255, 1243, 1243, + /* 0 */ 1663, 1663, 1663, 1491, 1254, 1367, 1254, 1254, 1254, 1254, + /* 10 */ 1491, 1491, 1491, 1254, 1254, 1254, 1254, 1254, 1254, 1397, + /* 20 */ 1397, 1544, 1287, 1254, 1254, 1254, 1254, 1254, 1254, 1254, + /* 30 */ 1254, 1254, 1254, 1254, 1254, 1490, 1254, 1254, 1254, 1254, + /* 40 */ 1578, 1578, 1254, 1254, 1254, 1254, 1254, 1563, 1562, 1254, + /* 50 */ 1254, 1254, 1406, 1254, 1413, 1254, 1254, 1254, 1254, 1254, + /* 60 */ 1492, 1493, 1254, 1254, 1254, 1543, 1545, 1508, 1420, 1419, + /* 70 */ 1418, 1417, 1526, 1385, 1411, 1404, 1408, 1487, 1488, 1486, + /* 80 */ 1641, 1493, 1492, 1254, 1407, 1455, 1471, 1454, 1254, 1254, + /* 90 */ 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, + /* 100 */ 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, + /* 110 */ 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, + /* 120 */ 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, + /* 130 */ 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, + /* 140 */ 1463, 1470, 1469, 1468, 1477, 1467, 1464, 1457, 1456, 1458, + /* 150 */ 1459, 1278, 1254, 1275, 1329, 1254, 1254, 1254, 1254, 1254, + /* 160 */ 1460, 1287, 1448, 1447, 1446, 1254, 1474, 1461, 1473, 1472, + /* 170 */ 1551, 1615, 1614, 1509, 1254, 1254, 1254, 1254, 1254, 1254, + /* 180 */ 1578, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, + /* 190 */ 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, + /* 200 */ 1254, 1254, 1254, 1387, 1578, 1578, 1254, 1287, 1578, 1578, + /* 210 */ 1388, 1388, 1283, 1283, 1391, 1558, 1358, 1358, 1358, 1358, + /* 220 */ 1367, 1358, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, + /* 230 */ 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1548, 1546, 1254, + /* 240 */ 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, + /* 250 */ 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, + /* 260 */ 1254, 1254, 1254, 1254, 1254, 1254, 1363, 1254, 1254, 1254, + /* 270 */ 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1608, 1254, + /* 280 */ 1521, 1343, 1363, 1363, 1363, 1363, 1365, 1344, 1342, 1357, + /* 290 */ 1288, 1261, 1655, 1423, 1412, 1364, 1412, 1652, 1410, 1423, + /* 300 */ 1423, 1410, 1423, 1364, 1652, 1304, 1630, 1299, 1397, 1397, + /* 310 */ 1397, 1387, 1387, 1387, 1387, 1391, 1391, 1489, 1364, 1357, + /* 320 */ 1254, 1655, 1655, 1373, 1373, 1654, 1654, 1373, 1509, 1638, + /* 330 */ 1432, 1332, 1338, 1338, 1338, 1338, 1373, 1272, 1410, 1638, + /* 340 */ 1638, 1410, 1432, 1332, 1410, 1332, 1410, 1373, 1272, 1525, + /* 350 */ 1649, 1373, 1272, 1499, 1373, 1272, 1373, 1272, 1499, 1330, + /* 360 */ 1330, 1330, 1319, 1254, 1254, 1499, 1330, 1304, 1330, 1319, + /* 370 */ 1330, 1330, 1596, 1254, 1503, 1503, 1499, 1373, 1588, 1588, + /* 380 */ 1400, 1400, 1405, 1391, 1494, 1373, 1254, 1405, 1403, 1401, + /* 390 */ 1410, 1322, 1611, 1611, 1607, 1607, 1607, 1660, 1660, 1558, + /* 400 */ 1623, 1287, 1287, 1287, 1287, 1623, 1306, 1306, 1288, 1288, + /* 410 */ 1287, 1623, 1254, 1254, 1254, 1254, 1254, 1254, 1618, 1254, + /* 420 */ 1553, 1510, 1377, 1254, 1254, 1254, 1254, 1254, 1254, 1254, + /* 430 */ 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1564, + /* 440 */ 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, + /* 450 */ 1254, 1437, 1254, 1257, 1555, 1254, 1254, 1254, 1254, 1254, + /* 460 */ 1254, 1254, 1254, 1414, 1415, 1378, 1254, 1254, 1254, 1254, + /* 470 */ 1254, 1254, 1254, 1429, 1254, 1254, 1254, 1424, 1254, 1254, + /* 480 */ 1254, 1254, 1254, 1254, 1254, 1254, 1651, 1254, 1254, 1254, + /* 490 */ 1254, 1254, 1254, 1524, 1523, 1254, 1254, 1375, 1254, 1254, + /* 500 */ 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, + /* 510 */ 1254, 1302, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, + /* 520 */ 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, + /* 530 */ 1254, 1254, 1254, 1254, 1254, 1402, 1254, 1254, 1254, 1254, + /* 540 */ 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, + /* 550 */ 1593, 1392, 1254, 1254, 1254, 1254, 1642, 1254, 1254, 1254, + /* 560 */ 1254, 1352, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, + /* 570 */ 1254, 1254, 1254, 1634, 1346, 1438, 1254, 1441, 1276, 1254, + /* 580 */ 1266, 1254, 1254, }; /********** End of lemon-generated parsing tables *****************************/ @@ -172521,8 +173902,8 @@ static const YYCODETYPE yyFallback[] = { 0, /* TRUEFALSE => nothing */ 0, /* ISNOT => nothing */ 0, /* FUNCTION => nothing */ - 0, /* UMINUS => nothing */ 0, /* UPLUS => nothing */ + 0, /* UMINUS => nothing */ 0, /* TRUTH => nothing */ 0, /* REGISTER => nothing */ 0, /* VECTOR => nothing */ @@ -172531,6 +173912,7 @@ static const YYCODETYPE yyFallback[] = { 0, /* ASTERISK => nothing */ 0, /* SPAN => nothing */ 0, /* ERROR => nothing */ + 0, /* QNUMBER => nothing */ 0, /* SPACE => nothing */ 0, /* ILLEGAL => nothing */ }; @@ -172573,14 +173955,9 @@ struct yyParser { #endif sqlite3ParserARG_SDECL /* A place to hold %extra_argument */ sqlite3ParserCTX_SDECL /* A place to hold %extra_context */ -#if YYSTACKDEPTH<=0 - int yystksz; /* Current side of the stack */ - yyStackEntry *yystack; /* The parser's stack */ - yyStackEntry yystk0; /* First stack entry */ -#else - yyStackEntry yystack[YYSTACKDEPTH]; /* The parser's stack */ - yyStackEntry *yystackEnd; /* Last entry in the stack */ -#endif + yyStackEntry *yystackEnd; /* Last entry in the stack */ + yyStackEntry *yystack; /* The parser stack */ + yyStackEntry yystk0[YYSTACKDEPTH]; /* Initial stack space */ }; typedef struct yyParser yyParser; @@ -172794,8 +174171,8 @@ static const char *const yyTokenName[] = { /* 170 */ "TRUEFALSE", /* 171 */ "ISNOT", /* 172 */ "FUNCTION", - /* 173 */ "UMINUS", - /* 174 */ "UPLUS", + /* 173 */ "UPLUS", + /* 174 */ "UMINUS", /* 175 */ "TRUTH", /* 176 */ "REGISTER", /* 177 */ "VECTOR", @@ -172804,142 +174181,145 @@ static const char *const yyTokenName[] = { /* 180 */ "ASTERISK", /* 181 */ "SPAN", /* 182 */ "ERROR", - /* 183 */ "SPACE", - /* 184 */ "ILLEGAL", - /* 185 */ "input", - /* 186 */ "cmdlist", - /* 187 */ "ecmd", - /* 188 */ "cmdx", - /* 189 */ "explain", - /* 190 */ "cmd", - /* 191 */ "transtype", - /* 192 */ "trans_opt", - /* 193 */ "nm", - /* 194 */ "savepoint_opt", - /* 195 */ "create_table", - /* 196 */ "create_table_args", - /* 197 */ "createkw", - /* 198 */ "temp", - /* 199 */ "ifnotexists", - /* 200 */ "dbnm", - /* 201 */ "columnlist", - /* 202 */ "conslist_opt", - /* 203 */ "table_option_set", - /* 204 */ "select", - /* 205 */ "table_option", - /* 206 */ "columnname", - /* 207 */ "carglist", - /* 208 */ "typetoken", - /* 209 */ "typename", - /* 210 */ "signed", - /* 211 */ "plus_num", - /* 212 */ "minus_num", - /* 213 */ "scanpt", - /* 214 */ "scantok", - /* 215 */ "ccons", - /* 216 */ "term", - /* 217 */ "expr", - /* 218 */ "onconf", - /* 219 */ "sortorder", - /* 220 */ "autoinc", - /* 221 */ "eidlist_opt", - /* 222 */ "refargs", - /* 223 */ "defer_subclause", - /* 224 */ "generated", - /* 225 */ "refarg", - /* 226 */ "refact", - /* 227 */ "init_deferred_pred_opt", - /* 228 */ "conslist", - /* 229 */ "tconscomma", - /* 230 */ "tcons", - /* 231 */ "sortlist", - /* 232 */ "eidlist", - /* 233 */ "defer_subclause_opt", - /* 234 */ "orconf", - /* 235 */ "resolvetype", - /* 236 */ "raisetype", - /* 237 */ "ifexists", - /* 238 */ "fullname", - /* 239 */ "selectnowith", - /* 240 */ "oneselect", - /* 241 */ "wqlist", - /* 242 */ "multiselect_op", - /* 243 */ "distinct", - /* 244 */ "selcollist", - /* 245 */ "from", - /* 246 */ "where_opt", - /* 247 */ "groupby_opt", - /* 248 */ "having_opt", - /* 249 */ "orderby_opt", - /* 250 */ "limit_opt", - /* 251 */ "window_clause", - /* 252 */ "values", - /* 253 */ "nexprlist", - /* 254 */ "sclp", - /* 255 */ "as", - /* 256 */ "seltablist", - /* 257 */ "stl_prefix", - /* 258 */ "joinop", - /* 259 */ "on_using", - /* 260 */ "indexed_by", - /* 261 */ "exprlist", - /* 262 */ "xfullname", - /* 263 */ "idlist", - /* 264 */ "indexed_opt", - /* 265 */ "nulls", - /* 266 */ "with", - /* 267 */ "where_opt_ret", - /* 268 */ "setlist", - /* 269 */ "insert_cmd", - /* 270 */ "idlist_opt", - /* 271 */ "upsert", - /* 272 */ "returning", - /* 273 */ "filter_over", - /* 274 */ "likeop", - /* 275 */ "between_op", - /* 276 */ "in_op", - /* 277 */ "paren_exprlist", - /* 278 */ "case_operand", - /* 279 */ "case_exprlist", - /* 280 */ "case_else", - /* 281 */ "uniqueflag", - /* 282 */ "collate", - /* 283 */ "vinto", - /* 284 */ "nmnum", - /* 285 */ "trigger_decl", - /* 286 */ "trigger_cmd_list", - /* 287 */ "trigger_time", - /* 288 */ "trigger_event", - /* 289 */ "foreach_clause", - /* 290 */ "when_clause", - /* 291 */ "trigger_cmd", - /* 292 */ "trnm", - /* 293 */ "tridxby", - /* 294 */ "database_kw_opt", - /* 295 */ "key_opt", - /* 296 */ "add_column_fullname", - /* 297 */ "kwcolumn_opt", - /* 298 */ "create_vtab", - /* 299 */ "vtabarglist", - /* 300 */ "vtabarg", - /* 301 */ "vtabargtoken", - /* 302 */ "lp", - /* 303 */ "anylist", - /* 304 */ "wqitem", - /* 305 */ "wqas", - /* 306 */ "windowdefn_list", - /* 307 */ "windowdefn", - /* 308 */ "window", - /* 309 */ "frame_opt", - /* 310 */ "part_opt", - /* 311 */ "filter_clause", - /* 312 */ "over_clause", - /* 313 */ "range_or_rows", - /* 314 */ "frame_bound", - /* 315 */ "frame_bound_s", - /* 316 */ "frame_bound_e", - /* 317 */ "frame_exclude_opt", - /* 318 */ "frame_exclude", + /* 183 */ "QNUMBER", + /* 184 */ "SPACE", + /* 185 */ "ILLEGAL", + /* 186 */ "input", + /* 187 */ "cmdlist", + /* 188 */ "ecmd", + /* 189 */ "cmdx", + /* 190 */ "explain", + /* 191 */ "cmd", + /* 192 */ "transtype", + /* 193 */ "trans_opt", + /* 194 */ "nm", + /* 195 */ "savepoint_opt", + /* 196 */ "create_table", + /* 197 */ "create_table_args", + /* 198 */ "createkw", + /* 199 */ "temp", + /* 200 */ "ifnotexists", + /* 201 */ "dbnm", + /* 202 */ "columnlist", + /* 203 */ "conslist_opt", + /* 204 */ "table_option_set", + /* 205 */ "select", + /* 206 */ "table_option", + /* 207 */ "columnname", + /* 208 */ "carglist", + /* 209 */ "typetoken", + /* 210 */ "typename", + /* 211 */ "signed", + /* 212 */ "plus_num", + /* 213 */ "minus_num", + /* 214 */ "scanpt", + /* 215 */ "scantok", + /* 216 */ "ccons", + /* 217 */ "term", + /* 218 */ "expr", + /* 219 */ "onconf", + /* 220 */ "sortorder", + /* 221 */ "autoinc", + /* 222 */ "eidlist_opt", + /* 223 */ "refargs", + /* 224 */ "defer_subclause", + /* 225 */ "generated", + /* 226 */ "refarg", + /* 227 */ "refact", + /* 228 */ "init_deferred_pred_opt", + /* 229 */ "conslist", + /* 230 */ "tconscomma", + /* 231 */ "tcons", + /* 232 */ "sortlist", + /* 233 */ "eidlist", + /* 234 */ "defer_subclause_opt", + /* 235 */ "orconf", + /* 236 */ "resolvetype", + /* 237 */ "raisetype", + /* 238 */ "ifexists", + /* 239 */ "fullname", + /* 240 */ "selectnowith", + /* 241 */ "oneselect", + /* 242 */ "wqlist", + /* 243 */ "multiselect_op", + /* 244 */ "distinct", + /* 245 */ "selcollist", + /* 246 */ "from", + /* 247 */ "where_opt", + /* 248 */ "groupby_opt", + /* 249 */ "having_opt", + /* 250 */ "orderby_opt", + /* 251 */ "limit_opt", + /* 252 */ "window_clause", + /* 253 */ "values", + /* 254 */ "nexprlist", + /* 255 */ "mvalues", + /* 256 */ "sclp", + /* 257 */ "as", + /* 258 */ "seltablist", + /* 259 */ "stl_prefix", + /* 260 */ "joinop", + /* 261 */ "on_using", + /* 262 */ "indexed_by", + /* 263 */ "exprlist", + /* 264 */ "xfullname", + /* 265 */ "idlist", + /* 266 */ "indexed_opt", + /* 267 */ "nulls", + /* 268 */ "with", + /* 269 */ "where_opt_ret", + /* 270 */ "setlist", + /* 271 */ "insert_cmd", + /* 272 */ "idlist_opt", + /* 273 */ "upsert", + /* 274 */ "returning", + /* 275 */ "filter_over", + /* 276 */ "likeop", + /* 277 */ "between_op", + /* 278 */ "in_op", + /* 279 */ "paren_exprlist", + /* 280 */ "case_operand", + /* 281 */ "case_exprlist", + /* 282 */ "case_else", + /* 283 */ "uniqueflag", + /* 284 */ "collate", + /* 285 */ "vinto", + /* 286 */ "nmnum", + /* 287 */ "trigger_decl", + /* 288 */ "trigger_cmd_list", + /* 289 */ "trigger_time", + /* 290 */ "trigger_event", + /* 291 */ "foreach_clause", + /* 292 */ "when_clause", + /* 293 */ "trigger_cmd", + /* 294 */ "trnm", + /* 295 */ "tridxby", + /* 296 */ "database_kw_opt", + /* 297 */ "key_opt", + /* 298 */ "add_column_fullname", + /* 299 */ "kwcolumn_opt", + /* 300 */ "create_vtab", + /* 301 */ "vtabarglist", + /* 302 */ "vtabarg", + /* 303 */ "vtabargtoken", + /* 304 */ "lp", + /* 305 */ "anylist", + /* 306 */ "wqitem", + /* 307 */ "wqas", + /* 308 */ "withnm", + /* 309 */ "windowdefn_list", + /* 310 */ "windowdefn", + /* 311 */ "window", + /* 312 */ "frame_opt", + /* 313 */ "part_opt", + /* 314 */ "filter_clause", + /* 315 */ "over_clause", + /* 316 */ "range_or_rows", + /* 317 */ "frame_bound", + /* 318 */ "frame_bound_s", + /* 319 */ "frame_bound_e", + /* 320 */ "frame_exclude_opt", + /* 321 */ "frame_exclude", }; #endif /* defined(YYCOVERAGE) || !defined(NDEBUG) */ @@ -173042,351 +174422,363 @@ static const char *const yyRuleName[] = { /* 92 */ "oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt orderby_opt limit_opt", /* 93 */ "oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt window_clause orderby_opt limit_opt", /* 94 */ "values ::= VALUES LP nexprlist RP", - /* 95 */ "values ::= values COMMA LP nexprlist RP", - /* 96 */ "distinct ::= DISTINCT", - /* 97 */ "distinct ::= ALL", - /* 98 */ "distinct ::=", - /* 99 */ "sclp ::=", - /* 100 */ "selcollist ::= sclp scanpt expr scanpt as", - /* 101 */ "selcollist ::= sclp scanpt STAR", - /* 102 */ "selcollist ::= sclp scanpt nm DOT STAR", - /* 103 */ "as ::= AS nm", - /* 104 */ "as ::=", - /* 105 */ "from ::=", - /* 106 */ "from ::= FROM seltablist", - /* 107 */ "stl_prefix ::= seltablist joinop", - /* 108 */ "stl_prefix ::=", - /* 109 */ "seltablist ::= stl_prefix nm dbnm as on_using", - /* 110 */ "seltablist ::= stl_prefix nm dbnm as indexed_by on_using", - /* 111 */ "seltablist ::= stl_prefix nm dbnm LP exprlist RP as on_using", - /* 112 */ "seltablist ::= stl_prefix LP select RP as on_using", - /* 113 */ "seltablist ::= stl_prefix LP seltablist RP as on_using", - /* 114 */ "dbnm ::=", - /* 115 */ "dbnm ::= DOT nm", - /* 116 */ "fullname ::= nm", - /* 117 */ "fullname ::= nm DOT nm", - /* 118 */ "xfullname ::= nm", - /* 119 */ "xfullname ::= nm DOT nm", - /* 120 */ "xfullname ::= nm DOT nm AS nm", - /* 121 */ "xfullname ::= nm AS nm", - /* 122 */ "joinop ::= COMMA|JOIN", - /* 123 */ "joinop ::= JOIN_KW JOIN", - /* 124 */ "joinop ::= JOIN_KW nm JOIN", - /* 125 */ "joinop ::= JOIN_KW nm nm JOIN", - /* 126 */ "on_using ::= ON expr", - /* 127 */ "on_using ::= USING LP idlist RP", - /* 128 */ "on_using ::=", - /* 129 */ "indexed_opt ::=", - /* 130 */ "indexed_by ::= INDEXED BY nm", - /* 131 */ "indexed_by ::= NOT INDEXED", - /* 132 */ "orderby_opt ::=", - /* 133 */ "orderby_opt ::= ORDER BY sortlist", - /* 134 */ "sortlist ::= sortlist COMMA expr sortorder nulls", - /* 135 */ "sortlist ::= expr sortorder nulls", - /* 136 */ "sortorder ::= ASC", - /* 137 */ "sortorder ::= DESC", - /* 138 */ "sortorder ::=", - /* 139 */ "nulls ::= NULLS FIRST", - /* 140 */ "nulls ::= NULLS LAST", - /* 141 */ "nulls ::=", - /* 142 */ "groupby_opt ::=", - /* 143 */ "groupby_opt ::= GROUP BY nexprlist", - /* 144 */ "having_opt ::=", - /* 145 */ "having_opt ::= HAVING expr", - /* 146 */ "limit_opt ::=", - /* 147 */ "limit_opt ::= LIMIT expr", - /* 148 */ "limit_opt ::= LIMIT expr OFFSET expr", - /* 149 */ "limit_opt ::= LIMIT expr COMMA expr", - /* 150 */ "cmd ::= with DELETE FROM xfullname indexed_opt where_opt_ret", - /* 151 */ "where_opt ::=", - /* 152 */ "where_opt ::= WHERE expr", - /* 153 */ "where_opt_ret ::=", - /* 154 */ "where_opt_ret ::= WHERE expr", - /* 155 */ "where_opt_ret ::= RETURNING selcollist", - /* 156 */ "where_opt_ret ::= WHERE expr RETURNING selcollist", - /* 157 */ "cmd ::= with UPDATE orconf xfullname indexed_opt SET setlist from where_opt_ret", - /* 158 */ "setlist ::= setlist COMMA nm EQ expr", - /* 159 */ "setlist ::= setlist COMMA LP idlist RP EQ expr", - /* 160 */ "setlist ::= nm EQ expr", - /* 161 */ "setlist ::= LP idlist RP EQ expr", - /* 162 */ "cmd ::= with insert_cmd INTO xfullname idlist_opt select upsert", - /* 163 */ "cmd ::= with insert_cmd INTO xfullname idlist_opt DEFAULT VALUES returning", - /* 164 */ "upsert ::=", - /* 165 */ "upsert ::= RETURNING selcollist", - /* 166 */ "upsert ::= ON CONFLICT LP sortlist RP where_opt DO UPDATE SET setlist where_opt upsert", - /* 167 */ "upsert ::= ON CONFLICT LP sortlist RP where_opt DO NOTHING upsert", - /* 168 */ "upsert ::= ON CONFLICT DO NOTHING returning", - /* 169 */ "upsert ::= ON CONFLICT DO UPDATE SET setlist where_opt returning", - /* 170 */ "returning ::= RETURNING selcollist", - /* 171 */ "insert_cmd ::= INSERT orconf", - /* 172 */ "insert_cmd ::= REPLACE", - /* 173 */ "idlist_opt ::=", - /* 174 */ "idlist_opt ::= LP idlist RP", - /* 175 */ "idlist ::= idlist COMMA nm", - /* 176 */ "idlist ::= nm", - /* 177 */ "expr ::= LP expr RP", - /* 178 */ "expr ::= ID|INDEXED|JOIN_KW", - /* 179 */ "expr ::= nm DOT nm", - /* 180 */ "expr ::= nm DOT nm DOT nm", - /* 181 */ "term ::= NULL|FLOAT|BLOB", - /* 182 */ "term ::= STRING", - /* 183 */ "term ::= INTEGER", - /* 184 */ "expr ::= VARIABLE", - /* 185 */ "expr ::= expr COLLATE ID|STRING", - /* 186 */ "expr ::= CAST LP expr AS typetoken RP", - /* 187 */ "expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP", - /* 188 */ "expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist ORDER BY sortlist RP", - /* 189 */ "expr ::= ID|INDEXED|JOIN_KW LP STAR RP", - /* 190 */ "expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP filter_over", - /* 191 */ "expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist ORDER BY sortlist RP filter_over", - /* 192 */ "expr ::= ID|INDEXED|JOIN_KW LP STAR RP filter_over", - /* 193 */ "term ::= CTIME_KW", - /* 194 */ "expr ::= LP nexprlist COMMA expr RP", - /* 195 */ "expr ::= expr AND expr", - /* 196 */ "expr ::= expr OR expr", - /* 197 */ "expr ::= expr LT|GT|GE|LE expr", - /* 198 */ "expr ::= expr EQ|NE expr", - /* 199 */ "expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr", - /* 200 */ "expr ::= expr PLUS|MINUS expr", - /* 201 */ "expr ::= expr STAR|SLASH|REM expr", - /* 202 */ "expr ::= expr CONCAT expr", - /* 203 */ "likeop ::= NOT LIKE_KW|MATCH", - /* 204 */ "expr ::= expr likeop expr", - /* 205 */ "expr ::= expr likeop expr ESCAPE expr", - /* 206 */ "expr ::= expr ISNULL|NOTNULL", - /* 207 */ "expr ::= expr NOT NULL", - /* 208 */ "expr ::= expr IS expr", - /* 209 */ "expr ::= expr IS NOT expr", - /* 210 */ "expr ::= expr IS NOT DISTINCT FROM expr", - /* 211 */ "expr ::= expr IS DISTINCT FROM expr", - /* 212 */ "expr ::= NOT expr", - /* 213 */ "expr ::= BITNOT expr", - /* 214 */ "expr ::= PLUS|MINUS expr", - /* 215 */ "expr ::= expr PTR expr", - /* 216 */ "between_op ::= BETWEEN", - /* 217 */ "between_op ::= NOT BETWEEN", - /* 218 */ "expr ::= expr between_op expr AND expr", - /* 219 */ "in_op ::= IN", - /* 220 */ "in_op ::= NOT IN", - /* 221 */ "expr ::= expr in_op LP exprlist RP", - /* 222 */ "expr ::= LP select RP", - /* 223 */ "expr ::= expr in_op LP select RP", - /* 224 */ "expr ::= expr in_op nm dbnm paren_exprlist", - /* 225 */ "expr ::= EXISTS LP select RP", - /* 226 */ "expr ::= CASE case_operand case_exprlist case_else END", - /* 227 */ "case_exprlist ::= case_exprlist WHEN expr THEN expr", - /* 228 */ "case_exprlist ::= WHEN expr THEN expr", - /* 229 */ "case_else ::= ELSE expr", - /* 230 */ "case_else ::=", - /* 231 */ "case_operand ::=", - /* 232 */ "exprlist ::=", - /* 233 */ "nexprlist ::= nexprlist COMMA expr", - /* 234 */ "nexprlist ::= expr", - /* 235 */ "paren_exprlist ::=", - /* 236 */ "paren_exprlist ::= LP exprlist RP", - /* 237 */ "cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt", - /* 238 */ "uniqueflag ::= UNIQUE", - /* 239 */ "uniqueflag ::=", - /* 240 */ "eidlist_opt ::=", - /* 241 */ "eidlist_opt ::= LP eidlist RP", - /* 242 */ "eidlist ::= eidlist COMMA nm collate sortorder", - /* 243 */ "eidlist ::= nm collate sortorder", - /* 244 */ "collate ::=", - /* 245 */ "collate ::= COLLATE ID|STRING", - /* 246 */ "cmd ::= DROP INDEX ifexists fullname", - /* 247 */ "cmd ::= VACUUM vinto", - /* 248 */ "cmd ::= VACUUM nm vinto", - /* 249 */ "vinto ::= INTO expr", - /* 250 */ "vinto ::=", - /* 251 */ "cmd ::= PRAGMA nm dbnm", - /* 252 */ "cmd ::= PRAGMA nm dbnm EQ nmnum", - /* 253 */ "cmd ::= PRAGMA nm dbnm LP nmnum RP", - /* 254 */ "cmd ::= PRAGMA nm dbnm EQ minus_num", - /* 255 */ "cmd ::= PRAGMA nm dbnm LP minus_num RP", - /* 256 */ "plus_num ::= PLUS INTEGER|FLOAT", - /* 257 */ "minus_num ::= MINUS INTEGER|FLOAT", - /* 258 */ "cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END", - /* 259 */ "trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause", - /* 260 */ "trigger_time ::= BEFORE|AFTER", - /* 261 */ "trigger_time ::= INSTEAD OF", - /* 262 */ "trigger_time ::=", - /* 263 */ "trigger_event ::= DELETE|INSERT", - /* 264 */ "trigger_event ::= UPDATE", - /* 265 */ "trigger_event ::= UPDATE OF idlist", - /* 266 */ "when_clause ::=", - /* 267 */ "when_clause ::= WHEN expr", - /* 268 */ "trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI", - /* 269 */ "trigger_cmd_list ::= trigger_cmd SEMI", - /* 270 */ "trnm ::= nm DOT nm", - /* 271 */ "tridxby ::= INDEXED BY nm", - /* 272 */ "tridxby ::= NOT INDEXED", - /* 273 */ "trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist from where_opt scanpt", - /* 274 */ "trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt", - /* 275 */ "trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt", - /* 276 */ "trigger_cmd ::= scanpt select scanpt", - /* 277 */ "expr ::= RAISE LP IGNORE RP", - /* 278 */ "expr ::= RAISE LP raisetype COMMA nm RP", - /* 279 */ "raisetype ::= ROLLBACK", - /* 280 */ "raisetype ::= ABORT", - /* 281 */ "raisetype ::= FAIL", - /* 282 */ "cmd ::= DROP TRIGGER ifexists fullname", - /* 283 */ "cmd ::= ATTACH database_kw_opt expr AS expr key_opt", - /* 284 */ "cmd ::= DETACH database_kw_opt expr", - /* 285 */ "key_opt ::=", - /* 286 */ "key_opt ::= KEY expr", - /* 287 */ "cmd ::= REINDEX", - /* 288 */ "cmd ::= REINDEX nm dbnm", - /* 289 */ "cmd ::= ANALYZE", - /* 290 */ "cmd ::= ANALYZE nm dbnm", - /* 291 */ "cmd ::= ALTER TABLE fullname RENAME TO nm", - /* 292 */ "cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist", - /* 293 */ "cmd ::= ALTER TABLE fullname DROP kwcolumn_opt nm", - /* 294 */ "add_column_fullname ::= fullname", - /* 295 */ "cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm", - /* 296 */ "cmd ::= create_vtab", - /* 297 */ "cmd ::= create_vtab LP vtabarglist RP", - /* 298 */ "create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm", - /* 299 */ "vtabarg ::=", - /* 300 */ "vtabargtoken ::= ANY", - /* 301 */ "vtabargtoken ::= lp anylist RP", - /* 302 */ "lp ::= LP", - /* 303 */ "with ::= WITH wqlist", - /* 304 */ "with ::= WITH RECURSIVE wqlist", - /* 305 */ "wqas ::= AS", - /* 306 */ "wqas ::= AS MATERIALIZED", - /* 307 */ "wqas ::= AS NOT MATERIALIZED", - /* 308 */ "wqitem ::= nm eidlist_opt wqas LP select RP", - /* 309 */ "wqlist ::= wqitem", - /* 310 */ "wqlist ::= wqlist COMMA wqitem", - /* 311 */ "windowdefn_list ::= windowdefn_list COMMA windowdefn", - /* 312 */ "windowdefn ::= nm AS LP window RP", - /* 313 */ "window ::= PARTITION BY nexprlist orderby_opt frame_opt", - /* 314 */ "window ::= nm PARTITION BY nexprlist orderby_opt frame_opt", - /* 315 */ "window ::= ORDER BY sortlist frame_opt", - /* 316 */ "window ::= nm ORDER BY sortlist frame_opt", - /* 317 */ "window ::= nm frame_opt", - /* 318 */ "frame_opt ::=", - /* 319 */ "frame_opt ::= range_or_rows frame_bound_s frame_exclude_opt", - /* 320 */ "frame_opt ::= range_or_rows BETWEEN frame_bound_s AND frame_bound_e frame_exclude_opt", - /* 321 */ "range_or_rows ::= RANGE|ROWS|GROUPS", - /* 322 */ "frame_bound_s ::= frame_bound", - /* 323 */ "frame_bound_s ::= UNBOUNDED PRECEDING", - /* 324 */ "frame_bound_e ::= frame_bound", - /* 325 */ "frame_bound_e ::= UNBOUNDED FOLLOWING", - /* 326 */ "frame_bound ::= expr PRECEDING|FOLLOWING", - /* 327 */ "frame_bound ::= CURRENT ROW", - /* 328 */ "frame_exclude_opt ::=", - /* 329 */ "frame_exclude_opt ::= EXCLUDE frame_exclude", - /* 330 */ "frame_exclude ::= NO OTHERS", - /* 331 */ "frame_exclude ::= CURRENT ROW", - /* 332 */ "frame_exclude ::= GROUP|TIES", - /* 333 */ "window_clause ::= WINDOW windowdefn_list", - /* 334 */ "filter_over ::= filter_clause over_clause", - /* 335 */ "filter_over ::= over_clause", - /* 336 */ "filter_over ::= filter_clause", - /* 337 */ "over_clause ::= OVER LP window RP", - /* 338 */ "over_clause ::= OVER nm", - /* 339 */ "filter_clause ::= FILTER LP WHERE expr RP", - /* 340 */ "input ::= cmdlist", - /* 341 */ "cmdlist ::= cmdlist ecmd", - /* 342 */ "cmdlist ::= ecmd", - /* 343 */ "ecmd ::= SEMI", - /* 344 */ "ecmd ::= cmdx SEMI", - /* 345 */ "ecmd ::= explain cmdx SEMI", - /* 346 */ "trans_opt ::=", - /* 347 */ "trans_opt ::= TRANSACTION", - /* 348 */ "trans_opt ::= TRANSACTION nm", - /* 349 */ "savepoint_opt ::= SAVEPOINT", - /* 350 */ "savepoint_opt ::=", - /* 351 */ "cmd ::= create_table create_table_args", - /* 352 */ "table_option_set ::= table_option", - /* 353 */ "columnlist ::= columnlist COMMA columnname carglist", - /* 354 */ "columnlist ::= columnname carglist", - /* 355 */ "nm ::= ID|INDEXED|JOIN_KW", - /* 356 */ "nm ::= STRING", - /* 357 */ "typetoken ::= typename", - /* 358 */ "typename ::= ID|STRING", - /* 359 */ "signed ::= plus_num", - /* 360 */ "signed ::= minus_num", - /* 361 */ "carglist ::= carglist ccons", - /* 362 */ "carglist ::=", - /* 363 */ "ccons ::= NULL onconf", - /* 364 */ "ccons ::= GENERATED ALWAYS AS generated", - /* 365 */ "ccons ::= AS generated", - /* 366 */ "conslist_opt ::= COMMA conslist", - /* 367 */ "conslist ::= conslist tconscomma tcons", - /* 368 */ "conslist ::= tcons", - /* 369 */ "tconscomma ::=", - /* 370 */ "defer_subclause_opt ::= defer_subclause", - /* 371 */ "resolvetype ::= raisetype", - /* 372 */ "selectnowith ::= oneselect", - /* 373 */ "oneselect ::= values", - /* 374 */ "sclp ::= selcollist COMMA", - /* 375 */ "as ::= ID|STRING", - /* 376 */ "indexed_opt ::= indexed_by", - /* 377 */ "returning ::=", - /* 378 */ "expr ::= term", - /* 379 */ "likeop ::= LIKE_KW|MATCH", - /* 380 */ "case_operand ::= expr", - /* 381 */ "exprlist ::= nexprlist", - /* 382 */ "nmnum ::= plus_num", - /* 383 */ "nmnum ::= nm", - /* 384 */ "nmnum ::= ON", - /* 385 */ "nmnum ::= DELETE", - /* 386 */ "nmnum ::= DEFAULT", - /* 387 */ "plus_num ::= INTEGER|FLOAT", - /* 388 */ "foreach_clause ::=", - /* 389 */ "foreach_clause ::= FOR EACH ROW", - /* 390 */ "trnm ::= nm", - /* 391 */ "tridxby ::=", - /* 392 */ "database_kw_opt ::= DATABASE", - /* 393 */ "database_kw_opt ::=", - /* 394 */ "kwcolumn_opt ::=", - /* 395 */ "kwcolumn_opt ::= COLUMNKW", - /* 396 */ "vtabarglist ::= vtabarg", - /* 397 */ "vtabarglist ::= vtabarglist COMMA vtabarg", - /* 398 */ "vtabarg ::= vtabarg vtabargtoken", - /* 399 */ "anylist ::=", - /* 400 */ "anylist ::= anylist LP anylist RP", - /* 401 */ "anylist ::= anylist ANY", - /* 402 */ "with ::=", - /* 403 */ "windowdefn_list ::= windowdefn", - /* 404 */ "window ::= frame_opt", + /* 95 */ "oneselect ::= mvalues", + /* 96 */ "mvalues ::= values COMMA LP nexprlist RP", + /* 97 */ "mvalues ::= mvalues COMMA LP nexprlist RP", + /* 98 */ "distinct ::= DISTINCT", + /* 99 */ "distinct ::= ALL", + /* 100 */ "distinct ::=", + /* 101 */ "sclp ::=", + /* 102 */ "selcollist ::= sclp scanpt expr scanpt as", + /* 103 */ "selcollist ::= sclp scanpt STAR", + /* 104 */ "selcollist ::= sclp scanpt nm DOT STAR", + /* 105 */ "as ::= AS nm", + /* 106 */ "as ::=", + /* 107 */ "from ::=", + /* 108 */ "from ::= FROM seltablist", + /* 109 */ "stl_prefix ::= seltablist joinop", + /* 110 */ "stl_prefix ::=", + /* 111 */ "seltablist ::= stl_prefix nm dbnm as on_using", + /* 112 */ "seltablist ::= stl_prefix nm dbnm as indexed_by on_using", + /* 113 */ "seltablist ::= stl_prefix nm dbnm LP exprlist RP as on_using", + /* 114 */ "seltablist ::= stl_prefix LP select RP as on_using", + /* 115 */ "seltablist ::= stl_prefix LP seltablist RP as on_using", + /* 116 */ "dbnm ::=", + /* 117 */ "dbnm ::= DOT nm", + /* 118 */ "fullname ::= nm", + /* 119 */ "fullname ::= nm DOT nm", + /* 120 */ "xfullname ::= nm", + /* 121 */ "xfullname ::= nm DOT nm", + /* 122 */ "xfullname ::= nm DOT nm AS nm", + /* 123 */ "xfullname ::= nm AS nm", + /* 124 */ "joinop ::= COMMA|JOIN", + /* 125 */ "joinop ::= JOIN_KW JOIN", + /* 126 */ "joinop ::= JOIN_KW nm JOIN", + /* 127 */ "joinop ::= JOIN_KW nm nm JOIN", + /* 128 */ "on_using ::= ON expr", + /* 129 */ "on_using ::= USING LP idlist RP", + /* 130 */ "on_using ::=", + /* 131 */ "indexed_opt ::=", + /* 132 */ "indexed_by ::= INDEXED BY nm", + /* 133 */ "indexed_by ::= NOT INDEXED", + /* 134 */ "orderby_opt ::=", + /* 135 */ "orderby_opt ::= ORDER BY sortlist", + /* 136 */ "sortlist ::= sortlist COMMA expr sortorder nulls", + /* 137 */ "sortlist ::= expr sortorder nulls", + /* 138 */ "sortorder ::= ASC", + /* 139 */ "sortorder ::= DESC", + /* 140 */ "sortorder ::=", + /* 141 */ "nulls ::= NULLS FIRST", + /* 142 */ "nulls ::= NULLS LAST", + /* 143 */ "nulls ::=", + /* 144 */ "groupby_opt ::=", + /* 145 */ "groupby_opt ::= GROUP BY nexprlist", + /* 146 */ "having_opt ::=", + /* 147 */ "having_opt ::= HAVING expr", + /* 148 */ "limit_opt ::=", + /* 149 */ "limit_opt ::= LIMIT expr", + /* 150 */ "limit_opt ::= LIMIT expr OFFSET expr", + /* 151 */ "limit_opt ::= LIMIT expr COMMA expr", + /* 152 */ "cmd ::= with DELETE FROM xfullname indexed_opt where_opt_ret", + /* 153 */ "where_opt ::=", + /* 154 */ "where_opt ::= WHERE expr", + /* 155 */ "where_opt_ret ::=", + /* 156 */ "where_opt_ret ::= WHERE expr", + /* 157 */ "where_opt_ret ::= RETURNING selcollist", + /* 158 */ "where_opt_ret ::= WHERE expr RETURNING selcollist", + /* 159 */ "cmd ::= with UPDATE orconf xfullname indexed_opt SET setlist from where_opt_ret", + /* 160 */ "setlist ::= setlist COMMA nm EQ expr", + /* 161 */ "setlist ::= setlist COMMA LP idlist RP EQ expr", + /* 162 */ "setlist ::= nm EQ expr", + /* 163 */ "setlist ::= LP idlist RP EQ expr", + /* 164 */ "cmd ::= with insert_cmd INTO xfullname idlist_opt select upsert", + /* 165 */ "cmd ::= with insert_cmd INTO xfullname idlist_opt DEFAULT VALUES returning", + /* 166 */ "upsert ::=", + /* 167 */ "upsert ::= RETURNING selcollist", + /* 168 */ "upsert ::= ON CONFLICT LP sortlist RP where_opt DO UPDATE SET setlist where_opt upsert", + /* 169 */ "upsert ::= ON CONFLICT LP sortlist RP where_opt DO NOTHING upsert", + /* 170 */ "upsert ::= ON CONFLICT DO NOTHING returning", + /* 171 */ "upsert ::= ON CONFLICT DO UPDATE SET setlist where_opt returning", + /* 172 */ "returning ::= RETURNING selcollist", + /* 173 */ "insert_cmd ::= INSERT orconf", + /* 174 */ "insert_cmd ::= REPLACE", + /* 175 */ "idlist_opt ::=", + /* 176 */ "idlist_opt ::= LP idlist RP", + /* 177 */ "idlist ::= idlist COMMA nm", + /* 178 */ "idlist ::= nm", + /* 179 */ "expr ::= LP expr RP", + /* 180 */ "expr ::= ID|INDEXED|JOIN_KW", + /* 181 */ "expr ::= nm DOT nm", + /* 182 */ "expr ::= nm DOT nm DOT nm", + /* 183 */ "term ::= NULL|FLOAT|BLOB", + /* 184 */ "term ::= STRING", + /* 185 */ "term ::= INTEGER", + /* 186 */ "expr ::= VARIABLE", + /* 187 */ "expr ::= expr COLLATE ID|STRING", + /* 188 */ "expr ::= CAST LP expr AS typetoken RP", + /* 189 */ "expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP", + /* 190 */ "expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist ORDER BY sortlist RP", + /* 191 */ "expr ::= ID|INDEXED|JOIN_KW LP STAR RP", + /* 192 */ "expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP filter_over", + /* 193 */ "expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist ORDER BY sortlist RP filter_over", + /* 194 */ "expr ::= ID|INDEXED|JOIN_KW LP STAR RP filter_over", + /* 195 */ "term ::= CTIME_KW", + /* 196 */ "expr ::= LP nexprlist COMMA expr RP", + /* 197 */ "expr ::= expr AND expr", + /* 198 */ "expr ::= expr OR expr", + /* 199 */ "expr ::= expr LT|GT|GE|LE expr", + /* 200 */ "expr ::= expr EQ|NE expr", + /* 201 */ "expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr", + /* 202 */ "expr ::= expr PLUS|MINUS expr", + /* 203 */ "expr ::= expr STAR|SLASH|REM expr", + /* 204 */ "expr ::= expr CONCAT expr", + /* 205 */ "likeop ::= NOT LIKE_KW|MATCH", + /* 206 */ "expr ::= expr likeop expr", + /* 207 */ "expr ::= expr likeop expr ESCAPE expr", + /* 208 */ "expr ::= expr ISNULL|NOTNULL", + /* 209 */ "expr ::= expr NOT NULL", + /* 210 */ "expr ::= expr IS expr", + /* 211 */ "expr ::= expr IS NOT expr", + /* 212 */ "expr ::= expr IS NOT DISTINCT FROM expr", + /* 213 */ "expr ::= expr IS DISTINCT FROM expr", + /* 214 */ "expr ::= NOT expr", + /* 215 */ "expr ::= BITNOT expr", + /* 216 */ "expr ::= PLUS|MINUS expr", + /* 217 */ "expr ::= expr PTR expr", + /* 218 */ "between_op ::= BETWEEN", + /* 219 */ "between_op ::= NOT BETWEEN", + /* 220 */ "expr ::= expr between_op expr AND expr", + /* 221 */ "in_op ::= IN", + /* 222 */ "in_op ::= NOT IN", + /* 223 */ "expr ::= expr in_op LP exprlist RP", + /* 224 */ "expr ::= LP select RP", + /* 225 */ "expr ::= expr in_op LP select RP", + /* 226 */ "expr ::= expr in_op nm dbnm paren_exprlist", + /* 227 */ "expr ::= EXISTS LP select RP", + /* 228 */ "expr ::= CASE case_operand case_exprlist case_else END", + /* 229 */ "case_exprlist ::= case_exprlist WHEN expr THEN expr", + /* 230 */ "case_exprlist ::= WHEN expr THEN expr", + /* 231 */ "case_else ::= ELSE expr", + /* 232 */ "case_else ::=", + /* 233 */ "case_operand ::=", + /* 234 */ "exprlist ::=", + /* 235 */ "nexprlist ::= nexprlist COMMA expr", + /* 236 */ "nexprlist ::= expr", + /* 237 */ "paren_exprlist ::=", + /* 238 */ "paren_exprlist ::= LP exprlist RP", + /* 239 */ "cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt", + /* 240 */ "uniqueflag ::= UNIQUE", + /* 241 */ "uniqueflag ::=", + /* 242 */ "eidlist_opt ::=", + /* 243 */ "eidlist_opt ::= LP eidlist RP", + /* 244 */ "eidlist ::= eidlist COMMA nm collate sortorder", + /* 245 */ "eidlist ::= nm collate sortorder", + /* 246 */ "collate ::=", + /* 247 */ "collate ::= COLLATE ID|STRING", + /* 248 */ "cmd ::= DROP INDEX ifexists fullname", + /* 249 */ "cmd ::= VACUUM vinto", + /* 250 */ "cmd ::= VACUUM nm vinto", + /* 251 */ "vinto ::= INTO expr", + /* 252 */ "vinto ::=", + /* 253 */ "cmd ::= PRAGMA nm dbnm", + /* 254 */ "cmd ::= PRAGMA nm dbnm EQ nmnum", + /* 255 */ "cmd ::= PRAGMA nm dbnm LP nmnum RP", + /* 256 */ "cmd ::= PRAGMA nm dbnm EQ minus_num", + /* 257 */ "cmd ::= PRAGMA nm dbnm LP minus_num RP", + /* 258 */ "plus_num ::= PLUS INTEGER|FLOAT", + /* 259 */ "minus_num ::= MINUS INTEGER|FLOAT", + /* 260 */ "cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END", + /* 261 */ "trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause", + /* 262 */ "trigger_time ::= BEFORE|AFTER", + /* 263 */ "trigger_time ::= INSTEAD OF", + /* 264 */ "trigger_time ::=", + /* 265 */ "trigger_event ::= DELETE|INSERT", + /* 266 */ "trigger_event ::= UPDATE", + /* 267 */ "trigger_event ::= UPDATE OF idlist", + /* 268 */ "when_clause ::=", + /* 269 */ "when_clause ::= WHEN expr", + /* 270 */ "trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI", + /* 271 */ "trigger_cmd_list ::= trigger_cmd SEMI", + /* 272 */ "trnm ::= nm DOT nm", + /* 273 */ "tridxby ::= INDEXED BY nm", + /* 274 */ "tridxby ::= NOT INDEXED", + /* 275 */ "trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist from where_opt scanpt", + /* 276 */ "trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt", + /* 277 */ "trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt", + /* 278 */ "trigger_cmd ::= scanpt select scanpt", + /* 279 */ "expr ::= RAISE LP IGNORE RP", + /* 280 */ "expr ::= RAISE LP raisetype COMMA nm RP", + /* 281 */ "raisetype ::= ROLLBACK", + /* 282 */ "raisetype ::= ABORT", + /* 283 */ "raisetype ::= FAIL", + /* 284 */ "cmd ::= DROP TRIGGER ifexists fullname", + /* 285 */ "cmd ::= ATTACH database_kw_opt expr AS expr key_opt", + /* 286 */ "cmd ::= DETACH database_kw_opt expr", + /* 287 */ "key_opt ::=", + /* 288 */ "key_opt ::= KEY expr", + /* 289 */ "cmd ::= REINDEX", + /* 290 */ "cmd ::= REINDEX nm dbnm", + /* 291 */ "cmd ::= ANALYZE", + /* 292 */ "cmd ::= ANALYZE nm dbnm", + /* 293 */ "cmd ::= ALTER TABLE fullname RENAME TO nm", + /* 294 */ "cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist", + /* 295 */ "cmd ::= ALTER TABLE fullname DROP kwcolumn_opt nm", + /* 296 */ "add_column_fullname ::= fullname", + /* 297 */ "cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm", + /* 298 */ "cmd ::= create_vtab", + /* 299 */ "cmd ::= create_vtab LP vtabarglist RP", + /* 300 */ "create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm", + /* 301 */ "vtabarg ::=", + /* 302 */ "vtabargtoken ::= ANY", + /* 303 */ "vtabargtoken ::= lp anylist RP", + /* 304 */ "lp ::= LP", + /* 305 */ "with ::= WITH wqlist", + /* 306 */ "with ::= WITH RECURSIVE wqlist", + /* 307 */ "wqas ::= AS", + /* 308 */ "wqas ::= AS MATERIALIZED", + /* 309 */ "wqas ::= AS NOT MATERIALIZED", + /* 310 */ "wqitem ::= withnm eidlist_opt wqas LP select RP", + /* 311 */ "withnm ::= nm", + /* 312 */ "wqlist ::= wqitem", + /* 313 */ "wqlist ::= wqlist COMMA wqitem", + /* 314 */ "windowdefn_list ::= windowdefn_list COMMA windowdefn", + /* 315 */ "windowdefn ::= nm AS LP window RP", + /* 316 */ "window ::= PARTITION BY nexprlist orderby_opt frame_opt", + /* 317 */ "window ::= nm PARTITION BY nexprlist orderby_opt frame_opt", + /* 318 */ "window ::= ORDER BY sortlist frame_opt", + /* 319 */ "window ::= nm ORDER BY sortlist frame_opt", + /* 320 */ "window ::= nm frame_opt", + /* 321 */ "frame_opt ::=", + /* 322 */ "frame_opt ::= range_or_rows frame_bound_s frame_exclude_opt", + /* 323 */ "frame_opt ::= range_or_rows BETWEEN frame_bound_s AND frame_bound_e frame_exclude_opt", + /* 324 */ "range_or_rows ::= RANGE|ROWS|GROUPS", + /* 325 */ "frame_bound_s ::= frame_bound", + /* 326 */ "frame_bound_s ::= UNBOUNDED PRECEDING", + /* 327 */ "frame_bound_e ::= frame_bound", + /* 328 */ "frame_bound_e ::= UNBOUNDED FOLLOWING", + /* 329 */ "frame_bound ::= expr PRECEDING|FOLLOWING", + /* 330 */ "frame_bound ::= CURRENT ROW", + /* 331 */ "frame_exclude_opt ::=", + /* 332 */ "frame_exclude_opt ::= EXCLUDE frame_exclude", + /* 333 */ "frame_exclude ::= NO OTHERS", + /* 334 */ "frame_exclude ::= CURRENT ROW", + /* 335 */ "frame_exclude ::= GROUP|TIES", + /* 336 */ "window_clause ::= WINDOW windowdefn_list", + /* 337 */ "filter_over ::= filter_clause over_clause", + /* 338 */ "filter_over ::= over_clause", + /* 339 */ "filter_over ::= filter_clause", + /* 340 */ "over_clause ::= OVER LP window RP", + /* 341 */ "over_clause ::= OVER nm", + /* 342 */ "filter_clause ::= FILTER LP WHERE expr RP", + /* 343 */ "term ::= QNUMBER", + /* 344 */ "input ::= cmdlist", + /* 345 */ "cmdlist ::= cmdlist ecmd", + /* 346 */ "cmdlist ::= ecmd", + /* 347 */ "ecmd ::= SEMI", + /* 348 */ "ecmd ::= cmdx SEMI", + /* 349 */ "ecmd ::= explain cmdx SEMI", + /* 350 */ "trans_opt ::=", + /* 351 */ "trans_opt ::= TRANSACTION", + /* 352 */ "trans_opt ::= TRANSACTION nm", + /* 353 */ "savepoint_opt ::= SAVEPOINT", + /* 354 */ "savepoint_opt ::=", + /* 355 */ "cmd ::= create_table create_table_args", + /* 356 */ "table_option_set ::= table_option", + /* 357 */ "columnlist ::= columnlist COMMA columnname carglist", + /* 358 */ "columnlist ::= columnname carglist", + /* 359 */ "nm ::= ID|INDEXED|JOIN_KW", + /* 360 */ "nm ::= STRING", + /* 361 */ "typetoken ::= typename", + /* 362 */ "typename ::= ID|STRING", + /* 363 */ "signed ::= plus_num", + /* 364 */ "signed ::= minus_num", + /* 365 */ "carglist ::= carglist ccons", + /* 366 */ "carglist ::=", + /* 367 */ "ccons ::= NULL onconf", + /* 368 */ "ccons ::= GENERATED ALWAYS AS generated", + /* 369 */ "ccons ::= AS generated", + /* 370 */ "conslist_opt ::= COMMA conslist", + /* 371 */ "conslist ::= conslist tconscomma tcons", + /* 372 */ "conslist ::= tcons", + /* 373 */ "tconscomma ::=", + /* 374 */ "defer_subclause_opt ::= defer_subclause", + /* 375 */ "resolvetype ::= raisetype", + /* 376 */ "selectnowith ::= oneselect", + /* 377 */ "oneselect ::= values", + /* 378 */ "sclp ::= selcollist COMMA", + /* 379 */ "as ::= ID|STRING", + /* 380 */ "indexed_opt ::= indexed_by", + /* 381 */ "returning ::=", + /* 382 */ "expr ::= term", + /* 383 */ "likeop ::= LIKE_KW|MATCH", + /* 384 */ "case_operand ::= expr", + /* 385 */ "exprlist ::= nexprlist", + /* 386 */ "nmnum ::= plus_num", + /* 387 */ "nmnum ::= nm", + /* 388 */ "nmnum ::= ON", + /* 389 */ "nmnum ::= DELETE", + /* 390 */ "nmnum ::= DEFAULT", + /* 391 */ "plus_num ::= INTEGER|FLOAT", + /* 392 */ "foreach_clause ::=", + /* 393 */ "foreach_clause ::= FOR EACH ROW", + /* 394 */ "trnm ::= nm", + /* 395 */ "tridxby ::=", + /* 396 */ "database_kw_opt ::= DATABASE", + /* 397 */ "database_kw_opt ::=", + /* 398 */ "kwcolumn_opt ::=", + /* 399 */ "kwcolumn_opt ::= COLUMNKW", + /* 400 */ "vtabarglist ::= vtabarg", + /* 401 */ "vtabarglist ::= vtabarglist COMMA vtabarg", + /* 402 */ "vtabarg ::= vtabarg vtabargtoken", + /* 403 */ "anylist ::=", + /* 404 */ "anylist ::= anylist LP anylist RP", + /* 405 */ "anylist ::= anylist ANY", + /* 406 */ "with ::=", + /* 407 */ "windowdefn_list ::= windowdefn", + /* 408 */ "window ::= frame_opt", }; #endif /* NDEBUG */ -#if YYSTACKDEPTH<=0 +#if YYGROWABLESTACK /* ** Try to increase the size of the parser stack. Return the number ** of errors. Return 0 on success. */ static int yyGrowStack(yyParser *p){ + int oldSize = 1 + (int)(p->yystackEnd - p->yystack); int newSize; int idx; yyStackEntry *pNew; - newSize = p->yystksz*2 + 100; - idx = p->yytos ? (int)(p->yytos - p->yystack) : 0; - if( p->yystack==&p->yystk0 ){ - pNew = malloc(newSize*sizeof(pNew[0])); - if( pNew ) pNew[0] = p->yystk0; + newSize = oldSize*2 + 100; + idx = (int)(p->yytos - p->yystack); + if( p->yystack==p->yystk0 ){ + pNew = YYREALLOC(0, newSize*sizeof(pNew[0])); + if( pNew==0 ) return 1; + memcpy(pNew, p->yystack, oldSize*sizeof(pNew[0])); }else{ - pNew = realloc(p->yystack, newSize*sizeof(pNew[0])); + pNew = YYREALLOC(p->yystack, newSize*sizeof(pNew[0])); + if( pNew==0 ) return 1; } - if( pNew ){ - p->yystack = pNew; - p->yytos = &p->yystack[idx]; + p->yystack = pNew; + p->yytos = &p->yystack[idx]; #ifndef NDEBUG - if( yyTraceFILE ){ - fprintf(yyTraceFILE,"%sStack grows from %d to %d entries.\n", - yyTracePrompt, p->yystksz, newSize); - } -#endif - p->yystksz = newSize; + if( yyTraceFILE ){ + fprintf(yyTraceFILE,"%sStack grows from %d to %d entries.\n", + yyTracePrompt, oldSize, newSize); } - return pNew==0; +#endif + p->yystackEnd = &p->yystack[newSize-1]; + return 0; } +#endif /* YYGROWABLESTACK */ + +#if !YYGROWABLESTACK +/* For builds that do no have a growable stack, yyGrowStack always +** returns an error. +*/ +# define yyGrowStack(X) 1 #endif /* Datatype of the argument to the memory allocated passed as the @@ -173406,24 +174798,14 @@ SQLITE_PRIVATE void sqlite3ParserInit(void *yypRawParser sqlite3ParserCTX_PDECL) #ifdef YYTRACKMAXSTACKDEPTH yypParser->yyhwm = 0; #endif -#if YYSTACKDEPTH<=0 - yypParser->yytos = NULL; - yypParser->yystack = NULL; - yypParser->yystksz = 0; - if( yyGrowStack(yypParser) ){ - yypParser->yystack = &yypParser->yystk0; - yypParser->yystksz = 1; - } -#endif + yypParser->yystack = yypParser->yystk0; + yypParser->yystackEnd = &yypParser->yystack[YYSTACKDEPTH-1]; #ifndef YYNOERRORRECOVERY yypParser->yyerrcnt = -1; #endif yypParser->yytos = yypParser->yystack; yypParser->yystack[0].stateno = 0; yypParser->yystack[0].major = 0; -#if YYSTACKDEPTH>0 - yypParser->yystackEnd = &yypParser->yystack[YYSTACKDEPTH-1]; -#endif } #ifndef sqlite3Parser_ENGINEALWAYSONSTACK @@ -173477,97 +174859,98 @@ static void yy_destructor( ** inside the C code. */ /********* Begin destructor definitions ***************************************/ - case 204: /* select */ - case 239: /* selectnowith */ - case 240: /* oneselect */ - case 252: /* values */ + case 205: /* select */ + case 240: /* selectnowith */ + case 241: /* oneselect */ + case 253: /* values */ + case 255: /* mvalues */ { -sqlite3SelectDelete(pParse->db, (yypminor->yy47)); +sqlite3SelectDelete(pParse->db, (yypminor->yy555)); } break; - case 216: /* term */ - case 217: /* expr */ - case 246: /* where_opt */ - case 248: /* having_opt */ - case 267: /* where_opt_ret */ - case 278: /* case_operand */ - case 280: /* case_else */ - case 283: /* vinto */ - case 290: /* when_clause */ - case 295: /* key_opt */ - case 311: /* filter_clause */ + case 217: /* term */ + case 218: /* expr */ + case 247: /* where_opt */ + case 249: /* having_opt */ + case 269: /* where_opt_ret */ + case 280: /* case_operand */ + case 282: /* case_else */ + case 285: /* vinto */ + case 292: /* when_clause */ + case 297: /* key_opt */ + case 314: /* filter_clause */ { -sqlite3ExprDelete(pParse->db, (yypminor->yy528)); +sqlite3ExprDelete(pParse->db, (yypminor->yy454)); } break; - case 221: /* eidlist_opt */ - case 231: /* sortlist */ - case 232: /* eidlist */ - case 244: /* selcollist */ - case 247: /* groupby_opt */ - case 249: /* orderby_opt */ - case 253: /* nexprlist */ - case 254: /* sclp */ - case 261: /* exprlist */ - case 268: /* setlist */ - case 277: /* paren_exprlist */ - case 279: /* case_exprlist */ - case 310: /* part_opt */ + case 222: /* eidlist_opt */ + case 232: /* sortlist */ + case 233: /* eidlist */ + case 245: /* selcollist */ + case 248: /* groupby_opt */ + case 250: /* orderby_opt */ + case 254: /* nexprlist */ + case 256: /* sclp */ + case 263: /* exprlist */ + case 270: /* setlist */ + case 279: /* paren_exprlist */ + case 281: /* case_exprlist */ + case 313: /* part_opt */ { -sqlite3ExprListDelete(pParse->db, (yypminor->yy322)); +sqlite3ExprListDelete(pParse->db, (yypminor->yy14)); } break; - case 238: /* fullname */ - case 245: /* from */ - case 256: /* seltablist */ - case 257: /* stl_prefix */ - case 262: /* xfullname */ + case 239: /* fullname */ + case 246: /* from */ + case 258: /* seltablist */ + case 259: /* stl_prefix */ + case 264: /* xfullname */ { -sqlite3SrcListDelete(pParse->db, (yypminor->yy131)); +sqlite3SrcListDelete(pParse->db, (yypminor->yy203)); } break; - case 241: /* wqlist */ + case 242: /* wqlist */ { -sqlite3WithDelete(pParse->db, (yypminor->yy521)); +sqlite3WithDelete(pParse->db, (yypminor->yy59)); } break; - case 251: /* window_clause */ - case 306: /* windowdefn_list */ + case 252: /* window_clause */ + case 309: /* windowdefn_list */ { -sqlite3WindowListDelete(pParse->db, (yypminor->yy41)); +sqlite3WindowListDelete(pParse->db, (yypminor->yy211)); } break; - case 263: /* idlist */ - case 270: /* idlist_opt */ + case 265: /* idlist */ + case 272: /* idlist_opt */ { -sqlite3IdListDelete(pParse->db, (yypminor->yy254)); +sqlite3IdListDelete(pParse->db, (yypminor->yy132)); } break; - case 273: /* filter_over */ - case 307: /* windowdefn */ - case 308: /* window */ - case 309: /* frame_opt */ - case 312: /* over_clause */ + case 275: /* filter_over */ + case 310: /* windowdefn */ + case 311: /* window */ + case 312: /* frame_opt */ + case 315: /* over_clause */ { -sqlite3WindowDelete(pParse->db, (yypminor->yy41)); +sqlite3WindowDelete(pParse->db, (yypminor->yy211)); } break; - case 286: /* trigger_cmd_list */ - case 291: /* trigger_cmd */ + case 288: /* trigger_cmd_list */ + case 293: /* trigger_cmd */ { -sqlite3DeleteTriggerStep(pParse->db, (yypminor->yy33)); +sqlite3DeleteTriggerStep(pParse->db, (yypminor->yy427)); } break; - case 288: /* trigger_event */ + case 290: /* trigger_event */ { -sqlite3IdListDelete(pParse->db, (yypminor->yy180).b); +sqlite3IdListDelete(pParse->db, (yypminor->yy286).b); } break; - case 314: /* frame_bound */ - case 315: /* frame_bound_s */ - case 316: /* frame_bound_e */ + case 317: /* frame_bound */ + case 318: /* frame_bound_s */ + case 319: /* frame_bound_e */ { -sqlite3ExprDelete(pParse->db, (yypminor->yy595).pExpr); +sqlite3ExprDelete(pParse->db, (yypminor->yy509).pExpr); } break; /********* End destructor definitions *****************************************/ @@ -173601,9 +174984,26 @@ static void yy_pop_parser_stack(yyParser *pParser){ */ SQLITE_PRIVATE void sqlite3ParserFinalize(void *p){ yyParser *pParser = (yyParser*)p; - while( pParser->yytos>pParser->yystack ) yy_pop_parser_stack(pParser); -#if YYSTACKDEPTH<=0 - if( pParser->yystack!=&pParser->yystk0 ) free(pParser->yystack); + + /* In-lined version of calling yy_pop_parser_stack() for each + ** element left in the stack */ + yyStackEntry *yytos = pParser->yytos; + while( yytos>pParser->yystack ){ +#ifndef NDEBUG + if( yyTraceFILE ){ + fprintf(yyTraceFILE,"%sPopping %s\n", + yyTracePrompt, + yyTokenName[yytos->major]); + } +#endif + if( yytos->major>=YY_MIN_DSTRCTR ){ + yy_destructor(pParser, yytos->major, &yytos->minor); + } + yytos--; + } + +#if YYGROWABLESTACK + if( pParser->yystack!=pParser->yystk0 ) YYFREE(pParser->yystack); #endif } @@ -173786,7 +175186,7 @@ static void yyStackOverflow(yyParser *yypParser){ ** stack every overflows */ /******** Begin %stack_overflow code ******************************************/ - sqlite3ErrorMsg(pParse, "parser stack overflow"); + sqlite3OomFault(pParse->db); /******** End %stack_overflow code ********************************************/ sqlite3ParserARG_STORE /* Suppress warning about unused %extra_argument var */ sqlite3ParserCTX_STORE @@ -173830,25 +175230,19 @@ static void yy_shift( assert( yypParser->yyhwm == (int)(yypParser->yytos - yypParser->yystack) ); } #endif -#if YYSTACKDEPTH>0 - if( yypParser->yytos>yypParser->yystackEnd ){ - yypParser->yytos--; - yyStackOverflow(yypParser); - return; - } -#else - if( yypParser->yytos>=&yypParser->yystack[yypParser->yystksz] ){ + yytos = yypParser->yytos; + if( yytos>yypParser->yystackEnd ){ if( yyGrowStack(yypParser) ){ yypParser->yytos--; yyStackOverflow(yypParser); return; } + yytos = yypParser->yytos; + assert( yytos <= yypParser->yystackEnd ); } -#endif if( yyNewState > YY_MAX_SHIFT ){ yyNewState += YY_MIN_REDUCE - YY_MIN_SHIFTREDUCE; } - yytos = yypParser->yytos; yytos->stateno = yyNewState; yytos->major = yyMajor; yytos->minor.yy0 = yyMinor; @@ -173858,411 +175252,415 @@ static void yy_shift( /* For rule J, yyRuleInfoLhs[J] contains the symbol on the left-hand side ** of that rule */ static const YYCODETYPE yyRuleInfoLhs[] = { - 189, /* (0) explain ::= EXPLAIN */ - 189, /* (1) explain ::= EXPLAIN QUERY PLAN */ - 188, /* (2) cmdx ::= cmd */ - 190, /* (3) cmd ::= BEGIN transtype trans_opt */ - 191, /* (4) transtype ::= */ - 191, /* (5) transtype ::= DEFERRED */ - 191, /* (6) transtype ::= IMMEDIATE */ - 191, /* (7) transtype ::= EXCLUSIVE */ - 190, /* (8) cmd ::= COMMIT|END trans_opt */ - 190, /* (9) cmd ::= ROLLBACK trans_opt */ - 190, /* (10) cmd ::= SAVEPOINT nm */ - 190, /* (11) cmd ::= RELEASE savepoint_opt nm */ - 190, /* (12) cmd ::= ROLLBACK trans_opt TO savepoint_opt nm */ - 195, /* (13) create_table ::= createkw temp TABLE ifnotexists nm dbnm */ - 197, /* (14) createkw ::= CREATE */ - 199, /* (15) ifnotexists ::= */ - 199, /* (16) ifnotexists ::= IF NOT EXISTS */ - 198, /* (17) temp ::= TEMP */ - 198, /* (18) temp ::= */ - 196, /* (19) create_table_args ::= LP columnlist conslist_opt RP table_option_set */ - 196, /* (20) create_table_args ::= AS select */ - 203, /* (21) table_option_set ::= */ - 203, /* (22) table_option_set ::= table_option_set COMMA table_option */ - 205, /* (23) table_option ::= WITHOUT nm */ - 205, /* (24) table_option ::= nm */ - 206, /* (25) columnname ::= nm typetoken */ - 208, /* (26) typetoken ::= */ - 208, /* (27) typetoken ::= typename LP signed RP */ - 208, /* (28) typetoken ::= typename LP signed COMMA signed RP */ - 209, /* (29) typename ::= typename ID|STRING */ - 213, /* (30) scanpt ::= */ - 214, /* (31) scantok ::= */ - 215, /* (32) ccons ::= CONSTRAINT nm */ - 215, /* (33) ccons ::= DEFAULT scantok term */ - 215, /* (34) ccons ::= DEFAULT LP expr RP */ - 215, /* (35) ccons ::= DEFAULT PLUS scantok term */ - 215, /* (36) ccons ::= DEFAULT MINUS scantok term */ - 215, /* (37) ccons ::= DEFAULT scantok ID|INDEXED */ - 215, /* (38) ccons ::= NOT NULL onconf */ - 215, /* (39) ccons ::= PRIMARY KEY sortorder onconf autoinc */ - 215, /* (40) ccons ::= UNIQUE onconf */ - 215, /* (41) ccons ::= CHECK LP expr RP */ - 215, /* (42) ccons ::= REFERENCES nm eidlist_opt refargs */ - 215, /* (43) ccons ::= defer_subclause */ - 215, /* (44) ccons ::= COLLATE ID|STRING */ - 224, /* (45) generated ::= LP expr RP */ - 224, /* (46) generated ::= LP expr RP ID */ - 220, /* (47) autoinc ::= */ - 220, /* (48) autoinc ::= AUTOINCR */ - 222, /* (49) refargs ::= */ - 222, /* (50) refargs ::= refargs refarg */ - 225, /* (51) refarg ::= MATCH nm */ - 225, /* (52) refarg ::= ON INSERT refact */ - 225, /* (53) refarg ::= ON DELETE refact */ - 225, /* (54) refarg ::= ON UPDATE refact */ - 226, /* (55) refact ::= SET NULL */ - 226, /* (56) refact ::= SET DEFAULT */ - 226, /* (57) refact ::= CASCADE */ - 226, /* (58) refact ::= RESTRICT */ - 226, /* (59) refact ::= NO ACTION */ - 223, /* (60) defer_subclause ::= NOT DEFERRABLE init_deferred_pred_opt */ - 223, /* (61) defer_subclause ::= DEFERRABLE init_deferred_pred_opt */ - 227, /* (62) init_deferred_pred_opt ::= */ - 227, /* (63) init_deferred_pred_opt ::= INITIALLY DEFERRED */ - 227, /* (64) init_deferred_pred_opt ::= INITIALLY IMMEDIATE */ - 202, /* (65) conslist_opt ::= */ - 229, /* (66) tconscomma ::= COMMA */ - 230, /* (67) tcons ::= CONSTRAINT nm */ - 230, /* (68) tcons ::= PRIMARY KEY LP sortlist autoinc RP onconf */ - 230, /* (69) tcons ::= UNIQUE LP sortlist RP onconf */ - 230, /* (70) tcons ::= CHECK LP expr RP onconf */ - 230, /* (71) tcons ::= FOREIGN KEY LP eidlist RP REFERENCES nm eidlist_opt refargs defer_subclause_opt */ - 233, /* (72) defer_subclause_opt ::= */ - 218, /* (73) onconf ::= */ - 218, /* (74) onconf ::= ON CONFLICT resolvetype */ - 234, /* (75) orconf ::= */ - 234, /* (76) orconf ::= OR resolvetype */ - 235, /* (77) resolvetype ::= IGNORE */ - 235, /* (78) resolvetype ::= REPLACE */ - 190, /* (79) cmd ::= DROP TABLE ifexists fullname */ - 237, /* (80) ifexists ::= IF EXISTS */ - 237, /* (81) ifexists ::= */ - 190, /* (82) cmd ::= createkw temp VIEW ifnotexists nm dbnm eidlist_opt AS select */ - 190, /* (83) cmd ::= DROP VIEW ifexists fullname */ - 190, /* (84) cmd ::= select */ - 204, /* (85) select ::= WITH wqlist selectnowith */ - 204, /* (86) select ::= WITH RECURSIVE wqlist selectnowith */ - 204, /* (87) select ::= selectnowith */ - 239, /* (88) selectnowith ::= selectnowith multiselect_op oneselect */ - 242, /* (89) multiselect_op ::= UNION */ - 242, /* (90) multiselect_op ::= UNION ALL */ - 242, /* (91) multiselect_op ::= EXCEPT|INTERSECT */ - 240, /* (92) oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt orderby_opt limit_opt */ - 240, /* (93) oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt window_clause orderby_opt limit_opt */ - 252, /* (94) values ::= VALUES LP nexprlist RP */ - 252, /* (95) values ::= values COMMA LP nexprlist RP */ - 243, /* (96) distinct ::= DISTINCT */ - 243, /* (97) distinct ::= ALL */ - 243, /* (98) distinct ::= */ - 254, /* (99) sclp ::= */ - 244, /* (100) selcollist ::= sclp scanpt expr scanpt as */ - 244, /* (101) selcollist ::= sclp scanpt STAR */ - 244, /* (102) selcollist ::= sclp scanpt nm DOT STAR */ - 255, /* (103) as ::= AS nm */ - 255, /* (104) as ::= */ - 245, /* (105) from ::= */ - 245, /* (106) from ::= FROM seltablist */ - 257, /* (107) stl_prefix ::= seltablist joinop */ - 257, /* (108) stl_prefix ::= */ - 256, /* (109) seltablist ::= stl_prefix nm dbnm as on_using */ - 256, /* (110) seltablist ::= stl_prefix nm dbnm as indexed_by on_using */ - 256, /* (111) seltablist ::= stl_prefix nm dbnm LP exprlist RP as on_using */ - 256, /* (112) seltablist ::= stl_prefix LP select RP as on_using */ - 256, /* (113) seltablist ::= stl_prefix LP seltablist RP as on_using */ - 200, /* (114) dbnm ::= */ - 200, /* (115) dbnm ::= DOT nm */ - 238, /* (116) fullname ::= nm */ - 238, /* (117) fullname ::= nm DOT nm */ - 262, /* (118) xfullname ::= nm */ - 262, /* (119) xfullname ::= nm DOT nm */ - 262, /* (120) xfullname ::= nm DOT nm AS nm */ - 262, /* (121) xfullname ::= nm AS nm */ - 258, /* (122) joinop ::= COMMA|JOIN */ - 258, /* (123) joinop ::= JOIN_KW JOIN */ - 258, /* (124) joinop ::= JOIN_KW nm JOIN */ - 258, /* (125) joinop ::= JOIN_KW nm nm JOIN */ - 259, /* (126) on_using ::= ON expr */ - 259, /* (127) on_using ::= USING LP idlist RP */ - 259, /* (128) on_using ::= */ - 264, /* (129) indexed_opt ::= */ - 260, /* (130) indexed_by ::= INDEXED BY nm */ - 260, /* (131) indexed_by ::= NOT INDEXED */ - 249, /* (132) orderby_opt ::= */ - 249, /* (133) orderby_opt ::= ORDER BY sortlist */ - 231, /* (134) sortlist ::= sortlist COMMA expr sortorder nulls */ - 231, /* (135) sortlist ::= expr sortorder nulls */ - 219, /* (136) sortorder ::= ASC */ - 219, /* (137) sortorder ::= DESC */ - 219, /* (138) sortorder ::= */ - 265, /* (139) nulls ::= NULLS FIRST */ - 265, /* (140) nulls ::= NULLS LAST */ - 265, /* (141) nulls ::= */ - 247, /* (142) groupby_opt ::= */ - 247, /* (143) groupby_opt ::= GROUP BY nexprlist */ - 248, /* (144) having_opt ::= */ - 248, /* (145) having_opt ::= HAVING expr */ - 250, /* (146) limit_opt ::= */ - 250, /* (147) limit_opt ::= LIMIT expr */ - 250, /* (148) limit_opt ::= LIMIT expr OFFSET expr */ - 250, /* (149) limit_opt ::= LIMIT expr COMMA expr */ - 190, /* (150) cmd ::= with DELETE FROM xfullname indexed_opt where_opt_ret */ - 246, /* (151) where_opt ::= */ - 246, /* (152) where_opt ::= WHERE expr */ - 267, /* (153) where_opt_ret ::= */ - 267, /* (154) where_opt_ret ::= WHERE expr */ - 267, /* (155) where_opt_ret ::= RETURNING selcollist */ - 267, /* (156) where_opt_ret ::= WHERE expr RETURNING selcollist */ - 190, /* (157) cmd ::= with UPDATE orconf xfullname indexed_opt SET setlist from where_opt_ret */ - 268, /* (158) setlist ::= setlist COMMA nm EQ expr */ - 268, /* (159) setlist ::= setlist COMMA LP idlist RP EQ expr */ - 268, /* (160) setlist ::= nm EQ expr */ - 268, /* (161) setlist ::= LP idlist RP EQ expr */ - 190, /* (162) cmd ::= with insert_cmd INTO xfullname idlist_opt select upsert */ - 190, /* (163) cmd ::= with insert_cmd INTO xfullname idlist_opt DEFAULT VALUES returning */ - 271, /* (164) upsert ::= */ - 271, /* (165) upsert ::= RETURNING selcollist */ - 271, /* (166) upsert ::= ON CONFLICT LP sortlist RP where_opt DO UPDATE SET setlist where_opt upsert */ - 271, /* (167) upsert ::= ON CONFLICT LP sortlist RP where_opt DO NOTHING upsert */ - 271, /* (168) upsert ::= ON CONFLICT DO NOTHING returning */ - 271, /* (169) upsert ::= ON CONFLICT DO UPDATE SET setlist where_opt returning */ - 272, /* (170) returning ::= RETURNING selcollist */ - 269, /* (171) insert_cmd ::= INSERT orconf */ - 269, /* (172) insert_cmd ::= REPLACE */ - 270, /* (173) idlist_opt ::= */ - 270, /* (174) idlist_opt ::= LP idlist RP */ - 263, /* (175) idlist ::= idlist COMMA nm */ - 263, /* (176) idlist ::= nm */ - 217, /* (177) expr ::= LP expr RP */ - 217, /* (178) expr ::= ID|INDEXED|JOIN_KW */ - 217, /* (179) expr ::= nm DOT nm */ - 217, /* (180) expr ::= nm DOT nm DOT nm */ - 216, /* (181) term ::= NULL|FLOAT|BLOB */ - 216, /* (182) term ::= STRING */ - 216, /* (183) term ::= INTEGER */ - 217, /* (184) expr ::= VARIABLE */ - 217, /* (185) expr ::= expr COLLATE ID|STRING */ - 217, /* (186) expr ::= CAST LP expr AS typetoken RP */ - 217, /* (187) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP */ - 217, /* (188) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist ORDER BY sortlist RP */ - 217, /* (189) expr ::= ID|INDEXED|JOIN_KW LP STAR RP */ - 217, /* (190) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP filter_over */ - 217, /* (191) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist ORDER BY sortlist RP filter_over */ - 217, /* (192) expr ::= ID|INDEXED|JOIN_KW LP STAR RP filter_over */ - 216, /* (193) term ::= CTIME_KW */ - 217, /* (194) expr ::= LP nexprlist COMMA expr RP */ - 217, /* (195) expr ::= expr AND expr */ - 217, /* (196) expr ::= expr OR expr */ - 217, /* (197) expr ::= expr LT|GT|GE|LE expr */ - 217, /* (198) expr ::= expr EQ|NE expr */ - 217, /* (199) expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr */ - 217, /* (200) expr ::= expr PLUS|MINUS expr */ - 217, /* (201) expr ::= expr STAR|SLASH|REM expr */ - 217, /* (202) expr ::= expr CONCAT expr */ - 274, /* (203) likeop ::= NOT LIKE_KW|MATCH */ - 217, /* (204) expr ::= expr likeop expr */ - 217, /* (205) expr ::= expr likeop expr ESCAPE expr */ - 217, /* (206) expr ::= expr ISNULL|NOTNULL */ - 217, /* (207) expr ::= expr NOT NULL */ - 217, /* (208) expr ::= expr IS expr */ - 217, /* (209) expr ::= expr IS NOT expr */ - 217, /* (210) expr ::= expr IS NOT DISTINCT FROM expr */ - 217, /* (211) expr ::= expr IS DISTINCT FROM expr */ - 217, /* (212) expr ::= NOT expr */ - 217, /* (213) expr ::= BITNOT expr */ - 217, /* (214) expr ::= PLUS|MINUS expr */ - 217, /* (215) expr ::= expr PTR expr */ - 275, /* (216) between_op ::= BETWEEN */ - 275, /* (217) between_op ::= NOT BETWEEN */ - 217, /* (218) expr ::= expr between_op expr AND expr */ - 276, /* (219) in_op ::= IN */ - 276, /* (220) in_op ::= NOT IN */ - 217, /* (221) expr ::= expr in_op LP exprlist RP */ - 217, /* (222) expr ::= LP select RP */ - 217, /* (223) expr ::= expr in_op LP select RP */ - 217, /* (224) expr ::= expr in_op nm dbnm paren_exprlist */ - 217, /* (225) expr ::= EXISTS LP select RP */ - 217, /* (226) expr ::= CASE case_operand case_exprlist case_else END */ - 279, /* (227) case_exprlist ::= case_exprlist WHEN expr THEN expr */ - 279, /* (228) case_exprlist ::= WHEN expr THEN expr */ - 280, /* (229) case_else ::= ELSE expr */ - 280, /* (230) case_else ::= */ - 278, /* (231) case_operand ::= */ - 261, /* (232) exprlist ::= */ - 253, /* (233) nexprlist ::= nexprlist COMMA expr */ - 253, /* (234) nexprlist ::= expr */ - 277, /* (235) paren_exprlist ::= */ - 277, /* (236) paren_exprlist ::= LP exprlist RP */ - 190, /* (237) cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt */ - 281, /* (238) uniqueflag ::= UNIQUE */ - 281, /* (239) uniqueflag ::= */ - 221, /* (240) eidlist_opt ::= */ - 221, /* (241) eidlist_opt ::= LP eidlist RP */ - 232, /* (242) eidlist ::= eidlist COMMA nm collate sortorder */ - 232, /* (243) eidlist ::= nm collate sortorder */ - 282, /* (244) collate ::= */ - 282, /* (245) collate ::= COLLATE ID|STRING */ - 190, /* (246) cmd ::= DROP INDEX ifexists fullname */ - 190, /* (247) cmd ::= VACUUM vinto */ - 190, /* (248) cmd ::= VACUUM nm vinto */ - 283, /* (249) vinto ::= INTO expr */ - 283, /* (250) vinto ::= */ - 190, /* (251) cmd ::= PRAGMA nm dbnm */ - 190, /* (252) cmd ::= PRAGMA nm dbnm EQ nmnum */ - 190, /* (253) cmd ::= PRAGMA nm dbnm LP nmnum RP */ - 190, /* (254) cmd ::= PRAGMA nm dbnm EQ minus_num */ - 190, /* (255) cmd ::= PRAGMA nm dbnm LP minus_num RP */ - 211, /* (256) plus_num ::= PLUS INTEGER|FLOAT */ - 212, /* (257) minus_num ::= MINUS INTEGER|FLOAT */ - 190, /* (258) cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END */ - 285, /* (259) trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause */ - 287, /* (260) trigger_time ::= BEFORE|AFTER */ - 287, /* (261) trigger_time ::= INSTEAD OF */ - 287, /* (262) trigger_time ::= */ - 288, /* (263) trigger_event ::= DELETE|INSERT */ - 288, /* (264) trigger_event ::= UPDATE */ - 288, /* (265) trigger_event ::= UPDATE OF idlist */ - 290, /* (266) when_clause ::= */ - 290, /* (267) when_clause ::= WHEN expr */ - 286, /* (268) trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */ - 286, /* (269) trigger_cmd_list ::= trigger_cmd SEMI */ - 292, /* (270) trnm ::= nm DOT nm */ - 293, /* (271) tridxby ::= INDEXED BY nm */ - 293, /* (272) tridxby ::= NOT INDEXED */ - 291, /* (273) trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist from where_opt scanpt */ - 291, /* (274) trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt */ - 291, /* (275) trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt */ - 291, /* (276) trigger_cmd ::= scanpt select scanpt */ - 217, /* (277) expr ::= RAISE LP IGNORE RP */ - 217, /* (278) expr ::= RAISE LP raisetype COMMA nm RP */ - 236, /* (279) raisetype ::= ROLLBACK */ - 236, /* (280) raisetype ::= ABORT */ - 236, /* (281) raisetype ::= FAIL */ - 190, /* (282) cmd ::= DROP TRIGGER ifexists fullname */ - 190, /* (283) cmd ::= ATTACH database_kw_opt expr AS expr key_opt */ - 190, /* (284) cmd ::= DETACH database_kw_opt expr */ - 295, /* (285) key_opt ::= */ - 295, /* (286) key_opt ::= KEY expr */ - 190, /* (287) cmd ::= REINDEX */ - 190, /* (288) cmd ::= REINDEX nm dbnm */ - 190, /* (289) cmd ::= ANALYZE */ - 190, /* (290) cmd ::= ANALYZE nm dbnm */ - 190, /* (291) cmd ::= ALTER TABLE fullname RENAME TO nm */ - 190, /* (292) cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist */ - 190, /* (293) cmd ::= ALTER TABLE fullname DROP kwcolumn_opt nm */ - 296, /* (294) add_column_fullname ::= fullname */ - 190, /* (295) cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm */ - 190, /* (296) cmd ::= create_vtab */ - 190, /* (297) cmd ::= create_vtab LP vtabarglist RP */ - 298, /* (298) create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm */ - 300, /* (299) vtabarg ::= */ - 301, /* (300) vtabargtoken ::= ANY */ - 301, /* (301) vtabargtoken ::= lp anylist RP */ - 302, /* (302) lp ::= LP */ - 266, /* (303) with ::= WITH wqlist */ - 266, /* (304) with ::= WITH RECURSIVE wqlist */ - 305, /* (305) wqas ::= AS */ - 305, /* (306) wqas ::= AS MATERIALIZED */ - 305, /* (307) wqas ::= AS NOT MATERIALIZED */ - 304, /* (308) wqitem ::= nm eidlist_opt wqas LP select RP */ - 241, /* (309) wqlist ::= wqitem */ - 241, /* (310) wqlist ::= wqlist COMMA wqitem */ - 306, /* (311) windowdefn_list ::= windowdefn_list COMMA windowdefn */ - 307, /* (312) windowdefn ::= nm AS LP window RP */ - 308, /* (313) window ::= PARTITION BY nexprlist orderby_opt frame_opt */ - 308, /* (314) window ::= nm PARTITION BY nexprlist orderby_opt frame_opt */ - 308, /* (315) window ::= ORDER BY sortlist frame_opt */ - 308, /* (316) window ::= nm ORDER BY sortlist frame_opt */ - 308, /* (317) window ::= nm frame_opt */ - 309, /* (318) frame_opt ::= */ - 309, /* (319) frame_opt ::= range_or_rows frame_bound_s frame_exclude_opt */ - 309, /* (320) frame_opt ::= range_or_rows BETWEEN frame_bound_s AND frame_bound_e frame_exclude_opt */ - 313, /* (321) range_or_rows ::= RANGE|ROWS|GROUPS */ - 315, /* (322) frame_bound_s ::= frame_bound */ - 315, /* (323) frame_bound_s ::= UNBOUNDED PRECEDING */ - 316, /* (324) frame_bound_e ::= frame_bound */ - 316, /* (325) frame_bound_e ::= UNBOUNDED FOLLOWING */ - 314, /* (326) frame_bound ::= expr PRECEDING|FOLLOWING */ - 314, /* (327) frame_bound ::= CURRENT ROW */ - 317, /* (328) frame_exclude_opt ::= */ - 317, /* (329) frame_exclude_opt ::= EXCLUDE frame_exclude */ - 318, /* (330) frame_exclude ::= NO OTHERS */ - 318, /* (331) frame_exclude ::= CURRENT ROW */ - 318, /* (332) frame_exclude ::= GROUP|TIES */ - 251, /* (333) window_clause ::= WINDOW windowdefn_list */ - 273, /* (334) filter_over ::= filter_clause over_clause */ - 273, /* (335) filter_over ::= over_clause */ - 273, /* (336) filter_over ::= filter_clause */ - 312, /* (337) over_clause ::= OVER LP window RP */ - 312, /* (338) over_clause ::= OVER nm */ - 311, /* (339) filter_clause ::= FILTER LP WHERE expr RP */ - 185, /* (340) input ::= cmdlist */ - 186, /* (341) cmdlist ::= cmdlist ecmd */ - 186, /* (342) cmdlist ::= ecmd */ - 187, /* (343) ecmd ::= SEMI */ - 187, /* (344) ecmd ::= cmdx SEMI */ - 187, /* (345) ecmd ::= explain cmdx SEMI */ - 192, /* (346) trans_opt ::= */ - 192, /* (347) trans_opt ::= TRANSACTION */ - 192, /* (348) trans_opt ::= TRANSACTION nm */ - 194, /* (349) savepoint_opt ::= SAVEPOINT */ - 194, /* (350) savepoint_opt ::= */ - 190, /* (351) cmd ::= create_table create_table_args */ - 203, /* (352) table_option_set ::= table_option */ - 201, /* (353) columnlist ::= columnlist COMMA columnname carglist */ - 201, /* (354) columnlist ::= columnname carglist */ - 193, /* (355) nm ::= ID|INDEXED|JOIN_KW */ - 193, /* (356) nm ::= STRING */ - 208, /* (357) typetoken ::= typename */ - 209, /* (358) typename ::= ID|STRING */ - 210, /* (359) signed ::= plus_num */ - 210, /* (360) signed ::= minus_num */ - 207, /* (361) carglist ::= carglist ccons */ - 207, /* (362) carglist ::= */ - 215, /* (363) ccons ::= NULL onconf */ - 215, /* (364) ccons ::= GENERATED ALWAYS AS generated */ - 215, /* (365) ccons ::= AS generated */ - 202, /* (366) conslist_opt ::= COMMA conslist */ - 228, /* (367) conslist ::= conslist tconscomma tcons */ - 228, /* (368) conslist ::= tcons */ - 229, /* (369) tconscomma ::= */ - 233, /* (370) defer_subclause_opt ::= defer_subclause */ - 235, /* (371) resolvetype ::= raisetype */ - 239, /* (372) selectnowith ::= oneselect */ - 240, /* (373) oneselect ::= values */ - 254, /* (374) sclp ::= selcollist COMMA */ - 255, /* (375) as ::= ID|STRING */ - 264, /* (376) indexed_opt ::= indexed_by */ - 272, /* (377) returning ::= */ - 217, /* (378) expr ::= term */ - 274, /* (379) likeop ::= LIKE_KW|MATCH */ - 278, /* (380) case_operand ::= expr */ - 261, /* (381) exprlist ::= nexprlist */ - 284, /* (382) nmnum ::= plus_num */ - 284, /* (383) nmnum ::= nm */ - 284, /* (384) nmnum ::= ON */ - 284, /* (385) nmnum ::= DELETE */ - 284, /* (386) nmnum ::= DEFAULT */ - 211, /* (387) plus_num ::= INTEGER|FLOAT */ - 289, /* (388) foreach_clause ::= */ - 289, /* (389) foreach_clause ::= FOR EACH ROW */ - 292, /* (390) trnm ::= nm */ - 293, /* (391) tridxby ::= */ - 294, /* (392) database_kw_opt ::= DATABASE */ - 294, /* (393) database_kw_opt ::= */ - 297, /* (394) kwcolumn_opt ::= */ - 297, /* (395) kwcolumn_opt ::= COLUMNKW */ - 299, /* (396) vtabarglist ::= vtabarg */ - 299, /* (397) vtabarglist ::= vtabarglist COMMA vtabarg */ - 300, /* (398) vtabarg ::= vtabarg vtabargtoken */ - 303, /* (399) anylist ::= */ - 303, /* (400) anylist ::= anylist LP anylist RP */ - 303, /* (401) anylist ::= anylist ANY */ - 266, /* (402) with ::= */ - 306, /* (403) windowdefn_list ::= windowdefn */ - 308, /* (404) window ::= frame_opt */ + 190, /* (0) explain ::= EXPLAIN */ + 190, /* (1) explain ::= EXPLAIN QUERY PLAN */ + 189, /* (2) cmdx ::= cmd */ + 191, /* (3) cmd ::= BEGIN transtype trans_opt */ + 192, /* (4) transtype ::= */ + 192, /* (5) transtype ::= DEFERRED */ + 192, /* (6) transtype ::= IMMEDIATE */ + 192, /* (7) transtype ::= EXCLUSIVE */ + 191, /* (8) cmd ::= COMMIT|END trans_opt */ + 191, /* (9) cmd ::= ROLLBACK trans_opt */ + 191, /* (10) cmd ::= SAVEPOINT nm */ + 191, /* (11) cmd ::= RELEASE savepoint_opt nm */ + 191, /* (12) cmd ::= ROLLBACK trans_opt TO savepoint_opt nm */ + 196, /* (13) create_table ::= createkw temp TABLE ifnotexists nm dbnm */ + 198, /* (14) createkw ::= CREATE */ + 200, /* (15) ifnotexists ::= */ + 200, /* (16) ifnotexists ::= IF NOT EXISTS */ + 199, /* (17) temp ::= TEMP */ + 199, /* (18) temp ::= */ + 197, /* (19) create_table_args ::= LP columnlist conslist_opt RP table_option_set */ + 197, /* (20) create_table_args ::= AS select */ + 204, /* (21) table_option_set ::= */ + 204, /* (22) table_option_set ::= table_option_set COMMA table_option */ + 206, /* (23) table_option ::= WITHOUT nm */ + 206, /* (24) table_option ::= nm */ + 207, /* (25) columnname ::= nm typetoken */ + 209, /* (26) typetoken ::= */ + 209, /* (27) typetoken ::= typename LP signed RP */ + 209, /* (28) typetoken ::= typename LP signed COMMA signed RP */ + 210, /* (29) typename ::= typename ID|STRING */ + 214, /* (30) scanpt ::= */ + 215, /* (31) scantok ::= */ + 216, /* (32) ccons ::= CONSTRAINT nm */ + 216, /* (33) ccons ::= DEFAULT scantok term */ + 216, /* (34) ccons ::= DEFAULT LP expr RP */ + 216, /* (35) ccons ::= DEFAULT PLUS scantok term */ + 216, /* (36) ccons ::= DEFAULT MINUS scantok term */ + 216, /* (37) ccons ::= DEFAULT scantok ID|INDEXED */ + 216, /* (38) ccons ::= NOT NULL onconf */ + 216, /* (39) ccons ::= PRIMARY KEY sortorder onconf autoinc */ + 216, /* (40) ccons ::= UNIQUE onconf */ + 216, /* (41) ccons ::= CHECK LP expr RP */ + 216, /* (42) ccons ::= REFERENCES nm eidlist_opt refargs */ + 216, /* (43) ccons ::= defer_subclause */ + 216, /* (44) ccons ::= COLLATE ID|STRING */ + 225, /* (45) generated ::= LP expr RP */ + 225, /* (46) generated ::= LP expr RP ID */ + 221, /* (47) autoinc ::= */ + 221, /* (48) autoinc ::= AUTOINCR */ + 223, /* (49) refargs ::= */ + 223, /* (50) refargs ::= refargs refarg */ + 226, /* (51) refarg ::= MATCH nm */ + 226, /* (52) refarg ::= ON INSERT refact */ + 226, /* (53) refarg ::= ON DELETE refact */ + 226, /* (54) refarg ::= ON UPDATE refact */ + 227, /* (55) refact ::= SET NULL */ + 227, /* (56) refact ::= SET DEFAULT */ + 227, /* (57) refact ::= CASCADE */ + 227, /* (58) refact ::= RESTRICT */ + 227, /* (59) refact ::= NO ACTION */ + 224, /* (60) defer_subclause ::= NOT DEFERRABLE init_deferred_pred_opt */ + 224, /* (61) defer_subclause ::= DEFERRABLE init_deferred_pred_opt */ + 228, /* (62) init_deferred_pred_opt ::= */ + 228, /* (63) init_deferred_pred_opt ::= INITIALLY DEFERRED */ + 228, /* (64) init_deferred_pred_opt ::= INITIALLY IMMEDIATE */ + 203, /* (65) conslist_opt ::= */ + 230, /* (66) tconscomma ::= COMMA */ + 231, /* (67) tcons ::= CONSTRAINT nm */ + 231, /* (68) tcons ::= PRIMARY KEY LP sortlist autoinc RP onconf */ + 231, /* (69) tcons ::= UNIQUE LP sortlist RP onconf */ + 231, /* (70) tcons ::= CHECK LP expr RP onconf */ + 231, /* (71) tcons ::= FOREIGN KEY LP eidlist RP REFERENCES nm eidlist_opt refargs defer_subclause_opt */ + 234, /* (72) defer_subclause_opt ::= */ + 219, /* (73) onconf ::= */ + 219, /* (74) onconf ::= ON CONFLICT resolvetype */ + 235, /* (75) orconf ::= */ + 235, /* (76) orconf ::= OR resolvetype */ + 236, /* (77) resolvetype ::= IGNORE */ + 236, /* (78) resolvetype ::= REPLACE */ + 191, /* (79) cmd ::= DROP TABLE ifexists fullname */ + 238, /* (80) ifexists ::= IF EXISTS */ + 238, /* (81) ifexists ::= */ + 191, /* (82) cmd ::= createkw temp VIEW ifnotexists nm dbnm eidlist_opt AS select */ + 191, /* (83) cmd ::= DROP VIEW ifexists fullname */ + 191, /* (84) cmd ::= select */ + 205, /* (85) select ::= WITH wqlist selectnowith */ + 205, /* (86) select ::= WITH RECURSIVE wqlist selectnowith */ + 205, /* (87) select ::= selectnowith */ + 240, /* (88) selectnowith ::= selectnowith multiselect_op oneselect */ + 243, /* (89) multiselect_op ::= UNION */ + 243, /* (90) multiselect_op ::= UNION ALL */ + 243, /* (91) multiselect_op ::= EXCEPT|INTERSECT */ + 241, /* (92) oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt orderby_opt limit_opt */ + 241, /* (93) oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt window_clause orderby_opt limit_opt */ + 253, /* (94) values ::= VALUES LP nexprlist RP */ + 241, /* (95) oneselect ::= mvalues */ + 255, /* (96) mvalues ::= values COMMA LP nexprlist RP */ + 255, /* (97) mvalues ::= mvalues COMMA LP nexprlist RP */ + 244, /* (98) distinct ::= DISTINCT */ + 244, /* (99) distinct ::= ALL */ + 244, /* (100) distinct ::= */ + 256, /* (101) sclp ::= */ + 245, /* (102) selcollist ::= sclp scanpt expr scanpt as */ + 245, /* (103) selcollist ::= sclp scanpt STAR */ + 245, /* (104) selcollist ::= sclp scanpt nm DOT STAR */ + 257, /* (105) as ::= AS nm */ + 257, /* (106) as ::= */ + 246, /* (107) from ::= */ + 246, /* (108) from ::= FROM seltablist */ + 259, /* (109) stl_prefix ::= seltablist joinop */ + 259, /* (110) stl_prefix ::= */ + 258, /* (111) seltablist ::= stl_prefix nm dbnm as on_using */ + 258, /* (112) seltablist ::= stl_prefix nm dbnm as indexed_by on_using */ + 258, /* (113) seltablist ::= stl_prefix nm dbnm LP exprlist RP as on_using */ + 258, /* (114) seltablist ::= stl_prefix LP select RP as on_using */ + 258, /* (115) seltablist ::= stl_prefix LP seltablist RP as on_using */ + 201, /* (116) dbnm ::= */ + 201, /* (117) dbnm ::= DOT nm */ + 239, /* (118) fullname ::= nm */ + 239, /* (119) fullname ::= nm DOT nm */ + 264, /* (120) xfullname ::= nm */ + 264, /* (121) xfullname ::= nm DOT nm */ + 264, /* (122) xfullname ::= nm DOT nm AS nm */ + 264, /* (123) xfullname ::= nm AS nm */ + 260, /* (124) joinop ::= COMMA|JOIN */ + 260, /* (125) joinop ::= JOIN_KW JOIN */ + 260, /* (126) joinop ::= JOIN_KW nm JOIN */ + 260, /* (127) joinop ::= JOIN_KW nm nm JOIN */ + 261, /* (128) on_using ::= ON expr */ + 261, /* (129) on_using ::= USING LP idlist RP */ + 261, /* (130) on_using ::= */ + 266, /* (131) indexed_opt ::= */ + 262, /* (132) indexed_by ::= INDEXED BY nm */ + 262, /* (133) indexed_by ::= NOT INDEXED */ + 250, /* (134) orderby_opt ::= */ + 250, /* (135) orderby_opt ::= ORDER BY sortlist */ + 232, /* (136) sortlist ::= sortlist COMMA expr sortorder nulls */ + 232, /* (137) sortlist ::= expr sortorder nulls */ + 220, /* (138) sortorder ::= ASC */ + 220, /* (139) sortorder ::= DESC */ + 220, /* (140) sortorder ::= */ + 267, /* (141) nulls ::= NULLS FIRST */ + 267, /* (142) nulls ::= NULLS LAST */ + 267, /* (143) nulls ::= */ + 248, /* (144) groupby_opt ::= */ + 248, /* (145) groupby_opt ::= GROUP BY nexprlist */ + 249, /* (146) having_opt ::= */ + 249, /* (147) having_opt ::= HAVING expr */ + 251, /* (148) limit_opt ::= */ + 251, /* (149) limit_opt ::= LIMIT expr */ + 251, /* (150) limit_opt ::= LIMIT expr OFFSET expr */ + 251, /* (151) limit_opt ::= LIMIT expr COMMA expr */ + 191, /* (152) cmd ::= with DELETE FROM xfullname indexed_opt where_opt_ret */ + 247, /* (153) where_opt ::= */ + 247, /* (154) where_opt ::= WHERE expr */ + 269, /* (155) where_opt_ret ::= */ + 269, /* (156) where_opt_ret ::= WHERE expr */ + 269, /* (157) where_opt_ret ::= RETURNING selcollist */ + 269, /* (158) where_opt_ret ::= WHERE expr RETURNING selcollist */ + 191, /* (159) cmd ::= with UPDATE orconf xfullname indexed_opt SET setlist from where_opt_ret */ + 270, /* (160) setlist ::= setlist COMMA nm EQ expr */ + 270, /* (161) setlist ::= setlist COMMA LP idlist RP EQ expr */ + 270, /* (162) setlist ::= nm EQ expr */ + 270, /* (163) setlist ::= LP idlist RP EQ expr */ + 191, /* (164) cmd ::= with insert_cmd INTO xfullname idlist_opt select upsert */ + 191, /* (165) cmd ::= with insert_cmd INTO xfullname idlist_opt DEFAULT VALUES returning */ + 273, /* (166) upsert ::= */ + 273, /* (167) upsert ::= RETURNING selcollist */ + 273, /* (168) upsert ::= ON CONFLICT LP sortlist RP where_opt DO UPDATE SET setlist where_opt upsert */ + 273, /* (169) upsert ::= ON CONFLICT LP sortlist RP where_opt DO NOTHING upsert */ + 273, /* (170) upsert ::= ON CONFLICT DO NOTHING returning */ + 273, /* (171) upsert ::= ON CONFLICT DO UPDATE SET setlist where_opt returning */ + 274, /* (172) returning ::= RETURNING selcollist */ + 271, /* (173) insert_cmd ::= INSERT orconf */ + 271, /* (174) insert_cmd ::= REPLACE */ + 272, /* (175) idlist_opt ::= */ + 272, /* (176) idlist_opt ::= LP idlist RP */ + 265, /* (177) idlist ::= idlist COMMA nm */ + 265, /* (178) idlist ::= nm */ + 218, /* (179) expr ::= LP expr RP */ + 218, /* (180) expr ::= ID|INDEXED|JOIN_KW */ + 218, /* (181) expr ::= nm DOT nm */ + 218, /* (182) expr ::= nm DOT nm DOT nm */ + 217, /* (183) term ::= NULL|FLOAT|BLOB */ + 217, /* (184) term ::= STRING */ + 217, /* (185) term ::= INTEGER */ + 218, /* (186) expr ::= VARIABLE */ + 218, /* (187) expr ::= expr COLLATE ID|STRING */ + 218, /* (188) expr ::= CAST LP expr AS typetoken RP */ + 218, /* (189) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP */ + 218, /* (190) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist ORDER BY sortlist RP */ + 218, /* (191) expr ::= ID|INDEXED|JOIN_KW LP STAR RP */ + 218, /* (192) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP filter_over */ + 218, /* (193) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist ORDER BY sortlist RP filter_over */ + 218, /* (194) expr ::= ID|INDEXED|JOIN_KW LP STAR RP filter_over */ + 217, /* (195) term ::= CTIME_KW */ + 218, /* (196) expr ::= LP nexprlist COMMA expr RP */ + 218, /* (197) expr ::= expr AND expr */ + 218, /* (198) expr ::= expr OR expr */ + 218, /* (199) expr ::= expr LT|GT|GE|LE expr */ + 218, /* (200) expr ::= expr EQ|NE expr */ + 218, /* (201) expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr */ + 218, /* (202) expr ::= expr PLUS|MINUS expr */ + 218, /* (203) expr ::= expr STAR|SLASH|REM expr */ + 218, /* (204) expr ::= expr CONCAT expr */ + 276, /* (205) likeop ::= NOT LIKE_KW|MATCH */ + 218, /* (206) expr ::= expr likeop expr */ + 218, /* (207) expr ::= expr likeop expr ESCAPE expr */ + 218, /* (208) expr ::= expr ISNULL|NOTNULL */ + 218, /* (209) expr ::= expr NOT NULL */ + 218, /* (210) expr ::= expr IS expr */ + 218, /* (211) expr ::= expr IS NOT expr */ + 218, /* (212) expr ::= expr IS NOT DISTINCT FROM expr */ + 218, /* (213) expr ::= expr IS DISTINCT FROM expr */ + 218, /* (214) expr ::= NOT expr */ + 218, /* (215) expr ::= BITNOT expr */ + 218, /* (216) expr ::= PLUS|MINUS expr */ + 218, /* (217) expr ::= expr PTR expr */ + 277, /* (218) between_op ::= BETWEEN */ + 277, /* (219) between_op ::= NOT BETWEEN */ + 218, /* (220) expr ::= expr between_op expr AND expr */ + 278, /* (221) in_op ::= IN */ + 278, /* (222) in_op ::= NOT IN */ + 218, /* (223) expr ::= expr in_op LP exprlist RP */ + 218, /* (224) expr ::= LP select RP */ + 218, /* (225) expr ::= expr in_op LP select RP */ + 218, /* (226) expr ::= expr in_op nm dbnm paren_exprlist */ + 218, /* (227) expr ::= EXISTS LP select RP */ + 218, /* (228) expr ::= CASE case_operand case_exprlist case_else END */ + 281, /* (229) case_exprlist ::= case_exprlist WHEN expr THEN expr */ + 281, /* (230) case_exprlist ::= WHEN expr THEN expr */ + 282, /* (231) case_else ::= ELSE expr */ + 282, /* (232) case_else ::= */ + 280, /* (233) case_operand ::= */ + 263, /* (234) exprlist ::= */ + 254, /* (235) nexprlist ::= nexprlist COMMA expr */ + 254, /* (236) nexprlist ::= expr */ + 279, /* (237) paren_exprlist ::= */ + 279, /* (238) paren_exprlist ::= LP exprlist RP */ + 191, /* (239) cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt */ + 283, /* (240) uniqueflag ::= UNIQUE */ + 283, /* (241) uniqueflag ::= */ + 222, /* (242) eidlist_opt ::= */ + 222, /* (243) eidlist_opt ::= LP eidlist RP */ + 233, /* (244) eidlist ::= eidlist COMMA nm collate sortorder */ + 233, /* (245) eidlist ::= nm collate sortorder */ + 284, /* (246) collate ::= */ + 284, /* (247) collate ::= COLLATE ID|STRING */ + 191, /* (248) cmd ::= DROP INDEX ifexists fullname */ + 191, /* (249) cmd ::= VACUUM vinto */ + 191, /* (250) cmd ::= VACUUM nm vinto */ + 285, /* (251) vinto ::= INTO expr */ + 285, /* (252) vinto ::= */ + 191, /* (253) cmd ::= PRAGMA nm dbnm */ + 191, /* (254) cmd ::= PRAGMA nm dbnm EQ nmnum */ + 191, /* (255) cmd ::= PRAGMA nm dbnm LP nmnum RP */ + 191, /* (256) cmd ::= PRAGMA nm dbnm EQ minus_num */ + 191, /* (257) cmd ::= PRAGMA nm dbnm LP minus_num RP */ + 212, /* (258) plus_num ::= PLUS INTEGER|FLOAT */ + 213, /* (259) minus_num ::= MINUS INTEGER|FLOAT */ + 191, /* (260) cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END */ + 287, /* (261) trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause */ + 289, /* (262) trigger_time ::= BEFORE|AFTER */ + 289, /* (263) trigger_time ::= INSTEAD OF */ + 289, /* (264) trigger_time ::= */ + 290, /* (265) trigger_event ::= DELETE|INSERT */ + 290, /* (266) trigger_event ::= UPDATE */ + 290, /* (267) trigger_event ::= UPDATE OF idlist */ + 292, /* (268) when_clause ::= */ + 292, /* (269) when_clause ::= WHEN expr */ + 288, /* (270) trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */ + 288, /* (271) trigger_cmd_list ::= trigger_cmd SEMI */ + 294, /* (272) trnm ::= nm DOT nm */ + 295, /* (273) tridxby ::= INDEXED BY nm */ + 295, /* (274) tridxby ::= NOT INDEXED */ + 293, /* (275) trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist from where_opt scanpt */ + 293, /* (276) trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt */ + 293, /* (277) trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt */ + 293, /* (278) trigger_cmd ::= scanpt select scanpt */ + 218, /* (279) expr ::= RAISE LP IGNORE RP */ + 218, /* (280) expr ::= RAISE LP raisetype COMMA nm RP */ + 237, /* (281) raisetype ::= ROLLBACK */ + 237, /* (282) raisetype ::= ABORT */ + 237, /* (283) raisetype ::= FAIL */ + 191, /* (284) cmd ::= DROP TRIGGER ifexists fullname */ + 191, /* (285) cmd ::= ATTACH database_kw_opt expr AS expr key_opt */ + 191, /* (286) cmd ::= DETACH database_kw_opt expr */ + 297, /* (287) key_opt ::= */ + 297, /* (288) key_opt ::= KEY expr */ + 191, /* (289) cmd ::= REINDEX */ + 191, /* (290) cmd ::= REINDEX nm dbnm */ + 191, /* (291) cmd ::= ANALYZE */ + 191, /* (292) cmd ::= ANALYZE nm dbnm */ + 191, /* (293) cmd ::= ALTER TABLE fullname RENAME TO nm */ + 191, /* (294) cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist */ + 191, /* (295) cmd ::= ALTER TABLE fullname DROP kwcolumn_opt nm */ + 298, /* (296) add_column_fullname ::= fullname */ + 191, /* (297) cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm */ + 191, /* (298) cmd ::= create_vtab */ + 191, /* (299) cmd ::= create_vtab LP vtabarglist RP */ + 300, /* (300) create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm */ + 302, /* (301) vtabarg ::= */ + 303, /* (302) vtabargtoken ::= ANY */ + 303, /* (303) vtabargtoken ::= lp anylist RP */ + 304, /* (304) lp ::= LP */ + 268, /* (305) with ::= WITH wqlist */ + 268, /* (306) with ::= WITH RECURSIVE wqlist */ + 307, /* (307) wqas ::= AS */ + 307, /* (308) wqas ::= AS MATERIALIZED */ + 307, /* (309) wqas ::= AS NOT MATERIALIZED */ + 306, /* (310) wqitem ::= withnm eidlist_opt wqas LP select RP */ + 308, /* (311) withnm ::= nm */ + 242, /* (312) wqlist ::= wqitem */ + 242, /* (313) wqlist ::= wqlist COMMA wqitem */ + 309, /* (314) windowdefn_list ::= windowdefn_list COMMA windowdefn */ + 310, /* (315) windowdefn ::= nm AS LP window RP */ + 311, /* (316) window ::= PARTITION BY nexprlist orderby_opt frame_opt */ + 311, /* (317) window ::= nm PARTITION BY nexprlist orderby_opt frame_opt */ + 311, /* (318) window ::= ORDER BY sortlist frame_opt */ + 311, /* (319) window ::= nm ORDER BY sortlist frame_opt */ + 311, /* (320) window ::= nm frame_opt */ + 312, /* (321) frame_opt ::= */ + 312, /* (322) frame_opt ::= range_or_rows frame_bound_s frame_exclude_opt */ + 312, /* (323) frame_opt ::= range_or_rows BETWEEN frame_bound_s AND frame_bound_e frame_exclude_opt */ + 316, /* (324) range_or_rows ::= RANGE|ROWS|GROUPS */ + 318, /* (325) frame_bound_s ::= frame_bound */ + 318, /* (326) frame_bound_s ::= UNBOUNDED PRECEDING */ + 319, /* (327) frame_bound_e ::= frame_bound */ + 319, /* (328) frame_bound_e ::= UNBOUNDED FOLLOWING */ + 317, /* (329) frame_bound ::= expr PRECEDING|FOLLOWING */ + 317, /* (330) frame_bound ::= CURRENT ROW */ + 320, /* (331) frame_exclude_opt ::= */ + 320, /* (332) frame_exclude_opt ::= EXCLUDE frame_exclude */ + 321, /* (333) frame_exclude ::= NO OTHERS */ + 321, /* (334) frame_exclude ::= CURRENT ROW */ + 321, /* (335) frame_exclude ::= GROUP|TIES */ + 252, /* (336) window_clause ::= WINDOW windowdefn_list */ + 275, /* (337) filter_over ::= filter_clause over_clause */ + 275, /* (338) filter_over ::= over_clause */ + 275, /* (339) filter_over ::= filter_clause */ + 315, /* (340) over_clause ::= OVER LP window RP */ + 315, /* (341) over_clause ::= OVER nm */ + 314, /* (342) filter_clause ::= FILTER LP WHERE expr RP */ + 217, /* (343) term ::= QNUMBER */ + 186, /* (344) input ::= cmdlist */ + 187, /* (345) cmdlist ::= cmdlist ecmd */ + 187, /* (346) cmdlist ::= ecmd */ + 188, /* (347) ecmd ::= SEMI */ + 188, /* (348) ecmd ::= cmdx SEMI */ + 188, /* (349) ecmd ::= explain cmdx SEMI */ + 193, /* (350) trans_opt ::= */ + 193, /* (351) trans_opt ::= TRANSACTION */ + 193, /* (352) trans_opt ::= TRANSACTION nm */ + 195, /* (353) savepoint_opt ::= SAVEPOINT */ + 195, /* (354) savepoint_opt ::= */ + 191, /* (355) cmd ::= create_table create_table_args */ + 204, /* (356) table_option_set ::= table_option */ + 202, /* (357) columnlist ::= columnlist COMMA columnname carglist */ + 202, /* (358) columnlist ::= columnname carglist */ + 194, /* (359) nm ::= ID|INDEXED|JOIN_KW */ + 194, /* (360) nm ::= STRING */ + 209, /* (361) typetoken ::= typename */ + 210, /* (362) typename ::= ID|STRING */ + 211, /* (363) signed ::= plus_num */ + 211, /* (364) signed ::= minus_num */ + 208, /* (365) carglist ::= carglist ccons */ + 208, /* (366) carglist ::= */ + 216, /* (367) ccons ::= NULL onconf */ + 216, /* (368) ccons ::= GENERATED ALWAYS AS generated */ + 216, /* (369) ccons ::= AS generated */ + 203, /* (370) conslist_opt ::= COMMA conslist */ + 229, /* (371) conslist ::= conslist tconscomma tcons */ + 229, /* (372) conslist ::= tcons */ + 230, /* (373) tconscomma ::= */ + 234, /* (374) defer_subclause_opt ::= defer_subclause */ + 236, /* (375) resolvetype ::= raisetype */ + 240, /* (376) selectnowith ::= oneselect */ + 241, /* (377) oneselect ::= values */ + 256, /* (378) sclp ::= selcollist COMMA */ + 257, /* (379) as ::= ID|STRING */ + 266, /* (380) indexed_opt ::= indexed_by */ + 274, /* (381) returning ::= */ + 218, /* (382) expr ::= term */ + 276, /* (383) likeop ::= LIKE_KW|MATCH */ + 280, /* (384) case_operand ::= expr */ + 263, /* (385) exprlist ::= nexprlist */ + 286, /* (386) nmnum ::= plus_num */ + 286, /* (387) nmnum ::= nm */ + 286, /* (388) nmnum ::= ON */ + 286, /* (389) nmnum ::= DELETE */ + 286, /* (390) nmnum ::= DEFAULT */ + 212, /* (391) plus_num ::= INTEGER|FLOAT */ + 291, /* (392) foreach_clause ::= */ + 291, /* (393) foreach_clause ::= FOR EACH ROW */ + 294, /* (394) trnm ::= nm */ + 295, /* (395) tridxby ::= */ + 296, /* (396) database_kw_opt ::= DATABASE */ + 296, /* (397) database_kw_opt ::= */ + 299, /* (398) kwcolumn_opt ::= */ + 299, /* (399) kwcolumn_opt ::= COLUMNKW */ + 301, /* (400) vtabarglist ::= vtabarg */ + 301, /* (401) vtabarglist ::= vtabarglist COMMA vtabarg */ + 302, /* (402) vtabarg ::= vtabarg vtabargtoken */ + 305, /* (403) anylist ::= */ + 305, /* (404) anylist ::= anylist LP anylist RP */ + 305, /* (405) anylist ::= anylist ANY */ + 268, /* (406) with ::= */ + 309, /* (407) windowdefn_list ::= windowdefn */ + 311, /* (408) window ::= frame_opt */ }; /* For rule J, yyRuleInfoNRhs[J] contains the negative of the number @@ -174363,316 +175761,320 @@ static const signed char yyRuleInfoNRhs[] = { -9, /* (92) oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt orderby_opt limit_opt */ -10, /* (93) oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt window_clause orderby_opt limit_opt */ -4, /* (94) values ::= VALUES LP nexprlist RP */ - -5, /* (95) values ::= values COMMA LP nexprlist RP */ - -1, /* (96) distinct ::= DISTINCT */ - -1, /* (97) distinct ::= ALL */ - 0, /* (98) distinct ::= */ - 0, /* (99) sclp ::= */ - -5, /* (100) selcollist ::= sclp scanpt expr scanpt as */ - -3, /* (101) selcollist ::= sclp scanpt STAR */ - -5, /* (102) selcollist ::= sclp scanpt nm DOT STAR */ - -2, /* (103) as ::= AS nm */ - 0, /* (104) as ::= */ - 0, /* (105) from ::= */ - -2, /* (106) from ::= FROM seltablist */ - -2, /* (107) stl_prefix ::= seltablist joinop */ - 0, /* (108) stl_prefix ::= */ - -5, /* (109) seltablist ::= stl_prefix nm dbnm as on_using */ - -6, /* (110) seltablist ::= stl_prefix nm dbnm as indexed_by on_using */ - -8, /* (111) seltablist ::= stl_prefix nm dbnm LP exprlist RP as on_using */ - -6, /* (112) seltablist ::= stl_prefix LP select RP as on_using */ - -6, /* (113) seltablist ::= stl_prefix LP seltablist RP as on_using */ - 0, /* (114) dbnm ::= */ - -2, /* (115) dbnm ::= DOT nm */ - -1, /* (116) fullname ::= nm */ - -3, /* (117) fullname ::= nm DOT nm */ - -1, /* (118) xfullname ::= nm */ - -3, /* (119) xfullname ::= nm DOT nm */ - -5, /* (120) xfullname ::= nm DOT nm AS nm */ - -3, /* (121) xfullname ::= nm AS nm */ - -1, /* (122) joinop ::= COMMA|JOIN */ - -2, /* (123) joinop ::= JOIN_KW JOIN */ - -3, /* (124) joinop ::= JOIN_KW nm JOIN */ - -4, /* (125) joinop ::= JOIN_KW nm nm JOIN */ - -2, /* (126) on_using ::= ON expr */ - -4, /* (127) on_using ::= USING LP idlist RP */ - 0, /* (128) on_using ::= */ - 0, /* (129) indexed_opt ::= */ - -3, /* (130) indexed_by ::= INDEXED BY nm */ - -2, /* (131) indexed_by ::= NOT INDEXED */ - 0, /* (132) orderby_opt ::= */ - -3, /* (133) orderby_opt ::= ORDER BY sortlist */ - -5, /* (134) sortlist ::= sortlist COMMA expr sortorder nulls */ - -3, /* (135) sortlist ::= expr sortorder nulls */ - -1, /* (136) sortorder ::= ASC */ - -1, /* (137) sortorder ::= DESC */ - 0, /* (138) sortorder ::= */ - -2, /* (139) nulls ::= NULLS FIRST */ - -2, /* (140) nulls ::= NULLS LAST */ - 0, /* (141) nulls ::= */ - 0, /* (142) groupby_opt ::= */ - -3, /* (143) groupby_opt ::= GROUP BY nexprlist */ - 0, /* (144) having_opt ::= */ - -2, /* (145) having_opt ::= HAVING expr */ - 0, /* (146) limit_opt ::= */ - -2, /* (147) limit_opt ::= LIMIT expr */ - -4, /* (148) limit_opt ::= LIMIT expr OFFSET expr */ - -4, /* (149) limit_opt ::= LIMIT expr COMMA expr */ - -6, /* (150) cmd ::= with DELETE FROM xfullname indexed_opt where_opt_ret */ - 0, /* (151) where_opt ::= */ - -2, /* (152) where_opt ::= WHERE expr */ - 0, /* (153) where_opt_ret ::= */ - -2, /* (154) where_opt_ret ::= WHERE expr */ - -2, /* (155) where_opt_ret ::= RETURNING selcollist */ - -4, /* (156) where_opt_ret ::= WHERE expr RETURNING selcollist */ - -9, /* (157) cmd ::= with UPDATE orconf xfullname indexed_opt SET setlist from where_opt_ret */ - -5, /* (158) setlist ::= setlist COMMA nm EQ expr */ - -7, /* (159) setlist ::= setlist COMMA LP idlist RP EQ expr */ - -3, /* (160) setlist ::= nm EQ expr */ - -5, /* (161) setlist ::= LP idlist RP EQ expr */ - -7, /* (162) cmd ::= with insert_cmd INTO xfullname idlist_opt select upsert */ - -8, /* (163) cmd ::= with insert_cmd INTO xfullname idlist_opt DEFAULT VALUES returning */ - 0, /* (164) upsert ::= */ - -2, /* (165) upsert ::= RETURNING selcollist */ - -12, /* (166) upsert ::= ON CONFLICT LP sortlist RP where_opt DO UPDATE SET setlist where_opt upsert */ - -9, /* (167) upsert ::= ON CONFLICT LP sortlist RP where_opt DO NOTHING upsert */ - -5, /* (168) upsert ::= ON CONFLICT DO NOTHING returning */ - -8, /* (169) upsert ::= ON CONFLICT DO UPDATE SET setlist where_opt returning */ - -2, /* (170) returning ::= RETURNING selcollist */ - -2, /* (171) insert_cmd ::= INSERT orconf */ - -1, /* (172) insert_cmd ::= REPLACE */ - 0, /* (173) idlist_opt ::= */ - -3, /* (174) idlist_opt ::= LP idlist RP */ - -3, /* (175) idlist ::= idlist COMMA nm */ - -1, /* (176) idlist ::= nm */ - -3, /* (177) expr ::= LP expr RP */ - -1, /* (178) expr ::= ID|INDEXED|JOIN_KW */ - -3, /* (179) expr ::= nm DOT nm */ - -5, /* (180) expr ::= nm DOT nm DOT nm */ - -1, /* (181) term ::= NULL|FLOAT|BLOB */ - -1, /* (182) term ::= STRING */ - -1, /* (183) term ::= INTEGER */ - -1, /* (184) expr ::= VARIABLE */ - -3, /* (185) expr ::= expr COLLATE ID|STRING */ - -6, /* (186) expr ::= CAST LP expr AS typetoken RP */ - -5, /* (187) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP */ - -8, /* (188) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist ORDER BY sortlist RP */ - -4, /* (189) expr ::= ID|INDEXED|JOIN_KW LP STAR RP */ - -6, /* (190) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP filter_over */ - -9, /* (191) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist ORDER BY sortlist RP filter_over */ - -5, /* (192) expr ::= ID|INDEXED|JOIN_KW LP STAR RP filter_over */ - -1, /* (193) term ::= CTIME_KW */ - -5, /* (194) expr ::= LP nexprlist COMMA expr RP */ - -3, /* (195) expr ::= expr AND expr */ - -3, /* (196) expr ::= expr OR expr */ - -3, /* (197) expr ::= expr LT|GT|GE|LE expr */ - -3, /* (198) expr ::= expr EQ|NE expr */ - -3, /* (199) expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr */ - -3, /* (200) expr ::= expr PLUS|MINUS expr */ - -3, /* (201) expr ::= expr STAR|SLASH|REM expr */ - -3, /* (202) expr ::= expr CONCAT expr */ - -2, /* (203) likeop ::= NOT LIKE_KW|MATCH */ - -3, /* (204) expr ::= expr likeop expr */ - -5, /* (205) expr ::= expr likeop expr ESCAPE expr */ - -2, /* (206) expr ::= expr ISNULL|NOTNULL */ - -3, /* (207) expr ::= expr NOT NULL */ - -3, /* (208) expr ::= expr IS expr */ - -4, /* (209) expr ::= expr IS NOT expr */ - -6, /* (210) expr ::= expr IS NOT DISTINCT FROM expr */ - -5, /* (211) expr ::= expr IS DISTINCT FROM expr */ - -2, /* (212) expr ::= NOT expr */ - -2, /* (213) expr ::= BITNOT expr */ - -2, /* (214) expr ::= PLUS|MINUS expr */ - -3, /* (215) expr ::= expr PTR expr */ - -1, /* (216) between_op ::= BETWEEN */ - -2, /* (217) between_op ::= NOT BETWEEN */ - -5, /* (218) expr ::= expr between_op expr AND expr */ - -1, /* (219) in_op ::= IN */ - -2, /* (220) in_op ::= NOT IN */ - -5, /* (221) expr ::= expr in_op LP exprlist RP */ - -3, /* (222) expr ::= LP select RP */ - -5, /* (223) expr ::= expr in_op LP select RP */ - -5, /* (224) expr ::= expr in_op nm dbnm paren_exprlist */ - -4, /* (225) expr ::= EXISTS LP select RP */ - -5, /* (226) expr ::= CASE case_operand case_exprlist case_else END */ - -5, /* (227) case_exprlist ::= case_exprlist WHEN expr THEN expr */ - -4, /* (228) case_exprlist ::= WHEN expr THEN expr */ - -2, /* (229) case_else ::= ELSE expr */ - 0, /* (230) case_else ::= */ - 0, /* (231) case_operand ::= */ - 0, /* (232) exprlist ::= */ - -3, /* (233) nexprlist ::= nexprlist COMMA expr */ - -1, /* (234) nexprlist ::= expr */ - 0, /* (235) paren_exprlist ::= */ - -3, /* (236) paren_exprlist ::= LP exprlist RP */ - -12, /* (237) cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt */ - -1, /* (238) uniqueflag ::= UNIQUE */ - 0, /* (239) uniqueflag ::= */ - 0, /* (240) eidlist_opt ::= */ - -3, /* (241) eidlist_opt ::= LP eidlist RP */ - -5, /* (242) eidlist ::= eidlist COMMA nm collate sortorder */ - -3, /* (243) eidlist ::= nm collate sortorder */ - 0, /* (244) collate ::= */ - -2, /* (245) collate ::= COLLATE ID|STRING */ - -4, /* (246) cmd ::= DROP INDEX ifexists fullname */ - -2, /* (247) cmd ::= VACUUM vinto */ - -3, /* (248) cmd ::= VACUUM nm vinto */ - -2, /* (249) vinto ::= INTO expr */ - 0, /* (250) vinto ::= */ - -3, /* (251) cmd ::= PRAGMA nm dbnm */ - -5, /* (252) cmd ::= PRAGMA nm dbnm EQ nmnum */ - -6, /* (253) cmd ::= PRAGMA nm dbnm LP nmnum RP */ - -5, /* (254) cmd ::= PRAGMA nm dbnm EQ minus_num */ - -6, /* (255) cmd ::= PRAGMA nm dbnm LP minus_num RP */ - -2, /* (256) plus_num ::= PLUS INTEGER|FLOAT */ - -2, /* (257) minus_num ::= MINUS INTEGER|FLOAT */ - -5, /* (258) cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END */ - -11, /* (259) trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause */ - -1, /* (260) trigger_time ::= BEFORE|AFTER */ - -2, /* (261) trigger_time ::= INSTEAD OF */ - 0, /* (262) trigger_time ::= */ - -1, /* (263) trigger_event ::= DELETE|INSERT */ - -1, /* (264) trigger_event ::= UPDATE */ - -3, /* (265) trigger_event ::= UPDATE OF idlist */ - 0, /* (266) when_clause ::= */ - -2, /* (267) when_clause ::= WHEN expr */ - -3, /* (268) trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */ - -2, /* (269) trigger_cmd_list ::= trigger_cmd SEMI */ - -3, /* (270) trnm ::= nm DOT nm */ - -3, /* (271) tridxby ::= INDEXED BY nm */ - -2, /* (272) tridxby ::= NOT INDEXED */ - -9, /* (273) trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist from where_opt scanpt */ - -8, /* (274) trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt */ - -6, /* (275) trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt */ - -3, /* (276) trigger_cmd ::= scanpt select scanpt */ - -4, /* (277) expr ::= RAISE LP IGNORE RP */ - -6, /* (278) expr ::= RAISE LP raisetype COMMA nm RP */ - -1, /* (279) raisetype ::= ROLLBACK */ - -1, /* (280) raisetype ::= ABORT */ - -1, /* (281) raisetype ::= FAIL */ - -4, /* (282) cmd ::= DROP TRIGGER ifexists fullname */ - -6, /* (283) cmd ::= ATTACH database_kw_opt expr AS expr key_opt */ - -3, /* (284) cmd ::= DETACH database_kw_opt expr */ - 0, /* (285) key_opt ::= */ - -2, /* (286) key_opt ::= KEY expr */ - -1, /* (287) cmd ::= REINDEX */ - -3, /* (288) cmd ::= REINDEX nm dbnm */ - -1, /* (289) cmd ::= ANALYZE */ - -3, /* (290) cmd ::= ANALYZE nm dbnm */ - -6, /* (291) cmd ::= ALTER TABLE fullname RENAME TO nm */ - -7, /* (292) cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist */ - -6, /* (293) cmd ::= ALTER TABLE fullname DROP kwcolumn_opt nm */ - -1, /* (294) add_column_fullname ::= fullname */ - -8, /* (295) cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm */ - -1, /* (296) cmd ::= create_vtab */ - -4, /* (297) cmd ::= create_vtab LP vtabarglist RP */ - -8, /* (298) create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm */ - 0, /* (299) vtabarg ::= */ - -1, /* (300) vtabargtoken ::= ANY */ - -3, /* (301) vtabargtoken ::= lp anylist RP */ - -1, /* (302) lp ::= LP */ - -2, /* (303) with ::= WITH wqlist */ - -3, /* (304) with ::= WITH RECURSIVE wqlist */ - -1, /* (305) wqas ::= AS */ - -2, /* (306) wqas ::= AS MATERIALIZED */ - -3, /* (307) wqas ::= AS NOT MATERIALIZED */ - -6, /* (308) wqitem ::= nm eidlist_opt wqas LP select RP */ - -1, /* (309) wqlist ::= wqitem */ - -3, /* (310) wqlist ::= wqlist COMMA wqitem */ - -3, /* (311) windowdefn_list ::= windowdefn_list COMMA windowdefn */ - -5, /* (312) windowdefn ::= nm AS LP window RP */ - -5, /* (313) window ::= PARTITION BY nexprlist orderby_opt frame_opt */ - -6, /* (314) window ::= nm PARTITION BY nexprlist orderby_opt frame_opt */ - -4, /* (315) window ::= ORDER BY sortlist frame_opt */ - -5, /* (316) window ::= nm ORDER BY sortlist frame_opt */ - -2, /* (317) window ::= nm frame_opt */ - 0, /* (318) frame_opt ::= */ - -3, /* (319) frame_opt ::= range_or_rows frame_bound_s frame_exclude_opt */ - -6, /* (320) frame_opt ::= range_or_rows BETWEEN frame_bound_s AND frame_bound_e frame_exclude_opt */ - -1, /* (321) range_or_rows ::= RANGE|ROWS|GROUPS */ - -1, /* (322) frame_bound_s ::= frame_bound */ - -2, /* (323) frame_bound_s ::= UNBOUNDED PRECEDING */ - -1, /* (324) frame_bound_e ::= frame_bound */ - -2, /* (325) frame_bound_e ::= UNBOUNDED FOLLOWING */ - -2, /* (326) frame_bound ::= expr PRECEDING|FOLLOWING */ - -2, /* (327) frame_bound ::= CURRENT ROW */ - 0, /* (328) frame_exclude_opt ::= */ - -2, /* (329) frame_exclude_opt ::= EXCLUDE frame_exclude */ - -2, /* (330) frame_exclude ::= NO OTHERS */ - -2, /* (331) frame_exclude ::= CURRENT ROW */ - -1, /* (332) frame_exclude ::= GROUP|TIES */ - -2, /* (333) window_clause ::= WINDOW windowdefn_list */ - -2, /* (334) filter_over ::= filter_clause over_clause */ - -1, /* (335) filter_over ::= over_clause */ - -1, /* (336) filter_over ::= filter_clause */ - -4, /* (337) over_clause ::= OVER LP window RP */ - -2, /* (338) over_clause ::= OVER nm */ - -5, /* (339) filter_clause ::= FILTER LP WHERE expr RP */ - -1, /* (340) input ::= cmdlist */ - -2, /* (341) cmdlist ::= cmdlist ecmd */ - -1, /* (342) cmdlist ::= ecmd */ - -1, /* (343) ecmd ::= SEMI */ - -2, /* (344) ecmd ::= cmdx SEMI */ - -3, /* (345) ecmd ::= explain cmdx SEMI */ - 0, /* (346) trans_opt ::= */ - -1, /* (347) trans_opt ::= TRANSACTION */ - -2, /* (348) trans_opt ::= TRANSACTION nm */ - -1, /* (349) savepoint_opt ::= SAVEPOINT */ - 0, /* (350) savepoint_opt ::= */ - -2, /* (351) cmd ::= create_table create_table_args */ - -1, /* (352) table_option_set ::= table_option */ - -4, /* (353) columnlist ::= columnlist COMMA columnname carglist */ - -2, /* (354) columnlist ::= columnname carglist */ - -1, /* (355) nm ::= ID|INDEXED|JOIN_KW */ - -1, /* (356) nm ::= STRING */ - -1, /* (357) typetoken ::= typename */ - -1, /* (358) typename ::= ID|STRING */ - -1, /* (359) signed ::= plus_num */ - -1, /* (360) signed ::= minus_num */ - -2, /* (361) carglist ::= carglist ccons */ - 0, /* (362) carglist ::= */ - -2, /* (363) ccons ::= NULL onconf */ - -4, /* (364) ccons ::= GENERATED ALWAYS AS generated */ - -2, /* (365) ccons ::= AS generated */ - -2, /* (366) conslist_opt ::= COMMA conslist */ - -3, /* (367) conslist ::= conslist tconscomma tcons */ - -1, /* (368) conslist ::= tcons */ - 0, /* (369) tconscomma ::= */ - -1, /* (370) defer_subclause_opt ::= defer_subclause */ - -1, /* (371) resolvetype ::= raisetype */ - -1, /* (372) selectnowith ::= oneselect */ - -1, /* (373) oneselect ::= values */ - -2, /* (374) sclp ::= selcollist COMMA */ - -1, /* (375) as ::= ID|STRING */ - -1, /* (376) indexed_opt ::= indexed_by */ - 0, /* (377) returning ::= */ - -1, /* (378) expr ::= term */ - -1, /* (379) likeop ::= LIKE_KW|MATCH */ - -1, /* (380) case_operand ::= expr */ - -1, /* (381) exprlist ::= nexprlist */ - -1, /* (382) nmnum ::= plus_num */ - -1, /* (383) nmnum ::= nm */ - -1, /* (384) nmnum ::= ON */ - -1, /* (385) nmnum ::= DELETE */ - -1, /* (386) nmnum ::= DEFAULT */ - -1, /* (387) plus_num ::= INTEGER|FLOAT */ - 0, /* (388) foreach_clause ::= */ - -3, /* (389) foreach_clause ::= FOR EACH ROW */ - -1, /* (390) trnm ::= nm */ - 0, /* (391) tridxby ::= */ - -1, /* (392) database_kw_opt ::= DATABASE */ - 0, /* (393) database_kw_opt ::= */ - 0, /* (394) kwcolumn_opt ::= */ - -1, /* (395) kwcolumn_opt ::= COLUMNKW */ - -1, /* (396) vtabarglist ::= vtabarg */ - -3, /* (397) vtabarglist ::= vtabarglist COMMA vtabarg */ - -2, /* (398) vtabarg ::= vtabarg vtabargtoken */ - 0, /* (399) anylist ::= */ - -4, /* (400) anylist ::= anylist LP anylist RP */ - -2, /* (401) anylist ::= anylist ANY */ - 0, /* (402) with ::= */ - -1, /* (403) windowdefn_list ::= windowdefn */ - -1, /* (404) window ::= frame_opt */ + -1, /* (95) oneselect ::= mvalues */ + -5, /* (96) mvalues ::= values COMMA LP nexprlist RP */ + -5, /* (97) mvalues ::= mvalues COMMA LP nexprlist RP */ + -1, /* (98) distinct ::= DISTINCT */ + -1, /* (99) distinct ::= ALL */ + 0, /* (100) distinct ::= */ + 0, /* (101) sclp ::= */ + -5, /* (102) selcollist ::= sclp scanpt expr scanpt as */ + -3, /* (103) selcollist ::= sclp scanpt STAR */ + -5, /* (104) selcollist ::= sclp scanpt nm DOT STAR */ + -2, /* (105) as ::= AS nm */ + 0, /* (106) as ::= */ + 0, /* (107) from ::= */ + -2, /* (108) from ::= FROM seltablist */ + -2, /* (109) stl_prefix ::= seltablist joinop */ + 0, /* (110) stl_prefix ::= */ + -5, /* (111) seltablist ::= stl_prefix nm dbnm as on_using */ + -6, /* (112) seltablist ::= stl_prefix nm dbnm as indexed_by on_using */ + -8, /* (113) seltablist ::= stl_prefix nm dbnm LP exprlist RP as on_using */ + -6, /* (114) seltablist ::= stl_prefix LP select RP as on_using */ + -6, /* (115) seltablist ::= stl_prefix LP seltablist RP as on_using */ + 0, /* (116) dbnm ::= */ + -2, /* (117) dbnm ::= DOT nm */ + -1, /* (118) fullname ::= nm */ + -3, /* (119) fullname ::= nm DOT nm */ + -1, /* (120) xfullname ::= nm */ + -3, /* (121) xfullname ::= nm DOT nm */ + -5, /* (122) xfullname ::= nm DOT nm AS nm */ + -3, /* (123) xfullname ::= nm AS nm */ + -1, /* (124) joinop ::= COMMA|JOIN */ + -2, /* (125) joinop ::= JOIN_KW JOIN */ + -3, /* (126) joinop ::= JOIN_KW nm JOIN */ + -4, /* (127) joinop ::= JOIN_KW nm nm JOIN */ + -2, /* (128) on_using ::= ON expr */ + -4, /* (129) on_using ::= USING LP idlist RP */ + 0, /* (130) on_using ::= */ + 0, /* (131) indexed_opt ::= */ + -3, /* (132) indexed_by ::= INDEXED BY nm */ + -2, /* (133) indexed_by ::= NOT INDEXED */ + 0, /* (134) orderby_opt ::= */ + -3, /* (135) orderby_opt ::= ORDER BY sortlist */ + -5, /* (136) sortlist ::= sortlist COMMA expr sortorder nulls */ + -3, /* (137) sortlist ::= expr sortorder nulls */ + -1, /* (138) sortorder ::= ASC */ + -1, /* (139) sortorder ::= DESC */ + 0, /* (140) sortorder ::= */ + -2, /* (141) nulls ::= NULLS FIRST */ + -2, /* (142) nulls ::= NULLS LAST */ + 0, /* (143) nulls ::= */ + 0, /* (144) groupby_opt ::= */ + -3, /* (145) groupby_opt ::= GROUP BY nexprlist */ + 0, /* (146) having_opt ::= */ + -2, /* (147) having_opt ::= HAVING expr */ + 0, /* (148) limit_opt ::= */ + -2, /* (149) limit_opt ::= LIMIT expr */ + -4, /* (150) limit_opt ::= LIMIT expr OFFSET expr */ + -4, /* (151) limit_opt ::= LIMIT expr COMMA expr */ + -6, /* (152) cmd ::= with DELETE FROM xfullname indexed_opt where_opt_ret */ + 0, /* (153) where_opt ::= */ + -2, /* (154) where_opt ::= WHERE expr */ + 0, /* (155) where_opt_ret ::= */ + -2, /* (156) where_opt_ret ::= WHERE expr */ + -2, /* (157) where_opt_ret ::= RETURNING selcollist */ + -4, /* (158) where_opt_ret ::= WHERE expr RETURNING selcollist */ + -9, /* (159) cmd ::= with UPDATE orconf xfullname indexed_opt SET setlist from where_opt_ret */ + -5, /* (160) setlist ::= setlist COMMA nm EQ expr */ + -7, /* (161) setlist ::= setlist COMMA LP idlist RP EQ expr */ + -3, /* (162) setlist ::= nm EQ expr */ + -5, /* (163) setlist ::= LP idlist RP EQ expr */ + -7, /* (164) cmd ::= with insert_cmd INTO xfullname idlist_opt select upsert */ + -8, /* (165) cmd ::= with insert_cmd INTO xfullname idlist_opt DEFAULT VALUES returning */ + 0, /* (166) upsert ::= */ + -2, /* (167) upsert ::= RETURNING selcollist */ + -12, /* (168) upsert ::= ON CONFLICT LP sortlist RP where_opt DO UPDATE SET setlist where_opt upsert */ + -9, /* (169) upsert ::= ON CONFLICT LP sortlist RP where_opt DO NOTHING upsert */ + -5, /* (170) upsert ::= ON CONFLICT DO NOTHING returning */ + -8, /* (171) upsert ::= ON CONFLICT DO UPDATE SET setlist where_opt returning */ + -2, /* (172) returning ::= RETURNING selcollist */ + -2, /* (173) insert_cmd ::= INSERT orconf */ + -1, /* (174) insert_cmd ::= REPLACE */ + 0, /* (175) idlist_opt ::= */ + -3, /* (176) idlist_opt ::= LP idlist RP */ + -3, /* (177) idlist ::= idlist COMMA nm */ + -1, /* (178) idlist ::= nm */ + -3, /* (179) expr ::= LP expr RP */ + -1, /* (180) expr ::= ID|INDEXED|JOIN_KW */ + -3, /* (181) expr ::= nm DOT nm */ + -5, /* (182) expr ::= nm DOT nm DOT nm */ + -1, /* (183) term ::= NULL|FLOAT|BLOB */ + -1, /* (184) term ::= STRING */ + -1, /* (185) term ::= INTEGER */ + -1, /* (186) expr ::= VARIABLE */ + -3, /* (187) expr ::= expr COLLATE ID|STRING */ + -6, /* (188) expr ::= CAST LP expr AS typetoken RP */ + -5, /* (189) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP */ + -8, /* (190) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist ORDER BY sortlist RP */ + -4, /* (191) expr ::= ID|INDEXED|JOIN_KW LP STAR RP */ + -6, /* (192) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP filter_over */ + -9, /* (193) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist ORDER BY sortlist RP filter_over */ + -5, /* (194) expr ::= ID|INDEXED|JOIN_KW LP STAR RP filter_over */ + -1, /* (195) term ::= CTIME_KW */ + -5, /* (196) expr ::= LP nexprlist COMMA expr RP */ + -3, /* (197) expr ::= expr AND expr */ + -3, /* (198) expr ::= expr OR expr */ + -3, /* (199) expr ::= expr LT|GT|GE|LE expr */ + -3, /* (200) expr ::= expr EQ|NE expr */ + -3, /* (201) expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr */ + -3, /* (202) expr ::= expr PLUS|MINUS expr */ + -3, /* (203) expr ::= expr STAR|SLASH|REM expr */ + -3, /* (204) expr ::= expr CONCAT expr */ + -2, /* (205) likeop ::= NOT LIKE_KW|MATCH */ + -3, /* (206) expr ::= expr likeop expr */ + -5, /* (207) expr ::= expr likeop expr ESCAPE expr */ + -2, /* (208) expr ::= expr ISNULL|NOTNULL */ + -3, /* (209) expr ::= expr NOT NULL */ + -3, /* (210) expr ::= expr IS expr */ + -4, /* (211) expr ::= expr IS NOT expr */ + -6, /* (212) expr ::= expr IS NOT DISTINCT FROM expr */ + -5, /* (213) expr ::= expr IS DISTINCT FROM expr */ + -2, /* (214) expr ::= NOT expr */ + -2, /* (215) expr ::= BITNOT expr */ + -2, /* (216) expr ::= PLUS|MINUS expr */ + -3, /* (217) expr ::= expr PTR expr */ + -1, /* (218) between_op ::= BETWEEN */ + -2, /* (219) between_op ::= NOT BETWEEN */ + -5, /* (220) expr ::= expr between_op expr AND expr */ + -1, /* (221) in_op ::= IN */ + -2, /* (222) in_op ::= NOT IN */ + -5, /* (223) expr ::= expr in_op LP exprlist RP */ + -3, /* (224) expr ::= LP select RP */ + -5, /* (225) expr ::= expr in_op LP select RP */ + -5, /* (226) expr ::= expr in_op nm dbnm paren_exprlist */ + -4, /* (227) expr ::= EXISTS LP select RP */ + -5, /* (228) expr ::= CASE case_operand case_exprlist case_else END */ + -5, /* (229) case_exprlist ::= case_exprlist WHEN expr THEN expr */ + -4, /* (230) case_exprlist ::= WHEN expr THEN expr */ + -2, /* (231) case_else ::= ELSE expr */ + 0, /* (232) case_else ::= */ + 0, /* (233) case_operand ::= */ + 0, /* (234) exprlist ::= */ + -3, /* (235) nexprlist ::= nexprlist COMMA expr */ + -1, /* (236) nexprlist ::= expr */ + 0, /* (237) paren_exprlist ::= */ + -3, /* (238) paren_exprlist ::= LP exprlist RP */ + -12, /* (239) cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt */ + -1, /* (240) uniqueflag ::= UNIQUE */ + 0, /* (241) uniqueflag ::= */ + 0, /* (242) eidlist_opt ::= */ + -3, /* (243) eidlist_opt ::= LP eidlist RP */ + -5, /* (244) eidlist ::= eidlist COMMA nm collate sortorder */ + -3, /* (245) eidlist ::= nm collate sortorder */ + 0, /* (246) collate ::= */ + -2, /* (247) collate ::= COLLATE ID|STRING */ + -4, /* (248) cmd ::= DROP INDEX ifexists fullname */ + -2, /* (249) cmd ::= VACUUM vinto */ + -3, /* (250) cmd ::= VACUUM nm vinto */ + -2, /* (251) vinto ::= INTO expr */ + 0, /* (252) vinto ::= */ + -3, /* (253) cmd ::= PRAGMA nm dbnm */ + -5, /* (254) cmd ::= PRAGMA nm dbnm EQ nmnum */ + -6, /* (255) cmd ::= PRAGMA nm dbnm LP nmnum RP */ + -5, /* (256) cmd ::= PRAGMA nm dbnm EQ minus_num */ + -6, /* (257) cmd ::= PRAGMA nm dbnm LP minus_num RP */ + -2, /* (258) plus_num ::= PLUS INTEGER|FLOAT */ + -2, /* (259) minus_num ::= MINUS INTEGER|FLOAT */ + -5, /* (260) cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END */ + -11, /* (261) trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause */ + -1, /* (262) trigger_time ::= BEFORE|AFTER */ + -2, /* (263) trigger_time ::= INSTEAD OF */ + 0, /* (264) trigger_time ::= */ + -1, /* (265) trigger_event ::= DELETE|INSERT */ + -1, /* (266) trigger_event ::= UPDATE */ + -3, /* (267) trigger_event ::= UPDATE OF idlist */ + 0, /* (268) when_clause ::= */ + -2, /* (269) when_clause ::= WHEN expr */ + -3, /* (270) trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */ + -2, /* (271) trigger_cmd_list ::= trigger_cmd SEMI */ + -3, /* (272) trnm ::= nm DOT nm */ + -3, /* (273) tridxby ::= INDEXED BY nm */ + -2, /* (274) tridxby ::= NOT INDEXED */ + -9, /* (275) trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist from where_opt scanpt */ + -8, /* (276) trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt */ + -6, /* (277) trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt */ + -3, /* (278) trigger_cmd ::= scanpt select scanpt */ + -4, /* (279) expr ::= RAISE LP IGNORE RP */ + -6, /* (280) expr ::= RAISE LP raisetype COMMA nm RP */ + -1, /* (281) raisetype ::= ROLLBACK */ + -1, /* (282) raisetype ::= ABORT */ + -1, /* (283) raisetype ::= FAIL */ + -4, /* (284) cmd ::= DROP TRIGGER ifexists fullname */ + -6, /* (285) cmd ::= ATTACH database_kw_opt expr AS expr key_opt */ + -3, /* (286) cmd ::= DETACH database_kw_opt expr */ + 0, /* (287) key_opt ::= */ + -2, /* (288) key_opt ::= KEY expr */ + -1, /* (289) cmd ::= REINDEX */ + -3, /* (290) cmd ::= REINDEX nm dbnm */ + -1, /* (291) cmd ::= ANALYZE */ + -3, /* (292) cmd ::= ANALYZE nm dbnm */ + -6, /* (293) cmd ::= ALTER TABLE fullname RENAME TO nm */ + -7, /* (294) cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist */ + -6, /* (295) cmd ::= ALTER TABLE fullname DROP kwcolumn_opt nm */ + -1, /* (296) add_column_fullname ::= fullname */ + -8, /* (297) cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm */ + -1, /* (298) cmd ::= create_vtab */ + -4, /* (299) cmd ::= create_vtab LP vtabarglist RP */ + -8, /* (300) create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm */ + 0, /* (301) vtabarg ::= */ + -1, /* (302) vtabargtoken ::= ANY */ + -3, /* (303) vtabargtoken ::= lp anylist RP */ + -1, /* (304) lp ::= LP */ + -2, /* (305) with ::= WITH wqlist */ + -3, /* (306) with ::= WITH RECURSIVE wqlist */ + -1, /* (307) wqas ::= AS */ + -2, /* (308) wqas ::= AS MATERIALIZED */ + -3, /* (309) wqas ::= AS NOT MATERIALIZED */ + -6, /* (310) wqitem ::= withnm eidlist_opt wqas LP select RP */ + -1, /* (311) withnm ::= nm */ + -1, /* (312) wqlist ::= wqitem */ + -3, /* (313) wqlist ::= wqlist COMMA wqitem */ + -3, /* (314) windowdefn_list ::= windowdefn_list COMMA windowdefn */ + -5, /* (315) windowdefn ::= nm AS LP window RP */ + -5, /* (316) window ::= PARTITION BY nexprlist orderby_opt frame_opt */ + -6, /* (317) window ::= nm PARTITION BY nexprlist orderby_opt frame_opt */ + -4, /* (318) window ::= ORDER BY sortlist frame_opt */ + -5, /* (319) window ::= nm ORDER BY sortlist frame_opt */ + -2, /* (320) window ::= nm frame_opt */ + 0, /* (321) frame_opt ::= */ + -3, /* (322) frame_opt ::= range_or_rows frame_bound_s frame_exclude_opt */ + -6, /* (323) frame_opt ::= range_or_rows BETWEEN frame_bound_s AND frame_bound_e frame_exclude_opt */ + -1, /* (324) range_or_rows ::= RANGE|ROWS|GROUPS */ + -1, /* (325) frame_bound_s ::= frame_bound */ + -2, /* (326) frame_bound_s ::= UNBOUNDED PRECEDING */ + -1, /* (327) frame_bound_e ::= frame_bound */ + -2, /* (328) frame_bound_e ::= UNBOUNDED FOLLOWING */ + -2, /* (329) frame_bound ::= expr PRECEDING|FOLLOWING */ + -2, /* (330) frame_bound ::= CURRENT ROW */ + 0, /* (331) frame_exclude_opt ::= */ + -2, /* (332) frame_exclude_opt ::= EXCLUDE frame_exclude */ + -2, /* (333) frame_exclude ::= NO OTHERS */ + -2, /* (334) frame_exclude ::= CURRENT ROW */ + -1, /* (335) frame_exclude ::= GROUP|TIES */ + -2, /* (336) window_clause ::= WINDOW windowdefn_list */ + -2, /* (337) filter_over ::= filter_clause over_clause */ + -1, /* (338) filter_over ::= over_clause */ + -1, /* (339) filter_over ::= filter_clause */ + -4, /* (340) over_clause ::= OVER LP window RP */ + -2, /* (341) over_clause ::= OVER nm */ + -5, /* (342) filter_clause ::= FILTER LP WHERE expr RP */ + -1, /* (343) term ::= QNUMBER */ + -1, /* (344) input ::= cmdlist */ + -2, /* (345) cmdlist ::= cmdlist ecmd */ + -1, /* (346) cmdlist ::= ecmd */ + -1, /* (347) ecmd ::= SEMI */ + -2, /* (348) ecmd ::= cmdx SEMI */ + -3, /* (349) ecmd ::= explain cmdx SEMI */ + 0, /* (350) trans_opt ::= */ + -1, /* (351) trans_opt ::= TRANSACTION */ + -2, /* (352) trans_opt ::= TRANSACTION nm */ + -1, /* (353) savepoint_opt ::= SAVEPOINT */ + 0, /* (354) savepoint_opt ::= */ + -2, /* (355) cmd ::= create_table create_table_args */ + -1, /* (356) table_option_set ::= table_option */ + -4, /* (357) columnlist ::= columnlist COMMA columnname carglist */ + -2, /* (358) columnlist ::= columnname carglist */ + -1, /* (359) nm ::= ID|INDEXED|JOIN_KW */ + -1, /* (360) nm ::= STRING */ + -1, /* (361) typetoken ::= typename */ + -1, /* (362) typename ::= ID|STRING */ + -1, /* (363) signed ::= plus_num */ + -1, /* (364) signed ::= minus_num */ + -2, /* (365) carglist ::= carglist ccons */ + 0, /* (366) carglist ::= */ + -2, /* (367) ccons ::= NULL onconf */ + -4, /* (368) ccons ::= GENERATED ALWAYS AS generated */ + -2, /* (369) ccons ::= AS generated */ + -2, /* (370) conslist_opt ::= COMMA conslist */ + -3, /* (371) conslist ::= conslist tconscomma tcons */ + -1, /* (372) conslist ::= tcons */ + 0, /* (373) tconscomma ::= */ + -1, /* (374) defer_subclause_opt ::= defer_subclause */ + -1, /* (375) resolvetype ::= raisetype */ + -1, /* (376) selectnowith ::= oneselect */ + -1, /* (377) oneselect ::= values */ + -2, /* (378) sclp ::= selcollist COMMA */ + -1, /* (379) as ::= ID|STRING */ + -1, /* (380) indexed_opt ::= indexed_by */ + 0, /* (381) returning ::= */ + -1, /* (382) expr ::= term */ + -1, /* (383) likeop ::= LIKE_KW|MATCH */ + -1, /* (384) case_operand ::= expr */ + -1, /* (385) exprlist ::= nexprlist */ + -1, /* (386) nmnum ::= plus_num */ + -1, /* (387) nmnum ::= nm */ + -1, /* (388) nmnum ::= ON */ + -1, /* (389) nmnum ::= DELETE */ + -1, /* (390) nmnum ::= DEFAULT */ + -1, /* (391) plus_num ::= INTEGER|FLOAT */ + 0, /* (392) foreach_clause ::= */ + -3, /* (393) foreach_clause ::= FOR EACH ROW */ + -1, /* (394) trnm ::= nm */ + 0, /* (395) tridxby ::= */ + -1, /* (396) database_kw_opt ::= DATABASE */ + 0, /* (397) database_kw_opt ::= */ + 0, /* (398) kwcolumn_opt ::= */ + -1, /* (399) kwcolumn_opt ::= COLUMNKW */ + -1, /* (400) vtabarglist ::= vtabarg */ + -3, /* (401) vtabarglist ::= vtabarglist COMMA vtabarg */ + -2, /* (402) vtabarg ::= vtabarg vtabargtoken */ + 0, /* (403) anylist ::= */ + -4, /* (404) anylist ::= anylist LP anylist RP */ + -2, /* (405) anylist ::= anylist ANY */ + 0, /* (406) with ::= */ + -1, /* (407) windowdefn_list ::= windowdefn */ + -1, /* (408) window ::= frame_opt */ }; static void yy_accept(yyParser*); /* Forward Declaration */ @@ -174724,16 +176126,16 @@ static YYACTIONTYPE yy_reduce( { sqlite3FinishCoding(pParse); } break; case 3: /* cmd ::= BEGIN transtype trans_opt */ -{sqlite3BeginTransaction(pParse, yymsp[-1].minor.yy394);} +{sqlite3BeginTransaction(pParse, yymsp[-1].minor.yy144);} break; case 4: /* transtype ::= */ -{yymsp[1].minor.yy394 = TK_DEFERRED;} +{yymsp[1].minor.yy144 = TK_DEFERRED;} break; case 5: /* transtype ::= DEFERRED */ case 6: /* transtype ::= IMMEDIATE */ yytestcase(yyruleno==6); case 7: /* transtype ::= EXCLUSIVE */ yytestcase(yyruleno==7); - case 321: /* range_or_rows ::= RANGE|ROWS|GROUPS */ yytestcase(yyruleno==321); -{yymsp[0].minor.yy394 = yymsp[0].major; /*A-overwrites-X*/} + case 324: /* range_or_rows ::= RANGE|ROWS|GROUPS */ yytestcase(yyruleno==324); +{yymsp[0].minor.yy144 = yymsp[0].major; /*A-overwrites-X*/} break; case 8: /* cmd ::= COMMIT|END trans_opt */ case 9: /* cmd ::= ROLLBACK trans_opt */ yytestcase(yyruleno==9); @@ -174756,7 +176158,7 @@ static YYACTIONTYPE yy_reduce( break; case 13: /* create_table ::= createkw temp TABLE ifnotexists nm dbnm */ { - sqlite3StartTable(pParse,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0,yymsp[-4].minor.yy394,0,0,yymsp[-2].minor.yy394); + sqlite3StartTable(pParse,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0,yymsp[-4].minor.yy144,0,0,yymsp[-2].minor.yy144); } break; case 14: /* createkw ::= CREATE */ @@ -174768,40 +176170,40 @@ static YYACTIONTYPE yy_reduce( case 62: /* init_deferred_pred_opt ::= */ yytestcase(yyruleno==62); case 72: /* defer_subclause_opt ::= */ yytestcase(yyruleno==72); case 81: /* ifexists ::= */ yytestcase(yyruleno==81); - case 98: /* distinct ::= */ yytestcase(yyruleno==98); - case 244: /* collate ::= */ yytestcase(yyruleno==244); -{yymsp[1].minor.yy394 = 0;} + case 100: /* distinct ::= */ yytestcase(yyruleno==100); + case 246: /* collate ::= */ yytestcase(yyruleno==246); +{yymsp[1].minor.yy144 = 0;} break; case 16: /* ifnotexists ::= IF NOT EXISTS */ -{yymsp[-2].minor.yy394 = 1;} +{yymsp[-2].minor.yy144 = 1;} break; case 17: /* temp ::= TEMP */ -{yymsp[0].minor.yy394 = pParse->db->init.busy==0;} +{yymsp[0].minor.yy144 = pParse->db->init.busy==0;} break; case 19: /* create_table_args ::= LP columnlist conslist_opt RP table_option_set */ { - sqlite3EndTable(pParse,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0,yymsp[0].minor.yy285,0); + sqlite3EndTable(pParse,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0,yymsp[0].minor.yy391,0); } break; case 20: /* create_table_args ::= AS select */ { - sqlite3EndTable(pParse,0,0,0,yymsp[0].minor.yy47); - sqlite3SelectDelete(pParse->db, yymsp[0].minor.yy47); + sqlite3EndTable(pParse,0,0,0,yymsp[0].minor.yy555); + sqlite3SelectDelete(pParse->db, yymsp[0].minor.yy555); } break; case 21: /* table_option_set ::= */ -{yymsp[1].minor.yy285 = 0;} +{yymsp[1].minor.yy391 = 0;} break; case 22: /* table_option_set ::= table_option_set COMMA table_option */ -{yylhsminor.yy285 = yymsp[-2].minor.yy285|yymsp[0].minor.yy285;} - yymsp[-2].minor.yy285 = yylhsminor.yy285; +{yylhsminor.yy391 = yymsp[-2].minor.yy391|yymsp[0].minor.yy391;} + yymsp[-2].minor.yy391 = yylhsminor.yy391; break; case 23: /* table_option ::= WITHOUT nm */ { if( yymsp[0].minor.yy0.n==5 && sqlite3_strnicmp(yymsp[0].minor.yy0.z,"rowid",5)==0 ){ - yymsp[-1].minor.yy285 = TF_WithoutRowid | TF_NoVisibleRowid; + yymsp[-1].minor.yy391 = TF_WithoutRowid | TF_NoVisibleRowid; }else{ - yymsp[-1].minor.yy285 = 0; + yymsp[-1].minor.yy391 = 0; sqlite3ErrorMsg(pParse, "unknown table option: %.*s", yymsp[0].minor.yy0.n, yymsp[0].minor.yy0.z); } } @@ -174809,20 +176211,20 @@ static YYACTIONTYPE yy_reduce( case 24: /* table_option ::= nm */ { if( yymsp[0].minor.yy0.n==6 && sqlite3_strnicmp(yymsp[0].minor.yy0.z,"strict",6)==0 ){ - yylhsminor.yy285 = TF_Strict; + yylhsminor.yy391 = TF_Strict; }else{ - yylhsminor.yy285 = 0; + yylhsminor.yy391 = 0; sqlite3ErrorMsg(pParse, "unknown table option: %.*s", yymsp[0].minor.yy0.n, yymsp[0].minor.yy0.z); } } - yymsp[0].minor.yy285 = yylhsminor.yy285; + yymsp[0].minor.yy391 = yylhsminor.yy391; break; case 25: /* columnname ::= nm typetoken */ {sqlite3AddColumn(pParse,yymsp[-1].minor.yy0,yymsp[0].minor.yy0);} break; case 26: /* typetoken ::= */ case 65: /* conslist_opt ::= */ yytestcase(yyruleno==65); - case 104: /* as ::= */ yytestcase(yyruleno==104); + case 106: /* as ::= */ yytestcase(yyruleno==106); {yymsp[1].minor.yy0.n = 0; yymsp[1].minor.yy0.z = 0;} break; case 27: /* typetoken ::= typename LP signed RP */ @@ -174841,7 +176243,7 @@ static YYACTIONTYPE yy_reduce( case 30: /* scanpt ::= */ { assert( yyLookahead!=YYNOCODE ); - yymsp[1].minor.yy522 = yyLookaheadToken.z; + yymsp[1].minor.yy168 = yyLookaheadToken.z; } break; case 31: /* scantok ::= */ @@ -174855,17 +176257,17 @@ static YYACTIONTYPE yy_reduce( {pParse->constraintName = yymsp[0].minor.yy0;} break; case 33: /* ccons ::= DEFAULT scantok term */ -{sqlite3AddDefaultValue(pParse,yymsp[0].minor.yy528,yymsp[-1].minor.yy0.z,&yymsp[-1].minor.yy0.z[yymsp[-1].minor.yy0.n]);} +{sqlite3AddDefaultValue(pParse,yymsp[0].minor.yy454,yymsp[-1].minor.yy0.z,&yymsp[-1].minor.yy0.z[yymsp[-1].minor.yy0.n]);} break; case 34: /* ccons ::= DEFAULT LP expr RP */ -{sqlite3AddDefaultValue(pParse,yymsp[-1].minor.yy528,yymsp[-2].minor.yy0.z+1,yymsp[0].minor.yy0.z);} +{sqlite3AddDefaultValue(pParse,yymsp[-1].minor.yy454,yymsp[-2].minor.yy0.z+1,yymsp[0].minor.yy0.z);} break; case 35: /* ccons ::= DEFAULT PLUS scantok term */ -{sqlite3AddDefaultValue(pParse,yymsp[0].minor.yy528,yymsp[-2].minor.yy0.z,&yymsp[-1].minor.yy0.z[yymsp[-1].minor.yy0.n]);} +{sqlite3AddDefaultValue(pParse,yymsp[0].minor.yy454,yymsp[-2].minor.yy0.z,&yymsp[-1].minor.yy0.z[yymsp[-1].minor.yy0.n]);} break; case 36: /* ccons ::= DEFAULT MINUS scantok term */ { - Expr *p = sqlite3PExpr(pParse, TK_UMINUS, yymsp[0].minor.yy528, 0); + Expr *p = sqlite3PExpr(pParse, TK_UMINUS, yymsp[0].minor.yy454, 0); sqlite3AddDefaultValue(pParse,p,yymsp[-2].minor.yy0.z,&yymsp[-1].minor.yy0.z[yymsp[-1].minor.yy0.n]); } break; @@ -174880,151 +176282,151 @@ static YYACTIONTYPE yy_reduce( } break; case 38: /* ccons ::= NOT NULL onconf */ -{sqlite3AddNotNull(pParse, yymsp[0].minor.yy394);} +{sqlite3AddNotNull(pParse, yymsp[0].minor.yy144);} break; case 39: /* ccons ::= PRIMARY KEY sortorder onconf autoinc */ -{sqlite3AddPrimaryKey(pParse,0,yymsp[-1].minor.yy394,yymsp[0].minor.yy394,yymsp[-2].minor.yy394);} +{sqlite3AddPrimaryKey(pParse,0,yymsp[-1].minor.yy144,yymsp[0].minor.yy144,yymsp[-2].minor.yy144);} break; case 40: /* ccons ::= UNIQUE onconf */ -{sqlite3CreateIndex(pParse,0,0,0,0,yymsp[0].minor.yy394,0,0,0,0, +{sqlite3CreateIndex(pParse,0,0,0,0,yymsp[0].minor.yy144,0,0,0,0, SQLITE_IDXTYPE_UNIQUE);} break; case 41: /* ccons ::= CHECK LP expr RP */ -{sqlite3AddCheckConstraint(pParse,yymsp[-1].minor.yy528,yymsp[-2].minor.yy0.z,yymsp[0].minor.yy0.z);} +{sqlite3AddCheckConstraint(pParse,yymsp[-1].minor.yy454,yymsp[-2].minor.yy0.z,yymsp[0].minor.yy0.z);} break; case 42: /* ccons ::= REFERENCES nm eidlist_opt refargs */ -{sqlite3CreateForeignKey(pParse,0,&yymsp[-2].minor.yy0,yymsp[-1].minor.yy322,yymsp[0].minor.yy394);} +{sqlite3CreateForeignKey(pParse,0,&yymsp[-2].minor.yy0,yymsp[-1].minor.yy14,yymsp[0].minor.yy144);} break; case 43: /* ccons ::= defer_subclause */ -{sqlite3DeferForeignKey(pParse,yymsp[0].minor.yy394);} +{sqlite3DeferForeignKey(pParse,yymsp[0].minor.yy144);} break; case 44: /* ccons ::= COLLATE ID|STRING */ {sqlite3AddCollateType(pParse, &yymsp[0].minor.yy0);} break; case 45: /* generated ::= LP expr RP */ -{sqlite3AddGenerated(pParse,yymsp[-1].minor.yy528,0);} +{sqlite3AddGenerated(pParse,yymsp[-1].minor.yy454,0);} break; case 46: /* generated ::= LP expr RP ID */ -{sqlite3AddGenerated(pParse,yymsp[-2].minor.yy528,&yymsp[0].minor.yy0);} +{sqlite3AddGenerated(pParse,yymsp[-2].minor.yy454,&yymsp[0].minor.yy0);} break; case 48: /* autoinc ::= AUTOINCR */ -{yymsp[0].minor.yy394 = 1;} +{yymsp[0].minor.yy144 = 1;} break; case 49: /* refargs ::= */ -{ yymsp[1].minor.yy394 = OE_None*0x0101; /* EV: R-19803-45884 */} +{ yymsp[1].minor.yy144 = OE_None*0x0101; /* EV: R-19803-45884 */} break; case 50: /* refargs ::= refargs refarg */ -{ yymsp[-1].minor.yy394 = (yymsp[-1].minor.yy394 & ~yymsp[0].minor.yy231.mask) | yymsp[0].minor.yy231.value; } +{ yymsp[-1].minor.yy144 = (yymsp[-1].minor.yy144 & ~yymsp[0].minor.yy383.mask) | yymsp[0].minor.yy383.value; } break; case 51: /* refarg ::= MATCH nm */ -{ yymsp[-1].minor.yy231.value = 0; yymsp[-1].minor.yy231.mask = 0x000000; } +{ yymsp[-1].minor.yy383.value = 0; yymsp[-1].minor.yy383.mask = 0x000000; } break; case 52: /* refarg ::= ON INSERT refact */ -{ yymsp[-2].minor.yy231.value = 0; yymsp[-2].minor.yy231.mask = 0x000000; } +{ yymsp[-2].minor.yy383.value = 0; yymsp[-2].minor.yy383.mask = 0x000000; } break; case 53: /* refarg ::= ON DELETE refact */ -{ yymsp[-2].minor.yy231.value = yymsp[0].minor.yy394; yymsp[-2].minor.yy231.mask = 0x0000ff; } +{ yymsp[-2].minor.yy383.value = yymsp[0].minor.yy144; yymsp[-2].minor.yy383.mask = 0x0000ff; } break; case 54: /* refarg ::= ON UPDATE refact */ -{ yymsp[-2].minor.yy231.value = yymsp[0].minor.yy394<<8; yymsp[-2].minor.yy231.mask = 0x00ff00; } +{ yymsp[-2].minor.yy383.value = yymsp[0].minor.yy144<<8; yymsp[-2].minor.yy383.mask = 0x00ff00; } break; case 55: /* refact ::= SET NULL */ -{ yymsp[-1].minor.yy394 = OE_SetNull; /* EV: R-33326-45252 */} +{ yymsp[-1].minor.yy144 = OE_SetNull; /* EV: R-33326-45252 */} break; case 56: /* refact ::= SET DEFAULT */ -{ yymsp[-1].minor.yy394 = OE_SetDflt; /* EV: R-33326-45252 */} +{ yymsp[-1].minor.yy144 = OE_SetDflt; /* EV: R-33326-45252 */} break; case 57: /* refact ::= CASCADE */ -{ yymsp[0].minor.yy394 = OE_Cascade; /* EV: R-33326-45252 */} +{ yymsp[0].minor.yy144 = OE_Cascade; /* EV: R-33326-45252 */} break; case 58: /* refact ::= RESTRICT */ -{ yymsp[0].minor.yy394 = OE_Restrict; /* EV: R-33326-45252 */} +{ yymsp[0].minor.yy144 = OE_Restrict; /* EV: R-33326-45252 */} break; case 59: /* refact ::= NO ACTION */ -{ yymsp[-1].minor.yy394 = OE_None; /* EV: R-33326-45252 */} +{ yymsp[-1].minor.yy144 = OE_None; /* EV: R-33326-45252 */} break; case 60: /* defer_subclause ::= NOT DEFERRABLE init_deferred_pred_opt */ -{yymsp[-2].minor.yy394 = 0;} +{yymsp[-2].minor.yy144 = 0;} break; case 61: /* defer_subclause ::= DEFERRABLE init_deferred_pred_opt */ case 76: /* orconf ::= OR resolvetype */ yytestcase(yyruleno==76); - case 171: /* insert_cmd ::= INSERT orconf */ yytestcase(yyruleno==171); -{yymsp[-1].minor.yy394 = yymsp[0].minor.yy394;} + case 173: /* insert_cmd ::= INSERT orconf */ yytestcase(yyruleno==173); +{yymsp[-1].minor.yy144 = yymsp[0].minor.yy144;} break; case 63: /* init_deferred_pred_opt ::= INITIALLY DEFERRED */ case 80: /* ifexists ::= IF EXISTS */ yytestcase(yyruleno==80); - case 217: /* between_op ::= NOT BETWEEN */ yytestcase(yyruleno==217); - case 220: /* in_op ::= NOT IN */ yytestcase(yyruleno==220); - case 245: /* collate ::= COLLATE ID|STRING */ yytestcase(yyruleno==245); -{yymsp[-1].minor.yy394 = 1;} + case 219: /* between_op ::= NOT BETWEEN */ yytestcase(yyruleno==219); + case 222: /* in_op ::= NOT IN */ yytestcase(yyruleno==222); + case 247: /* collate ::= COLLATE ID|STRING */ yytestcase(yyruleno==247); +{yymsp[-1].minor.yy144 = 1;} break; case 64: /* init_deferred_pred_opt ::= INITIALLY IMMEDIATE */ -{yymsp[-1].minor.yy394 = 0;} +{yymsp[-1].minor.yy144 = 0;} break; case 66: /* tconscomma ::= COMMA */ {pParse->constraintName.n = 0;} break; case 68: /* tcons ::= PRIMARY KEY LP sortlist autoinc RP onconf */ -{sqlite3AddPrimaryKey(pParse,yymsp[-3].minor.yy322,yymsp[0].minor.yy394,yymsp[-2].minor.yy394,0);} +{sqlite3AddPrimaryKey(pParse,yymsp[-3].minor.yy14,yymsp[0].minor.yy144,yymsp[-2].minor.yy144,0);} break; case 69: /* tcons ::= UNIQUE LP sortlist RP onconf */ -{sqlite3CreateIndex(pParse,0,0,0,yymsp[-2].minor.yy322,yymsp[0].minor.yy394,0,0,0,0, +{sqlite3CreateIndex(pParse,0,0,0,yymsp[-2].minor.yy14,yymsp[0].minor.yy144,0,0,0,0, SQLITE_IDXTYPE_UNIQUE);} break; case 70: /* tcons ::= CHECK LP expr RP onconf */ -{sqlite3AddCheckConstraint(pParse,yymsp[-2].minor.yy528,yymsp[-3].minor.yy0.z,yymsp[-1].minor.yy0.z);} +{sqlite3AddCheckConstraint(pParse,yymsp[-2].minor.yy454,yymsp[-3].minor.yy0.z,yymsp[-1].minor.yy0.z);} break; case 71: /* tcons ::= FOREIGN KEY LP eidlist RP REFERENCES nm eidlist_opt refargs defer_subclause_opt */ { - sqlite3CreateForeignKey(pParse, yymsp[-6].minor.yy322, &yymsp[-3].minor.yy0, yymsp[-2].minor.yy322, yymsp[-1].minor.yy394); - sqlite3DeferForeignKey(pParse, yymsp[0].minor.yy394); + sqlite3CreateForeignKey(pParse, yymsp[-6].minor.yy14, &yymsp[-3].minor.yy0, yymsp[-2].minor.yy14, yymsp[-1].minor.yy144); + sqlite3DeferForeignKey(pParse, yymsp[0].minor.yy144); } break; case 73: /* onconf ::= */ case 75: /* orconf ::= */ yytestcase(yyruleno==75); -{yymsp[1].minor.yy394 = OE_Default;} +{yymsp[1].minor.yy144 = OE_Default;} break; case 74: /* onconf ::= ON CONFLICT resolvetype */ -{yymsp[-2].minor.yy394 = yymsp[0].minor.yy394;} +{yymsp[-2].minor.yy144 = yymsp[0].minor.yy144;} break; case 77: /* resolvetype ::= IGNORE */ -{yymsp[0].minor.yy394 = OE_Ignore;} +{yymsp[0].minor.yy144 = OE_Ignore;} break; case 78: /* resolvetype ::= REPLACE */ - case 172: /* insert_cmd ::= REPLACE */ yytestcase(yyruleno==172); -{yymsp[0].minor.yy394 = OE_Replace;} + case 174: /* insert_cmd ::= REPLACE */ yytestcase(yyruleno==174); +{yymsp[0].minor.yy144 = OE_Replace;} break; case 79: /* cmd ::= DROP TABLE ifexists fullname */ { - sqlite3DropTable(pParse, yymsp[0].minor.yy131, 0, yymsp[-1].minor.yy394); + sqlite3DropTable(pParse, yymsp[0].minor.yy203, 0, yymsp[-1].minor.yy144); } break; case 82: /* cmd ::= createkw temp VIEW ifnotexists nm dbnm eidlist_opt AS select */ { - sqlite3CreateView(pParse, &yymsp[-8].minor.yy0, &yymsp[-4].minor.yy0, &yymsp[-3].minor.yy0, yymsp[-2].minor.yy322, yymsp[0].minor.yy47, yymsp[-7].minor.yy394, yymsp[-5].minor.yy394); + sqlite3CreateView(pParse, &yymsp[-8].minor.yy0, &yymsp[-4].minor.yy0, &yymsp[-3].minor.yy0, yymsp[-2].minor.yy14, yymsp[0].minor.yy555, yymsp[-7].minor.yy144, yymsp[-5].minor.yy144); } break; case 83: /* cmd ::= DROP VIEW ifexists fullname */ { - sqlite3DropTable(pParse, yymsp[0].minor.yy131, 1, yymsp[-1].minor.yy394); + sqlite3DropTable(pParse, yymsp[0].minor.yy203, 1, yymsp[-1].minor.yy144); } break; case 84: /* cmd ::= select */ { SelectDest dest = {SRT_Output, 0, 0, 0, 0, 0, 0}; - sqlite3Select(pParse, yymsp[0].minor.yy47, &dest); - sqlite3SelectDelete(pParse->db, yymsp[0].minor.yy47); + sqlite3Select(pParse, yymsp[0].minor.yy555, &dest); + sqlite3SelectDelete(pParse->db, yymsp[0].minor.yy555); } break; case 85: /* select ::= WITH wqlist selectnowith */ -{yymsp[-2].minor.yy47 = attachWithToSelect(pParse,yymsp[0].minor.yy47,yymsp[-1].minor.yy521);} +{yymsp[-2].minor.yy555 = attachWithToSelect(pParse,yymsp[0].minor.yy555,yymsp[-1].minor.yy59);} break; case 86: /* select ::= WITH RECURSIVE wqlist selectnowith */ -{yymsp[-3].minor.yy47 = attachWithToSelect(pParse,yymsp[0].minor.yy47,yymsp[-1].minor.yy521);} +{yymsp[-3].minor.yy555 = attachWithToSelect(pParse,yymsp[0].minor.yy555,yymsp[-1].minor.yy59);} break; case 87: /* select ::= selectnowith */ { - Select *p = yymsp[0].minor.yy47; + Select *p = yymsp[0].minor.yy555; if( p ){ parserDoubleLinkSelect(pParse, p); } @@ -175032,8 +176434,8 @@ static YYACTIONTYPE yy_reduce( break; case 88: /* selectnowith ::= selectnowith multiselect_op oneselect */ { - Select *pRhs = yymsp[0].minor.yy47; - Select *pLhs = yymsp[-2].minor.yy47; + Select *pRhs = yymsp[0].minor.yy555; + Select *pLhs = yymsp[-2].minor.yy555; if( pRhs && pRhs->pPrior ){ SrcList *pFrom; Token x; @@ -175043,148 +176445,145 @@ static YYACTIONTYPE yy_reduce( pRhs = sqlite3SelectNew(pParse,0,pFrom,0,0,0,0,0,0); } if( pRhs ){ - pRhs->op = (u8)yymsp[-1].minor.yy394; + pRhs->op = (u8)yymsp[-1].minor.yy144; pRhs->pPrior = pLhs; if( ALWAYS(pLhs) ) pLhs->selFlags &= ~SF_MultiValue; pRhs->selFlags &= ~SF_MultiValue; - if( yymsp[-1].minor.yy394!=TK_ALL ) pParse->hasCompound = 1; + if( yymsp[-1].minor.yy144!=TK_ALL ) pParse->hasCompound = 1; }else{ sqlite3SelectDelete(pParse->db, pLhs); } - yymsp[-2].minor.yy47 = pRhs; + yymsp[-2].minor.yy555 = pRhs; } break; case 89: /* multiselect_op ::= UNION */ case 91: /* multiselect_op ::= EXCEPT|INTERSECT */ yytestcase(yyruleno==91); -{yymsp[0].minor.yy394 = yymsp[0].major; /*A-overwrites-OP*/} +{yymsp[0].minor.yy144 = yymsp[0].major; /*A-overwrites-OP*/} break; case 90: /* multiselect_op ::= UNION ALL */ -{yymsp[-1].minor.yy394 = TK_ALL;} +{yymsp[-1].minor.yy144 = TK_ALL;} break; case 92: /* oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt orderby_opt limit_opt */ { - yymsp[-8].minor.yy47 = sqlite3SelectNew(pParse,yymsp[-6].minor.yy322,yymsp[-5].minor.yy131,yymsp[-4].minor.yy528,yymsp[-3].minor.yy322,yymsp[-2].minor.yy528,yymsp[-1].minor.yy322,yymsp[-7].minor.yy394,yymsp[0].minor.yy528); + yymsp[-8].minor.yy555 = sqlite3SelectNew(pParse,yymsp[-6].minor.yy14,yymsp[-5].minor.yy203,yymsp[-4].minor.yy454,yymsp[-3].minor.yy14,yymsp[-2].minor.yy454,yymsp[-1].minor.yy14,yymsp[-7].minor.yy144,yymsp[0].minor.yy454); } break; case 93: /* oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt window_clause orderby_opt limit_opt */ { - yymsp[-9].minor.yy47 = sqlite3SelectNew(pParse,yymsp[-7].minor.yy322,yymsp[-6].minor.yy131,yymsp[-5].minor.yy528,yymsp[-4].minor.yy322,yymsp[-3].minor.yy528,yymsp[-1].minor.yy322,yymsp[-8].minor.yy394,yymsp[0].minor.yy528); - if( yymsp[-9].minor.yy47 ){ - yymsp[-9].minor.yy47->pWinDefn = yymsp[-2].minor.yy41; + yymsp[-9].minor.yy555 = sqlite3SelectNew(pParse,yymsp[-7].minor.yy14,yymsp[-6].minor.yy203,yymsp[-5].minor.yy454,yymsp[-4].minor.yy14,yymsp[-3].minor.yy454,yymsp[-1].minor.yy14,yymsp[-8].minor.yy144,yymsp[0].minor.yy454); + if( yymsp[-9].minor.yy555 ){ + yymsp[-9].minor.yy555->pWinDefn = yymsp[-2].minor.yy211; }else{ - sqlite3WindowListDelete(pParse->db, yymsp[-2].minor.yy41); + sqlite3WindowListDelete(pParse->db, yymsp[-2].minor.yy211); } } break; case 94: /* values ::= VALUES LP nexprlist RP */ { - yymsp[-3].minor.yy47 = sqlite3SelectNew(pParse,yymsp[-1].minor.yy322,0,0,0,0,0,SF_Values,0); + yymsp[-3].minor.yy555 = sqlite3SelectNew(pParse,yymsp[-1].minor.yy14,0,0,0,0,0,SF_Values,0); } break; - case 95: /* values ::= values COMMA LP nexprlist RP */ + case 95: /* oneselect ::= mvalues */ { - Select *pRight, *pLeft = yymsp[-4].minor.yy47; - pRight = sqlite3SelectNew(pParse,yymsp[-1].minor.yy322,0,0,0,0,0,SF_Values|SF_MultiValue,0); - if( ALWAYS(pLeft) ) pLeft->selFlags &= ~SF_MultiValue; - if( pRight ){ - pRight->op = TK_ALL; - pRight->pPrior = pLeft; - yymsp[-4].minor.yy47 = pRight; - }else{ - yymsp[-4].minor.yy47 = pLeft; - } + sqlite3MultiValuesEnd(pParse, yymsp[0].minor.yy555); } break; - case 96: /* distinct ::= DISTINCT */ -{yymsp[0].minor.yy394 = SF_Distinct;} - break; - case 97: /* distinct ::= ALL */ -{yymsp[0].minor.yy394 = SF_All;} - break; - case 99: /* sclp ::= */ - case 132: /* orderby_opt ::= */ yytestcase(yyruleno==132); - case 142: /* groupby_opt ::= */ yytestcase(yyruleno==142); - case 232: /* exprlist ::= */ yytestcase(yyruleno==232); - case 235: /* paren_exprlist ::= */ yytestcase(yyruleno==235); - case 240: /* eidlist_opt ::= */ yytestcase(yyruleno==240); -{yymsp[1].minor.yy322 = 0;} - break; - case 100: /* selcollist ::= sclp scanpt expr scanpt as */ + case 96: /* mvalues ::= values COMMA LP nexprlist RP */ + case 97: /* mvalues ::= mvalues COMMA LP nexprlist RP */ yytestcase(yyruleno==97); { - yymsp[-4].minor.yy322 = sqlite3ExprListAppend(pParse, yymsp[-4].minor.yy322, yymsp[-2].minor.yy528); - if( yymsp[0].minor.yy0.n>0 ) sqlite3ExprListSetName(pParse, yymsp[-4].minor.yy322, &yymsp[0].minor.yy0, 1); - sqlite3ExprListSetSpan(pParse,yymsp[-4].minor.yy322,yymsp[-3].minor.yy522,yymsp[-1].minor.yy522); + yymsp[-4].minor.yy555 = sqlite3MultiValues(pParse, yymsp[-4].minor.yy555, yymsp[-1].minor.yy14); } break; - case 101: /* selcollist ::= sclp scanpt STAR */ + case 98: /* distinct ::= DISTINCT */ +{yymsp[0].minor.yy144 = SF_Distinct;} + break; + case 99: /* distinct ::= ALL */ +{yymsp[0].minor.yy144 = SF_All;} + break; + case 101: /* sclp ::= */ + case 134: /* orderby_opt ::= */ yytestcase(yyruleno==134); + case 144: /* groupby_opt ::= */ yytestcase(yyruleno==144); + case 234: /* exprlist ::= */ yytestcase(yyruleno==234); + case 237: /* paren_exprlist ::= */ yytestcase(yyruleno==237); + case 242: /* eidlist_opt ::= */ yytestcase(yyruleno==242); +{yymsp[1].minor.yy14 = 0;} + break; + case 102: /* selcollist ::= sclp scanpt expr scanpt as */ +{ + yymsp[-4].minor.yy14 = sqlite3ExprListAppend(pParse, yymsp[-4].minor.yy14, yymsp[-2].minor.yy454); + if( yymsp[0].minor.yy0.n>0 ) sqlite3ExprListSetName(pParse, yymsp[-4].minor.yy14, &yymsp[0].minor.yy0, 1); + sqlite3ExprListSetSpan(pParse,yymsp[-4].minor.yy14,yymsp[-3].minor.yy168,yymsp[-1].minor.yy168); +} + break; + case 103: /* selcollist ::= sclp scanpt STAR */ { Expr *p = sqlite3Expr(pParse->db, TK_ASTERISK, 0); sqlite3ExprSetErrorOffset(p, (int)(yymsp[0].minor.yy0.z - pParse->zTail)); - yymsp[-2].minor.yy322 = sqlite3ExprListAppend(pParse, yymsp[-2].minor.yy322, p); + yymsp[-2].minor.yy14 = sqlite3ExprListAppend(pParse, yymsp[-2].minor.yy14, p); } break; - case 102: /* selcollist ::= sclp scanpt nm DOT STAR */ + case 104: /* selcollist ::= sclp scanpt nm DOT STAR */ { Expr *pRight, *pLeft, *pDot; pRight = sqlite3PExpr(pParse, TK_ASTERISK, 0, 0); sqlite3ExprSetErrorOffset(pRight, (int)(yymsp[0].minor.yy0.z - pParse->zTail)); pLeft = tokenExpr(pParse, TK_ID, yymsp[-2].minor.yy0); pDot = sqlite3PExpr(pParse, TK_DOT, pLeft, pRight); - yymsp[-4].minor.yy322 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy322, pDot); + yymsp[-4].minor.yy14 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy14, pDot); } break; - case 103: /* as ::= AS nm */ - case 115: /* dbnm ::= DOT nm */ yytestcase(yyruleno==115); - case 256: /* plus_num ::= PLUS INTEGER|FLOAT */ yytestcase(yyruleno==256); - case 257: /* minus_num ::= MINUS INTEGER|FLOAT */ yytestcase(yyruleno==257); + case 105: /* as ::= AS nm */ + case 117: /* dbnm ::= DOT nm */ yytestcase(yyruleno==117); + case 258: /* plus_num ::= PLUS INTEGER|FLOAT */ yytestcase(yyruleno==258); + case 259: /* minus_num ::= MINUS INTEGER|FLOAT */ yytestcase(yyruleno==259); {yymsp[-1].minor.yy0 = yymsp[0].minor.yy0;} break; - case 105: /* from ::= */ - case 108: /* stl_prefix ::= */ yytestcase(yyruleno==108); -{yymsp[1].minor.yy131 = 0;} + case 107: /* from ::= */ + case 110: /* stl_prefix ::= */ yytestcase(yyruleno==110); +{yymsp[1].minor.yy203 = 0;} break; - case 106: /* from ::= FROM seltablist */ + case 108: /* from ::= FROM seltablist */ { - yymsp[-1].minor.yy131 = yymsp[0].minor.yy131; - sqlite3SrcListShiftJoinType(pParse,yymsp[-1].minor.yy131); + yymsp[-1].minor.yy203 = yymsp[0].minor.yy203; + sqlite3SrcListShiftJoinType(pParse,yymsp[-1].minor.yy203); } break; - case 107: /* stl_prefix ::= seltablist joinop */ + case 109: /* stl_prefix ::= seltablist joinop */ { - if( ALWAYS(yymsp[-1].minor.yy131 && yymsp[-1].minor.yy131->nSrc>0) ) yymsp[-1].minor.yy131->a[yymsp[-1].minor.yy131->nSrc-1].fg.jointype = (u8)yymsp[0].minor.yy394; + if( ALWAYS(yymsp[-1].minor.yy203 && yymsp[-1].minor.yy203->nSrc>0) ) yymsp[-1].minor.yy203->a[yymsp[-1].minor.yy203->nSrc-1].fg.jointype = (u8)yymsp[0].minor.yy144; } break; - case 109: /* seltablist ::= stl_prefix nm dbnm as on_using */ + case 111: /* seltablist ::= stl_prefix nm dbnm as on_using */ { - yymsp[-4].minor.yy131 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-4].minor.yy131,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0,0,&yymsp[0].minor.yy561); + yymsp[-4].minor.yy203 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-4].minor.yy203,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0,0,&yymsp[0].minor.yy269); } break; - case 110: /* seltablist ::= stl_prefix nm dbnm as indexed_by on_using */ + case 112: /* seltablist ::= stl_prefix nm dbnm as indexed_by on_using */ { - yymsp[-5].minor.yy131 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-5].minor.yy131,&yymsp[-4].minor.yy0,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,0,&yymsp[0].minor.yy561); - sqlite3SrcListIndexedBy(pParse, yymsp[-5].minor.yy131, &yymsp[-1].minor.yy0); + yymsp[-5].minor.yy203 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-5].minor.yy203,&yymsp[-4].minor.yy0,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,0,&yymsp[0].minor.yy269); + sqlite3SrcListIndexedBy(pParse, yymsp[-5].minor.yy203, &yymsp[-1].minor.yy0); } break; - case 111: /* seltablist ::= stl_prefix nm dbnm LP exprlist RP as on_using */ + case 113: /* seltablist ::= stl_prefix nm dbnm LP exprlist RP as on_using */ { - yymsp[-7].minor.yy131 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-7].minor.yy131,&yymsp[-6].minor.yy0,&yymsp[-5].minor.yy0,&yymsp[-1].minor.yy0,0,&yymsp[0].minor.yy561); - sqlite3SrcListFuncArgs(pParse, yymsp[-7].minor.yy131, yymsp[-3].minor.yy322); + yymsp[-7].minor.yy203 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-7].minor.yy203,&yymsp[-6].minor.yy0,&yymsp[-5].minor.yy0,&yymsp[-1].minor.yy0,0,&yymsp[0].minor.yy269); + sqlite3SrcListFuncArgs(pParse, yymsp[-7].minor.yy203, yymsp[-3].minor.yy14); } break; - case 112: /* seltablist ::= stl_prefix LP select RP as on_using */ + case 114: /* seltablist ::= stl_prefix LP select RP as on_using */ { - yymsp[-5].minor.yy131 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-5].minor.yy131,0,0,&yymsp[-1].minor.yy0,yymsp[-3].minor.yy47,&yymsp[0].minor.yy561); + yymsp[-5].minor.yy203 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-5].minor.yy203,0,0,&yymsp[-1].minor.yy0,yymsp[-3].minor.yy555,&yymsp[0].minor.yy269); } break; - case 113: /* seltablist ::= stl_prefix LP seltablist RP as on_using */ + case 115: /* seltablist ::= stl_prefix LP seltablist RP as on_using */ { - if( yymsp[-5].minor.yy131==0 && yymsp[-1].minor.yy0.n==0 && yymsp[0].minor.yy561.pOn==0 && yymsp[0].minor.yy561.pUsing==0 ){ - yymsp[-5].minor.yy131 = yymsp[-3].minor.yy131; - }else if( ALWAYS(yymsp[-3].minor.yy131!=0) && yymsp[-3].minor.yy131->nSrc==1 ){ - yymsp[-5].minor.yy131 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-5].minor.yy131,0,0,&yymsp[-1].minor.yy0,0,&yymsp[0].minor.yy561); - if( yymsp[-5].minor.yy131 ){ - SrcItem *pNew = &yymsp[-5].minor.yy131->a[yymsp[-5].minor.yy131->nSrc-1]; - SrcItem *pOld = yymsp[-3].minor.yy131->a; + if( yymsp[-5].minor.yy203==0 && yymsp[-1].minor.yy0.n==0 && yymsp[0].minor.yy269.pOn==0 && yymsp[0].minor.yy269.pUsing==0 ){ + yymsp[-5].minor.yy203 = yymsp[-3].minor.yy203; + }else if( ALWAYS(yymsp[-3].minor.yy203!=0) && yymsp[-3].minor.yy203->nSrc==1 ){ + yymsp[-5].minor.yy203 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-5].minor.yy203,0,0,&yymsp[-1].minor.yy0,0,&yymsp[0].minor.yy269); + if( yymsp[-5].minor.yy203 ){ + SrcItem *pNew = &yymsp[-5].minor.yy203->a[yymsp[-5].minor.yy203->nSrc-1]; + SrcItem *pOld = yymsp[-3].minor.yy203->a; pNew->zName = pOld->zName; pNew->zDatabase = pOld->zDatabase; pNew->pSelect = pOld->pSelect; @@ -175200,153 +176599,153 @@ static YYACTIONTYPE yy_reduce( pOld->zName = pOld->zDatabase = 0; pOld->pSelect = 0; } - sqlite3SrcListDelete(pParse->db, yymsp[-3].minor.yy131); + sqlite3SrcListDelete(pParse->db, yymsp[-3].minor.yy203); }else{ Select *pSubquery; - sqlite3SrcListShiftJoinType(pParse,yymsp[-3].minor.yy131); - pSubquery = sqlite3SelectNew(pParse,0,yymsp[-3].minor.yy131,0,0,0,0,SF_NestedFrom,0); - yymsp[-5].minor.yy131 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-5].minor.yy131,0,0,&yymsp[-1].minor.yy0,pSubquery,&yymsp[0].minor.yy561); + sqlite3SrcListShiftJoinType(pParse,yymsp[-3].minor.yy203); + pSubquery = sqlite3SelectNew(pParse,0,yymsp[-3].minor.yy203,0,0,0,0,SF_NestedFrom,0); + yymsp[-5].minor.yy203 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-5].minor.yy203,0,0,&yymsp[-1].minor.yy0,pSubquery,&yymsp[0].minor.yy269); } } break; - case 114: /* dbnm ::= */ - case 129: /* indexed_opt ::= */ yytestcase(yyruleno==129); + case 116: /* dbnm ::= */ + case 131: /* indexed_opt ::= */ yytestcase(yyruleno==131); {yymsp[1].minor.yy0.z=0; yymsp[1].minor.yy0.n=0;} break; - case 116: /* fullname ::= nm */ + case 118: /* fullname ::= nm */ { - yylhsminor.yy131 = sqlite3SrcListAppend(pParse,0,&yymsp[0].minor.yy0,0); - if( IN_RENAME_OBJECT && yylhsminor.yy131 ) sqlite3RenameTokenMap(pParse, yylhsminor.yy131->a[0].zName, &yymsp[0].minor.yy0); + yylhsminor.yy203 = sqlite3SrcListAppend(pParse,0,&yymsp[0].minor.yy0,0); + if( IN_RENAME_OBJECT && yylhsminor.yy203 ) sqlite3RenameTokenMap(pParse, yylhsminor.yy203->a[0].zName, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy131 = yylhsminor.yy131; + yymsp[0].minor.yy203 = yylhsminor.yy203; break; - case 117: /* fullname ::= nm DOT nm */ + case 119: /* fullname ::= nm DOT nm */ { - yylhsminor.yy131 = sqlite3SrcListAppend(pParse,0,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0); - if( IN_RENAME_OBJECT && yylhsminor.yy131 ) sqlite3RenameTokenMap(pParse, yylhsminor.yy131->a[0].zName, &yymsp[0].minor.yy0); + yylhsminor.yy203 = sqlite3SrcListAppend(pParse,0,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0); + if( IN_RENAME_OBJECT && yylhsminor.yy203 ) sqlite3RenameTokenMap(pParse, yylhsminor.yy203->a[0].zName, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy131 = yylhsminor.yy131; + yymsp[-2].minor.yy203 = yylhsminor.yy203; break; - case 118: /* xfullname ::= nm */ -{yymsp[0].minor.yy131 = sqlite3SrcListAppend(pParse,0,&yymsp[0].minor.yy0,0); /*A-overwrites-X*/} + case 120: /* xfullname ::= nm */ +{yymsp[0].minor.yy203 = sqlite3SrcListAppend(pParse,0,&yymsp[0].minor.yy0,0); /*A-overwrites-X*/} break; - case 119: /* xfullname ::= nm DOT nm */ -{yymsp[-2].minor.yy131 = sqlite3SrcListAppend(pParse,0,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0); /*A-overwrites-X*/} + case 121: /* xfullname ::= nm DOT nm */ +{yymsp[-2].minor.yy203 = sqlite3SrcListAppend(pParse,0,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0); /*A-overwrites-X*/} break; - case 120: /* xfullname ::= nm DOT nm AS nm */ + case 122: /* xfullname ::= nm DOT nm AS nm */ { - yymsp[-4].minor.yy131 = sqlite3SrcListAppend(pParse,0,&yymsp[-4].minor.yy0,&yymsp[-2].minor.yy0); /*A-overwrites-X*/ - if( yymsp[-4].minor.yy131 ) yymsp[-4].minor.yy131->a[0].zAlias = sqlite3NameFromToken(pParse->db, &yymsp[0].minor.yy0); + yymsp[-4].minor.yy203 = sqlite3SrcListAppend(pParse,0,&yymsp[-4].minor.yy0,&yymsp[-2].minor.yy0); /*A-overwrites-X*/ + if( yymsp[-4].minor.yy203 ) yymsp[-4].minor.yy203->a[0].zAlias = sqlite3NameFromToken(pParse->db, &yymsp[0].minor.yy0); } break; - case 121: /* xfullname ::= nm AS nm */ + case 123: /* xfullname ::= nm AS nm */ { - yymsp[-2].minor.yy131 = sqlite3SrcListAppend(pParse,0,&yymsp[-2].minor.yy0,0); /*A-overwrites-X*/ - if( yymsp[-2].minor.yy131 ) yymsp[-2].minor.yy131->a[0].zAlias = sqlite3NameFromToken(pParse->db, &yymsp[0].minor.yy0); + yymsp[-2].minor.yy203 = sqlite3SrcListAppend(pParse,0,&yymsp[-2].minor.yy0,0); /*A-overwrites-X*/ + if( yymsp[-2].minor.yy203 ) yymsp[-2].minor.yy203->a[0].zAlias = sqlite3NameFromToken(pParse->db, &yymsp[0].minor.yy0); } break; - case 122: /* joinop ::= COMMA|JOIN */ -{ yymsp[0].minor.yy394 = JT_INNER; } + case 124: /* joinop ::= COMMA|JOIN */ +{ yymsp[0].minor.yy144 = JT_INNER; } break; - case 123: /* joinop ::= JOIN_KW JOIN */ -{yymsp[-1].minor.yy394 = sqlite3JoinType(pParse,&yymsp[-1].minor.yy0,0,0); /*X-overwrites-A*/} + case 125: /* joinop ::= JOIN_KW JOIN */ +{yymsp[-1].minor.yy144 = sqlite3JoinType(pParse,&yymsp[-1].minor.yy0,0,0); /*X-overwrites-A*/} break; - case 124: /* joinop ::= JOIN_KW nm JOIN */ -{yymsp[-2].minor.yy394 = sqlite3JoinType(pParse,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0,0); /*X-overwrites-A*/} + case 126: /* joinop ::= JOIN_KW nm JOIN */ +{yymsp[-2].minor.yy144 = sqlite3JoinType(pParse,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0,0); /*X-overwrites-A*/} break; - case 125: /* joinop ::= JOIN_KW nm nm JOIN */ -{yymsp[-3].minor.yy394 = sqlite3JoinType(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0);/*X-overwrites-A*/} + case 127: /* joinop ::= JOIN_KW nm nm JOIN */ +{yymsp[-3].minor.yy144 = sqlite3JoinType(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0);/*X-overwrites-A*/} break; - case 126: /* on_using ::= ON expr */ -{yymsp[-1].minor.yy561.pOn = yymsp[0].minor.yy528; yymsp[-1].minor.yy561.pUsing = 0;} + case 128: /* on_using ::= ON expr */ +{yymsp[-1].minor.yy269.pOn = yymsp[0].minor.yy454; yymsp[-1].minor.yy269.pUsing = 0;} break; - case 127: /* on_using ::= USING LP idlist RP */ -{yymsp[-3].minor.yy561.pOn = 0; yymsp[-3].minor.yy561.pUsing = yymsp[-1].minor.yy254;} + case 129: /* on_using ::= USING LP idlist RP */ +{yymsp[-3].minor.yy269.pOn = 0; yymsp[-3].minor.yy269.pUsing = yymsp[-1].minor.yy132;} break; - case 128: /* on_using ::= */ -{yymsp[1].minor.yy561.pOn = 0; yymsp[1].minor.yy561.pUsing = 0;} + case 130: /* on_using ::= */ +{yymsp[1].minor.yy269.pOn = 0; yymsp[1].minor.yy269.pUsing = 0;} break; - case 130: /* indexed_by ::= INDEXED BY nm */ + case 132: /* indexed_by ::= INDEXED BY nm */ {yymsp[-2].minor.yy0 = yymsp[0].minor.yy0;} break; - case 131: /* indexed_by ::= NOT INDEXED */ + case 133: /* indexed_by ::= NOT INDEXED */ {yymsp[-1].minor.yy0.z=0; yymsp[-1].minor.yy0.n=1;} break; - case 133: /* orderby_opt ::= ORDER BY sortlist */ - case 143: /* groupby_opt ::= GROUP BY nexprlist */ yytestcase(yyruleno==143); -{yymsp[-2].minor.yy322 = yymsp[0].minor.yy322;} + case 135: /* orderby_opt ::= ORDER BY sortlist */ + case 145: /* groupby_opt ::= GROUP BY nexprlist */ yytestcase(yyruleno==145); +{yymsp[-2].minor.yy14 = yymsp[0].minor.yy14;} break; - case 134: /* sortlist ::= sortlist COMMA expr sortorder nulls */ + case 136: /* sortlist ::= sortlist COMMA expr sortorder nulls */ { - yymsp[-4].minor.yy322 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy322,yymsp[-2].minor.yy528); - sqlite3ExprListSetSortOrder(yymsp[-4].minor.yy322,yymsp[-1].minor.yy394,yymsp[0].minor.yy394); + yymsp[-4].minor.yy14 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy14,yymsp[-2].minor.yy454); + sqlite3ExprListSetSortOrder(yymsp[-4].minor.yy14,yymsp[-1].minor.yy144,yymsp[0].minor.yy144); } break; - case 135: /* sortlist ::= expr sortorder nulls */ + case 137: /* sortlist ::= expr sortorder nulls */ { - yymsp[-2].minor.yy322 = sqlite3ExprListAppend(pParse,0,yymsp[-2].minor.yy528); /*A-overwrites-Y*/ - sqlite3ExprListSetSortOrder(yymsp[-2].minor.yy322,yymsp[-1].minor.yy394,yymsp[0].minor.yy394); + yymsp[-2].minor.yy14 = sqlite3ExprListAppend(pParse,0,yymsp[-2].minor.yy454); /*A-overwrites-Y*/ + sqlite3ExprListSetSortOrder(yymsp[-2].minor.yy14,yymsp[-1].minor.yy144,yymsp[0].minor.yy144); } break; - case 136: /* sortorder ::= ASC */ -{yymsp[0].minor.yy394 = SQLITE_SO_ASC;} + case 138: /* sortorder ::= ASC */ +{yymsp[0].minor.yy144 = SQLITE_SO_ASC;} break; - case 137: /* sortorder ::= DESC */ -{yymsp[0].minor.yy394 = SQLITE_SO_DESC;} + case 139: /* sortorder ::= DESC */ +{yymsp[0].minor.yy144 = SQLITE_SO_DESC;} break; - case 138: /* sortorder ::= */ - case 141: /* nulls ::= */ yytestcase(yyruleno==141); -{yymsp[1].minor.yy394 = SQLITE_SO_UNDEFINED;} + case 140: /* sortorder ::= */ + case 143: /* nulls ::= */ yytestcase(yyruleno==143); +{yymsp[1].minor.yy144 = SQLITE_SO_UNDEFINED;} break; - case 139: /* nulls ::= NULLS FIRST */ -{yymsp[-1].minor.yy394 = SQLITE_SO_ASC;} + case 141: /* nulls ::= NULLS FIRST */ +{yymsp[-1].minor.yy144 = SQLITE_SO_ASC;} break; - case 140: /* nulls ::= NULLS LAST */ -{yymsp[-1].minor.yy394 = SQLITE_SO_DESC;} + case 142: /* nulls ::= NULLS LAST */ +{yymsp[-1].minor.yy144 = SQLITE_SO_DESC;} break; - case 144: /* having_opt ::= */ - case 146: /* limit_opt ::= */ yytestcase(yyruleno==146); - case 151: /* where_opt ::= */ yytestcase(yyruleno==151); - case 153: /* where_opt_ret ::= */ yytestcase(yyruleno==153); - case 230: /* case_else ::= */ yytestcase(yyruleno==230); - case 231: /* case_operand ::= */ yytestcase(yyruleno==231); - case 250: /* vinto ::= */ yytestcase(yyruleno==250); -{yymsp[1].minor.yy528 = 0;} + case 146: /* having_opt ::= */ + case 148: /* limit_opt ::= */ yytestcase(yyruleno==148); + case 153: /* where_opt ::= */ yytestcase(yyruleno==153); + case 155: /* where_opt_ret ::= */ yytestcase(yyruleno==155); + case 232: /* case_else ::= */ yytestcase(yyruleno==232); + case 233: /* case_operand ::= */ yytestcase(yyruleno==233); + case 252: /* vinto ::= */ yytestcase(yyruleno==252); +{yymsp[1].minor.yy454 = 0;} break; - case 145: /* having_opt ::= HAVING expr */ - case 152: /* where_opt ::= WHERE expr */ yytestcase(yyruleno==152); - case 154: /* where_opt_ret ::= WHERE expr */ yytestcase(yyruleno==154); - case 229: /* case_else ::= ELSE expr */ yytestcase(yyruleno==229); - case 249: /* vinto ::= INTO expr */ yytestcase(yyruleno==249); -{yymsp[-1].minor.yy528 = yymsp[0].minor.yy528;} + case 147: /* having_opt ::= HAVING expr */ + case 154: /* where_opt ::= WHERE expr */ yytestcase(yyruleno==154); + case 156: /* where_opt_ret ::= WHERE expr */ yytestcase(yyruleno==156); + case 231: /* case_else ::= ELSE expr */ yytestcase(yyruleno==231); + case 251: /* vinto ::= INTO expr */ yytestcase(yyruleno==251); +{yymsp[-1].minor.yy454 = yymsp[0].minor.yy454;} break; - case 147: /* limit_opt ::= LIMIT expr */ -{yymsp[-1].minor.yy528 = sqlite3PExpr(pParse,TK_LIMIT,yymsp[0].minor.yy528,0);} + case 149: /* limit_opt ::= LIMIT expr */ +{yymsp[-1].minor.yy454 = sqlite3PExpr(pParse,TK_LIMIT,yymsp[0].minor.yy454,0);} break; - case 148: /* limit_opt ::= LIMIT expr OFFSET expr */ -{yymsp[-3].minor.yy528 = sqlite3PExpr(pParse,TK_LIMIT,yymsp[-2].minor.yy528,yymsp[0].minor.yy528);} + case 150: /* limit_opt ::= LIMIT expr OFFSET expr */ +{yymsp[-3].minor.yy454 = sqlite3PExpr(pParse,TK_LIMIT,yymsp[-2].minor.yy454,yymsp[0].minor.yy454);} break; - case 149: /* limit_opt ::= LIMIT expr COMMA expr */ -{yymsp[-3].minor.yy528 = sqlite3PExpr(pParse,TK_LIMIT,yymsp[0].minor.yy528,yymsp[-2].minor.yy528);} + case 151: /* limit_opt ::= LIMIT expr COMMA expr */ +{yymsp[-3].minor.yy454 = sqlite3PExpr(pParse,TK_LIMIT,yymsp[0].minor.yy454,yymsp[-2].minor.yy454);} break; - case 150: /* cmd ::= with DELETE FROM xfullname indexed_opt where_opt_ret */ + case 152: /* cmd ::= with DELETE FROM xfullname indexed_opt where_opt_ret */ { - sqlite3SrcListIndexedBy(pParse, yymsp[-2].minor.yy131, &yymsp[-1].minor.yy0); - sqlite3DeleteFrom(pParse,yymsp[-2].minor.yy131,yymsp[0].minor.yy528,0,0); + sqlite3SrcListIndexedBy(pParse, yymsp[-2].minor.yy203, &yymsp[-1].minor.yy0); + sqlite3DeleteFrom(pParse,yymsp[-2].minor.yy203,yymsp[0].minor.yy454,0,0); } break; - case 155: /* where_opt_ret ::= RETURNING selcollist */ -{sqlite3AddReturning(pParse,yymsp[0].minor.yy322); yymsp[-1].minor.yy528 = 0;} + case 157: /* where_opt_ret ::= RETURNING selcollist */ +{sqlite3AddReturning(pParse,yymsp[0].minor.yy14); yymsp[-1].minor.yy454 = 0;} break; - case 156: /* where_opt_ret ::= WHERE expr RETURNING selcollist */ -{sqlite3AddReturning(pParse,yymsp[0].minor.yy322); yymsp[-3].minor.yy528 = yymsp[-2].minor.yy528;} + case 158: /* where_opt_ret ::= WHERE expr RETURNING selcollist */ +{sqlite3AddReturning(pParse,yymsp[0].minor.yy14); yymsp[-3].minor.yy454 = yymsp[-2].minor.yy454;} break; - case 157: /* cmd ::= with UPDATE orconf xfullname indexed_opt SET setlist from where_opt_ret */ + case 159: /* cmd ::= with UPDATE orconf xfullname indexed_opt SET setlist from where_opt_ret */ { - sqlite3SrcListIndexedBy(pParse, yymsp[-5].minor.yy131, &yymsp[-4].minor.yy0); - sqlite3ExprListCheckLength(pParse,yymsp[-2].minor.yy322,"set list"); - if( yymsp[-1].minor.yy131 ){ - SrcList *pFromClause = yymsp[-1].minor.yy131; + sqlite3SrcListIndexedBy(pParse, yymsp[-5].minor.yy203, &yymsp[-4].minor.yy0); + sqlite3ExprListCheckLength(pParse,yymsp[-2].minor.yy14,"set list"); + if( yymsp[-1].minor.yy203 ){ + SrcList *pFromClause = yymsp[-1].minor.yy203; if( pFromClause->nSrc>1 ){ Select *pSubquery; Token as; @@ -175355,92 +176754,92 @@ static YYACTIONTYPE yy_reduce( as.z = 0; pFromClause = sqlite3SrcListAppendFromTerm(pParse,0,0,0,&as,pSubquery,0); } - yymsp[-5].minor.yy131 = sqlite3SrcListAppendList(pParse, yymsp[-5].minor.yy131, pFromClause); + yymsp[-5].minor.yy203 = sqlite3SrcListAppendList(pParse, yymsp[-5].minor.yy203, pFromClause); } - sqlite3Update(pParse,yymsp[-5].minor.yy131,yymsp[-2].minor.yy322,yymsp[0].minor.yy528,yymsp[-6].minor.yy394,0,0,0); + sqlite3Update(pParse,yymsp[-5].minor.yy203,yymsp[-2].minor.yy14,yymsp[0].minor.yy454,yymsp[-6].minor.yy144,0,0,0); } break; - case 158: /* setlist ::= setlist COMMA nm EQ expr */ + case 160: /* setlist ::= setlist COMMA nm EQ expr */ { - yymsp[-4].minor.yy322 = sqlite3ExprListAppend(pParse, yymsp[-4].minor.yy322, yymsp[0].minor.yy528); - sqlite3ExprListSetName(pParse, yymsp[-4].minor.yy322, &yymsp[-2].minor.yy0, 1); + yymsp[-4].minor.yy14 = sqlite3ExprListAppend(pParse, yymsp[-4].minor.yy14, yymsp[0].minor.yy454); + sqlite3ExprListSetName(pParse, yymsp[-4].minor.yy14, &yymsp[-2].minor.yy0, 1); } break; - case 159: /* setlist ::= setlist COMMA LP idlist RP EQ expr */ + case 161: /* setlist ::= setlist COMMA LP idlist RP EQ expr */ { - yymsp[-6].minor.yy322 = sqlite3ExprListAppendVector(pParse, yymsp[-6].minor.yy322, yymsp[-3].minor.yy254, yymsp[0].minor.yy528); + yymsp[-6].minor.yy14 = sqlite3ExprListAppendVector(pParse, yymsp[-6].minor.yy14, yymsp[-3].minor.yy132, yymsp[0].minor.yy454); } break; - case 160: /* setlist ::= nm EQ expr */ + case 162: /* setlist ::= nm EQ expr */ { - yylhsminor.yy322 = sqlite3ExprListAppend(pParse, 0, yymsp[0].minor.yy528); - sqlite3ExprListSetName(pParse, yylhsminor.yy322, &yymsp[-2].minor.yy0, 1); + yylhsminor.yy14 = sqlite3ExprListAppend(pParse, 0, yymsp[0].minor.yy454); + sqlite3ExprListSetName(pParse, yylhsminor.yy14, &yymsp[-2].minor.yy0, 1); } - yymsp[-2].minor.yy322 = yylhsminor.yy322; + yymsp[-2].minor.yy14 = yylhsminor.yy14; break; - case 161: /* setlist ::= LP idlist RP EQ expr */ + case 163: /* setlist ::= LP idlist RP EQ expr */ { - yymsp[-4].minor.yy322 = sqlite3ExprListAppendVector(pParse, 0, yymsp[-3].minor.yy254, yymsp[0].minor.yy528); + yymsp[-4].minor.yy14 = sqlite3ExprListAppendVector(pParse, 0, yymsp[-3].minor.yy132, yymsp[0].minor.yy454); } break; - case 162: /* cmd ::= with insert_cmd INTO xfullname idlist_opt select upsert */ + case 164: /* cmd ::= with insert_cmd INTO xfullname idlist_opt select upsert */ { - sqlite3Insert(pParse, yymsp[-3].minor.yy131, yymsp[-1].minor.yy47, yymsp[-2].minor.yy254, yymsp[-5].minor.yy394, yymsp[0].minor.yy444); + sqlite3Insert(pParse, yymsp[-3].minor.yy203, yymsp[-1].minor.yy555, yymsp[-2].minor.yy132, yymsp[-5].minor.yy144, yymsp[0].minor.yy122); } break; - case 163: /* cmd ::= with insert_cmd INTO xfullname idlist_opt DEFAULT VALUES returning */ + case 165: /* cmd ::= with insert_cmd INTO xfullname idlist_opt DEFAULT VALUES returning */ { - sqlite3Insert(pParse, yymsp[-4].minor.yy131, 0, yymsp[-3].minor.yy254, yymsp[-6].minor.yy394, 0); + sqlite3Insert(pParse, yymsp[-4].minor.yy203, 0, yymsp[-3].minor.yy132, yymsp[-6].minor.yy144, 0); } break; - case 164: /* upsert ::= */ -{ yymsp[1].minor.yy444 = 0; } + case 166: /* upsert ::= */ +{ yymsp[1].minor.yy122 = 0; } break; - case 165: /* upsert ::= RETURNING selcollist */ -{ yymsp[-1].minor.yy444 = 0; sqlite3AddReturning(pParse,yymsp[0].minor.yy322); } + case 167: /* upsert ::= RETURNING selcollist */ +{ yymsp[-1].minor.yy122 = 0; sqlite3AddReturning(pParse,yymsp[0].minor.yy14); } break; - case 166: /* upsert ::= ON CONFLICT LP sortlist RP where_opt DO UPDATE SET setlist where_opt upsert */ -{ yymsp[-11].minor.yy444 = sqlite3UpsertNew(pParse->db,yymsp[-8].minor.yy322,yymsp[-6].minor.yy528,yymsp[-2].minor.yy322,yymsp[-1].minor.yy528,yymsp[0].minor.yy444);} + case 168: /* upsert ::= ON CONFLICT LP sortlist RP where_opt DO UPDATE SET setlist where_opt upsert */ +{ yymsp[-11].minor.yy122 = sqlite3UpsertNew(pParse->db,yymsp[-8].minor.yy14,yymsp[-6].minor.yy454,yymsp[-2].minor.yy14,yymsp[-1].minor.yy454,yymsp[0].minor.yy122);} break; - case 167: /* upsert ::= ON CONFLICT LP sortlist RP where_opt DO NOTHING upsert */ -{ yymsp[-8].minor.yy444 = sqlite3UpsertNew(pParse->db,yymsp[-5].minor.yy322,yymsp[-3].minor.yy528,0,0,yymsp[0].minor.yy444); } + case 169: /* upsert ::= ON CONFLICT LP sortlist RP where_opt DO NOTHING upsert */ +{ yymsp[-8].minor.yy122 = sqlite3UpsertNew(pParse->db,yymsp[-5].minor.yy14,yymsp[-3].minor.yy454,0,0,yymsp[0].minor.yy122); } break; - case 168: /* upsert ::= ON CONFLICT DO NOTHING returning */ -{ yymsp[-4].minor.yy444 = sqlite3UpsertNew(pParse->db,0,0,0,0,0); } + case 170: /* upsert ::= ON CONFLICT DO NOTHING returning */ +{ yymsp[-4].minor.yy122 = sqlite3UpsertNew(pParse->db,0,0,0,0,0); } break; - case 169: /* upsert ::= ON CONFLICT DO UPDATE SET setlist where_opt returning */ -{ yymsp[-7].minor.yy444 = sqlite3UpsertNew(pParse->db,0,0,yymsp[-2].minor.yy322,yymsp[-1].minor.yy528,0);} + case 171: /* upsert ::= ON CONFLICT DO UPDATE SET setlist where_opt returning */ +{ yymsp[-7].minor.yy122 = sqlite3UpsertNew(pParse->db,0,0,yymsp[-2].minor.yy14,yymsp[-1].minor.yy454,0);} break; - case 170: /* returning ::= RETURNING selcollist */ -{sqlite3AddReturning(pParse,yymsp[0].minor.yy322);} + case 172: /* returning ::= RETURNING selcollist */ +{sqlite3AddReturning(pParse,yymsp[0].minor.yy14);} break; - case 173: /* idlist_opt ::= */ -{yymsp[1].minor.yy254 = 0;} + case 175: /* idlist_opt ::= */ +{yymsp[1].minor.yy132 = 0;} break; - case 174: /* idlist_opt ::= LP idlist RP */ -{yymsp[-2].minor.yy254 = yymsp[-1].minor.yy254;} + case 176: /* idlist_opt ::= LP idlist RP */ +{yymsp[-2].minor.yy132 = yymsp[-1].minor.yy132;} break; - case 175: /* idlist ::= idlist COMMA nm */ -{yymsp[-2].minor.yy254 = sqlite3IdListAppend(pParse,yymsp[-2].minor.yy254,&yymsp[0].minor.yy0);} + case 177: /* idlist ::= idlist COMMA nm */ +{yymsp[-2].minor.yy132 = sqlite3IdListAppend(pParse,yymsp[-2].minor.yy132,&yymsp[0].minor.yy0);} break; - case 176: /* idlist ::= nm */ -{yymsp[0].minor.yy254 = sqlite3IdListAppend(pParse,0,&yymsp[0].minor.yy0); /*A-overwrites-Y*/} + case 178: /* idlist ::= nm */ +{yymsp[0].minor.yy132 = sqlite3IdListAppend(pParse,0,&yymsp[0].minor.yy0); /*A-overwrites-Y*/} break; - case 177: /* expr ::= LP expr RP */ -{yymsp[-2].minor.yy528 = yymsp[-1].minor.yy528;} + case 179: /* expr ::= LP expr RP */ +{yymsp[-2].minor.yy454 = yymsp[-1].minor.yy454;} break; - case 178: /* expr ::= ID|INDEXED|JOIN_KW */ -{yymsp[0].minor.yy528=tokenExpr(pParse,TK_ID,yymsp[0].minor.yy0); /*A-overwrites-X*/} + case 180: /* expr ::= ID|INDEXED|JOIN_KW */ +{yymsp[0].minor.yy454=tokenExpr(pParse,TK_ID,yymsp[0].minor.yy0); /*A-overwrites-X*/} break; - case 179: /* expr ::= nm DOT nm */ + case 181: /* expr ::= nm DOT nm */ { Expr *temp1 = tokenExpr(pParse,TK_ID,yymsp[-2].minor.yy0); Expr *temp2 = tokenExpr(pParse,TK_ID,yymsp[0].minor.yy0); - yylhsminor.yy528 = sqlite3PExpr(pParse, TK_DOT, temp1, temp2); + yylhsminor.yy454 = sqlite3PExpr(pParse, TK_DOT, temp1, temp2); } - yymsp[-2].minor.yy528 = yylhsminor.yy528; + yymsp[-2].minor.yy454 = yylhsminor.yy454; break; - case 180: /* expr ::= nm DOT nm DOT nm */ + case 182: /* expr ::= nm DOT nm DOT nm */ { Expr *temp1 = tokenExpr(pParse,TK_ID,yymsp[-4].minor.yy0); Expr *temp2 = tokenExpr(pParse,TK_ID,yymsp[-2].minor.yy0); @@ -175449,27 +176848,27 @@ static YYACTIONTYPE yy_reduce( if( IN_RENAME_OBJECT ){ sqlite3RenameTokenRemap(pParse, 0, temp1); } - yylhsminor.yy528 = sqlite3PExpr(pParse, TK_DOT, temp1, temp4); + yylhsminor.yy454 = sqlite3PExpr(pParse, TK_DOT, temp1, temp4); } - yymsp[-4].minor.yy528 = yylhsminor.yy528; + yymsp[-4].minor.yy454 = yylhsminor.yy454; break; - case 181: /* term ::= NULL|FLOAT|BLOB */ - case 182: /* term ::= STRING */ yytestcase(yyruleno==182); -{yymsp[0].minor.yy528=tokenExpr(pParse,yymsp[0].major,yymsp[0].minor.yy0); /*A-overwrites-X*/} + case 183: /* term ::= NULL|FLOAT|BLOB */ + case 184: /* term ::= STRING */ yytestcase(yyruleno==184); +{yymsp[0].minor.yy454=tokenExpr(pParse,yymsp[0].major,yymsp[0].minor.yy0); /*A-overwrites-X*/} break; - case 183: /* term ::= INTEGER */ + case 185: /* term ::= INTEGER */ { - yylhsminor.yy528 = sqlite3ExprAlloc(pParse->db, TK_INTEGER, &yymsp[0].minor.yy0, 1); - if( yylhsminor.yy528 ) yylhsminor.yy528->w.iOfst = (int)(yymsp[0].minor.yy0.z - pParse->zTail); + yylhsminor.yy454 = sqlite3ExprAlloc(pParse->db, TK_INTEGER, &yymsp[0].minor.yy0, 1); + if( yylhsminor.yy454 ) yylhsminor.yy454->w.iOfst = (int)(yymsp[0].minor.yy0.z - pParse->zTail); } - yymsp[0].minor.yy528 = yylhsminor.yy528; + yymsp[0].minor.yy454 = yylhsminor.yy454; break; - case 184: /* expr ::= VARIABLE */ + case 186: /* expr ::= VARIABLE */ { if( !(yymsp[0].minor.yy0.z[0]=='#' && sqlite3Isdigit(yymsp[0].minor.yy0.z[1])) ){ u32 n = yymsp[0].minor.yy0.n; - yymsp[0].minor.yy528 = tokenExpr(pParse, TK_VARIABLE, yymsp[0].minor.yy0); - sqlite3ExprAssignVarNumber(pParse, yymsp[0].minor.yy528, n); + yymsp[0].minor.yy454 = tokenExpr(pParse, TK_VARIABLE, yymsp[0].minor.yy0); + sqlite3ExprAssignVarNumber(pParse, yymsp[0].minor.yy454, n); }else{ /* When doing a nested parse, one can include terms in an expression ** that look like this: #1 #2 ... These terms refer to registers @@ -175478,194 +176877,203 @@ static YYACTIONTYPE yy_reduce( assert( t.n>=2 ); if( pParse->nested==0 ){ sqlite3ErrorMsg(pParse, "near \"%T\": syntax error", &t); - yymsp[0].minor.yy528 = 0; + yymsp[0].minor.yy454 = 0; }else{ - yymsp[0].minor.yy528 = sqlite3PExpr(pParse, TK_REGISTER, 0, 0); - if( yymsp[0].minor.yy528 ) sqlite3GetInt32(&t.z[1], &yymsp[0].minor.yy528->iTable); + yymsp[0].minor.yy454 = sqlite3PExpr(pParse, TK_REGISTER, 0, 0); + if( yymsp[0].minor.yy454 ) sqlite3GetInt32(&t.z[1], &yymsp[0].minor.yy454->iTable); } } } break; - case 185: /* expr ::= expr COLLATE ID|STRING */ + case 187: /* expr ::= expr COLLATE ID|STRING */ { - yymsp[-2].minor.yy528 = sqlite3ExprAddCollateToken(pParse, yymsp[-2].minor.yy528, &yymsp[0].minor.yy0, 1); + yymsp[-2].minor.yy454 = sqlite3ExprAddCollateToken(pParse, yymsp[-2].minor.yy454, &yymsp[0].minor.yy0, 1); } break; - case 186: /* expr ::= CAST LP expr AS typetoken RP */ + case 188: /* expr ::= CAST LP expr AS typetoken RP */ { - yymsp[-5].minor.yy528 = sqlite3ExprAlloc(pParse->db, TK_CAST, &yymsp[-1].minor.yy0, 1); - sqlite3ExprAttachSubtrees(pParse->db, yymsp[-5].minor.yy528, yymsp[-3].minor.yy528, 0); + yymsp[-5].minor.yy454 = sqlite3ExprAlloc(pParse->db, TK_CAST, &yymsp[-1].minor.yy0, 1); + sqlite3ExprAttachSubtrees(pParse->db, yymsp[-5].minor.yy454, yymsp[-3].minor.yy454, 0); } break; - case 187: /* expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP */ + case 189: /* expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP */ { - yylhsminor.yy528 = sqlite3ExprFunction(pParse, yymsp[-1].minor.yy322, &yymsp[-4].minor.yy0, yymsp[-2].minor.yy394); + yylhsminor.yy454 = sqlite3ExprFunction(pParse, yymsp[-1].minor.yy14, &yymsp[-4].minor.yy0, yymsp[-2].minor.yy144); } - yymsp[-4].minor.yy528 = yylhsminor.yy528; + yymsp[-4].minor.yy454 = yylhsminor.yy454; break; - case 188: /* expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist ORDER BY sortlist RP */ + case 190: /* expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist ORDER BY sortlist RP */ { - yylhsminor.yy528 = sqlite3ExprFunction(pParse, yymsp[-4].minor.yy322, &yymsp[-7].minor.yy0, yymsp[-5].minor.yy394); - sqlite3ExprAddFunctionOrderBy(pParse, yylhsminor.yy528, yymsp[-1].minor.yy322); + yylhsminor.yy454 = sqlite3ExprFunction(pParse, yymsp[-4].minor.yy14, &yymsp[-7].minor.yy0, yymsp[-5].minor.yy144); + sqlite3ExprAddFunctionOrderBy(pParse, yylhsminor.yy454, yymsp[-1].minor.yy14); } - yymsp[-7].minor.yy528 = yylhsminor.yy528; + yymsp[-7].minor.yy454 = yylhsminor.yy454; break; - case 189: /* expr ::= ID|INDEXED|JOIN_KW LP STAR RP */ + case 191: /* expr ::= ID|INDEXED|JOIN_KW LP STAR RP */ { - yylhsminor.yy528 = sqlite3ExprFunction(pParse, 0, &yymsp[-3].minor.yy0, 0); + yylhsminor.yy454 = sqlite3ExprFunction(pParse, 0, &yymsp[-3].minor.yy0, 0); } - yymsp[-3].minor.yy528 = yylhsminor.yy528; + yymsp[-3].minor.yy454 = yylhsminor.yy454; break; - case 190: /* expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP filter_over */ + case 192: /* expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP filter_over */ { - yylhsminor.yy528 = sqlite3ExprFunction(pParse, yymsp[-2].minor.yy322, &yymsp[-5].minor.yy0, yymsp[-3].minor.yy394); - sqlite3WindowAttach(pParse, yylhsminor.yy528, yymsp[0].minor.yy41); + yylhsminor.yy454 = sqlite3ExprFunction(pParse, yymsp[-2].minor.yy14, &yymsp[-5].minor.yy0, yymsp[-3].minor.yy144); + sqlite3WindowAttach(pParse, yylhsminor.yy454, yymsp[0].minor.yy211); } - yymsp[-5].minor.yy528 = yylhsminor.yy528; + yymsp[-5].minor.yy454 = yylhsminor.yy454; break; - case 191: /* expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist ORDER BY sortlist RP filter_over */ + case 193: /* expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist ORDER BY sortlist RP filter_over */ { - yylhsminor.yy528 = sqlite3ExprFunction(pParse, yymsp[-5].minor.yy322, &yymsp[-8].minor.yy0, yymsp[-6].minor.yy394); - sqlite3WindowAttach(pParse, yylhsminor.yy528, yymsp[0].minor.yy41); - sqlite3ExprAddFunctionOrderBy(pParse, yylhsminor.yy528, yymsp[-2].minor.yy322); + yylhsminor.yy454 = sqlite3ExprFunction(pParse, yymsp[-5].minor.yy14, &yymsp[-8].minor.yy0, yymsp[-6].minor.yy144); + sqlite3WindowAttach(pParse, yylhsminor.yy454, yymsp[0].minor.yy211); + sqlite3ExprAddFunctionOrderBy(pParse, yylhsminor.yy454, yymsp[-2].minor.yy14); } - yymsp[-8].minor.yy528 = yylhsminor.yy528; + yymsp[-8].minor.yy454 = yylhsminor.yy454; break; - case 192: /* expr ::= ID|INDEXED|JOIN_KW LP STAR RP filter_over */ + case 194: /* expr ::= ID|INDEXED|JOIN_KW LP STAR RP filter_over */ { - yylhsminor.yy528 = sqlite3ExprFunction(pParse, 0, &yymsp[-4].minor.yy0, 0); - sqlite3WindowAttach(pParse, yylhsminor.yy528, yymsp[0].minor.yy41); + yylhsminor.yy454 = sqlite3ExprFunction(pParse, 0, &yymsp[-4].minor.yy0, 0); + sqlite3WindowAttach(pParse, yylhsminor.yy454, yymsp[0].minor.yy211); } - yymsp[-4].minor.yy528 = yylhsminor.yy528; + yymsp[-4].minor.yy454 = yylhsminor.yy454; break; - case 193: /* term ::= CTIME_KW */ + case 195: /* term ::= CTIME_KW */ { - yylhsminor.yy528 = sqlite3ExprFunction(pParse, 0, &yymsp[0].minor.yy0, 0); + yylhsminor.yy454 = sqlite3ExprFunction(pParse, 0, &yymsp[0].minor.yy0, 0); } - yymsp[0].minor.yy528 = yylhsminor.yy528; + yymsp[0].minor.yy454 = yylhsminor.yy454; break; - case 194: /* expr ::= LP nexprlist COMMA expr RP */ + case 196: /* expr ::= LP nexprlist COMMA expr RP */ { - ExprList *pList = sqlite3ExprListAppend(pParse, yymsp[-3].minor.yy322, yymsp[-1].minor.yy528); - yymsp[-4].minor.yy528 = sqlite3PExpr(pParse, TK_VECTOR, 0, 0); - if( yymsp[-4].minor.yy528 ){ - yymsp[-4].minor.yy528->x.pList = pList; + ExprList *pList = sqlite3ExprListAppend(pParse, yymsp[-3].minor.yy14, yymsp[-1].minor.yy454); + yymsp[-4].minor.yy454 = sqlite3PExpr(pParse, TK_VECTOR, 0, 0); + if( yymsp[-4].minor.yy454 ){ + yymsp[-4].minor.yy454->x.pList = pList; if( ALWAYS(pList->nExpr) ){ - yymsp[-4].minor.yy528->flags |= pList->a[0].pExpr->flags & EP_Propagate; + yymsp[-4].minor.yy454->flags |= pList->a[0].pExpr->flags & EP_Propagate; } }else{ sqlite3ExprListDelete(pParse->db, pList); } } break; - case 195: /* expr ::= expr AND expr */ -{yymsp[-2].minor.yy528=sqlite3ExprAnd(pParse,yymsp[-2].minor.yy528,yymsp[0].minor.yy528);} + case 197: /* expr ::= expr AND expr */ +{yymsp[-2].minor.yy454=sqlite3ExprAnd(pParse,yymsp[-2].minor.yy454,yymsp[0].minor.yy454);} break; - case 196: /* expr ::= expr OR expr */ - case 197: /* expr ::= expr LT|GT|GE|LE expr */ yytestcase(yyruleno==197); - case 198: /* expr ::= expr EQ|NE expr */ yytestcase(yyruleno==198); - case 199: /* expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr */ yytestcase(yyruleno==199); - case 200: /* expr ::= expr PLUS|MINUS expr */ yytestcase(yyruleno==200); - case 201: /* expr ::= expr STAR|SLASH|REM expr */ yytestcase(yyruleno==201); - case 202: /* expr ::= expr CONCAT expr */ yytestcase(yyruleno==202); -{yymsp[-2].minor.yy528=sqlite3PExpr(pParse,yymsp[-1].major,yymsp[-2].minor.yy528,yymsp[0].minor.yy528);} + case 198: /* expr ::= expr OR expr */ + case 199: /* expr ::= expr LT|GT|GE|LE expr */ yytestcase(yyruleno==199); + case 200: /* expr ::= expr EQ|NE expr */ yytestcase(yyruleno==200); + case 201: /* expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr */ yytestcase(yyruleno==201); + case 202: /* expr ::= expr PLUS|MINUS expr */ yytestcase(yyruleno==202); + case 203: /* expr ::= expr STAR|SLASH|REM expr */ yytestcase(yyruleno==203); + case 204: /* expr ::= expr CONCAT expr */ yytestcase(yyruleno==204); +{yymsp[-2].minor.yy454=sqlite3PExpr(pParse,yymsp[-1].major,yymsp[-2].minor.yy454,yymsp[0].minor.yy454);} break; - case 203: /* likeop ::= NOT LIKE_KW|MATCH */ + case 205: /* likeop ::= NOT LIKE_KW|MATCH */ {yymsp[-1].minor.yy0=yymsp[0].minor.yy0; yymsp[-1].minor.yy0.n|=0x80000000; /*yymsp[-1].minor.yy0-overwrite-yymsp[0].minor.yy0*/} break; - case 204: /* expr ::= expr likeop expr */ + case 206: /* expr ::= expr likeop expr */ { ExprList *pList; int bNot = yymsp[-1].minor.yy0.n & 0x80000000; yymsp[-1].minor.yy0.n &= 0x7fffffff; - pList = sqlite3ExprListAppend(pParse,0, yymsp[0].minor.yy528); - pList = sqlite3ExprListAppend(pParse,pList, yymsp[-2].minor.yy528); - yymsp[-2].minor.yy528 = sqlite3ExprFunction(pParse, pList, &yymsp[-1].minor.yy0, 0); - if( bNot ) yymsp[-2].minor.yy528 = sqlite3PExpr(pParse, TK_NOT, yymsp[-2].minor.yy528, 0); - if( yymsp[-2].minor.yy528 ) yymsp[-2].minor.yy528->flags |= EP_InfixFunc; + pList = sqlite3ExprListAppend(pParse,0, yymsp[0].minor.yy454); + pList = sqlite3ExprListAppend(pParse,pList, yymsp[-2].minor.yy454); + yymsp[-2].minor.yy454 = sqlite3ExprFunction(pParse, pList, &yymsp[-1].minor.yy0, 0); + if( bNot ) yymsp[-2].minor.yy454 = sqlite3PExpr(pParse, TK_NOT, yymsp[-2].minor.yy454, 0); + if( yymsp[-2].minor.yy454 ) yymsp[-2].minor.yy454->flags |= EP_InfixFunc; } break; - case 205: /* expr ::= expr likeop expr ESCAPE expr */ + case 207: /* expr ::= expr likeop expr ESCAPE expr */ { ExprList *pList; int bNot = yymsp[-3].minor.yy0.n & 0x80000000; yymsp[-3].minor.yy0.n &= 0x7fffffff; - pList = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy528); - pList = sqlite3ExprListAppend(pParse,pList, yymsp[-4].minor.yy528); - pList = sqlite3ExprListAppend(pParse,pList, yymsp[0].minor.yy528); - yymsp[-4].minor.yy528 = sqlite3ExprFunction(pParse, pList, &yymsp[-3].minor.yy0, 0); - if( bNot ) yymsp[-4].minor.yy528 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy528, 0); - if( yymsp[-4].minor.yy528 ) yymsp[-4].minor.yy528->flags |= EP_InfixFunc; + pList = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy454); + pList = sqlite3ExprListAppend(pParse,pList, yymsp[-4].minor.yy454); + pList = sqlite3ExprListAppend(pParse,pList, yymsp[0].minor.yy454); + yymsp[-4].minor.yy454 = sqlite3ExprFunction(pParse, pList, &yymsp[-3].minor.yy0, 0); + if( bNot ) yymsp[-4].minor.yy454 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy454, 0); + if( yymsp[-4].minor.yy454 ) yymsp[-4].minor.yy454->flags |= EP_InfixFunc; } break; - case 206: /* expr ::= expr ISNULL|NOTNULL */ -{yymsp[-1].minor.yy528 = sqlite3PExpr(pParse,yymsp[0].major,yymsp[-1].minor.yy528,0);} + case 208: /* expr ::= expr ISNULL|NOTNULL */ +{yymsp[-1].minor.yy454 = sqlite3PExpr(pParse,yymsp[0].major,yymsp[-1].minor.yy454,0);} break; - case 207: /* expr ::= expr NOT NULL */ -{yymsp[-2].minor.yy528 = sqlite3PExpr(pParse,TK_NOTNULL,yymsp[-2].minor.yy528,0);} + case 209: /* expr ::= expr NOT NULL */ +{yymsp[-2].minor.yy454 = sqlite3PExpr(pParse,TK_NOTNULL,yymsp[-2].minor.yy454,0);} break; - case 208: /* expr ::= expr IS expr */ + case 210: /* expr ::= expr IS expr */ { - yymsp[-2].minor.yy528 = sqlite3PExpr(pParse,TK_IS,yymsp[-2].minor.yy528,yymsp[0].minor.yy528); - binaryToUnaryIfNull(pParse, yymsp[0].minor.yy528, yymsp[-2].minor.yy528, TK_ISNULL); + yymsp[-2].minor.yy454 = sqlite3PExpr(pParse,TK_IS,yymsp[-2].minor.yy454,yymsp[0].minor.yy454); + binaryToUnaryIfNull(pParse, yymsp[0].minor.yy454, yymsp[-2].minor.yy454, TK_ISNULL); } break; - case 209: /* expr ::= expr IS NOT expr */ + case 211: /* expr ::= expr IS NOT expr */ { - yymsp[-3].minor.yy528 = sqlite3PExpr(pParse,TK_ISNOT,yymsp[-3].minor.yy528,yymsp[0].minor.yy528); - binaryToUnaryIfNull(pParse, yymsp[0].minor.yy528, yymsp[-3].minor.yy528, TK_NOTNULL); + yymsp[-3].minor.yy454 = sqlite3PExpr(pParse,TK_ISNOT,yymsp[-3].minor.yy454,yymsp[0].minor.yy454); + binaryToUnaryIfNull(pParse, yymsp[0].minor.yy454, yymsp[-3].minor.yy454, TK_NOTNULL); } break; - case 210: /* expr ::= expr IS NOT DISTINCT FROM expr */ + case 212: /* expr ::= expr IS NOT DISTINCT FROM expr */ { - yymsp[-5].minor.yy528 = sqlite3PExpr(pParse,TK_IS,yymsp[-5].minor.yy528,yymsp[0].minor.yy528); - binaryToUnaryIfNull(pParse, yymsp[0].minor.yy528, yymsp[-5].minor.yy528, TK_ISNULL); + yymsp[-5].minor.yy454 = sqlite3PExpr(pParse,TK_IS,yymsp[-5].minor.yy454,yymsp[0].minor.yy454); + binaryToUnaryIfNull(pParse, yymsp[0].minor.yy454, yymsp[-5].minor.yy454, TK_ISNULL); } break; - case 211: /* expr ::= expr IS DISTINCT FROM expr */ + case 213: /* expr ::= expr IS DISTINCT FROM expr */ { - yymsp[-4].minor.yy528 = sqlite3PExpr(pParse,TK_ISNOT,yymsp[-4].minor.yy528,yymsp[0].minor.yy528); - binaryToUnaryIfNull(pParse, yymsp[0].minor.yy528, yymsp[-4].minor.yy528, TK_NOTNULL); + yymsp[-4].minor.yy454 = sqlite3PExpr(pParse,TK_ISNOT,yymsp[-4].minor.yy454,yymsp[0].minor.yy454); + binaryToUnaryIfNull(pParse, yymsp[0].minor.yy454, yymsp[-4].minor.yy454, TK_NOTNULL); } break; - case 212: /* expr ::= NOT expr */ - case 213: /* expr ::= BITNOT expr */ yytestcase(yyruleno==213); -{yymsp[-1].minor.yy528 = sqlite3PExpr(pParse, yymsp[-1].major, yymsp[0].minor.yy528, 0);/*A-overwrites-B*/} + case 214: /* expr ::= NOT expr */ + case 215: /* expr ::= BITNOT expr */ yytestcase(yyruleno==215); +{yymsp[-1].minor.yy454 = sqlite3PExpr(pParse, yymsp[-1].major, yymsp[0].minor.yy454, 0);/*A-overwrites-B*/} break; - case 214: /* expr ::= PLUS|MINUS expr */ + case 216: /* expr ::= PLUS|MINUS expr */ { - yymsp[-1].minor.yy528 = sqlite3PExpr(pParse, yymsp[-1].major==TK_PLUS ? TK_UPLUS : TK_UMINUS, yymsp[0].minor.yy528, 0); - /*A-overwrites-B*/ + Expr *p = yymsp[0].minor.yy454; + u8 op = yymsp[-1].major + (TK_UPLUS-TK_PLUS); + assert( TK_UPLUS>TK_PLUS ); + assert( TK_UMINUS == TK_MINUS + (TK_UPLUS - TK_PLUS) ); + if( p && p->op==TK_UPLUS ){ + p->op = op; + yymsp[-1].minor.yy454 = p; + }else{ + yymsp[-1].minor.yy454 = sqlite3PExpr(pParse, op, p, 0); + /*A-overwrites-B*/ + } } break; - case 215: /* expr ::= expr PTR expr */ + case 217: /* expr ::= expr PTR expr */ { - ExprList *pList = sqlite3ExprListAppend(pParse, 0, yymsp[-2].minor.yy528); - pList = sqlite3ExprListAppend(pParse, pList, yymsp[0].minor.yy528); - yylhsminor.yy528 = sqlite3ExprFunction(pParse, pList, &yymsp[-1].minor.yy0, 0); + ExprList *pList = sqlite3ExprListAppend(pParse, 0, yymsp[-2].minor.yy454); + pList = sqlite3ExprListAppend(pParse, pList, yymsp[0].minor.yy454); + yylhsminor.yy454 = sqlite3ExprFunction(pParse, pList, &yymsp[-1].minor.yy0, 0); } - yymsp[-2].minor.yy528 = yylhsminor.yy528; + yymsp[-2].minor.yy454 = yylhsminor.yy454; break; - case 216: /* between_op ::= BETWEEN */ - case 219: /* in_op ::= IN */ yytestcase(yyruleno==219); -{yymsp[0].minor.yy394 = 0;} + case 218: /* between_op ::= BETWEEN */ + case 221: /* in_op ::= IN */ yytestcase(yyruleno==221); +{yymsp[0].minor.yy144 = 0;} break; - case 218: /* expr ::= expr between_op expr AND expr */ + case 220: /* expr ::= expr between_op expr AND expr */ { - ExprList *pList = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy528); - pList = sqlite3ExprListAppend(pParse,pList, yymsp[0].minor.yy528); - yymsp[-4].minor.yy528 = sqlite3PExpr(pParse, TK_BETWEEN, yymsp[-4].minor.yy528, 0); - if( yymsp[-4].minor.yy528 ){ - yymsp[-4].minor.yy528->x.pList = pList; + ExprList *pList = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy454); + pList = sqlite3ExprListAppend(pParse,pList, yymsp[0].minor.yy454); + yymsp[-4].minor.yy454 = sqlite3PExpr(pParse, TK_BETWEEN, yymsp[-4].minor.yy454, 0); + if( yymsp[-4].minor.yy454 ){ + yymsp[-4].minor.yy454->x.pList = pList; }else{ sqlite3ExprListDelete(pParse->db, pList); } - if( yymsp[-3].minor.yy394 ) yymsp[-4].minor.yy528 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy528, 0); + if( yymsp[-3].minor.yy144 ) yymsp[-4].minor.yy454 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy454, 0); } break; - case 221: /* expr ::= expr in_op LP exprlist RP */ + case 223: /* expr ::= expr in_op LP exprlist RP */ { - if( yymsp[-1].minor.yy322==0 ){ + if( yymsp[-1].minor.yy14==0 ){ /* Expressions of the form ** ** expr1 IN () @@ -175674,208 +177082,208 @@ static YYACTIONTYPE yy_reduce( ** simplify to constants 0 (false) and 1 (true), respectively, ** regardless of the value of expr1. */ - sqlite3ExprUnmapAndDelete(pParse, yymsp[-4].minor.yy528); - yymsp[-4].minor.yy528 = sqlite3Expr(pParse->db, TK_STRING, yymsp[-3].minor.yy394 ? "true" : "false"); - if( yymsp[-4].minor.yy528 ) sqlite3ExprIdToTrueFalse(yymsp[-4].minor.yy528); + sqlite3ExprUnmapAndDelete(pParse, yymsp[-4].minor.yy454); + yymsp[-4].minor.yy454 = sqlite3Expr(pParse->db, TK_STRING, yymsp[-3].minor.yy144 ? "true" : "false"); + if( yymsp[-4].minor.yy454 ) sqlite3ExprIdToTrueFalse(yymsp[-4].minor.yy454); }else{ - Expr *pRHS = yymsp[-1].minor.yy322->a[0].pExpr; - if( yymsp[-1].minor.yy322->nExpr==1 && sqlite3ExprIsConstant(pRHS) && yymsp[-4].minor.yy528->op!=TK_VECTOR ){ - yymsp[-1].minor.yy322->a[0].pExpr = 0; - sqlite3ExprListDelete(pParse->db, yymsp[-1].minor.yy322); + Expr *pRHS = yymsp[-1].minor.yy14->a[0].pExpr; + if( yymsp[-1].minor.yy14->nExpr==1 && sqlite3ExprIsConstant(pParse,pRHS) && yymsp[-4].minor.yy454->op!=TK_VECTOR ){ + yymsp[-1].minor.yy14->a[0].pExpr = 0; + sqlite3ExprListDelete(pParse->db, yymsp[-1].minor.yy14); pRHS = sqlite3PExpr(pParse, TK_UPLUS, pRHS, 0); - yymsp[-4].minor.yy528 = sqlite3PExpr(pParse, TK_EQ, yymsp[-4].minor.yy528, pRHS); - }else if( yymsp[-1].minor.yy322->nExpr==1 && pRHS->op==TK_SELECT ){ - yymsp[-4].minor.yy528 = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy528, 0); - sqlite3PExprAddSelect(pParse, yymsp[-4].minor.yy528, pRHS->x.pSelect); + yymsp[-4].minor.yy454 = sqlite3PExpr(pParse, TK_EQ, yymsp[-4].minor.yy454, pRHS); + }else if( yymsp[-1].minor.yy14->nExpr==1 && pRHS->op==TK_SELECT ){ + yymsp[-4].minor.yy454 = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy454, 0); + sqlite3PExprAddSelect(pParse, yymsp[-4].minor.yy454, pRHS->x.pSelect); pRHS->x.pSelect = 0; - sqlite3ExprListDelete(pParse->db, yymsp[-1].minor.yy322); + sqlite3ExprListDelete(pParse->db, yymsp[-1].minor.yy14); }else{ - yymsp[-4].minor.yy528 = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy528, 0); - if( yymsp[-4].minor.yy528==0 ){ - sqlite3ExprListDelete(pParse->db, yymsp[-1].minor.yy322); - }else if( yymsp[-4].minor.yy528->pLeft->op==TK_VECTOR ){ - int nExpr = yymsp[-4].minor.yy528->pLeft->x.pList->nExpr; - Select *pSelectRHS = sqlite3ExprListToValues(pParse, nExpr, yymsp[-1].minor.yy322); + yymsp[-4].minor.yy454 = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy454, 0); + if( yymsp[-4].minor.yy454==0 ){ + sqlite3ExprListDelete(pParse->db, yymsp[-1].minor.yy14); + }else if( yymsp[-4].minor.yy454->pLeft->op==TK_VECTOR ){ + int nExpr = yymsp[-4].minor.yy454->pLeft->x.pList->nExpr; + Select *pSelectRHS = sqlite3ExprListToValues(pParse, nExpr, yymsp[-1].minor.yy14); if( pSelectRHS ){ parserDoubleLinkSelect(pParse, pSelectRHS); - sqlite3PExprAddSelect(pParse, yymsp[-4].minor.yy528, pSelectRHS); + sqlite3PExprAddSelect(pParse, yymsp[-4].minor.yy454, pSelectRHS); } }else{ - yymsp[-4].minor.yy528->x.pList = yymsp[-1].minor.yy322; - sqlite3ExprSetHeightAndFlags(pParse, yymsp[-4].minor.yy528); + yymsp[-4].minor.yy454->x.pList = yymsp[-1].minor.yy14; + sqlite3ExprSetHeightAndFlags(pParse, yymsp[-4].minor.yy454); } } - if( yymsp[-3].minor.yy394 ) yymsp[-4].minor.yy528 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy528, 0); + if( yymsp[-3].minor.yy144 ) yymsp[-4].minor.yy454 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy454, 0); } } break; - case 222: /* expr ::= LP select RP */ + case 224: /* expr ::= LP select RP */ { - yymsp[-2].minor.yy528 = sqlite3PExpr(pParse, TK_SELECT, 0, 0); - sqlite3PExprAddSelect(pParse, yymsp[-2].minor.yy528, yymsp[-1].minor.yy47); + yymsp[-2].minor.yy454 = sqlite3PExpr(pParse, TK_SELECT, 0, 0); + sqlite3PExprAddSelect(pParse, yymsp[-2].minor.yy454, yymsp[-1].minor.yy555); } break; - case 223: /* expr ::= expr in_op LP select RP */ + case 225: /* expr ::= expr in_op LP select RP */ { - yymsp[-4].minor.yy528 = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy528, 0); - sqlite3PExprAddSelect(pParse, yymsp[-4].minor.yy528, yymsp[-1].minor.yy47); - if( yymsp[-3].minor.yy394 ) yymsp[-4].minor.yy528 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy528, 0); + yymsp[-4].minor.yy454 = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy454, 0); + sqlite3PExprAddSelect(pParse, yymsp[-4].minor.yy454, yymsp[-1].minor.yy555); + if( yymsp[-3].minor.yy144 ) yymsp[-4].minor.yy454 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy454, 0); } break; - case 224: /* expr ::= expr in_op nm dbnm paren_exprlist */ + case 226: /* expr ::= expr in_op nm dbnm paren_exprlist */ { SrcList *pSrc = sqlite3SrcListAppend(pParse, 0,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0); Select *pSelect = sqlite3SelectNew(pParse, 0,pSrc,0,0,0,0,0,0); - if( yymsp[0].minor.yy322 ) sqlite3SrcListFuncArgs(pParse, pSelect ? pSrc : 0, yymsp[0].minor.yy322); - yymsp[-4].minor.yy528 = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy528, 0); - sqlite3PExprAddSelect(pParse, yymsp[-4].minor.yy528, pSelect); - if( yymsp[-3].minor.yy394 ) yymsp[-4].minor.yy528 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy528, 0); + if( yymsp[0].minor.yy14 ) sqlite3SrcListFuncArgs(pParse, pSelect ? pSrc : 0, yymsp[0].minor.yy14); + yymsp[-4].minor.yy454 = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy454, 0); + sqlite3PExprAddSelect(pParse, yymsp[-4].minor.yy454, pSelect); + if( yymsp[-3].minor.yy144 ) yymsp[-4].minor.yy454 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy454, 0); } break; - case 225: /* expr ::= EXISTS LP select RP */ + case 227: /* expr ::= EXISTS LP select RP */ { Expr *p; - p = yymsp[-3].minor.yy528 = sqlite3PExpr(pParse, TK_EXISTS, 0, 0); - sqlite3PExprAddSelect(pParse, p, yymsp[-1].minor.yy47); + p = yymsp[-3].minor.yy454 = sqlite3PExpr(pParse, TK_EXISTS, 0, 0); + sqlite3PExprAddSelect(pParse, p, yymsp[-1].minor.yy555); } break; - case 226: /* expr ::= CASE case_operand case_exprlist case_else END */ + case 228: /* expr ::= CASE case_operand case_exprlist case_else END */ { - yymsp[-4].minor.yy528 = sqlite3PExpr(pParse, TK_CASE, yymsp[-3].minor.yy528, 0); - if( yymsp[-4].minor.yy528 ){ - yymsp[-4].minor.yy528->x.pList = yymsp[-1].minor.yy528 ? sqlite3ExprListAppend(pParse,yymsp[-2].minor.yy322,yymsp[-1].minor.yy528) : yymsp[-2].minor.yy322; - sqlite3ExprSetHeightAndFlags(pParse, yymsp[-4].minor.yy528); + yymsp[-4].minor.yy454 = sqlite3PExpr(pParse, TK_CASE, yymsp[-3].minor.yy454, 0); + if( yymsp[-4].minor.yy454 ){ + yymsp[-4].minor.yy454->x.pList = yymsp[-1].minor.yy454 ? sqlite3ExprListAppend(pParse,yymsp[-2].minor.yy14,yymsp[-1].minor.yy454) : yymsp[-2].minor.yy14; + sqlite3ExprSetHeightAndFlags(pParse, yymsp[-4].minor.yy454); }else{ - sqlite3ExprListDelete(pParse->db, yymsp[-2].minor.yy322); - sqlite3ExprDelete(pParse->db, yymsp[-1].minor.yy528); + sqlite3ExprListDelete(pParse->db, yymsp[-2].minor.yy14); + sqlite3ExprDelete(pParse->db, yymsp[-1].minor.yy454); } } break; - case 227: /* case_exprlist ::= case_exprlist WHEN expr THEN expr */ + case 229: /* case_exprlist ::= case_exprlist WHEN expr THEN expr */ { - yymsp[-4].minor.yy322 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy322, yymsp[-2].minor.yy528); - yymsp[-4].minor.yy322 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy322, yymsp[0].minor.yy528); + yymsp[-4].minor.yy14 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy14, yymsp[-2].minor.yy454); + yymsp[-4].minor.yy14 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy14, yymsp[0].minor.yy454); } break; - case 228: /* case_exprlist ::= WHEN expr THEN expr */ + case 230: /* case_exprlist ::= WHEN expr THEN expr */ { - yymsp[-3].minor.yy322 = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy528); - yymsp[-3].minor.yy322 = sqlite3ExprListAppend(pParse,yymsp[-3].minor.yy322, yymsp[0].minor.yy528); + yymsp[-3].minor.yy14 = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy454); + yymsp[-3].minor.yy14 = sqlite3ExprListAppend(pParse,yymsp[-3].minor.yy14, yymsp[0].minor.yy454); } break; - case 233: /* nexprlist ::= nexprlist COMMA expr */ -{yymsp[-2].minor.yy322 = sqlite3ExprListAppend(pParse,yymsp[-2].minor.yy322,yymsp[0].minor.yy528);} + case 235: /* nexprlist ::= nexprlist COMMA expr */ +{yymsp[-2].minor.yy14 = sqlite3ExprListAppend(pParse,yymsp[-2].minor.yy14,yymsp[0].minor.yy454);} break; - case 234: /* nexprlist ::= expr */ -{yymsp[0].minor.yy322 = sqlite3ExprListAppend(pParse,0,yymsp[0].minor.yy528); /*A-overwrites-Y*/} + case 236: /* nexprlist ::= expr */ +{yymsp[0].minor.yy14 = sqlite3ExprListAppend(pParse,0,yymsp[0].minor.yy454); /*A-overwrites-Y*/} break; - case 236: /* paren_exprlist ::= LP exprlist RP */ - case 241: /* eidlist_opt ::= LP eidlist RP */ yytestcase(yyruleno==241); -{yymsp[-2].minor.yy322 = yymsp[-1].minor.yy322;} + case 238: /* paren_exprlist ::= LP exprlist RP */ + case 243: /* eidlist_opt ::= LP eidlist RP */ yytestcase(yyruleno==243); +{yymsp[-2].minor.yy14 = yymsp[-1].minor.yy14;} break; - case 237: /* cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt */ + case 239: /* cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt */ { sqlite3CreateIndex(pParse, &yymsp[-7].minor.yy0, &yymsp[-6].minor.yy0, - sqlite3SrcListAppend(pParse,0,&yymsp[-4].minor.yy0,0), yymsp[-2].minor.yy322, yymsp[-10].minor.yy394, - &yymsp[-11].minor.yy0, yymsp[0].minor.yy528, SQLITE_SO_ASC, yymsp[-8].minor.yy394, SQLITE_IDXTYPE_APPDEF); + sqlite3SrcListAppend(pParse,0,&yymsp[-4].minor.yy0,0), yymsp[-2].minor.yy14, yymsp[-10].minor.yy144, + &yymsp[-11].minor.yy0, yymsp[0].minor.yy454, SQLITE_SO_ASC, yymsp[-8].minor.yy144, SQLITE_IDXTYPE_APPDEF); if( IN_RENAME_OBJECT && pParse->pNewIndex ){ sqlite3RenameTokenMap(pParse, pParse->pNewIndex->zName, &yymsp[-4].minor.yy0); } } break; - case 238: /* uniqueflag ::= UNIQUE */ - case 280: /* raisetype ::= ABORT */ yytestcase(yyruleno==280); -{yymsp[0].minor.yy394 = OE_Abort;} + case 240: /* uniqueflag ::= UNIQUE */ + case 282: /* raisetype ::= ABORT */ yytestcase(yyruleno==282); +{yymsp[0].minor.yy144 = OE_Abort;} break; - case 239: /* uniqueflag ::= */ -{yymsp[1].minor.yy394 = OE_None;} + case 241: /* uniqueflag ::= */ +{yymsp[1].minor.yy144 = OE_None;} break; - case 242: /* eidlist ::= eidlist COMMA nm collate sortorder */ + case 244: /* eidlist ::= eidlist COMMA nm collate sortorder */ { - yymsp[-4].minor.yy322 = parserAddExprIdListTerm(pParse, yymsp[-4].minor.yy322, &yymsp[-2].minor.yy0, yymsp[-1].minor.yy394, yymsp[0].minor.yy394); + yymsp[-4].minor.yy14 = parserAddExprIdListTerm(pParse, yymsp[-4].minor.yy14, &yymsp[-2].minor.yy0, yymsp[-1].minor.yy144, yymsp[0].minor.yy144); } break; - case 243: /* eidlist ::= nm collate sortorder */ + case 245: /* eidlist ::= nm collate sortorder */ { - yymsp[-2].minor.yy322 = parserAddExprIdListTerm(pParse, 0, &yymsp[-2].minor.yy0, yymsp[-1].minor.yy394, yymsp[0].minor.yy394); /*A-overwrites-Y*/ + yymsp[-2].minor.yy14 = parserAddExprIdListTerm(pParse, 0, &yymsp[-2].minor.yy0, yymsp[-1].minor.yy144, yymsp[0].minor.yy144); /*A-overwrites-Y*/ } break; - case 246: /* cmd ::= DROP INDEX ifexists fullname */ -{sqlite3DropIndex(pParse, yymsp[0].minor.yy131, yymsp[-1].minor.yy394);} + case 248: /* cmd ::= DROP INDEX ifexists fullname */ +{sqlite3DropIndex(pParse, yymsp[0].minor.yy203, yymsp[-1].minor.yy144);} break; - case 247: /* cmd ::= VACUUM vinto */ -{sqlite3Vacuum(pParse,0,yymsp[0].minor.yy528);} + case 249: /* cmd ::= VACUUM vinto */ +{sqlite3Vacuum(pParse,0,yymsp[0].minor.yy454);} break; - case 248: /* cmd ::= VACUUM nm vinto */ -{sqlite3Vacuum(pParse,&yymsp[-1].minor.yy0,yymsp[0].minor.yy528);} + case 250: /* cmd ::= VACUUM nm vinto */ +{sqlite3Vacuum(pParse,&yymsp[-1].minor.yy0,yymsp[0].minor.yy454);} break; - case 251: /* cmd ::= PRAGMA nm dbnm */ + case 253: /* cmd ::= PRAGMA nm dbnm */ {sqlite3Pragma(pParse,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0,0,0);} break; - case 252: /* cmd ::= PRAGMA nm dbnm EQ nmnum */ + case 254: /* cmd ::= PRAGMA nm dbnm EQ nmnum */ {sqlite3Pragma(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0,0);} break; - case 253: /* cmd ::= PRAGMA nm dbnm LP nmnum RP */ + case 255: /* cmd ::= PRAGMA nm dbnm LP nmnum RP */ {sqlite3Pragma(pParse,&yymsp[-4].minor.yy0,&yymsp[-3].minor.yy0,&yymsp[-1].minor.yy0,0);} break; - case 254: /* cmd ::= PRAGMA nm dbnm EQ minus_num */ + case 256: /* cmd ::= PRAGMA nm dbnm EQ minus_num */ {sqlite3Pragma(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0,1);} break; - case 255: /* cmd ::= PRAGMA nm dbnm LP minus_num RP */ + case 257: /* cmd ::= PRAGMA nm dbnm LP minus_num RP */ {sqlite3Pragma(pParse,&yymsp[-4].minor.yy0,&yymsp[-3].minor.yy0,&yymsp[-1].minor.yy0,1);} break; - case 258: /* cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END */ + case 260: /* cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END */ { Token all; all.z = yymsp[-3].minor.yy0.z; all.n = (int)(yymsp[0].minor.yy0.z - yymsp[-3].minor.yy0.z) + yymsp[0].minor.yy0.n; - sqlite3FinishTrigger(pParse, yymsp[-1].minor.yy33, &all); + sqlite3FinishTrigger(pParse, yymsp[-1].minor.yy427, &all); } break; - case 259: /* trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause */ + case 261: /* trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause */ { - sqlite3BeginTrigger(pParse, &yymsp[-7].minor.yy0, &yymsp[-6].minor.yy0, yymsp[-5].minor.yy394, yymsp[-4].minor.yy180.a, yymsp[-4].minor.yy180.b, yymsp[-2].minor.yy131, yymsp[0].minor.yy528, yymsp[-10].minor.yy394, yymsp[-8].minor.yy394); + sqlite3BeginTrigger(pParse, &yymsp[-7].minor.yy0, &yymsp[-6].minor.yy0, yymsp[-5].minor.yy144, yymsp[-4].minor.yy286.a, yymsp[-4].minor.yy286.b, yymsp[-2].minor.yy203, yymsp[0].minor.yy454, yymsp[-10].minor.yy144, yymsp[-8].minor.yy144); yymsp[-10].minor.yy0 = (yymsp[-6].minor.yy0.n==0?yymsp[-7].minor.yy0:yymsp[-6].minor.yy0); /*A-overwrites-T*/ } break; - case 260: /* trigger_time ::= BEFORE|AFTER */ -{ yymsp[0].minor.yy394 = yymsp[0].major; /*A-overwrites-X*/ } + case 262: /* trigger_time ::= BEFORE|AFTER */ +{ yymsp[0].minor.yy144 = yymsp[0].major; /*A-overwrites-X*/ } break; - case 261: /* trigger_time ::= INSTEAD OF */ -{ yymsp[-1].minor.yy394 = TK_INSTEAD;} + case 263: /* trigger_time ::= INSTEAD OF */ +{ yymsp[-1].minor.yy144 = TK_INSTEAD;} break; - case 262: /* trigger_time ::= */ -{ yymsp[1].minor.yy394 = TK_BEFORE; } + case 264: /* trigger_time ::= */ +{ yymsp[1].minor.yy144 = TK_BEFORE; } break; - case 263: /* trigger_event ::= DELETE|INSERT */ - case 264: /* trigger_event ::= UPDATE */ yytestcase(yyruleno==264); -{yymsp[0].minor.yy180.a = yymsp[0].major; /*A-overwrites-X*/ yymsp[0].minor.yy180.b = 0;} + case 265: /* trigger_event ::= DELETE|INSERT */ + case 266: /* trigger_event ::= UPDATE */ yytestcase(yyruleno==266); +{yymsp[0].minor.yy286.a = yymsp[0].major; /*A-overwrites-X*/ yymsp[0].minor.yy286.b = 0;} break; - case 265: /* trigger_event ::= UPDATE OF idlist */ -{yymsp[-2].minor.yy180.a = TK_UPDATE; yymsp[-2].minor.yy180.b = yymsp[0].minor.yy254;} + case 267: /* trigger_event ::= UPDATE OF idlist */ +{yymsp[-2].minor.yy286.a = TK_UPDATE; yymsp[-2].minor.yy286.b = yymsp[0].minor.yy132;} break; - case 266: /* when_clause ::= */ - case 285: /* key_opt ::= */ yytestcase(yyruleno==285); -{ yymsp[1].minor.yy528 = 0; } + case 268: /* when_clause ::= */ + case 287: /* key_opt ::= */ yytestcase(yyruleno==287); +{ yymsp[1].minor.yy454 = 0; } break; - case 267: /* when_clause ::= WHEN expr */ - case 286: /* key_opt ::= KEY expr */ yytestcase(yyruleno==286); -{ yymsp[-1].minor.yy528 = yymsp[0].minor.yy528; } + case 269: /* when_clause ::= WHEN expr */ + case 288: /* key_opt ::= KEY expr */ yytestcase(yyruleno==288); +{ yymsp[-1].minor.yy454 = yymsp[0].minor.yy454; } break; - case 268: /* trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */ + case 270: /* trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */ { - assert( yymsp[-2].minor.yy33!=0 ); - yymsp[-2].minor.yy33->pLast->pNext = yymsp[-1].minor.yy33; - yymsp[-2].minor.yy33->pLast = yymsp[-1].minor.yy33; + assert( yymsp[-2].minor.yy427!=0 ); + yymsp[-2].minor.yy427->pLast->pNext = yymsp[-1].minor.yy427; + yymsp[-2].minor.yy427->pLast = yymsp[-1].minor.yy427; } break; - case 269: /* trigger_cmd_list ::= trigger_cmd SEMI */ + case 271: /* trigger_cmd_list ::= trigger_cmd SEMI */ { - assert( yymsp[-1].minor.yy33!=0 ); - yymsp[-1].minor.yy33->pLast = yymsp[-1].minor.yy33; + assert( yymsp[-1].minor.yy427!=0 ); + yymsp[-1].minor.yy427->pLast = yymsp[-1].minor.yy427; } break; - case 270: /* trnm ::= nm DOT nm */ + case 272: /* trnm ::= nm DOT nm */ { yymsp[-2].minor.yy0 = yymsp[0].minor.yy0; sqlite3ErrorMsg(pParse, @@ -175883,367 +177291,377 @@ static YYACTIONTYPE yy_reduce( "statements within triggers"); } break; - case 271: /* tridxby ::= INDEXED BY nm */ + case 273: /* tridxby ::= INDEXED BY nm */ { sqlite3ErrorMsg(pParse, "the INDEXED BY clause is not allowed on UPDATE or DELETE statements " "within triggers"); } break; - case 272: /* tridxby ::= NOT INDEXED */ + case 274: /* tridxby ::= NOT INDEXED */ { sqlite3ErrorMsg(pParse, "the NOT INDEXED clause is not allowed on UPDATE or DELETE statements " "within triggers"); } break; - case 273: /* trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist from where_opt scanpt */ -{yylhsminor.yy33 = sqlite3TriggerUpdateStep(pParse, &yymsp[-6].minor.yy0, yymsp[-2].minor.yy131, yymsp[-3].minor.yy322, yymsp[-1].minor.yy528, yymsp[-7].minor.yy394, yymsp[-8].minor.yy0.z, yymsp[0].minor.yy522);} - yymsp[-8].minor.yy33 = yylhsminor.yy33; + case 275: /* trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist from where_opt scanpt */ +{yylhsminor.yy427 = sqlite3TriggerUpdateStep(pParse, &yymsp[-6].minor.yy0, yymsp[-2].minor.yy203, yymsp[-3].minor.yy14, yymsp[-1].minor.yy454, yymsp[-7].minor.yy144, yymsp[-8].minor.yy0.z, yymsp[0].minor.yy168);} + yymsp[-8].minor.yy427 = yylhsminor.yy427; break; - case 274: /* trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt */ + case 276: /* trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt */ { - yylhsminor.yy33 = sqlite3TriggerInsertStep(pParse,&yymsp[-4].minor.yy0,yymsp[-3].minor.yy254,yymsp[-2].minor.yy47,yymsp[-6].minor.yy394,yymsp[-1].minor.yy444,yymsp[-7].minor.yy522,yymsp[0].minor.yy522);/*yylhsminor.yy33-overwrites-yymsp[-6].minor.yy394*/ + yylhsminor.yy427 = sqlite3TriggerInsertStep(pParse,&yymsp[-4].minor.yy0,yymsp[-3].minor.yy132,yymsp[-2].minor.yy555,yymsp[-6].minor.yy144,yymsp[-1].minor.yy122,yymsp[-7].minor.yy168,yymsp[0].minor.yy168);/*yylhsminor.yy427-overwrites-yymsp[-6].minor.yy144*/ } - yymsp[-7].minor.yy33 = yylhsminor.yy33; + yymsp[-7].minor.yy427 = yylhsminor.yy427; break; - case 275: /* trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt */ -{yylhsminor.yy33 = sqlite3TriggerDeleteStep(pParse, &yymsp[-3].minor.yy0, yymsp[-1].minor.yy528, yymsp[-5].minor.yy0.z, yymsp[0].minor.yy522);} - yymsp[-5].minor.yy33 = yylhsminor.yy33; + case 277: /* trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt */ +{yylhsminor.yy427 = sqlite3TriggerDeleteStep(pParse, &yymsp[-3].minor.yy0, yymsp[-1].minor.yy454, yymsp[-5].minor.yy0.z, yymsp[0].minor.yy168);} + yymsp[-5].minor.yy427 = yylhsminor.yy427; break; - case 276: /* trigger_cmd ::= scanpt select scanpt */ -{yylhsminor.yy33 = sqlite3TriggerSelectStep(pParse->db, yymsp[-1].minor.yy47, yymsp[-2].minor.yy522, yymsp[0].minor.yy522); /*yylhsminor.yy33-overwrites-yymsp[-1].minor.yy47*/} - yymsp[-2].minor.yy33 = yylhsminor.yy33; + case 278: /* trigger_cmd ::= scanpt select scanpt */ +{yylhsminor.yy427 = sqlite3TriggerSelectStep(pParse->db, yymsp[-1].minor.yy555, yymsp[-2].minor.yy168, yymsp[0].minor.yy168); /*yylhsminor.yy427-overwrites-yymsp[-1].minor.yy555*/} + yymsp[-2].minor.yy427 = yylhsminor.yy427; break; - case 277: /* expr ::= RAISE LP IGNORE RP */ + case 279: /* expr ::= RAISE LP IGNORE RP */ { - yymsp[-3].minor.yy528 = sqlite3PExpr(pParse, TK_RAISE, 0, 0); - if( yymsp[-3].minor.yy528 ){ - yymsp[-3].minor.yy528->affExpr = OE_Ignore; + yymsp[-3].minor.yy454 = sqlite3PExpr(pParse, TK_RAISE, 0, 0); + if( yymsp[-3].minor.yy454 ){ + yymsp[-3].minor.yy454->affExpr = OE_Ignore; } } break; - case 278: /* expr ::= RAISE LP raisetype COMMA nm RP */ + case 280: /* expr ::= RAISE LP raisetype COMMA nm RP */ { - yymsp[-5].minor.yy528 = sqlite3ExprAlloc(pParse->db, TK_RAISE, &yymsp[-1].minor.yy0, 1); - if( yymsp[-5].minor.yy528 ) { - yymsp[-5].minor.yy528->affExpr = (char)yymsp[-3].minor.yy394; + yymsp[-5].minor.yy454 = sqlite3ExprAlloc(pParse->db, TK_RAISE, &yymsp[-1].minor.yy0, 1); + if( yymsp[-5].minor.yy454 ) { + yymsp[-5].minor.yy454->affExpr = (char)yymsp[-3].minor.yy144; } } break; - case 279: /* raisetype ::= ROLLBACK */ -{yymsp[0].minor.yy394 = OE_Rollback;} + case 281: /* raisetype ::= ROLLBACK */ +{yymsp[0].minor.yy144 = OE_Rollback;} break; - case 281: /* raisetype ::= FAIL */ -{yymsp[0].minor.yy394 = OE_Fail;} + case 283: /* raisetype ::= FAIL */ +{yymsp[0].minor.yy144 = OE_Fail;} break; - case 282: /* cmd ::= DROP TRIGGER ifexists fullname */ + case 284: /* cmd ::= DROP TRIGGER ifexists fullname */ { - sqlite3DropTrigger(pParse,yymsp[0].minor.yy131,yymsp[-1].minor.yy394); + sqlite3DropTrigger(pParse,yymsp[0].minor.yy203,yymsp[-1].minor.yy144); } break; - case 283: /* cmd ::= ATTACH database_kw_opt expr AS expr key_opt */ + case 285: /* cmd ::= ATTACH database_kw_opt expr AS expr key_opt */ { - sqlite3Attach(pParse, yymsp[-3].minor.yy528, yymsp[-1].minor.yy528, yymsp[0].minor.yy528); + sqlite3Attach(pParse, yymsp[-3].minor.yy454, yymsp[-1].minor.yy454, yymsp[0].minor.yy454); } break; - case 284: /* cmd ::= DETACH database_kw_opt expr */ + case 286: /* cmd ::= DETACH database_kw_opt expr */ { - sqlite3Detach(pParse, yymsp[0].minor.yy528); + sqlite3Detach(pParse, yymsp[0].minor.yy454); } break; - case 287: /* cmd ::= REINDEX */ + case 289: /* cmd ::= REINDEX */ {sqlite3Reindex(pParse, 0, 0);} break; - case 288: /* cmd ::= REINDEX nm dbnm */ + case 290: /* cmd ::= REINDEX nm dbnm */ {sqlite3Reindex(pParse, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0);} break; - case 289: /* cmd ::= ANALYZE */ + case 291: /* cmd ::= ANALYZE */ {sqlite3Analyze(pParse, 0, 0);} break; - case 290: /* cmd ::= ANALYZE nm dbnm */ + case 292: /* cmd ::= ANALYZE nm dbnm */ {sqlite3Analyze(pParse, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0);} break; - case 291: /* cmd ::= ALTER TABLE fullname RENAME TO nm */ + case 293: /* cmd ::= ALTER TABLE fullname RENAME TO nm */ { - sqlite3AlterRenameTable(pParse,yymsp[-3].minor.yy131,&yymsp[0].minor.yy0); + sqlite3AlterRenameTable(pParse,yymsp[-3].minor.yy203,&yymsp[0].minor.yy0); } break; - case 292: /* cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist */ + case 294: /* cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist */ { yymsp[-1].minor.yy0.n = (int)(pParse->sLastToken.z-yymsp[-1].minor.yy0.z) + pParse->sLastToken.n; sqlite3AlterFinishAddColumn(pParse, &yymsp[-1].minor.yy0); } break; - case 293: /* cmd ::= ALTER TABLE fullname DROP kwcolumn_opt nm */ + case 295: /* cmd ::= ALTER TABLE fullname DROP kwcolumn_opt nm */ { - sqlite3AlterDropColumn(pParse, yymsp[-3].minor.yy131, &yymsp[0].minor.yy0); + sqlite3AlterDropColumn(pParse, yymsp[-3].minor.yy203, &yymsp[0].minor.yy0); } break; - case 294: /* add_column_fullname ::= fullname */ + case 296: /* add_column_fullname ::= fullname */ { disableLookaside(pParse); - sqlite3AlterBeginAddColumn(pParse, yymsp[0].minor.yy131); + sqlite3AlterBeginAddColumn(pParse, yymsp[0].minor.yy203); } break; - case 295: /* cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm */ + case 297: /* cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm */ { - sqlite3AlterRenameColumn(pParse, yymsp[-5].minor.yy131, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); + sqlite3AlterRenameColumn(pParse, yymsp[-5].minor.yy203, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); } break; - case 296: /* cmd ::= create_vtab */ + case 298: /* cmd ::= create_vtab */ {sqlite3VtabFinishParse(pParse,0);} break; - case 297: /* cmd ::= create_vtab LP vtabarglist RP */ + case 299: /* cmd ::= create_vtab LP vtabarglist RP */ {sqlite3VtabFinishParse(pParse,&yymsp[0].minor.yy0);} break; - case 298: /* create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm */ + case 300: /* create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm */ { - sqlite3VtabBeginParse(pParse, &yymsp[-3].minor.yy0, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, yymsp[-4].minor.yy394); + sqlite3VtabBeginParse(pParse, &yymsp[-3].minor.yy0, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, yymsp[-4].minor.yy144); } break; - case 299: /* vtabarg ::= */ + case 301: /* vtabarg ::= */ {sqlite3VtabArgInit(pParse);} break; - case 300: /* vtabargtoken ::= ANY */ - case 301: /* vtabargtoken ::= lp anylist RP */ yytestcase(yyruleno==301); - case 302: /* lp ::= LP */ yytestcase(yyruleno==302); + case 302: /* vtabargtoken ::= ANY */ + case 303: /* vtabargtoken ::= lp anylist RP */ yytestcase(yyruleno==303); + case 304: /* lp ::= LP */ yytestcase(yyruleno==304); {sqlite3VtabArgExtend(pParse,&yymsp[0].minor.yy0);} break; - case 303: /* with ::= WITH wqlist */ - case 304: /* with ::= WITH RECURSIVE wqlist */ yytestcase(yyruleno==304); -{ sqlite3WithPush(pParse, yymsp[0].minor.yy521, 1); } + case 305: /* with ::= WITH wqlist */ + case 306: /* with ::= WITH RECURSIVE wqlist */ yytestcase(yyruleno==306); +{ sqlite3WithPush(pParse, yymsp[0].minor.yy59, 1); } break; - case 305: /* wqas ::= AS */ -{yymsp[0].minor.yy516 = M10d_Any;} + case 307: /* wqas ::= AS */ +{yymsp[0].minor.yy462 = M10d_Any;} break; - case 306: /* wqas ::= AS MATERIALIZED */ -{yymsp[-1].minor.yy516 = M10d_Yes;} + case 308: /* wqas ::= AS MATERIALIZED */ +{yymsp[-1].minor.yy462 = M10d_Yes;} break; - case 307: /* wqas ::= AS NOT MATERIALIZED */ -{yymsp[-2].minor.yy516 = M10d_No;} + case 309: /* wqas ::= AS NOT MATERIALIZED */ +{yymsp[-2].minor.yy462 = M10d_No;} break; - case 308: /* wqitem ::= nm eidlist_opt wqas LP select RP */ + case 310: /* wqitem ::= withnm eidlist_opt wqas LP select RP */ { - yymsp[-5].minor.yy385 = sqlite3CteNew(pParse, &yymsp[-5].minor.yy0, yymsp[-4].minor.yy322, yymsp[-1].minor.yy47, yymsp[-3].minor.yy516); /*A-overwrites-X*/ + yymsp[-5].minor.yy67 = sqlite3CteNew(pParse, &yymsp[-5].minor.yy0, yymsp[-4].minor.yy14, yymsp[-1].minor.yy555, yymsp[-3].minor.yy462); /*A-overwrites-X*/ } break; - case 309: /* wqlist ::= wqitem */ + case 311: /* withnm ::= nm */ +{pParse->bHasWith = 1;} + break; + case 312: /* wqlist ::= wqitem */ { - yymsp[0].minor.yy521 = sqlite3WithAdd(pParse, 0, yymsp[0].minor.yy385); /*A-overwrites-X*/ + yymsp[0].minor.yy59 = sqlite3WithAdd(pParse, 0, yymsp[0].minor.yy67); /*A-overwrites-X*/ } break; - case 310: /* wqlist ::= wqlist COMMA wqitem */ + case 313: /* wqlist ::= wqlist COMMA wqitem */ { - yymsp[-2].minor.yy521 = sqlite3WithAdd(pParse, yymsp[-2].minor.yy521, yymsp[0].minor.yy385); + yymsp[-2].minor.yy59 = sqlite3WithAdd(pParse, yymsp[-2].minor.yy59, yymsp[0].minor.yy67); } break; - case 311: /* windowdefn_list ::= windowdefn_list COMMA windowdefn */ + case 314: /* windowdefn_list ::= windowdefn_list COMMA windowdefn */ { - assert( yymsp[0].minor.yy41!=0 ); - sqlite3WindowChain(pParse, yymsp[0].minor.yy41, yymsp[-2].minor.yy41); - yymsp[0].minor.yy41->pNextWin = yymsp[-2].minor.yy41; - yylhsminor.yy41 = yymsp[0].minor.yy41; + assert( yymsp[0].minor.yy211!=0 ); + sqlite3WindowChain(pParse, yymsp[0].minor.yy211, yymsp[-2].minor.yy211); + yymsp[0].minor.yy211->pNextWin = yymsp[-2].minor.yy211; + yylhsminor.yy211 = yymsp[0].minor.yy211; } - yymsp[-2].minor.yy41 = yylhsminor.yy41; + yymsp[-2].minor.yy211 = yylhsminor.yy211; break; - case 312: /* windowdefn ::= nm AS LP window RP */ + case 315: /* windowdefn ::= nm AS LP window RP */ { - if( ALWAYS(yymsp[-1].minor.yy41) ){ - yymsp[-1].minor.yy41->zName = sqlite3DbStrNDup(pParse->db, yymsp[-4].minor.yy0.z, yymsp[-4].minor.yy0.n); + if( ALWAYS(yymsp[-1].minor.yy211) ){ + yymsp[-1].minor.yy211->zName = sqlite3DbStrNDup(pParse->db, yymsp[-4].minor.yy0.z, yymsp[-4].minor.yy0.n); } - yylhsminor.yy41 = yymsp[-1].minor.yy41; + yylhsminor.yy211 = yymsp[-1].minor.yy211; } - yymsp[-4].minor.yy41 = yylhsminor.yy41; + yymsp[-4].minor.yy211 = yylhsminor.yy211; break; - case 313: /* window ::= PARTITION BY nexprlist orderby_opt frame_opt */ + case 316: /* window ::= PARTITION BY nexprlist orderby_opt frame_opt */ { - yymsp[-4].minor.yy41 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy41, yymsp[-2].minor.yy322, yymsp[-1].minor.yy322, 0); + yymsp[-4].minor.yy211 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy211, yymsp[-2].minor.yy14, yymsp[-1].minor.yy14, 0); } break; - case 314: /* window ::= nm PARTITION BY nexprlist orderby_opt frame_opt */ + case 317: /* window ::= nm PARTITION BY nexprlist orderby_opt frame_opt */ { - yylhsminor.yy41 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy41, yymsp[-2].minor.yy322, yymsp[-1].minor.yy322, &yymsp[-5].minor.yy0); + yylhsminor.yy211 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy211, yymsp[-2].minor.yy14, yymsp[-1].minor.yy14, &yymsp[-5].minor.yy0); } - yymsp[-5].minor.yy41 = yylhsminor.yy41; + yymsp[-5].minor.yy211 = yylhsminor.yy211; break; - case 315: /* window ::= ORDER BY sortlist frame_opt */ + case 318: /* window ::= ORDER BY sortlist frame_opt */ { - yymsp[-3].minor.yy41 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy41, 0, yymsp[-1].minor.yy322, 0); + yymsp[-3].minor.yy211 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy211, 0, yymsp[-1].minor.yy14, 0); } break; - case 316: /* window ::= nm ORDER BY sortlist frame_opt */ + case 319: /* window ::= nm ORDER BY sortlist frame_opt */ { - yylhsminor.yy41 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy41, 0, yymsp[-1].minor.yy322, &yymsp[-4].minor.yy0); + yylhsminor.yy211 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy211, 0, yymsp[-1].minor.yy14, &yymsp[-4].minor.yy0); } - yymsp[-4].minor.yy41 = yylhsminor.yy41; + yymsp[-4].minor.yy211 = yylhsminor.yy211; break; - case 317: /* window ::= nm frame_opt */ + case 320: /* window ::= nm frame_opt */ { - yylhsminor.yy41 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy41, 0, 0, &yymsp[-1].minor.yy0); + yylhsminor.yy211 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy211, 0, 0, &yymsp[-1].minor.yy0); } - yymsp[-1].minor.yy41 = yylhsminor.yy41; + yymsp[-1].minor.yy211 = yylhsminor.yy211; break; - case 318: /* frame_opt ::= */ + case 321: /* frame_opt ::= */ { - yymsp[1].minor.yy41 = sqlite3WindowAlloc(pParse, 0, TK_UNBOUNDED, 0, TK_CURRENT, 0, 0); + yymsp[1].minor.yy211 = sqlite3WindowAlloc(pParse, 0, TK_UNBOUNDED, 0, TK_CURRENT, 0, 0); } break; - case 319: /* frame_opt ::= range_or_rows frame_bound_s frame_exclude_opt */ + case 322: /* frame_opt ::= range_or_rows frame_bound_s frame_exclude_opt */ { - yylhsminor.yy41 = sqlite3WindowAlloc(pParse, yymsp[-2].minor.yy394, yymsp[-1].minor.yy595.eType, yymsp[-1].minor.yy595.pExpr, TK_CURRENT, 0, yymsp[0].minor.yy516); + yylhsminor.yy211 = sqlite3WindowAlloc(pParse, yymsp[-2].minor.yy144, yymsp[-1].minor.yy509.eType, yymsp[-1].minor.yy509.pExpr, TK_CURRENT, 0, yymsp[0].minor.yy462); } - yymsp[-2].minor.yy41 = yylhsminor.yy41; + yymsp[-2].minor.yy211 = yylhsminor.yy211; break; - case 320: /* frame_opt ::= range_or_rows BETWEEN frame_bound_s AND frame_bound_e frame_exclude_opt */ + case 323: /* frame_opt ::= range_or_rows BETWEEN frame_bound_s AND frame_bound_e frame_exclude_opt */ { - yylhsminor.yy41 = sqlite3WindowAlloc(pParse, yymsp[-5].minor.yy394, yymsp[-3].minor.yy595.eType, yymsp[-3].minor.yy595.pExpr, yymsp[-1].minor.yy595.eType, yymsp[-1].minor.yy595.pExpr, yymsp[0].minor.yy516); + yylhsminor.yy211 = sqlite3WindowAlloc(pParse, yymsp[-5].minor.yy144, yymsp[-3].minor.yy509.eType, yymsp[-3].minor.yy509.pExpr, yymsp[-1].minor.yy509.eType, yymsp[-1].minor.yy509.pExpr, yymsp[0].minor.yy462); } - yymsp[-5].minor.yy41 = yylhsminor.yy41; + yymsp[-5].minor.yy211 = yylhsminor.yy211; break; - case 322: /* frame_bound_s ::= frame_bound */ - case 324: /* frame_bound_e ::= frame_bound */ yytestcase(yyruleno==324); -{yylhsminor.yy595 = yymsp[0].minor.yy595;} - yymsp[0].minor.yy595 = yylhsminor.yy595; + case 325: /* frame_bound_s ::= frame_bound */ + case 327: /* frame_bound_e ::= frame_bound */ yytestcase(yyruleno==327); +{yylhsminor.yy509 = yymsp[0].minor.yy509;} + yymsp[0].minor.yy509 = yylhsminor.yy509; break; - case 323: /* frame_bound_s ::= UNBOUNDED PRECEDING */ - case 325: /* frame_bound_e ::= UNBOUNDED FOLLOWING */ yytestcase(yyruleno==325); - case 327: /* frame_bound ::= CURRENT ROW */ yytestcase(yyruleno==327); -{yylhsminor.yy595.eType = yymsp[-1].major; yylhsminor.yy595.pExpr = 0;} - yymsp[-1].minor.yy595 = yylhsminor.yy595; + case 326: /* frame_bound_s ::= UNBOUNDED PRECEDING */ + case 328: /* frame_bound_e ::= UNBOUNDED FOLLOWING */ yytestcase(yyruleno==328); + case 330: /* frame_bound ::= CURRENT ROW */ yytestcase(yyruleno==330); +{yylhsminor.yy509.eType = yymsp[-1].major; yylhsminor.yy509.pExpr = 0;} + yymsp[-1].minor.yy509 = yylhsminor.yy509; break; - case 326: /* frame_bound ::= expr PRECEDING|FOLLOWING */ -{yylhsminor.yy595.eType = yymsp[0].major; yylhsminor.yy595.pExpr = yymsp[-1].minor.yy528;} - yymsp[-1].minor.yy595 = yylhsminor.yy595; + case 329: /* frame_bound ::= expr PRECEDING|FOLLOWING */ +{yylhsminor.yy509.eType = yymsp[0].major; yylhsminor.yy509.pExpr = yymsp[-1].minor.yy454;} + yymsp[-1].minor.yy509 = yylhsminor.yy509; break; - case 328: /* frame_exclude_opt ::= */ -{yymsp[1].minor.yy516 = 0;} + case 331: /* frame_exclude_opt ::= */ +{yymsp[1].minor.yy462 = 0;} break; - case 329: /* frame_exclude_opt ::= EXCLUDE frame_exclude */ -{yymsp[-1].minor.yy516 = yymsp[0].minor.yy516;} + case 332: /* frame_exclude_opt ::= EXCLUDE frame_exclude */ +{yymsp[-1].minor.yy462 = yymsp[0].minor.yy462;} break; - case 330: /* frame_exclude ::= NO OTHERS */ - case 331: /* frame_exclude ::= CURRENT ROW */ yytestcase(yyruleno==331); -{yymsp[-1].minor.yy516 = yymsp[-1].major; /*A-overwrites-X*/} + case 333: /* frame_exclude ::= NO OTHERS */ + case 334: /* frame_exclude ::= CURRENT ROW */ yytestcase(yyruleno==334); +{yymsp[-1].minor.yy462 = yymsp[-1].major; /*A-overwrites-X*/} break; - case 332: /* frame_exclude ::= GROUP|TIES */ -{yymsp[0].minor.yy516 = yymsp[0].major; /*A-overwrites-X*/} + case 335: /* frame_exclude ::= GROUP|TIES */ +{yymsp[0].minor.yy462 = yymsp[0].major; /*A-overwrites-X*/} break; - case 333: /* window_clause ::= WINDOW windowdefn_list */ -{ yymsp[-1].minor.yy41 = yymsp[0].minor.yy41; } + case 336: /* window_clause ::= WINDOW windowdefn_list */ +{ yymsp[-1].minor.yy211 = yymsp[0].minor.yy211; } break; - case 334: /* filter_over ::= filter_clause over_clause */ + case 337: /* filter_over ::= filter_clause over_clause */ { - if( yymsp[0].minor.yy41 ){ - yymsp[0].minor.yy41->pFilter = yymsp[-1].minor.yy528; + if( yymsp[0].minor.yy211 ){ + yymsp[0].minor.yy211->pFilter = yymsp[-1].minor.yy454; }else{ - sqlite3ExprDelete(pParse->db, yymsp[-1].minor.yy528); + sqlite3ExprDelete(pParse->db, yymsp[-1].minor.yy454); } - yylhsminor.yy41 = yymsp[0].minor.yy41; + yylhsminor.yy211 = yymsp[0].minor.yy211; } - yymsp[-1].minor.yy41 = yylhsminor.yy41; + yymsp[-1].minor.yy211 = yylhsminor.yy211; break; - case 335: /* filter_over ::= over_clause */ + case 338: /* filter_over ::= over_clause */ { - yylhsminor.yy41 = yymsp[0].minor.yy41; + yylhsminor.yy211 = yymsp[0].minor.yy211; } - yymsp[0].minor.yy41 = yylhsminor.yy41; + yymsp[0].minor.yy211 = yylhsminor.yy211; break; - case 336: /* filter_over ::= filter_clause */ + case 339: /* filter_over ::= filter_clause */ { - yylhsminor.yy41 = (Window*)sqlite3DbMallocZero(pParse->db, sizeof(Window)); - if( yylhsminor.yy41 ){ - yylhsminor.yy41->eFrmType = TK_FILTER; - yylhsminor.yy41->pFilter = yymsp[0].minor.yy528; + yylhsminor.yy211 = (Window*)sqlite3DbMallocZero(pParse->db, sizeof(Window)); + if( yylhsminor.yy211 ){ + yylhsminor.yy211->eFrmType = TK_FILTER; + yylhsminor.yy211->pFilter = yymsp[0].minor.yy454; }else{ - sqlite3ExprDelete(pParse->db, yymsp[0].minor.yy528); + sqlite3ExprDelete(pParse->db, yymsp[0].minor.yy454); } } - yymsp[0].minor.yy41 = yylhsminor.yy41; + yymsp[0].minor.yy211 = yylhsminor.yy211; break; - case 337: /* over_clause ::= OVER LP window RP */ + case 340: /* over_clause ::= OVER LP window RP */ { - yymsp[-3].minor.yy41 = yymsp[-1].minor.yy41; - assert( yymsp[-3].minor.yy41!=0 ); + yymsp[-3].minor.yy211 = yymsp[-1].minor.yy211; + assert( yymsp[-3].minor.yy211!=0 ); } break; - case 338: /* over_clause ::= OVER nm */ + case 341: /* over_clause ::= OVER nm */ { - yymsp[-1].minor.yy41 = (Window*)sqlite3DbMallocZero(pParse->db, sizeof(Window)); - if( yymsp[-1].minor.yy41 ){ - yymsp[-1].minor.yy41->zName = sqlite3DbStrNDup(pParse->db, yymsp[0].minor.yy0.z, yymsp[0].minor.yy0.n); + yymsp[-1].minor.yy211 = (Window*)sqlite3DbMallocZero(pParse->db, sizeof(Window)); + if( yymsp[-1].minor.yy211 ){ + yymsp[-1].minor.yy211->zName = sqlite3DbStrNDup(pParse->db, yymsp[0].minor.yy0.z, yymsp[0].minor.yy0.n); } } break; - case 339: /* filter_clause ::= FILTER LP WHERE expr RP */ -{ yymsp[-4].minor.yy528 = yymsp[-1].minor.yy528; } + case 342: /* filter_clause ::= FILTER LP WHERE expr RP */ +{ yymsp[-4].minor.yy454 = yymsp[-1].minor.yy454; } + break; + case 343: /* term ::= QNUMBER */ +{ + yylhsminor.yy454=tokenExpr(pParse,yymsp[0].major,yymsp[0].minor.yy0); + sqlite3DequoteNumber(pParse, yylhsminor.yy454); +} + yymsp[0].minor.yy454 = yylhsminor.yy454; break; default: - /* (340) input ::= cmdlist */ yytestcase(yyruleno==340); - /* (341) cmdlist ::= cmdlist ecmd */ yytestcase(yyruleno==341); - /* (342) cmdlist ::= ecmd (OPTIMIZED OUT) */ assert(yyruleno!=342); - /* (343) ecmd ::= SEMI */ yytestcase(yyruleno==343); - /* (344) ecmd ::= cmdx SEMI */ yytestcase(yyruleno==344); - /* (345) ecmd ::= explain cmdx SEMI (NEVER REDUCES) */ assert(yyruleno!=345); - /* (346) trans_opt ::= */ yytestcase(yyruleno==346); - /* (347) trans_opt ::= TRANSACTION */ yytestcase(yyruleno==347); - /* (348) trans_opt ::= TRANSACTION nm */ yytestcase(yyruleno==348); - /* (349) savepoint_opt ::= SAVEPOINT */ yytestcase(yyruleno==349); - /* (350) savepoint_opt ::= */ yytestcase(yyruleno==350); - /* (351) cmd ::= create_table create_table_args */ yytestcase(yyruleno==351); - /* (352) table_option_set ::= table_option (OPTIMIZED OUT) */ assert(yyruleno!=352); - /* (353) columnlist ::= columnlist COMMA columnname carglist */ yytestcase(yyruleno==353); - /* (354) columnlist ::= columnname carglist */ yytestcase(yyruleno==354); - /* (355) nm ::= ID|INDEXED|JOIN_KW */ yytestcase(yyruleno==355); - /* (356) nm ::= STRING */ yytestcase(yyruleno==356); - /* (357) typetoken ::= typename */ yytestcase(yyruleno==357); - /* (358) typename ::= ID|STRING */ yytestcase(yyruleno==358); - /* (359) signed ::= plus_num (OPTIMIZED OUT) */ assert(yyruleno!=359); - /* (360) signed ::= minus_num (OPTIMIZED OUT) */ assert(yyruleno!=360); - /* (361) carglist ::= carglist ccons */ yytestcase(yyruleno==361); - /* (362) carglist ::= */ yytestcase(yyruleno==362); - /* (363) ccons ::= NULL onconf */ yytestcase(yyruleno==363); - /* (364) ccons ::= GENERATED ALWAYS AS generated */ yytestcase(yyruleno==364); - /* (365) ccons ::= AS generated */ yytestcase(yyruleno==365); - /* (366) conslist_opt ::= COMMA conslist */ yytestcase(yyruleno==366); - /* (367) conslist ::= conslist tconscomma tcons */ yytestcase(yyruleno==367); - /* (368) conslist ::= tcons (OPTIMIZED OUT) */ assert(yyruleno!=368); - /* (369) tconscomma ::= */ yytestcase(yyruleno==369); - /* (370) defer_subclause_opt ::= defer_subclause (OPTIMIZED OUT) */ assert(yyruleno!=370); - /* (371) resolvetype ::= raisetype (OPTIMIZED OUT) */ assert(yyruleno!=371); - /* (372) selectnowith ::= oneselect (OPTIMIZED OUT) */ assert(yyruleno!=372); - /* (373) oneselect ::= values */ yytestcase(yyruleno==373); - /* (374) sclp ::= selcollist COMMA */ yytestcase(yyruleno==374); - /* (375) as ::= ID|STRING */ yytestcase(yyruleno==375); - /* (376) indexed_opt ::= indexed_by (OPTIMIZED OUT) */ assert(yyruleno!=376); - /* (377) returning ::= */ yytestcase(yyruleno==377); - /* (378) expr ::= term (OPTIMIZED OUT) */ assert(yyruleno!=378); - /* (379) likeop ::= LIKE_KW|MATCH */ yytestcase(yyruleno==379); - /* (380) case_operand ::= expr */ yytestcase(yyruleno==380); - /* (381) exprlist ::= nexprlist */ yytestcase(yyruleno==381); - /* (382) nmnum ::= plus_num (OPTIMIZED OUT) */ assert(yyruleno!=382); - /* (383) nmnum ::= nm (OPTIMIZED OUT) */ assert(yyruleno!=383); - /* (384) nmnum ::= ON */ yytestcase(yyruleno==384); - /* (385) nmnum ::= DELETE */ yytestcase(yyruleno==385); - /* (386) nmnum ::= DEFAULT */ yytestcase(yyruleno==386); - /* (387) plus_num ::= INTEGER|FLOAT */ yytestcase(yyruleno==387); - /* (388) foreach_clause ::= */ yytestcase(yyruleno==388); - /* (389) foreach_clause ::= FOR EACH ROW */ yytestcase(yyruleno==389); - /* (390) trnm ::= nm */ yytestcase(yyruleno==390); - /* (391) tridxby ::= */ yytestcase(yyruleno==391); - /* (392) database_kw_opt ::= DATABASE */ yytestcase(yyruleno==392); - /* (393) database_kw_opt ::= */ yytestcase(yyruleno==393); - /* (394) kwcolumn_opt ::= */ yytestcase(yyruleno==394); - /* (395) kwcolumn_opt ::= COLUMNKW */ yytestcase(yyruleno==395); - /* (396) vtabarglist ::= vtabarg */ yytestcase(yyruleno==396); - /* (397) vtabarglist ::= vtabarglist COMMA vtabarg */ yytestcase(yyruleno==397); - /* (398) vtabarg ::= vtabarg vtabargtoken */ yytestcase(yyruleno==398); - /* (399) anylist ::= */ yytestcase(yyruleno==399); - /* (400) anylist ::= anylist LP anylist RP */ yytestcase(yyruleno==400); - /* (401) anylist ::= anylist ANY */ yytestcase(yyruleno==401); - /* (402) with ::= */ yytestcase(yyruleno==402); - /* (403) windowdefn_list ::= windowdefn (OPTIMIZED OUT) */ assert(yyruleno!=403); - /* (404) window ::= frame_opt (OPTIMIZED OUT) */ assert(yyruleno!=404); + /* (344) input ::= cmdlist */ yytestcase(yyruleno==344); + /* (345) cmdlist ::= cmdlist ecmd */ yytestcase(yyruleno==345); + /* (346) cmdlist ::= ecmd (OPTIMIZED OUT) */ assert(yyruleno!=346); + /* (347) ecmd ::= SEMI */ yytestcase(yyruleno==347); + /* (348) ecmd ::= cmdx SEMI */ yytestcase(yyruleno==348); + /* (349) ecmd ::= explain cmdx SEMI (NEVER REDUCES) */ assert(yyruleno!=349); + /* (350) trans_opt ::= */ yytestcase(yyruleno==350); + /* (351) trans_opt ::= TRANSACTION */ yytestcase(yyruleno==351); + /* (352) trans_opt ::= TRANSACTION nm */ yytestcase(yyruleno==352); + /* (353) savepoint_opt ::= SAVEPOINT */ yytestcase(yyruleno==353); + /* (354) savepoint_opt ::= */ yytestcase(yyruleno==354); + /* (355) cmd ::= create_table create_table_args */ yytestcase(yyruleno==355); + /* (356) table_option_set ::= table_option (OPTIMIZED OUT) */ assert(yyruleno!=356); + /* (357) columnlist ::= columnlist COMMA columnname carglist */ yytestcase(yyruleno==357); + /* (358) columnlist ::= columnname carglist */ yytestcase(yyruleno==358); + /* (359) nm ::= ID|INDEXED|JOIN_KW */ yytestcase(yyruleno==359); + /* (360) nm ::= STRING */ yytestcase(yyruleno==360); + /* (361) typetoken ::= typename */ yytestcase(yyruleno==361); + /* (362) typename ::= ID|STRING */ yytestcase(yyruleno==362); + /* (363) signed ::= plus_num (OPTIMIZED OUT) */ assert(yyruleno!=363); + /* (364) signed ::= minus_num (OPTIMIZED OUT) */ assert(yyruleno!=364); + /* (365) carglist ::= carglist ccons */ yytestcase(yyruleno==365); + /* (366) carglist ::= */ yytestcase(yyruleno==366); + /* (367) ccons ::= NULL onconf */ yytestcase(yyruleno==367); + /* (368) ccons ::= GENERATED ALWAYS AS generated */ yytestcase(yyruleno==368); + /* (369) ccons ::= AS generated */ yytestcase(yyruleno==369); + /* (370) conslist_opt ::= COMMA conslist */ yytestcase(yyruleno==370); + /* (371) conslist ::= conslist tconscomma tcons */ yytestcase(yyruleno==371); + /* (372) conslist ::= tcons (OPTIMIZED OUT) */ assert(yyruleno!=372); + /* (373) tconscomma ::= */ yytestcase(yyruleno==373); + /* (374) defer_subclause_opt ::= defer_subclause (OPTIMIZED OUT) */ assert(yyruleno!=374); + /* (375) resolvetype ::= raisetype (OPTIMIZED OUT) */ assert(yyruleno!=375); + /* (376) selectnowith ::= oneselect (OPTIMIZED OUT) */ assert(yyruleno!=376); + /* (377) oneselect ::= values */ yytestcase(yyruleno==377); + /* (378) sclp ::= selcollist COMMA */ yytestcase(yyruleno==378); + /* (379) as ::= ID|STRING */ yytestcase(yyruleno==379); + /* (380) indexed_opt ::= indexed_by (OPTIMIZED OUT) */ assert(yyruleno!=380); + /* (381) returning ::= */ yytestcase(yyruleno==381); + /* (382) expr ::= term (OPTIMIZED OUT) */ assert(yyruleno!=382); + /* (383) likeop ::= LIKE_KW|MATCH */ yytestcase(yyruleno==383); + /* (384) case_operand ::= expr */ yytestcase(yyruleno==384); + /* (385) exprlist ::= nexprlist */ yytestcase(yyruleno==385); + /* (386) nmnum ::= plus_num (OPTIMIZED OUT) */ assert(yyruleno!=386); + /* (387) nmnum ::= nm (OPTIMIZED OUT) */ assert(yyruleno!=387); + /* (388) nmnum ::= ON */ yytestcase(yyruleno==388); + /* (389) nmnum ::= DELETE */ yytestcase(yyruleno==389); + /* (390) nmnum ::= DEFAULT */ yytestcase(yyruleno==390); + /* (391) plus_num ::= INTEGER|FLOAT */ yytestcase(yyruleno==391); + /* (392) foreach_clause ::= */ yytestcase(yyruleno==392); + /* (393) foreach_clause ::= FOR EACH ROW */ yytestcase(yyruleno==393); + /* (394) trnm ::= nm */ yytestcase(yyruleno==394); + /* (395) tridxby ::= */ yytestcase(yyruleno==395); + /* (396) database_kw_opt ::= DATABASE */ yytestcase(yyruleno==396); + /* (397) database_kw_opt ::= */ yytestcase(yyruleno==397); + /* (398) kwcolumn_opt ::= */ yytestcase(yyruleno==398); + /* (399) kwcolumn_opt ::= COLUMNKW */ yytestcase(yyruleno==399); + /* (400) vtabarglist ::= vtabarg */ yytestcase(yyruleno==400); + /* (401) vtabarglist ::= vtabarglist COMMA vtabarg */ yytestcase(yyruleno==401); + /* (402) vtabarg ::= vtabarg vtabargtoken */ yytestcase(yyruleno==402); + /* (403) anylist ::= */ yytestcase(yyruleno==403); + /* (404) anylist ::= anylist LP anylist RP */ yytestcase(yyruleno==404); + /* (405) anylist ::= anylist ANY */ yytestcase(yyruleno==405); + /* (406) with ::= */ yytestcase(yyruleno==406); + /* (407) windowdefn_list ::= windowdefn (OPTIMIZED OUT) */ assert(yyruleno!=407); + /* (408) window ::= frame_opt (OPTIMIZED OUT) */ assert(yyruleno!=408); break; /********** End reduce actions ************************************************/ }; @@ -176430,19 +177848,12 @@ SQLITE_PRIVATE void sqlite3Parser( (int)(yypParser->yytos - yypParser->yystack)); } #endif -#if YYSTACKDEPTH>0 if( yypParser->yytos>=yypParser->yystackEnd ){ - yyStackOverflow(yypParser); - break; - } -#else - if( yypParser->yytos>=&yypParser->yystack[yypParser->yystksz-1] ){ if( yyGrowStack(yypParser) ){ yyStackOverflow(yypParser); break; } } -#endif } yyact = yy_reduce(yypParser,yyruleno,yymajor,yyminor sqlite3ParserCTX_PARAM); }else if( yyact <= YY_MAX_SHIFTREDUCE ){ @@ -177513,27 +178924,58 @@ SQLITE_PRIVATE int sqlite3GetToken(const unsigned char *z, int *tokenType){ *tokenType = TK_INTEGER; #ifndef SQLITE_OMIT_HEX_INTEGER if( z[0]=='0' && (z[1]=='x' || z[1]=='X') && sqlite3Isxdigit(z[2]) ){ - for(i=3; sqlite3Isxdigit(z[i]); i++){} - return i; - } + for(i=3; 1; i++){ + if( sqlite3Isxdigit(z[i])==0 ){ + if( z[i]==SQLITE_DIGIT_SEPARATOR ){ + *tokenType = TK_QNUMBER; + }else{ + break; + } + } + } + }else #endif - for(i=0; sqlite3Isdigit(z[i]); i++){} + { + for(i=0; 1; i++){ + if( sqlite3Isdigit(z[i])==0 ){ + if( z[i]==SQLITE_DIGIT_SEPARATOR ){ + *tokenType = TK_QNUMBER; + }else{ + break; + } + } + } #ifndef SQLITE_OMIT_FLOATING_POINT - if( z[i]=='.' ){ - i++; - while( sqlite3Isdigit(z[i]) ){ i++; } - *tokenType = TK_FLOAT; - } - if( (z[i]=='e' || z[i]=='E') && - ( sqlite3Isdigit(z[i+1]) - || ((z[i+1]=='+' || z[i+1]=='-') && sqlite3Isdigit(z[i+2])) - ) - ){ - i += 2; - while( sqlite3Isdigit(z[i]) ){ i++; } - *tokenType = TK_FLOAT; - } + if( z[i]=='.' ){ + if( *tokenType==TK_INTEGER ) *tokenType = TK_FLOAT; + for(i++; 1; i++){ + if( sqlite3Isdigit(z[i])==0 ){ + if( z[i]==SQLITE_DIGIT_SEPARATOR ){ + *tokenType = TK_QNUMBER; + }else{ + break; + } + } + } + } + if( (z[i]=='e' || z[i]=='E') && + ( sqlite3Isdigit(z[i+1]) + || ((z[i+1]=='+' || z[i+1]=='-') && sqlite3Isdigit(z[i+2])) + ) + ){ + if( *tokenType==TK_INTEGER ) *tokenType = TK_FLOAT; + for(i+=2; 1; i++){ + if( sqlite3Isdigit(z[i])==0 ){ + if( z[i]==SQLITE_DIGIT_SEPARATOR ){ + *tokenType = TK_QNUMBER; + }else{ + break; + } + } + } + } #endif + } while( IdChar(z[i]) ){ *tokenType = TK_ILLEGAL; i++; @@ -177698,10 +179140,13 @@ SQLITE_PRIVATE int sqlite3RunParser(Parse *pParse, const char *zSql){ if( tokenType>=TK_WINDOW ){ assert( tokenType==TK_SPACE || tokenType==TK_OVER || tokenType==TK_FILTER || tokenType==TK_ILLEGAL || tokenType==TK_WINDOW + || tokenType==TK_QNUMBER ); #else if( tokenType>=TK_SPACE ){ - assert( tokenType==TK_SPACE || tokenType==TK_ILLEGAL ); + assert( tokenType==TK_SPACE || tokenType==TK_ILLEGAL + || tokenType==TK_QNUMBER + ); #endif /* SQLITE_OMIT_WINDOWFUNC */ if( AtomicLoad(&db->u1.isInterrupted) ){ pParse->rc = SQLITE_INTERRUPT; @@ -177734,7 +179179,7 @@ SQLITE_PRIVATE int sqlite3RunParser(Parse *pParse, const char *zSql){ assert( n==6 ); tokenType = analyzeFilterKeyword((const u8*)&zSql[6], lastTokenParsed); #endif /* SQLITE_OMIT_WINDOWFUNC */ - }else{ + }else if( tokenType!=TK_QNUMBER ){ Token x; x.z = zSql; x.n = n; @@ -188727,22 +190172,24 @@ static int fts3IntegrityMethod( char **pzErr /* Write error message here */ ){ Fts3Table *p = (Fts3Table*)pVtab; - int rc; + int rc = SQLITE_OK; int bOk = 0; UNUSED_PARAMETER(isQuick); rc = sqlite3Fts3IntegrityCheck(p, &bOk); - assert( rc!=SQLITE_CORRUPT_VTAB || bOk==0 ); - if( rc!=SQLITE_OK && rc!=SQLITE_CORRUPT_VTAB ){ + assert( rc!=SQLITE_CORRUPT_VTAB ); + if( rc==SQLITE_ERROR || (rc&0xFF)==SQLITE_CORRUPT ){ *pzErr = sqlite3_mprintf("unable to validate the inverted index for" " FTS%d table %s.%s: %s", p->bFts4 ? 4 : 3, zSchema, zTabname, sqlite3_errstr(rc)); - }else if( bOk==0 ){ + if( *pzErr ) rc = SQLITE_OK; + }else if( rc==SQLITE_OK && bOk==0 ){ *pzErr = sqlite3_mprintf("malformed inverted index for FTS%d table %s.%s", p->bFts4 ? 4 : 3, zSchema, zTabname); + if( *pzErr==0 ) rc = SQLITE_NOMEM; } sqlite3Fts3SegmentsClose(p); - return SQLITE_OK; + return rc; } @@ -200404,7 +201851,12 @@ SQLITE_PRIVATE int sqlite3Fts3IntegrityCheck(Fts3Table *p, int *pbOk){ sqlite3_finalize(pStmt); } - *pbOk = (rc==SQLITE_OK && cksum1==cksum2); + if( rc==SQLITE_CORRUPT_VTAB ){ + rc = SQLITE_OK; + *pbOk = 0; + }else{ + *pbOk = (rc==SQLITE_OK && cksum1==cksum2); + } return rc; } @@ -201310,7 +202762,7 @@ static void fts3SnippetDetails( } mCover |= mPhrase; - for(j=0; jnToken; j++){ + for(j=0; jnToken && jnSnippet; j++){ mHighlight |= (mPos>>j); } @@ -203971,7 +205423,6 @@ static void jsonAppendRawNZ(JsonString *p, const char *zIn, u32 N){ } } - /* Append formatted text (not to exceed N bytes) to the JsonString. */ static void jsonPrintf(int N, JsonString *p, const char *zFormat, ...){ @@ -204029,6 +205480,40 @@ static void jsonAppendSeparator(JsonString *p){ jsonAppendChar(p, ','); } +/* c is a control character. Append the canonical JSON representation +** of that control character to p. +** +** This routine assumes that the output buffer has already been enlarged +** sufficiently to hold the worst-case encoding plus a nul terminator. +*/ +static void jsonAppendControlChar(JsonString *p, u8 c){ + static const char aSpecial[] = { + 0, 0, 0, 0, 0, 0, 0, 0, 'b', 't', 'n', 0, 'f', 'r', 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + }; + assert( sizeof(aSpecial)==32 ); + assert( aSpecial['\b']=='b' ); + assert( aSpecial['\f']=='f' ); + assert( aSpecial['\n']=='n' ); + assert( aSpecial['\r']=='r' ); + assert( aSpecial['\t']=='t' ); + assert( c>=0 && cnUsed+7 <= p->nAlloc ); + if( aSpecial[c] ){ + p->zBuf[p->nUsed] = '\\'; + p->zBuf[p->nUsed+1] = aSpecial[c]; + p->nUsed += 2; + }else{ + p->zBuf[p->nUsed] = '\\'; + p->zBuf[p->nUsed+1] = 'u'; + p->zBuf[p->nUsed+2] = '0'; + p->zBuf[p->nUsed+3] = '0'; + p->zBuf[p->nUsed+4] = "0123456789abcdef"[c>>4]; + p->zBuf[p->nUsed+5] = "0123456789abcdef"[c&0xf]; + p->nUsed += 6; + } +} + /* Append the N-byte string in zIn to the end of the JsonString string ** under construction. Enclose the string in double-quotes ("...") and ** escape any double-quotes or backslash characters contained within the @@ -204088,35 +205573,14 @@ static void jsonAppendString(JsonString *p, const char *zIn, u32 N){ } c = z[0]; if( c=='"' || c=='\\' ){ - json_simple_escape: if( (p->nUsed+N+3 > p->nAlloc) && jsonStringGrow(p,N+3)!=0 ) return; p->zBuf[p->nUsed++] = '\\'; p->zBuf[p->nUsed++] = c; }else if( c=='\'' ){ p->zBuf[p->nUsed++] = c; }else{ - static const char aSpecial[] = { - 0, 0, 0, 0, 0, 0, 0, 0, 'b', 't', 'n', 0, 'f', 'r', 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 - }; - assert( sizeof(aSpecial)==32 ); - assert( aSpecial['\b']=='b' ); - assert( aSpecial['\f']=='f' ); - assert( aSpecial['\n']=='n' ); - assert( aSpecial['\r']=='r' ); - assert( aSpecial['\t']=='t' ); - assert( c>=0 && cnUsed+N+7 > p->nAlloc) && jsonStringGrow(p,N+7)!=0 ) return; - p->zBuf[p->nUsed++] = '\\'; - p->zBuf[p->nUsed++] = 'u'; - p->zBuf[p->nUsed++] = '0'; - p->zBuf[p->nUsed++] = '0'; - p->zBuf[p->nUsed++] = "0123456789abcdef"[c>>4]; - p->zBuf[p->nUsed++] = "0123456789abcdef"[c&0xf]; + jsonAppendControlChar(p, c); } z++; N--; @@ -204817,7 +206281,10 @@ static u32 jsonbValidityCheck( if( !jsonIsOk[z[j]] && z[j]!='\'' ){ if( z[j]=='"' ){ if( x==JSONB_TEXTJ ) return j+1; - }else if( z[j]!='\\' || j+1>=k ){ + }else if( z[j]<=0x1f ){ + /* Control characters in JSON5 string literals are ok */ + if( x==JSONB_TEXTJ ) return j+1; + }else if( NEVER(z[j]!='\\') || j+1>=k ){ return j+1; }else if( strchr("\"\\/bfnrt",z[j+1])!=0 ){ j++; @@ -205112,9 +206579,14 @@ json_parse_restart: return -1; } }else if( c<=0x1f ){ - /* Control characters are not allowed in strings */ - pParse->iErr = j; - return -1; + if( c==0 ){ + pParse->iErr = j; + return -1; + } + /* Control characters are not allowed in canonical JSON string + ** literals, but are allowed in JSON5 string literals. */ + opcode = JSONB_TEXT5; + pParse->hasNonstd = 1; }else if( c=='"' ){ opcode = JSONB_TEXT5; } @@ -205330,6 +206802,7 @@ json_parse_restart: return i+4; } /* fall-through into the default case that checks for NaN */ + /* no break */ deliberate_fall_through } default: { u32 k; @@ -205598,7 +207071,7 @@ static u32 jsonTranslateBlobToText( zIn = (const char*)&pParse->aBlob[i+n]; jsonAppendChar(pOut, '"'); while( sz2>0 ){ - for(k=0; k0 ){ jsonAppendRawNZ(pOut, zIn, k); if( k>=sz2 ){ @@ -205613,6 +207086,13 @@ static u32 jsonTranslateBlobToText( sz2--; continue; } + if( zIn[0]<=0x1f ){ + if( pOut->nUsed+7>pOut->nAlloc && jsonStringGrow(pOut,7) ) break; + jsonAppendControlChar(pOut, zIn[0]); + zIn++; + sz2--; + continue; + } assert( zIn[0]=='\\' ); assert( sz2>=1 ); if( sz2<2 ){ @@ -205715,6 +207195,112 @@ static u32 jsonTranslateBlobToText( return i+n+sz; } +/* Context for recursion of json_pretty() +*/ +typedef struct JsonPretty JsonPretty; +struct JsonPretty { + JsonParse *pParse; /* The BLOB being rendered */ + JsonString *pOut; /* Generate pretty output into this string */ + const char *zIndent; /* Use this text for indentation */ + u32 szIndent; /* Bytes in zIndent[] */ + u32 nIndent; /* Current level of indentation */ +}; + +/* Append indentation to the pretty JSON under construction */ +static void jsonPrettyIndent(JsonPretty *pPretty){ + u32 jj; + for(jj=0; jjnIndent; jj++){ + jsonAppendRaw(pPretty->pOut, pPretty->zIndent, pPretty->szIndent); + } +} + +/* +** Translate the binary JSONB representation of JSON beginning at +** pParse->aBlob[i] into a JSON text string. Append the JSON +** text onto the end of pOut. Return the index in pParse->aBlob[] +** of the first byte past the end of the element that is translated. +** +** This is a variant of jsonTranslateBlobToText() that "pretty-prints" +** the output. Extra whitespace is inserted to make the JSON easier +** for humans to read. +** +** If an error is detected in the BLOB input, the pOut->eErr flag +** might get set to JSTRING_MALFORMED. But not all BLOB input errors +** are detected. So a malformed JSONB input might either result +** in an error, or in incorrect JSON. +** +** The pOut->eErr JSTRING_OOM flag is set on a OOM. +*/ +static u32 jsonTranslateBlobToPrettyText( + JsonPretty *pPretty, /* Pretty-printing context */ + u32 i /* Start rendering at this index */ +){ + u32 sz, n, j, iEnd; + const JsonParse *pParse = pPretty->pParse; + JsonString *pOut = pPretty->pOut; + n = jsonbPayloadSize(pParse, i, &sz); + if( n==0 ){ + pOut->eErr |= JSTRING_MALFORMED; + return pParse->nBlob+1; + } + switch( pParse->aBlob[i] & 0x0f ){ + case JSONB_ARRAY: { + j = i+n; + iEnd = j+sz; + jsonAppendChar(pOut, '['); + if( jnIndent++; + while( pOut->eErr==0 ){ + jsonPrettyIndent(pPretty); + j = jsonTranslateBlobToPrettyText(pPretty, j); + if( j>=iEnd ) break; + jsonAppendRawNZ(pOut, ",\n", 2); + } + jsonAppendChar(pOut, '\n'); + pPretty->nIndent--; + jsonPrettyIndent(pPretty); + } + jsonAppendChar(pOut, ']'); + i = iEnd; + break; + } + case JSONB_OBJECT: { + j = i+n; + iEnd = j+sz; + jsonAppendChar(pOut, '{'); + if( jnIndent++; + while( pOut->eErr==0 ){ + jsonPrettyIndent(pPretty); + j = jsonTranslateBlobToText(pParse, j, pOut); + if( j>iEnd ){ + pOut->eErr |= JSTRING_MALFORMED; + break; + } + jsonAppendRawNZ(pOut, ": ", 2); + j = jsonTranslateBlobToPrettyText(pPretty, j); + if( j>=iEnd ) break; + jsonAppendRawNZ(pOut, ",\n", 2); + } + jsonAppendChar(pOut, '\n'); + pPretty->nIndent--; + jsonPrettyIndent(pPretty); + } + jsonAppendChar(pOut, '}'); + i = iEnd; + break; + } + default: { + i = jsonTranslateBlobToText(pParse, i, pOut); + break; + } + } + return i; +} + + /* Return true if the input pJson ** ** For performance reasons, this routine does not do a detailed check of the @@ -206965,11 +208551,12 @@ static void jsonParseFunc( if( p==0 ) return; if( argc==1 ){ jsonDebugPrintBlob(p, 0, p->nBlob, 0, &out); - sqlite3_result_text64(ctx, out.zText, out.nChar, SQLITE_DYNAMIC, SQLITE_UTF8); + sqlite3_result_text64(ctx,out.zText,out.nChar,SQLITE_TRANSIENT,SQLITE_UTF8); }else{ jsonShowParse(p); } jsonParseFree(p); + sqlite3_str_reset(&out); } #endif /* SQLITE_DEBUG */ @@ -207068,13 +208655,6 @@ static void jsonArrayLengthFunc( jsonParseFree(p); } -/* True if the string is all digits */ -static int jsonAllDigits(const char *z, int n){ - int i; - for(i=0; i $[NUMBER] // Not PG. Purely for convenience */ jsonStringInit(&jx, ctx); - if( jsonAllDigits(zPath, nPath) ){ + if( sqlite3_value_type(argv[i])==SQLITE_INTEGER ){ jsonAppendRawNZ(&jx, "[", 1); jsonAppendRaw(&jx, zPath, nPath); jsonAppendRawNZ(&jx, "]", 2); @@ -207633,6 +209213,40 @@ json_type_done: jsonParseFree(p); } +/* +** json_pretty(JSON) +** json_pretty(JSON, INDENT) +** +** Return text that is a pretty-printed rendering of the input JSON. +** If the argument is not valid JSON, return NULL. +** +** The INDENT argument is text that is used for indentation. If omitted, +** it defaults to four spaces (the same as PostgreSQL). +*/ +static void jsonPrettyFunc( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + JsonString s; /* The output string */ + JsonPretty x; /* Pretty printing context */ + + memset(&x, 0, sizeof(x)); + x.pParse = jsonParseFuncArg(ctx, argv[0], 0); + if( x.pParse==0 ) return; + x.pOut = &s; + jsonStringInit(&s, ctx); + if( argc==1 || (x.zIndent = (const char*)sqlite3_value_text(argv[1]))==0 ){ + x.zIndent = " "; + x.szIndent = 4; + }else{ + x.szIndent = (u32)strlen(x.zIndent); + } + jsonTranslateBlobToPrettyText(&x, 0); + jsonReturnString(&s, 0, 0); + jsonParseFree(x.pParse); +} + /* ** json_valid(JSON) ** json_valid(JSON, FLAGS) @@ -208647,6 +210261,8 @@ SQLITE_PRIVATE void sqlite3RegisterJsonFunctions(void){ JFUNCTION(jsonb_object, -1,0,1, 1,1,0, jsonObjectFunc), JFUNCTION(json_patch, 2,1,1, 0,0,0, jsonPatchFunc), JFUNCTION(jsonb_patch, 2,1,0, 0,1,0, jsonPatchFunc), + JFUNCTION(json_pretty, 1,1,0, 0,0,0, jsonPrettyFunc), + JFUNCTION(json_pretty, 2,1,0, 0,0,0, jsonPrettyFunc), JFUNCTION(json_quote, 1,0,1, 1,0,0, jsonQuoteFunc), JFUNCTION(json_remove, -1,1,1, 0,0,0, jsonRemoveFunc), JFUNCTION(jsonb_remove, -1,1,0, 0,1,0, jsonRemoveFunc), @@ -210546,6 +212162,8 @@ static int deserializeGeometry(sqlite3_value *pValue, RtreeConstraint *pCons){ return SQLITE_OK; } +SQLITE_PRIVATE int sqlite3IntFloatCompare(i64,double); + /* ** Rtree virtual table module xFilter method. */ @@ -210575,7 +212193,8 @@ static int rtreeFilter( i64 iNode = 0; int eType = sqlite3_value_numeric_type(argv[0]); if( eType==SQLITE_INTEGER - || (eType==SQLITE_FLOAT && sqlite3_value_double(argv[0])==iRowid) + || (eType==SQLITE_FLOAT + && 0==sqlite3IntFloatCompare(iRowid,sqlite3_value_double(argv[0]))) ){ rc = findLeafNode(pRtree, iRowid, &pLeaf, &iNode); }else{ @@ -211930,6 +213549,7 @@ constraint: */ static int rtreeBeginTransaction(sqlite3_vtab *pVtab){ Rtree *pRtree = (Rtree *)pVtab; + assert( pRtree->inWrTrans==0 ); pRtree->inWrTrans = 1; return SQLITE_OK; } @@ -215484,7 +217104,7 @@ static void icuLoadCollation( UCollator *pUCollator; /* ICU library collation object */ int rc; /* Return code from sqlite3_create_collation_x() */ - assert(nArg==2); + assert(nArg==2 || nArg==3); (void)nArg; /* Unused parameter */ zLocale = (const char *)sqlite3_value_text(apArg[0]); zName = (const char *)sqlite3_value_text(apArg[1]); @@ -215499,7 +217119,39 @@ static void icuLoadCollation( return; } assert(p); - + if(nArg==3){ + const char *zOption = (const char*)sqlite3_value_text(apArg[2]); + static const struct { + const char *zName; + UColAttributeValue val; + } aStrength[] = { + { "PRIMARY", UCOL_PRIMARY }, + { "SECONDARY", UCOL_SECONDARY }, + { "TERTIARY", UCOL_TERTIARY }, + { "DEFAULT", UCOL_DEFAULT_STRENGTH }, + { "QUARTERNARY", UCOL_QUATERNARY }, + { "IDENTICAL", UCOL_IDENTICAL }, + }; + unsigned int i; + for(i=0; i=sizeof(aStrength)/sizeof(aStrength[0]) ){ + sqlite3_str *pStr = sqlite3_str_new(sqlite3_context_db_handle(p)); + sqlite3_str_appendf(pStr, + "unknown collation strength \"%s\" - should be one of:", + zOption); + for(i=0; ipTblIter, &p->zErrmsg); pIter->zTbl = 0; + pIter->zDataTbl = 0; }else{ pIter->zTbl = (const char*)sqlite3_column_text(pIter->pTblIter, 0); pIter->zDataTbl = (const char*)sqlite3_column_text(pIter->pTblIter,1); @@ -219452,7 +221107,7 @@ static i64 rbuShmChecksum(sqlite3rbu *p){ u32 volatile *ptr; p->rc = pDb->pMethods->xShmMap(pDb, 0, 32*1024, 0, (void volatile**)&ptr); if( p->rc==SQLITE_OK ){ - iRet = ((i64)ptr[10] << 32) + ptr[11]; + iRet = (i64)(((u64)ptr[10] << 32) + ptr[11]); } } return iRet; @@ -226923,14 +228578,14 @@ static int sessionChangesetNextOne( p->rc = sessionInputBuffer(&p->in, 2); if( p->rc!=SQLITE_OK ) return p->rc; + sessionDiscardData(&p->in); + p->in.iCurrent = p->in.iNext; + /* If the iterator is already at the end of the changeset, return DONE. */ if( p->in.iNext>=p->in.nData ){ return SQLITE_DONE; } - sessionDiscardData(&p->in); - p->in.iCurrent = p->in.iNext; - op = p->in.aData[p->in.iNext++]; while( op=='T' || op=='P' ){ if( pbNew ) *pbNew = 1; @@ -228665,6 +230320,7 @@ struct sqlite3_changegroup { int rc; /* Error code */ int bPatch; /* True to accumulate patchsets */ SessionTable *pList; /* List of tables in current patch */ + SessionBuffer rec; sqlite3 *db; /* Configured by changegroup_schema() */ char *zDb; /* Configured by changegroup_schema() */ @@ -228963,108 +230619,128 @@ static int sessionChangesetExtendRecord( } /* -** Add all changes in the changeset traversed by the iterator passed as -** the first argument to the changegroup hash tables. +** Locate or create a SessionTable object that may be used to add the +** change currently pointed to by iterator pIter to changegroup pGrp. +** If successful, set output variable (*ppTab) to point to the table +** object and return SQLITE_OK. Otherwise, if some error occurs, return +** an SQLite error code and leave (*ppTab) set to NULL. */ -static int sessionChangesetToHash( - sqlite3_changeset_iter *pIter, /* Iterator to read from */ - sqlite3_changegroup *pGrp, /* Changegroup object to add changeset to */ - int bRebase /* True if hash table is for rebasing */ +static int sessionChangesetFindTable( + sqlite3_changegroup *pGrp, + const char *zTab, + sqlite3_changeset_iter *pIter, + SessionTable **ppTab ){ - u8 *aRec; - int nRec; int rc = SQLITE_OK; SessionTable *pTab = 0; - SessionBuffer rec = {0, 0, 0}; + int nTab = (int)strlen(zTab); + u8 *abPK = 0; + int nCol = 0; - while( SQLITE_ROW==sessionChangesetNext(pIter, &aRec, &nRec, 0) ){ - const char *zNew; - int nCol; - int op; - int iHash; - int bIndirect; - SessionChange *pChange; - SessionChange *pExist = 0; - SessionChange **pp; + *ppTab = 0; + sqlite3changeset_pk(pIter, &abPK, &nCol); - /* Ensure that only changesets, or only patchsets, but not a mixture - ** of both, are being combined. It is an error to try to combine a - ** changeset and a patchset. */ - if( pGrp->pList==0 ){ - pGrp->bPatch = pIter->bPatchset; - }else if( pIter->bPatchset!=pGrp->bPatch ){ - rc = SQLITE_ERROR; - break; + /* Search the list for an existing table */ + for(pTab = pGrp->pList; pTab; pTab=pTab->pNext){ + if( 0==sqlite3_strnicmp(pTab->zName, zTab, nTab+1) ) break; + } + + /* If one was not found above, create a new table now */ + if( !pTab ){ + SessionTable **ppNew; + + pTab = sqlite3_malloc64(sizeof(SessionTable) + nCol + nTab+1); + if( !pTab ){ + return SQLITE_NOMEM; } + memset(pTab, 0, sizeof(SessionTable)); + pTab->nCol = nCol; + pTab->abPK = (u8*)&pTab[1]; + memcpy(pTab->abPK, abPK, nCol); + pTab->zName = (char*)&pTab->abPK[nCol]; + memcpy(pTab->zName, zTab, nTab+1); - sqlite3changeset_op(pIter, &zNew, &nCol, &op, &bIndirect); - if( !pTab || sqlite3_stricmp(zNew, pTab->zName) ){ - /* Search the list for a matching table */ - int nNew = (int)strlen(zNew); - u8 *abPK; - - sqlite3changeset_pk(pIter, &abPK, 0); - for(pTab = pGrp->pList; pTab; pTab=pTab->pNext){ - if( 0==sqlite3_strnicmp(pTab->zName, zNew, nNew+1) ) break; - } - if( !pTab ){ - SessionTable **ppTab; - - pTab = sqlite3_malloc64(sizeof(SessionTable) + nCol + nNew+1); - if( !pTab ){ - rc = SQLITE_NOMEM; - break; - } - memset(pTab, 0, sizeof(SessionTable)); - pTab->nCol = nCol; - pTab->abPK = (u8*)&pTab[1]; - memcpy(pTab->abPK, abPK, nCol); - pTab->zName = (char*)&pTab->abPK[nCol]; - memcpy(pTab->zName, zNew, nNew+1); - - if( pGrp->db ){ - pTab->nCol = 0; - rc = sessionInitTable(0, pTab, pGrp->db, pGrp->zDb); - if( rc ){ - assert( pTab->azCol==0 ); - sqlite3_free(pTab); - break; - } - } - - /* The new object must be linked on to the end of the list, not - ** simply added to the start of it. This is to ensure that the - ** tables within the output of sqlite3changegroup_output() are in - ** the right order. */ - for(ppTab=&pGrp->pList; *ppTab; ppTab=&(*ppTab)->pNext); - *ppTab = pTab; - } - - if( !sessionChangesetCheckCompat(pTab, nCol, abPK) ){ - rc = SQLITE_SCHEMA; - break; + if( pGrp->db ){ + pTab->nCol = 0; + rc = sessionInitTable(0, pTab, pGrp->db, pGrp->zDb); + if( rc ){ + assert( pTab->azCol==0 ); + sqlite3_free(pTab); + return rc; } } - if( nColnCol ){ - assert( pGrp->db ); - rc = sessionChangesetExtendRecord(pGrp, pTab, nCol, op, aRec, nRec, &rec); - if( rc ) break; - aRec = rec.aBuf; - nRec = rec.nBuf; - } + /* The new object must be linked on to the end of the list, not + ** simply added to the start of it. This is to ensure that the + ** tables within the output of sqlite3changegroup_output() are in + ** the right order. */ + for(ppNew=&pGrp->pList; *ppNew; ppNew=&(*ppNew)->pNext); + *ppNew = pTab; + } - if( sessionGrowHash(0, pIter->bPatchset, pTab) ){ - rc = SQLITE_NOMEM; - break; - } + /* Check that the table is compatible. */ + if( !sessionChangesetCheckCompat(pTab, nCol, abPK) ){ + rc = SQLITE_SCHEMA; + } + + *ppTab = pTab; + return rc; +} + +/* +** Add the change currently indicated by iterator pIter to the hash table +** belonging to changegroup pGrp. +*/ +static int sessionOneChangeToHash( + sqlite3_changegroup *pGrp, + sqlite3_changeset_iter *pIter, + int bRebase +){ + int rc = SQLITE_OK; + int nCol = 0; + int op = 0; + int iHash = 0; + int bIndirect = 0; + SessionChange *pChange = 0; + SessionChange *pExist = 0; + SessionChange **pp = 0; + SessionTable *pTab = 0; + u8 *aRec = &pIter->in.aData[pIter->in.iCurrent + 2]; + int nRec = (pIter->in.iNext - pIter->in.iCurrent) - 2; + + /* Ensure that only changesets, or only patchsets, but not a mixture + ** of both, are being combined. It is an error to try to combine a + ** changeset and a patchset. */ + if( pGrp->pList==0 ){ + pGrp->bPatch = pIter->bPatchset; + }else if( pIter->bPatchset!=pGrp->bPatch ){ + rc = SQLITE_ERROR; + } + + if( rc==SQLITE_OK ){ + const char *zTab = 0; + sqlite3changeset_op(pIter, &zTab, &nCol, &op, &bIndirect); + rc = sessionChangesetFindTable(pGrp, zTab, pIter, &pTab); + } + + if( rc==SQLITE_OK && nColnCol ){ + SessionBuffer *pBuf = &pGrp->rec; + rc = sessionChangesetExtendRecord(pGrp, pTab, nCol, op, aRec, nRec, pBuf); + aRec = pBuf->aBuf; + nRec = pBuf->nBuf; + assert( pGrp->db ); + } + + if( rc==SQLITE_OK && sessionGrowHash(0, pIter->bPatchset, pTab) ){ + rc = SQLITE_NOMEM; + } + + if( rc==SQLITE_OK ){ + /* Search for existing entry. If found, remove it from the hash table. + ** Code below may link it back in. */ iHash = sessionChangeHash( pTab, (pIter->bPatchset && op==SQLITE_DELETE), aRec, pTab->nChange ); - - /* Search for existing entry. If found, remove it from the hash table. - ** Code below may link it back in. - */ for(pp=&pTab->apChange[iHash]; *pp; pp=&(*pp)->pNext){ int bPkOnly1 = 0; int bPkOnly2 = 0; @@ -229079,19 +230755,41 @@ static int sessionChangesetToHash( break; } } + } + if( rc==SQLITE_OK ){ rc = sessionChangeMerge(pTab, bRebase, pIter->bPatchset, pExist, op, bIndirect, aRec, nRec, &pChange ); - if( rc ) break; - if( pChange ){ - pChange->pNext = pTab->apChange[iHash]; - pTab->apChange[iHash] = pChange; - pTab->nEntry++; - } + } + if( rc==SQLITE_OK && pChange ){ + pChange->pNext = pTab->apChange[iHash]; + pTab->apChange[iHash] = pChange; + pTab->nEntry++; + } + + if( rc==SQLITE_OK ) rc = pIter->rc; + return rc; +} + +/* +** Add all changes in the changeset traversed by the iterator passed as +** the first argument to the changegroup hash tables. +*/ +static int sessionChangesetToHash( + sqlite3_changeset_iter *pIter, /* Iterator to read from */ + sqlite3_changegroup *pGrp, /* Changegroup object to add changeset to */ + int bRebase /* True if hash table is for rebasing */ +){ + u8 *aRec; + int nRec; + int rc = SQLITE_OK; + + while( SQLITE_ROW==(sessionChangesetNext(pIter, &aRec, &nRec, 0)) ){ + rc = sessionOneChangeToHash(pGrp, pIter, bRebase); + if( rc!=SQLITE_OK ) break; } - sqlite3_free(rec.aBuf); if( rc==SQLITE_OK ) rc = pIter->rc; return rc; } @@ -229219,6 +230917,23 @@ SQLITE_API int sqlite3changegroup_add(sqlite3_changegroup *pGrp, int nData, void return rc; } +/* +** Add a single change to a changeset-group. +*/ +SQLITE_API int sqlite3changegroup_add_change( + sqlite3_changegroup *pGrp, + sqlite3_changeset_iter *pIter +){ + if( pIter->in.iCurrent==pIter->in.iNext + || pIter->rc!=SQLITE_OK + || pIter->bInvert + ){ + /* Iterator does not point to any valid entry or is an INVERT iterator. */ + return SQLITE_ERROR; + } + return sessionOneChangeToHash(pGrp, pIter, 0); +} + /* ** Obtain a buffer containing a changeset representing the concatenation ** of all changesets added to the group so far. @@ -229268,6 +230983,7 @@ SQLITE_API void sqlite3changegroup_delete(sqlite3_changegroup *pGrp){ if( pGrp ){ sqlite3_free(pGrp->zDb); sessionDeleteTable(0, pGrp->pList); + sqlite3_free(pGrp->rec.aBuf); sqlite3_free(pGrp); } } @@ -229669,6 +231385,7 @@ SQLITE_API int sqlite3rebaser_rebase_strm( SQLITE_API void sqlite3rebaser_delete(sqlite3_rebaser *p){ if( p ){ sessionDeleteTable(0, p->grp.pList); + sqlite3_free(p->grp.rec.aBuf); sqlite3_free(p); } } @@ -229766,8 +231483,8 @@ struct Fts5PhraseIter { ** EXTENSION API FUNCTIONS ** ** xUserData(pFts): -** Return a copy of the context pointer the extension function was -** registered with. +** Return a copy of the pUserData pointer passed to the xCreateFunction() +** API when the extension function was registered. ** ** xColumnTotalSize(pFts, iCol, pnToken): ** If parameter iCol is less than zero, set output variable *pnToken @@ -231363,6 +233080,9 @@ static void sqlite3Fts5UnicodeAscii(u8*, u8*); ** sqlite3Fts5ParserARG_STORE Code to store %extra_argument into fts5yypParser ** sqlite3Fts5ParserARG_FETCH Code to extract %extra_argument from fts5yypParser ** sqlite3Fts5ParserCTX_* As sqlite3Fts5ParserARG_ except for %extra_context +** fts5YYREALLOC Name of the realloc() function to use +** fts5YYFREE Name of the free() function to use +** fts5YYDYNSTACK True if stack space should be extended on heap ** fts5YYERRORSYMBOL is the code number of the error symbol. If not ** defined, then do no error processing. ** fts5YYNSTATE the combined number of states. @@ -231376,6 +233096,8 @@ static void sqlite3Fts5UnicodeAscii(u8*, u8*); ** fts5YY_NO_ACTION The fts5yy_action[] code for no-op ** fts5YY_MIN_REDUCE Minimum value for reduce actions ** fts5YY_MAX_REDUCE Maximum value for reduce actions +** fts5YY_MIN_DSTRCTR Minimum symbol value that has a destructor +** fts5YY_MAX_DSTRCTR Maximum symbol value that has a destructor */ #ifndef INTERFACE # define INTERFACE 1 @@ -231402,6 +233124,9 @@ typedef union { #define sqlite3Fts5ParserARG_PARAM ,pParse #define sqlite3Fts5ParserARG_FETCH Fts5Parse *pParse=fts5yypParser->pParse; #define sqlite3Fts5ParserARG_STORE fts5yypParser->pParse=pParse; +#define fts5YYREALLOC realloc +#define fts5YYFREE free +#define fts5YYDYNSTACK 0 #define sqlite3Fts5ParserCTX_SDECL #define sqlite3Fts5ParserCTX_PDECL #define sqlite3Fts5ParserCTX_PARAM @@ -231419,6 +233144,8 @@ typedef union { #define fts5YY_NO_ACTION 82 #define fts5YY_MIN_REDUCE 83 #define fts5YY_MAX_REDUCE 110 +#define fts5YY_MIN_DSTRCTR 16 +#define fts5YY_MAX_DSTRCTR 24 /************* End control #defines *******************************************/ #define fts5YY_NLOOKAHEAD ((int)(sizeof(fts5yy_lookahead)/sizeof(fts5yy_lookahead[0]))) @@ -231434,6 +233161,22 @@ typedef union { # define fts5yytestcase(X) #endif +/* Macro to determine if stack space has the ability to grow using +** heap memory. +*/ +#if fts5YYSTACKDEPTH<=0 || fts5YYDYNSTACK +# define fts5YYGROWABLESTACK 1 +#else +# define fts5YYGROWABLESTACK 0 +#endif + +/* Guarantee a minimum number of initial stack slots. +*/ +#if fts5YYSTACKDEPTH<=0 +# undef fts5YYSTACKDEPTH +# define fts5YYSTACKDEPTH 2 /* Need a minimum stack size */ +#endif + /* Next are the tables used to determine what action to take based on the ** current state and lookahead token. These tables are used to implement @@ -231594,14 +233337,9 @@ struct fts5yyParser { #endif sqlite3Fts5ParserARG_SDECL /* A place to hold %extra_argument */ sqlite3Fts5ParserCTX_SDECL /* A place to hold %extra_context */ -#if fts5YYSTACKDEPTH<=0 - int fts5yystksz; /* Current side of the stack */ - fts5yyStackEntry *fts5yystack; /* The parser's stack */ - fts5yyStackEntry fts5yystk0; /* First stack entry */ -#else - fts5yyStackEntry fts5yystack[fts5YYSTACKDEPTH]; /* The parser's stack */ - fts5yyStackEntry *fts5yystackEnd; /* Last entry in the stack */ -#endif + fts5yyStackEntry *fts5yystackEnd; /* Last entry in the stack */ + fts5yyStackEntry *fts5yystack; /* The parser stack */ + fts5yyStackEntry fts5yystk0[fts5YYSTACKDEPTH]; /* Initial stack space */ }; typedef struct fts5yyParser fts5yyParser; @@ -231708,37 +233446,45 @@ static const char *const fts5yyRuleName[] = { #endif /* NDEBUG */ -#if fts5YYSTACKDEPTH<=0 +#if fts5YYGROWABLESTACK /* ** Try to increase the size of the parser stack. Return the number ** of errors. Return 0 on success. */ static int fts5yyGrowStack(fts5yyParser *p){ + int oldSize = 1 + (int)(p->fts5yystackEnd - p->fts5yystack); int newSize; int idx; fts5yyStackEntry *pNew; - newSize = p->fts5yystksz*2 + 100; - idx = p->fts5yytos ? (int)(p->fts5yytos - p->fts5yystack) : 0; - if( p->fts5yystack==&p->fts5yystk0 ){ - pNew = malloc(newSize*sizeof(pNew[0])); - if( pNew ) pNew[0] = p->fts5yystk0; + newSize = oldSize*2 + 100; + idx = (int)(p->fts5yytos - p->fts5yystack); + if( p->fts5yystack==p->fts5yystk0 ){ + pNew = fts5YYREALLOC(0, newSize*sizeof(pNew[0])); + if( pNew==0 ) return 1; + memcpy(pNew, p->fts5yystack, oldSize*sizeof(pNew[0])); }else{ - pNew = realloc(p->fts5yystack, newSize*sizeof(pNew[0])); + pNew = fts5YYREALLOC(p->fts5yystack, newSize*sizeof(pNew[0])); + if( pNew==0 ) return 1; } - if( pNew ){ - p->fts5yystack = pNew; - p->fts5yytos = &p->fts5yystack[idx]; + p->fts5yystack = pNew; + p->fts5yytos = &p->fts5yystack[idx]; #ifndef NDEBUG - if( fts5yyTraceFILE ){ - fprintf(fts5yyTraceFILE,"%sStack grows from %d to %d entries.\n", - fts5yyTracePrompt, p->fts5yystksz, newSize); - } -#endif - p->fts5yystksz = newSize; + if( fts5yyTraceFILE ){ + fprintf(fts5yyTraceFILE,"%sStack grows from %d to %d entries.\n", + fts5yyTracePrompt, oldSize, newSize); } - return pNew==0; +#endif + p->fts5yystackEnd = &p->fts5yystack[newSize-1]; + return 0; } +#endif /* fts5YYGROWABLESTACK */ + +#if !fts5YYGROWABLESTACK +/* For builds that do no have a growable stack, fts5yyGrowStack always +** returns an error. +*/ +# define fts5yyGrowStack(X) 1 #endif /* Datatype of the argument to the memory allocated passed as the @@ -231758,24 +233504,14 @@ static void sqlite3Fts5ParserInit(void *fts5yypRawParser sqlite3Fts5ParserCTX_PD #ifdef fts5YYTRACKMAXSTACKDEPTH fts5yypParser->fts5yyhwm = 0; #endif -#if fts5YYSTACKDEPTH<=0 - fts5yypParser->fts5yytos = NULL; - fts5yypParser->fts5yystack = NULL; - fts5yypParser->fts5yystksz = 0; - if( fts5yyGrowStack(fts5yypParser) ){ - fts5yypParser->fts5yystack = &fts5yypParser->fts5yystk0; - fts5yypParser->fts5yystksz = 1; - } -#endif + fts5yypParser->fts5yystack = fts5yypParser->fts5yystk0; + fts5yypParser->fts5yystackEnd = &fts5yypParser->fts5yystack[fts5YYSTACKDEPTH-1]; #ifndef fts5YYNOERRORRECOVERY fts5yypParser->fts5yyerrcnt = -1; #endif fts5yypParser->fts5yytos = fts5yypParser->fts5yystack; fts5yypParser->fts5yystack[0].stateno = 0; fts5yypParser->fts5yystack[0].major = 0; -#if fts5YYSTACKDEPTH>0 - fts5yypParser->fts5yystackEnd = &fts5yypParser->fts5yystack[fts5YYSTACKDEPTH-1]; -#endif } #ifndef sqlite3Fts5Parser_ENGINEALWAYSONSTACK @@ -231889,9 +233625,26 @@ static void fts5yy_pop_parser_stack(fts5yyParser *pParser){ */ static void sqlite3Fts5ParserFinalize(void *p){ fts5yyParser *pParser = (fts5yyParser*)p; - while( pParser->fts5yytos>pParser->fts5yystack ) fts5yy_pop_parser_stack(pParser); -#if fts5YYSTACKDEPTH<=0 - if( pParser->fts5yystack!=&pParser->fts5yystk0 ) free(pParser->fts5yystack); + + /* In-lined version of calling fts5yy_pop_parser_stack() for each + ** element left in the stack */ + fts5yyStackEntry *fts5yytos = pParser->fts5yytos; + while( fts5yytos>pParser->fts5yystack ){ +#ifndef NDEBUG + if( fts5yyTraceFILE ){ + fprintf(fts5yyTraceFILE,"%sPopping %s\n", + fts5yyTracePrompt, + fts5yyTokenName[fts5yytos->major]); + } +#endif + if( fts5yytos->major>=fts5YY_MIN_DSTRCTR ){ + fts5yy_destructor(pParser, fts5yytos->major, &fts5yytos->minor); + } + fts5yytos--; + } + +#if fts5YYGROWABLESTACK + if( pParser->fts5yystack!=pParser->fts5yystk0 ) fts5YYFREE(pParser->fts5yystack); #endif } @@ -232118,25 +233871,19 @@ static void fts5yy_shift( assert( fts5yypParser->fts5yyhwm == (int)(fts5yypParser->fts5yytos - fts5yypParser->fts5yystack) ); } #endif -#if fts5YYSTACKDEPTH>0 - if( fts5yypParser->fts5yytos>fts5yypParser->fts5yystackEnd ){ - fts5yypParser->fts5yytos--; - fts5yyStackOverflow(fts5yypParser); - return; - } -#else - if( fts5yypParser->fts5yytos>=&fts5yypParser->fts5yystack[fts5yypParser->fts5yystksz] ){ + fts5yytos = fts5yypParser->fts5yytos; + if( fts5yytos>fts5yypParser->fts5yystackEnd ){ if( fts5yyGrowStack(fts5yypParser) ){ fts5yypParser->fts5yytos--; fts5yyStackOverflow(fts5yypParser); return; } + fts5yytos = fts5yypParser->fts5yytos; + assert( fts5yytos <= fts5yypParser->fts5yystackEnd ); } -#endif if( fts5yyNewState > fts5YY_MAX_SHIFT ){ fts5yyNewState += fts5YY_MIN_REDUCE - fts5YY_MIN_SHIFTREDUCE; } - fts5yytos = fts5yypParser->fts5yytos; fts5yytos->stateno = fts5yyNewState; fts5yytos->major = fts5yyMajor; fts5yytos->minor.fts5yy0 = fts5yyMinor; @@ -232573,19 +234320,12 @@ static void sqlite3Fts5Parser( (int)(fts5yypParser->fts5yytos - fts5yypParser->fts5yystack)); } #endif -#if fts5YYSTACKDEPTH>0 if( fts5yypParser->fts5yytos>=fts5yypParser->fts5yystackEnd ){ - fts5yyStackOverflow(fts5yypParser); - break; - } -#else - if( fts5yypParser->fts5yytos>=&fts5yypParser->fts5yystack[fts5yypParser->fts5yystksz-1] ){ if( fts5yyGrowStack(fts5yypParser) ){ fts5yyStackOverflow(fts5yypParser); break; } } -#endif } fts5yyact = fts5yy_reduce(fts5yypParser,fts5yyruleno,fts5yymajor,fts5yyminor sqlite3Fts5ParserCTX_PARAM); }else if( fts5yyact <= fts5YY_MAX_SHIFTREDUCE ){ @@ -250797,7 +252537,7 @@ static void fts5SourceIdFunc( ){ assert( nArg==0 ); UNUSED_PARAM2(nArg, apUnused); - sqlite3_result_text(pCtx, "fts5: 2024-04-15 13:34:05 8653b758870e6ef0c98d46b3ace27849054af85da891eb121e9aaa537f1e8355", -1, SQLITE_TRANSIENT); + sqlite3_result_text(pCtx, "fts5: 2024-05-23 13:25:27 96c92aba00c8375bc32fafcdf12429c58bd8aabfcadab6683e35bbb9cdebf19e", -1, SQLITE_TRANSIENT); } /* @@ -250836,6 +252576,7 @@ static int fts5IntegrityMethod( if( (rc&0xff)==SQLITE_CORRUPT ){ *pzErr = sqlite3_mprintf("malformed inverted index for FTS5 table %s.%s", zSchema, zTabname); + rc = (*pzErr) ? SQLITE_OK : SQLITE_NOMEM; }else if( rc!=SQLITE_OK ){ *pzErr = sqlite3_mprintf("unable to validate the inverted index for" " FTS5 table %s.%s: %s", @@ -250843,7 +252584,7 @@ static int fts5IntegrityMethod( } sqlite3Fts5IndexCloseReader(pTab->p.pIndex); - return SQLITE_OK; + return rc; } static int fts5Init(sqlite3 *db){ diff --git a/src/database/sqlite3.h b/src/database/sqlite3.h index 2618b37a..57df8dcf 100644 --- a/src/database/sqlite3.h +++ b/src/database/sqlite3.h @@ -146,9 +146,9 @@ extern "C" { ** [sqlite3_libversion_number()], [sqlite3_sourceid()], ** [sqlite_version()] and [sqlite_source_id()]. */ -#define SQLITE_VERSION "3.45.3" -#define SQLITE_VERSION_NUMBER 3045003 -#define SQLITE_SOURCE_ID "2024-04-15 13:34:05 8653b758870e6ef0c98d46b3ace27849054af85da891eb121e9aaa537f1e8355" +#define SQLITE_VERSION "3.46.0" +#define SQLITE_VERSION_NUMBER 3046000 +#define SQLITE_SOURCE_ID "2024-05-23 13:25:27 96c92aba00c8375bc32fafcdf12429c58bd8aabfcadab6683e35bbb9cdebf19e" /* ** CAPI3REF: Run-Time Library Version Numbers @@ -764,11 +764,11 @@ struct sqlite3_file { ** ** xLock() upgrades the database file lock. In other words, xLock() moves the ** database file lock in the direction NONE toward EXCLUSIVE. The argument to -** xLock() is always on of SHARED, RESERVED, PENDING, or EXCLUSIVE, never +** xLock() is always one of SHARED, RESERVED, PENDING, or EXCLUSIVE, never ** SQLITE_LOCK_NONE. If the database file lock is already at or above the ** requested lock, then the call to xLock() is a no-op. ** xUnlock() downgrades the database file lock to either SHARED or NONE. -* If the lock is already at or below the requested lock state, then the call +** If the lock is already at or below the requested lock state, then the call ** to xUnlock() is a no-op. ** The xCheckReservedLock() method checks whether any database connection, ** either in this process or in some other process, is holding a RESERVED, @@ -3305,8 +3305,8 @@ SQLITE_API int sqlite3_set_authorizer( #define SQLITE_RECURSIVE 33 /* NULL NULL */ /* -** CAPI3REF: Tracing And Profiling Functions -** METHOD: sqlite3 +** CAPI3REF: Deprecated Tracing And Profiling Functions +** DEPRECATED ** ** These routines are deprecated. Use the [sqlite3_trace_v2()] interface ** instead of the routines described here. @@ -6887,6 +6887,12 @@ SQLITE_API int sqlite3_autovacuum_pages( ** The exceptions defined in this paragraph might change in a future ** release of SQLite. ** +** Whether the update hook is invoked before or after the +** corresponding change is currently unspecified and may differ +** depending on the type of change. Do not rely on the order of the +** hook call with regards to the final result of the operation which +** triggers the hook. +** ** The update hook implementation must not do anything that will modify ** the database connection that invoked the update hook. Any actions ** to modify the database connection must be deferred until after the @@ -8357,7 +8363,7 @@ SQLITE_API int sqlite3_test_control(int op, ...); ** The sqlite3_keyword_count() interface returns the number of distinct ** keywords understood by SQLite. ** -** The sqlite3_keyword_name(N,Z,L) interface finds the N-th keyword and +** The sqlite3_keyword_name(N,Z,L) interface finds the 0-based N-th keyword and ** makes *Z point to that keyword expressed as UTF8 and writes the number ** of bytes in the keyword into *L. The string that *Z points to is not ** zero-terminated. The sqlite3_keyword_name(N,Z,L) routine returns @@ -9936,24 +9942,45 @@ SQLITE_API const char *sqlite3_vtab_collation(sqlite3_index_info*,int); **

  • ** ^(If the sqlite3_vtab_distinct() interface returns 2, that means ** that the query planner does not need the rows returned in any particular -** order, as long as rows with the same values in all "aOrderBy" columns -** are adjacent.)^ ^(Furthermore, only a single row for each particular -** combination of values in the columns identified by the "aOrderBy" field -** needs to be returned.)^ ^It is always ok for two or more rows with the same -** values in all "aOrderBy" columns to be returned, as long as all such rows -** are adjacent. ^The virtual table may, if it chooses, omit extra rows -** that have the same value for all columns identified by "aOrderBy". -** ^However omitting the extra rows is optional. +** order, as long as rows with the same values in all columns identified +** by "aOrderBy" are adjacent.)^ ^(Furthermore, when two or more rows +** contain the same values for all columns identified by "colUsed", all but +** one such row may optionally be omitted from the result.)^ +** The virtual table is not required to omit rows that are duplicates +** over the "colUsed" columns, but if the virtual table can do that without +** too much extra effort, it could potentially help the query to run faster. ** This mode is used for a DISTINCT query. **

  • -** ^(If the sqlite3_vtab_distinct() interface returns 3, that means -** that the query planner needs only distinct rows but it does need the -** rows to be sorted.)^ ^The virtual table implementation is free to omit -** rows that are identical in all aOrderBy columns, if it wants to, but -** it is not required to omit any rows. This mode is used for queries +** ^(If the sqlite3_vtab_distinct() interface returns 3, that means the +** virtual table must return rows in the order defined by "aOrderBy" as +** if the sqlite3_vtab_distinct() interface had returned 0. However if +** two or more rows in the result have the same values for all columns +** identified by "colUsed", then all but one such row may optionally be +** omitted.)^ Like when the return value is 2, the virtual table +** is not required to omit rows that are duplicates over the "colUsed" +** columns, but if the virtual table can do that without +** too much extra effort, it could potentially help the query to run faster. +** This mode is used for queries ** that have both DISTINCT and ORDER BY clauses. ** ** +**

    The following table summarizes the conditions under which the +** virtual table is allowed to set the "orderByConsumed" flag based on +** the value returned by sqlite3_vtab_distinct(). This table is a +** restatement of the previous four paragraphs: +** +** +** +**
    sqlite3_vtab_distinct() return value +** Rows are returned in aOrderBy order +** Rows with the same value in all aOrderBy columns are adjacent +** Duplicates over all colUsed columns may be omitted +**
    0yesyesno +**
    1noyesno +**
    2noyesyes +**
    3yesyesyes +**
    +** ** ^For the purposes of comparing virtual table output values to see if the ** values are same value for sorting purposes, two NULL values are considered ** to be the same. In other words, the comparison operator is "IS" @@ -11998,6 +12025,30 @@ SQLITE_API int sqlite3changegroup_schema(sqlite3_changegroup*, sqlite3*, const c */ SQLITE_API int sqlite3changegroup_add(sqlite3_changegroup*, int nData, void *pData); +/* +** CAPI3REF: Add A Single Change To A Changegroup +** METHOD: sqlite3_changegroup +** +** This function adds the single change currently indicated by the iterator +** passed as the second argument to the changegroup object. The rules for +** adding the change are just as described for [sqlite3changegroup_add()]. +** +** If the change is successfully added to the changegroup, SQLITE_OK is +** returned. Otherwise, an SQLite error code is returned. +** +** The iterator must point to a valid entry when this function is called. +** If it does not, SQLITE_ERROR is returned and no change is added to the +** changegroup. Additionally, the iterator must not have been opened with +** the SQLITE_CHANGESETAPPLY_INVERT flag. In this case SQLITE_ERROR is also +** returned. +*/ +SQLITE_API int sqlite3changegroup_add_change( + sqlite3_changegroup*, + sqlite3_changeset_iter* +); + + + /* ** CAPI3REF: Obtain A Composite Changeset From A Changegroup ** METHOD: sqlite3_changegroup @@ -12802,8 +12853,8 @@ struct Fts5PhraseIter { ** EXTENSION API FUNCTIONS ** ** xUserData(pFts): -** Return a copy of the context pointer the extension function was -** registered with. +** Return a copy of the pUserData pointer passed to the xCreateFunction() +** API when the extension function was registered. ** ** xColumnTotalSize(pFts, iCol, pnToken): ** If parameter iCol is less than zero, set output variable *pnToken From dc204a41b059f97c8a3e13c36628ee2536cc1a27 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 25 May 2024 09:54:37 +0200 Subject: [PATCH 099/339] Use v2.6 CI containers and nightly for the devcontainer Signed-off-by: DL6ER --- .devcontainer/devcontainer.json | 2 +- .github/Dockerfile | 2 +- ...int-FTL-version-in-interactive-shell.patch | 2 +- src/api/stats_database.c | 10 +++++----- src/config/config.h | 2 +- src/config/legacy_reader.c | 20 ++++++++++--------- src/config/password.c | 2 +- src/database/aliasclients.c | 8 ++++---- src/database/aliasclients.h | 2 +- src/database/gravity-db.c | 14 ++++++------- src/database/network-table.c | 18 ++++++++--------- src/database/query-table.c | 4 ++-- src/database/sqlite3-ext.c | 4 ++-- src/dnsmasq/forward.c | 2 +- src/dnsmasq/helper.c | 2 +- src/dnsmasq/network.c | 4 ++-- src/dnsmasq/option.c | 2 +- src/dnsmasq/rfc1035.c | 2 +- src/syscalls/CMakeLists.txt | 1 + src/syscalls/accept.c | 4 ++-- src/syscalls/asprintf.c | 4 ++-- src/syscalls/calloc.c | 4 ++-- src/syscalls/fopen.c | 4 ++-- src/syscalls/fprintf.c | 4 ++-- src/syscalls/free.c | 4 ++-- src/syscalls/ftlallocate.c | 4 ++-- src/syscalls/pthread_mutex_lock.c | 4 ++-- src/syscalls/realloc.c | 4 ++-- src/syscalls/recv.c | 4 ++-- src/syscalls/recvfrom.c | 4 ++-- src/syscalls/select.c | 4 ++-- src/syscalls/sendto.c | 4 ++-- src/syscalls/snprintf.c | 4 ++-- src/syscalls/sprintf.c | 4 ++-- src/syscalls/strdup.c | 4 ++-- src/syscalls/string.c | 4 ++-- src/syscalls/vasprintf.c | 4 ++-- src/syscalls/vfprintf.c | 4 ++-- src/syscalls/vsnprintf.c | 4 ++-- src/syscalls/vsprintf.c | 4 ++-- src/syscalls/write.c | 4 ++-- 41 files changed, 97 insertions(+), 94 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 3fe6ed7f..bb8c5892 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,6 +1,6 @@ { "name": "FTL x86_64 Build Env", - "image": "ghcr.io/pi-hole/ftl-build:new-clang", + "image": "ghcr.io/pi-hole/ftl-build:nightly", "runArgs": [ "--cap-add=SYS_PTRACE", "--security-opt", "seccomp=unconfined" ], "customizations": { "vscode": { diff --git a/.github/Dockerfile b/.github/Dockerfile index 1cbde40e..046e45f6 100644 --- a/.github/Dockerfile +++ b/.github/Dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/pi-hole/ftl-build:new-clang AS builder +FROM ghcr.io/pi-hole/ftl-build:v2.6 AS builder WORKDIR /app diff --git a/patch/sqlite3/0001-print-FTL-version-in-interactive-shell.patch b/patch/sqlite3/0001-print-FTL-version-in-interactive-shell.patch index 1efac968..c7aaa292 100644 --- a/patch/sqlite3/0001-print-FTL-version-in-interactive-shell.patch +++ b/patch/sqlite3/0001-print-FTL-version-in-interactive-shell.patch @@ -7,7 +7,7 @@ index 6280ebf6..a5e82f70 100644 #include #include +// print_FTL_version() -+#include "../log.h" ++#include "log.h" #if !defined(_WIN32) && !defined(WIN32) # include diff --git a/src/api/stats_database.c b/src/api/stats_database.c index 5a30b105..61213c1e 100644 --- a/src/api/stats_database.c +++ b/src/api/stats_database.c @@ -8,16 +8,16 @@ * This file is copyright under the latest version of the EUPL. * Please see LICENSE file for your rights under this license. */ -#include "../FTL.h" -#include "../webserver/http-common.h" -#include "../webserver/json_macros.h" +#include "FTL.h" +#include "webserver/http-common.h" +#include "webserver/json_macros.h" #include "api.h" // querytypes[] -#include "../datastructure.h" +#include "datastructure.h" // logging routines #include "log.h" // db -#include "../database/common.h" +#include "database/common.h" // SQL Query type filters for the database #define FILTER_STATUS_NOT_BLOCKED "status IN (0,2,3,12,13,14,17)" diff --git a/src/config/config.h b/src/config/config.h index 6cd4f05e..013414ed 100644 --- a/src/config/config.h +++ b/src/config/config.h @@ -11,7 +11,7 @@ #define CONFIG_H // enum privacy_level -#include "../enums.h" +#include "enums.h" #include // typedef int16_t #include diff --git a/src/config/legacy_reader.c b/src/config/legacy_reader.c index 06173fea..75e9118a 100644 --- a/src/config/legacy_reader.c +++ b/src/config/legacy_reader.c @@ -28,7 +28,7 @@ static pthread_mutex_t lock; // Private prototypes static char *parseFTLconf(FILE *fp, const char *key); static void releaseConfigMemory(void); -static char *getPath(FILE* fp, const char *option, char *ptr); +static char *__attribute__((nonnull(1,2,3), malloc, warn_unused_result)) getPath(FILE* fp, const char *option, char *ptr); static bool parseBool(const char *option, bool *ptr); static void readDebugingSettingsLegacy(FILE *fp); static void getBlockingModeLegacy(FILE *fp); @@ -593,7 +593,7 @@ const char *readFTLlegacy(struct config *conf) return path; } -static char *getPath(FILE* fp, const char *option, char *path_default) +static char *__attribute__((nonnull(1,2,3), malloc, warn_unused_result)) getPath(FILE* fp, const char *option, char *path_default) { // This subroutine is used to read paths from pihole-FTL.conf // fp: File path to opened and readable config file @@ -604,22 +604,24 @@ static char *getPath(FILE* fp, const char *option, char *path_default) errno = 0; // Use sscanf() to obtain filename from config file parameter only if buffer != NULL char *val_ptr = calloc(128, sizeof(char)); - if(buffer == NULL || sscanf(buffer, "%127s", val_ptr) != 1) - { - // Use standard path if no custom path was obtained from the config file - return path_default; - } // Test if memory allocation was successful if(val_ptr == NULL) { - log_crit("Allocating memory for %s failed (%s, %i). Exiting.", option, strerror(errno), errno); + log_crit("Allocating memory for %s failed (%s, %i). Exiting.", + option, strerror(errno), errno); exit(EXIT_FAILURE); } - else if(strlen(val_ptr) == 0) + + if(buffer == NULL || sscanf(buffer, "%127s", val_ptr) != 1 || strlen(val_ptr) == 0) { + // Use standard path if no custom path was obtained from the config file log_info(" %s: Empty path is not possible, using default", option); + + strncpy(val_ptr, path_default, 127); + val_ptr[127] = '\0'; + return val_ptr; } return val_ptr; diff --git a/src/config/password.c b/src/config/password.c index c1231ed7..5c32411c 100644 --- a/src/config/password.c +++ b/src/config/password.c @@ -315,7 +315,7 @@ char * __attribute__((malloc)) create_password(const char *password) enum password_result verify_login(const char *password) { enum password_result pw = verify_password(password, config.webserver.api.pwhash.v.s, true); - log_debug(DEBUG_API, pw == PASSWORD_CORRECT ? "Password correct" : "Password incorrect"); + log_debug(DEBUG_API, "Password %s correct", pw == PASSWORD_CORRECT ? "" : "not"); // Check if an application password is set and if it matches if(pw == PASSWORD_INCORRECT && diff --git a/src/database/aliasclients.c b/src/database/aliasclients.c index f060e217..7e8fddeb 100644 --- a/src/database/aliasclients.c +++ b/src/database/aliasclients.c @@ -8,15 +8,15 @@ * This file is copyright under the latest version of the EUPL. * Please see LICENSE file for your rights under this license. */ -#include "../FTL.h" +#include "FTL.h" #include "aliasclients.h" #include "common.h" // global counters variable -#include "../shmem.h" +#include "shmem.h" // global config variable -#include "../config/config.h" +#include "config/config.h" // logging routines -#include "../log.h" +#include "log.h" // getAliasclientIDfromIP() #include "network-table.h" diff --git a/src/database/aliasclients.h b/src/database/aliasclients.h index 8eb6d2f7..a03617fc 100644 --- a/src/database/aliasclients.h +++ b/src/database/aliasclients.h @@ -11,7 +11,7 @@ #define ALIASCLIENTS_TABLE_H // type clientsData -#include "../datastructure.h" +#include "datastructure.h" bool create_aliasclients_table(sqlite3 *db); diff --git a/src/database/gravity-db.c b/src/database/gravity-db.c index fef07869..d922e2d7 100644 --- a/src/database/gravity-db.c +++ b/src/database/gravity-db.c @@ -8,29 +8,29 @@ * This file is copyright under the latest version of the EUPL. * Please see LICENSE file for your rights under this license. */ -#include "../FTL.h" +#include "FTL.h" #include "sqlite3.h" #include "gravity-db.h" // struct config -#include "../config/config.h" +#include "config/config.h" // logging routines -#include "../log.h" +#include "log.h" // getstr() -#include "../shmem.h" +#include "shmem.h" // SQLite3 prepared statement vectors -#include "../vector.h" +#include "vector.h" // log_subnet_warning() // logg_inaccessible_adlist #include "message-table.h" // getMACfromIP() #include "network-table.h" // struct DNSCacheData -#include "../datastructure.h" +#include "datastructure.h" // reset_aliasclient() #include "aliasclients.h" // Definition of struct regexData -#include "../regex_r.h" +#include "regex_r.h" // Prefix of interface names in the client table #define INTERFACE_SEP ":" diff --git a/src/database/network-table.c b/src/database/network-table.c index 1e2a8817..2899d4aa 100644 --- a/src/database/network-table.c +++ b/src/database/network-table.c @@ -8,21 +8,21 @@ * This file is copyright under the latest version of the EUPL. * Please see LICENSE file for your rights under this license. */ -#include "../FTL.h" +#include "FTL.h" #include "network-table.h" #include "common.h" -#include "../shmem.h" -#include "../log.h" +#include "shmem.h" +#include "log.h" // timer_elapsed_msec() -#include "../timers.h" -#include "../config/config.h" -#include "../datastructure.h" +#include "timers.h" +#include "config/config.h" +#include "datastructure.h" // struct config -#include "../config/config.h" +#include "config/config.h" // resolve_this_name() -#include "../resolve.h" +#include "resolve.h" // killed -#include "../signals.h" +#include "signals.h" // Private prototypes static char *getMACVendor(const char *hwaddr) __attribute__ ((malloc)); diff --git a/src/database/query-table.c b/src/database/query-table.c index bcce7dfc..342730a4 100644 --- a/src/database/query-table.c +++ b/src/database/query-table.c @@ -1120,13 +1120,13 @@ void DB_read_queries(void) (buffer = (const char *)sqlite3_column_text(stmt, 6)) != NULL) { // Get IP address and port of upstream destination - char serv_addr[INET6_ADDRSTRLEN + 1] = { 0 }; + char serv_addr[INET6_ADDRSTRLEN + 16] = { 0 }; unsigned int serv_port = 53; // We limit the number of bytes written into the serv_addr buffer // to prevent buffer overflows. If there is no port available in // the database, we skip extracting them and use the default port sscanf(buffer, "%"xstr(INET6_ADDRSTRLEN)"[^#]#%u", serv_addr, &serv_port); - serv_addr[INET6_ADDRSTRLEN-1] = '\0'; + serv_addr[INET6_ADDRSTRLEN + 15] = '\0'; upstreamID = findUpstreamID(serv_addr, (in_port_t)serv_port); } diff --git a/src/database/sqlite3-ext.c b/src/database/sqlite3-ext.c index e3e28498..b065576e 100644 --- a/src/database/sqlite3-ext.c +++ b/src/database/sqlite3-ext.c @@ -22,9 +22,9 @@ // free() #include // logging routines -#include "../log.h" +#include "log.h" // struct config -#include "../config/config.h" +#include "config/config.h" // isMAC() #include "network-table.h" diff --git a/src/dnsmasq/forward.c b/src/dnsmasq/forward.c index f9316a08..15713ec2 100644 --- a/src/dnsmasq/forward.c +++ b/src/dnsmasq/forward.c @@ -15,7 +15,7 @@ */ #include "dnsmasq.h" -#include "../dnsmasq_interface.h" +#include "dnsmasq_interface.h" static struct frec *get_new_frec(time_t now, struct server *serv, int force); static struct frec *lookup_frec(unsigned short id, int fd, void *hash, int *firstp, int *lastp); diff --git a/src/dnsmasq/helper.c b/src/dnsmasq/helper.c index a59a0a78..65727ba4 100644 --- a/src/dnsmasq/helper.c +++ b/src/dnsmasq/helper.c @@ -15,7 +15,7 @@ */ #include "dnsmasq.h" -#include "../log.h" +#include "log.h" #ifdef HAVE_SCRIPT diff --git a/src/dnsmasq/network.c b/src/dnsmasq/network.c index 60799d48..9e009f77 100644 --- a/src/dnsmasq/network.c +++ b/src/dnsmasq/network.c @@ -15,8 +15,8 @@ */ #include "dnsmasq.h" -#include "../dnsmasq_interface.h" -#include "../log.h" +#include "dnsmasq_interface.h" +#include "log.h" #ifdef HAVE_LINUX_NETWORK diff --git a/src/dnsmasq/option.c b/src/dnsmasq/option.c index 249a6f35..8e7377dc 100644 --- a/src/dnsmasq/option.c +++ b/src/dnsmasq/option.c @@ -20,7 +20,7 @@ #include /* Pi-hole modification */ -#include "../log.h" +#include "log.h" /************************/ static volatile int mem_recover = 0; diff --git a/src/dnsmasq/rfc1035.c b/src/dnsmasq/rfc1035.c index 06d3067c..ef618ff0 100644 --- a/src/dnsmasq/rfc1035.c +++ b/src/dnsmasq/rfc1035.c @@ -15,7 +15,7 @@ */ #include "dnsmasq.h" -#include "../dnsmasq_interface.h" +#include "dnsmasq_interface.h" int extract_name(struct dns_header *header, size_t plen, unsigned char **pp, char *name, int isExtract, int extrabytes) diff --git a/src/syscalls/CMakeLists.txt b/src/syscalls/CMakeLists.txt index 10103094..7ba43aa4 100644 --- a/src/syscalls/CMakeLists.txt +++ b/src/syscalls/CMakeLists.txt @@ -36,3 +36,4 @@ set(sources add_library(syscalls OBJECT ${sources}) target_compile_options(syscalls PRIVATE ${EXTRAWARN}) +target_include_directories(syscalls PRIVATE ${PROJECT_SOURCE_DIR}/src) diff --git a/src/syscalls/accept.c b/src/syscalls/accept.c index 60edbf62..ef53bad2 100644 --- a/src/syscalls/accept.c +++ b/src/syscalls/accept.c @@ -8,9 +8,9 @@ * This file is copyright under the latest version of the EUPL. * Please see LICENSE file for your rights under this license. */ -#include "../FTL.h" +#include "FTL.h" //#include "syscalls.h" is implicitly done in FTL.h -#include "../log.h" +#include "log.h" #undef accept int FTLaccept(int sockfd, struct sockaddr *addr, socklen_t *addrlen, const char *file, const char *func, const int line) diff --git a/src/syscalls/asprintf.c b/src/syscalls/asprintf.c index 4da1ea82..67585bb0 100644 --- a/src/syscalls/asprintf.c +++ b/src/syscalls/asprintf.c @@ -8,9 +8,9 @@ * This file is copyright under the latest version of the EUPL. * Please see LICENSE file for your rights under this license. */ -#include "../FTL.h" +#include "FTL.h" //#include "syscalls.h" is implicitly done in FTL.h -#include "../log.h" +#include "log.h" int FTLasprintf(const char *file, const char *func, const int line, char **buffer, const char *format, ...) { diff --git a/src/syscalls/calloc.c b/src/syscalls/calloc.c index 60c2d2f0..e8de9821 100644 --- a/src/syscalls/calloc.c +++ b/src/syscalls/calloc.c @@ -8,9 +8,9 @@ * This file is copyright under the latest version of the EUPL. * Please see LICENSE file for your rights under this license. */ -#include "../FTL.h" +#include "FTL.h" //#include "syscalls.h" is implicitly done in FTL.h -#include "../log.h" +#include "log.h" #undef calloc void* __attribute__((malloc)) __attribute__((alloc_size(1,2))) FTLcalloc(const size_t nmemb, const size_t size, const char *file, const char *func, const int line) diff --git a/src/syscalls/fopen.c b/src/syscalls/fopen.c index 71912a69..9dd603a4 100644 --- a/src/syscalls/fopen.c +++ b/src/syscalls/fopen.c @@ -8,9 +8,9 @@ * This file is copyright under the latest version of the EUPL. * Please see LICENSE file for your rights under this license. */ -#include "../FTL.h" +#include "FTL.h" //#include "syscalls.h" is implicitly done in FTL.h -#include "../log.h" +#include "log.h" static uint8_t already_writing = 0; diff --git a/src/syscalls/fprintf.c b/src/syscalls/fprintf.c index be3bee68..d64d85ab 100644 --- a/src/syscalls/fprintf.c +++ b/src/syscalls/fprintf.c @@ -8,9 +8,9 @@ * This file is copyright under the latest version of the EUPL. * Please see LICENSE file for your rights under this license. */ -#include "../FTL.h" +#include "FTL.h" //#include "syscalls.h" is implicitly done in FTL.h -#include "../log.h" +#include "log.h" int FTLfprintf(FILE *stream, const char *file, const char *func, const int line, const char *format, ...) { diff --git a/src/syscalls/free.c b/src/syscalls/free.c index 1091aa14..360170ea 100644 --- a/src/syscalls/free.c +++ b/src/syscalls/free.c @@ -8,9 +8,9 @@ * This file is copyright under the latest version of the EUPL. * Please see LICENSE file for your rights under this license. */ -#include "../FTL.h" +#include "FTL.h" //#include "syscalls.h" is implicitly done in FTL.h -#include "../log.h" +#include "log.h" #undef free void FTLfree(void **ptr, const char *file, const char *func, const int line) diff --git a/src/syscalls/ftlallocate.c b/src/syscalls/ftlallocate.c index 8140f1b8..b1330ae7 100644 --- a/src/syscalls/ftlallocate.c +++ b/src/syscalls/ftlallocate.c @@ -8,9 +8,9 @@ * This file is copyright under the latest version of the EUPL. * Please see LICENSE file for your rights under this license. */ -#include "../FTL.h" +#include "FTL.h" //#include "syscalls.h" is implicitly done in FTL.h -#include "../log.h" +#include "log.h" #include // off_t is automatically set as off64_t when this is a 64bit system diff --git a/src/syscalls/pthread_mutex_lock.c b/src/syscalls/pthread_mutex_lock.c index ab4b0112..1fc1821a 100644 --- a/src/syscalls/pthread_mutex_lock.c +++ b/src/syscalls/pthread_mutex_lock.c @@ -8,9 +8,9 @@ * This file is copyright under the latest version of the EUPL. * Please see LICENSE file for your rights under this license. */ -#include "../FTL.h" +#include "FTL.h" //#include "syscalls.h" is implicitly done in FTL.h -#include "../log.h" +#include "log.h" #include diff --git a/src/syscalls/realloc.c b/src/syscalls/realloc.c index 77f83f4a..aa2b12a3 100644 --- a/src/syscalls/realloc.c +++ b/src/syscalls/realloc.c @@ -8,9 +8,9 @@ * This file is copyright under the latest version of the EUPL. * Please see LICENSE file for your rights under this license. */ -#include "../FTL.h" +#include "FTL.h" //#include "syscalls.h" is implicitly done in FTL.h -#include "../log.h" +#include "log.h" #undef realloc void __attribute__((alloc_size(2))) *FTLrealloc(void *ptr_in, const size_t size, const char * file, const char * func, const int line) diff --git a/src/syscalls/recv.c b/src/syscalls/recv.c index bdae2b25..0dce546d 100644 --- a/src/syscalls/recv.c +++ b/src/syscalls/recv.c @@ -8,9 +8,9 @@ * This file is copyright under the latest version of the EUPL. * Please see LICENSE file for your rights under this license. */ -#include "../FTL.h" +#include "FTL.h" //#include "syscalls.h" is implicitly done in FTL.h -#include "../log.h" +#include "log.h" #include diff --git a/src/syscalls/recvfrom.c b/src/syscalls/recvfrom.c index b78f8bf5..b703ad04 100644 --- a/src/syscalls/recvfrom.c +++ b/src/syscalls/recvfrom.c @@ -8,9 +8,9 @@ * This file is copyright under the latest version of the EUPL. * Please see LICENSE file for your rights under this license. */ -#include "../FTL.h" +#include "FTL.h" //#include "syscalls.h" is implicitly done in FTL.h -#include "../log.h" +#include "log.h" #include #include diff --git a/src/syscalls/select.c b/src/syscalls/select.c index b6889fb6..1907ba51 100644 --- a/src/syscalls/select.c +++ b/src/syscalls/select.c @@ -8,9 +8,9 @@ * This file is copyright under the latest version of the EUPL. * Please see LICENSE file for your rights under this license. */ -#include "../FTL.h" +#include "FTL.h" //#include "syscalls.h" is implicitly done in FTL.h -#include "../log.h" +#include "log.h" #include diff --git a/src/syscalls/sendto.c b/src/syscalls/sendto.c index b0cf0141..c3d0710b 100644 --- a/src/syscalls/sendto.c +++ b/src/syscalls/sendto.c @@ -8,9 +8,9 @@ * This file is copyright under the latest version of the EUPL. * Please see LICENSE file for your rights under this license. */ -#include "../FTL.h" +#include "FTL.h" //#include "syscalls.h" is implicitly done in FTL.h -#include "../log.h" +#include "log.h" #include #include diff --git a/src/syscalls/snprintf.c b/src/syscalls/snprintf.c index 699d942c..7384eec4 100644 --- a/src/syscalls/snprintf.c +++ b/src/syscalls/snprintf.c @@ -8,9 +8,9 @@ * This file is copyright under the latest version of the EUPL. * Please see LICENSE file for your rights under this license. */ -#include "../FTL.h" +#include "FTL.h" //#include "syscalls.h" is implicitly done in FTL.h -#include "../log.h" +#include "log.h" int FTLsnprintf(const char *file, const char *func, const int line, char *__restrict__ buffer, const size_t maxlen, const char *format, ...) { diff --git a/src/syscalls/sprintf.c b/src/syscalls/sprintf.c index a6cc4094..c3ef0563 100644 --- a/src/syscalls/sprintf.c +++ b/src/syscalls/sprintf.c @@ -8,9 +8,9 @@ * This file is copyright under the latest version of the EUPL. * Please see LICENSE file for your rights under this license. */ -#include "../FTL.h" +#include "FTL.h" //#include "syscalls.h" is implicitly done in FTL.h -#include "../log.h" +#include "log.h" int FTLsprintf(const char *file, const char *func, const int line, char *__restrict__ buffer, const char *format, ...) { diff --git a/src/syscalls/strdup.c b/src/syscalls/strdup.c index bd2d7b41..dc912f5d 100644 --- a/src/syscalls/strdup.c +++ b/src/syscalls/strdup.c @@ -8,9 +8,9 @@ * This file is copyright under the latest version of the EUPL. * Please see LICENSE file for your rights under this license. */ -#include "../FTL.h" +#include "FTL.h" //#include "syscalls.h" is implicitly done in FTL.h -#include "../log.h" +#include "log.h" char* __attribute__((malloc)) FTLstrdup(const char *src, const char *file, const char *func, const int line) { diff --git a/src/syscalls/string.c b/src/syscalls/string.c index 88254e35..1d394252 100644 --- a/src/syscalls/string.c +++ b/src/syscalls/string.c @@ -8,9 +8,9 @@ * This file is copyright under the latest version of the EUPL. * Please see LICENSE file for your rights under this license. */ -#include "../FTL.h" +#include "FTL.h" //#include "syscalls.h" is implicitly done in FTL.h -#include "../log.h" +#include "log.h" #undef strlen size_t FTLstrlen(const char *s, const char *file, const char *func, const int line) diff --git a/src/syscalls/vasprintf.c b/src/syscalls/vasprintf.c index 3ac340e6..d1a268e9 100644 --- a/src/syscalls/vasprintf.c +++ b/src/syscalls/vasprintf.c @@ -8,9 +8,9 @@ * This file is copyright under the latest version of the EUPL. * Please see LICENSE file for your rights under this license. */ -#include "../FTL.h" +#include "FTL.h" //#include "syscalls.h" is implicitly done in FTL.h -#include "../log.h" +#include "log.h" #undef vasprintf int FTLvasprintf(const char *file, const char *func, const int line, char **buffer, const char *format, va_list args) diff --git a/src/syscalls/vfprintf.c b/src/syscalls/vfprintf.c index c7a82de6..516271dc 100644 --- a/src/syscalls/vfprintf.c +++ b/src/syscalls/vfprintf.c @@ -8,9 +8,9 @@ * This file is copyright under the latest version of the EUPL. * Please see LICENSE file for your rights under this license. */ -#include "../FTL.h" +#include "FTL.h" //#include "syscalls.h" is implicitly done in FTL.h -#include "../log.h" +#include "log.h" // itoa implementation using only static memory // taken from Kernighan and Ritchie's "The C Programming Language" diff --git a/src/syscalls/vsnprintf.c b/src/syscalls/vsnprintf.c index 4f4badfc..690d90f0 100644 --- a/src/syscalls/vsnprintf.c +++ b/src/syscalls/vsnprintf.c @@ -8,9 +8,9 @@ * This file is copyright under the latest version of the EUPL. * Please see LICENSE file for your rights under this license. */ -#include "../FTL.h" +#include "FTL.h" //#include "syscalls.h" is implicitly done in FTL.h -#include "../log.h" +#include "log.h" #undef vsnprintf int FTLvsnprintf(const char *file, const char *func, const int line, char *__restrict__ buffer, const size_t maxlen, const char *format, va_list args) diff --git a/src/syscalls/vsprintf.c b/src/syscalls/vsprintf.c index cce0b35f..72aee733 100644 --- a/src/syscalls/vsprintf.c +++ b/src/syscalls/vsprintf.c @@ -8,9 +8,9 @@ * This file is copyright under the latest version of the EUPL. * Please see LICENSE file for your rights under this license. */ -#include "../FTL.h" +#include "FTL.h" //#include "syscalls.h" is implicitly done in FTL.h -#include "../log.h" +#include "log.h" #undef vsprintf int FTLvsprintf(const char *file, const char *func, const int line, char *__restrict__ buffer, const char *format, va_list args) diff --git a/src/syscalls/write.c b/src/syscalls/write.c index e145007f..bdb8eafc 100644 --- a/src/syscalls/write.c +++ b/src/syscalls/write.c @@ -8,9 +8,9 @@ * This file is copyright under the latest version of the EUPL. * Please see LICENSE file for your rights under this license. */ -#include "../FTL.h" +#include "FTL.h" //#include "syscalls.h" is implicitly done in FTL.h -#include "../log.h" +#include "log.h" #undef write ssize_t FTLwrite(int fd, const void *buf, size_t total, const char *file, const char *func, const int line) From 40eee1ac9c7c3663fa994ebc316deec791a83740 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 25 May 2024 09:57:46 +0200 Subject: [PATCH 100/339] Addres spellchecker complaints Signed-off-by: DL6ER --- .github/.codespellignore_lines | 2 ++ src/config/toml_helper.c | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/.codespellignore_lines b/.github/.codespellignore_lines index 49f7eebf..a592a395 100644 --- a/.github/.codespellignore_lines +++ b/.github/.codespellignore_lines @@ -1 +1,3 @@ self.errors.append("Exception when GETing from FTL: " + str(e)) +// sitten -> sittin (substitution of "i" for "e"), +// sittin -> sitting (insertion of "g" at the end). diff --git a/src/config/toml_helper.c b/src/config/toml_helper.c index 04ade29b..26c6b1dd 100644 --- a/src/config/toml_helper.c +++ b/src/config/toml_helper.c @@ -209,7 +209,7 @@ void print_comment(FILE *fp, const char *str, const char *intro, const unsigned // If this the first line? If not, add a newline if (i > 0) fputc('\n', fp); - // Add intendation + // Add indentation for (unsigned int j = 0; j != 2*indent; ++j) fputc(' ', fp); // Start a new line @@ -428,7 +428,7 @@ void writeTOMLvalue(FILE * fp, const int indent, const enum conf_type t, union c if(strlen(item->valuestring) == 0) continue; - // Add intendation (if we are indenting) + // Add indentation (if we are indenting) if(indent > -1) indentTOML(fp, indent + 1); From 36cb3a4f793f61f98aeb63c44ee198d08c6862f0 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 26 May 2024 11:36:46 +0200 Subject: [PATCH 101/339] Add more debugging for the internal name resolution process: (1) about the PTR we sent (and where), (2) about the status of the answer, (3) whether the answer was truncated (and, hence, ignored), (4) about answers skipped because they are not of type PTR, and (5) when we are trying to get client host names from the database (without any actual PTR lookups) Signed-off-by: DL6ER --- src/database/network-table.c | 1 + src/resolve.c | 89 ++++++++++++++++++++++++++++++++++-- 2 files changed, 85 insertions(+), 5 deletions(-) diff --git a/src/database/network-table.c b/src/database/network-table.c index 1e2a8817..54c71b81 100644 --- a/src/database/network-table.c +++ b/src/database/network-table.c @@ -2004,6 +2004,7 @@ char *__attribute__((malloc)) getNameFromIP(sqlite3 *db, const char *ipaddr) // Return early if database is known to be broken if(FTLDBerror()) return NULL; + log_debug(DEBUG_RESOLVER, "Trying to obtain host name of \"%s\" from network_addresses table", ipaddr); // Check if we want to resolve host names if(!resolve_this_name(ipaddr)) diff --git a/src/resolve.c b/src/resolve.c index b083b0cc..342ff9a6 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -95,6 +95,64 @@ struct RES_RECORD uint8_t *rdata; }; +// see https://www.iana.org/assignments/dns-parameters/dns-parameters.xhtml +static const char *getDNScode(int code) +{ + switch(code) + { + case 0: + return "NoError"; + case 1: + return "FormErr (Format Error)"; + case 2: + return "ServFail (Server Failure)"; + case 3: + return "NXDomain (Non-Existent Domain)"; + case 4: + return "NotImp (Not Implemented)"; + case 5: + return "Refused (Query Refused)"; + case 6: + return "YXDomain (Name Exists when it should not)"; + case 7: + return "YXRRSet (RR Set Exists when it should not)"; + case 8: + return "NXRRSet (RR Set that should exist does not)"; + case 9: + return "NotAuth (Server Not Authoritative for zone)"; + case 10: + return "NotZone (Name not contained in zone)"; + case 11: + return "DSOTYPENI (DSO-TYPE Not Implemented)"; + case 16: + return "BADVERS (Bad OPT Version) -or- BADSIG (TSIG Signature Failure)"; + case 17: + return "BADKEY (Key not recognized)"; + case 18: + return "BADTIME (Signature out of time window)"; + case 19: + return "BADMODE (Bad TKEY Mode)"; + case 20: + return "BADNAME (Duplicate key name)"; + case 21: + return "BADALG (Algorithm not supported)"; + case 22: + return "BADTRUNC (Bad Truncation)"; + case 23: + return "BADCOOKIE (Bad/missing Server Cookie)"; + default: + ; + } + + if((code >= 24 && code <= 3840) || (code >= 4096 && code <= 65535)) + return "Unassigned"; + else if(code >= 3841 && code <= 4095) + return "Reserved for Private Use"; + + // else: + return "Unknown"; +} + // Validate given hostname static bool valid_hostname(char* name, const char* clientip) { @@ -212,6 +270,10 @@ static char *__attribute__((malloc)) ngethostbyname(const char *host, const char dest.sin_addr.s_addr = htonl(INADDR_LOOPBACK); // 127.0.0.1 dest.sin_port = htons(config.dns.port.v.u16); // Configured DNS port + // Log query in debug mode + log_debug(DEBUG_RESOLVER, "Resolving PTR \"%s\" on 127.0.0.1#%u", host, config.dns.port.v.u16); + + // Send the query const size_t questionlen = sizeof(struct DNS_HEADER) + (strlen((const char*)qname) + 1) + sizeof(struct QUESTION); if(sendto(s, buf, questionlen, 0, (struct sockaddr*)&dest, sizeof(dest)) < 0) { @@ -237,6 +299,18 @@ static char *__attribute__((malloc)) ngethostbyname(const char *host, const char // Move ahead of the dns header and the query field reader = &buf[questionlen]; + // Log the status of the query + log_debug(DEBUG_RESOLVER, "DNS query for PTR \"%s\" returned status %s (%i)", + host, getDNScode(dns->rcode), dns->rcode); + + // Abort if the query was not successful + if(dns->tc != 0) + { + log_debug(DEBUG_RESOLVER, "Internal name lookup for %s was unsuccessful: DNS response was truncated", + ipaddr); + return strdup(""); + } + // Start reading answers uint16_t stop = 0; char *name = NULL; @@ -249,16 +323,21 @@ static char *__attribute__((malloc)) ngethostbyname(const char *host, const char reader = reader + sizeof(struct R_DATA); // We only care about PTR answers and ignore all others - if(ntohs(answers[i].resource->type) != T_PTR) + const uint16_t rtype = ntohs(answers[i].resource->type); + if(rtype != T_PTR) + { + log_debug(DEBUG_RESOLVER, "Answer %u is not of type PTR but %u (skipping)", + i, rtype); continue; + } // Read the answer and convert from network to host representation answers[i].rdata = name_fromDNS(reader, buf, &stop); reader = reader + stop; name = (char *)answers[i].rdata; - log_debug(DEBUG_RESOLVER, "Resolving %s (PTR \"%s\"): %u = \"%s\"", - ipaddr, answers[i].name, i, answers[i].rdata); + log_debug(DEBUG_RESOLVER, "Answer %u is PTR \"%s\" => \"%s\"", + i, answers[i].name, answers[i].rdata); // We break out of the loop if this is a valid hostname if(strlen(name) > 0 && valid_hostname(name, ipaddr)) @@ -652,14 +731,14 @@ static void resolveClients(const bool onlynew, const bool force_refreshing) continue; } - unlock_shm(); - // Check if we want to resolve an IPv6 address bool IPv6 = false; const char *ipaddr = NULL; if((ipaddr = getstr(ippos)) != NULL && strstr(ipaddr,":") != NULL) IPv6 = true; + unlock_shm(); + // If we're in refreshing mode (onlynew == false), we skip clients if // 1. We should not refresh any hostnames // 2. We should only refresh IPv4 client, but this client is IPv6 From 2843fb0203d9aa973ea16e5a78f5d1eddf0c87c4 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 26 May 2024 11:42:11 +0200 Subject: [PATCH 102/339] Fix forgotten update of lastQuery of upstream servers Signed-off-by: DL6ER --- src/dnsmasq_interface.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c index 4bef6109..a7edf4d8 100644 --- a/src/dnsmasq_interface.c +++ b/src/dnsmasq_interface.c @@ -1723,7 +1723,10 @@ static void FTL_forwarded(const unsigned int flags, const char *name, const unio upstreamsData *upstream = getUpstream(upstreamID, true); if(upstream != NULL) + { upstream->count++; + upstream->lastQuery = now; + } // Proceed only if // - current query has not been marked as replied to so far From 5db4ade0578dbebdd34bf9c4dd8582401b89357f Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 26 May 2024 21:22:29 +0200 Subject: [PATCH 103/339] Use TCP queries for internal name resolution to avoid truncated replies Signed-off-by: DL6ER --- .github/.codespellignore | 1 + src/args.c | 15 ++- src/resolve.c | 203 ++++++++++++++++++++++++++++++++------- src/resolve.h | 4 +- test/test_suite.bats | 17 +++- 5 files changed, 201 insertions(+), 39 deletions(-) diff --git a/.github/.codespellignore b/.github/.codespellignore index 645d300f..0dd61bef 100644 --- a/.github/.codespellignore +++ b/.github/.codespellignore @@ -8,3 +8,4 @@ requestors punycode bitap mmapped +dnsmasq diff --git a/src/args.c b/src/args.c index e4bbd0cd..15f596c3 100644 --- a/src/args.c +++ b/src/args.c @@ -499,7 +499,7 @@ void parse_args(int argc, char* argv[]) } // Local reverse name resolver - if(argc == 3 && strcasecmp(argv[1], "ptr") == 0) + if((argc == 3 || argc == 4) && strcasecmp(argv[1], "ptr") == 0) { // Enable stdout printing cli_mode = true; @@ -507,7 +507,18 @@ void parse_args(int argc, char* argv[]) // Need to get dns.port and the resolver settings readFTLconf(&config, false); - char *name = resolveHostname(argv[2], true); + // TCP or UDP (default)? + const bool tcp = argc == 4 && strcasecmp(argv[3], "tcp") == 0; + + // Create a socket + struct sockaddr_in dest; + const int sock = create_socket(tcp, &dest); + char *name = resolveHostname(sock, &dest, tcp, argv[2], true); + + // Close the socket + close(sock); + + // Exit early if no name was found if(name == NULL) exit(EXIT_FAILURE); diff --git a/src/resolve.c b/src/resolve.c index 342ff9a6..d8b51a83 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -33,6 +33,8 @@ #include "regex_r.h" // statis_assert() #include +// TCP_MAX_QUERIES +#include "dnsmasq/config.h" // Function Prototypes static void name_toDNS(unsigned char *dns, const size_t dnslen, const char *host, const size_t hostlen) __attribute__((nonnull(1,3))); @@ -212,10 +214,49 @@ bool __attribute__((pure)) resolve_this_name(const char *ipaddr) return true; } -// Perform a name lookup by sending a packet to ourselves -static char *__attribute__((malloc)) ngethostbyname(const char *host, const char *ipaddr) +int create_socket(bool tcp, struct sockaddr_in *dest) { - uint8_t buf[1024] = { 0 }; + // Create a UDP (datagram) or TCP (stream) socket + const int sock = socket(AF_INET, tcp ? SOCK_STREAM : SOCK_DGRAM, tcp ? IPPROTO_TCP : IPPROTO_UDP); + if(sock < 0) + { + log_err("Unable to create DNS resolver socket: %s", strerror(errno)); + return -1; + } + + // Set timeout for socket (2 seconds) + struct timeval tv; + tv.tv_sec = 2; + tv.tv_usec = 0; + if(setsockopt(sock, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv)) < 0) + { + log_err("Unable to set DNS resolver socket timeout: %s", strerror(errno)); + close(sock); + return -1; + } + + // Create socket destination structure + memset(dest, 0, sizeof(*dest)); + dest->sin_family = AF_INET; // IPv4 + dest->sin_addr.s_addr = htonl(INADDR_LOOPBACK); // 127.0.0.1 + dest->sin_port = htons(config.dns.port.v.u16); // Configured DNS port + + // Connect to the DNS server (only done for TCP as UDP is + // connectionless) + if(tcp && connect(sock, (struct sockaddr*)dest, sizeof(*dest)) < 0) + { + log_err("Unable to connect to DNS resolver: %s", strerror(errno)); + close(sock); + return -1; + } + + return sock; +} + +// Perform a name lookup by sending a packet to ourselves +static char *__attribute__((malloc)) ngethostbyname(const int sock, struct sockaddr_in *dest, const bool tcp, const char *host, const char *ipaddr) +{ + uint8_t buf[4096] = { 0 }; // buffer for DNS query uint8_t *qname = NULL, *reader = NULL; struct RES_RECORD answers[20] = { 0 }; // buffer for DNS replies struct DNS_HEADER *dns = NULL; @@ -262,42 +303,73 @@ static char *__attribute__((malloc)) ngethostbyname(const char *host, const char qinfo->qtype = htons(T_PTR); // Type of the query, A, MX, CNAME, NS etc qinfo->qclass = htons(1); // IN - - // UDP packet for DNS queries - const int s = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP); - struct sockaddr_in dest = { 0 }; - dest.sin_family = AF_INET; // IPv4 - dest.sin_addr.s_addr = htonl(INADDR_LOOPBACK); // 127.0.0.1 - dest.sin_port = htons(config.dns.port.v.u16); // Configured DNS port + const size_t len = sizeof(struct DNS_HEADER) + (strlen((const char*)qname) + 1) + sizeof(struct QUESTION); // Log query in debug mode - log_debug(DEBUG_RESOLVER, "Resolving PTR \"%s\" on 127.0.0.1#%u", host, config.dns.port.v.u16); + log_debug(DEBUG_RESOLVER, "Resolving PTR \"%s\" on 127.0.0.1#%u (%s)", + host, config.dns.port.v.u16, tcp ? "TCP" : "UDP"); - // Send the query - const size_t questionlen = sizeof(struct DNS_HEADER) + (strlen((const char*)qname) + 1) + sizeof(struct QUESTION); - if(sendto(s, buf, questionlen, 0, (struct sockaddr*)&dest, sizeof(dest)) < 0) + if(!tcp) { - perror("sendto failed"); - close(s); - return strdup(""); - } + // Send the query + socklen_t addrlen = sizeof(*dest); + if(sendto(sock, buf, len, 0, (struct sockaddr*)dest, addrlen) < 0) + { + log_err("Cannot send UDP DNS query: %s", strerror(errno)); + return strdup(""); + } - // Receive the answer - socklen_t addrlen = sizeof(dest); - if(recvfrom (s, buf, sizeof(buf), 0, (struct sockaddr*)&dest, &addrlen) < 0) + // Receive the answer + if(recvfrom (sock, buf, sizeof(buf), 0, (struct sockaddr*)dest, &addrlen) < 0) + { + log_err("Cannot receive UDP DNS reply: %s", strerror(errno)); + return strdup(""); + } + } + else { - perror("recvfrom failed"); - close(s); - return strdup(""); - } + // Send the query + // For TCP streams, we first have to send the length of the data + // we are sending. The reason for this is that with TCP, we are + // not sending messages (datagrams) but a continuous stream of + // bytes. We therefore need a way to tell the receiver about + // this length of the message. + uint16_t prefix = htons(len & 0xffffu); + if(send(sock, &prefix, sizeof(prefix), 0) < 0 || + send(sock, buf, len, 0) < 0) + { + log_err("Cannot send TCP DNS query: %s", strerror(errno)); + return strdup(""); + } - // Close socket - close(s); + // Receive the answer, first the length of the message ... + prefix = 0; + if(recv(sock, &prefix, sizeof(prefix), 0) < 0) + { + log_err("Cannot receive TCP DNS reply (1): %s", strerror(errno)); + return strdup(""); + } + prefix = ntohs(prefix); + + // Sanity check the length of the message + if(prefix > sizeof(buf)) + { + log_err("Received TCP DNS reply is too long (%u bytes)", prefix); + return strdup(""); + } + bzero(buf, prefix + 1); + // ... then the message itself + if(recv(sock, buf, sizeof(buf), 0) < 0) + { + log_err("Cannot receive TCP DNS reply (2): %s", strerror(errno)); + return strdup(""); + } + } // Parse the reply dns = (struct DNS_HEADER*) buf; // Move ahead of the dns header and the query field - reader = &buf[questionlen]; + reader = &buf[len]; // Log the status of the query log_debug(DEBUG_RESOLVER, "DNS query for PTR \"%s\" returned status %s (%i)", @@ -447,7 +519,7 @@ static u_char * __attribute__((malloc)) __attribute__((nonnull(1,2,3))) name_fro } // Strip off the trailing dot - name[i-1] = '\0'; + name[i > 0 ? i-1 : i] = '\0'; return name; } @@ -477,7 +549,8 @@ static void __attribute__((nonnull(1,3))) name_toDNS(unsigned char *dns, const s *dns++='\0'; } -char *__attribute__((malloc)) resolveHostname(const char *addr, const bool force) +char *__attribute__((malloc)) resolveHostname(const int sock, struct sockaddr_in *dest, + const bool tcp, const char *addr, const bool force) { // Get host name char *hostn = NULL; @@ -600,11 +673,11 @@ char *__attribute__((malloc)) resolveHostname(const char *addr, const bool force // Get host name by making a reverse lookup to ourselves (server at 127.0.0.1 with port 53) // We implement a minimalistic resolver here as we cannot rely on the system resolver using whatever // nameserver we configured in /etc/resolv.conf - return ngethostbyname(inaddr, addr); + return ngethostbyname(sock, dest, tcp, inaddr, addr); } // Resolve upstream destination host names -static size_t resolveAndAddHostname(size_t ippos, size_t oldnamepos) +static size_t resolveAndAddHostname(const int sock, struct sockaddr_in *dest, const bool tcp, size_t ippos, size_t oldnamepos) { // Get IP and host name strings. They are cloned in case shared memory is // resized before the next lock @@ -629,7 +702,7 @@ static size_t resolveAndAddHostname(size_t ippos, size_t oldnamepos) // Important: Don't hold a lock while resolving as the main thread // (dnsmasq) needs to be operable during the call to resolveHostname() - char *newname = resolveHostname(ipaddr, false); + char *newname = resolveHostname(sock, dest, tcp, ipaddr, false); // If no hostname was found, try to obtain hostname from the network table // This may be disabled due to a user setting @@ -680,7 +753,18 @@ static void resolveClients(const bool onlynew, const bool force_refreshing) int clientscount = counters->clients; unlock_shm(); + // Create DNS client socket + const bool tcp = true; + struct sockaddr_in dest = { 0 }; + int sock = create_socket(tcp, &dest); + if(sock < 0) + { + log_err("Unable to create DNS resolver socket, client host name resolution failed"); + return; + } + int skipped = 0; + unsigned int queries = 0u; for(int clientID = 0; clientID < clientscount; clientID++) { // Memory access needs to get locked @@ -768,8 +852,24 @@ static void resolveClients(const bool onlynew, const bool force_refreshing) continue; } + // We need to reconnect after a certain number of queries due to + // dnsmasq-internal limits + if(tcp && ++queries > TCP_MAX_QUERIES - 1) + { + close(sock); + sock = create_socket(tcp, &dest); + if(sock < 0) + { + log_err("Unable to recreate to DNS resolver socket, client host name resolution failed"); + return; + } + + // Reset query counter + queries = 0; + } + // Obtain/update hostname of this client - size_t newnamepos = resolveAndAddHostname(ippos, oldnamepos); + size_t newnamepos = resolveAndAddHostname(sock, &dest, tcp, ippos, oldnamepos); lock_shm(); // Get client pointer for the second time (writing data) @@ -795,6 +895,9 @@ static void resolveClients(const bool onlynew, const bool force_refreshing) unlock_shm(); } + // Close socket + close(sock); + log_debug(DEBUG_RESOLVER, "%i / %i client host names resolved", clientscount-skipped, clientscount); } @@ -808,7 +911,18 @@ static void resolveUpstreams(const bool onlynew) int upstreams = counters->upstreams; unlock_shm(); + // Create socket + const bool tcp = false; + struct sockaddr_in dest = { 0 }; + int sock = create_socket(tcp, &dest); + if(sock < 0) + { + log_err("Unable to create DNS resolver socket, upstream host name resolution failed"); + return; + } + int skipped = 0; + unsigned int queries = 0u; for(int upstreamID = 0; upstreamID < upstreams; upstreamID++) { // Memory access needs to get locked @@ -853,8 +967,24 @@ static void resolveUpstreams(const bool onlynew) continue; } + // We need to reconnect after a certain number of queries due to + // dnsmasq-internal limits + if(tcp && ++queries > TCP_MAX_QUERIES - 1) + { + close(sock); + sock = create_socket(tcp, &dest); + if(sock < 0) + { + log_err("Unable to recreate to DNS resolver socket, client host name resolution failed"); + return; + } + + // Reset query counter + queries = 0; + } + // Obtain/update hostname of this client - size_t newnamepos = resolveAndAddHostname(ippos, oldnamepos); + size_t newnamepos = resolveAndAddHostname(sock, &dest, tcp, ippos, oldnamepos); lock_shm(); // Get upstream pointer for the second time (writing data) @@ -880,6 +1010,9 @@ static void resolveUpstreams(const bool onlynew) unlock_shm(); } + // Close socket + close(sock); + log_debug(DEBUG_RESOLVER, "%i / %i upstream server host names resolved", upstreams-skipped, upstreams); } diff --git a/src/resolve.h b/src/resolve.h index 2dfc50e1..64b972aa 100644 --- a/src/resolve.h +++ b/src/resolve.h @@ -11,7 +11,9 @@ #define RESOLVE_H void *DNSclient_thread(void *val); -char *resolveHostname(const char *addr, const bool force) __attribute__((malloc)); +int create_socket(bool tcp, struct sockaddr_in *dest); +char *resolveHostname(const int sock, struct sockaddr_in *dest, const bool tcp, + const char *addr, const bool force) __attribute__((malloc)); bool resolve_names(void) __attribute__((pure)); bool resolve_this_name(const char *ipaddr) __attribute__((pure)); diff --git a/test/test_suite.bats b/test/test_suite.bats index 94c04019..da994606 100644 --- a/test/test_suite.bats +++ b/test/test_suite.bats @@ -1729,15 +1729,30 @@ [[ ${lines[0]} == "ce4c01340ef46bf3bc26831f7c53763d57c863528826aa795f1da5e16d6e7b2d test/test.pem" ]] } -@test "Internal IP -> name resolution works" { +@test "Internal IP -> name resolution works (UDP IPv4)" { run bash -c "./pihole-FTL ptr 127.0.0.1 | tail -n1" printf "%s\n" "${lines[@]}" [[ ${lines[0]} == "localhost" ]] +} + +@test "Internal IP -> name resolution works (UDP IPv6)" { run bash -c "./pihole-FTL ptr ::1 | tail -n1" printf "%s\n" "${lines[@]}" [[ ${lines[0]} == "localhost" ]] } +@test "Internal IP -> name resolution works (TCP IPv4)" { + run bash -c "./pihole-FTL ptr 127.0.0.1 tcp | tail -n1" + printf "%s\n" "${lines[@]}" + [[ ${lines[0]} == "localhost" ]] +} + +@test "Internal IP -> name resolution works (TCP IPv6)" { + run bash -c "./pihole-FTL ptr ::1 tcp | tail -n1" + printf "%s\n" "${lines[@]}" + [[ ${lines[0]} == "localhost" ]] +} + @test "API validation" { run python3 test/api/checkAPI.py printf "%s\n" "${lines[@]}" From 090c1adabf6172cf2d861ce7cb0c18658c5d6f4b Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 27 May 2024 08:25:30 +0200 Subject: [PATCH 104/339] Implement later retrying if name resolution attempt failed temporarily (e.g., broken pipe). Also use TCP protocol for upstream server name resolution Signed-off-by: DL6ER --- src/resolve.c | 80 +++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 62 insertions(+), 18 deletions(-) diff --git a/src/resolve.c b/src/resolve.c index d8b51a83..ba2d5523 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -291,7 +291,7 @@ static char *__attribute__((malloc)) ngethostbyname(const int sock, struct socka if(hname == NULL) { log_err("Unable to allocate memory for hname"); - return strdup(""); + return NULL; } strncpy(hname, host, hnamelen); strncat(hname, ".", hnamelen - strlen(hname)); @@ -316,14 +316,14 @@ static char *__attribute__((malloc)) ngethostbyname(const int sock, struct socka if(sendto(sock, buf, len, 0, (struct sockaddr*)dest, addrlen) < 0) { log_err("Cannot send UDP DNS query: %s", strerror(errno)); - return strdup(""); + return NULL; } // Receive the answer if(recvfrom (sock, buf, sizeof(buf), 0, (struct sockaddr*)dest, &addrlen) < 0) { log_err("Cannot receive UDP DNS reply: %s", strerror(errno)); - return strdup(""); + return NULL; } } else @@ -339,7 +339,7 @@ static char *__attribute__((malloc)) ngethostbyname(const int sock, struct socka send(sock, buf, len, 0) < 0) { log_err("Cannot send TCP DNS query: %s", strerror(errno)); - return strdup(""); + return NULL; } // Receive the answer, first the length of the message ... @@ -347,7 +347,7 @@ static char *__attribute__((malloc)) ngethostbyname(const int sock, struct socka if(recv(sock, &prefix, sizeof(prefix), 0) < 0) { log_err("Cannot receive TCP DNS reply (1): %s", strerror(errno)); - return strdup(""); + return NULL; } prefix = ntohs(prefix); @@ -355,14 +355,14 @@ static char *__attribute__((malloc)) ngethostbyname(const int sock, struct socka if(prefix > sizeof(buf)) { log_err("Received TCP DNS reply is too long (%u bytes)", prefix); - return strdup(""); + return NULL; } bzero(buf, prefix + 1); // ... then the message itself if(recv(sock, buf, sizeof(buf), 0) < 0) { log_err("Cannot receive TCP DNS reply (2): %s", strerror(errno)); - return strdup(""); + return NULL; } } @@ -380,7 +380,7 @@ static char *__attribute__((malloc)) ngethostbyname(const int sock, struct socka { log_debug(DEBUG_RESOLVER, "Internal name lookup for %s was unsuccessful: DNS response was truncated", ipaddr); - return strdup(""); + return NULL; } // Start reading answers @@ -608,7 +608,7 @@ char *__attribute__((malloc)) resolveHostname(const int sock, struct sockaddr_in if(inaddr == NULL) { log_err("Unable to allocate memory for reverse lookup"); - return strdup(""); + return NULL; } // Convert IPv6 address to reverse lookup format @@ -658,7 +658,7 @@ char *__attribute__((malloc)) resolveHostname(const int sock, struct sockaddr_in if(inaddr == NULL) { log_err("Unable to allocate memory for reverse lookup"); - return strdup(""); + return NULL; } // Convert IPv4 address to reverse lookup format @@ -677,7 +677,8 @@ char *__attribute__((malloc)) resolveHostname(const int sock, struct sockaddr_in } // Resolve upstream destination host names -static size_t resolveAndAddHostname(const int sock, struct sockaddr_in *dest, const bool tcp, size_t ippos, size_t oldnamepos) +static size_t resolveAndAddHostname(const int sock, struct sockaddr_in *dest, const bool tcp, + size_t ippos, size_t oldnamepos, bool *success) { // Get IP and host name strings. They are cloned in case shared memory is // resized before the next lock @@ -703,6 +704,18 @@ static size_t resolveAndAddHostname(const int sock, struct sockaddr_in *dest, co // Important: Don't hold a lock while resolving as the main thread // (dnsmasq) needs to be operable during the call to resolveHostname() char *newname = resolveHostname(sock, dest, tcp, ipaddr, false); + if(newname == NULL) + { + // We could not resolve the hostname, so we keep the old one + // and mark the entry as not new + log_debug(DEBUG_RESOLVER, " ---> \"%s\" (failed to resolve)", oldname); + + // Free allocated memory + *success = false; + free(ipaddr); + free(oldname); + return oldnamepos; + } // If no hostname was found, try to obtain hostname from the network table // This may be disabled due to a user setting @@ -721,6 +734,8 @@ static size_t resolveAndAddHostname(const int sock, struct sockaddr_in *dest, co { lock_shm(); size_t newnamepos = addstr(newname); + + // Free allocated memory // newname has already been checked against NULL // so we can safely free it free(newname); @@ -795,7 +810,7 @@ static void resolveClients(const bool onlynew, const bool force_refreshing) // Limit for a "recently active" client is two hours ago if(!force_refreshing && !onlynew && client->lastQuery < now - 2*60*60) { - log_debug(DEBUG_RESOLVER, "Skipping client %s (%s) because it was inactive for %i seconds", + log_debug(DEBUG_RESOLVER, "Skipping client %s -> \"%s\" because it was inactive for %i seconds", getstr(ippos), getstr(oldnamepos), (int)(now - client->lastQuery)); unlock_shm(); @@ -807,7 +822,7 @@ static void resolveClients(const bool onlynew, const bool force_refreshing) // If not, we will try to re-resolve all known clients if(!force_refreshing && onlynew && !newflag) { - log_debug(DEBUG_RESOLVER, "Skipping client %s (%s) because it is not new", + log_debug(DEBUG_RESOLVER, "Skipping client %s -> \"%s\" because it is not new", getstr(ippos), getstr(oldnamepos)); unlock_shm(); @@ -844,7 +859,7 @@ static void resolveClients(const bool onlynew, const bool force_refreshing) reason = "Looking only for unknown hostnames"; lock_shm(); - log_debug(DEBUG_RESOLVER, "Skipping client %s (%s) because it should not be refreshed: %s", + log_debug(DEBUG_RESOLVER, "Skipping client %s -> \"%s\" because it should not be refreshed: %s", getstr(ippos), getstr(oldnamepos), reason); unlock_shm(); } @@ -869,7 +884,8 @@ static void resolveClients(const bool onlynew, const bool force_refreshing) } // Obtain/update hostname of this client - size_t newnamepos = resolveAndAddHostname(sock, &dest, tcp, ippos, oldnamepos); + bool success = true; + size_t newnamepos = resolveAndAddHostname(sock, &dest, tcp, ippos, oldnamepos, &success); lock_shm(); // Get client pointer for the second time (writing data) @@ -885,6 +901,20 @@ static void resolveClients(const bool onlynew, const bool force_refreshing) continue; } + if(!success) + { + // We could not resolve the hostname, so we keep the old one + // and mark the entry as not new - it will be retried later + client->flags.new = false; + + log_debug(DEBUG_RESOLVER, "Client %s -> \"%s\" could not be resolved, retrying later", + getstr(ippos), getstr(oldnamepos)); + + unlock_shm(); + continue; + } + + // else: // Store obtained host name (may be unchanged) client->namepos = newnamepos; // Mark entry as not new @@ -912,7 +942,7 @@ static void resolveUpstreams(const bool onlynew) unlock_shm(); // Create socket - const bool tcp = false; + const bool tcp = true; struct sockaddr_in dest = { 0 }; int sock = create_socket(tcp, &dest); if(sock < 0) @@ -945,7 +975,7 @@ static void resolveUpstreams(const bool onlynew) // Limit for a "recently active" upstream server is two hours ago if(upstream->lastQuery < now - 2*60*60) { - log_debug(DEBUG_RESOLVER, "Skipping upstream %s (%s) because it was inactive for %i seconds", + log_debug(DEBUG_RESOLVER, "Skipping upstream %s -> \"%s\" because it was inactive for %i seconds", getstr(ippos), getstr(oldnamepos), (int)(now - upstream->lastQuery)); unlock_shm(); @@ -984,7 +1014,8 @@ static void resolveUpstreams(const bool onlynew) } // Obtain/update hostname of this client - size_t newnamepos = resolveAndAddHostname(sock, &dest, tcp, ippos, oldnamepos); + bool success = true; + size_t newnamepos = resolveAndAddHostname(sock, &dest, tcp, ippos, oldnamepos, &success); lock_shm(); // Get upstream pointer for the second time (writing data) @@ -1000,6 +1031,19 @@ static void resolveUpstreams(const bool onlynew) continue; } + if(!success) + { + // We could not resolve the hostname, so we keep the old one + // and mark the entry as not new - it will be retried later + upstream->flags.new = false; + + log_debug(DEBUG_RESOLVER, "Upstream %s -> \"%s\" could not be resolved, retrying later", + getstr(ippos), getstr(oldnamepos)); + + unlock_shm(); + continue; + } + // Store obtained host name (may be unchanged) upstream->namepos = newnamepos; // Mark entry as not new From 5a539a9d5c93ebec67d9d87e3d1580e09ca7d9f9 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 28 May 2024 19:49:15 +0200 Subject: [PATCH 105/339] Add NTP-server/client implementation Signed-off-by: DL6ER --- src/CMakeLists.txt | 2 + src/api/docs/content/specs/config.yaml | 26 ++ src/args.c | 12 + src/config/config.c | 32 ++ src/config/config.h | 11 + src/config/dnsmasq_config.c | 9 + src/dnsmasq_interface.c | 7 +- src/ntp/CMakeLists.txt | 19 ++ src/ntp/client.c | 226 +++++++++++++++ src/ntp/ntp.h | 29 ++ src/ntp/server.c | 387 +++++++++++++++++++++++++ test/pihole.toml | 20 ++ test/test_suite.bats | 20 +- 13 files changed, 793 insertions(+), 7 deletions(-) create mode 100644 src/ntp/CMakeLists.txt create mode 100644 src/ntp/client.c create mode 100644 src/ntp/ntp.h create mode 100644 src/ntp/server.c diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index c040c688..cdfe9493 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -274,6 +274,7 @@ add_executable(pihole-FTL $ $ $ + $ ) if(STATIC) set_target_properties(pihole-FTL PROPERTIES LINK_SEARCH_START_STATIC ON) @@ -314,6 +315,7 @@ add_subdirectory(tre-regex) add_subdirectory(syscalls) add_subdirectory(config) add_subdirectory(tools) +add_subdirectory(ntp) find_library(LIBREADLINE NAMES libreadline${CMAKE_STATIC_LIBRARY_SUFFIX} readline) find_library(LIBHISTORY NAMES libhistory${CMAKE_STATIC_LIBRARY_SUFFIX} history) diff --git a/src/api/docs/content/specs/config.yaml b/src/api/docs/content/specs/config.yaml index 0b785754..853c6450 100644 --- a/src/api/docs/content/specs/config.yaml +++ b/src/api/docs/content/specs/config.yaml @@ -324,6 +324,25 @@ components: type: array items: type: string + ntp: + type: object + properties: + ipv4: + type: object + properties: + active: + type: boolean + address: + type: string + x-format: ipv4 + ipv6: + type: object + properties: + active: + type: boolean + address: + type: string + x-format: ipv6 resolver: type: object properties: @@ -656,6 +675,13 @@ components: hosts: - "11:22:33:44:55:66,192.168.1.123" - "11:22:33:44:55:67,192.168.1.124,hostname" + ntp: + ipv4: + active: true + address: "" + ipv6: + active: true + address: "" resolver: resolveIPv4: true resolveIPv6: true diff --git a/src/args.c b/src/args.c index e4bbd0cd..1e7431ae 100644 --- a/src/args.c +++ b/src/args.c @@ -66,6 +66,8 @@ #include "files.h" // resolveHostname() #include "resolve.h" +// ntp_client() +#include "ntp/ntp.h" // defined in dnsmasq.c extern void print_dnsmasq_version(const char *yellow, const char *green, const char *bold, const char *normal); @@ -305,6 +307,16 @@ void parse_args(int argc, char* argv[]) exit(write_teleporter_zip_to_disk() ? EXIT_SUCCESS : EXIT_FAILURE); } + // Create test NTP client + if((argc == 2 || argc == 3) && strcmp(argv[1], "ntp-client") == 0) + { + // Enable stdout printing + cli_mode = true; + log_ctrl(false, true); + const char *server = argc == 3 ? argv[2] : "127.0.0.1"; + exit(ntp_client(server) ? EXIT_SUCCESS : EXIT_FAILURE); + } + // Import teleporter archive through CLI if(argc == 3 && strcmp(argv[1], "--teleporter") == 0) { diff --git a/src/config/config.c b/src/config/config.c index f5bb3fa1..444f2b38 100644 --- a/src/config/config.c +++ b/src/config/config.c @@ -794,6 +794,38 @@ void initConfig(struct config *conf) conf->dhcp.hosts.c = validate_stub; // Type-based checking + dnsmasq syntax checking + // struct ntp + conf->ntp.ipv4.active.k = "ntp.ipv4.active"; + conf->ntp.ipv4.active.h = "Should FTL act as an NTP server (IPv4)?"; + conf->ntp.ipv4.active.t = CONF_BOOL; + conf->ntp.ipv4.active.f = FLAG_RESTART_FTL; + conf->ntp.ipv4.active.d.b = true; + conf->ntp.ipv4.active.c = validate_stub; // Only type-based checking + + conf->ntp.ipv4.address.k = "ntp.ipv4.address"; + conf->ntp.ipv4.address.h = "IPv4 address to listen on for NTP requests"; + conf->ntp.ipv4.address.a = cJSON_CreateStringReference(" or empty string (\"\") for wildcard"); + conf->ntp.ipv4.address.t = CONF_STRUCT_IN_ADDR; + conf->ntp.ipv4.address.f = FLAG_RESTART_FTL; + memset(&conf->ntp.ipv4.address.d.in_addr, 0, sizeof(struct in_addr)); + conf->ntp.ipv4.address.c = validate_stub; // Only type-based checking + + conf->ntp.ipv6.active.k = "ntp.ipv6.active"; + conf->ntp.ipv6.active.h = "Should FTL act as an NTP server (IPv6)?"; + conf->ntp.ipv6.active.t = CONF_BOOL; + conf->ntp.ipv6.active.f = FLAG_RESTART_FTL; + conf->ntp.ipv6.active.d.b = true; + conf->ntp.ipv6.active.c = validate_stub; // Only type-based checking + + conf->ntp.ipv6.address.k = "ntp.ipv6.address"; + conf->ntp.ipv6.address.h = "IPv6 address to listen on for NTP requests"; + conf->ntp.ipv6.address.a = cJSON_CreateStringReference(" or empty string (\"\") for wildcard"); + conf->ntp.ipv6.address.t = CONF_STRUCT_IN6_ADDR; + conf->ntp.ipv6.address.f = FLAG_RESTART_FTL; + memset(&conf->ntp.ipv6.address.d.in6_addr, 0, sizeof(struct in6_addr)); + conf->ntp.ipv6.address.c = validate_stub; // Only type-based checking + + // struct resolver conf->resolver.resolveIPv6.k = "resolver.resolveIPv6"; conf->resolver.resolveIPv6.h = "Should FTL try to resolve IPv6 addresses to hostnames?"; diff --git a/src/config/config.h b/src/config/config.h index 6cd4f05e..b22cddf0 100644 --- a/src/config/config.h +++ b/src/config/config.h @@ -190,6 +190,17 @@ struct config { struct conf_item hosts; } dhcp; + struct { + struct { + struct conf_item active; + struct conf_item address; + } ipv4; + struct { + struct conf_item active; + struct conf_item address; + } ipv6; + } ntp; + struct { struct conf_item resolveIPv4; struct conf_item resolveIPv6; diff --git a/src/config/dnsmasq_config.c b/src/config/dnsmasq_config.c index 8943c23f..a9da85ea 100644 --- a/src/config/dnsmasq_config.c +++ b/src/config/dnsmasq_config.c @@ -582,6 +582,15 @@ bool __attribute__((const)) write_dnsmasq_config(struct config *conf, bool test_ fputs("log-dhcp\n\n", pihole_conf); } + // Check if IPv4 NTP server is active and broadcast it as DHCP option + if(conf->ntp.ipv4.active.v.b) + { + fputs("# Add NTP server to DHCP\n", pihole_conf); + // The special address 0.0.0.0 is taken to mean "the + // address of the machine running the DHCP server" + fputs("dhcp-option=option:ntp-server,0.0.0.0", pihole_conf); + } + // Add per-host parameters if(cJSON_GetArraySize(conf->dhcp.hosts.v.json) > 0) { diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c index 4bef6109..6dc59c7a 100644 --- a/src/dnsmasq_interface.c +++ b/src/dnsmasq_interface.c @@ -58,6 +58,8 @@ #include "config/config.h" // FTL_fork_and_bind_sockets() #include "main.h" +// ntp_server_start() +#include "ntp/ntp.h" // Private prototypes static void print_flags(const unsigned int flags); @@ -2888,6 +2890,9 @@ void FTL_fork_and_bind_sockets(struct passwd *ent_pw, bool dnsmasq_start) // so they will not listen to real-time signals handle_realtime_signals(); + // Initialize NTP server + ntp_server_start(); + // We will use the attributes object later to start all threads in // detached mode pthread_attr_t attr; @@ -3601,4 +3606,4 @@ void FTL_connection_error(const char *reason, const union mysockaddr *addr) if(server != NULL) free(server); } -} \ No newline at end of file +} diff --git a/src/ntp/CMakeLists.txt b/src/ntp/CMakeLists.txt new file mode 100644 index 00000000..7eca589a --- /dev/null +++ b/src/ntp/CMakeLists.txt @@ -0,0 +1,19 @@ +# Pi-hole: A black hole for Internet advertisements +# (c) 2024 Pi-hole, LLC (https://pi-hole.net) +# Network-wide ad blocking via your own hardware. +# +# FTL Engine +# /src/ntp/CMakeList.txt +# +# This file is copyright under the latest version of the EUPL. +# Please see LICENSE file for your rights under this license. + +set(ntp_sources + server.c + client.c + ntp.h + ) + +add_library(ntp OBJECT ${ntp_sources}) +target_compile_options(ntp PRIVATE "${EXTRAWARN}") +target_include_directories(ntp PRIVATE ${PROJECT_SOURCE_DIR}/src) diff --git a/src/ntp/client.c b/src/ntp/client.c new file mode 100644 index 00000000..9919daf1 --- /dev/null +++ b/src/ntp/client.c @@ -0,0 +1,226 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2024 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* NTP client routines +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ + +// close() +#include +// clock_gettime() +#include +// socket(), connect(), send(), recv(), AF_INET, SOCK_DGRAM, IPPROTO_UDP +#include +// getaddrinfo(), freeaddrinfo(), struct addrinfo +#include +// memcpy() +#include +// pow() +#include +// ctime() +#include +// errno +#include + +#include "ntp.h" +#include "log.h" + +// Create minimal NTP request, see server implementation for details about the +// packet structure +static bool request(int fd, uint32_t org[2]) +{ + // NTP Packet buffer + unsigned char buf[48] = {0}; + + // LI = 0, VN = 4 (current version), Mode = 3 (Client) + buf[0] = 0x23; + + // Set Origin Timestamp + gettime32(org, true); + memcpy(&buf[40], &org[0], 2 * sizeof(uint32_t)); + + // Send request + if(send(fd, buf, 48, 0) != 48) + { + log_warn("Failed to send data to NTP server: %s", strerror(errno)); + return false; + } + + return true; +} + +static bool get_reply(int fd, uint32_t org_[2]) +{ + // NTP Packet buffer + unsigned char buf[48]; + // NTP Packet buffer as uint32_t + uint32_t *pt = (uint32_t *)((void*)&buf[24]);; + + // Receive reply + if(recv(fd, buf, 48, 0) < 48) + { + log_warn("Failed to receive data from NTP server: %s", strerror(errno)); + return false; + } + + // Extract precision of server clock + signed char rho = (signed char)buf[3]; + if(rho < -32 || rho > 0) + { + // Accepted limits are 2^-32 (~ 0.2 nanoseconds) + // to 2^0 (= 1 second) + log_warn("Received NTP reply has invalid precision: 2^(%i), assuming microsecond accuracy", rho); + rho = -19; + } + // Compute precision of server clock in seconds 2^rho + const double s_rho = pow(2, rho); + + // Extract Transmit Timestamp + // org = Origin Timestamp (Transmit Timestamp @ Client) + uint32_t org[2]; + org[0] = ntohl(*pt++); + org[1] = ntohl(*pt++); + // rec = Receive Timestamp (Receive Timestamp @ Server) + uint32_t rec[2]; + rec[0] = ntohl(*pt++); + rec[1] = ntohl(*pt++); + // xmt = Transmit Timestamp (Transmit Timestamp @ Server) + uint32_t xmt[2]; + xmt[0] = ntohl(*pt++); + xmt[1] = ntohl(*pt++); + + // dst = Destination Timestamp (Receive Timestamp @ Client) + uint32_t dst[2]; + gettime32(dst, false); + + // Check org_ and org are identical (otherwise, the reply corresponds to + // a different request and should be ignored), note that the byte order + // of the received packet is already converted while org_ is still in + // network byte order + if(ntohl(org_[0]) != org[0] || ntohl(org_[1]) != org[1]) + { + log_warn("Received NTP reply does not match request"); + return false; + } + + // Check stratum, mode, version, etc. + if((buf[0] & 0x07) != 4) + { + log_warn("Received NTP reply has invalid version"); + return false; + } + + // Calculate delay and offset + const double tfrac = 4294967296.0; // 2^32 as double + const double T1 = org[0] + org[1] / tfrac; + const double T2 = rec[0] + rec[1] / tfrac; + const double T3 = xmt[0] + xmt[1] / tfrac; + const double T4 = dst[0] + dst[1] / tfrac; + + // RFC 5905, Section 8: On-wire protocol + // It is recommended to use double precision floating point arithmetic + // for the calculations to allow unambiguous interpretation of the + // results within the maximum adjustment range of 68 years. + + // Compute offset of client clock relative to server clock + const double theta = ( ( T2 - T1 ) + ( T3 - T4 ) ) / 2; + // Compute round-trip delay + double delta = ( T4 - T1 ) - ( T3 - T2 ); + + // In some scenarios where the initial frequency offset of the client is + // relatively large and the actual propagation time small, it is + // possible for the delay computation to become negative. For instance, + // if the frequency difference is 100 ppm and the interval T4-T1 is 64 + // s, the apparent delay is -6.4 ms. Since negative values are + // misleading in subsequent computations, the value of delta should be + // clamped not less than s.rho, where s.rho is the system precision + // described in Section 11.1, expressed in seconds. + if(delta < s_rho) + { + log_warn("Negative delay detected, clamping to 0"); + delta = 0; + } + + // Print current time at client + char client_time_str[26]; + const time_t client_time = dst[0]; + ctime_r(&client_time, client_time_str); + // Remove trailing newline + client_time_str[24] = '\0'; + log_info("Current time at client: %s", client_time_str); + + // Print current time at server + char server_time_str[26]; + const time_t server_time = xmt[0]; + // Remove trailing newline + server_time_str[24] = '\0'; + ctime_r(&server_time, server_time_str); + log_info("Current time at server: %s", server_time_str); + + // Print offset and delay + log_info("Time offset: %e s", theta); + log_info("Round-trip delay: %e s", delta); + + // Offset and delay larger than 0.1 seconds are considered as invalid + // during local testing + return theta < 0.1 && delta < 0.1; +} + +bool ntp_client(const char *server) +{ + const int protocol = strchr(server, ':') != NULL ? AF_INET6 : AF_INET; + + // Create UDP socket + const int s = socket(protocol, SOCK_DGRAM, IPPROTO_UDP); + if(s == -1) + { + log_err("Cannot create UDP socket"); + return false; + } + + // Set socket timeout to 2 seconds + struct timeval tv; + tv.tv_sec = 2; + tv.tv_usec = 0; + if(setsockopt(s, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv)) != 0) + { + log_err("Cannot set socket timeout"); + close(s); + return false; + } + + // Resolve server address + struct addrinfo *saddr; + if(getaddrinfo(server, "123", NULL, &saddr) != 0) + { + log_err("Cannot resolve NTP server address"); + close(s); + return false; + } + + // Set address to send to/receive from + if(connect(s, saddr->ai_addr, saddr->ai_addrlen) != 0) + { + log_err("Cannot connect to NTP server"); + close(s); + return false; + } + freeaddrinfo(saddr); + + // Send request + uint32_t org[2]; + if(!request(s, org)) + { + close(s); + return false; + } + + // Get reply + const bool status = get_reply(s, org); + close(s); + + return status; +} diff --git a/src/ntp/ntp.h b/src/ntp/ntp.h new file mode 100644 index 00000000..c6e159c5 --- /dev/null +++ b/src/ntp/ntp.h @@ -0,0 +1,29 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2024 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* NTP prototypes +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ + +#ifndef NTP_H +#define NTP_H + +// uint64_t +#include +// bool +#include + +//uint64_t gettime32(void); +void gettime32(uint32_t ts[], const bool netorder); +//uint64_t gettime64(void); + +bool ntp_server_start(void); +bool ntp_client(const char *server); + +#endif // NTP_H + + + diff --git a/src/ntp/server.c b/src/ntp/server.c new file mode 100644 index 00000000..86c54667 --- /dev/null +++ b/src/ntp/server.c @@ -0,0 +1,387 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2024 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* NTP server routines +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ + +// exit(0) +#include +// memcpy() +#include +// close() +#include +// fork(), wait() +#include +// clock_gettime() +#include +//#include +#include +// wait() +#include +// htonl(), etc. +#include +// errno +#include +// ctime() +#include +// log2() +#include +// pthread_create +#include +// PR_SET_NAME +#include + +#include "ntp.h" +#include "log.h" +#include "config/config.h" + +// Retrieves the current system time, adjusts it to a 1900 epoch, converts it to +// a 32-bit fraction of a second, and optionally converts it to network byte +// order. +void gettime32(uint32_t tv[], const bool netorder) +{ + struct timespec ts; + // CLOCK_REALTIME is the system-wide realtime clock. + // It is both affected by discontinuous jumps in the system time (e.g., + // if the system administrator manually changes the clock), and by the + // incremental adjustments performed by adjtime(3) and NTP. + clock_gettime(CLOCK_REALTIME, &ts); + + // Set the epoch to 1900 (add seconds from 1900 to 1970) + tv[0] = ts.tv_sec + 2208988800ULL; + // Convert microseconds to 32 bit fraction of a second + tv[1] = (ts.tv_nsec * 0x100000000ULL) / 1000000000ULL; + + if (netorder) + { + tv[0] = htonl(tv[0]); + tv[1] = htonl(tv[1]); + } +} + +// Create and send an NTP reply to the client +static int ntp_reply(const int socket_fd, const struct sockaddr *saddr_p, const socklen_t saddrlen, + const unsigned char recv_buf[], const uint32_t recv_time[2]) +{ + // Buffer for the response + unsigned char send_buf[48]; + memset(send_buf, 0, sizeof(send_buf)); + + // DWORD-aligned pointer to the send buffer + uint32_t *u32p = (uint32_t*)((void*)&send_buf[0]); + // DWORD-aligned read-only pointer to the receive buffer + const uint32_t *u32r = (uint32_t*)((void*)&recv_buf[0]); + +// NTP Packet Header Format (RFC 5905), page 18 +// 0 1 2 3 +// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// |LI | VN |Mode | Stratum | Poll | Precision | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + + // Check if the first byte is valid: mode is expected to be 3 ("client") + if ((recv_buf[0] & 0x07) != 0x3) { + log_warn("Received invalid NTP request: not from an NTP client, ignoring"); + return 1; + } + + // set LI = 0 (no warning about leap seconds), set version-number to + // 4 and set mode = 4 ("server") + send_buf[0] = (0x04 << 3) + 0x04; + + // Set stratum to "secondary server" as we have derived time via + // external NTP as well. May be set to 1 if we want to be a primary + // server (synchronized by a hardware clock with GPS, etc.) + send_buf[1] = 0x02; + + // Copy Poll value from client + send_buf[2] = recv_buf[2]; + + // Precision in Nanoseconds from CLOCK_REALTIME + struct timespec ts; + clock_getres(CLOCK_REALTIME, &ts); + // Precision in log2 seconds + signed char precision = (signed char)(1.0*log2(1e-9*ts.tv_nsec)); + // Precision in log2 seconds + send_buf[3] = precision; + + // Advance 32 bit pointer to the next field + u32p++; + +// 0 1 2 3 +// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | Root Delay | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | Root Dispersion | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + + /* zur Vereinfachung , Root Delay = 0, Root Dispersion = 0 */ + *u32p++ = 0; + *u32p++ = 0; + +// 0 1 2 3 +// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | Reference ID | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + + // Reference ID = 'LOCL" (LOCAL CLOCK) + // A four-octet, left-justified, zero-padded ASCII string assigned to + // the reference clock + memcpy(u32p++, "LOCL", 4); + +// 0 1 2 3 +// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | | +// + Reference Timestamp (64) + +// | | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + // Time when the system clock was last set or corrected, in NTP + // timestamp format. As this is not a stratum 1 server, we don't have + // a hardware clock to set this value. +#ifdef MOCK_REFTIME + // Mock this timestamp with the current time of the server minus 1 + // minute. + uint32_t ref_time[2]; + gettime32(ref_time, true); + ref_time[0] = ref_time[0] - htonl(60); // subtract 60 seconds + memcpy(u32p, ref_time, 2 * sizeof(uint32_t)); + u32p += 2; +#else + // A stateless server copies T3 and T4 from the client packet to T1 and + // T2 of the server packet and tacks on the transmit timestamp T3 before + // sending it to the client. + *u32p++ = u32r[8]; + *u32p++ = u32r[9]; +#endif +// 0 1 2 3 +// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | | +// + Origin Timestamp (64) + +// | | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + // Time at the client when the request departed for the server, in NTP + // timestamp format. (this is the client's transmit time) + *u32p++ = u32r[10]; + *u32p++ = u32r[11]; + +// 0 1 2 3 +// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | | +// + Receive Timestamp (64) + +// | | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + // Time at the server when the request arrived from the client, in NTP + // timestamp format. (this is the server's receive time) + memcpy(u32p, recv_time, 2 * sizeof(uint32_t)); + u32p += 2; + +// 0 1 2 3 +// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | | +// + Transmit Timestamp (64) + +// | | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + // Time at the server when the response left for the client, in NTP + // timestamp format. (this is the server's transmit time) + uint32_t transmit_time[2]; + gettime32(transmit_time, true); + memcpy(u32p, transmit_time, 2 * sizeof(uint32_t)); + u32p += 2; + +// 0 1 2 3 +// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | | +// . . +// . Extension Field 1 (variable) . +// . . +// | | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | | +// . . +// . Extension Field 2 (variable) . +// . . +// | | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | Key Identifier | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | | +// | dgst (128) | +// | | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// +// Figure 8: Packet Header Format + + // Send the response + errno = 0; + if(sendto(socket_fd, send_buf, sizeof(send_buf), 0, saddr_p, saddrlen) < 48) + { + log_err("NTP send error: %s", strerror(errno)); + return 1; + } + + return 0; +} + +// Process incoming NTP requests +static void request_process_loop(int fd, const char *ipstr, const int protocol) +{ + log_info("NTP server listening on %s:123 (%s)", ipstr, protocol == AF_INET ? "IPv4" : "IPv6"); + while (true) + { + unsigned char buf[48]; + struct sockaddr src_addr; + socklen_t src_addrlen = sizeof(src_addr); + while(recvfrom(fd, buf, sizeof(buf), 0, &src_addr, &src_addrlen) < 48); // ignore invalid requests + + // Get the current time in NTP format + uint32_t recv_time[2]; + gettime32(recv_time, true); + + struct sockaddr_in sin; + memcpy(&sin, &src_addr, sizeof(sin)); + // printf("Request from %s\n", inet_ntoa(sin.sin_addr)); + + const pid_t pid = fork(); + if (pid == 0) { + /* Child */ + ntp_reply(fd, &src_addr , src_addrlen, buf, recv_time); + exit(0); + } else if (pid == -1) { + log_err("fork() error"); + return; + } + // return to parent + } +} +/* +// Wait for a child process to exit +static void wait_wrapper(int _a) +{ + int s; + wait(&s); +}*/ + +// Start the NTP server +static void *ntp_bind_and_listen(void *param) +{ +// signal(SIGCHLD, wait_wrapper); + const int protocol = param == 0 ? AF_INET : AF_INET6; + + // Create a socket + errno = 0; + const int s = socket(protocol, SOCK_DGRAM, IPPROTO_UDP); + if(s == -1) + { + log_warn("Cannot create NTP socket (%s), IPv%i NTP server not available", + strerror(errno), protocol == AF_INET ? 4 : 6); + return NULL; + } + + // Bind the socket to the NTP port + char ipstr[INET6_ADDRSTRLEN + 1]; + memset(ipstr, 0, sizeof(ipstr)); + if(protocol == AF_INET) + { + // IPv4 - set thread name + prctl(PR_SET_NAME, "NTP (IPv4)", 0, 0, 0); + + // Prepare the bind address + struct sockaddr_in bind_addr; + memset(&bind_addr, 0, sizeof(bind_addr)); + bind_addr.sin_family = AF_INET; // IPv4 + bind_addr.sin_port = htons(123); // NTP port + memcpy(&bind_addr.sin_addr, &config.ntp.ipv4.address.v.in_addr, sizeof(bind_addr.sin_addr)); + inet_ntop(AF_INET, &bind_addr.sin_addr, ipstr, sizeof(ipstr) - 1); + + // Bind the socket + errno = 0; + if(bind(s, (struct sockaddr *)&bind_addr, sizeof(bind_addr)) != 0) + { + log_warn("Cannot bind to IPv4 address %s:123 (%s), IPv4 NTP server not available", + ipstr, strerror(errno)); + return NULL; + } + } + else + { + // IPv6 - set thread name + prctl(PR_SET_NAME, "NTP (IPv6)", 0, 0, 0); + + // Set socket options to allow IPv6 only, otherwise it will bind + // to both IPv4 and IPv6 and show IPv4 addresses as + // v4-mapped-on-v6 addresses + int opt = 1; + if(setsockopt(s, IPPROTO_IPV6, IPV6_V6ONLY, &opt, sizeof(opt)) != 0) + { + log_warn("Cannot set socket option IPV6_V6ONLY (%s), IPv6 NTP server not available", strerror(errno)); + return NULL; + } + + // Prepare the bind address + struct sockaddr_in6 bind_addr; + memset(&bind_addr, 0, sizeof(bind_addr)); + bind_addr.sin6_family = AF_INET6; // IPv6 + bind_addr.sin6_port = htons(123); // NTP port + memcpy(&bind_addr.sin6_addr, &config.ntp.ipv6.address.v.in6_addr, sizeof(bind_addr.sin6_addr)); + inet_ntop(AF_INET6, &bind_addr.sin6_addr, ipstr, sizeof(ipstr) - 1); + + // Bind the socket + errno = 0; + if(bind(s, (struct sockaddr *)&bind_addr, sizeof(bind_addr)) != 0) + { + log_warn("Cannot bind to IPv6 address %s:123 (%s), IPv6 NTP server not available", + ipstr, strerror(errno)); + return NULL; + } + } + + request_process_loop(s, ipstr, protocol); + close(s); + + return NULL; +} + +// Start the NTP server +bool ntp_server_start(void) +{ + // Spawn two pthreads, one for IPv4 and one for IPv6 + + // IPv4 + if(config.ntp.ipv4.active.v.b) + { + // Create a thread for the IPv4 NTP server + pthread_t thread; + if (pthread_create(&thread, NULL, ntp_bind_and_listen, (void *)0) != 0) + { + log_err("Can not create NTP server thread for IPv4"); + return false; + } + } + + // IPv6 + if(config.ntp.ipv6.active.v.b) + { + // Create a thread for the IPv6 NTP server + pthread_t thread; + if (pthread_create(&thread, NULL, ntp_bind_and_listen, (void *)1) != 0) + { + log_err("Can not create NTP server thread for IPv6"); + return false; + } + } + + sleep(10); + + return true; +} diff --git a/test/pihole.toml b/test/pihole.toml index 75f8244c..364d542d 100644 --- a/test/pihole.toml +++ b/test/pihole.toml @@ -442,6 +442,26 @@ # "[][,id:|*][,set:][,tag:][,][,][,][,ignore]" hosts = [] + [ntp.ipv4] + # Should FTL act as an NTP server (IPv4)? + active = true + + # IPv4 address to listen on for NTP requests + # + # Possible values are: + # or empty string ("") for wildcard (0.0.0.0) + address = "" + + [ntp.ipv6] + # Should FTL act as an NTP server (IPv6)? + active = true + + # IPv6 address to listen on for NTP requests + # + # Possible values are: + # or empty string ("") for wildcard (::) + address = "" + [resolver] # Should FTL try to resolve IPv4 addresses to hostnames? resolveIPv4 = false ### CHANGED, default = true diff --git a/test/test_suite.bats b/test/test_suite.bats index 94c04019..1e0b99d1 100644 --- a/test/test_suite.bats +++ b/test/test_suite.bats @@ -26,7 +26,7 @@ #} # @test "Running a second instance is detected and prevented" { - run bash -c 'su pihole -s /bin/sh -c "/home/pihole/pihole-FTL -f"' + run bash -c 'su pihole -s /bin/sh -c "./pihole-FTL -f"' printf "%s\n" "${lines[@]}" [[ "${lines[@]}" == *"CRIT: Initialization of shared memory failed."* ]] [[ "${lines[@]}" == *"INFO: pihole-FTL is already running"* ]] @@ -54,7 +54,7 @@ @test "Number of compiled regex filters as expected" { run bash -c 'grep "Compiled [0-9]* allow" /var/log/pihole/FTL.log' printf "%s\n" "${lines[@]}" - [[ ${lines[0]} == *"Compiled 2 allow and 11 deny regex for 1 client in "* ]] + [[ ${lines[0]} == *"Compiled 2 allow and 11 deny regex"* ]] } @test "denied domain is blocked" { @@ -490,15 +490,15 @@ } @test "Test fail on invalid CLI argument" { - run bash -c '/home/pihole/pihole-FTL abc' + run bash -c './pihole-FTL abc' printf "%s\n" "${lines[@]}" [[ ${lines[0]} == "pihole-FTL: invalid option -- 'abc'" ]] - [[ ${lines[1]} == "Command: '/home/pihole/pihole-FTL abc'" ]] - [[ ${lines[2]} == "Try '/home/pihole/pihole-FTL --help' for more information" ]] + [[ ${lines[1]} == "Command: './pihole-FTL abc'" ]] + [[ ${lines[2]} == "Try './pihole-FTL --help' for more information" ]] } @test "Help CLI argument return help text" { - run bash -c '/home/pihole/pihole-FTL help' + run bash -c './pihole-FTL help' printf "%s\n" "${lines[@]}" [[ ${lines[0]} == "The Pi-hole FTL engine - "* ]] } @@ -1390,6 +1390,14 @@ [[ ${lines[0]} == '{"error":{"key":"bad_request","message":"Config items set via environment variables cannot be changed via the API","hint":"misc.nice"},"took":'*'}' ]] } +@test "Check NTP server is broadcasting correct time" { + run bash -c './pihole-FTL ntp-client 127.0.0.1' + printf "%s\n" "${lines[@]}" + [[ $status == 0 ]] +} + +# We cannot easily test IPv6 as it may not be available in docker (CI) + @test "API domain search: Non-existing domain returns expected JSON" { run bash -c 'curl -s 127.0.0.1/api/search/non.existent' printf "%s\n" "${lines[@]}" From 3de9e12f70c0312665bc628c8284fefe32cbd3cb Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 28 May 2024 21:32:04 +0200 Subject: [PATCH 106/339] Change magic comment put next to config options which are forced through the environment Signed-off-by: DL6ER --- src/config/toml_writer.c | 14 ++++++-------- test/test_suite.bats | 5 ++--- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/src/config/toml_writer.c b/src/config/toml_writer.c index afc6a35b..e8cde7dc 100644 --- a/src/config/toml_writer.c +++ b/src/config/toml_writer.c @@ -82,13 +82,6 @@ bool writeFTLtoml(const bool verbose) print_toml_allowed_values(conf_item->a, fp, 85, level-1); } - // Print info if this value is overwritten by an env var - if(conf_item->f & FLAG_ENV_VAR) - { - print_comment(fp, ">>> This config is overwritten by an environmental variable <<<", "", 85, level-1); - env_vars++; - } - // Write value indentTOML(fp, level-1); fprintf(fp, "%s = ", conf_item->p[level-1]); @@ -105,7 +98,12 @@ bool writeFTLtoml(const bool verbose) if(changed) { - fprintf(fp, " ### CHANGED, default = "); + + // Print info if this value is overwritten by an env var + if(conf_item->f & FLAG_ENV_VAR) + env_vars++; + + fprintf(fp, " ### CHANGED%s, default = ", conf_item->f & FLAG_ENV_VAR ? " (env)" : ""); writeTOMLvalue(fp, -1, conf_item->t, &conf_item->d); modified++; } diff --git a/test/test_suite.bats b/test/test_suite.bats index 94c04019..72b102bb 100644 --- a/test/test_suite.bats +++ b/test/test_suite.bats @@ -1326,10 +1326,9 @@ @test "Environmental variable is favored over config file" { # The config file has -10 but we set FTLCONF_misc_nice="-11" - run bash -c 'grep -B1 "nice = -11" /etc/pihole/pihole.toml' + run bash -c 'grep "nice = -11" /etc/pihole/pihole.toml' printf "%s\n" "${lines[@]}" - [[ ${lines[0]} == " # >>> This config is overwritten by an environmental variable <<<" ]] - [[ ${lines[1]} == " nice = -11 ### CHANGED, default = -10" ]] + [[ ${lines[2]} == " nice = -11 ### CHANGED (env), default = -10" ]] } @test "Correct number of environmental variables is logged" { From 2cb6b36dca5ee164d0520f8b6706129287962295 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 28 May 2024 21:33:45 +0200 Subject: [PATCH 107/339] Print config file statistics at the end of the config file Signed-off-by: DL6ER --- src/config/toml_writer.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/config/toml_writer.c b/src/config/toml_writer.c index e8cde7dc..f6e37b1c 100644 --- a/src/config/toml_writer.c +++ b/src/config/toml_writer.c @@ -112,6 +112,15 @@ bool writeFTLtoml(const bool verbose) fputs("\n\n", fp); } + // Print config file statistics at the end of the file as comment + fputs("# Configuration statistics:\n", fp); + fprintf(fp, "# %zu total entries out of which %zu %s default\n", + CONFIG_ELEMENTS, CONFIG_ELEMENTS - modified, + CONFIG_ELEMENTS - modified == 1 ? "entry is" : "entries are"); + fprintf(fp, "# --> %u %s modified (%u %s forced through environment variables)\n", + modified, modified == 1 ? "entry is" : "entries are", + env_vars, env_vars == 1 ? "entry is" : "entries are"); + // Log some statistics in verbose mode if(verbose || config.debug.config.v.b) { From cdc7d00f8e26249cc3f3388cc388cef7badf29ac Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 28 May 2024 21:37:29 +0200 Subject: [PATCH 108/339] Add missing help text for new ntp-client option Signed-off-by: DL6ER --- src/args.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/args.c b/src/args.c index 1e7431ae..6427ac9b 100644 --- a/src/args.c +++ b/src/args.c @@ -1016,6 +1016,8 @@ void parse_args(int argc, char* argv[]) printf("%sOther:%s\n", yellow, normal); printf("\t%sptr %sIP%s Resolve IP address to hostname\n", green, cyan, normal); printf("\t%ssha256sum %sfile%s Calculate SHA256 checksum of a file\n", green, cyan, normal); + printf("\t%sntp-client %s[server]%s Request network time from %sserver%s\n", green, cyan, normal, cyan, normal); + printf("\t defaults to 127.0.0.1 if omitted\n"); printf("\t%sdhcp-discover%s Discover DHCP servers in the local\n", green, normal); printf("\t network\n"); printf("\t%sarp-scan %s[-a/-x]%s Use ARP to scan local network for\n", green, cyan, normal); From a5f5092f5a3109b2fe92e8436768585ee2d8d6a9 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 28 May 2024 21:47:11 +0200 Subject: [PATCH 109/339] List forced environment variables at end of the config file Signed-off-by: DL6ER --- src/config/toml_writer.c | 32 +++++++++++++++++++++++++------- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/src/config/toml_writer.c b/src/config/toml_writer.c index f6e37b1c..cc5fc07d 100644 --- a/src/config/toml_writer.c +++ b/src/config/toml_writer.c @@ -47,7 +47,8 @@ bool writeFTLtoml(const bool verbose) // Iterate over configuration and store it into the file char *last_path = (char*)""; - unsigned int modified = 0, env_vars = 0; + unsigned int modified = 0; + cJSON *env_vars = cJSON_CreateArray(); for(unsigned int i = 0; i < CONFIG_ELEMENTS; i++) { // Get pointer to memory location of this conf_item @@ -101,7 +102,7 @@ bool writeFTLtoml(const bool verbose) // Print info if this value is overwritten by an env var if(conf_item->f & FLAG_ENV_VAR) - env_vars++; + cJSON_AddItemToArray(env_vars, cJSON_CreateStringReference(conf_item->k)); fprintf(fp, " ### CHANGED%s, default = ", conf_item->f & FLAG_ENV_VAR ? " (env)" : ""); writeTOMLvalue(fp, -1, conf_item->t, &conf_item->d); @@ -117,9 +118,23 @@ bool writeFTLtoml(const bool verbose) fprintf(fp, "# %zu total entries out of which %zu %s default\n", CONFIG_ELEMENTS, CONFIG_ELEMENTS - modified, CONFIG_ELEMENTS - modified == 1 ? "entry is" : "entries are"); - fprintf(fp, "# --> %u %s modified (%u %s forced through environment variables)\n", - modified, modified == 1 ? "entry is" : "entries are", - env_vars, env_vars == 1 ? "entry is" : "entries are"); + fprintf(fp, "# --> %u %s modified\n", + modified, modified == 1 ? "entry is" : "entries are"); + + const unsigned int num_env_vars = cJSON_GetArraySize(env_vars); + if(num_env_vars > 0) + { + fprintf(fp, "# %u %s forced through environment:\n", + num_env_vars, num_env_vars == 1 ? "entry is" : "entries are"); + + for(unsigned int i = 0; i < num_env_vars; i++) + { + const char *env_var = cJSON_GetArrayItem(env_vars, i)->valuestring; + fprintf(fp, "# - %s\n", env_var); + } + } + else + fputc('\n', fp); // Log some statistics in verbose mode if(verbose || config.debug.config.v.b) @@ -130,10 +145,13 @@ bool writeFTLtoml(const bool verbose) CONFIG_ELEMENTS - modified == 1 ? "entry is" : "entries are"); log_info(" - %u %s modified", modified, modified == 1 ? "entry is" : "entries are"); - log_info(" - %u %s forced through environment", env_vars, - env_vars == 1 ? "entry is" : "entries are"); + log_info(" - %u %s forced through environment", num_env_vars, + num_env_vars == 1 ? "entry is" : "entries are"); } + // Free cJSON array + cJSON_Delete(env_vars); + // Close file and release exclusive lock closeFTLtoml(fp); From 7bfe4dc26e291e74bcc53dbb01dea3aa13c94338 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 28 May 2024 21:59:41 +0200 Subject: [PATCH 110/339] Reset config options previously forced by env vars but not any longer Signed-off-by: DL6ER --- src/config/env.c | 96 ++++++++++++++++++++++++++++++++++++++-- src/config/env.h | 3 +- src/config/toml_reader.c | 18 +++++++- test/test_suite.bats | 2 +- 4 files changed, 112 insertions(+), 7 deletions(-) diff --git a/src/config/env.c b/src/config/env.c index a58650fe..f1e7bb5f 100644 --- a/src/config/env.c +++ b/src/config/env.c @@ -19,6 +19,10 @@ #include "args.h" // suggest_closest() #include "config/suggest.h" +// LINE_MAX +#include +// openFTLtoml() +#include "config/toml_helper.h" struct env_item { bool used; @@ -161,16 +165,51 @@ void freeEnvVars(void) } } -bool readEnvValue(struct conf_item *conf_item, struct config *newconf) +bool __attribute__((nonnull(1,2,3))) readEnvValue(struct conf_item *conf_item, struct config *newconf, cJSON *forced_vars, bool *reset) { // First check if a environmental variable with the given key exists by // iterating over the list of FTLCONF_ variables struct env_item *item = getFTLenv(conf_item->e); - // Return early if this environment variable does not exist if(item == NULL) - return false; + { + // Environment variable does not exist + // Check if this was a forced setting before + // If so, we revert the config option to default + for(int i = 0; i < cJSON_GetArraySize(forced_vars); i++) + { + const char *forced_var = cJSON_GetArrayItem(forced_vars, i)->valuestring; + if(strcmp(forced_var, conf_item->k) == 0) + { + log_info("Resetting %s to default (not forced anymore)", conf_item->k); + + // Revert to default + if(conf_item->t == CONF_STRING_ALLOCATED) + { + // Free previously allocated string + free(conf_item->v.s); + // Make a duplicate of the default value + conf_item->v.s = strdup(conf_item->d.s); + } + else + { + // Revert to default value + memcpy(&conf_item->v, &conf_item->d, sizeof(conf_item->v)); + } + + // Mark this environment variable as reset to + // default + if(reset != NULL) + *reset = true; + break; + } + } + + // Return false as this setting is not forced by an environment + // variable + return false; + } // Mark this environment variable as used item->used = true; @@ -561,3 +600,54 @@ bool readEnvValue(struct conf_item *conf_item, struct config *newconf) return true; } + +cJSON *read_forced_vars(const unsigned int version) +{ + // Create cJSON array to store forced variables + cJSON *env_vars = cJSON_CreateArray(); + + // Try to open default config file. Use fallback if not found + FILE *fp; + if((fp = openFTLtoml("r", version)) == NULL) + { + // Return empty cJSON array + return env_vars; + } + + // Read file line by line until we get to the end of the file where the + // statistics are stored, specifically, the line starting with + // "# X entr{y is,ies are} forced through environment" + char line[LINE_MAX] = { 0 }; + while(fgets(line, sizeof(line), fp) != NULL) + { + // Check if this is the line we are looking for + if(strncmp(line, "# ", 2) == 0) + { + // Check if this is the line we are looking for + if(strstr(line, "forced through environment:") != NULL) + break; + } + } + + // Read the next lines to extract the variables + while(fgets(line, sizeof(line), fp) != NULL) + { + // Check if this is the line we are looking for + if(strncmp(line, "# - ", 6) != 0) + { + // We are done, break out of the loop + break; + } + + // else: Add the variable to the cJSON array + // Trim the string (remove leading "# - " and trailing newline) + line[strcspn(line, "\n")] = '\0'; + cJSON_AddItemToArray(env_vars, cJSON_CreateString(line + 6)); + } + + // Close file and release exclusive lock + closeFTLtoml(fp); + + // Return cJSON array + return env_vars; +} diff --git a/src/config/env.h b/src/config/env.h index d26c7e75..b7b580af 100644 --- a/src/config/env.h +++ b/src/config/env.h @@ -23,6 +23,7 @@ int dist(const char *str); void getEnvVars(void); void freeEnvVars(void); void printFTLenv(void); -bool readEnvValue(struct conf_item *conf_item, struct config *newconf); +bool readEnvValue(struct conf_item *conf_item, struct config *newconf, cJSON *forced_vars, bool *reset) __attribute__((nonnull(1,2,3))); +cJSON *read_forced_vars(const unsigned int version); #endif //CONFIG_ENV_H diff --git a/src/config/toml_reader.c b/src/config/toml_reader.c index 2c24e102..d2ceac0a 100644 --- a/src/config/toml_reader.c +++ b/src/config/toml_reader.c @@ -115,10 +115,14 @@ bool readFTLtoml(struct config *oldconf, struct config *newconf, return false; } + // First, get an array of keys of config items that have been forced + // through environment variables + cJSON *env_vars = read_forced_vars(version); + // Try to read debug config. This is done before the full config // parsing to allow for debug output further down // First try to read env variable, if this fails, read TOML - if(teleporter || !readEnvValue(&newconf->debug.config, newconf)) + if(teleporter || !readEnvValue(&newconf->debug.config, newconf, env_vars, NULL)) { toml_table_t *conf_debug = toml_table_in(toml, "debug"); if(conf_debug) @@ -140,12 +144,21 @@ bool readFTLtoml(struct config *oldconf, struct config *newconf, // First try to read this config option from an environment variable // Skip reading environment variables when importing from Teleporter // If this succeeds, skip searching the TOML file for this config item - if(!teleporter && readEnvValue(new_conf_item, newconf)) + bool reset = false; + if(!teleporter && readEnvValue(new_conf_item, newconf, env_vars, &reset)) { new_conf_item->f |= FLAG_ENV_VAR; continue; } + // Skip this variable if it has been reset (forced by + // environment variable before but not anymore) + if(reset) + { + log_info("Skipping %s as it has been reset", new_conf_item->k); + continue; + } + // Get config path depth unsigned int level = config_path_depth(new_conf_item->p); @@ -200,6 +213,7 @@ bool readFTLtoml(struct config *oldconf, struct config *newconf, // Free memory allocated by the TOML parser and return success toml_free(toml); + cJSON_Delete(env_vars); return true; } diff --git a/test/test_suite.bats b/test/test_suite.bats index 72b102bb..9765723d 100644 --- a/test/test_suite.bats +++ b/test/test_suite.bats @@ -1328,7 +1328,7 @@ # The config file has -10 but we set FTLCONF_misc_nice="-11" run bash -c 'grep "nice = -11" /etc/pihole/pihole.toml' printf "%s\n" "${lines[@]}" - [[ ${lines[2]} == " nice = -11 ### CHANGED (env), default = -10" ]] + [[ ${lines[0]} == " nice = -11 ### CHANGED (env), default = -10" ]] } @test "Correct number of environmental variables is logged" { From 980cc84fcd1d6e17d6cc48ab9c62812798c77d8e Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 29 May 2024 19:09:44 +0200 Subject: [PATCH 111/339] Reset special debug.all in a similar way Signed-off-by: DL6ER --- src/config/toml_reader.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/config/toml_reader.c b/src/config/toml_reader.c index d2ceac0a..6f3f98e3 100644 --- a/src/config/toml_reader.c +++ b/src/config/toml_reader.c @@ -155,6 +155,13 @@ bool readFTLtoml(struct config *oldconf, struct config *newconf, // environment variable before but not anymore) if(reset) { + if(new_conf_item->t == CONF_ALL_DEBUG_BOOL) + { + // Reset all debug flags to false if debug.all + // has been reset + set_all_debug(newconf, false); + set_debug_flags(newconf); + } log_info("Skipping %s as it has been reset", new_conf_item->k); continue; } From 19715ead9a3e455cfd5a9a2d5cf18b84e955a7ba Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 29 May 2024 20:06:50 +0200 Subject: [PATCH 112/339] Add help description of new optional ptr IP [tcp] flag Signed-off-by: DL6ER --- src/args.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/args.c b/src/args.c index c33ed4ed..7abe42f5 100644 --- a/src/args.c +++ b/src/args.c @@ -1018,7 +1018,8 @@ void parse_args(int argc, char* argv[]) printf(" Decoding: %spihole-FTL idn2 -d %spunycode%s\n\n", green, cyan, normal); printf("%sOther:%s\n", yellow, normal); - printf("\t%sptr %sIP%s Resolve IP address to hostname\n", green, cyan, normal); + printf("\t%sptr %sIP%s %s[tcp]%s Resolve IP address to hostname\n", green, cyan, normal, purple, normal); + printf("\t Append %stcp%s to use TCP instead of UDP\n", purple, normal); printf("\t%ssha256sum %sfile%s Calculate SHA256 checksum of a file\n", green, cyan, normal); printf("\t%sdhcp-discover%s Discover DHCP servers in the local\n", green, normal); printf("\t network\n"); From 51de04c18c4f66c8d2b6241e8790841212abfc89 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 29 May 2024 20:24:09 +0200 Subject: [PATCH 113/339] Fix include paths Signed-off-by: DL6ER --- src/ntp/client.c | 2 +- src/ntp/server.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ntp/client.c b/src/ntp/client.c index 0f586d13..092df60f 100644 --- a/src/ntp/client.c +++ b/src/ntp/client.c @@ -26,7 +26,7 @@ // errno #include -#include "ntp.h" +#include "ntp/ntp.h" #include "log.h" // Create minimal NTP request, see server implementation for details about the diff --git a/src/ntp/server.c b/src/ntp/server.c index eb9b8a33..6814f7d4 100644 --- a/src/ntp/server.c +++ b/src/ntp/server.c @@ -36,7 +36,7 @@ // PR_SET_NAME #include -#include "ntp.h" +#include "ntp/ntp.h" #include "log.h" #include "config/config.h" From 1a5e7f6bf81e72edb44b7ed0cd66aafe8be67c12 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 30 May 2024 12:26:40 +0200 Subject: [PATCH 114/339] Add new dhcp.ignoreUnknownClients option Signed-off-by: DL6ER --- src/api/docs/content/specs/config.yaml | 3 +++ src/config/config.c | 7 +++++++ src/config/config.h | 1 + src/config/dnsmasq_config.c | 7 +++++++ 4 files changed, 18 insertions(+) diff --git a/src/api/docs/content/specs/config.yaml b/src/api/docs/content/specs/config.yaml index 0b785754..08d5dd1b 100644 --- a/src/api/docs/content/specs/config.yaml +++ b/src/api/docs/content/specs/config.yaml @@ -320,6 +320,8 @@ components: type: boolean logging: type: boolean + ignoreUnknownClients: + type: boolean hosts: type: array items: @@ -653,6 +655,7 @@ components: rapidCommit: false multiDNS: false logging: false + ignoreUnknownClients: false hosts: - "11:22:33:44:55:66,192.168.1.123" - "11:22:33:44:55:67,192.168.1.124,hostname" diff --git a/src/config/config.c b/src/config/config.c index f5bb3fa1..b81cf769 100644 --- a/src/config/config.c +++ b/src/config/config.c @@ -785,6 +785,13 @@ void initConfig(struct config *conf) conf->dhcp.logging.d.b = false; conf->dhcp.logging.c = validate_stub; // Only type-based checking + conf->dhcp.ignoreUnknownClients.k = "dhcp.ignoreUnknownClients"; + conf->dhcp.ignoreUnknownClients.h = "Ignore unknown DHCP clients.\n If this option is set, Pi-hole ignores all clients which are not explicitly configured through dhcp.hosts. This can be useful to prevent unauthorized clients from getting an IP address from the DHCP server.\n It should be noted that this option is not a security feature, as clients can still assign themselves an IP address and use the network. It is merely a convenience feature to prevent unknown clients from getting a valid IP configuration assigned automatically.\n Note that you will need to configure new clients manually in dhcp.hosts before they can use the network when this feature is enabled."; + conf->dhcp.ignoreUnknownClients.t = CONF_BOOL; + conf->dhcp.ignoreUnknownClients.f = FLAG_RESTART_FTL; + conf->dhcp.ignoreUnknownClients.d.b = false; + conf->dhcp.ignoreUnknownClients.c = validate_stub; // Only type-based checking + conf->dhcp.hosts.k = "dhcp.hosts"; conf->dhcp.hosts.h = "Per host parameters for the DHCP server. This allows a machine with a particular hardware address to be always allocated the same hostname, IP address and lease time or to specify static DHCP leases"; conf->dhcp.hosts.a = cJSON_CreateStringReference("Array of static leases each on in one of the following forms: \"[][,id:|*][,set:][,tag:][,][,][,][,ignore]\""); diff --git a/src/config/config.h b/src/config/config.h index 013414ed..884f1faa 100644 --- a/src/config/config.h +++ b/src/config/config.h @@ -187,6 +187,7 @@ struct config { struct conf_item rapidCommit; struct conf_item multiDNS; struct conf_item logging; + struct conf_item ignoreUnknownClients; struct conf_item hosts; } dhcp; diff --git a/src/config/dnsmasq_config.c b/src/config/dnsmasq_config.c index 5dda68b0..516b924c 100644 --- a/src/config/dnsmasq_config.c +++ b/src/config/dnsmasq_config.c @@ -582,6 +582,13 @@ bool __attribute__((const)) write_dnsmasq_config(struct config *conf, bool test_ fputs("log-dhcp\n\n", pihole_conf); } + // Add option to ignore unknown clients if enabled + if(conf->dhcp.ignoreUnknownClients.v.b) + { + fputs("# Ignore clients not configured below\n", pihole_conf); + fputs("dhcp-ignore=tag:!known\n", pihole_conf); + } + // Add per-host parameters if(cJSON_GetArraySize(conf->dhcp.hosts.v.json) > 0) { From 96da0d4f943fde54a6f8fa18f33d16f0f051082c Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 30 May 2024 12:31:09 +0200 Subject: [PATCH 115/339] Synchronize test/pihole.toml with the many config file changes that have been made in between and add a test ensuring they remain in sync in the future Signed-off-by: DL6ER --- test/pihole.toml | 185 +++++++++++++++++++++++++------------------ test/run.sh | 10 +-- test/test_suite.bats | 37 +++------ 3 files changed, 121 insertions(+), 111 deletions(-) diff --git a/test/pihole.toml b/test/pihole.toml index 75f8244c..e1d72c8b 100644 --- a/test/pihole.toml +++ b/test/pihole.toml @@ -1,14 +1,11 @@ +# Pi-hole configuration file (v5.25.2-1891-g7ff016f2-dirty) +# Encoding: UTF-8 # This file is managed by pihole-FTL -# -# Do not edit the file while FTL is -# running or your changes may be overwritten -# -# Last updated on 2023-01-23 14:51:44 -# by FTL v5.20.1-552-g5184ed28 +# Last updated on 2024-05-30 11:37:59 [dns] # Array of upstream DNS servers used by Pi-hole - # Example: [ "8.8.8.8", "127.0.0.1#5353", "docker-resolver" ] + # Example: [ "8.8.8.8", "127.0.0.1#5335", "docker-resolver" ] # # Possible values are: # array of IP addresses and/or hostnames, optionally with a port (#...) @@ -57,16 +54,18 @@ # Pi-hole will not respond automatically on PTR requests to local interface # addresses. Ensure pi.hole and/or hostname records exist elsewhere. # - "HOSTNAME" - # Pi-hole will not respond automatically on PTR requests to local interface - # addresses. Ensure pi.hole and/or hostname records exist elsewhere. + # Serve the machine's hostname. The hostname is queried from the kernel through + # uname(2)->nodename. If the machine has multiple network interfaces, it can + # also have multiple nodenames. In this case, it is unspecified and up to the + # kernel which one will be returned. On Linux, the returned string is what has + # been set using sethostname(2) which is typically what has been set in + # /etc/hostname. # - "HOSTNAMEFQDN" - # Serve the machine's global hostname as fully qualified domain by adding the - # local suffix. If no local suffix has been defined, FTL appends the local - # domain .no_fqdn_available. In this case you should either add - # domain=whatever.com to a custom config file inside /etc/dnsmasq.d/ (to set - # whatever.com as local domain) or use domain=# which will try to derive the - # local domain from /etc/resolv.conf (or whatever is set with resolv-file, when - # multiple search directives exist, the first one is used). + # Serve the machine's hostname (see limitations above) as fully qualified domain + # by adding the local domain. If no local domain has been defined (config option + # dns.domain), FTL tries to query the domain name from the kernel using + # getdomainname(2). If this fails, FTL appends ".no_fqdn_available" to the + # hostname. # - "PI.HOLE" # Respond with "pi.hole". piholePTR = "PI.HOLE" @@ -140,7 +139,7 @@ bogusPriv = true # Validate DNS replies using DNSSEC? - dnssec = true + dnssec = true ### CHANGED, default = false # Interface to use for DNS (see also dnsmasq.listening.mode) and DHCP (if enabled) # @@ -197,11 +196,10 @@ # given, it overwrites the value of local-ttl # # Possible values are: - # Array of static leases each on in one of the following forms: - # ",[,]" + # Array of CNAMEs each on in one of the following forms: ",[,]" cnameRecords = [ - "brücke.com,äste.com,2", - ] + "brücke.com,äste.com,2" + ] ### CHANGED, default = [] # Port used by the DNS server port = 53 @@ -225,7 +223,10 @@ # : Domain used for the reverse server feature (e.g., "fritz.box") # Example: "fritz.box" # - # A valid line could look like this: "true,192.168.0.0/24,192.168.0.1,fritz.box" + # Possible values are: + # array of reverse servers each one in one of the following forms: + # ",[/],[#],", e.g., + # "true,192.168.0.0/24,192.168.0.1,fritz.box" revServers = [] [dns.cache] @@ -239,11 +240,15 @@ # expired only recently, the data will be used anyway (a refreshing from upstream is # triggered). This can improve DNS query delays especially over unreliable Internet # connections. This feature comes at the expense of possibly sometimes returning - # out-of-date data and less efficient cache utilisation, since old data cannot be + # out-of-date data and less efficient cache utilization, since old data cannot be # flushed when its TTL expires, so the cache becomes mostly least-recently-used. To # mitigate issues caused by massively outdated DNS replies, the maximum overaging of # cached records is limited. We strongly recommend staying below 86400 (1 day) with # this option. + # Setting the TTL excess time to zero will serve stale cache data regardless how long + # it has expired. This is not recommended as it may lead to stale data being served + # for a long time. Setting this option to any negative value will disable this feature + # altogether. optimizer = 3600 [dns.blocking] @@ -259,7 +264,7 @@ # (0.0.0.0 or ::). The "unspecified address" is a reserved IP address specified # by RFC 3513 - Internet Protocol Version 6 (IPv6) Addressing Architecture, # section 2.5.2. - # - "IP-NODATA-AAAA" + # - "IP_NODATA_AAAA" # In IP-NODATA-AAAA mode, blocked queries will be answered with the local IPv4 # addresses of your Pi-hole. Blocked AAAA queries will be answered with # NODATA-IPV6 and clients will only try to reach your Pi-hole over its static @@ -267,7 +272,7 @@ # - "IP" # In IP mode, blocked queries will be answered with the local IP addresses of # your Pi-hole. - # - "NXDOMAIN" + # - "NX" # In NXDOMAIN mode, blocked queries will be answered with an empty response # (i.e., there won't be an answer section) and status NXDOMAIN. A NXDOMAIN # response should indicate that there is no such domain to the client making the @@ -300,21 +305,21 @@ # "pi.hole.", "." ] force4 = true ### CHANGED, default = false - # Use a specific IPv6 address for the Pi-hole host? See description for the IPv4 - # variant above for further details. - force6 = true ### CHANGED, default = false - # Custom IPv4 address for the Pi-hole host # # Possible values are: # or empty string ("") - IPv4 = "10.100.0.10" ### CHANGED, default = "0.0.0.0" + IPv4 = "10.100.0.10" ### CHANGED, default = "" + + # Use a specific IPv6 address for the Pi-hole host? See description for the IPv4 + # variant above for further details. + force6 = true ### CHANGED, default = false # Custom IPv6 address for the Pi-hole host # # Possible values are: # or empty string ("") - IPv6 = "fe80::10" ### CHANGED, default = "::" + IPv6 = "fe80::10" ### CHANGED, default = "" [dns.reply.blocking] # Use a specific IPv4 address in IP blocking mode? By default, FTL determines the @@ -325,26 +330,26 @@ # blocked, regular expressions with the ;reply=IP regex extension. force4 = true ### CHANGED, default = false - # Use a specific IPv6 address in IP blocking mode? See description for the IPv4 variant - # above for further details. - force6 = true ### CHANGED, default = false - # Custom IPv4 address for IP blocking mode # # Possible values are: # or empty string ("") - IPv4 = "10.100.0.11" ### CHANGED, default = "0.0.0.0" + IPv4 = "10.100.0.11" ### CHANGED, default = "" + + # Use a specific IPv6 address in IP blocking mode? See description for the IPv4 variant + # above for further details. + force6 = true ### CHANGED, default = false # Custom IPv6 address for IP blocking mode # # Possible values are: # or empty string ("") - IPv6 = "fe80::11" ### CHANGED, default = "::" + IPv6 = "fe80::11" ### CHANGED, default = "" [dns.rateLimit] # Rate-limited queries are answered with a REFUSED reply and not further processed by # FTL. - #The default settings for FTL's rate-limiting are to permit no more than 1000 queries + # The default settings for FTL's rate-limiting are to permit no more than 1000 queries # in 60 seconds. Both numbers can be customized independently. It is important to note # that rate-limiting is happening on a per-client basis. Other clients can continue to # use FTL while rate-limited clients are short-circuited at the same time. @@ -377,32 +382,33 @@ # Start address of the DHCP address pool # # Possible values are: - # , e.g., "192.168.0.10" + # or empty string (""), e.g., "192.168.0.10" start = "" # End address of the DHCP address pool # # Possible values are: - # , e.g., "192.168.0.250" + # or empty string (""), e.g., "192.168.0.250" end = "" # Address of the gateway to be used (typically the address of your router in a home # installation) # # Possible values are: - # , e.g., "192.168.0.1" + # or empty string (""), e.g., "192.168.0.1" router = "" # The netmask used by your Pi-hole. For directly connected networks (i.e., networks on # which the machine running Pi-hole has an interface) the netmask is optional and may - # be set to "0.0.0.0": it will then be determined from the interface configuration - # itself. For networks which receive DHCP service via a relay agent, we cannot - # determine the netmask itself, so it should explicitly be specified, otherwise + # be set to an empty string (""): it will then be determined from the interface + # configuration itself. For networks which receive DHCP service via a relay agent, we + # cannot determine the netmask itself, so it should explicitly be specified, otherwise # Pi-hole guesses based on the class (A, B or C) of the network address. # # Possible values are: - # , e.g., "255.255.255.0" or "0.0.0.0" for auto-discovery - netmask = "0.0.0.0" + # (e.g., "255.255.255.0") or empty string ("") for + # auto-discovery + netmask = "" # If the lease time is given, then leases will be given for that length of time. If not # given, the default lease time is one hour for IPv4 and one day for IPv6. @@ -433,6 +439,18 @@ # the file specified by files.log.dnsmasq below. logging = false + # Ignore unknown DHCP clients. + # If this option is set, Pi-hole ignores all clients which are not explicitly + # configured through dhcp.hosts. This can be useful to prevent unauthorized clients + # from getting an IP address from the DHCP server. + # It should be noted that this option is not a security feature, as clients can still + # assign themselves an IP address and use the network. It is merely a convenience + # feature to prevent unknown clients from getting a valid IP configuration assigned + # automatically. + # Note that you will need to configure new clients manually in dhcp.hosts before they + # can use the network when this feature is enabled. + ignoreUnknownClients = false + # Per host parameters for the DHCP server. This allows a machine with a particular # hardware address to be always allocated the same hostname, IP address and lease time # or to specify static DHCP leases @@ -451,10 +469,11 @@ # Control whether FTL should use the fallback option to try to obtain client names from # checking the network table. This behavior can be disabled with this option. - #Assume an IPv6 client without a host names. However, the network table knows - though - # the client's MAC address - that this is the same device where we have a host name - # for another IP address (e.g., a DHCP server managed IPv4 address). In this case, we - # use the host name associated to the other address as this is the same device. + # Assume an IPv6 client without a host names. However, the network table knows - + # though the client's MAC address - that this is the same device where we have a host + # name for another IP address (e.g., a DHCP server managed IPv4 address). In this + # case, we use the host name associated to the other address as this is the same + # device. networkNames = false ### CHANGED, default = true # With this option, you can change how (and if) hourly PTR requests are made to check @@ -485,8 +504,7 @@ DBimport = true # How long should queries be stored in the database [days]? - # Setting this to 0 disables exporting queries to the database. - maxDBdays = 365 + maxDBdays = 91 # How often do we store queries in FTL's database [seconds]? DBinterval = 60 @@ -508,7 +526,7 @@ # How long should IP addresses be kept in the network_addresses table [days]? IP # addresses (and associated host names) older than the specified number of days are # removed to avoid dead entries in the network overview table. - expire = 365 + expire = 91 [webserver] # On which domain is the web interface served? @@ -573,7 +591,7 @@ # the total number of concurrent sessions is limited so setting this value too high # may result in users being rejected and unable to log in if there are already too # many sessions active. - timeout = 300 + timeout = 300 ### CHANGED, default = 1800 # Should Pi-hole backup and restore sessions from the database? This is useful if you # want to keep your sessions after a restart of the web interface. @@ -599,7 +617,7 @@ # # Possible values are: # - cert = "/etc/pihole/test.pem" + cert = "/etc/pihole/test.pem" ### CHANGED, default = "/etc/pihole/tls.pem" [webserver.paths] # Server root on the host @@ -622,23 +640,24 @@ # # Possible values are: # - "default-auto" - # Pi-hole auto theme (light/dark, default) + # Pi-hole auto # - "default-light" - # Pi-hole day theme (light) + # Pi-hole day # - "default-dark" - # Pi-hole midnight theme (dark) + # Pi-hole midnight # - "default-darker" - # Pi-hole deep-midnight theme (dark) + # Pi-hole deep-midnight # - "high-contrast" - # High-contrast theme (light) + # High-contrast light # - "high-contrast-dark" - # High-contrast theme (dark) + # High-contrast dark # - "lcars" - # Star Trek LCARS theme (dark) + # Star Trek LCARS theme = "default-auto" [webserver.api] - # Does local clients need to authenticate to access the API? + # Do local clients need to authenticate to access the API? This settings allows local + # clients to use the API without authentication. localAPIauth = true # Do local clients need to authenticate to access the search API? This settings allows @@ -708,13 +727,15 @@ # array of regular expressions describing domains excludeDomains = [] - # How much history should be imported from the database [seconds]? (max 24*60*60 = - # 86400) + # How much history should be imported from the database and returned by the API + # [seconds]? (max 24*60*60 = 86400) maxHistory = 86400 # Up to how many clients should be returned in the activity graph endpoint # (/api/history/clients)? - # This setting can be overwritten at run-time using the parameter N + # This setting can be overwritten at run-time using the parameter N. Setting this to 0 + # will always send all clients. Be aware that this may be challenging for the GUI if + # you have many (think > 1.000 clients) in your network maxClients = 10 # How should the API compute the most active clients? If set to true, the API will @@ -765,7 +786,7 @@ # directory must be writable by the user running gravity (typically pihole). # # Possible values are: - # + # gravity_tmp = "/tmp" # The database containing MAC -> Vendor information for the network table @@ -774,7 +795,7 @@ # macvendor = "/etc/pihole/macvendor.db" - # The config file of Pi-hole + # The old config file of Pi-hole used before v6.0 # # Possible values are: # @@ -812,11 +833,11 @@ [misc] # Using privacy levels you can specify which level of detail you want to see in your - # Pi-hole statistics. + # Pi-hole statistics. Changing this setting will trigger a restart of FTL # # Possible values are: # - 0 - # Doesn't hide anything, all statistics are available. + # Don't hide anything, all statistics are available. # - 1 # Hide domains. This setting disables Top Domains and Top Ads # - 2 @@ -841,7 +862,7 @@ # CPU scheduler to favor or disfavor a process in scheduling decisions. The range of # the nice value varies across UNIX systems. On modern Linux, the range is -20 (high # priority = not very nice to other processes) to +19 (low priority). - nice = -999 ### CHANGED, default = -10 + nice = -11 ### CHANGED (env), default = -10 # Should FTL translate its own stack addresses into code lines during the bug # backtrace? This improves the analysis of crashed significantly. It is recommended to @@ -853,11 +874,15 @@ # Should FTL load additional dnsmasq configuration files from /etc/dnsmasq.d/? etc_dnsmasq_d = true ### CHANGED, default = false - # Additional lines to inject into the generated dnsmasq configuration. Warning: This is - # an advanced setting and should only be used with care. Incorrectly formatted or - # duplicated lines as well as lines conflicting with the automatic configuration of - # Pi-hole can break the embedded dnsmasq and will stop DNS resolution from working. + # Additional lines to inject into the generated dnsmasq configuration. + # Warning: This is an advanced setting and should only be used with care. Incorrectly + # formatted or duplicated lines as well as lines conflicting with the automatic + # configuration of Pi-hole can break the embedded dnsmasq and will stop DNS resolution + # from working. # Use this option with extra care. + # + # Possible values are: + # array of valid dnsmasq config line options dnsmasq_lines = [] # Log additional information about queries and replies to pihole.log @@ -937,7 +962,7 @@ # when debugging specific API issues and can be helpful, e.g., when a client cannot # connect due to an obscure API error. Furthermore, this setting enables logging of # all API requests (auth log) and details about user authentication attempts. - api = true ### CHANGED, default = false + api = true ### CHANGED (env), default = false # Print extra debugging information about TLS connections. This includes the TLS # version, the cipher suite, the certificate chain and much more. This very verbose @@ -997,7 +1022,7 @@ # Debug monitoring of /etc/pihole filesystem events inotify = true ### CHANGED, default = false - # Logging of webserver (CivetWeb) debug messages + # Debug monitoring of the webserver (CivetWeb) events webserver = true ### CHANGED, default = false # Temporary flag that may print additional information. This debug flag is meant to be @@ -1013,3 +1038,9 @@ # *remaining* debug flags but unsetting it will disable *all* debug flags. all = true ### CHANGED, default = false +# Configuration statistics: +# 136 total entries out of which 82 entries are default +# --> 54 entries are modified +# 2 entries are forced through environment: +# - misc.nice +# - debug.api diff --git a/test/run.sh b/test/run.sh index 24976551..13341bbb 100755 --- a/test/run.sh +++ b/test/run.sh @@ -23,13 +23,12 @@ done rm -rf /etc/pihole /var/log/pihole /dev/shm/FTL-* # Create necessary directories and files -mkdir -p /home/pihole /etc/pihole /run/pihole /var/log/pihole +mkdir -p /home/pihole /etc/pihole /run/pihole /var/log/pihole /etc/pihole/config_backups echo "" > /var/log/pihole/FTL.log echo "" > /var/log/pihole/pihole.log touch /run/pihole-FTL.pid /run/pihole-FTL.port dig.log ptr.log -touch /var/log/pihole/HTTP_info.log /var/log/pihole/PH7.log /etc/pihole/dhcp.leases -chown pihole:pihole /etc/pihole /run/pihole /var/log/pihole/pihole.log /var/log/pihole/FTL.log /run/pihole-FTL.pid /run/pihole-FTL.port -chown pihole:pihole /var/log/pihole/HTTP_info.log /var/log/pihole/PH7.log /etc/pihole/dhcp.leases +touch /var/log/pihole/HTTP_info.log /etc/pihole/dhcp.leases +chown -R pihole:pihole /etc/pihole /run/pihole /var/log/pihole # Copy binary into a location the new user pihole can access cp ./pihole-FTL /home/pihole/pihole-FTL @@ -128,9 +127,6 @@ if [[ $RET != 0 ]]; then echo -n "HTTP_info.log: " curl_to_tricorder /var/log/pihole/HTTP_info.log echo "" - echo -n "PH7.log: " - curl_to_tricorder /var/log/pihole/PH7.log - echo "" echo -n "pihole.toml: " curl_to_tricorder /etc/pihole/pihole.toml echo "" diff --git a/test/test_suite.bats b/test/test_suite.bats index bdcc267d..e020846c 100644 --- a/test/test_suite.bats +++ b/test/test_suite.bats @@ -1,30 +1,13 @@ #!./test/libs/bats/bin/bats -#@test "Version, Tag, Branch, Hash, Date is reported" { -# run bash -c 'echo ">version >quit" | nc -v 127.0.0.1 4711' -# printf "%s\n" "${lines[@]}" -# [[ ${lines[1]} == "version "* ]] -# [[ ${lines[2]} == "tag "* ]] -# [[ ${lines[3]} == "branch "* ]] -# [[ ${lines[4]} == "hash "* ]] -# [[ ${lines[5]} == "date "* ]] -# [[ ${lines[6]} == "" ]] -#} -# -#@test "DNS server port is reported over Telnet API" { -# run bash -c 'echo ">dns-port >quit" | nc -v 127.0.0.1 4711' -# printf "%s\n" "${lines[@]}" -# [[ ${lines[1]} == "53" ]] -# [[ ${lines[2]} == "" ]] -#} -# -#@test "Maxlogage value is reported over Telnet API" { -# run bash -c 'echo ">maxlogage >quit" | nc -v 127.0.0.1 4711' -# printf "%s\n" "${lines[@]}" -# [[ ${lines[1]} == "86400" ]] -# [[ ${lines[2]} == "" ]] -#} -# +@test "Compare template and test TOML config files" { + # We skip the first 5 lines of the files as they contain the version and + # timestamp of the file creation/modification + run bash -c 'diff <(tail -n +6 test/pihole.toml) <(tail -n +6 /etc/pihole/pihole.toml)' + printf "%s\n" "${lines[@]}" + [[ "${lines[@]}" == "" ]] +} + @test "Running a second instance is detected and prevented" { run bash -c 'su pihole -s /bin/sh -c "/home/pihole/pihole-FTL -f"' printf "%s\n" "${lines[@]}" @@ -1793,10 +1776,10 @@ @test "Expected number of config file rotations" { run bash -c 'grep -c "INFO: Config file written to /etc/pihole/pihole.toml" /var/log/pihole/FTL.log' printf "%s\n" "${lines[@]}" - [[ ${lines[0]} == "3" ]] + [[ ${lines[0]} == "2" ]] run bash -c 'grep -c "DEBUG_CONFIG: pihole.toml unchanged" /var/log/pihole/FTL.log' printf "%s\n" "${lines[@]}" - [[ ${lines[0]} == "3" ]] + [[ ${lines[0]} == "4" ]] run bash -c 'grep -c "DEBUG_CONFIG: Config file written to /etc/pihole/dnsmasq.conf" /var/log/pihole/FTL.log' printf "%s\n" "${lines[@]}" [[ ${lines[0]} == "1" ]] From 1f6d9c115e9823b8eed481dd27f6876fc8329dbb Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 30 May 2024 19:25:18 +0200 Subject: [PATCH 116/339] Explicitly chown PID and remove old PORT file Signed-off-by: DL6ER --- test/run.sh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/test/run.sh b/test/run.sh index 13341bbb..5d490a82 100755 --- a/test/run.sh +++ b/test/run.sh @@ -26,9 +26,11 @@ rm -rf /etc/pihole /var/log/pihole /dev/shm/FTL-* mkdir -p /home/pihole /etc/pihole /run/pihole /var/log/pihole /etc/pihole/config_backups echo "" > /var/log/pihole/FTL.log echo "" > /var/log/pihole/pihole.log -touch /run/pihole-FTL.pid /run/pihole-FTL.port dig.log ptr.log -touch /var/log/pihole/HTTP_info.log /etc/pihole/dhcp.leases +echo "" > /var/log/pihole/webserver.log +touch /run/pihole-FTL.pid dig.log ptr.log +touch /etc/pihole/dhcp.leases chown -R pihole:pihole /etc/pihole /run/pihole /var/log/pihole +chown pihole:pihole /run/pihole-FTL.pid # Copy binary into a location the new user pihole can access cp ./pihole-FTL /home/pihole/pihole-FTL @@ -124,8 +126,8 @@ if [[ $RET != 0 ]]; then echo -n "ptr.log: " curl_to_tricorder ./ptr.log echo "" - echo -n "HTTP_info.log: " - curl_to_tricorder /var/log/pihole/HTTP_info.log + echo -n "webserver.log: " + curl_to_tricorder /var/log/pihole/webserver.log echo "" echo -n "pihole.toml: " curl_to_tricorder /etc/pihole/pihole.toml From 306710e74a89df3a743c4592c171ef4851783c93 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 30 May 2024 19:46:33 +0200 Subject: [PATCH 117/339] Add timezone information where this is appropriate. We do not include this in the log files to not needlessly change for format. Open for discussion Signed-off-by: DL6ER --- src/api/auth.c | 8 ++++---- src/config/dnsmasq_config.c | 4 ++-- src/config/toml_writer.c | 4 ++-- src/gc.c | 4 ++-- src/log.c | 23 ++++++++++++++++------- src/log.h | 5 +++-- src/overTime.c | 8 ++++---- src/procps.c | 2 +- src/zip/teleporter.c | 4 ++-- test/pihole.toml | 2 +- 10 files changed, 37 insertions(+), 27 deletions(-) diff --git a/src/api/auth.c b/src/api/auth.c index 88c8d10b..a4752d35 100644 --- a/src/api/auth.c +++ b/src/api/auth.c @@ -268,8 +268,8 @@ int check_client_auth(struct ftl_conn *api, const bool is_api) // Debug logging if(config.debug.api.v.b) { - char timestr[128]; - get_timestr(timestr, auth_data[user_id].valid_until, false, false); + char timestr[TIMESTR_SIZE]; + get_timestr(timestr, auth_data[user_id].valid_until, false, false, true); log_debug(DEBUG_API, "Recognized known user: user_id %i, valid_until: %s, remote_addr %s (%s at login)", user_id, timestr, api->request->remote_addr, auth_data[user_id].remote_addr); } @@ -631,8 +631,8 @@ int api_auth(struct ftl_conn *api) // Debug logging if(config.debug.api.v.b && user_id > API_AUTH_UNAUTHORIZED) { - char timestr[128]; - get_timestr(timestr, auth_data[user_id].valid_until, false, false); + char timestr[TIMESTR_SIZE]; + get_timestr(timestr, auth_data[user_id].valid_until, false, false, true); log_debug(DEBUG_API, "API: Registered new user: user_id %i valid_until: %s remote_addr %s (accepted due to %s)", user_id, timestr, auth_data[user_id].remote_addr, empty_password ? "empty password" : "correct response"); diff --git a/src/config/dnsmasq_config.c b/src/config/dnsmasq_config.c index 5dda68b0..60f9498f 100644 --- a/src/config/dnsmasq_config.c +++ b/src/config/dnsmasq_config.c @@ -191,8 +191,8 @@ char *get_dnsmasq_line(const unsigned int lineno) static void write_config_header(FILE *fp, const char *description) { const time_t now = time(NULL); - char timestring[TIMESTR_SIZE] = ""; - get_timestr(timestring, now, false, false); + char timestring[TIMESTR_SIZE]; + get_timestr(timestring, now, false, false, true); fputs("# Pi-hole: A black hole for Internet advertisements\n", fp); fprintf(fp, "# (c) %u Pi-hole, LLC (https://pi-hole.net)\n", get_year(now)); fputs("# Network-wide ad blocking via your own hardware.\n", fp); diff --git a/src/config/toml_writer.c b/src/config/toml_writer.c index cc5fc07d..5d995a82 100644 --- a/src/config/toml_writer.c +++ b/src/config/toml_writer.c @@ -39,8 +39,8 @@ bool writeFTLtoml(const bool verbose) fprintf(fp, "# Pi-hole configuration file (%s)\n", get_FTL_version()); fputs("# Encoding: UTF-8\n", fp); fputs("# This file is managed by pihole-FTL\n", fp); - char timestring[TIMESTR_SIZE] = ""; - get_timestr(timestring, time(NULL), false, false); + char timestring[TIMESTR_SIZE]; + get_timestr(timestring, time(NULL), false, false, true); fputs("# Last updated on ", fp); fputs(timestring, fp); fputs("\n\n", fp); diff --git a/src/gc.c b/src/gc.c index af1680b3..e656222b 100644 --- a/src/gc.c +++ b/src/gc.c @@ -298,8 +298,8 @@ void runGC(const time_t now, time_t *lastGCrun, const bool flush) if(config.debug.gc.v.b) { timer_start(GC_TIMER); - char timestring[TIMESTR_SIZE] = ""; - get_timestr(timestring, mintime, false, false); + char timestring[TIMESTR_SIZE]; + get_timestr(timestring, mintime, false, false, true); log_debug(DEBUG_GC, "GC starting, mintime: %s (%lu), counters->queries = %d", timestring, (unsigned long)mintime, counters->queries); } diff --git a/src/log.c b/src/log.c index f57004a2..a12f9345 100644 --- a/src/log.c +++ b/src/log.c @@ -85,9 +85,9 @@ double double_time(void) return tp.tv_sec + 1e-9*tp.tv_nsec; } -// The size of 84 bytes has been carefully selected for all possible timestamps -// to always fit into the available space without buffer overflows -void get_timestr(char timestring[TIMESTR_SIZE], const time_t timein, const bool millis, const bool uri_compatible) +// Get a human-readable time string +void get_timestr(char timestring[TIMESTR_SIZE], const time_t timein, + const bool millis, const bool uri_compatible, const bool timezone) { struct tm tm; localtime_r(&timein, &tm); @@ -115,6 +115,15 @@ void get_timestr(char timestring[TIMESTR_SIZE], const time_t timein, const bool tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, space, tm.tm_hour, colon, tm.tm_min, colon, tm.tm_sec); } + + // Append timezone if requested + if(timezone) + snprintf(timestring + strlen(timestring), + TIMESTR_SIZE - strlen(timestring), + "%c%s", space, tm.tm_zone); + + // Ensure that the string is zero-terminated + timestring[TIMESTR_SIZE - 1] = '\0'; } // Return the current year @@ -227,7 +236,7 @@ const char *debugstr(const enum debug_flag flag) void __attribute__ ((format (printf, 3, 4))) _FTL_log(const int priority, const enum debug_flag flag, const char *format, ...) { - char timestring[TIMESTR_SIZE] = ""; + char timestring[TIMESTR_SIZE]; va_list args; // We have been explicitly asked to not print anything to the log @@ -235,7 +244,7 @@ void __attribute__ ((format (printf, 3, 4))) _FTL_log(const int priority, const return; // Get human-readable time - get_timestr(timestring, time(NULL), true, false); + get_timestr(timestring, time(NULL), true, false, false); // Get and log PID of current process to avoid ambiguities when more than one // pihole-FTL instance is logging into the same file @@ -324,7 +333,7 @@ void __attribute__ ((format (printf, 3, 4))) _FTL_log(const int priority, const void __attribute__ ((format (printf, 1, 2))) log_web(const char *format, ...) { - char timestring[TIMESTR_SIZE] = ""; + char timestring[TIMESTR_SIZE]; const time_t now = time(NULL); va_list args; @@ -336,7 +345,7 @@ void __attribute__ ((format (printf, 1, 2))) log_web(const char *format, ...) add_to_fifo_buffer(FIFO_WEBSERVER, buffer, NULL, len > MAX_MSG_FIFO ? MAX_MSG_FIFO : len); // Get human-readable time - get_timestr(timestring, now, true, false); + get_timestr(timestring, now, true, false, false); // Get and log PID of current process to avoid ambiguities when more than one // pihole-FTL instance is logging into the same file diff --git a/src/log.h b/src/log.h index d794191b..4b9162c1 100644 --- a/src/log.h +++ b/src/log.h @@ -19,7 +19,7 @@ #include #define DEBUG_ANY 0 -#define TIMESTR_SIZE 84 +#define TIMESTR_SIZE 128 // Credit: https://stackoverflow.com/a/75116514 #define LEFT(str, w) \ @@ -51,7 +51,8 @@ unsigned int get_year(const time_t timein); const char *get_FTL_version(void); void log_FTL_version(bool crashreport); double double_time(void); -void get_timestr(char timestring[TIMESTR_SIZE], const time_t timein, const bool millis, const bool uri_compatible); +void get_timestr(char timestring[TIMESTR_SIZE], const time_t timein, + const bool millis, const bool uri_compatible, const bool timezone); const char *debugstr(const enum debug_flag flag) __attribute__((const)); void log_web(const char *format, ...) __attribute__ ((format (printf, 1, 2))); const char *get_ordinal_suffix(unsigned int number) __attribute__ ((const)); diff --git a/src/overTime.c b/src/overTime.c index 39681311..b1e42386 100644 --- a/src/overTime.c +++ b/src/overTime.c @@ -119,12 +119,12 @@ unsigned int _getOverTimeID(time_t timestamp, const char *file, const int line) // This is definitely wrong. We warn about this (but only once) if(!warned_about_hwclock) { - char timestampStr[TIMESTR_SIZE] = ""; - get_timestr(timestampStr, timestamp, false, false); + char timestampStr[TIMESTR_SIZE]; + get_timestr(timestampStr, timestamp, false, false, true); const time_t lastTimestamp = overTime[OVERTIME_SLOTS-1].timestamp; - char lastTimestampStr[TIMESTR_SIZE] = ""; - get_timestr(lastTimestampStr, lastTimestamp, false, false); + char lastTimestampStr[TIMESTR_SIZE]; + get_timestr(lastTimestampStr, lastTimestamp, false, false, true); log_warn("Found database entries in the future (%s (%lu), last timestamp for importing: %s (%lu)). " "Your over-time statistics may be incorrect (found in %s:%d)", diff --git a/src/procps.c b/src/procps.c index 8e3656a4..2aaaf3b9 100644 --- a/src/procps.c +++ b/src/procps.c @@ -109,7 +109,7 @@ static bool get_process_creation_time(const pid_t pid, char timestr[TIMESTR_SIZE struct stat st; if(stat(filename, &st) < 0) return false; - get_timestr(timestr, st.st_ctim.tv_sec, false, false); + get_timestr(timestr, st.st_ctim.tv_sec, false, false, true); return true; } diff --git a/src/zip/teleporter.c b/src/zip/teleporter.c index 5eb02317..a82e5855 100644 --- a/src/zip/teleporter.c +++ b/src/zip/teleporter.c @@ -277,8 +277,8 @@ const char *generate_teleporter_zip(mz_zip_archive *zip, char filename[128], voi // Generate filename for ZIP archive (it has both the hostname and the // current datetime) - char timestr[TIMESTR_SIZE] = ""; - get_timestr(timestr, time(NULL), false, true); + char timestr[TIMESTR_SIZE]; + get_timestr(timestr, time(NULL), false, true, true); snprintf(filename, 128, "pi-hole_%s_teleporter_%s.zip", hostname(), timestr); // Everything worked well diff --git a/test/pihole.toml b/test/pihole.toml index 75f8244c..748cbfa0 100644 --- a/test/pihole.toml +++ b/test/pihole.toml @@ -3,7 +3,7 @@ # Do not edit the file while FTL is # running or your changes may be overwritten # -# Last updated on 2023-01-23 14:51:44 +# Last updated on 2023-01-23 14:51:44 CET # by FTL v5.20.1-552-g5184ed28 [dns] From 7db4483c82e5ec90067e8e721c50d9ef246c99d3 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 31 May 2024 13:20:12 +0200 Subject: [PATCH 118/339] Update macvendor script Signed-off-by: DL6ER --- tools/macvendor.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/tools/macvendor.py b/tools/macvendor.py index 003f4e7a..d0cabeed 100644 --- a/tools/macvendor.py +++ b/tools/macvendor.py @@ -12,21 +12,19 @@ import os import re -import urllib.request +import requests import sqlite3 # Download raw data from Wireshark's website # We use the official URL recommended in the header of this file -# Thanks to mibere for the update +URL = "https://www.wireshark.org/download/automated/data/manuf" +# User-Agent string to use for the request +USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36" print("Downloading...") -opener = urllib.request.build_opener() -opener.addheaders = [('User-agent', 'Mozilla/5.0')] -urllib.request.install_opener(opener) -urllib.request.urlretrieve("https://gitlab.com/wireshark/wireshark/-/raw/master/manuf", "manuf.data") +manuf = requests.get(URL, headers={"User-Agent": USER_AGENT}).text.splitlines() print("...done") # Read file into memory and process lines -manuf = open("manuf.data", "r") data = [] print("Processing...") for line in manuf: @@ -63,7 +61,6 @@ for line in manuf: else: data.append([mac, desc_short]) print("...done") -manuf.close() # Create database database = "macvendor.db" From d406327ae41ceec180e3561991e0294efe7b377f Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 1 Jun 2024 10:48:05 +0200 Subject: [PATCH 119/339] Synchronize pihole.toml and config.c Signed-off-by: DL6ER --- src/config/config.c | 4 ++-- test/pihole.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/config/config.c b/src/config/config.c index 4437be01..72aac826 100644 --- a/src/config/config.c +++ b/src/config/config.c @@ -811,7 +811,7 @@ void initConfig(struct config *conf) conf->ntp.ipv4.address.k = "ntp.ipv4.address"; conf->ntp.ipv4.address.h = "IPv4 address to listen on for NTP requests"; - conf->ntp.ipv4.address.a = cJSON_CreateStringReference(" or empty string (\"\") for wildcard"); + conf->ntp.ipv4.address.a = cJSON_CreateStringReference(" or empty string (\"\") for wildcard (0.0.0.0)"); conf->ntp.ipv4.address.t = CONF_STRUCT_IN_ADDR; conf->ntp.ipv4.address.f = FLAG_RESTART_FTL; memset(&conf->ntp.ipv4.address.d.in_addr, 0, sizeof(struct in_addr)); @@ -826,7 +826,7 @@ void initConfig(struct config *conf) conf->ntp.ipv6.address.k = "ntp.ipv6.address"; conf->ntp.ipv6.address.h = "IPv6 address to listen on for NTP requests"; - conf->ntp.ipv6.address.a = cJSON_CreateStringReference(" or empty string (\"\") for wildcard"); + conf->ntp.ipv6.address.a = cJSON_CreateStringReference(" or empty string (\"\") for wildcard (::)"); conf->ntp.ipv6.address.t = CONF_STRUCT_IN6_ADDR; conf->ntp.ipv6.address.f = FLAG_RESTART_FTL; memset(&conf->ntp.ipv6.address.d.in6_addr, 0, sizeof(struct in6_addr)); diff --git a/test/pihole.toml b/test/pihole.toml index eef8df8b..3c4b651a 100644 --- a/test/pihole.toml +++ b/test/pihole.toml @@ -1059,7 +1059,7 @@ all = true ### CHANGED, default = false # Configuration statistics: -# 136 total entries out of which 82 entries are default +# 140 total entries out of which 86 entries are default # --> 54 entries are modified # 2 entries are forced through environment: # - misc.nice From 4be0ff60823dd3bf9dc930eab9f42287627ad2af Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 1 Jun 2024 10:35:49 +0000 Subject: [PATCH 120/339] Bump eps1lon/actions-label-merge-conflict Bumps the github_action-dependencies group with 1 update: [eps1lon/actions-label-merge-conflict](https://github.com/eps1lon/actions-label-merge-conflict). Updates `eps1lon/actions-label-merge-conflict` from 3.0.1 to 3.0.2 - [Release notes](https://github.com/eps1lon/actions-label-merge-conflict/releases) - [Changelog](https://github.com/eps1lon/actions-label-merge-conflict/blob/main/CHANGELOG.md) - [Commits](https://github.com/eps1lon/actions-label-merge-conflict/compare/v3.0.1...v3.0.2) --- updated-dependencies: - dependency-name: eps1lon/actions-label-merge-conflict dependency-type: direct:production update-type: version-update:semver-patch dependency-group: github_action-dependencies ... Signed-off-by: dependabot[bot] --- .github/workflows/merge-conflict.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/merge-conflict.yml b/.github/workflows/merge-conflict.yml index c2d3444f..24b299fc 100644 --- a/.github/workflows/merge-conflict.yml +++ b/.github/workflows/merge-conflict.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check if PRs are have merge conflicts - uses: eps1lon/actions-label-merge-conflict@v3.0.1 + uses: eps1lon/actions-label-merge-conflict@v3.0.2 with: dirtyLabel: "Merge conflicts" repoToken: "${{ secrets.GITHUB_TOKEN }}" From daa26ae9cba27718dc0609aa83e49956aeea0717 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 2 Jun 2024 05:56:58 +0200 Subject: [PATCH 121/339] Implement time updating via optional --update flag and switch to unsigned 64 bit and double computations as mandated by RFC 5905 (page 28) Signed-off-by: DL6ER --- src/args.c | 20 +++++-- src/ntp/client.c | 126 +++++++++++++++++++++++++++++-------------- src/ntp/ntp.h | 30 +++++++++-- src/ntp/server.c | 88 +++++++++++------------------- test/test_suite.bats | 2 +- 5 files changed, 157 insertions(+), 109 deletions(-) diff --git a/src/args.c b/src/args.c index a563192f..ab7d6c44 100644 --- a/src/args.c +++ b/src/args.c @@ -308,13 +308,17 @@ void parse_args(int argc, char* argv[]) } // Create test NTP client - if((argc == 2 || argc == 3) && strcmp(argv[1], "ntp-client") == 0) + if((argc > 1 && argc < 5) && strcmp(argv[1], "ntp") == 0) { // Enable stdout printing cli_mode = true; log_ctrl(false, true); - const char *server = argc == 3 ? argv[2] : "127.0.0.1"; - exit(ntp_client(server) ? EXIT_SUCCESS : EXIT_FAILURE); + const bool update = (argc > 2 && strcmp(argv[2], "--update") == 0) || + (argc > 3 && strcmp(argv[3], "--update") == 0); + const char *server = "127.0.0.1"; + if(argc > 2 && strcmp(argv[2], "--update") != 0) + server = argv[2]; + exit(ntp_client(server, update) ? EXIT_SUCCESS : EXIT_FAILURE); } // Import teleporter archive through CLI @@ -1029,12 +1033,18 @@ void parse_args(int argc, char* argv[]) printf(" Encoding: %spihole-FTL idn2 %sdomain%s\n", green, cyan, normal); printf(" Decoding: %spihole-FTL idn2 -d %spunycode%s\n\n", green, cyan, normal); + printf("%sNTP client:%s\n", yellow, normal); + printf(" Query an NTP server for the current time and print the\n"); + printf(" result in human-readable format. An optional %sserver%s may be\n", cyan, normal); + printf(" as argument. If the server is omitted, 127.0.0.1 is used.\n\n"); + printf(" The system time is updated on the system when the optional\n"); + printf(" %s--update%s flag is given.\n\n", purple, normal); + printf(" Usage: %spihole-FTL ntp %s[server]%s %s[--update]%s\n\n", green, cyan, normal, purple, normal); + printf("%sOther:%s\n", yellow, normal); printf("\t%sptr %sIP%s %s[tcp]%s Resolve IP address to hostname\n", green, cyan, normal, purple, normal); printf("\t Append %stcp%s to use TCP instead of UDP\n", purple, normal); printf("\t%ssha256sum %sfile%s Calculate SHA256 checksum of a file\n", green, cyan, normal); - printf("\t%sntp-client %s[server]%s Request network time from %sserver%s\n", green, cyan, normal, cyan, normal); - printf("\t defaults to 127.0.0.1 if omitted\n"); printf("\t%sdhcp-discover%s Discover DHCP servers in the local\n", green, normal); printf("\t network\n"); printf("\t%sarp-scan %s[-a/-x]%s Use ARP to scan local network for\n", green, cyan, normal); diff --git a/src/ntp/client.c b/src/ntp/client.c index 092df60f..f5a960d8 100644 --- a/src/ntp/client.c +++ b/src/ntp/client.c @@ -25,13 +25,15 @@ #include // errno #include +// PRIi64 +#include #include "ntp/ntp.h" #include "log.h" // Create minimal NTP request, see server implementation for details about the // packet structure -static bool request(int fd, uint32_t org[2]) +static bool request(int fd, uint64_t *org) { // NTP Packet buffer unsigned char buf[48] = {0}; @@ -39,9 +41,14 @@ static bool request(int fd, uint32_t org[2]) // LI = 0, VN = 4 (current version), Mode = 3 (Client) buf[0] = 0x23; + // Minimum poll interval (2^6 = 64 seconds) + buf[2] = 0x06; + // Set Origin Timestamp - gettime32(org, true); - memcpy(&buf[40], &org[0], 2 * sizeof(uint32_t)); + *org = gettime64(); + //memcpy(&buf[40], &org[0], 2 * sizeof(uint32_t)); + const uint64_t norg = hton64(*org); + memcpy(&buf[40], &norg, sizeof(norg)); // Send request if(send(fd, buf, 48, 0) != 48) @@ -53,12 +60,10 @@ static bool request(int fd, uint32_t org[2]) return true; } -static bool get_reply(int fd, uint32_t org_[2]) +static bool reply(int fd, uint64_t *org_, const bool settime) { // NTP Packet buffer unsigned char buf[48]; - // NTP Packet buffer as uint32_t - uint32_t *pt = (uint32_t *)((void*)&buf[24]);; // Receive reply if(recv(fd, buf, 48, 0) < 48) @@ -81,27 +86,24 @@ static bool get_reply(int fd, uint32_t org_[2]) // Extract Transmit Timestamp // org = Origin Timestamp (Transmit Timestamp @ Client) - uint32_t org[2]; - org[0] = ntohl(*pt++); - org[1] = ntohl(*pt++); + uint64_t netbuffer; + memcpy(&netbuffer, &buf[24], sizeof(netbuffer)); + const uint64_t org = ntoh64(netbuffer); // rec = Receive Timestamp (Receive Timestamp @ Server) - uint32_t rec[2]; - rec[0] = ntohl(*pt++); - rec[1] = ntohl(*pt++); + memcpy(&netbuffer, &buf[32], sizeof(netbuffer)); + const uint64_t rec = ntoh64(netbuffer); // xmt = Transmit Timestamp (Transmit Timestamp @ Server) - uint32_t xmt[2]; - xmt[0] = ntohl(*pt++); - xmt[1] = ntohl(*pt++); + memcpy(&netbuffer, &buf[40], sizeof(netbuffer)); + const uint64_t xmt = ntoh64(netbuffer); // dst = Destination Timestamp (Receive Timestamp @ Client) - uint32_t dst[2]; - gettime32(dst, false); + uint64_t dst = gettime64(); // Check org_ and org are identical (otherwise, the reply corresponds to // a different request and should be ignored), note that the byte order // of the received packet is already converted while org_ is still in // network byte order - if(ntohl(org_[0]) != org[0] || ntohl(org_[1]) != org[1]) + if(*org_ != org) { log_warn("Received NTP reply does not match request"); return false; @@ -115,11 +117,10 @@ static bool get_reply(int fd, uint32_t org_[2]) } // Calculate delay and offset - const double tfrac = 4294967296.0; // 2^32 as double - const double T1 = org[0] + org[1] / tfrac; - const double T2 = rec[0] + rec[1] / tfrac; - const double T3 = xmt[0] + xmt[1] / tfrac; - const double T4 = dst[0] + dst[1] / tfrac; + const double T1 = org / FRAC; + const double T2 = rec / FRAC; + const double T3 = xmt / FRAC; + const double T4 = dst / FRAC; // RFC 5905, Section 8: On-wire protocol // It is recommended to use double precision floating point arithmetic @@ -128,7 +129,9 @@ static bool get_reply(int fd, uint32_t org_[2]) // Compute offset of client clock relative to server clock const double theta = ( ( T2 - T1 ) + ( T3 - T4 ) ) / 2; - // Compute round-trip delay + // Compute round-trip delay, which represents the delay of the packet + // passing through the network, which can be due switches and network + // technologies are highly variable double delta = ( T4 - T1 ) - ( T3 - T2 ); // In some scenarios where the initial frequency offset of the client is @@ -140,37 +143,78 @@ static bool get_reply(int fd, uint32_t org_[2]) // clamped not less than s.rho, where s.rho is the system precision // described in Section 11.1, expressed in seconds. if(delta < s_rho) - { - log_warn("Negative delay detected, clamping to 0"); delta = 0; - } // Print current time at client - char client_time_str[26]; - const time_t client_time = dst[0]; - strncpy(client_time_str, ctime(&client_time), sizeof(client_time_str) -1); - // Remove trailing newline - client_time_str[24] = '\0'; + char client_time_str[128]; + struct timeval client_time; + client_time.tv_sec = NTPtoSEC(dst); + client_time.tv_usec = NTPtoUSEC(dst); + struct tm *client_tm = localtime(&client_time.tv_sec); + snprintf(client_time_str, sizeof(client_time_str), "%04i-%02i-%02i %02i:%02i:%02i.%06"PRIi64" %s", + client_tm->tm_year + 1900, client_tm->tm_mon + 1, client_tm->tm_mday, + client_tm->tm_hour, client_tm->tm_min, client_tm->tm_sec, client_time.tv_usec, + client_tm->tm_zone); + client_time_str[sizeof(client_time_str) - 1] = '\0'; log_info("Current time at client: %s", client_time_str); // Print current time at server - char server_time_str[26]; - const time_t server_time = xmt[0]; - strncpy(server_time_str, ctime(&server_time), sizeof(server_time_str) -1); - // Remove trailing newline - server_time_str[24] = '\0'; + char server_time_str[128]; + struct timeval server_time; + server_time.tv_sec = NTPtoSEC(xmt); + server_time.tv_usec = NTPtoUSEC(xmt); + struct tm *server_tm = localtime(&server_time.tv_sec); + snprintf(server_time_str, sizeof(server_time_str), "%04i-%02i-%02i %02i:%02i:%02i.%06"PRIi64" %s", + server_tm->tm_year + 1900, server_tm->tm_mon + 1, server_tm->tm_mday, + server_tm->tm_hour, server_tm->tm_min, server_tm->tm_sec, server_time.tv_usec, + server_tm->tm_zone); + server_time_str[sizeof(server_time_str) - 1] = '\0'; log_info("Current time at server: %s", server_time_str); // Print offset and delay log_info("Time offset: %e s", theta); log_info("Round-trip delay: %e s", delta); + // Set time if requested + if(settime) + { + // Get current time + struct timeval unix_time; + gettimeofday(&unix_time, NULL); + + // Convert from double to native format (signed) and add to the + // current time. Note the addition is done in native format to + // avoid overflow or loss of precision. + const uint64_t ntp_time = D2LFP(theta) + U2LFP(unix_time); + + // Convert NTP to native format + unix_time.tv_sec = NTPtoSEC(ntp_time); + unix_time.tv_usec = NTPtoUSEC(ntp_time); + + // Print new time + char new_time_str[128]; + struct tm *new_time_tm = localtime(&unix_time.tv_sec); + snprintf(new_time_str, sizeof(new_time_str), "%04i-%02i-%02i %02i:%02i:%02i.%06"PRIi64" %s", + new_time_tm->tm_year + 1900, new_time_tm->tm_mon + 1, new_time_tm->tm_mday, + new_time_tm->tm_hour, new_time_tm->tm_min, new_time_tm->tm_sec, unix_time.tv_usec, + new_time_tm->tm_zone); + new_time_str[sizeof(new_time_str) - 1] = '\0'; + + // Set time + if(settimeofday(&unix_time, NULL) != 0) + { + log_warn("Failed to set time to %s: %s", new_time_str, strerror(errno)); + return false; + } + log_info("Updated time at client: %s", new_time_str); + } + // Offset and delay larger than 0.1 seconds are considered as invalid // during local testing return theta < 0.1 && delta < 0.1; } -bool ntp_client(const char *server) +bool ntp_client(const char *server, const bool settime) { const int protocol = strchr(server, ':') != NULL ? AF_INET6 : AF_INET; @@ -212,15 +256,15 @@ bool ntp_client(const char *server) freeaddrinfo(saddr); // Send request - uint32_t org[2]; - if(!request(s, org)) + uint64_t org; + if(!request(s, &org)) { close(s); return false; } // Get reply - const bool status = get_reply(s, org); + const bool status = reply(s, &org, settime); close(s); return status; diff --git a/src/ntp/ntp.h b/src/ntp/ntp.h index c6e159c5..c71d3740 100644 --- a/src/ntp/ntp.h +++ b/src/ntp/ntp.h @@ -16,12 +16,34 @@ // bool #include -//uint64_t gettime32(void); -void gettime32(uint32_t ts[], const bool netorder); -//uint64_t gettime64(void); +// Get current time in NTP (64bit) format +uint64_t gettime64(void); +// Start NTP server bool ntp_server_start(void); -bool ntp_client(const char *server); + +// Start NTP client +bool ntp_client(const char *server, const bool settime); + +// number of seconds between 1900 and 1970 (MSB=1) +#define DIFF_SEC_1900_1970 (2208988800UL) +// number of seconds between 1970 and Feb 7, 2036 (6:28:16 UTC) (MSB=0) +#define DIFF_SEC_1970_2036 (2085978496UL) + +// Timestamp conversion macroni (RFC 5905, Appendix A) +#define FRAC 4294967296. // 2^32 as double +#define D2LFP(a) ((uint64_t)((a) * FRAC)) // NTP timestamp +#define LFP2D(a) ((double)(a) / FRAC) +#define U2LFP(a) (((uint64_t)((a).tv_sec + DIFF_SEC_1900_1970) << 32) + (uint64_t) ((a).tv_usec / 1e6 * FRAC)) + +// Convert NTP timestamp to seconds and microseconds +//#define NTPtoSEC(x) (((x & 0x80000000) != 0) ? ((x >> 32) - DIFF_SEC_1900_1970) : ((x >> 32) + DIFF_SEC_1970_2036)) +#define NTPtoSEC(x) ((x >> 32) - DIFF_SEC_1900_1970) +#define NTPtoUSEC(x) (suseconds_t)((LFP2D(x & 0xFFFFFFFF) * 1e6)) + +// Convert uint64_t to network byte order and vice versa +#define hton64(x) ((((uint64_t)htonl(x)) << 32) + htonl((x) >> 32)) +#define ntoh64(x) ((((uint64_t)ntohl(x)) << 32) + ntohl((x) >> 32)) #endif // NTP_H diff --git a/src/ntp/server.c b/src/ntp/server.c index 6814f7d4..fd924076 100644 --- a/src/ntp/server.c +++ b/src/ntp/server.c @@ -29,8 +29,6 @@ #include // ctime() #include -// log2() -#include // pthread_create #include // PR_SET_NAME @@ -40,33 +38,17 @@ #include "log.h" #include "config/config.h" -// Retrieves the current system time, adjusts it to a 1900 epoch, converts it to -// a 32-bit fraction of a second, and optionally converts it to network byte -// order. -void gettime32(uint32_t tv[], const bool netorder) +// RFC 5905 Appendix A.4: Kernel System Clock Interface +uint64_t gettime64(void) { - struct timespec ts; - // CLOCK_REALTIME is the system-wide realtime clock. - // It is both affected by discontinuous jumps in the system time (e.g., - // if the system administrator manually changes the clock), and by the - // incremental adjustments performed by adjtime(3) and NTP. - clock_gettime(CLOCK_REALTIME, &ts); - - // Set the epoch to 1900 (add seconds from 1900 to 1970) - tv[0] = ts.tv_sec + 2208988800ULL; - // Convert microseconds to 32 bit fraction of a second - tv[1] = (ts.tv_nsec * 0x100000000ULL) / 1000000000ULL; - - if (netorder) - { - tv[0] = htonl(tv[0]); - tv[1] = htonl(tv[1]); - } + struct timeval unix_time; + gettimeofday(&unix_time, NULL); + return (U2LFP(unix_time)); } // Create and send an NTP reply to the client static int ntp_reply(const int socket_fd, const struct sockaddr *saddr_p, const socklen_t saddrlen, - const unsigned char recv_buf[], const uint32_t recv_time[2]) + const unsigned char recv_buf[], const uint64_t *recv_time) { // Buffer for the response unsigned char send_buf[48]; @@ -102,13 +84,8 @@ static int ntp_reply(const int socket_fd, const struct sockaddr *saddr_p, const // Copy Poll value from client send_buf[2] = recv_buf[2]; - // Precision in Nanoseconds from CLOCK_REALTIME - struct timespec ts; - clock_getres(CLOCK_REALTIME, &ts); - // Precision in log2 seconds - signed char precision = (signed char)(1.0*log2(1e-9*ts.tv_nsec)); - // Precision in log2 seconds - send_buf[3] = precision; + // Precision (log2(1e-6) = -19.931568569324174) + send_buf[3] = (signed char)(-19); // Advance 32 bit pointer to the next field u32p++; @@ -143,23 +120,23 @@ static int ntp_reply(const int socket_fd, const struct sockaddr *saddr_p, const // + Reference Timestamp (64) + // | | // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + // Time when the system clock was last set or corrected, in NTP // timestamp format. As this is not a stratum 1 server, we don't have // a hardware clock to set this value. #ifdef MOCK_REFTIME // Mock this timestamp with the current time of the server minus 1 // minute. - uint32_t ref_time[2]; - gettime32(ref_time, true); - ref_time[0] = ref_time[0] - htonl(60); // subtract 60 seconds - memcpy(u32p, ref_time, 2 * sizeof(uint32_t)); + const uint64_t ref_time = gettime64() - 60 * 1000000; + const uint64_t net_ref_time = hton64(ref_time); + memcpy(u32p, &net_ref_time, sizeof(uint64_t)); u32p += 2; #else // A stateless server copies T3 and T4 from the client packet to T1 and // T2 of the server packet and tacks on the transmit timestamp T3 before // sending it to the client. - *u32p++ = u32r[8]; - *u32p++ = u32r[9]; + memcpy(u32p, &u32r[8], sizeof(uint64_t)); + u32p += 2; #endif // 0 1 2 3 // 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 @@ -168,10 +145,11 @@ static int ntp_reply(const int socket_fd, const struct sockaddr *saddr_p, const // + Origin Timestamp (64) + // | | // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + // Time at the client when the request departed for the server, in NTP // timestamp format. (this is the client's transmit time) - *u32p++ = u32r[10]; - *u32p++ = u32r[11]; + memcpy(u32p, &u32r[10], sizeof(uint64_t)); + u32p += 2; // 0 1 2 3 // 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 @@ -180,9 +158,11 @@ static int ntp_reply(const int socket_fd, const struct sockaddr *saddr_p, const // + Receive Timestamp (64) + // | | // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + // Time at the server when the request arrived from the client, in NTP // timestamp format. (this is the server's receive time) - memcpy(u32p, recv_time, 2 * sizeof(uint32_t)); + const uint64_t net_recv_time = hton64(*recv_time); + memcpy(u32p, &net_recv_time, sizeof(uint64_t)); u32p += 2; // 0 1 2 3 @@ -192,12 +172,13 @@ static int ntp_reply(const int socket_fd, const struct sockaddr *saddr_p, const // + Transmit Timestamp (64) + // | | // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + // Time at the server when the response left for the client, in NTP // timestamp format. (this is the server's transmit time) - uint32_t transmit_time[2]; - gettime32(transmit_time, true); - memcpy(u32p, transmit_time, 2 * sizeof(uint32_t)); - u32p += 2; + const uint64_t transmit_time = gettime64(); + const uint64_t net_transmit_time = hton64(transmit_time); + memcpy(u32p, &net_transmit_time, sizeof(uint64_t)); + // u32p += 2; // 0 1 2 3 // 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 @@ -245,18 +226,17 @@ static void request_process_loop(int fd, const char *ipstr, const int protocol) socklen_t src_addrlen = sizeof(src_addr); while(recvfrom(fd, buf, sizeof(buf), 0, &src_addr, &src_addrlen) < 48); // ignore invalid requests - // Get the current time in NTP format - uint32_t recv_time[2]; - gettime32(recv_time, true); + // Get the current time in NTP format directly after receiving + // the request + const uint64_t recv_time = gettime64(); struct sockaddr_in sin; memcpy(&sin, &src_addr, sizeof(sin)); - // printf("Request from %s\n", inet_ntoa(sin.sin_addr)); const pid_t pid = fork(); if (pid == 0) { - /* Child */ - ntp_reply(fd, &src_addr , src_addrlen, buf, recv_time); + // Child + ntp_reply(fd, &src_addr , src_addrlen, buf, &recv_time); exit(0); } else if (pid == -1) { log_err("fork() error"); @@ -265,18 +245,10 @@ static void request_process_loop(int fd, const char *ipstr, const int protocol) // return to parent } } -/* -// Wait for a child process to exit -static void wait_wrapper(int _a) -{ - int s; - wait(&s); -}*/ // Start the NTP server static void *ntp_bind_and_listen(void *param) { -// signal(SIGCHLD, wait_wrapper); const int protocol = param == 0 ? AF_INET : AF_INET6; // Create a socket diff --git a/test/test_suite.bats b/test/test_suite.bats index 92cd34e1..f16e4d7f 100644 --- a/test/test_suite.bats +++ b/test/test_suite.bats @@ -1360,7 +1360,7 @@ } @test "Check NTP server is broadcasting correct time" { - run bash -c './pihole-FTL ntp-client 127.0.0.1' + run bash -c './pihole-FTL ntp 127.0.0.1' printf "%s\n" "${lines[@]}" [[ $status == 0 ]] } From 79c966e2e66bff2c530b6b4e279283dfd0937d22 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 2 Jun 2024 07:24:05 +0200 Subject: [PATCH 122/339] Reduce code duplication Signed-off-by: DL6ER --- src/ntp/client.c | 51 +++++++++++++++++++----------------------------- 1 file changed, 20 insertions(+), 31 deletions(-) diff --git a/src/ntp/client.c b/src/ntp/client.c index f5a960d8..632a067c 100644 --- a/src/ntp/client.c +++ b/src/ntp/client.c @@ -60,6 +60,22 @@ static bool request(int fd, uint64_t *org) return true; } +// Display NTP time in human-readable format +static void display_time(const char *description, const uint64_t ntp_time) +{ + char client_time_str[128]; + struct timeval client_time; + client_time.tv_sec = NTPtoSEC(ntp_time); + client_time.tv_usec = NTPtoUSEC(ntp_time); + struct tm *client_tm = localtime(&client_time.tv_sec); + snprintf(client_time_str, sizeof(client_time_str), "%04i-%02i-%02i %02i:%02i:%02i.%06"PRIi64" %s", + client_tm->tm_year + 1900, client_tm->tm_mon + 1, client_tm->tm_mday, + client_tm->tm_hour, client_tm->tm_min, client_tm->tm_sec, client_time.tv_usec, + client_tm->tm_zone); + client_time_str[sizeof(client_time_str) - 1] = '\0'; + log_info("%s: %s", description, client_time_str); +} + static bool reply(int fd, uint64_t *org_, const bool settime) { // NTP Packet buffer @@ -146,30 +162,10 @@ static bool reply(int fd, uint64_t *org_, const bool settime) delta = 0; // Print current time at client - char client_time_str[128]; - struct timeval client_time; - client_time.tv_sec = NTPtoSEC(dst); - client_time.tv_usec = NTPtoUSEC(dst); - struct tm *client_tm = localtime(&client_time.tv_sec); - snprintf(client_time_str, sizeof(client_time_str), "%04i-%02i-%02i %02i:%02i:%02i.%06"PRIi64" %s", - client_tm->tm_year + 1900, client_tm->tm_mon + 1, client_tm->tm_mday, - client_tm->tm_hour, client_tm->tm_min, client_tm->tm_sec, client_time.tv_usec, - client_tm->tm_zone); - client_time_str[sizeof(client_time_str) - 1] = '\0'; - log_info("Current time at client: %s", client_time_str); + display_time("Current time at client", dst); // Print current time at server - char server_time_str[128]; - struct timeval server_time; - server_time.tv_sec = NTPtoSEC(xmt); - server_time.tv_usec = NTPtoUSEC(xmt); - struct tm *server_tm = localtime(&server_time.tv_sec); - snprintf(server_time_str, sizeof(server_time_str), "%04i-%02i-%02i %02i:%02i:%02i.%06"PRIi64" %s", - server_tm->tm_year + 1900, server_tm->tm_mon + 1, server_tm->tm_mday, - server_tm->tm_hour, server_tm->tm_min, server_tm->tm_sec, server_time.tv_usec, - server_tm->tm_zone); - server_time_str[sizeof(server_time_str) - 1] = '\0'; - log_info("Current time at server: %s", server_time_str); + display_time("Current time at server", xmt); // Print offset and delay log_info("Time offset: %e s", theta); @@ -192,21 +188,14 @@ static bool reply(int fd, uint64_t *org_, const bool settime) unix_time.tv_usec = NTPtoUSEC(ntp_time); // Print new time - char new_time_str[128]; - struct tm *new_time_tm = localtime(&unix_time.tv_sec); - snprintf(new_time_str, sizeof(new_time_str), "%04i-%02i-%02i %02i:%02i:%02i.%06"PRIi64" %s", - new_time_tm->tm_year + 1900, new_time_tm->tm_mon + 1, new_time_tm->tm_mday, - new_time_tm->tm_hour, new_time_tm->tm_min, new_time_tm->tm_sec, unix_time.tv_usec, - new_time_tm->tm_zone); - new_time_str[sizeof(new_time_str) - 1] = '\0'; + display_time("Setting time to", ntp_time); // Set time if(settimeofday(&unix_time, NULL) != 0) { - log_warn("Failed to set time to %s: %s", new_time_str, strerror(errno)); + log_warn("Failed to set time: %s", strerror(errno)); return false; } - log_info("Updated time at client: %s", new_time_str); } // Offset and delay larger than 0.1 seconds are considered as invalid From f8990e769ae519fc65955b77d69f9cbfcc7252be Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 2 Jun 2024 07:31:55 +0200 Subject: [PATCH 123/339] Average over up to eight successive NTP queries to reduce total time error during synchronization Signed-off-by: DL6ER --- src/ntp/client.c | 242 ++++++++++++++++++++++++++++++----------------- src/ntp/ntp.h | 4 + src/ntp/server.c | 24 ++--- 3 files changed, 171 insertions(+), 99 deletions(-) diff --git a/src/ntp/client.c b/src/ntp/client.c index 632a067c..41089c0f 100644 --- a/src/ntp/client.c +++ b/src/ntp/client.c @@ -31,9 +31,18 @@ #include "ntp/ntp.h" #include "log.h" +struct ntp_sync +{ + uint64_t org; + uint64_t xmt; + double theta; + double delta; + double precision; +}; + // Create minimal NTP request, see server implementation for details about the // packet structure -static bool request(int fd, uint64_t *org) +static bool request(int fd, struct ntp_sync *ntp) { // NTP Packet buffer unsigned char buf[48] = {0}; @@ -44,16 +53,19 @@ static bool request(int fd, uint64_t *org) // Minimum poll interval (2^6 = 64 seconds) buf[2] = 0x06; - // Set Origin Timestamp - *org = gettime64(); - //memcpy(&buf[40], &org[0], 2 * sizeof(uint32_t)); - const uint64_t norg = hton64(*org); + // Set Reference Timestamp (ref) to 0 + // This is the time at which the local clock was last set or corrected. + memset(&buf[8], 0, sizeof(uint64_t)); + + // Set Origin Timestamp (org) in NTP format + ntp->org = gettime64(); + const uint64_t norg = hton64(ntp->org); memcpy(&buf[40], &norg, sizeof(norg)); // Send request if(send(fd, buf, 48, 0) != 48) { - log_warn("Failed to send data to NTP server: %s", strerror(errno)); + printf("Failed to send data to NTP server: %s\n", strerror(errno)); return false; } @@ -61,6 +73,8 @@ static bool request(int fd, uint64_t *org) } // Display NTP time in human-readable format +// This function is similar to get_timestr() in src/log.c but differs in that it +// includes microseconds whereas get_timestr() only includes milliseconds static void display_time(const char *description, const uint64_t ntp_time) { char client_time_str[128]; @@ -73,10 +87,10 @@ static void display_time(const char *description, const uint64_t ntp_time) client_tm->tm_hour, client_tm->tm_min, client_tm->tm_sec, client_time.tv_usec, client_tm->tm_zone); client_time_str[sizeof(client_time_str) - 1] = '\0'; - log_info("%s: %s", description, client_time_str); + printf("%s: %s\n", description, client_time_str); } -static bool reply(int fd, uint64_t *org_, const bool settime) +static bool reply(int fd, struct ntp_sync *ntp, const bool verbose) { // NTP Packet buffer unsigned char buf[48]; @@ -84,7 +98,7 @@ static bool reply(int fd, uint64_t *org_, const bool settime) // Receive reply if(recv(fd, buf, 48, 0) < 48) { - log_warn("Failed to receive data from NTP server: %s", strerror(errno)); + printf("Failed to receive data from NTP server: %s\n", strerror(errno)); return false; } @@ -94,11 +108,11 @@ static bool reply(int fd, uint64_t *org_, const bool settime) { // Accepted limits are 2^-32 (~ 0.2 nanoseconds) // to 2^0 (= 1 second) - log_warn("Received NTP reply has invalid precision: 2^(%i), assuming microsecond accuracy", rho); + printf("Received NTP reply has invalid precision: 2^(%i), assuming microsecond accuracy\n", rho); rho = -19; } // Compute precision of server clock in seconds 2^rho - const double s_rho = pow(2, rho); + ntp->precision = pow(2, rho); // Extract Transmit Timestamp // org = Origin Timestamp (Transmit Timestamp @ Client) @@ -110,7 +124,7 @@ static bool reply(int fd, uint64_t *org_, const bool settime) const uint64_t rec = ntoh64(netbuffer); // xmt = Transmit Timestamp (Transmit Timestamp @ Server) memcpy(&netbuffer, &buf[40], sizeof(netbuffer)); - const uint64_t xmt = ntoh64(netbuffer); + ntp->xmt = ntoh64(netbuffer); // dst = Destination Timestamp (Receive Timestamp @ Client) uint64_t dst = gettime64(); @@ -119,23 +133,23 @@ static bool reply(int fd, uint64_t *org_, const bool settime) // a different request and should be ignored), note that the byte order // of the received packet is already converted while org_ is still in // network byte order - if(*org_ != org) + if(ntp->org != org) { - log_warn("Received NTP reply does not match request"); + printf("Received NTP reply does not match request (request %"PRIx64", reply %"PRIx64")\n", ntp->org, org); return false; } // Check stratum, mode, version, etc. if((buf[0] & 0x07) != 4) { - log_warn("Received NTP reply has invalid version"); + printf("Received NTP reply has invalid version\n"); return false; } // Calculate delay and offset - const double T1 = org / FRAC; + const double T1 = ntp->org / FRAC; const double T2 = rec / FRAC; - const double T3 = xmt / FRAC; + const double T3 = ntp->xmt / FRAC; const double T4 = dst / FRAC; // RFC 5905, Section 8: On-wire protocol @@ -144,11 +158,11 @@ static bool reply(int fd, uint64_t *org_, const bool settime) // results within the maximum adjustment range of 68 years. // Compute offset of client clock relative to server clock - const double theta = ( ( T2 - T1 ) + ( T3 - T4 ) ) / 2; + ntp->theta = ( ( T2 - T1 ) + ( T3 - T4 ) ) / 2; // Compute round-trip delay, which represents the delay of the packet // passing through the network, which can be due switches and network // technologies are highly variable - double delta = ( T4 - T1 ) - ( T3 - T2 ); + ntp->delta = ( T4 - T1 ) - ( T3 - T2 ); // In some scenarios where the initial frequency offset of the client is // relatively large and the actual propagation time small, it is @@ -158,18 +172,131 @@ static bool reply(int fd, uint64_t *org_, const bool settime) // misleading in subsequent computations, the value of delta should be // clamped not less than s.rho, where s.rho is the system precision // described in Section 11.1, expressed in seconds. - if(delta < s_rho) - delta = 0; + if(ntp->delta < ntp->precision) + ntp->delta = 0; + +# // Return early if not verbose + if(!verbose) + return true; // Print current time at client display_time("Current time at client", dst); // Print current time at server - display_time("Current time at server", xmt); + display_time("Current time at server", ntp->xmt); // Print offset and delay - log_info("Time offset: %e s", theta); - log_info("Round-trip delay: %e s", delta); + printf("Time offset: %e s\n", ntp->theta); + printf("Round-trip delay: %e s\n", ntp->delta); + + return true; +} + +bool ntp_client(const char *server, const bool settime) +{ + const int protocol = strchr(server, ':') != NULL ? AF_INET6 : AF_INET; + + // Create UDP socket + const int s = socket(protocol, SOCK_DGRAM, IPPROTO_UDP); + if(s == -1) + { + printf("ERROR: Cannot create UDP socket\n"); + return false; + } + + // Set socket timeout to 2 seconds + struct timeval tv; + tv.tv_sec = 2; + tv.tv_usec = 0; + if(setsockopt(s, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv)) != 0) + { + printf("ERROR: Cannot set socket timeout\n"); + close(s); + return false; + } + + // Resolve server address + struct addrinfo *saddr; + if(getaddrinfo(server, "123", NULL, &saddr) != 0) + { + printf("ERROR: Cannot resolve NTP server address\n"); + close(s); + return false; + } + + // Set address to send to/receive from + if(connect(s, saddr->ai_addr, saddr->ai_addrlen) != 0) + { + printf("ERROR: Cannot connect to NTP server\n"); + close(s); + return false; + } + freeaddrinfo(saddr); + + struct ntp_sync ntp[NTP_AVERGAGE_COUNT]; + memset(&ntp, 0, sizeof(ntp)); + for(unsigned int i = 0; i < NTP_AVERGAGE_COUNT; i++) + { + // Send request + if(!request(s, &ntp[i])) + { + close(s); + return false; + } + // Get reply + if(!reply(s, &ntp[i], false)) + continue; + + // Sleep for 100 ms to avoid flooding the server + printf("."); + fflush(stdout); + usleep(100000); + } + printf("\n"); + + // Close socket + close(s); + + // Compute average and standard deviation + unsigned int valid = 0; + double theta_avg = 0.0, theta_stdev = 0.0; + double delta_avg = 0.0, delta_stdev = 0.0; + for(unsigned int i = 0; i < NTP_AVERGAGE_COUNT; i++) + { + // Skip invalid values + if(fabs(ntp[i].theta) < ntp[i].precision || + fabs(ntp[i].delta) < ntp[i].precision) + continue; + + theta_avg += ntp[i].theta; + delta_avg += ntp[i].delta; + valid++; + } + + if(valid == 0) + { + printf("No valid NTP replies received, check server and network connectivity\n\n"); + return false; + } + printf("Received %u/%d valid NTP replies\n\n", valid, NTP_AVERGAGE_COUNT); + + theta_avg /= valid; + delta_avg /= valid; + for(unsigned int i = 0; i < NTP_AVERGAGE_COUNT; i++) + { + // Skip invalid values + if(fabs(ntp[i].theta) < ntp[i].precision || + fabs(ntp[i].delta) < ntp[i].precision) + continue; + + theta_stdev += pow(ntp[i].theta - theta_avg, 2); + delta_stdev += pow(ntp[i].delta - delta_avg, 2); + } + theta_stdev = sqrt(theta_stdev / valid); + delta_stdev = sqrt(delta_stdev / valid); + + printf("Average time offset: (%e +/- %e s)\n", theta_avg, theta_stdev); + printf("Average round-trip delay: (%e +/- %e s)\n", delta_avg, delta_stdev); // Set time if requested if(settime) @@ -181,80 +308,25 @@ static bool reply(int fd, uint64_t *org_, const bool settime) // Convert from double to native format (signed) and add to the // current time. Note the addition is done in native format to // avoid overflow or loss of precision. - const uint64_t ntp_time = D2LFP(theta) + U2LFP(unix_time); + const uint64_t ntp_time = U2LFP(unix_time) + D2LFP(theta_avg); // Convert NTP to native format unix_time.tv_sec = NTPtoSEC(ntp_time); unix_time.tv_usec = NTPtoUSEC(ntp_time); // Print new time - display_time("Setting time to", ntp_time); + display_time("Setting local time to", ntp_time); // Set time if(settimeofday(&unix_time, NULL) != 0) { - log_warn("Failed to set time: %s", strerror(errno)); + printf("Failed to set time: %s\n", + errno == EPERM ? "Insufficient permissions, try running with sudo" : strerror(errno)); return false; } } // Offset and delay larger than 0.1 seconds are considered as invalid - // during local testing - return theta < 0.1 && delta < 0.1; -} - -bool ntp_client(const char *server, const bool settime) -{ - const int protocol = strchr(server, ':') != NULL ? AF_INET6 : AF_INET; - - // Create UDP socket - const int s = socket(protocol, SOCK_DGRAM, IPPROTO_UDP); - if(s == -1) - { - log_err("Cannot create UDP socket"); - return false; - } - - // Set socket timeout to 2 seconds - struct timeval tv; - tv.tv_sec = 2; - tv.tv_usec = 0; - if(setsockopt(s, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv)) != 0) - { - log_err("Cannot set socket timeout"); - close(s); - return false; - } - - // Resolve server address - struct addrinfo *saddr; - if(getaddrinfo(server, "123", NULL, &saddr) != 0) - { - log_err("Cannot resolve NTP server address"); - close(s); - return false; - } - - // Set address to send to/receive from - if(connect(s, saddr->ai_addr, saddr->ai_addrlen) != 0) - { - log_err("Cannot connect to NTP server"); - close(s); - return false; - } - freeaddrinfo(saddr); - - // Send request - uint64_t org; - if(!request(s, &org)) - { - close(s); - return false; - } - - // Get reply - const bool status = reply(s, &org, settime); - close(s); - - return status; + // during local testing (e.g., when the server is on the same machine) + return theta_avg < 0.1 && delta_avg < 0.1; } diff --git a/src/ntp/ntp.h b/src/ntp/ntp.h index c71d3740..cc82c9b9 100644 --- a/src/ntp/ntp.h +++ b/src/ntp/ntp.h @@ -25,6 +25,10 @@ bool ntp_server_start(void); // Start NTP client bool ntp_client(const char *server, const bool settime); +// Number of NTP queries to average. The more queries, the more accurate the +// time, but the longer it takes to synchronize. The minimum is 1. +#define NTP_AVERGAGE_COUNT 8 + // number of seconds between 1900 and 1970 (MSB=1) #define DIFF_SEC_1900_1970 (2208988800UL) // number of seconds between 1970 and Feb 7, 2036 (6:28:16 UTC) (MSB=0) diff --git a/src/ntp/server.c b/src/ntp/server.c index fd924076..e1d593a5 100644 --- a/src/ntp/server.c +++ b/src/ntp/server.c @@ -84,8 +84,10 @@ static int ntp_reply(const int socket_fd, const struct sockaddr *saddr_p, const // Copy Poll value from client send_buf[2] = recv_buf[2]; - // Precision (log2(1e-6) = -19.931568569324174) - send_buf[3] = (signed char)(-19); + // Precision: the precision of the local clock, in seconds to the + // nearest power of two. + // log2(1 usec = 1e-6 s) = -19.931568569324174 + send_buf[3] = (signed char)(-20); // Advance 32 bit pointer to the next field u32p++; @@ -98,9 +100,11 @@ static int ntp_reply(const int socket_fd, const struct sockaddr *saddr_p, const // | Root Dispersion | // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - /* zur Vereinfachung , Root Delay = 0, Root Dispersion = 0 */ - *u32p++ = 0; - *u32p++ = 0; + // Assume Root Delay (total roundtrip delay to the primary reference + // source) = 0, Root Dispersion (the nominal error relative to the + // primary reference source) = 0 as we don't have these numbers + *u32p++ = 0.0; + *u32p++ = 0.0; // 0 1 2 3 // 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 @@ -124,20 +128,12 @@ static int ntp_reply(const int socket_fd, const struct sockaddr *saddr_p, const // Time when the system clock was last set or corrected, in NTP // timestamp format. As this is not a stratum 1 server, we don't have // a hardware clock to set this value. -#ifdef MOCK_REFTIME - // Mock this timestamp with the current time of the server minus 1 - // minute. - const uint64_t ref_time = gettime64() - 60 * 1000000; - const uint64_t net_ref_time = hton64(ref_time); - memcpy(u32p, &net_ref_time, sizeof(uint64_t)); - u32p += 2; -#else // A stateless server copies T3 and T4 from the client packet to T1 and // T2 of the server packet and tacks on the transmit timestamp T3 before // sending it to the client. memcpy(u32p, &u32r[8], sizeof(uint64_t)); u32p += 2; -#endif + // 0 1 2 3 // 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ From 14ea246f9a6c1034227a35fab377fd7d09a30b25 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 2 Jun 2024 10:19:26 +0200 Subject: [PATCH 124/339] Always include timezone in human-readable timestamps Signed-off-by: DL6ER --- src/api/auth.c | 4 ++-- src/config/dnsmasq_config.c | 2 +- src/config/toml_writer.c | 2 +- src/gc.c | 2 +- src/log.c | 21 +++++++-------------- src/log.h | 3 +-- src/overTime.c | 4 ++-- src/procps.c | 2 +- src/zip/teleporter.c | 2 +- 9 files changed, 17 insertions(+), 25 deletions(-) diff --git a/src/api/auth.c b/src/api/auth.c index a4752d35..227b2906 100644 --- a/src/api/auth.c +++ b/src/api/auth.c @@ -269,7 +269,7 @@ int check_client_auth(struct ftl_conn *api, const bool is_api) if(config.debug.api.v.b) { char timestr[TIMESTR_SIZE]; - get_timestr(timestr, auth_data[user_id].valid_until, false, false, true); + get_timestr(timestr, auth_data[user_id].valid_until, false, false); log_debug(DEBUG_API, "Recognized known user: user_id %i, valid_until: %s, remote_addr %s (%s at login)", user_id, timestr, api->request->remote_addr, auth_data[user_id].remote_addr); } @@ -632,7 +632,7 @@ int api_auth(struct ftl_conn *api) if(config.debug.api.v.b && user_id > API_AUTH_UNAUTHORIZED) { char timestr[TIMESTR_SIZE]; - get_timestr(timestr, auth_data[user_id].valid_until, false, false, true); + get_timestr(timestr, auth_data[user_id].valid_until, false, false); log_debug(DEBUG_API, "API: Registered new user: user_id %i valid_until: %s remote_addr %s (accepted due to %s)", user_id, timestr, auth_data[user_id].remote_addr, empty_password ? "empty password" : "correct response"); diff --git a/src/config/dnsmasq_config.c b/src/config/dnsmasq_config.c index cdaf5092..8c2e2686 100644 --- a/src/config/dnsmasq_config.c +++ b/src/config/dnsmasq_config.c @@ -192,7 +192,7 @@ static void write_config_header(FILE *fp, const char *description) { const time_t now = time(NULL); char timestring[TIMESTR_SIZE]; - get_timestr(timestring, now, false, false, true); + get_timestr(timestring, now, false, false); fputs("# Pi-hole: A black hole for Internet advertisements\n", fp); fprintf(fp, "# (c) %u Pi-hole, LLC (https://pi-hole.net)\n", get_year(now)); fputs("# Network-wide ad blocking via your own hardware.\n", fp); diff --git a/src/config/toml_writer.c b/src/config/toml_writer.c index 5d995a82..638e8fd6 100644 --- a/src/config/toml_writer.c +++ b/src/config/toml_writer.c @@ -40,7 +40,7 @@ bool writeFTLtoml(const bool verbose) fputs("# Encoding: UTF-8\n", fp); fputs("# This file is managed by pihole-FTL\n", fp); char timestring[TIMESTR_SIZE]; - get_timestr(timestring, time(NULL), false, false, true); + get_timestr(timestring, time(NULL), false, false); fputs("# Last updated on ", fp); fputs(timestring, fp); fputs("\n\n", fp); diff --git a/src/gc.c b/src/gc.c index e656222b..a142ca23 100644 --- a/src/gc.c +++ b/src/gc.c @@ -299,7 +299,7 @@ void runGC(const time_t now, time_t *lastGCrun, const bool flush) { timer_start(GC_TIMER); char timestring[TIMESTR_SIZE]; - get_timestr(timestring, mintime, false, false, true); + get_timestr(timestring, mintime, false, false); log_debug(DEBUG_GC, "GC starting, mintime: %s (%lu), counters->queries = %d", timestring, (unsigned long)mintime, counters->queries); } diff --git a/src/log.c b/src/log.c index a12f9345..a10fe836 100644 --- a/src/log.c +++ b/src/log.c @@ -86,8 +86,7 @@ double double_time(void) } // Get a human-readable time string -void get_timestr(char timestring[TIMESTR_SIZE], const time_t timein, - const bool millis, const bool uri_compatible, const bool timezone) +void get_timestr(char timestring[TIMESTR_SIZE], const time_t timein, const bool millis, const bool uri_compatible) { struct tm tm; localtime_r(&timein, &tm); @@ -105,23 +104,17 @@ void get_timestr(char timestring[TIMESTR_SIZE], const time_t timein, gettimeofday(&tv, NULL); const int millisec = tv.tv_usec/1000; - sprintf(timestring,"%d-%02d-%02d%c%02d%c%02d%c%02d.%03i", + snprintf(timestring, TIMESTR_SIZE, "%d-%02d-%02d%c%02d%c%02d%c%02d.%03i%c%s", tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, space, - tm.tm_hour, colon, tm.tm_min, colon, tm.tm_sec, millisec); + tm.tm_hour, colon, tm.tm_min, colon, tm.tm_sec, millisec, space, tm.tm_zone); } else { - sprintf(timestring,"%d-%02d-%02d%c%02d%c%02d%c%02d", + snprintf(timestring, TIMESTR_SIZE, "%d-%02d-%02d%c%02d%c%02d%c%02d%c%s", tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, space, - tm.tm_hour, colon, tm.tm_min, colon, tm.tm_sec); + tm.tm_hour, colon, tm.tm_min, colon, tm.tm_sec, space, tm.tm_zone); } - // Append timezone if requested - if(timezone) - snprintf(timestring + strlen(timestring), - TIMESTR_SIZE - strlen(timestring), - "%c%s", space, tm.tm_zone); - // Ensure that the string is zero-terminated timestring[TIMESTR_SIZE - 1] = '\0'; } @@ -244,7 +237,7 @@ void __attribute__ ((format (printf, 3, 4))) _FTL_log(const int priority, const return; // Get human-readable time - get_timestr(timestring, time(NULL), true, false, false); + get_timestr(timestring, time(NULL), true, false); // Get and log PID of current process to avoid ambiguities when more than one // pihole-FTL instance is logging into the same file @@ -345,7 +338,7 @@ void __attribute__ ((format (printf, 1, 2))) log_web(const char *format, ...) add_to_fifo_buffer(FIFO_WEBSERVER, buffer, NULL, len > MAX_MSG_FIFO ? MAX_MSG_FIFO : len); // Get human-readable time - get_timestr(timestring, now, true, false, false); + get_timestr(timestring, now, true, false); // Get and log PID of current process to avoid ambiguities when more than one // pihole-FTL instance is logging into the same file diff --git a/src/log.h b/src/log.h index 4b9162c1..6e550ad9 100644 --- a/src/log.h +++ b/src/log.h @@ -51,8 +51,7 @@ unsigned int get_year(const time_t timein); const char *get_FTL_version(void); void log_FTL_version(bool crashreport); double double_time(void); -void get_timestr(char timestring[TIMESTR_SIZE], const time_t timein, - const bool millis, const bool uri_compatible, const bool timezone); +void get_timestr(char timestring[TIMESTR_SIZE], const time_t timein, const bool millis, const bool uri_compatible); const char *debugstr(const enum debug_flag flag) __attribute__((const)); void log_web(const char *format, ...) __attribute__ ((format (printf, 1, 2))); const char *get_ordinal_suffix(unsigned int number) __attribute__ ((const)); diff --git a/src/overTime.c b/src/overTime.c index b1e42386..a7ad1a7e 100644 --- a/src/overTime.c +++ b/src/overTime.c @@ -120,11 +120,11 @@ unsigned int _getOverTimeID(time_t timestamp, const char *file, const int line) if(!warned_about_hwclock) { char timestampStr[TIMESTR_SIZE]; - get_timestr(timestampStr, timestamp, false, false, true); + get_timestr(timestampStr, timestamp, false, false); const time_t lastTimestamp = overTime[OVERTIME_SLOTS-1].timestamp; char lastTimestampStr[TIMESTR_SIZE]; - get_timestr(lastTimestampStr, lastTimestamp, false, false, true); + get_timestr(lastTimestampStr, lastTimestamp, false, false); log_warn("Found database entries in the future (%s (%lu), last timestamp for importing: %s (%lu)). " "Your over-time statistics may be incorrect (found in %s:%d)", diff --git a/src/procps.c b/src/procps.c index 2aaaf3b9..8e3656a4 100644 --- a/src/procps.c +++ b/src/procps.c @@ -109,7 +109,7 @@ static bool get_process_creation_time(const pid_t pid, char timestr[TIMESTR_SIZE struct stat st; if(stat(filename, &st) < 0) return false; - get_timestr(timestr, st.st_ctim.tv_sec, false, false, true); + get_timestr(timestr, st.st_ctim.tv_sec, false, false); return true; } diff --git a/src/zip/teleporter.c b/src/zip/teleporter.c index a82e5855..47f2edc3 100644 --- a/src/zip/teleporter.c +++ b/src/zip/teleporter.c @@ -278,7 +278,7 @@ const char *generate_teleporter_zip(mz_zip_archive *zip, char filename[128], voi // Generate filename for ZIP archive (it has both the hostname and the // current datetime) char timestr[TIMESTR_SIZE]; - get_timestr(timestr, time(NULL), false, true, true); + get_timestr(timestr, time(NULL), false, true); snprintf(filename, 128, "pi-hole_%s_teleporter_%s.zip", hostname(), timestr); // Everything worked well From 58c59a0aef6160530095c38a80dc7e14854e89b0 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 2 Jun 2024 11:56:53 +0200 Subject: [PATCH 125/339] Skip certificate domain check when TLS is not actually used even if a certificate is available Signed-off-by: DL6ER --- src/webserver/webserver.c | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/webserver/webserver.c b/src/webserver/webserver.c index 76ca6197..33d3ffcc 100644 --- a/src/webserver/webserver.c +++ b/src/webserver/webserver.c @@ -421,7 +421,17 @@ void http_init(void) #ifdef HAVE_MBEDTLS // Add TLS options if configured - if(config.webserver.tls.cert.v.s != NULL && + + // TLS is used when webserver.port contains "s" (e.g. "443s") + const bool tls_used = config.webserver.port.v.s != NULL && + strchr(config.webserver.port.v.s, 's') != NULL; + + // Check certificate domain if + // - TLS is used + // - A certificate is configured + // - The certificate is readable + if(tls_used && + config.webserver.tls.cert.v.s != NULL && strlen(config.webserver.tls.cert.v.s) > 0) { // Try to generate certificate if not present @@ -439,6 +449,8 @@ void http_init(void) } } + // Check if the certificate is readable (we may have just + // created it) if(file_readable(config.webserver.tls.cert.v.s)) { if(read_certificate(config.webserver.tls.cert.v.s, config.webserver.domain.v.s, false) != CERT_DOMAIN_MATCH) From 3d2fd6d6a27b8b57e61a6026b1907fd95593a54f Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 2 Jun 2024 12:27:20 +0200 Subject: [PATCH 126/339] Add new misc.readOnly config option to force the configuration to be read-only. It can only be modified through the config file but neither the API nor the CLI as long as read-only mode is enabled Signed-off-by: DL6ER --- src/api/config.c | 20 ++++++++++++++++++++ src/api/docs/content/specs/config.yaml | 3 +++ src/config/cli.c | 17 +++++++++++++++++ src/config/config.c | 7 +++++++ src/config/config.h | 2 ++ src/config/toml_writer.c | 7 +++++++ test/pihole.toml | 8 +++++++- 7 files changed, 63 insertions(+), 1 deletion(-) diff --git a/src/api/config.c b/src/api/config.c index f413e502..4d530db4 100644 --- a/src/api/config.c +++ b/src/api/config.c @@ -670,6 +670,16 @@ static int api_config_patch(struct ftl_conn *api) NULL); } + // Return early if the user tries to change some settings but the config + // is in read-only mode + if(config.misc.readOnly.v.b) + { + return send_json_error(api, 403, + "forbidden", + "The config is currently in read-only mode", + NULL); + } + // Read all known config items bool config_changed = false; bool dnsmasq_changed = false; @@ -696,6 +706,16 @@ static int api_config_patch(struct ftl_conn *api) continue; } + if(new_item->f & FLAG_READ_ONLY && cJSON_IsBool(elem) && elem->valueint == 1) + { + char *key = strdup(new_item->k); + free_config(&newconf); + return send_json_error_free(api, 400, + "bad_request", + "This config option can only be set in pihole.toml, not via the API", + key, true); + } + // Check if this is a write-only config item with the placeholder value if(new_item->f & FLAG_WRITE_ONLY && cJSON_IsString(elem) && strcmp(elem->valuestring, PASSWORD_VALUE) == 0) diff --git a/src/api/docs/content/specs/config.yaml b/src/api/docs/content/specs/config.yaml index 08d5dd1b..07633ef2 100644 --- a/src/api/docs/content/specs/config.yaml +++ b/src/api/docs/content/specs/config.yaml @@ -481,6 +481,8 @@ components: type: string extraLogging: type: boolean + readOnly: + type: boolean check: type: object properties: @@ -726,6 +728,7 @@ components: etc_dnsmasq_d: false dnsmasq_lines: [ ] extraLogging: false + readOnly: false check: load: true shmem: 90 diff --git a/src/config/cli.c b/src/config/cli.c index 98dd765b..0bcc249d 100644 --- a/src/config/cli.c +++ b/src/config/cli.c @@ -397,6 +397,14 @@ int set_config_from_CLI(const char *key, const char *value) return EXIT_FAILURE; } + // Return early if the user tries to change some settings but the config + // is in read-only mode + if(config.misc.readOnly.v.b) + { + printf("Config is in read-only mode, changes are not allowed (misc.readOnly = true)\n"); + return EXIT_FAILURE; + } + // Identify config option struct config newconf; duplicate_config(&newconf, &config); @@ -410,6 +418,7 @@ int set_config_from_CLI(const char *key, const char *value) if(strcmp(item->k, key) != 0) continue; + // Check if this is a read-only config option (forced by env var) if(item->f & FLAG_ENV_VAR) { log_err("Config option %s is read-only (set via environmental variable)", key); @@ -417,6 +426,14 @@ int set_config_from_CLI(const char *key, const char *value) return ENV_VAR_FORCED; } + // Check if this the special read-only config option + if(item->f & FLAG_READ_ONLY) + { + log_err("Config option %s can only be set in pihole.toml, not via the CLI", key); + free_config(&newconf); + return EXIT_FAILURE; + } + // This is the config option we are looking for new_item = item; diff --git a/src/config/config.c b/src/config/config.c index b81cf769..5c11bb05 100644 --- a/src/config/config.c +++ b/src/config/config.c @@ -1238,6 +1238,13 @@ void initConfig(struct config *conf) conf->misc.extraLogging.d.b = false; conf->misc.extraLogging.c = validate_stub; // Only type-based checking + conf->misc.readOnly.k = "misc.readOnly"; + conf->misc.readOnly.h = "Put configuration into read-only mode. This will prevent any changes to the configuration file via the API or CLI. This setting useful when a configuration is to be forced/modified by some third-party application (like infrastructure-as-code providers) and should not be changed by any means."; + conf->misc.readOnly.t = CONF_BOOL; + conf->misc.readOnly.f = FLAG_READ_ONLY; + conf->misc.readOnly.d.b = false; + conf->misc.readOnly.c = validate_stub; // Only type-based checking + // sub-struct misc.check conf->misc.check.load.k = "misc.check.load"; conf->misc.check.load.h = "Pi-hole is very lightweight on resources. Nevertheless, this does not mean that you should run Pi-hole on a server that is otherwise extremely busy as queuing on the system can lead to unnecessary delays in DNS operation as the system becomes less and less usable as the system load increases because all resources are permanently in use. To account for this, FTL regularly checks the system load. To bring this to your attention, FTL warns about excessive load when the 15 minute system load average exceeds the number of cores.\n This check can be disabled with this setting."; diff --git a/src/config/config.h b/src/config/config.h index 884f1faa..9c06492a 100644 --- a/src/config/config.h +++ b/src/config/config.h @@ -96,6 +96,7 @@ enum conf_type { #define FLAG_WRITE_ONLY (1 << 4) #define FLAG_ENV_VAR (1 << 5) #define FLAG_CONF_IMPORTED (1 << 6) +#define FLAG_READ_ONLY (1 << 7) struct conf_item { const char *k; // item Key @@ -274,6 +275,7 @@ struct config { struct conf_item etc_dnsmasq_d; struct conf_item dnsmasq_lines; struct conf_item extraLogging; + struct conf_item readOnly; struct { struct conf_item load; struct conf_item shmem; diff --git a/src/config/toml_writer.c b/src/config/toml_writer.c index cc5fc07d..34d8fa70 100644 --- a/src/config/toml_writer.c +++ b/src/config/toml_writer.c @@ -27,6 +27,13 @@ extern uint8_t last_checksum[SHA256_DIGEST_SIZE]; bool writeFTLtoml(const bool verbose) { + // Return early without writing if we are in config read-only mode + if(config.misc.readOnly.v.b) + { + log_debug(DEBUG_CONFIG, "Config file is read-only, not writing"); + return true; + } + // Try to open a temporary config file for writing FILE *fp; if((fp = openFTLtoml("w", 0)) == NULL) diff --git a/test/pihole.toml b/test/pihole.toml index e1d72c8b..d4d2fdf0 100644 --- a/test/pihole.toml +++ b/test/pihole.toml @@ -893,6 +893,12 @@ # debugging and is not recommended for normal use. extraLogging = false + # Put configuration into read-only mode. This will prevent any changes to the + # configuration file via the API or CLI. This setting useful when a configuration is + # to be forced/modified by some third-party application (like infrastructure-as-code + # providers) and should not be changed by any means. + readOnly = false + [misc.check] # Pi-hole is very lightweight on resources. Nevertheless, this does not mean that you # should run Pi-hole on a server that is otherwise extremely busy as queuing on the @@ -1039,7 +1045,7 @@ all = true ### CHANGED, default = false # Configuration statistics: -# 136 total entries out of which 82 entries are default +# 137 total entries out of which 83 entries are default # --> 54 entries are modified # 2 entries are forced through environment: # - misc.nice From d3439593a1851a91030ac3f0d3e543d78d6a5546 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 2 Jun 2024 19:55:28 +0200 Subject: [PATCH 127/339] Default to resolve internal PTRs using UDP, fall back to TCP for individual queries on UDP truncation (commonly seen in conjunction with DNSSEC) Signed-off-by: DL6ER --- src/args.c | 2 +- src/database/network-table.c | 21 +++++---- src/resolve.c | 89 ++++++++++++++---------------------- src/resolve.h | 4 +- 4 files changed, 50 insertions(+), 66 deletions(-) diff --git a/src/args.c b/src/args.c index 7abe42f5..d643ebc1 100644 --- a/src/args.c +++ b/src/args.c @@ -513,7 +513,7 @@ void parse_args(int argc, char* argv[]) // Create a socket struct sockaddr_in dest; const int sock = create_socket(tcp, &dest); - char *name = resolveHostname(sock, &dest, tcp, argv[2], true); + char *name = resolveHostname(sock, tcp, &dest, argv[2], true, NULL); // Close the socket close(sock); diff --git a/src/database/network-table.c b/src/database/network-table.c index 19a8e099..9592b627 100644 --- a/src/database/network-table.c +++ b/src/database/network-table.c @@ -2009,7 +2009,7 @@ char *__attribute__((malloc)) getNameFromIP(sqlite3 *db, const char *ipaddr) // Check if we want to resolve host names if(!resolve_this_name(ipaddr)) { - log_debug(DEBUG_DATABASE, "getNameFromIP(\"%s\") - configured to not resolve host name", ipaddr); + log_debug(DEBUG_RESOLVER, "getNameFromIP(\"%s\") - configured to not resolve host name", ipaddr); return NULL; } @@ -2056,6 +2056,8 @@ char *__attribute__((malloc)) getNameFromIP(sqlite3 *db, const char *ipaddr) return NULL; } + log_debug(DEBUG_RESOLVER, "Check for a host name associated with IP address %s", ipaddr); + char *name = NULL; rc = sqlite3_step(stmt); if(rc == SQLITE_ROW) @@ -2063,7 +2065,7 @@ char *__attribute__((malloc)) getNameFromIP(sqlite3 *db, const char *ipaddr) // Database record found (result might be empty) name = strdup((char*)sqlite3_column_text(stmt, 0)); - log_debug(DEBUG_DATABASE, "Found database host name (same address) %s -> %s", ipaddr, name); + log_debug(DEBUG_RESOLVER, "Found database host name (same address) %s -> %s", ipaddr, name); } else if(rc != SQLITE_DONE) { @@ -2084,6 +2086,8 @@ char *__attribute__((malloc)) getNameFromIP(sqlite3 *db, const char *ipaddr) return name; } + log_debug(DEBUG_RESOLVER, " ---> not found"); + // Nothing found for the exact IP address // Check for a host name associated with the same device (but another IP address) querystr = "SELECT name FROM network_addresses " @@ -2114,6 +2118,8 @@ char *__attribute__((malloc)) getNameFromIP(sqlite3 *db, const char *ipaddr) return NULL; } + log_debug(DEBUG_RESOLVER, "Checking for a host name associated with the same device (but another IP address)"); + rc = sqlite3_step(stmt); if(rc == SQLITE_ROW) { @@ -2153,8 +2159,6 @@ char *__attribute__((malloc)) getNameFromMAC(const char *client) if(FTLDBerror()) return NULL; - log_debug(DEBUG_DATABASE,"Looking up host name for %s", client); - // Open pihole-FTL.db database file sqlite3 *db = NULL; if((db = dbopen(false, false)) == NULL) @@ -2192,6 +2196,8 @@ char *__attribute__((malloc)) getNameFromMAC(const char *client) return NULL; } + log_debug(DEBUG_RESOLVER, "Check for a host name associated with MAC address %s", client); + char *name = NULL; rc = sqlite3_step(stmt); if(rc == SQLITE_ROW) @@ -2262,11 +2268,8 @@ char *__attribute__((malloc)) getIfaceFromIP(sqlite3 *db, const char *ipaddr) return NULL; } - if(config.debug.resolver.v.b) - { - log_debug(DEBUG_RESOLVER, "getIfaceFromIP(): \"%s\" with ? = \"%s\"", - querystr, ipaddr); - } + log_debug(DEBUG_DATABASE, "getIfaceFromIP(): \"%s\" with ? = \"%s\"", + querystr, ipaddr); // Bind ipaddr to prepared statement if((rc = sqlite3_bind_text(stmt, 1, ipaddr, -1, SQLITE_STATIC)) != SQLITE_OK) diff --git a/src/resolve.c b/src/resolve.c index e19543bb..f3a66134 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -261,7 +261,8 @@ int create_socket(bool tcp, struct sockaddr_in *dest) } // Perform a name lookup by sending a packet to ourselves -static char *__attribute__((malloc)) ngethostbyname(const int sock, struct sockaddr_in *dest, const bool tcp, const char *host, const char *ipaddr) +static char *__attribute__((malloc)) ngethostbyname(const int sock, const bool tcp, struct sockaddr_in *dest, + const char *host, const char *ipaddr, bool *truncated) { uint8_t buf[4096] = { 0 }; // buffer for DNS query uint8_t *qname = NULL, *reader = NULL; @@ -385,8 +386,9 @@ static char *__attribute__((malloc)) ngethostbyname(const int sock, struct socka // Abort if the query was not successful if(dns->tc != 0) { - log_debug(DEBUG_RESOLVER, "Internal name lookup for %s was unsuccessful: DNS response was truncated", - ipaddr); + log_debug(DEBUG_RESOLVER, " --> DNS response truncated"); + if(truncated != NULL) + *truncated = true; return NULL; } @@ -556,8 +558,8 @@ static void __attribute__((nonnull(1,3))) name_toDNS(unsigned char *dns, const s *dns++='\0'; } -char *__attribute__((malloc)) resolveHostname(const int sock, struct sockaddr_in *dest, - const bool tcp, const char *addr, const bool force) +char *__attribute__((malloc)) resolveHostname(const int sock, const bool tcp, struct sockaddr_in *dest, + const char *addr, const bool force, bool *truncated) { // Get host name char *hostn = NULL; @@ -680,11 +682,17 @@ char *__attribute__((malloc)) resolveHostname(const int sock, struct sockaddr_in // Get host name by making a reverse lookup to ourselves (server at 127.0.0.1 with port 53) // We implement a minimalistic resolver here as we cannot rely on the system resolver using whatever // nameserver we configured in /etc/resolv.conf - return ngethostbyname(sock, dest, tcp, inaddr, addr); + hostn = ngethostbyname(sock, tcp, dest, inaddr, addr, truncated); + + // Free allocated memory + free(inaddr); + + // Return obtained host name + return hostn; } // Resolve upstream destination host names -static size_t resolveAndAddHostname(const int sock, struct sockaddr_in *dest, const bool tcp, +static size_t resolveAndAddHostname(const int udp_sock, struct sockaddr_in *dest, size_t ippos, size_t oldnamepos, bool *success) { // Get IP and host name strings. They are cloned in case shared memory is @@ -710,12 +718,21 @@ static size_t resolveAndAddHostname(const int sock, struct sockaddr_in *dest, co // Important: Don't hold a lock while resolving as the main thread // (dnsmasq) needs to be operable during the call to resolveHostname() - char *newname = resolveHostname(sock, dest, tcp, ipaddr, false); + bool truncated = false; + char *newname = resolveHostname(udp_sock, false, dest, ipaddr, false, &truncated); + if(newname == NULL && truncated) + { + // Retry with TCP if UDP failed due to truncation (RFC 7766) + const int tcp_sock = create_socket(true, dest); + newname = resolveHostname(tcp_sock, true, dest, ipaddr, false, NULL); + close(tcp_sock); + } + if(newname == NULL) { // We could not resolve the hostname, so we keep the old one // and mark the entry as not new - log_debug(DEBUG_RESOLVER, " ---> \"%s\" (failed to resolve)", oldname); + log_debug(DEBUG_RESOLVER, " ---> \"%s\" (failed to resolve via UDP, too)", oldname); // Free allocated memory *success = false; @@ -776,17 +793,15 @@ static void resolveClients(const bool onlynew, const bool force_refreshing) unlock_shm(); // Create DNS client socket - const bool tcp = true; struct sockaddr_in dest = { 0 }; - int sock = create_socket(tcp, &dest); - if(sock < 0) + const int udp_sock = create_socket(false, &dest); + if(udp_sock < 0) { log_err("Unable to create DNS resolver socket, client host name resolution failed"); return; } int skipped = 0; - unsigned int queries = 0u; for(int clientID = 0; clientID < clientscount; clientID++) { // Memory access needs to get locked @@ -874,25 +889,9 @@ static void resolveClients(const bool onlynew, const bool force_refreshing) continue; } - // We need to reconnect after a certain number of queries due to - // dnsmasq-internal limits - if(tcp && ++queries > TCP_MAX_QUERIES - 1) - { - close(sock); - sock = create_socket(tcp, &dest); - if(sock < 0) - { - log_err("Unable to recreate to DNS resolver socket, client host name resolution failed"); - return; - } - - // Reset query counter - queries = 0; - } - // Obtain/update hostname of this client bool success = true; - size_t newnamepos = resolveAndAddHostname(sock, &dest, tcp, ippos, oldnamepos, &success); + size_t newnamepos = resolveAndAddHostname(udp_sock, &dest, ippos, oldnamepos, &success); lock_shm(); // Get client pointer for the second time (writing data) @@ -933,7 +932,7 @@ static void resolveClients(const bool onlynew, const bool force_refreshing) } // Close socket - close(sock); + close(udp_sock); log_debug(DEBUG_RESOLVER, "%i / %i client host names resolved", clientscount-skipped, clientscount); @@ -949,17 +948,15 @@ static void resolveUpstreams(const bool onlynew) unlock_shm(); // Create socket - const bool tcp = true; struct sockaddr_in dest = { 0 }; - int sock = create_socket(tcp, &dest); - if(sock < 0) + const int udp_sock = create_socket(false, &dest); + if(udp_sock < 0) { - log_err("Unable to create DNS resolver socket, upstream host name resolution failed"); + log_err("Unable to create DNS resolver socket, client host name resolution failed"); return; } int skipped = 0; - unsigned int queries = 0u; for(int upstreamID = 0; upstreamID < upstreams; upstreamID++) { // Memory access needs to get locked @@ -1004,25 +1001,9 @@ static void resolveUpstreams(const bool onlynew) continue; } - // We need to reconnect after a certain number of queries due to - // dnsmasq-internal limits - if(tcp && ++queries > TCP_MAX_QUERIES - 1) - { - close(sock); - sock = create_socket(tcp, &dest); - if(sock < 0) - { - log_err("Unable to recreate to DNS resolver socket, client host name resolution failed"); - return; - } - - // Reset query counter - queries = 0; - } - // Obtain/update hostname of this client bool success = true; - size_t newnamepos = resolveAndAddHostname(sock, &dest, tcp, ippos, oldnamepos, &success); + size_t newnamepos = resolveAndAddHostname(udp_sock, &dest, ippos, oldnamepos, &success); lock_shm(); // Get upstream pointer for the second time (writing data) @@ -1062,7 +1043,7 @@ static void resolveUpstreams(const bool onlynew) } // Close socket - close(sock); + close(udp_sock); log_debug(DEBUG_RESOLVER, "%i / %i upstream server host names resolved", upstreams-skipped, upstreams); diff --git a/src/resolve.h b/src/resolve.h index 64b972aa..d80654b7 100644 --- a/src/resolve.h +++ b/src/resolve.h @@ -12,8 +12,8 @@ void *DNSclient_thread(void *val); int create_socket(bool tcp, struct sockaddr_in *dest); -char *resolveHostname(const int sock, struct sockaddr_in *dest, const bool tcp, - const char *addr, const bool force) __attribute__((malloc)); +char *resolveHostname(const int sock, const bool tcp, struct sockaddr_in *dest, + const char *addr, const bool force, bool *truncated) __attribute__((malloc)); bool resolve_names(void) __attribute__((pure)); bool resolve_this_name(const char *ipaddr) __attribute__((pure)); From 5b9df0237248dc5cd891ef724787e8e2a220a85f Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 2 Jun 2024 20:07:41 +0200 Subject: [PATCH 128/339] Add missing newlines in dnsmasq config Signed-off-by: DL6ER --- src/config/dnsmasq_config.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/config/dnsmasq_config.c b/src/config/dnsmasq_config.c index 86b9952f..5f3244bf 100644 --- a/src/config/dnsmasq_config.c +++ b/src/config/dnsmasq_config.c @@ -588,14 +588,14 @@ bool __attribute__((const)) write_dnsmasq_config(struct config *conf, bool test_ fputs("# Add NTP server to DHCP\n", pihole_conf); // The special address 0.0.0.0 is taken to mean "the // address of the machine running the DHCP server" - fputs("dhcp-option=option:ntp-server,0.0.0.0", pihole_conf); + fputs("dhcp-option=option:ntp-server,0.0.0.0\n\n", pihole_conf); } - + // Add option to ignore unknown clients if enabled if(conf->dhcp.ignoreUnknownClients.v.b) { fputs("# Ignore clients not configured below\n", pihole_conf); - fputs("dhcp-ignore=tag:!known\n", pihole_conf); + fputs("dhcp-ignore=tag:!known\n\n", pihole_conf); } // Add per-host parameters From f2c14cce253bb4c6d03ec3623a5e9a1739b830a5 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 3 Jun 2024 09:48:13 +0200 Subject: [PATCH 129/339] Fix incorrect unification of regex warnings Signed-off-by: DL6ER --- src/database/message-table.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/database/message-table.c b/src/database/message-table.c index f3cc4103..9271bd96 100644 --- a/src/database/message-table.c +++ b/src/database/message-table.c @@ -137,9 +137,9 @@ static enum message_type get_message_type_from_string(const char *typestr) static unsigned char message_blob_types[MAX_MESSAGE][5] = { - { // REGEX_MESSAGE: The message column contains the regex warning text + { // REGEX_MESSAGE: The message column contains the regex text (the erroring regex filter itself) SQLITE_TEXT, // regex type ("deny", "allow") - SQLITE_TEXT, // regex text (the erroring regex filter itself) + SQLITE_TEXT, // regex warning text SQLITE_INTEGER, // database index of regex (so the dashboard can show a link) SQLITE_NULL, // not used SQLITE_NULL // not used @@ -993,9 +993,9 @@ bool format_messages(cJSON *array) { case REGEX_MESSAGE: { - const char *warning = (const char*)sqlite3_column_text(stmt, 3); + const char *regex = (const char*)sqlite3_column_text(stmt, 3); const char *type = (const char*)sqlite3_column_text(stmt, 4); - const char *regex = (const char*)sqlite3_column_text(stmt, 5); + const char *warning = (const char*)sqlite3_column_text(stmt, 5); const int dbindex = sqlite3_column_int(stmt, 6); format_regex_message(plain, sizeof(plain), html, sizeof(html), @@ -1206,7 +1206,7 @@ void logg_regex_warning(const char *type, const char *warning, const int dbindex return; // Add to database - const int rowid = add_message(REGEX_MESSAGE, warning, type, regex, dbindex); + const int rowid = add_message(REGEX_MESSAGE, regex, type, warning, dbindex); if(rowid == -1) log_err("logg_regex_warning(): Failed to add message to database"); } From 189979284f2910f748282261263107e223efbc00 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 3 Jun 2024 11:46:39 +0200 Subject: [PATCH 130/339] Store correct database ID when issuing a warning Signed-off-by: DL6ER --- src/regex.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/regex.c b/src/regex.c index 0d14d07b..ec80032f 100644 --- a/src/regex.c +++ b/src/regex.c @@ -788,7 +788,7 @@ static void read_regex_table(const enum regex_type regexid) if(!compile_regex(regex_string, ®ex[index], &message) && message != NULL) { logg_regex_warning(regextype[regexid], message, - regex->database_id, regex_string); + rowid, regex_string); free(message); } From af0468eb365f4cc5bb1cd83aaaf13979e07e2080 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 3 Jun 2024 13:29:06 +0200 Subject: [PATCH 131/339] Fix very long DNS names (>64 bytes) potentially crashing the internal name resolving mechanism, the new limit is 256 bytes with proper boundary checking Signed-off-by: DL6ER --- src/resolve.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/resolve.c b/src/resolve.c index e19543bb..457db27c 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -449,7 +449,8 @@ static char *__attribute__((malloc)) ngethostbyname(const int sock, struct socka // 3www6google3com -> www.google.com static u_char * __attribute__((malloc)) __attribute__((nonnull(1,2,3))) name_fromDNS(unsigned char *reader, unsigned char *buffer, uint16_t *count) { - unsigned char *name = calloc(MAXHOSTNAMELEN, sizeof(char)); + const size_t MAXNAMELEN = 256; + unsigned char *name = calloc(MAXNAMELEN, sizeof(char)); unsigned int p = 0, jumped = 0; // Initialize count @@ -462,7 +463,7 @@ static u_char * __attribute__((malloc)) __attribute__((nonnull(1,2,3))) name_fro // Instead, each label is preceded by a byte containing its length, and // the name is terminated by a zero-length label representing the root // zone. - while(*reader != 0) + while(*reader != 0 && p < MAXNAMELEN - 2) { if(*reader >= 0xC0) { From 7c0d7e87e844089c8ae105c55899450e39aaeec9 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 4 Jun 2024 07:55:06 +0200 Subject: [PATCH 132/339] Add debug.ntp flag Signed-off-by: DL6ER --- src/api/docs/content/specs/config.yaml | 3 + src/args.c | 1 + src/config/config.c | 7 ++ src/config/config.h | 1 + src/enums.h | 1 + src/log.c | 2 + src/ntp/client.c | 90 +++++++++++++++++--------- src/ntp/ntp.h | 7 ++ src/ntp/server.c | 55 ++++++++++++---- test/pihole.toml | 7 +- 10 files changed, 128 insertions(+), 46 deletions(-) diff --git a/src/api/docs/content/specs/config.yaml b/src/api/docs/content/specs/config.yaml index babf5f4c..c1b4edfa 100644 --- a/src/api/docs/content/specs/config.yaml +++ b/src/api/docs/content/specs/config.yaml @@ -566,6 +566,8 @@ components: type: boolean reserved: type: boolean + ntp: + type: boolean all: type: boolean topics: @@ -784,6 +786,7 @@ components: webserver: false extra: false reserved: false + ntp: false all: false config_one: summary: One option diff --git a/src/args.c b/src/args.c index 66ebcd37..622c894b 100644 --- a/src/args.c +++ b/src/args.c @@ -313,6 +313,7 @@ void parse_args(int argc, char* argv[]) // Enable stdout printing cli_mode = true; log_ctrl(false, true); + readFTLconf(&config, false); const bool update = (argc > 2 && strcmp(argv[2], "--update") == 0) || (argc > 3 && strcmp(argv[3], "--update") == 0); const char *server = "127.0.0.1"; diff --git a/src/config/config.c b/src/config/config.c index 72aac826..a7f29c02 100644 --- a/src/config/config.c +++ b/src/config/config.c @@ -1480,6 +1480,13 @@ void initConfig(struct config *conf) conf->debug.reserved.d.b = false; conf->debug.reserved.c = validate_stub; // Only type-based checking + conf->debug.ntp.k = "debug.ntp"; + conf->debug.ntp.h = "Print information about NTP synchronization"; + conf->debug.ntp.t = CONF_BOOL; + conf->debug.ntp.f = FLAG_ADVANCED_SETTING; + conf->debug.ntp.d.b = false; + conf->debug.ntp.c = validate_stub; // Only type-based checking + conf->debug.all.k = "debug.all"; conf->debug.all.h = "Set all debug flags at once. This is a convenience option to enable all debug flags at once. Note that this option is not persistent, setting it to true will enable all *remaining* debug flags but unsetting it will disable *all* debug flags."; conf->debug.all.t = CONF_ALL_DEBUG_BOOL; diff --git a/src/config/config.h b/src/config/config.h index 7ce87d1e..6c000146 100644 --- a/src/config/config.h +++ b/src/config/config.h @@ -323,6 +323,7 @@ struct config { struct conf_item webserver; struct conf_item extra; struct conf_item reserved; + struct conf_item ntp; // all must be the last item in this struct struct conf_item all; } debug; diff --git a/src/enums.h b/src/enums.h index 09769a9c..67dd1fbc 100644 --- a/src/enums.h +++ b/src/enums.h @@ -162,6 +162,7 @@ enum debug_flag { DEBUG_WEBSERVER, DEBUG_EXTRA, DEBUG_RESERVED, + DEBUG_NTP, DEBUG_MAX } __attribute__ ((packed)); diff --git a/src/log.c b/src/log.c index a10fe836..0fbbe447 100644 --- a/src/log.c +++ b/src/log.c @@ -219,6 +219,8 @@ const char *debugstr(const enum debug_flag flag) return "DEBUG_WEBSERVER"; case DEBUG_RESERVED: return "DEBUG_RESERVED"; + case DEBUG_NTP: + return "DEBUG_NTP"; case DEBUG_MAX: return "DEBUG_MAX"; case DEBUG_NONE: // fall through diff --git a/src/ntp/client.c b/src/ntp/client.c index 41089c0f..0a1fd4d5 100644 --- a/src/ntp/client.c +++ b/src/ntp/client.c @@ -8,7 +8,7 @@ * This file is copyright under the latest version of the EUPL. * Please see LICENSE file for your rights under this license. */ -#include "FTL.h" +#include "ntp/ntp.h" // close() #include // clock_gettime() @@ -27,9 +27,8 @@ #include // PRIi64 #include - -#include "ntp/ntp.h" -#include "log.h" +// config struct +#include "config/config.h" struct ntp_sync { @@ -65,7 +64,7 @@ static bool request(int fd, struct ntp_sync *ntp) // Send request if(send(fd, buf, 48, 0) != 48) { - printf("Failed to send data to NTP server: %s\n", strerror(errno)); + log_err("Failed to send data to NTP server: %s", strerror(errno)); return false; } @@ -75,19 +74,44 @@ static bool request(int fd, struct ntp_sync *ntp) // Display NTP time in human-readable format // This function is similar to get_timestr() in src/log.c but differs in that it // includes microseconds whereas get_timestr() only includes milliseconds -static void display_time(const char *description, const uint64_t ntp_time) +static void format_NTP_time(char time_str[TIMESTR_SIZE], const uint64_t ntp_time) { - char client_time_str[128]; struct timeval client_time; client_time.tv_sec = NTPtoSEC(ntp_time); client_time.tv_usec = NTPtoUSEC(ntp_time); struct tm *client_tm = localtime(&client_time.tv_sec); - snprintf(client_time_str, sizeof(client_time_str), "%04i-%02i-%02i %02i:%02i:%02i.%06"PRIi64" %s", - client_tm->tm_year + 1900, client_tm->tm_mon + 1, client_tm->tm_mday, - client_tm->tm_hour, client_tm->tm_min, client_tm->tm_sec, client_time.tv_usec, - client_tm->tm_zone); - client_time_str[sizeof(client_time_str) - 1] = '\0'; - printf("%s: %s\n", description, client_time_str); + snprintf(time_str, TIMESTR_SIZE, "%04i-%02i-%02i %02i:%02i:%02i.%06"PRIi64" %s", + client_tm->tm_year + 1900, client_tm->tm_mon + 1, client_tm->tm_mday, + client_tm->tm_hour, client_tm->tm_min, client_tm->tm_sec, client_time.tv_usec, + client_tm->tm_zone); + time_str[TIMESTR_SIZE - 1] = '\0'; +} + +// Print NTP timestamp in human-readable form for debugging +void print_debug_time(const char *label, const uint32_t *u32p, const uint64_t ntp_time) +{ + // Get the time from the appropriate buffer + uint64_t timevar; + if(u32p != NULL) + { + memcpy(&timevar, u32p, sizeof(uint64_t)); + // Convert to host byte order + timevar = ntoh64(timevar); + } + else + { + // Use the provided time (already in host byte order) + timevar = ntp_time; + } + + + // Format the time + char time_str[TIMESTR_SIZE]; + format_NTP_time(time_str, timevar); + + // Print the time + log_debug(DEBUG_NTP, "%s: %08"PRIx64".%08"PRIx64" = %s", label, + (timevar >> 32) & 0xFFFFFFFF, timevar & 0xFFFFFFFF, time_str); } static bool reply(int fd, struct ntp_sync *ntp, const bool verbose) @@ -98,7 +122,7 @@ static bool reply(int fd, struct ntp_sync *ntp, const bool verbose) // Receive reply if(recv(fd, buf, 48, 0) < 48) { - printf("Failed to receive data from NTP server: %s\n", strerror(errno)); + log_err("Failed to receive data from NTP server: %s", strerror(errno)); return false; } @@ -108,7 +132,7 @@ static bool reply(int fd, struct ntp_sync *ntp, const bool verbose) { // Accepted limits are 2^-32 (~ 0.2 nanoseconds) // to 2^0 (= 1 second) - printf("Received NTP reply has invalid precision: 2^(%i), assuming microsecond accuracy\n", rho); + log_warn("Received NTP reply has invalid precision: 2^(%i), assuming microsecond accuracy", rho); rho = -19; } // Compute precision of server clock in seconds 2^rho @@ -135,14 +159,14 @@ static bool reply(int fd, struct ntp_sync *ntp, const bool verbose) // network byte order if(ntp->org != org) { - printf("Received NTP reply does not match request (request %"PRIx64", reply %"PRIx64")\n", ntp->org, org); + log_warn("Received NTP reply does not match request (request %"PRIx64", reply %"PRIx64")", ntp->org, org); return false; } // Check stratum, mode, version, etc. if((buf[0] & 0x07) != 4) { - printf("Received NTP reply has invalid version\n"); + log_warn("Received NTP reply has invalid version"); return false; } @@ -176,18 +200,18 @@ static bool reply(int fd, struct ntp_sync *ntp, const bool verbose) ntp->delta = 0; # // Return early if not verbose - if(!verbose) + if(!config.debug.ntp.v.b) return true; // Print current time at client - display_time("Current time at client", dst); + print_debug_time("Current time at client", NULL, dst); // Print current time at server - display_time("Current time at server", ntp->xmt); + print_debug_time("Current time at server", NULL, ntp->xmt); // Print offset and delay - printf("Time offset: %e s\n", ntp->theta); - printf("Round-trip delay: %e s\n", ntp->delta); + log_debug(DEBUG_NTP, "Time offset: %e s", ntp->theta); + log_debug(DEBUG_NTP, "Round-trip delay: %e s", ntp->delta); return true; } @@ -200,7 +224,7 @@ bool ntp_client(const char *server, const bool settime) const int s = socket(protocol, SOCK_DGRAM, IPPROTO_UDP); if(s == -1) { - printf("ERROR: Cannot create UDP socket\n"); + log_err("Cannot create UDP socket\n"); return false; } @@ -210,7 +234,7 @@ bool ntp_client(const char *server, const bool settime) tv.tv_usec = 0; if(setsockopt(s, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv)) != 0) { - printf("ERROR: Cannot set socket timeout\n"); + log_err("Cannot set socket timeout\n"); close(s); return false; } @@ -219,7 +243,7 @@ bool ntp_client(const char *server, const bool settime) struct addrinfo *saddr; if(getaddrinfo(server, "123", NULL, &saddr) != 0) { - printf("ERROR: Cannot resolve NTP server address\n"); + log_err("Cannot resolve NTP server address\n"); close(s); return false; } @@ -227,7 +251,7 @@ bool ntp_client(const char *server, const bool settime) // Set address to send to/receive from if(connect(s, saddr->ai_addr, saddr->ai_addrlen) != 0) { - printf("ERROR: Cannot connect to NTP server\n"); + log_err("Cannot connect to NTP server\n"); close(s); return false; } @@ -275,10 +299,10 @@ bool ntp_client(const char *server, const bool settime) if(valid == 0) { - printf("No valid NTP replies received, check server and network connectivity\n\n"); + log_err("No valid NTP replies received, check server and network connectivity\n"); return false; } - printf("Received %u/%d valid NTP replies\n\n", valid, NTP_AVERGAGE_COUNT); + log_info("Received %u/%d valid NTP replies\n", valid, NTP_AVERGAGE_COUNT); theta_avg /= valid; delta_avg /= valid; @@ -295,8 +319,8 @@ bool ntp_client(const char *server, const bool settime) theta_stdev = sqrt(theta_stdev / valid); delta_stdev = sqrt(delta_stdev / valid); - printf("Average time offset: (%e +/- %e s)\n", theta_avg, theta_stdev); - printf("Average round-trip delay: (%e +/- %e s)\n", delta_avg, delta_stdev); + log_info("Average time offset: (%e +/- %e s)", theta_avg, theta_stdev); + log_info("Average round-trip delay: (%e +/- %e s)", delta_avg, delta_stdev); // Set time if requested if(settime) @@ -315,12 +339,14 @@ bool ntp_client(const char *server, const bool settime) unix_time.tv_usec = NTPtoUSEC(ntp_time); // Print new time - display_time("Setting local time to", ntp_time); + char time_str[TIMESTR_SIZE]; + format_NTP_time(time_str, ntp_time); + log_info("Setting local time to: %s", time_str); // Set time if(settimeofday(&unix_time, NULL) != 0) { - printf("Failed to set time: %s\n", + log_err("Failed to set time: %s", errno == EPERM ? "Insufficient permissions, try running with sudo" : strerror(errno)); return false; } diff --git a/src/ntp/ntp.h b/src/ntp/ntp.h index cc82c9b9..caca5169 100644 --- a/src/ntp/ntp.h +++ b/src/ntp/ntp.h @@ -11,6 +11,10 @@ #ifndef NTP_H #define NTP_H +#include "FTL.h" +// TIMESTR_SIZE +#include "log.h" + // uint64_t #include // bool @@ -19,6 +23,9 @@ // Get current time in NTP (64bit) format uint64_t gettime64(void); +// Print NTP timestamp in human-readable form +void print_debug_time(const char *label, const uint32_t *u32p, const uint64_t ntp_time); + // Start NTP server bool ntp_server_start(void); diff --git a/src/ntp/server.c b/src/ntp/server.c index e1d593a5..f6e70937 100644 --- a/src/ntp/server.c +++ b/src/ntp/server.c @@ -8,7 +8,7 @@ * This file is copyright under the latest version of the EUPL. * Please see LICENSE file for your rights under this license. */ -#include "FTL.h" +#include "ntp/ntp.h" // exit(0) #include // memcpy() @@ -33,10 +33,10 @@ #include // PR_SET_NAME #include - -#include "ntp/ntp.h" -#include "log.h" +// config struct #include "config/config.h" +// PRIi64 +#include // RFC 5905 Appendix A.4: Kernel System Clock Interface uint64_t gettime64(void) @@ -47,8 +47,8 @@ uint64_t gettime64(void) } // Create and send an NTP reply to the client -static int ntp_reply(const int socket_fd, const struct sockaddr *saddr_p, const socklen_t saddrlen, - const unsigned char recv_buf[], const uint64_t *recv_time) +static bool ntp_reply(const int socket_fd, const struct sockaddr *saddr_p, const socklen_t saddrlen, + const unsigned char recv_buf[], const uint64_t *recv_time) { // Buffer for the response unsigned char send_buf[48]; @@ -69,7 +69,7 @@ static int ntp_reply(const int socket_fd, const struct sockaddr *saddr_p, const // Check if the first byte is valid: mode is expected to be 3 ("client") if ((recv_buf[0] & 0x07) != 0x3) { log_warn("Received invalid NTP request: not from an NTP client, ignoring"); - return 1; + return false; } // set LI = 0 (no warning about leap seconds), set version-number to @@ -132,6 +132,8 @@ static int ntp_reply(const int socket_fd, const struct sockaddr *saddr_p, const // T2 of the server packet and tacks on the transmit timestamp T3 before // sending it to the client. memcpy(u32p, &u32r[8], sizeof(uint64_t)); + if(config.debug.ntp.v.b) + print_debug_time("Reference Timestamp", u32p, 0); u32p += 2; // 0 1 2 3 @@ -145,6 +147,8 @@ static int ntp_reply(const int socket_fd, const struct sockaddr *saddr_p, const // Time at the client when the request departed for the server, in NTP // timestamp format. (this is the client's transmit time) memcpy(u32p, &u32r[10], sizeof(uint64_t)); + if(config.debug.ntp.v.b) + print_debug_time("Origin Timestamp", u32p, 0); u32p += 2; // 0 1 2 3 @@ -159,6 +163,8 @@ static int ntp_reply(const int socket_fd, const struct sockaddr *saddr_p, const // timestamp format. (this is the server's receive time) const uint64_t net_recv_time = hton64(*recv_time); memcpy(u32p, &net_recv_time, sizeof(uint64_t)); + if(config.debug.ntp.v.b) + print_debug_time("Receive Timestamp", u32p, 0); u32p += 2; // 0 1 2 3 @@ -174,7 +180,9 @@ static int ntp_reply(const int socket_fd, const struct sockaddr *saddr_p, const const uint64_t transmit_time = gettime64(); const uint64_t net_transmit_time = hton64(transmit_time); memcpy(u32p, &net_transmit_time, sizeof(uint64_t)); - // u32p += 2; + if(config.debug.ntp.v.b) + print_debug_time("Transmit Timestamp", u32p, 0); + u32p += 2; // 0 1 2 3 // 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 @@ -205,10 +213,10 @@ static int ntp_reply(const int socket_fd, const struct sockaddr *saddr_p, const if(sendto(socket_fd, send_buf, sizeof(send_buf), 0, saddr_p, saddrlen) < 48) { log_err("NTP send error: %s", strerror(errno)); - return 1; + return false; } - return 0; + return true; } // Process incoming NTP requests @@ -226,9 +234,32 @@ static void request_process_loop(int fd, const char *ipstr, const int protocol) // the request const uint64_t recv_time = gettime64(); - struct sockaddr_in sin; - memcpy(&sin, &src_addr, sizeof(sin)); + // Print the request + if(config.debug.ntp.v.b) + { + if(protocol == AF_INET6) + { + struct sockaddr_in6 sin6; + memcpy(&sin6, &src_addr, sizeof(sin6)); + char ip[INET6_ADDRSTRLEN]; + const in_port_t port = ntohs(sin6.sin6_port); + inet_ntop(protocol, &sin6.sin6_addr, ip, sizeof(ip)); + log_debug(DEBUG_NTP, "Received NTP request from [%s]:%u", ip, port); + } + else + { + struct sockaddr_in sin; + memcpy(&sin, &src_addr, sizeof(sin)); + + char ip[INET6_ADDRSTRLEN]; + const in_port_t port = ntohs(sin.sin_port); + inet_ntop(protocol, &sin.sin_addr, ip, sizeof(ip)); + log_debug(DEBUG_NTP, "Received NTP request from %s:%u", ip, port); + } + } + + // Fork a child to handle the request const pid_t pid = fork(); if (pid == 0) { // Child diff --git a/test/pihole.toml b/test/pihole.toml index 3c4b651a..9e5b06fb 100644 --- a/test/pihole.toml +++ b/test/pihole.toml @@ -1053,14 +1053,17 @@ # Reserved debug flag reserved = true ### CHANGED, default = false + # Print information about NTP synchronization + ntp = true ### CHANGED, default = false + # Set all debug flags at once. This is a convenience option to enable all debug flags # at once. Note that this option is not persistent, setting it to true will enable all # *remaining* debug flags but unsetting it will disable *all* debug flags. all = true ### CHANGED, default = false # Configuration statistics: -# 140 total entries out of which 86 entries are default -# --> 54 entries are modified +# 141 total entries out of which 86 entries are default +# --> 55 entries are modified # 2 entries are forced through environment: # - misc.nice # - debug.api From 10e4c732f28de5679dc777bf6b74081390a5d51b Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 4 Jun 2024 13:14:50 +0200 Subject: [PATCH 133/339] Use trimmed mean to compute time offset to exclude outliers where packets traveled unusual paths Signed-off-by: DL6ER --- src/ntp/client.c | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/src/ntp/client.c b/src/ntp/client.c index 0a1fd4d5..7120fc54 100644 --- a/src/ntp/client.c +++ b/src/ntp/client.c @@ -322,6 +322,33 @@ bool ntp_client(const char *server, const bool settime) log_info("Average time offset: (%e +/- %e s)", theta_avg, theta_stdev); log_info("Average round-trip delay: (%e +/- %e s)", delta_avg, delta_stdev); + // Compute trimmed mean (average excluding outliers) + double theta_trim = 0.0, delta_trim = 0.0; + unsigned int trim = 0; + for(unsigned int i = 0; i < NTP_AVERGAGE_COUNT; i++) + { + // Skip invalid values + if(fabs(ntp[i].theta) < ntp[i].precision || + fabs(ntp[i].delta) < ntp[i].precision) + continue; + + // Skip outliers + // We consider values > 2 standard deviations from the mean as + // outliers + if(fabs(ntp[i].theta - theta_avg) > 2 * theta_stdev || + fabs(ntp[i].delta - delta_avg) > 2 * delta_stdev) + continue; + + theta_trim += ntp[i].theta; + delta_trim += ntp[i].delta; + trim++; + } + theta_trim /= trim; + delta_trim /= trim; + + log_info("Trimmed mean time offset: %e s (excluded %u outliers)", theta_trim, NTP_AVERGAGE_COUNT - trim); + log_info("Trimmed mean round-trip delay: %e s (excluded %u outliers)", delta_trim, NTP_AVERGAGE_COUNT - trim); + // Set time if requested if(settime) { @@ -332,7 +359,7 @@ bool ntp_client(const char *server, const bool settime) // Convert from double to native format (signed) and add to the // current time. Note the addition is done in native format to // avoid overflow or loss of precision. - const uint64_t ntp_time = U2LFP(unix_time) + D2LFP(theta_avg); + const uint64_t ntp_time = U2LFP(unix_time) + D2LFP(theta_trim); // Convert NTP to native format unix_time.tv_sec = NTPtoSEC(ntp_time); From de45ba840af9a98b5ac89c9c66e69c0c99fcfa28 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 4 Jun 2024 13:17:09 +0200 Subject: [PATCH 134/339] Add CAP_SYS_TIME to required capabilities to set the system time without being root Signed-off-by: DL6ER --- src/CMakeLists.txt | 2 +- src/capabilities.c | 7 +++++++ test/test_suite.bats | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 8d823bd0..008e6d7d 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -375,5 +375,5 @@ find_program(SETCAP setcap) install(TARGETS pihole-FTL RUNTIME DESTINATION bin PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE) -install(CODE "execute_process(COMMAND ${SETCAP} CAP_NET_BIND_SERVICE,CAP_NET_RAW,CAP_NET_ADMIN,CAP_SYS_NICE,CAP_CHOWN+eip \$ENV{DESTDIR}\${CMAKE_INSTALL_PREFIX}/bin/pihole-FTL)") +install(CODE "execute_process(COMMAND ${SETCAP} CAP_NET_BIND_SERVICE,CAP_NET_RAW,CAP_NET_ADMIN,CAP_SYS_NICE,CAP_CHOWN,CAP_SYS_TIME+eip \$ENV{DESTDIR}\${CMAKE_INSTALL_PREFIX}/bin/pihole-FTL)") diff --git a/src/capabilities.c b/src/capabilities.c index 79d0532e..57631edb 100644 --- a/src/capabilities.c +++ b/src/capabilities.c @@ -141,6 +141,13 @@ bool check_capabilities(void) log_warn("Required Linux capability CAP_CHOWN not available"); capabilities_okay = false; } + if (!(data->permitted & (1 << CAP_SYS_TIME)) || + !(data->effective & (1 << CAP_SYS_TIME))) + { + // Necessary for setting the system time in the NTP client + log_warn("Required Linux capability CAP_SYS_TIME not available"); + capabilities_okay = false; + } // Free allocated memory free(hdr); diff --git a/test/test_suite.bats b/test/test_suite.bats index f16e4d7f..0456c18e 100644 --- a/test/test_suite.bats +++ b/test/test_suite.bats @@ -487,7 +487,7 @@ } @test "No WARNING messages in FTL.log (besides known warnings)" { - run bash -c 'grep "WARNING:" /var/log/pihole/FTL.log | grep -v -E "CAP_NET_ADMIN|CAP_NET_RAW|CAP_SYS_NICE|CAP_IPC_LOCK|CAP_CHOWN|CAP_NET_BIND_SERVICE|(Cannot set process priority)|FTLCONF_"' + run bash -c 'grep "WARNING:" /var/log/pihole/FTL.log | grep -v -E "CAP_NET_ADMIN|CAP_NET_RAW|CAP_SYS_NICE|CAP_IPC_LOCK|CAP_CHOWN|CAP_NET_BIND_SERVICE|CAP_SYS_TIME|(Cannot set process priority)|FTLCONF_"' printf "%s\n" "${lines[@]}" [[ "${lines[@]}" == "" ]] } From 12758da50b47de2536e6c43deb02fbc52a444268 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 4 Jun 2024 17:48:58 +0200 Subject: [PATCH 135/339] Use David L. Mills' clock adjustment algorithm (RFC 5905) for gradual clock adjustments if the deviation is small Signed-off-by: DL6ER --- src/ntp/client.c | 92 ++++++++++++++++++++++++++++++++++++------------ 1 file changed, 69 insertions(+), 23 deletions(-) diff --git a/src/ntp/client.c b/src/ntp/client.c index 7120fc54..a3b5fd7f 100644 --- a/src/ntp/client.c +++ b/src/ntp/client.c @@ -29,6 +29,8 @@ #include // config struct #include "config/config.h" +// ntp_adjtime() +#include struct ntp_sync { @@ -114,6 +116,61 @@ void print_debug_time(const char *label, const uint32_t *u32p, const uint64_t nt (timevar >> 32) & 0xFFFFFFFF, timevar & 0xFFFFFFFF, time_str); } +static bool settime_step(const double offset) +{ + // Get current time + struct timeval unix_time; + gettimeofday(&unix_time, NULL); + + // Convert from double to native format (signed) and add to the + // current time. Note the addition is done in native format to + // avoid overflow or loss of precision. + const uint64_t ntp_time = U2LFP(unix_time) + D2LFP(offset); + + // Convert NTP to native format + unix_time.tv_sec = NTPtoSEC(ntp_time); + unix_time.tv_usec = NTPtoUSEC(ntp_time); + log_debug(DEBUG_NTP, "Stepping system time by %e s", offset); + + // Set time immediately + if(settimeofday(&unix_time, NULL) != 0) + { + log_err("Failed to set time: %s", + errno == EPERM ? "Insufficient permissions, try running with sudo" : strerror(errno)); + return false; + } + + return true; +} + +static bool settime_skew(const double offset) +{ + // Gradually adjust time using ntp_adjtime() using David + // L. Mills' clock adjustment algorithm (see RFC 5905) + // Deviations will only gradually be corrected at + // maximum slew rate of 500ppm (0.05%), i.e., no faster + // than a correction of 500 microseconds per second, or, in + // other words, 1000 seconds (16 minutes and 40 seconds) to + // correct a 0.5 second offset. + struct timex tx; + memset(&tx, 0, sizeof(tx)); + + // Set mode to adjust time offset + tx.modes = MOD_CLKA | MOD_MICRO; + tx.offset = 1e6 * offset; // Convert to microseconds + log_debug(DEBUG_NTP, "Gradually adjusting system time by %li usec within the next %.1f seconds)", + tx.offset, fabs(1e6 * offset / 500)); + + if(ntp_adjtime(&tx) < 0) + { + log_err("Failed to adjust time: %s", + errno == EPERM ? "Insufficient permissions, try running with sudo" : strerror(errno)); + return false; + } + + return true; +} + static bool reply(int fd, struct ntp_sync *ntp, const bool verbose) { // NTP Packet buffer @@ -352,31 +409,20 @@ bool ntp_client(const char *server, const bool settime) // Set time if requested if(settime) { - // Get current time - struct timeval unix_time; - gettimeofday(&unix_time, NULL); + // If the clock deviates more than 0.5 seconds from the NTP server, + // the time is updated immediately. Otherwise, the time is updated + // gradually to avoid sudden jumps in the system clock. + // The threshold of 0.5 seconds is hard-wired into the kernel + // since Linux 2.6.26, see man ntp_adjtime(2) for details. + bool success; + if(fabs(theta_trim) > 0.5) + success = settime_step(theta_trim); + else + success = settime_skew(theta_trim); - // Convert from double to native format (signed) and add to the - // current time. Note the addition is done in native format to - // avoid overflow or loss of precision. - const uint64_t ntp_time = U2LFP(unix_time) + D2LFP(theta_trim); - - // Convert NTP to native format - unix_time.tv_sec = NTPtoSEC(ntp_time); - unix_time.tv_usec = NTPtoUSEC(ntp_time); - - // Print new time - char time_str[TIMESTR_SIZE]; - format_NTP_time(time_str, ntp_time); - log_info("Setting local time to: %s", time_str); - - // Set time - if(settimeofday(&unix_time, NULL) != 0) - { - log_err("Failed to set time: %s", - errno == EPERM ? "Insufficient permissions, try running with sudo" : strerror(errno)); + // Return early if time could not be set + if(!success) return false; - } } // Offset and delay larger than 0.1 seconds are considered as invalid From 3971e67849ef731f087b8489f24e16b4e9a1c700 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 4 Jun 2024 23:32:07 +0200 Subject: [PATCH 136/339] Add NTP background synchronization Signed-off-by: DL6ER --- src/api/docs/content/specs/config.yaml | 13 +++ src/config/config.c | 19 ++++ src/config/config.h | 5 + src/dnsmasq_interface.c | 3 + src/ntp/client.c | 130 ++++++++++++++++++++----- src/ntp/ntp.h | 3 + test/pihole.toml | 16 ++- test/test_suite.bats | 2 +- 8 files changed, 162 insertions(+), 29 deletions(-) diff --git a/src/api/docs/content/specs/config.yaml b/src/api/docs/content/specs/config.yaml index c1b4edfa..9901324d 100644 --- a/src/api/docs/content/specs/config.yaml +++ b/src/api/docs/content/specs/config.yaml @@ -345,6 +345,15 @@ components: address: type: string x-format: ipv6 + sync: + type: object + properties: + server: + type: string + interval: + type: integer + count: + type: integer resolver: type: object properties: @@ -687,6 +696,10 @@ components: ipv6: active: true address: "" + sync: + server: "pool.ntp.org" + interval: 3600 + count: 8 resolver: resolveIPv4: true resolveIPv6: true diff --git a/src/config/config.c b/src/config/config.c index a7f29c02..df4d7ef4 100644 --- a/src/config/config.c +++ b/src/config/config.c @@ -832,6 +832,25 @@ void initConfig(struct config *conf) memset(&conf->ntp.ipv6.address.d.in6_addr, 0, sizeof(struct in6_addr)); conf->ntp.ipv6.address.c = validate_stub; // Only type-based checking + conf->ntp.sync.server.k = "ntp.sync.server"; + conf->ntp.sync.server.h = "NTP server (hostname, IPv4 or IPv6) to sync with, e.g., \"pool.ntp.org\" or \"[2001:4860:4860::8888]\""; + conf->ntp.sync.server.a = cJSON_CreateStringReference("valid NTP upstream server"); + conf->ntp.sync.server.t = CONF_STRING; + conf->ntp.sync.server.d.s = (char*)"pool.ntp.org"; + conf->ntp.sync.server.c = validate_stub; // Only type-based checking + + conf->ntp.sync.interval.k = "ntp.sync.interval"; + conf->ntp.sync.interval.h = "Interval in seconds to sync with the NTP server"; + conf->ntp.sync.interval.t = CONF_UINT; + conf->ntp.sync.interval.d.ui = 3600; + conf->ntp.sync.interval.c = validate_stub; // Only type-based checking + + conf->ntp.sync.count.k = "ntp.sync.count"; + conf->ntp.sync.count.h = "Number of NTP syncs to perform and average before updating the system time"; + conf->ntp.sync.count.t = CONF_UINT; + conf->ntp.sync.count.d.ui = 8; + conf->ntp.sync.count.c = validate_stub; // Only type-based checking + // struct resolver conf->resolver.resolveIPv6.k = "resolver.resolveIPv6"; diff --git a/src/config/config.h b/src/config/config.h index 6c000146..88bdefd9 100644 --- a/src/config/config.h +++ b/src/config/config.h @@ -200,6 +200,11 @@ struct config { struct conf_item active; struct conf_item address; } ipv6; + struct { + struct conf_item server; + struct conf_item interval; + struct conf_item count; + } sync; } ntp; struct { diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c index cc527ca4..58384edd 100644 --- a/src/dnsmasq_interface.c +++ b/src/dnsmasq_interface.c @@ -2899,6 +2899,9 @@ void FTL_fork_and_bind_sockets(struct passwd *ent_pw, bool dnsmasq_start) // Initialize NTP server ntp_server_start(); + // Start NTP sync thread + ntp_start_sync_thread(); + // We will use the attributes object later to start all threads in // detached mode pthread_attr_t attr; diff --git a/src/ntp/client.c b/src/ntp/client.c index a3b5fd7f..5ce79a87 100644 --- a/src/ntp/client.c +++ b/src/ntp/client.c @@ -29,8 +29,8 @@ #include // config struct #include "config/config.h" -// ntp_adjtime() -#include +// adjtime() +#include struct ntp_sync { @@ -145,23 +145,41 @@ static bool settime_step(const double offset) static bool settime_skew(const double offset) { - // Gradually adjust time using ntp_adjtime() using David - // L. Mills' clock adjustment algorithm (see RFC 5905) - // Deviations will only gradually be corrected at - // maximum slew rate of 500ppm (0.05%), i.e., no faster - // than a correction of 500 microseconds per second, or, in - // other words, 1000 seconds (16 minutes and 40 seconds) to - // correct a 0.5 second offset. - struct timex tx; - memset(&tx, 0, sizeof(tx)); + // This function gradually adjusts the system clock. + // + // If the adjustment in delta is positive, then the system clock is + // speeded up by some small percentage (i.e., by adding a small amount + // of time to the clock value in each second) until the adjustment has + // been completed. If the adjustment in delta is negative, then the + // clock is slowed down in a similar fashion. + // + // If a clock adjustment from an earlier adjtime() call is already in + // progress at the time of a later adjtime() call, and delta is not NULL + // for the later call, then the earlier adjustment is stopped, but any + // al‐ ready completed part of that adjustment is not undone. + // + // The adjustment that adjtime() makes to the clock is carried out in + // such a manner that the clock is always monotonically increasing. + // Using adjtime() to adjust the time prevents the problems that can be + // caused for certain applications (e.g., make(1)) by abrupt positive or + // negative jumps in the system time. + // + // adjtime() is intended to be used to make small adjustments to the + // system time. The actual time adjustment rate is implementation-specific + // but is typically on the order of 500 ppm, i.e., 0.5 ms/s. + struct timeval tx; + tx.tv_sec = (long int)offset; + tx.tv_usec = (offset - tx.tv_sec) * 1e6; + if(tx.tv_usec < 0) + { + // Adjust seconds if microseconds are negative + tx.tv_sec--; + tx.tv_usec += 1000000000; + } + log_debug(DEBUG_NTP, "Gradually adjusting system time by %li.%06li s", + (long int)tx.tv_sec, (long int)tx.tv_usec); - // Set mode to adjust time offset - tx.modes = MOD_CLKA | MOD_MICRO; - tx.offset = 1e6 * offset; // Convert to microseconds - log_debug(DEBUG_NTP, "Gradually adjusting system time by %li usec within the next %.1f seconds)", - tx.offset, fabs(1e6 * offset / 500)); - - if(ntp_adjtime(&tx) < 0) + if(adjtime(&tx, NULL) < 0) { log_err("Failed to adjust time: %s", errno == EPERM ? "Insufficient permissions, try running with sudo" : strerror(errno)); @@ -314,14 +332,23 @@ bool ntp_client(const char *server, const bool settime) } freeaddrinfo(saddr); - struct ntp_sync ntp[NTP_AVERGAGE_COUNT]; - memset(&ntp, 0, sizeof(ntp)); - for(unsigned int i = 0; i < NTP_AVERGAGE_COUNT; i++) + // Send and receive NTP packets + const unsigned int count = config.ntp.sync.count.v.ui; + struct ntp_sync *ntp = calloc(count, sizeof(struct ntp_sync)); + if(ntp == NULL) + { + log_err("Cannot allocate memory for NTP client\n"); + close(s); + return false; + } + memset(ntp, 0, count*sizeof(*ntp)); + for(unsigned int i = 0; i < count; i++) { // Send request if(!request(s, &ntp[i])) { close(s); + free(ntp); return false; } // Get reply @@ -342,7 +369,7 @@ bool ntp_client(const char *server, const bool settime) unsigned int valid = 0; double theta_avg = 0.0, theta_stdev = 0.0; double delta_avg = 0.0, delta_stdev = 0.0; - for(unsigned int i = 0; i < NTP_AVERGAGE_COUNT; i++) + for(unsigned int i = 0; i < count; i++) { // Skip invalid values if(fabs(ntp[i].theta) < ntp[i].precision || @@ -357,13 +384,14 @@ bool ntp_client(const char *server, const bool settime) if(valid == 0) { log_err("No valid NTP replies received, check server and network connectivity\n"); + free(ntp); return false; } - log_info("Received %u/%d valid NTP replies\n", valid, NTP_AVERGAGE_COUNT); + log_info("Received %u/%u valid NTP replies\n", valid, count); theta_avg /= valid; delta_avg /= valid; - for(unsigned int i = 0; i < NTP_AVERGAGE_COUNT; i++) + for(unsigned int i = 0; i < count; i++) { // Skip invalid values if(fabs(ntp[i].theta) < ntp[i].precision || @@ -382,7 +410,7 @@ bool ntp_client(const char *server, const bool settime) // Compute trimmed mean (average excluding outliers) double theta_trim = 0.0, delta_trim = 0.0; unsigned int trim = 0; - for(unsigned int i = 0; i < NTP_AVERGAGE_COUNT; i++) + for(unsigned int i = 0; i < count; i++) { // Skip invalid values if(fabs(ntp[i].theta) < ntp[i].precision || @@ -403,8 +431,11 @@ bool ntp_client(const char *server, const bool settime) theta_trim /= trim; delta_trim /= trim; - log_info("Trimmed mean time offset: %e s (excluded %u outliers)", theta_trim, NTP_AVERGAGE_COUNT - trim); - log_info("Trimmed mean round-trip delay: %e s (excluded %u outliers)", delta_trim, NTP_AVERGAGE_COUNT - trim); + // Free allocated memory + free(ntp); + + log_info("Trimmed mean time offset: %e s (excluded %u outliers)", theta_trim, count - trim); + log_info("Trimmed mean round-trip delay: %e s (excluded %u outliers)", delta_trim, count - trim); // Set time if requested if(settime) @@ -429,3 +460,48 @@ bool ntp_client(const char *server, const bool settime) // during local testing (e.g., when the server is on the same machine) return theta_avg < 0.1 && delta_avg < 0.1; } + +static void *ntp_client_thread(void *arg) +{ + // Set thread name + pthread_setname_np(pthread_self(), "NTP sync"); + + // Run NTP client + while(true) + { + // Run NTP client + if(ntp_client(config.ntp.sync.server.v.s, true)) + break; + + // Sleep before retrying + sleep(config.ntp.sync.interval.v.ui); + } + + return NULL; +} + +bool ntp_start_sync_thread(void) +{ + // Return early if NTP client is disabled + if(config.ntp.sync.server.v.s == NULL || + strlen(config.ntp.sync.server.v.s) == 0 || + config.ntp.sync.interval.v.ui == 0) + return false; + + // Create thread + pthread_t thread; + if(pthread_create(&thread, NULL, ntp_client_thread, NULL) != 0) + { + log_err("Cannot create NTP client thread\n"); + return false; + } + + // Detach thread + if(pthread_detach(thread) != 0) + { + log_err("Cannot detach NTP client thread\n"); + return false; + } + + return true; +} \ No newline at end of file diff --git a/src/ntp/ntp.h b/src/ntp/ntp.h index caca5169..363f2d3e 100644 --- a/src/ntp/ntp.h +++ b/src/ntp/ntp.h @@ -32,6 +32,9 @@ bool ntp_server_start(void); // Start NTP client bool ntp_client(const char *server, const bool settime); +// Start NTP sync thread +bool ntp_start_sync_thread(void); + // Number of NTP queries to average. The more queries, the more accurate the // time, but the longer it takes to synchronize. The minimum is 1. #define NTP_AVERGAGE_COUNT 8 diff --git a/test/pihole.toml b/test/pihole.toml index 9e5b06fb..f98191d6 100644 --- a/test/pihole.toml +++ b/test/pihole.toml @@ -480,6 +480,20 @@ # or empty string ("") for wildcard (::) address = "" + [ntp.sync] + # NTP server (hostname, IPv4 or IPv6) to sync with, e.g., "pool.ntp.org" or + # "[2001:4860:4860::8888]" + # + # Possible values are: + # valid NTP upstream server + server = "pool.ntp.org" + + # Interval in seconds to sync with the NTP server + interval = 3600 + + # Number of NTP syncs to perform and average before updating the system time + count = 8 + [resolver] # Should FTL try to resolve IPv4 addresses to hostnames? resolveIPv4 = false ### CHANGED, default = true @@ -1062,7 +1076,7 @@ all = true ### CHANGED, default = false # Configuration statistics: -# 141 total entries out of which 86 entries are default +# 144 total entries out of which 89 entries are default # --> 55 entries are modified # 2 entries are forced through environment: # - misc.nice diff --git a/test/test_suite.bats b/test/test_suite.bats index 0456c18e..5969ba79 100644 --- a/test/test_suite.bats +++ b/test/test_suite.bats @@ -1171,7 +1171,7 @@ @test "No ERROR messages in FTL.log (besides known/intended error)" { run bash -c 'grep "ERROR: " /var/log/pihole/FTL.log' printf "%s\n" "${lines[@]}" - run bash -c 'grep "ERROR: " /var/log/pihole/FTL.log | grep -c -v -E "(index\.html)|(Failed to create shared memory object)|(FTLCONF_debug_api is invalid)"' + run bash -c 'grep "ERROR: " /var/log/pihole/FTL.log | grep -c -v -E "(index\.html)|(Failed to create shared memory object)|(FTLCONF_debug_api is invalid)|(Failed to adjust time: Insufficient permissions)"' printf "count: %s\n" "${lines[@]}" [[ ${lines[0]} == "0" ]] } From a872c03091404b51d2b99f869cce20fb791b6084 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 4 Jun 2024 23:48:44 +0200 Subject: [PATCH 137/339] Fix formating error Signed-off-by: DL6ER --- src/ntp/client.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ntp/client.c b/src/ntp/client.c index 5ce79a87..995dd26f 100644 --- a/src/ntp/client.c +++ b/src/ntp/client.c @@ -274,7 +274,7 @@ static bool reply(int fd, struct ntp_sync *ntp, const bool verbose) if(ntp->delta < ntp->precision) ntp->delta = 0; -# // Return early if not verbose + // Return early if not verbose if(!config.debug.ntp.v.b) return true; @@ -504,4 +504,4 @@ bool ntp_start_sync_thread(void) } return true; -} \ No newline at end of file +} From 2269caeb9a2a33208f5294cc5a8e3dd4ffb6f9d6 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 4 Jun 2024 23:56:11 +0200 Subject: [PATCH 138/339] Make NTP sync thread cancelable Signed-off-by: DL6ER --- src/database/database-thread.c | 1 - src/enums.h | 1 + src/ntp/client.c | 19 ++++++++++++++----- src/signals.h | 2 ++ 4 files changed, 17 insertions(+), 6 deletions(-) diff --git a/src/database/database-thread.c b/src/database/database-thread.c index efdd1bd6..81e80484 100644 --- a/src/database/database-thread.c +++ b/src/database/database-thread.c @@ -78,7 +78,6 @@ static bool analyze_database(sqlite3 *db) } #define DBOPEN_OR_AGAIN() { if(!db) db = dbopen(false, false); if(!db) { thread_sleepms(DB, 5000); continue; } } -#define BREAK_IF_KILLED() { if(killed) break; } #define DBCLOSE_OR_BREAK() { dbclose(&db); BREAK_IF_KILLED(); } void *DB_thread(void *val) diff --git a/src/enums.h b/src/enums.h index 67dd1fbc..d8977f83 100644 --- a/src/enums.h +++ b/src/enums.h @@ -252,6 +252,7 @@ enum thread_types { DNSclient, CONF_READER, TIMER, + NTP, THREADS_MAX } __attribute__ ((packed)); diff --git a/src/ntp/client.c b/src/ntp/client.c index 995dd26f..1c3c031b 100644 --- a/src/ntp/client.c +++ b/src/ntp/client.c @@ -31,7 +31,8 @@ #include "config/config.h" // adjtime() #include - +// thread_names[] +#include "signals.h" struct ntp_sync { uint64_t org; @@ -464,19 +465,27 @@ bool ntp_client(const char *server, const bool settime) static void *ntp_client_thread(void *arg) { // Set thread name + thread_names[NTP] = "ntp-client"; + thread_running[NTP] = true; + prctl(PR_SET_NAME, thread_names[DB], 0, 0, 0); pthread_setname_np(pthread_self(), "NTP sync"); // Run NTP client - while(true) + while(!killed) { // Run NTP client - if(ntp_client(config.ntp.sync.server.v.s, true)) - break; + ntp_client(config.ntp.sync.server.v.s, true); + + // Intermediate cancellation-point + BREAK_IF_KILLED(); // Sleep before retrying - sleep(config.ntp.sync.interval.v.ui); + thread_sleepms(NTP, 1000 * config.ntp.sync.interval.v.ui); } + log_info("Terminating NTP thread"); + thread_running[NTP] = false; + return NULL; } diff --git a/src/signals.h b/src/signals.h index 4a08e4b9..f52fb2d9 100644 --- a/src/signals.h +++ b/src/signals.h @@ -32,4 +32,6 @@ extern volatile sig_atomic_t thread_cancellable[THREADS_MAX]; extern volatile sig_atomic_t thread_running[THREADS_MAX]; extern const char *thread_names[THREADS_MAX]; +#define BREAK_IF_KILLED() { if(killed) break; } + #endif //SIGNALS_H From f088b79e8f7bf60d4f02ba2c310571019f5636b8 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 6 Jun 2024 06:13:52 +0200 Subject: [PATCH 139/339] Exit synchronization early if no trimmed time offest average is avalable, print progress only when printing to the CLI, and increase delay between successive NTP requests to 0.5 seconds Signed-off-by: DL6ER --- src/args.c | 2 +- src/ntp/client.c | 68 +++++++++++++++++++++++++----------------------- src/ntp/ntp.h | 5 +++- 3 files changed, 41 insertions(+), 34 deletions(-) diff --git a/src/args.c b/src/args.c index 622c894b..f6e252a1 100644 --- a/src/args.c +++ b/src/args.c @@ -319,7 +319,7 @@ void parse_args(int argc, char* argv[]) const char *server = "127.0.0.1"; if(argc > 2 && strcmp(argv[2], "--update") != 0) server = argv[2]; - exit(ntp_client(server, update) ? EXIT_SUCCESS : EXIT_FAILURE); + exit(ntp_client(server, update, true) ? EXIT_SUCCESS : EXIT_FAILURE); } // Import teleporter archive through CLI diff --git a/src/ntp/client.c b/src/ntp/client.c index 1c3c031b..55c7ce14 100644 --- a/src/ntp/client.c +++ b/src/ntp/client.c @@ -67,7 +67,8 @@ static bool request(int fd, struct ntp_sync *ntp) // Send request if(send(fd, buf, 48, 0) != 48) { - log_err("Failed to send data to NTP server: %s", strerror(errno)); + log_err("Failed to send data to NTP server: %s", + errno == EAGAIN ? "Timeout" : strerror(errno)); return false; } @@ -171,14 +172,7 @@ static bool settime_skew(const double offset) struct timeval tx; tx.tv_sec = (long int)offset; tx.tv_usec = (offset - tx.tv_sec) * 1e6; - if(tx.tv_usec < 0) - { - // Adjust seconds if microseconds are negative - tx.tv_sec--; - tx.tv_usec += 1000000000; - } - log_debug(DEBUG_NTP, "Gradually adjusting system time by %li.%06li s", - (long int)tx.tv_sec, (long int)tx.tv_usec); + log_debug(DEBUG_NTP, "Gradually adjusting system time by %.3f ms", 1e3 * offset); if(adjtime(&tx, NULL) < 0) { @@ -198,7 +192,8 @@ static bool reply(int fd, struct ntp_sync *ntp, const bool verbose) // Receive reply if(recv(fd, buf, 48, 0) < 48) { - log_err("Failed to receive data from NTP server: %s", strerror(errno)); + log_err("Failed to receive data from NTP server: %s", + errno == EAGAIN ? "Timeout" : strerror(errno)); return false; } @@ -235,14 +230,15 @@ static bool reply(int fd, struct ntp_sync *ntp, const bool verbose) // network byte order if(ntp->org != org) { - log_warn("Received NTP reply does not match request (request %"PRIx64", reply %"PRIx64")", ntp->org, org); + log_warn("Received NTP reply does not match request (request %"PRIx64", reply %"PRIx64"), ignoring", + ntp->org, org); return false; } // Check stratum, mode, version, etc. if((buf[0] & 0x07) != 4) { - log_warn("Received NTP reply has invalid version"); + log_warn("Received NTP reply has invalid version, ignoring"); return false; } @@ -292,7 +288,7 @@ static bool reply(int fd, struct ntp_sync *ntp, const bool verbose) return true; } -bool ntp_client(const char *server, const bool settime) +bool ntp_client(const char *server, const bool settime, const bool print) { const int protocol = strchr(server, ':') != NULL ? AF_INET6 : AF_INET; @@ -300,26 +296,26 @@ bool ntp_client(const char *server, const bool settime) const int s = socket(protocol, SOCK_DGRAM, IPPROTO_UDP); if(s == -1) { - log_err("Cannot create UDP socket\n"); + log_err("Cannot create UDP socket"); return false; } - // Set socket timeout to 2 seconds + // Set socket timeout to 5 seconds struct timeval tv; - tv.tv_sec = 2; + tv.tv_sec = 5; tv.tv_usec = 0; if(setsockopt(s, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv)) != 0) { - log_err("Cannot set socket timeout\n"); + log_err("Cannot set socket timeout"); close(s); return false; } // Resolve server address struct addrinfo *saddr; - if(getaddrinfo(server, "123", NULL, &saddr) != 0) + if(getaddrinfo(server, "ntp", NULL, &saddr) != 0) { - log_err("Cannot resolve NTP server address\n"); + log_err("Cannot resolve NTP server address"); close(s); return false; } @@ -327,7 +323,7 @@ bool ntp_client(const char *server, const bool settime) // Set address to send to/receive from if(connect(s, saddr->ai_addr, saddr->ai_addrlen) != 0) { - log_err("Cannot connect to NTP server\n"); + log_err("Cannot connect to NTP server"); close(s); return false; } @@ -338,7 +334,7 @@ bool ntp_client(const char *server, const bool settime) struct ntp_sync *ntp = calloc(count, sizeof(struct ntp_sync)); if(ntp == NULL) { - log_err("Cannot allocate memory for NTP client\n"); + log_err("Cannot allocate memory for NTP client"); close(s); return false; } @@ -356,12 +352,14 @@ bool ntp_client(const char *server, const bool settime) if(!reply(s, &ntp[i], false)) continue; - // Sleep for 100 ms to avoid flooding the server - printf("."); + // Sleep for some time to avoid flooding the server + if(print) + printf("."); fflush(stdout); - usleep(100000); + usleep(NTP_DELAY); } - printf("\n"); + if(print) + printf("\n"); // Close socket close(s); @@ -384,11 +382,11 @@ bool ntp_client(const char *server, const bool settime) if(valid == 0) { - log_err("No valid NTP replies received, check server and network connectivity\n"); + log_warn("No valid NTP replies received, check server and network connectivity"); free(ntp); return false; } - log_info("Received %u/%u valid NTP replies\n", valid, count); + log_info("Received %u/%u valid NTP replies", valid, count); theta_avg /= valid; delta_avg /= valid; @@ -429,12 +427,18 @@ bool ntp_client(const char *server, const bool settime) delta_trim += ntp[i].delta; trim++; } - theta_trim /= trim; - delta_trim /= trim; // Free allocated memory free(ntp); + if(trim == 0) + { + log_warn("No valid NTP replies after outlier removal, check server and network connectivity"); + return false; + } + theta_trim /= trim; + delta_trim /= trim; + log_info("Trimmed mean time offset: %e s (excluded %u outliers)", theta_trim, count - trim); log_info("Trimmed mean round-trip delay: %e s (excluded %u outliers)", delta_trim, count - trim); @@ -474,7 +478,7 @@ static void *ntp_client_thread(void *arg) while(!killed) { // Run NTP client - ntp_client(config.ntp.sync.server.v.s, true); + ntp_client(config.ntp.sync.server.v.s, true, false); // Intermediate cancellation-point BREAK_IF_KILLED(); @@ -501,14 +505,14 @@ bool ntp_start_sync_thread(void) pthread_t thread; if(pthread_create(&thread, NULL, ntp_client_thread, NULL) != 0) { - log_err("Cannot create NTP client thread\n"); + log_err("Cannot create NTP client thread"); return false; } // Detach thread if(pthread_detach(thread) != 0) { - log_err("Cannot detach NTP client thread\n"); + log_err("Cannot detach NTP client thread"); return false; } diff --git a/src/ntp/ntp.h b/src/ntp/ntp.h index 363f2d3e..34d6f050 100644 --- a/src/ntp/ntp.h +++ b/src/ntp/ntp.h @@ -30,7 +30,7 @@ void print_debug_time(const char *label, const uint32_t *u32p, const uint64_t nt bool ntp_server_start(void); // Start NTP client -bool ntp_client(const char *server, const bool settime); +bool ntp_client(const char *server, const bool settime, const bool print); // Start NTP sync thread bool ntp_start_sync_thread(void); @@ -39,6 +39,9 @@ bool ntp_start_sync_thread(void); // time, but the longer it takes to synchronize. The minimum is 1. #define NTP_AVERGAGE_COUNT 8 +// Delay between consecutive NTP queries in microseconds +#define NTP_DELAY 500000 + // number of seconds between 1900 and 1970 (MSB=1) #define DIFF_SEC_1900_1970 (2208988800UL) // number of seconds between 1970 and Feb 7, 2036 (6:28:16 UTC) (MSB=0) From ee8f9899ddb21ed093c445217518882d3e460527 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 6 Jun 2024 07:21:37 +0200 Subject: [PATCH 140/339] Add NTP settings category to the API and create all threads in detached mode Signed-off-by: DL6ER --- src/api/config.c | 1 + src/dnsmasq_interface.c | 14 ++++++++------ src/ntp/client.c | 14 ++++---------- src/ntp/ntp.h | 4 ++-- src/ntp/server.c | 8 +++----- src/tools/arp-scan.c | 2 ++ src/tools/dhcp-discover.c | 2 ++ 7 files changed, 22 insertions(+), 23 deletions(-) diff --git a/src/api/config.c b/src/api/config.c index f413e502..aa881a82 100644 --- a/src/api/config.c +++ b/src/api/config.c @@ -37,6 +37,7 @@ static struct { { { "dns", "DNS", "DNS server settings" }, { "dhcp", "DHCP", "DHCP server settings" }, + { "ntp", "NTP", "Network Time Sync settings" }, { "resolver", "Resolver", "Resolver settings" }, { "database", "Database", "Database settings" }, { "webserver", "HTTP/API", "Webserver and API settings" }, diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c index 58384edd..14a82284 100644 --- a/src/dnsmasq_interface.c +++ b/src/dnsmasq_interface.c @@ -2896,17 +2896,19 @@ void FTL_fork_and_bind_sockets(struct passwd *ent_pw, bool dnsmasq_start) // so they will not listen to real-time signals handle_realtime_signals(); - // Initialize NTP server - ntp_server_start(); - - // Start NTP sync thread - ntp_start_sync_thread(); - // We will use the attributes object later to start all threads in // detached mode pthread_attr_t attr; // Initialize thread attributes object with default attribute values pthread_attr_init(&attr); + // Set thread attributes to detached mode + pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); + + // Initialize NTP server + ntp_server_start(&attr); + + // Start NTP sync thread + ntp_start_sync_thread(&attr); // Start database thread if database is used if(pthread_create( &threads[DB], &attr, DB_thread, NULL ) != 0) diff --git a/src/ntp/client.c b/src/ntp/client.c index 55c7ce14..46cc9e0b 100644 --- a/src/ntp/client.c +++ b/src/ntp/client.c @@ -31,6 +31,8 @@ #include "config/config.h" // adjtime() #include +// threads[] +#include "daemon.h" // thread_names[] #include "signals.h" struct ntp_sync @@ -493,7 +495,7 @@ static void *ntp_client_thread(void *arg) return NULL; } -bool ntp_start_sync_thread(void) +bool ntp_start_sync_thread(pthread_attr_t *attr) { // Return early if NTP client is disabled if(config.ntp.sync.server.v.s == NULL || @@ -502,19 +504,11 @@ bool ntp_start_sync_thread(void) return false; // Create thread - pthread_t thread; - if(pthread_create(&thread, NULL, ntp_client_thread, NULL) != 0) + if(pthread_create(&threads[NTP], attr, ntp_client_thread, NULL) != 0) { log_err("Cannot create NTP client thread"); return false; } - // Detach thread - if(pthread_detach(thread) != 0) - { - log_err("Cannot detach NTP client thread"); - return false; - } - return true; } diff --git a/src/ntp/ntp.h b/src/ntp/ntp.h index 34d6f050..3fde4c07 100644 --- a/src/ntp/ntp.h +++ b/src/ntp/ntp.h @@ -27,13 +27,13 @@ uint64_t gettime64(void); void print_debug_time(const char *label, const uint32_t *u32p, const uint64_t ntp_time); // Start NTP server -bool ntp_server_start(void); +bool ntp_server_start(pthread_attr_t *attr); // Start NTP client bool ntp_client(const char *server, const bool settime, const bool print); // Start NTP sync thread -bool ntp_start_sync_thread(void); +bool ntp_start_sync_thread(pthread_attr_t *attr); // Number of NTP queries to average. The more queries, the more accurate the // time, but the longer it takes to synchronize. The minimum is 1. diff --git a/src/ntp/server.c b/src/ntp/server.c index f6e70937..294ff005 100644 --- a/src/ntp/server.c +++ b/src/ntp/server.c @@ -353,7 +353,7 @@ static void *ntp_bind_and_listen(void *param) } // Start the NTP server -bool ntp_server_start(void) +bool ntp_server_start(pthread_attr_t *attr) { // Spawn two pthreads, one for IPv4 and one for IPv6 @@ -362,7 +362,7 @@ bool ntp_server_start(void) { // Create a thread for the IPv4 NTP server pthread_t thread; - if (pthread_create(&thread, NULL, ntp_bind_and_listen, (void *)0) != 0) + if (pthread_create(&thread, attr, ntp_bind_and_listen, (void *)0) != 0) { log_err("Can not create NTP server thread for IPv4"); return false; @@ -374,14 +374,12 @@ bool ntp_server_start(void) { // Create a thread for the IPv6 NTP server pthread_t thread; - if (pthread_create(&thread, NULL, ntp_bind_and_listen, (void *)1) != 0) + if (pthread_create(&thread, attr, ntp_bind_and_listen, (void *)1) != 0) { log_err("Can not create NTP server thread for IPv6"); return false; } } - sleep(10); - return true; } diff --git a/src/tools/arp-scan.c b/src/tools/arp-scan.c index 923625a3..c5ac6ec3 100644 --- a/src/tools/arp-scan.c +++ b/src/tools/arp-scan.c @@ -616,6 +616,8 @@ int run_arp_scan(const bool scan_all, const bool extreme_mode) pthread_attr_t attr; // Initialize thread attributes object with default attribute values pthread_attr_init(&attr); + // Set thread attributes to detached mode + pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); struct ifaddrs *addrs, *tmp; getifaddrs(&addrs); diff --git a/src/tools/dhcp-discover.c b/src/tools/dhcp-discover.c index c68a74f9..045c8c14 100644 --- a/src/tools/dhcp-discover.c +++ b/src/tools/dhcp-discover.c @@ -725,6 +725,8 @@ int run_dhcp_discover(void) pthread_attr_t attr; // Initialize thread attributes object with default attribute values pthread_attr_init(&attr); + // Set thread attributes to detached mode + pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); // Create processing/printfing lock pthread_mutexattr_t lock_attr; From d923904291b1ddfbbac65ff5f38f939e7949a64e Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 6 Jun 2024 07:32:23 +0200 Subject: [PATCH 141/339] Tweak config option description Signed-off-by: DL6ER --- src/config/config.c | 8 ++++---- test/pihole.toml | 11 ++++++----- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/config/config.c b/src/config/config.c index df4d7ef4..a8ad6e0f 100644 --- a/src/config/config.c +++ b/src/config/config.c @@ -803,7 +803,7 @@ void initConfig(struct config *conf) // struct ntp conf->ntp.ipv4.active.k = "ntp.ipv4.active"; - conf->ntp.ipv4.active.h = "Should FTL act as an NTP server (IPv4)?"; + conf->ntp.ipv4.active.h = "Should FTL act as network time protocol (NTP) server (IPv4)?"; conf->ntp.ipv4.active.t = CONF_BOOL; conf->ntp.ipv4.active.f = FLAG_RESTART_FTL; conf->ntp.ipv4.active.d.b = true; @@ -818,7 +818,7 @@ void initConfig(struct config *conf) conf->ntp.ipv4.address.c = validate_stub; // Only type-based checking conf->ntp.ipv6.active.k = "ntp.ipv6.active"; - conf->ntp.ipv6.active.h = "Should FTL act as an NTP server (IPv6)?"; + conf->ntp.ipv6.active.h = "Should FTL act as network time protocol (NTP) server (IPv6)?"; conf->ntp.ipv6.active.t = CONF_BOOL; conf->ntp.ipv6.active.f = FLAG_RESTART_FTL; conf->ntp.ipv6.active.d.b = true; @@ -833,14 +833,14 @@ void initConfig(struct config *conf) conf->ntp.ipv6.address.c = validate_stub; // Only type-based checking conf->ntp.sync.server.k = "ntp.sync.server"; - conf->ntp.sync.server.h = "NTP server (hostname, IPv4 or IPv6) to sync with, e.g., \"pool.ntp.org\" or \"[2001:4860:4860::8888]\""; + conf->ntp.sync.server.h = "NTP upstream server to sync with, e.g., \"pool.ntp.org\". Note that the NTP server should be located as close as possible to you in order to minimize the time offset possibly introduced by different routing paths."; conf->ntp.sync.server.a = cJSON_CreateStringReference("valid NTP upstream server"); conf->ntp.sync.server.t = CONF_STRING; conf->ntp.sync.server.d.s = (char*)"pool.ntp.org"; conf->ntp.sync.server.c = validate_stub; // Only type-based checking conf->ntp.sync.interval.k = "ntp.sync.interval"; - conf->ntp.sync.interval.h = "Interval in seconds to sync with the NTP server"; + conf->ntp.sync.interval.h = "Interval in seconds between successive syncronization attempts with the NTP server"; conf->ntp.sync.interval.t = CONF_UINT; conf->ntp.sync.interval.d.ui = 3600; conf->ntp.sync.interval.c = validate_stub; // Only type-based checking diff --git a/test/pihole.toml b/test/pihole.toml index f98191d6..b765e555 100644 --- a/test/pihole.toml +++ b/test/pihole.toml @@ -461,7 +461,7 @@ hosts = [] [ntp.ipv4] - # Should FTL act as an NTP server (IPv4)? + # Should FTL act as network time protocol (NTP) server (IPv4)? active = true # IPv4 address to listen on for NTP requests @@ -471,7 +471,7 @@ address = "" [ntp.ipv6] - # Should FTL act as an NTP server (IPv6)? + # Should FTL act as network time protocol (NTP) server (IPv6)? active = true # IPv6 address to listen on for NTP requests @@ -481,14 +481,15 @@ address = "" [ntp.sync] - # NTP server (hostname, IPv4 or IPv6) to sync with, e.g., "pool.ntp.org" or - # "[2001:4860:4860::8888]" + # NTP upstream server to sync with, e.g., "pool.ntp.org". Note that the NTP server + # should be located as close as possible to you in order to minimize the time offset + # possibly introduced by different routing paths. # # Possible values are: # valid NTP upstream server server = "pool.ntp.org" - # Interval in seconds to sync with the NTP server + # Interval in seconds between successive syncronization attempts with the NTP server interval = 3600 # Number of NTP syncs to perform and average before updating the system time From 08f2e37d9fd7e112b5bb28c2f99b415ab21d05c9 Mon Sep 17 00:00:00 2001 From: Dominik Date: Thu, 6 Jun 2024 08:54:31 +0200 Subject: [PATCH 142/339] Apply suggestions from code review Co-authored-by: RD WebDesign Signed-off-by: Dominik --- src/config/config.c | 2 +- test/pihole.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/config/config.c b/src/config/config.c index a8ad6e0f..19b17086 100644 --- a/src/config/config.c +++ b/src/config/config.c @@ -840,7 +840,7 @@ void initConfig(struct config *conf) conf->ntp.sync.server.c = validate_stub; // Only type-based checking conf->ntp.sync.interval.k = "ntp.sync.interval"; - conf->ntp.sync.interval.h = "Interval in seconds between successive syncronization attempts with the NTP server"; + conf->ntp.sync.interval.h = "Interval in seconds between successive synchronization attempts with the NTP server"; conf->ntp.sync.interval.t = CONF_UINT; conf->ntp.sync.interval.d.ui = 3600; conf->ntp.sync.interval.c = validate_stub; // Only type-based checking diff --git a/test/pihole.toml b/test/pihole.toml index b765e555..d29dc427 100644 --- a/test/pihole.toml +++ b/test/pihole.toml @@ -489,7 +489,7 @@ # valid NTP upstream server server = "pool.ntp.org" - # Interval in seconds between successive syncronization attempts with the NTP server + # Interval in seconds between successive synchronization attempts with the NTP server interval = 3600 # Number of NTP syncs to perform and average before updating the system time From 44d57e5b37f19b4fff91eba0735fca2a1a33b9f1 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 6 Jun 2024 06:45:53 +0200 Subject: [PATCH 143/339] Add checking of return status of sqlite3_open_v2 to ensure we are not trying to use the database when it failed to open Signed-off-by: DL6ER --- src/database/gravity-db.c | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/database/gravity-db.c b/src/database/gravity-db.c index d922e2d7..f0ac7398 100644 --- a/src/database/gravity-db.c +++ b/src/database/gravity-db.c @@ -28,9 +28,10 @@ #include "datastructure.h" // reset_aliasclient() #include "aliasclients.h" - // Definition of struct regexData #include "regex_r.h" +// file_readable() +#include "files.h" // Prefix of interface names in the client table #define INTERFACE_SEP ":" @@ -2715,9 +2716,17 @@ bool gravity_updated(void) sqlite3 *db = NULL; sqlite3_stmt *query_stmt = NULL; + // Check if database is a readable file + if(file_readable(config.files.gravity.v.s) == false) + { + log_err("Cannot read gravity database at %s - file does not exist or is not readable", + config.files.gravity.v.s); + return false; + } + // Open database int rc = sqlite3_open_v2(config.files.gravity.v.s, &db, SQLITE_OPEN_READONLY, NULL); - if(db == NULL) + if(db == NULL || rc != SQLITE_OK) { log_err("gravity_updated(): %s - SQL error open: %s", config.files.gravity.v.s, sqlite3_errstr(rc)); return false; From f9eea51dbdb1630755ed3e26c01b477593581fb8 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 6 Jun 2024 20:25:59 +0200 Subject: [PATCH 144/339] Ensure we also recalculate te checksum of the config file when in read-only mode Signed-off-by: DL6ER --- src/config/toml_writer.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/config/toml_writer.c b/src/config/toml_writer.c index 681ee1eb..8361c324 100644 --- a/src/config/toml_writer.c +++ b/src/config/toml_writer.c @@ -31,6 +31,12 @@ bool writeFTLtoml(const bool verbose) if(config.misc.readOnly.v.b) { log_debug(DEBUG_CONFIG, "Config file is read-only, not writing"); + + // We need to (re-)calculate the checksum here as it'd otherwise + // be outdated (in non-read-only mode, it's calculated at the + // end of this function) + if(!sha256sum(GLOBALTOMLPATH, last_checksum)) + log_err("Unable to create checksum of %s", GLOBALTOMLPATH); return true; } From 2406e1a70ee9ef83b98ed672df2705d02b016da2 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 6 Jun 2024 20:42:06 +0200 Subject: [PATCH 145/339] Check if the newly set password is the same as the old one Signed-off-by: DL6ER --- src/config/password.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/config/password.c b/src/config/password.c index 5c32411c..4ecf7169 100644 --- a/src/config/password.c +++ b/src/config/password.c @@ -601,6 +601,13 @@ int run_performance_test(void) bool set_and_check_password(struct conf_item *conf_item, const char *password) { + // Check if the newly set password is the same as the old one + if(verify_password(password, config.webserver.api.pwhash.v.s, false) == PASSWORD_CORRECT) + { + log_debug(DEBUG_CONFIG, "Password unchanged, not updating"); + return true; + } + // Get password hash as allocated string (an empty string is hashed to an empty string) char *pwhash = strlen(password) > 0 ? create_password(password) : strdup(""); From 93f751f90a278facc08a48cb87ac60c777d36853 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 7 Jun 2024 18:49:11 +0200 Subject: [PATCH 146/339] Use adjtimex instead of adjtime as the latter uses the former (see http://git.musl-libc.org/cgit/musl/tree/src/linux/adjtime.c and https://codebrowser.dev/glibc/glibc/time/adjtime.c.html). Also add comment from man rtc(4) about how RTCs are updated at the same time Signed-off-by: DL6ER --- src/ntp/client.c | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/src/ntp/client.c b/src/ntp/client.c index 46cc9e0b..3b331062 100644 --- a/src/ntp/client.c +++ b/src/ntp/client.c @@ -35,6 +35,8 @@ #include "daemon.h" // thread_names[] #include "signals.h" +// adjtimex() +#include struct ntp_sync { uint64_t org; @@ -151,6 +153,7 @@ static bool settime_skew(const double offset) { // This function gradually adjusts the system clock. // + // Linux uses David L. Mills' clock adjustment algorithm (see RFC 5905). // If the adjustment in delta is positive, then the system clock is // speeded up by some small percentage (i.e., by adding a small amount // of time to the clock value in each second) until the adjustment has @@ -160,23 +163,30 @@ static bool settime_skew(const double offset) // If a clock adjustment from an earlier adjtime() call is already in // progress at the time of a later adjtime() call, and delta is not NULL // for the later call, then the earlier adjustment is stopped, but any - // al‐ ready completed part of that adjustment is not undone. + // already completed part of that adjustment is not undone. // - // The adjustment that adjtime() makes to the clock is carried out in + // The adjustment that adjtimex() makes to the clock is carried out in // such a manner that the clock is always monotonically increasing. - // Using adjtime() to adjust the time prevents the problems that can be + // Using adjtimex() to adjust the time prevents the problems that can be // caused for certain applications (e.g., make(1)) by abrupt positive or // negative jumps in the system time. // - // adjtime() is intended to be used to make small adjustments to the + // adjtimex() is intended to be used to make small adjustments to the // system time. The actual time adjustment rate is implementation-specific // but is typically on the order of 500 ppm, i.e., 0.5 ms/s. - struct timeval tx; - tx.tv_sec = (long int)offset; - tx.tv_usec = (offset - tx.tv_sec) * 1e6; - log_debug(DEBUG_NTP, "Gradually adjusting system time by %.3f ms", 1e3 * offset); + // + // man rtc(4) adds: + // When the kernel's system time is synchronized with an external + // reference using adjtimex() it will update a designated RTC + // periodically every 11 minutes. - if(adjtime(&tx, NULL) < 0) + struct timex tx = { 0 }; + tx.offset = 1000000 * offset; + tx.modes = ADJ_OFFSET_SINGLESHOT; + + log_debug(DEBUG_NTP, "Gradually adjusting system time by %ld us", tx.offset); + + if(adjtimex(&tx) < 0) { log_err("Failed to adjust time: %s", errno == EPERM ? "Insufficient permissions, try running with sudo" : strerror(errno)); From 671771ceb0fde2bd4ec2f5b9e6bc14fc269aecef Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 7 Jun 2024 18:50:14 +0200 Subject: [PATCH 147/339] Improve NTP synchronization by rejecting synchronization if the standard deviation of the time offset or round-trip delay is larger than 1 second. This ensures the time cannot go off even in cases where the network connectivity is really bad Signed-off-by: DL6ER --- src/ntp/client.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/ntp/client.c b/src/ntp/client.c index 3b331062..b860a90b 100644 --- a/src/ntp/client.c +++ b/src/ntp/client.c @@ -418,6 +418,15 @@ bool ntp_client(const char *server, const bool settime, const bool print) log_info("Average time offset: (%e +/- %e s)", theta_avg, theta_stdev); log_info("Average round-trip delay: (%e +/- %e s)", delta_avg, delta_stdev); + // Reject synchronization if the standard deviation of the time offset + // or round-trip delay is larger than 1 second + if(theta_stdev > 1.0 || delta_stdev > 1.0) + { + log_warn("Standard deviation of time offset is too large, rejecting synchronization"); + free(ntp); + return false; + } + // Compute trimmed mean (average excluding outliers) double theta_trim = 0.0, delta_trim = 0.0; unsigned int trim = 0; From 791e3a80979655e3ab8e80415b702c02a3fbdf37 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 7 Jun 2024 20:01:04 +0200 Subject: [PATCH 148/339] Add RTC synchronization Signed-off-by: DL6ER --- src/api/docs/content/specs/config.yaml | 13 ++ src/config/config.c | 19 ++ src/config/config.h | 5 + src/ntp/CMakeLists.txt | 1 + src/ntp/client.c | 4 + src/ntp/ntp.h | 3 + src/ntp/rtc.c | 296 +++++++++++++++++++++++++ test/pihole.toml | 15 +- 8 files changed, 355 insertions(+), 1 deletion(-) create mode 100644 src/ntp/rtc.c diff --git a/src/api/docs/content/specs/config.yaml b/src/api/docs/content/specs/config.yaml index 9901324d..dfc4ae7d 100644 --- a/src/api/docs/content/specs/config.yaml +++ b/src/api/docs/content/specs/config.yaml @@ -354,6 +354,15 @@ components: type: integer count: type: integer + rtc: + type: object + properties: + set: + type: boolean + device: + type: string + utc: + type: boolean resolver: type: object properties: @@ -700,6 +709,10 @@ components: server: "pool.ntp.org" interval: 3600 count: 8 + rtc: + set: true + device: "" + utc: true resolver: resolveIPv4: true resolveIPv6: true diff --git a/src/config/config.c b/src/config/config.c index 19b17086..d1e51857 100644 --- a/src/config/config.c +++ b/src/config/config.c @@ -851,6 +851,25 @@ void initConfig(struct config *conf) conf->ntp.sync.count.d.ui = 8; conf->ntp.sync.count.c = validate_stub; // Only type-based checking + conf->ntp.rtc.set.k = "ntp.rtc.set"; + conf->ntp.rtc.set.h = "Should FTL update a real-time clock (RTC) if available?"; + conf->ntp.rtc.set.t = CONF_BOOL; + conf->ntp.rtc.set.d.b = true; + conf->ntp.rtc.set.c = validate_stub; // Only type-based checking + + conf->ntp.rtc.device.k = "ntp.rtc.device"; + conf->ntp.rtc.device.h = "Path to the RTC device to update. Leave empty for auto-discovery"; + conf->ntp.rtc.device.a = cJSON_CreateStringReference("Path to the RTC device, e.g., \"/dev/rtc0\""); + conf->ntp.rtc.device.t = CONF_STRING; + conf->ntp.rtc.device.d.s = (char*)""; + conf->ntp.rtc.device.c = validate_stub; // Only type-based checking + + conf->ntp.rtc.utc.k = "ntp.rtc.utc"; + conf->ntp.rtc.utc.h = "Should the RTC be set to UTC?"; + conf->ntp.rtc.utc.t = CONF_BOOL; + conf->ntp.rtc.utc.d.b = true; + conf->ntp.rtc.utc.c = validate_stub; // Only type-based checking + // struct resolver conf->resolver.resolveIPv6.k = "resolver.resolveIPv6"; diff --git a/src/config/config.h b/src/config/config.h index 88bdefd9..3a68fa8e 100644 --- a/src/config/config.h +++ b/src/config/config.h @@ -205,6 +205,11 @@ struct config { struct conf_item interval; struct conf_item count; } sync; + struct { + struct conf_item set; + struct conf_item device; + struct conf_item utc; + } rtc; } ntp; struct { diff --git a/src/ntp/CMakeLists.txt b/src/ntp/CMakeLists.txt index 7eca589a..5cdc5d12 100644 --- a/src/ntp/CMakeLists.txt +++ b/src/ntp/CMakeLists.txt @@ -11,6 +11,7 @@ set(ntp_sources server.c client.c + rtc.c ntp.h ) diff --git a/src/ntp/client.c b/src/ntp/client.c index b860a90b..a23b37ae 100644 --- a/src/ntp/client.c +++ b/src/ntp/client.c @@ -480,6 +480,10 @@ bool ntp_client(const char *server, const bool settime, const bool print) // Return early if time could not be set if(!success) return false; + + // Finally, adjust RTC if configured + if(config.ntp.rtc.set.v.b) + ntp_sync_rtc(); } // Offset and delay larger than 0.1 seconds are considered as invalid diff --git a/src/ntp/ntp.h b/src/ntp/ntp.h index 3fde4c07..b87a7878 100644 --- a/src/ntp/ntp.h +++ b/src/ntp/ntp.h @@ -35,6 +35,9 @@ bool ntp_client(const char *server, const bool settime, const bool print); // Start NTP sync thread bool ntp_start_sync_thread(pthread_attr_t *attr); +// Sync RTC time +bool ntp_sync_rtc(void); + // Number of NTP queries to average. The more queries, the more accurate the // time, but the longer it takes to synchronize. The minimum is 1. #define NTP_AVERGAGE_COUNT 8 diff --git a/src/ntp/rtc.c b/src/ntp/rtc.c new file mode 100644 index 00000000..abeb457e --- /dev/null +++ b/src/ntp/rtc.c @@ -0,0 +1,296 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2024 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* Real Time Clock (RTC) functions +* The routines in this file have been inspired by man pages +* and the source of the hwclock which is part of the util-linux +* project (https://github.com/util-linux/util-linux/) +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ + +#include "ntp/ntp.h" + +// ioctl() +#include +// RTC +#include +// O_WRONLY +#include +// struct config +#include "config/config.h" + +// List of RTC devices from +// https://github.com/util-linux/util-linux/blob/41e7686c9ad1ea7892b9d8941c266869bf6a28dd/sys-utils/hwclock-rtc.c#L85-L93 +static const char * const rtc_devices[] = { +#ifdef __ia64__ + "/dev/efirtc", + "/dev/misc/efirtc", +#endif + "/dev/rtc0", + "/dev/rtc", + "/dev/misc/rtc" +}; + +static void print_tm_time(const char *label, const struct tm *tm) +{ + char timestr[TIMESTR_SIZE] = { 0 }; + strftime(timestr, sizeof(timestr), "%Y-%m-%d %H:%M:%S", tm); + log_info("%s %s", label, timestr); +} + +// Try to find the RTC device and open it +static int open_rtc(void) +{ + int rtc_fd = -1; + + // Get current user's UID and GID + const uid_t uid = getuid(); + const gid_t gid = getgid(); + + // If the user has specified an RTC device, try to open it + if(config.ntp.rtc.device.v.s != NULL && + strlen(config.ntp.rtc.device.v.s) > 0) + { + // Open the RTC device + rtc_fd = open(config.ntp.rtc.device.v.s, O_RDONLY); + if (rtc_fd != -1) + { + log_debug(DEBUG_NTP, "Successfully opened RTC at \"%s\"", + config.ntp.rtc.device.v.s); + return rtc_fd; + } + + // If the open failed because of permissions, try to change them + // momentarily. On some embedded systems, the RTC device is owned by + // root exclusively and users do not have permission to even open it. + // Without being able to access the RTC, the capability to set the + // time (CAP_SYS_TIME) is useless. + if(errno == EACCES) + { + // Get current owner of the device + struct stat st = { 0 }; + if(stat(config.ntp.rtc.device.v.s, &st) == -1) + { + log_debug(DEBUG_NTP, "stat(\"%s\") failed: %s", + config.ntp.rtc.device.v.s, strerror(errno)); + return -1; + } + + if(chown(config.ntp.rtc.device.v.s, uid, gid) == -1) + { + log_debug(DEBUG_NTP, "chown(\"%s\", %u, %u) failed: %s", + config.ntp.rtc.device.v.s, uid, gid, strerror(errno)); + return -1; + } + + rtc_fd = open(config.ntp.rtc.device.v.s, O_RDONLY); + if (rtc_fd != -1) + { + log_debug(DEBUG_NTP, "Successfully opened RTC at \"%s\"", + config.ntp.rtc.device.v.s); + } + + // Chown the device back to the original owner + if(chown(config.ntp.rtc.device.v.s, st.st_uid, st.st_gid) == -1) + { + log_debug(DEBUG_NTP, "chown(\"%s\", %u, %u) failed: %s", + config.ntp.rtc.device.v.s, st.st_uid, st.st_gid, strerror(errno)); + return -1; + } + + // Return the RTC file descriptor (can be -1) + return rtc_fd; + } + + log_debug(DEBUG_NTP, "Failed to open RTC at \"%s\": %s", + config.ntp.rtc.device.v.s, strerror(errno)); + + return -1; + } + + // If the user has not specified an RTC device, try to open the default + // ones + for(size_t i = 0; i < ArraySize(rtc_devices); i++) + { + rtc_fd = open(rtc_devices[i], O_RDONLY); + if (rtc_fd != -1) + { + log_debug(DEBUG_NTP, "Successfully opened RTC at \"%s\"", + rtc_devices[i]); + break; + } + + // If the open failed because of permissions, try to change them + // momentarily + if(errno == EACCES) + { + // Get current owner of the device + struct stat st = { 0 }; + if(stat(rtc_devices[i], &st) == -1) + { + log_debug(DEBUG_NTP, "stat(\"%s\") failed: %s", + rtc_devices[i], strerror(errno)); + return -1; + } + + if(chown(rtc_devices[i], uid, gid) == -1) + { + log_debug(DEBUG_NTP, "chown(\"%s\", %u, %u) failed: %s", + rtc_devices[i], uid, gid, strerror(errno)); + return -1; + } + + rtc_fd = open(rtc_devices[i], O_RDONLY); + if (rtc_fd != -1) + { + log_debug(DEBUG_NTP, "Successfully opened RTC at \"%s\"", + rtc_devices[i]); + } + + // Chown the device back to the original owner + if(chown(rtc_devices[i], st.st_uid, st.st_gid) == -1) + { + log_debug(DEBUG_NTP, "chown(\"%s\", %u, %u) failed: %s", + rtc_devices[i], st.st_uid, st.st_gid, strerror(errno)); + return -1; + } + + // Return the RTC file descriptor (can be -1) + return rtc_fd; + } + + log_debug(DEBUG_NTP, "Failed to open RTC at \"%s\": %s", + rtc_devices[i], strerror(errno)); + } + + return rtc_fd; +} + +static bool read_rtc(struct tm *tm) +{ + // Open the RTC device + const int rtc_fd = open_rtc(); + if(rtc_fd == -1) + return false; + + // Read the RTC time + struct rtc_time rtc_tm = { 0 }; + const int rc = ioctl(rtc_fd, RTC_RD_TIME, &rtc_tm); + if(rc == -1) + { + log_debug(DEBUG_NTP, "ioctl(RTC_RD_NAME) failed: %s", + strerror(errno)); + close(rtc_fd); + return false; + } + + // Convert the kernel's struct tm to the standard struct tm + tm->tm_sec = rtc_tm.tm_sec; + tm->tm_min = rtc_tm.tm_min; + tm->tm_hour = rtc_tm.tm_hour; + tm->tm_mday = rtc_tm.tm_mday; + tm->tm_mon = rtc_tm.tm_mon; + tm->tm_year = rtc_tm.tm_year; + tm->tm_wday = rtc_tm.tm_wday; + tm->tm_yday = rtc_tm.tm_yday; + tm->tm_isdst = -1; // the RTC does not provide this information + print_tm_time("Current RTC time is", tm); + + // Close the RTC device + close(rtc_fd); + + return true; +} + +// Set the Hardware Clock to the broken down time . +// Use ioctls to "rtc" device to set the time. +static bool set_rtc(const struct tm *new_time) +{ + // Open the RTC device + const int rtc_fd = open_rtc(); + if(rtc_fd == -1) + return false; + + // Set the RTC time from the broken down time + struct rtc_time rtc_tm = { 0 }; + rtc_tm.tm_sec = new_time->tm_sec; + rtc_tm.tm_min = new_time->tm_min; + rtc_tm.tm_hour = new_time->tm_hour; + rtc_tm.tm_mday = new_time->tm_mday; + rtc_tm.tm_mon = new_time->tm_mon; + rtc_tm.tm_year = new_time->tm_year; + rtc_tm.tm_wday = new_time->tm_wday; + rtc_tm.tm_yday = new_time->tm_yday; + rtc_tm.tm_isdst = new_time->tm_isdst; + + // Set the RTC time + const int rc = ioctl(rtc_fd, RTC_SET_TIME, &rtc_tm); + if(rc == -1) + { + log_debug(DEBUG_NTP, "ioctl(RTC_SET_TIME) failed: %s", + strerror(errno)); + close(rtc_fd); + return false; + } + print_tm_time("RTC time set to", new_time); + + // Close the RTC device + close(rtc_fd); + return true; +} + +bool ntp_sync_rtc(void) +{ + // Wait until the beginning of the next second as the RTC only has a + // resolution of one second + struct timespec ts = { 0 }; + clock_gettime(CLOCK_REALTIME, &ts); + ts.tv_sec++; + ts.tv_nsec = 0; + clock_nanosleep(CLOCK_REALTIME, TIMER_ABSTIME, &ts, NULL); + + // Time to which we will set Hardware Clock, in broken down format + struct tm new_time = { 0 }; + const time_t newtime = time(NULL); + if(config.ntp.rtc.utc.v.b) + // UTC + gmtime_r(&newtime, &new_time); + else + // Local time + localtime_r(&newtime, &new_time); + + // Read the current time from the RTC + struct tm rtc_time = { 0 }; + if(!read_rtc(&rtc_time)) + { + log_debug(DEBUG_NTP, "Failed to read RTC time"); + return false; + } + + // If the RTC time is the same as the current time, we don't need to set + // it. We don't use memcmp() here because the tm struct may contain + // additional fields that are not filled in by the RTC (e.g. tm_isdst). + if(rtc_time.tm_sec == new_time.tm_sec && + rtc_time.tm_min == new_time.tm_min && + rtc_time.tm_hour == new_time.tm_hour && + rtc_time.tm_mday == new_time.tm_mday && + rtc_time.tm_mon == new_time.tm_mon && + rtc_time.tm_year == new_time.tm_year) + { + // The RTC time is already correct, return early + log_debug(DEBUG_NTP, "RTC time is already correct"); + return true; + } + + // Set the RTC time + if(!set_rtc(&new_time)) + { + log_debug(DEBUG_NTP, "Failed to set RTC time"); + return false; + } + + return true; +} diff --git a/test/pihole.toml b/test/pihole.toml index d29dc427..681e3a0c 100644 --- a/test/pihole.toml +++ b/test/pihole.toml @@ -495,6 +495,19 @@ # Number of NTP syncs to perform and average before updating the system time count = 8 + [ntp.rtc] + # Should FTL update a real-time clock (RTC) if available? + set = true + + # Path to the RTC device to update. Leave empty for auto-discovery + # + # Possible values are: + # Path to the RTC device, e.g., "/dev/rtc0" + device = "" + + # Should the RTC be set to UTC? + utc = true + [resolver] # Should FTL try to resolve IPv4 addresses to hostnames? resolveIPv4 = false ### CHANGED, default = true @@ -1077,7 +1090,7 @@ all = true ### CHANGED, default = false # Configuration statistics: -# 144 total entries out of which 89 entries are default +# 147 total entries out of which 92 entries are default # --> 55 entries are modified # 2 entries are forced through environment: # - misc.nice From a05bf8dd24439f5faca14b4b0aeca91b059eff5a Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 10 Jun 2024 16:57:11 +0200 Subject: [PATCH 149/339] Improve shutdown sequence of threads Signed-off-by: DL6ER --- src/daemon.c | 10 +++++++--- src/ntp/client.c | 3 +-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/daemon.c b/src/daemon.c index 0345e046..f329f1ae 100644 --- a/src/daemon.c +++ b/src/daemon.c @@ -272,9 +272,14 @@ static void terminate_threads(void) log_info("Waiting for threads to join"); for(int i = 0; i < THREADS_MAX; i++) { + log_debug(DEBUG_EXTRA, "Joining %s thread (%d)", thread_names[i], i); // Skip threads that have never been started or which are already stopped - if(!thread_running[i]) + if(threads[i] == 0 || !thread_running[i]) + { + log_debug(DEBUG_EXTRA, "Skipping thread as it %s", + threads[i] == 0 ? "was never started" : "is not running"); continue; + } // Cancel thread if it is idle if(thread_cancellable[i]) @@ -297,8 +302,7 @@ static void terminate_threads(void) ts.tv_sec += 2; // Try to join thread and cancel it if it is still busy - const int s = pthread_timedjoin_np(threads[i], NULL, &ts); - if(s != 0) + if(pthread_timedjoin_np(threads[i], NULL, &ts) != 0) { log_info("Thread %s (%d) is still busy, cancelling it.", thread_names[i], i); diff --git a/src/ntp/client.c b/src/ntp/client.c index a23b37ae..4f253cd2 100644 --- a/src/ntp/client.c +++ b/src/ntp/client.c @@ -496,8 +496,7 @@ static void *ntp_client_thread(void *arg) // Set thread name thread_names[NTP] = "ntp-client"; thread_running[NTP] = true; - prctl(PR_SET_NAME, thread_names[DB], 0, 0, 0); - pthread_setname_np(pthread_self(), "NTP sync"); + prctl(PR_SET_NAME, thread_names[NTP], 0, 0, 0); // Run NTP client while(!killed) From 27db8a43cec224f04a057a2cb65addc7e8ebffa5 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 10 Jun 2024 17:12:09 +0200 Subject: [PATCH 150/339] Copy root delay/dispersion errors from upstream server (after first upstream NTP synchronization) Signed-off-by: DL6ER --- src/ntp/client.c | 46 ++++++++++++++++++++++++++++++++++++---------- src/ntp/ntp.h | 4 ++++ src/ntp/server.c | 26 ++++++++++++++------------ 3 files changed, 54 insertions(+), 22 deletions(-) diff --git a/src/ntp/client.c b/src/ntp/client.c index 4f253cd2..dc3db113 100644 --- a/src/ntp/client.c +++ b/src/ntp/client.c @@ -122,24 +122,29 @@ void print_debug_time(const char *label, const uint32_t *u32p, const uint64_t nt (timevar >> 32) & 0xFFFFFFFF, timevar & 0xFFFFFFFF, time_str); } -static bool settime_step(const double offset) +static uint64_t get_new_time(struct timeval *unix_time, const double offset) { // Get current time - struct timeval unix_time; - gettimeofday(&unix_time, NULL); + gettimeofday(unix_time, NULL); // Convert from double to native format (signed) and add to the // current time. Note the addition is done in native format to // avoid overflow or loss of precision. - const uint64_t ntp_time = U2LFP(unix_time) + D2LFP(offset); + const uint64_t ntp_time = U2LFP(*unix_time) + D2LFP(offset); // Convert NTP to native format - unix_time.tv_sec = NTPtoSEC(ntp_time); - unix_time.tv_usec = NTPtoUSEC(ntp_time); + unix_time->tv_sec = NTPtoSEC(ntp_time); + unix_time->tv_usec = NTPtoUSEC(ntp_time); + + return ntp_time; +} + +static bool settime_step(struct timeval *unix_time, const double offset) +{ log_debug(DEBUG_NTP, "Stepping system time by %e s", offset); // Set time immediately - if(settimeofday(&unix_time, NULL) != 0) + if(settimeofday(unix_time, NULL) != 0) { log_err("Failed to set time: %s", errno == EPERM ? "Insufficient permissions, try running with sudo" : strerror(errno)); @@ -221,9 +226,16 @@ static bool reply(int fd, struct ntp_sync *ntp, const bool verbose) // Compute precision of server clock in seconds 2^rho ntp->precision = pow(2, rho); + // Get root delay and dispersion (in network-byte-order !) + memcpy(&ntp_root_delay, &buf[4], sizeof(ntp_root_delay)); + memcpy(&ntp_root_dispersion, &buf[8], sizeof(ntp_root_dispersion)); + // Extract Transmit Timestamp - // org = Origin Timestamp (Transmit Timestamp @ Client) uint64_t netbuffer; + // ref = Reference Timestamp (Time at which the clock was last set or corrected) + memcpy(&netbuffer, &buf[16], sizeof(netbuffer)); + const uint64_t ref = ntoh64(netbuffer); + // org = Origin Timestamp (Transmit Timestamp @ Client) memcpy(&netbuffer, &buf[24], sizeof(netbuffer)); const uint64_t org = ntoh64(netbuffer); // rec = Receive Timestamp (Receive Timestamp @ Server) @@ -287,6 +299,9 @@ static bool reply(int fd, struct ntp_sync *ntp, const bool verbose) if(!config.debug.ntp.v.b) return true; + // Print current time at server + print_debug_time("Server reference time", NULL, ref); + // Print current time at client print_debug_time("Current time at client", NULL, dst); @@ -296,6 +311,10 @@ static bool reply(int fd, struct ntp_sync *ntp, const bool verbose) // Print offset and delay log_debug(DEBUG_NTP, "Time offset: %e s", ntp->theta); log_debug(DEBUG_NTP, "Round-trip delay: %e s", ntp->delta); + const uint32_t root_delay = ntohl(ntp_root_delay); + log_debug(DEBUG_NTP, "Root delay: %e s", LFP2D(root_delay)); + const uint32_t root_dispersion = ntohl(ntp_root_dispersion); + log_debug(DEBUG_NTP, "Root dispersion: %e s", LFP2D(root_dispersion)); return true; } @@ -398,7 +417,7 @@ bool ntp_client(const char *server, const bool settime, const bool print) free(ntp); return false; } - log_info("Received %u/%u valid NTP replies", valid, count); + log_info("Received %u/%u valid NTP replies from %s", valid, count, server); theta_avg /= valid; delta_avg /= valid; @@ -466,6 +485,10 @@ bool ntp_client(const char *server, const bool settime, const bool print) // Set time if requested if(settime) { + // Calculate corrected time + struct timeval unix_time; + const uint64_t ntp_time = get_new_time(&unix_time, theta_trim); + // If the clock deviates more than 0.5 seconds from the NTP server, // the time is updated immediately. Otherwise, the time is updated // gradually to avoid sudden jumps in the system clock. @@ -473,7 +496,7 @@ bool ntp_client(const char *server, const bool settime, const bool print) // since Linux 2.6.26, see man ntp_adjtime(2) for details. bool success; if(fabs(theta_trim) > 0.5) - success = settime_step(theta_trim); + success = settime_step(&unix_time, theta_trim); else success = settime_skew(theta_trim); @@ -481,6 +504,9 @@ bool ntp_client(const char *server, const bool settime, const bool print) if(!success) return false; + // Update last NTP sync time + ntp_last_sync = ntp_time; + // Finally, adjust RTC if configured if(config.ntp.rtc.set.v.b) ntp_sync_rtc(); diff --git a/src/ntp/ntp.h b/src/ntp/ntp.h index b87a7878..d8dbf451 100644 --- a/src/ntp/ntp.h +++ b/src/ntp/ntp.h @@ -65,6 +65,10 @@ bool ntp_sync_rtc(void); #define hton64(x) ((((uint64_t)htonl(x)) << 32) + htonl((x) >> 32)) #define ntoh64(x) ((((uint64_t)ntohl(x)) << 32) + ntohl((x) >> 32)) +extern uint64_t ntp_last_sync; +extern uint32_t ntp_root_delay; +extern uint32_t ntp_root_dispersion; + #endif // NTP_H diff --git a/src/ntp/server.c b/src/ntp/server.c index 294ff005..048a4bdf 100644 --- a/src/ntp/server.c +++ b/src/ntp/server.c @@ -38,6 +38,10 @@ // PRIi64 #include +uint64_t ntp_last_sync = 0u; +uint32_t ntp_root_delay = 0u; +uint32_t ntp_root_dispersion = 0u; + // RFC 5905 Appendix A.4: Kernel System Clock Interface uint64_t gettime64(void) { @@ -100,11 +104,12 @@ static bool ntp_reply(const int socket_fd, const struct sockaddr *saddr_p, const // | Root Dispersion | // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - // Assume Root Delay (total roundtrip delay to the primary reference - // source) = 0, Root Dispersion (the nominal error relative to the - // primary reference source) = 0 as we don't have these numbers - *u32p++ = 0.0; - *u32p++ = 0.0; + // Set Root Delay (total roundtrip delay to the primary reference + // source) and Root Dispersion (the nominal error relative to the + // primary reference source) to the values obtained from the upstream + // NTP server. These values are already in network byte order. + *u32p++ = ntp_root_delay; + *u32p++ = ntp_root_dispersion; // 0 1 2 3 // 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 @@ -115,7 +120,7 @@ static bool ntp_reply(const int socket_fd, const struct sockaddr *saddr_p, const // Reference ID = 'LOCL" (LOCAL CLOCK) // A four-octet, left-justified, zero-padded ASCII string assigned to // the reference clock - memcpy(u32p++, "LOCL", 4); + memcpy(u32p++, "LOCL", sizeof(uint32_t)); // 0 1 2 3 // 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 @@ -126,12 +131,9 @@ static bool ntp_reply(const int socket_fd, const struct sockaddr *saddr_p, const // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ // Time when the system clock was last set or corrected, in NTP - // timestamp format. As this is not a stratum 1 server, we don't have - // a hardware clock to set this value. - // A stateless server copies T3 and T4 from the client packet to T1 and - // T2 of the server packet and tacks on the transmit timestamp T3 before - // sending it to the client. - memcpy(u32p, &u32r[8], sizeof(uint64_t)); + // timestamp format. + const uint64_t last_sync = hton64(ntp_last_sync); + memcpy(u32p, &last_sync, sizeof(uint64_t)); if(config.debug.ntp.v.b) print_debug_time("Reference Timestamp", u32p, 0); u32p += 2; From fcc0a5ab2f19528b05817a3c37b7d14107d52ab3 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 10 Jun 2024 17:18:41 +0200 Subject: [PATCH 151/339] Use "fresh" sockets for NTP client requests to avoid reusing the same ephermal port for mulitple requests Signed-off-by: DL6ER --- src/ntp/client.c | 54 ++++++++++++++++++++++++++++++------------------ 1 file changed, 34 insertions(+), 20 deletions(-) diff --git a/src/ntp/client.c b/src/ntp/client.c index dc3db113..d2e22879 100644 --- a/src/ntp/client.c +++ b/src/ntp/client.c @@ -319,16 +319,15 @@ static bool reply(int fd, struct ntp_sync *ntp, const bool verbose) return true; } -bool ntp_client(const char *server, const bool settime, const bool print) +static int getsock(const struct addrinfo *saddr) { - const int protocol = strchr(server, ':') != NULL ? AF_INET6 : AF_INET; - // Create UDP socket + const int protocol = saddr->ai_addrlen == sizeof(struct sockaddr_in6) ? AF_INET6 : AF_INET; const int s = socket(protocol, SOCK_DGRAM, IPPROTO_UDP); if(s == -1) { log_err("Cannot create UDP socket"); - return false; + return -1; } // Set socket timeout to 5 seconds @@ -339,16 +338,7 @@ bool ntp_client(const char *server, const bool settime, const bool print) { log_err("Cannot set socket timeout"); close(s); - return false; - } - - // Resolve server address - struct addrinfo *saddr; - if(getaddrinfo(server, "ntp", NULL, &saddr) != 0) - { - log_err("Cannot resolve NTP server address"); - close(s); - return false; + return -1; } // Set address to send to/receive from @@ -356,32 +346,56 @@ bool ntp_client(const char *server, const bool settime, const bool print) { log_err("Cannot connect to NTP server"); close(s); + return -1; + } + + // Return socket + return s; +} + +bool ntp_client(const char *server, const bool settime, const bool print) +{ + // Resolve server address + struct addrinfo *saddr; + if(getaddrinfo(server, "ntp", NULL, &saddr) != 0) + { + log_err("Cannot resolve NTP server address"); return false; } - freeaddrinfo(saddr); - // Send and receive NTP packets const unsigned int count = config.ntp.sync.count.v.ui; struct ntp_sync *ntp = calloc(count, sizeof(struct ntp_sync)); if(ntp == NULL) { log_err("Cannot allocate memory for NTP client"); - close(s); return false; } - memset(ntp, 0, count*sizeof(*ntp)); + + // Send and receive NTP packets for(unsigned int i = 0; i < count; i++) { + // Create socket + const int s = getsock(saddr); + if(s == -1) + continue; + // Send request if(!request(s, &ntp[i])) { close(s); free(ntp); + freeaddrinfo(saddr); return false; } // Get reply if(!reply(s, &ntp[i], false)) + { + close(s); continue; + } + + // Close socket + close(s); // Sleep for some time to avoid flooding the server if(print) @@ -392,8 +406,8 @@ bool ntp_client(const char *server, const bool settime, const bool print) if(print) printf("\n"); - // Close socket - close(s); + // Free allocated memory + freeaddrinfo(saddr); // Compute average and standard deviation unsigned int valid = 0; From d737bf524cbe341033950eb0027d346a6eaa523c Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 10 Jun 2024 17:23:58 +0200 Subject: [PATCH 152/339] Determine root dispersion and error based on our own most recent time synchronization as described by RFC 5905, Scn. 4 (page 9) Signed-off-by: DL6ER --- src/ntp/client.c | 13 +++++++++---- src/ntp/ntp.h | 3 +++ src/ntp/server.c | 6 +++--- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/src/ntp/client.c b/src/ntp/client.c index d2e22879..817f701f 100644 --- a/src/ntp/client.c +++ b/src/ntp/client.c @@ -226,10 +226,6 @@ static bool reply(int fd, struct ntp_sync *ntp, const bool verbose) // Compute precision of server clock in seconds 2^rho ntp->precision = pow(2, rho); - // Get root delay and dispersion (in network-byte-order !) - memcpy(&ntp_root_delay, &buf[4], sizeof(ntp_root_delay)); - memcpy(&ntp_root_dispersion, &buf[8], sizeof(ntp_root_dispersion)); - // Extract Transmit Timestamp uint64_t netbuffer; // ref = Reference Timestamp (Time at which the clock was last set or corrected) @@ -521,6 +517,15 @@ bool ntp_client(const char *server, const bool settime, const bool print) // Update last NTP sync time ntp_last_sync = ntp_time; + // Compute our server's root dispersion and delay + // Both quantities are the maximum error and maximum delay of + // the server's time relative to the reference time. The root + // dispersion is the maximum error of the server's time relative + // to the reference time, while the root delay is the maximum + // delay of the server's time relative to the reference time + ntp_root_delay = D2FP(theta_trim); + ntp_root_dispersion = D2FP(theta_stdev); + // Finally, adjust RTC if configured if(config.ntp.rtc.set.v.b) ntp_sync_rtc(); diff --git a/src/ntp/ntp.h b/src/ntp/ntp.h index d8dbf451..72445423 100644 --- a/src/ntp/ntp.h +++ b/src/ntp/ntp.h @@ -51,6 +51,9 @@ bool ntp_sync_rtc(void); #define DIFF_SEC_1970_2036 (2085978496UL) // Timestamp conversion macroni (RFC 5905, Appendix A) +#define FRIC 65536. // 2^16 as a double +#define D2FP(r) ((uint32_t)((r) * FRIC)) // NTP short +#define FP2D(r) ((double)(r) / FRIC) #define FRAC 4294967296. // 2^32 as double #define D2LFP(a) ((uint64_t)((a) * FRAC)) // NTP timestamp #define LFP2D(a) ((double)(a) / FRAC) diff --git a/src/ntp/server.c b/src/ntp/server.c index 048a4bdf..87b34831 100644 --- a/src/ntp/server.c +++ b/src/ntp/server.c @@ -107,9 +107,9 @@ static bool ntp_reply(const int socket_fd, const struct sockaddr *saddr_p, const // Set Root Delay (total roundtrip delay to the primary reference // source) and Root Dispersion (the nominal error relative to the // primary reference source) to the values obtained from the upstream - // NTP server. These values are already in network byte order. - *u32p++ = ntp_root_delay; - *u32p++ = ntp_root_dispersion; + // NTP server. + *u32p++ = htonl(ntp_root_delay); + *u32p++ = htonl(ntp_root_dispersion); // 0 1 2 3 // 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 From b8eee89e07117d6ede18ff69eecc1aea7503115a Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 10 Jun 2024 19:35:40 +0200 Subject: [PATCH 153/339] Do not detach threads because we want to join them during shutdown Signed-off-by: DL6ER --- src/daemon.c | 3 ++- src/dnsmasq_interface.c | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/daemon.c b/src/daemon.c index f329f1ae..a4d294ab 100644 --- a/src/daemon.c +++ b/src/daemon.c @@ -265,7 +265,6 @@ pid_t FTL_gettid(void) static void terminate_threads(void) { - struct timespec ts; // Terminate threads before closing database connections and finishing shared memory killed = true; // Try to join threads to ensure cancellation has succeeded @@ -290,6 +289,8 @@ static void terminate_threads(void) } // Cancel thread if we cannot set a timeout for joining + struct timespec ts; + memset(&ts, 0, sizeof(ts)); if (clock_gettime(CLOCK_REALTIME, &ts) == -1) { log_info("Thread %s (%d) is busy, cancelling it (cannot set timeout).", diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c index 14a82284..853fba2d 100644 --- a/src/dnsmasq_interface.c +++ b/src/dnsmasq_interface.c @@ -2900,9 +2900,9 @@ void FTL_fork_and_bind_sockets(struct passwd *ent_pw, bool dnsmasq_start) // detached mode pthread_attr_t attr; // Initialize thread attributes object with default attribute values + // Do NOT detach threads as we want to join them during shutdown with a + // fixed timeout to give them time to clean up and finish their work pthread_attr_init(&attr); - // Set thread attributes to detached mode - pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); // Initialize NTP server ntp_server_start(&attr); From ec6750d42f4c2bb50b51ff455ac2c2eefb927859 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 10 Jun 2024 20:06:41 +0200 Subject: [PATCH 154/339] Make definition of __USE_MISC conditional Signed-off-by: DL6ER --- src/zip/gzip.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/zip/gzip.c b/src/zip/gzip.c index 4ec97c74..a192df5a 100644 --- a/src/zip/gzip.c +++ b/src/zip/gzip.c @@ -16,7 +16,9 @@ #include #include // le32toh and friends +#ifndef __USE_MISC #define __USE_MISC +#endif #include static int mz_uncompress2_raw(unsigned char *pDest, mz_ulong *pDest_len, const unsigned char *pSource, mz_ulong *pSource_len); From 9bc0d4c25d38e2694244fef547d9c389f9806928 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 11 Jun 2024 04:18:55 +0200 Subject: [PATCH 155/339] Fix root delay/dispersion debug printing Signed-off-by: DL6ER --- src/ntp/client.c | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/ntp/client.c b/src/ntp/client.c index 817f701f..2028edc0 100644 --- a/src/ntp/client.c +++ b/src/ntp/client.c @@ -226,6 +226,11 @@ static bool reply(int fd, struct ntp_sync *ntp, const bool verbose) // Compute precision of server clock in seconds 2^rho ntp->precision = pow(2, rho); + // Extract root delay and root dispersion of server clock + uint32_t srv_root_delay, srv_root_dispersion; + memcpy(&srv_root_delay, &buf[4], sizeof(srv_root_delay)); + memcpy(&srv_root_dispersion, &buf[8], sizeof(srv_root_dispersion)); + // Extract Transmit Timestamp uint64_t netbuffer; // ref = Reference Timestamp (Time at which the clock was last set or corrected) @@ -307,10 +312,10 @@ static bool reply(int fd, struct ntp_sync *ntp, const bool verbose) // Print offset and delay log_debug(DEBUG_NTP, "Time offset: %e s", ntp->theta); log_debug(DEBUG_NTP, "Round-trip delay: %e s", ntp->delta); - const uint32_t root_delay = ntohl(ntp_root_delay); - log_debug(DEBUG_NTP, "Root delay: %e s", LFP2D(root_delay)); - const uint32_t root_dispersion = ntohl(ntp_root_dispersion); - log_debug(DEBUG_NTP, "Root dispersion: %e s", LFP2D(root_dispersion)); + const uint32_t root_delay = ntohl(srv_root_delay); + log_debug(DEBUG_NTP, "Root delay: %e s", FP2D(root_delay)); + const uint32_t root_dispersion = ntohl(srv_root_dispersion); + log_debug(DEBUG_NTP, "Root dispersion: %e s", FP2D(root_dispersion)); return true; } @@ -539,7 +544,6 @@ bool ntp_client(const char *server, const bool settime, const bool print) static void *ntp_client_thread(void *arg) { // Set thread name - thread_names[NTP] = "ntp-client"; thread_running[NTP] = true; prctl(PR_SET_NAME, thread_names[NTP], 0, 0, 0); From 90ba90a0c760d4a547beceb8f5d7e9bac384f92b Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 11 Jun 2024 04:20:04 +0200 Subject: [PATCH 156/339] Pre-define thread names so they can always be shown during shutdown, even if a thread was never started, remove unused CONF_READER thread slot Signed-off-by: DL6ER --- src/database/database-thread.c | 1 - src/enums.h | 1 - src/gc.c | 1 - src/resolve.c | 1 - src/signals.c | 8 +++++++- src/signals.h | 2 +- 6 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/database/database-thread.c b/src/database/database-thread.c index 81e80484..f8768902 100644 --- a/src/database/database-thread.c +++ b/src/database/database-thread.c @@ -83,7 +83,6 @@ static bool analyze_database(sqlite3 *db) void *DB_thread(void *val) { // Set thread name - thread_names[DB] = "database"; thread_running[DB] = true; prctl(PR_SET_NAME, thread_names[DB], 0, 0, 0); diff --git a/src/enums.h b/src/enums.h index d8977f83..b9ad1443 100644 --- a/src/enums.h +++ b/src/enums.h @@ -250,7 +250,6 @@ enum thread_types { DB, GC, DNSclient, - CONF_READER, TIMER, NTP, THREADS_MAX diff --git a/src/gc.c b/src/gc.c index a142ca23..3d4f0962 100644 --- a/src/gc.c +++ b/src/gc.c @@ -481,7 +481,6 @@ static bool check_files_on_same_device(const char *path1, const char *path2) void *GC_thread(void *val) { // Set thread name - thread_names[GC] = "housekeeper"; thread_running[GC] = true; prctl(PR_SET_NAME, thread_names[GC], 0, 0, 0); diff --git a/src/resolve.c b/src/resolve.c index f77cadfb..453e477e 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -1053,7 +1053,6 @@ static void resolveUpstreams(const bool onlynew) void *DNSclient_thread(void *val) { // Set thread name - thread_names[DNSclient] = "DNS client"; thread_running[DNSclient] = true; prctl(PR_SET_NAME, thread_names[DNSclient], 0, 0, 0); diff --git a/src/signals.c b/src/signals.c index 9c638445..ec3b35f8 100644 --- a/src/signals.c +++ b/src/signals.c @@ -35,7 +35,13 @@ volatile int exit_code = EXIT_SUCCESS; volatile sig_atomic_t thread_cancellable[THREADS_MAX] = { false }; volatile sig_atomic_t thread_running[THREADS_MAX] = { false }; -const char *thread_names[THREADS_MAX] = { "" }; +const char * const thread_names[THREADS_MAX] = { + "database", + "housekeeper", + "DNS client", + "timer", + "NTP client" + }; // Return the (null-terminated) name of the calling thread // The name is stored in the buffer as well as returned for convenience diff --git a/src/signals.h b/src/signals.h index f52fb2d9..76664887 100644 --- a/src/signals.h +++ b/src/signals.h @@ -30,7 +30,7 @@ extern volatile sig_atomic_t want_to_reload_lists; extern volatile sig_atomic_t thread_cancellable[THREADS_MAX]; extern volatile sig_atomic_t thread_running[THREADS_MAX]; -extern const char *thread_names[THREADS_MAX]; +extern const char * const thread_names[THREADS_MAX]; #define BREAK_IF_KILLED() { if(killed) break; } From 2516dcf3e290074547e9fe1cb8db4bbe9b72333b Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 11 Jun 2024 04:23:40 +0200 Subject: [PATCH 157/339] Always join threads if they have ever been started to avoid resource leaking Signed-off-by: DL6ER --- src/daemon.c | 5 ++--- src/signals.c | 4 ++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/daemon.c b/src/daemon.c index a4d294ab..5e7cf2db 100644 --- a/src/daemon.c +++ b/src/daemon.c @@ -273,10 +273,9 @@ static void terminate_threads(void) { log_debug(DEBUG_EXTRA, "Joining %s thread (%d)", thread_names[i], i); // Skip threads that have never been started or which are already stopped - if(threads[i] == 0 || !thread_running[i]) + if(threads[i] == 0) { - log_debug(DEBUG_EXTRA, "Skipping thread as it %s", - threads[i] == 0 ? "was never started" : "is not running"); + log_debug(DEBUG_EXTRA, "Skipping thread as it was never started"); continue; } diff --git a/src/signals.c b/src/signals.c index ec3b35f8..8e14cfe2 100644 --- a/src/signals.c +++ b/src/signals.c @@ -38,9 +38,9 @@ volatile sig_atomic_t thread_running[THREADS_MAX] = { false }; const char * const thread_names[THREADS_MAX] = { "database", "housekeeper", - "DNS client", + "dns-client", "timer", - "NTP client" + "ntp-client" }; // Return the (null-terminated) name of the calling thread From 126d4d87ce84445d3f97a368a29182cc64fe36b8 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 12 Jun 2024 18:21:46 +0200 Subject: [PATCH 158/339] Mark timer thread as running Signed-off-by: DL6ER --- src/timers.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/timers.c b/src/timers.c index fe66d760..17b16704 100644 --- a/src/timers.c +++ b/src/timers.c @@ -84,7 +84,8 @@ void get_blockingmode_timer(double *delay, bool *target_status) void *timer(void *val) { // Set thread name - prctl(PR_SET_NAME, "int.timer", 0, 0, 0); + thread_running[GC] = true; + prctl(PR_SET_NAME, thread_names[TIMER], 0, 0, 0); // Save timestamp as we do not want to store immediately // to the database @@ -105,9 +106,11 @@ void *timer(void *val) set_blockingstatus(timer_target_status); timer_delay = -1.0; } - sleepms(SLEEPING_TIME * 1000); + thread_sleepms(TIMER, SLEEPING_TIME * 1000); } + log_info("Terminating timer thread"); + thread_running[GC] = false; return NULL; } From e70c364af40f71a3669a0a56f1f675f9f44aaa43 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 13 Jun 2024 06:26:28 +0200 Subject: [PATCH 159/339] Add message table entries for selected NTP warnings/errors Signed-off-by: DL6ER --- src/database/message-table.c | 60 ++++++++++++++++++++++++++++++++++++ src/database/message-table.h | 1 + src/enums.h | 1 + src/ntp/client.c | 59 +++++++++++++++++++++++++++-------- src/ntp/server.c | 30 ++++++++++++++---- 5 files changed, 133 insertions(+), 18 deletions(-) diff --git a/src/database/message-table.c b/src/database/message-table.c index 9271bd96..e6dab7de 100644 --- a/src/database/message-table.c +++ b/src/database/message-table.c @@ -97,6 +97,8 @@ static const char *get_message_type_str(const enum message_type type) return "CERTIFICATE_DOMAIN_MISMATCH"; case CONNECTION_ERROR_MESSAGE: return "CONNECTION_ERROR"; + case NTP_MESSAGE: + return "NTP"; case MAX_MESSAGE: default: return "UNKNOWN"; @@ -131,6 +133,8 @@ static enum message_type get_message_type_from_string(const char *typestr) return CERTIFICATE_DOMAIN_MISMATCH_MESSAGE; else if (strcmp(typestr, "CONNECTION_ERROR") == 0) return CONNECTION_ERROR_MESSAGE; + else if (strcmp(typestr, "NTP") == 0) + return NTP_MESSAGE; else return MAX_MESSAGE; } @@ -230,6 +234,14 @@ static unsigned char message_blob_types[MAX_MESSAGE][5] = SQLITE_NULL, // not used SQLITE_NULL, // not used SQLITE_NULL // not used + }, + { + // NTP: The message column contains the warning/error + SQLITE_TEXT, // level (warning/error) + SQLITE_TEXT, // component (server/client) + SQLITE_NULL, // not used + SQLITE_NULL, // not used + SQLITE_NULL // not used } }; // Create message table in the database @@ -900,6 +912,20 @@ static void format_connection_error(char *plain, const int sizeof_plain, char *h free(escaped_server); } +static void format_ntp_message(char *plain, const int sizeof_plain, char *html, const int sizeof_html, + const char *message, const char *level, const char *who) +{ + if(snprintf(plain, sizeof_plain, "%s NTP %s: %s", level, who, message) > sizeof_plain) + log_warn("format_ntp_message(): Buffer too small to hold plain message, warning truncated"); + + // Return early if HTML text is not required + if(sizeof_html < 1 || html == NULL) + return; + + if(snprintf(html, sizeof_html, "%s in NTP %s:

    %s
    ", level, who, message) > sizeof_html) + log_warn("format_ntp_message(): Buffer too small to hold HTML message, warning truncated"); +} + int count_messages(const bool filter_dnsmasq_warnings) { int count = 0; @@ -1147,6 +1173,18 @@ bool format_messages(cJSON *array) break; } + case NTP_MESSAGE: + { + const char *message = (const char*)sqlite3_column_text(stmt, 3); + const char *level = (const char*)sqlite3_column_text(stmt, 4); + const char *who = (const char*)sqlite3_column_text(stmt, 5); + + format_ntp_message(plain, sizeof(plain), html, sizeof(html), + message, level, who); + + break; + } + case MAX_MESSAGE: // Fall through default: log_warn("format_messages() - Unknown message type: %s", mtypestr); @@ -1423,3 +1461,25 @@ void log_connection_error(const char *server, const char *reason, const char *er if(rowid == -1) log_err("logg_connection_error(): Failed to add message to database"); } + +void log_ntp_message(const bool error, const bool server, const char *message) +{ + const char *who = server ? "server" : "client"; + const char *level = error ? "Error" : "Warning"; + + // Create message + char buf[2048]; + format_ntp_message(buf, sizeof(buf), NULL, 0, message, level, who); + + // Log to FTL.log + if(error) + log_err("%s", buf); + else + log_warn("%s", buf); + + // Log to database + const int rowid = add_message(NTP_MESSAGE, message, level, who); + + if(rowid == -1) + log_err("log_ntp_message(): Failed to add message to database"); +} diff --git a/src/database/message-table.h b/src/database/message-table.h index d92bbe8d..5354af6f 100644 --- a/src/database/message-table.h +++ b/src/database/message-table.h @@ -30,5 +30,6 @@ void log_resource_shortage(const double load, const int nprocs, const int shmem, void logg_inaccessible_adlist(const int dbindex, const char *address); void log_certificate_domain_mismatch(const char *certfile, const char *domain); void log_connection_error(const char *server, const char *reason, const char *error); +void log_ntp_message(const bool error, const bool server, const char *message); #endif //MESSAGETABLE_H diff --git a/src/enums.h b/src/enums.h index b9ad1443..406e8f31 100644 --- a/src/enums.h +++ b/src/enums.h @@ -276,6 +276,7 @@ enum message_type { DISK_MESSAGE_EXTENDED, CERTIFICATE_DOMAIN_MISMATCH_MESSAGE, CONNECTION_ERROR_MESSAGE, + NTP_MESSAGE, MAX_MESSAGE, } __attribute__ ((packed)); diff --git a/src/ntp/client.c b/src/ntp/client.c index 2028edc0..c6d53c25 100644 --- a/src/ntp/client.c +++ b/src/ntp/client.c @@ -37,6 +37,8 @@ #include "signals.h" // adjtimex() #include +// log_ntp_message() +#include "database/message-table.h" struct ntp_sync { uint64_t org; @@ -146,8 +148,11 @@ static bool settime_step(struct timeval *unix_time, const double offset) // Set time immediately if(settimeofday(unix_time, NULL) != 0) { - log_err("Failed to set time: %s", - errno == EPERM ? "Insufficient permissions, try running with sudo" : strerror(errno)); + char errbuf[1024]; + strncpy(errbuf, "Failed to set time during NTP sync: ", sizeof(errbuf)); + strncat(errbuf, errno == EPERM ? "Insufficient permissions" : strerror(errno), sizeof(errbuf) - strlen(errbuf) - 1); + errbuf[sizeof(errbuf) - 1] = '\0'; + log_ntp_message(true, false, errbuf); return false; } @@ -193,8 +198,11 @@ static bool settime_skew(const double offset) if(adjtimex(&tx) < 0) { - log_err("Failed to adjust time: %s", - errno == EPERM ? "Insufficient permissions, try running with sudo" : strerror(errno)); + char errbuf[1024]; + strncpy(errbuf, "Failed to adjust time during NTP sync: ", sizeof(errbuf)); + strncat(errbuf, errno == EPERM ? "Insufficient permissions" : strerror(errno), sizeof(errbuf) - strlen(errbuf) - 1); + errbuf[sizeof(errbuf) - 1] = '\0'; + log_ntp_message(true, false, errbuf); return false; } @@ -220,7 +228,10 @@ static bool reply(int fd, struct ntp_sync *ntp, const bool verbose) { // Accepted limits are 2^-32 (~ 0.2 nanoseconds) // to 2^0 (= 1 second) - log_warn("Received NTP reply has invalid precision: 2^(%i), assuming microsecond accuracy", rho); + char errbuf[1024]; + snprintf(errbuf, sizeof(errbuf), "Received NTP reply has invalid precision: 2^(%i), assuming microsecond accuracy", rho); + errbuf[sizeof(errbuf) - 1] = '\0'; + log_ntp_message(false, false, errbuf); rho = -19; } // Compute precision of server clock in seconds 2^rho @@ -327,7 +338,11 @@ static int getsock(const struct addrinfo *saddr) const int s = socket(protocol, SOCK_DGRAM, IPPROTO_UDP); if(s == -1) { - log_err("Cannot create UDP socket"); + char errbuf[1024]; + strncpy(errbuf, "Cannot create UDP socket: ", sizeof(errbuf)); + strncat(errbuf, strerror(errno), sizeof(errbuf) - strlen(errbuf) - 1); + errbuf[sizeof(errbuf) - 1] = '\0'; + log_ntp_message(true, false, errbuf); return -1; } @@ -337,7 +352,11 @@ static int getsock(const struct addrinfo *saddr) tv.tv_usec = 0; if(setsockopt(s, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv)) != 0) { - log_err("Cannot set socket timeout"); + char errbuf[1024]; + strncpy(errbuf, "Cannot set socket timeout: ", sizeof(errbuf)); + strncat(errbuf, strerror(errno), sizeof(errbuf) - strlen(errbuf) - 1); + errbuf[sizeof(errbuf) - 1] = '\0'; + log_ntp_message(true, false, errbuf); close(s); return -1; } @@ -345,7 +364,11 @@ static int getsock(const struct addrinfo *saddr) // Set address to send to/receive from if(connect(s, saddr->ai_addr, saddr->ai_addrlen) != 0) { - log_err("Cannot connect to NTP server"); + char errbuf[1024]; + strncpy(errbuf, "Canot connect to NTP server: ", sizeof(errbuf)); + strncat(errbuf, strerror(errno), sizeof(errbuf) - strlen(errbuf) - 1); + errbuf[sizeof(errbuf) - 1] = '\0'; + log_ntp_message(true, false, errbuf); close(s); return -1; } @@ -357,10 +380,22 @@ static int getsock(const struct addrinfo *saddr) bool ntp_client(const char *server, const bool settime, const bool print) { // Resolve server address + int eai; struct addrinfo *saddr; - if(getaddrinfo(server, "ntp", NULL, &saddr) != 0) + if((eai = getaddrinfo(server, "ntp", NULL, &saddr)) != 0) { - log_err("Cannot resolve NTP server address"); + char errbuf[1024]; + strncpy(errbuf, "Cannot resolve NTP server address: ", sizeof(errbuf)); + strncat(errbuf, errno == EAI_SYSTEM ? strerror(errno) : gai_strerror(eai), + sizeof(errbuf) - strlen(errbuf) - 1); + if(eai == EAI_NONAME || eai == EAI_NODATA) + { + strncat(errbuf, " \"", sizeof(errbuf) - strlen(errbuf) - 1); + strncat(errbuf, server, sizeof(errbuf) - strlen(errbuf) - 1); + strncat(errbuf, "\"", sizeof(errbuf) - strlen(errbuf) - 1); + } + errbuf[sizeof(errbuf) - 1] = '\0'; + log_ntp_message(true, false, errbuf); return false; } @@ -428,7 +463,7 @@ bool ntp_client(const char *server, const bool settime, const bool print) if(valid == 0) { - log_warn("No valid NTP replies received, check server and network connectivity"); + log_ntp_message(false, false, "No valid NTP replies received, check server and network connectivity"); free(ntp); return false; } @@ -456,7 +491,7 @@ bool ntp_client(const char *server, const bool settime, const bool print) // or round-trip delay is larger than 1 second if(theta_stdev > 1.0 || delta_stdev > 1.0) { - log_warn("Standard deviation of time offset is too large, rejecting synchronization"); + log_ntp_message(false, false, "Standard deviation of time offset is too large, rejecting synchronization"); free(ntp); return false; } diff --git a/src/ntp/server.c b/src/ntp/server.c index 87b34831..652c9525 100644 --- a/src/ntp/server.c +++ b/src/ntp/server.c @@ -37,6 +37,8 @@ #include "config/config.h" // PRIi64 #include +// log_ntp_message() +#include "database/message-table.h" uint64_t ntp_last_sync = 0u; uint32_t ntp_root_delay = 0u; @@ -285,8 +287,12 @@ static void *ntp_bind_and_listen(void *param) const int s = socket(protocol, SOCK_DGRAM, IPPROTO_UDP); if(s == -1) { - log_warn("Cannot create NTP socket (%s), IPv%i NTP server not available", + char errbuf[1024]; + snprintf(errbuf, sizeof(errbuf), + "Cannot create NTP socket (%s), IPv%i NTP server not available", strerror(errno), protocol == AF_INET ? 4 : 6); + errbuf[sizeof(errbuf) - 1] = '\0'; + log_ntp_message(true, true, errbuf); return NULL; } @@ -310,8 +316,12 @@ static void *ntp_bind_and_listen(void *param) errno = 0; if(bind(s, (struct sockaddr *)&bind_addr, sizeof(bind_addr)) != 0) { - log_warn("Cannot bind to IPv4 address %s:123 (%s), IPv4 NTP server not available", + char errbuf[1024]; + snprintf(errbuf, sizeof(errbuf), + "Cannot bind to IPv4 address %s:123 (%s), IPv4 NTP server not available", ipstr, strerror(errno)); + errbuf[sizeof(errbuf) - 1] = '\0'; + log_ntp_message(true, true, errbuf); return NULL; } } @@ -326,7 +336,11 @@ static void *ntp_bind_and_listen(void *param) int opt = 1; if(setsockopt(s, IPPROTO_IPV6, IPV6_V6ONLY, &opt, sizeof(opt)) != 0) { - log_warn("Cannot set socket option IPV6_V6ONLY (%s), IPv6 NTP server not available", strerror(errno)); + char errbuf[1024]; + strncpy(errbuf, "Cannot set socket option IPV6_V6ONLY, IPv6 NTP server not available: ", sizeof(errbuf)); + strncat(errbuf, strerror(errno), sizeof(errbuf) - strlen(errbuf) - 1); + errbuf[sizeof(errbuf) - 1] = '\0'; + log_ntp_message(true, true, errbuf); return NULL; } @@ -342,8 +356,12 @@ static void *ntp_bind_and_listen(void *param) errno = 0; if(bind(s, (struct sockaddr *)&bind_addr, sizeof(bind_addr)) != 0) { - log_warn("Cannot bind to IPv6 address %s:123 (%s), IPv6 NTP server not available", + char errbuf[1024]; + snprintf(errbuf, sizeof(errbuf), + "Cannot bind to IPv6 address %s:123 (%s), IPv6 NTP server not available", ipstr, strerror(errno)); + errbuf[sizeof(errbuf) - 1] = '\0'; + log_ntp_message(true, true, errbuf); return NULL; } } @@ -366,7 +384,7 @@ bool ntp_server_start(pthread_attr_t *attr) pthread_t thread; if (pthread_create(&thread, attr, ntp_bind_and_listen, (void *)0) != 0) { - log_err("Can not create NTP server thread for IPv4"); + log_ntp_message(true, true, "Cannot create NTP server thread for IPv4"); return false; } } @@ -378,7 +396,7 @@ bool ntp_server_start(pthread_attr_t *attr) pthread_t thread; if (pthread_create(&thread, attr, ntp_bind_and_listen, (void *)1) != 0) { - log_err("Can not create NTP server thread for IPv6"); + log_ntp_message(true, true, "Cannot create NTP server thread for IPv6"); return false; } } From 3ae3afa1b529adccb3d47136ea4aef2dc2649003 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 13 Jun 2024 06:30:43 +0200 Subject: [PATCH 160/339] Fix harmless incorrect warning when generating HTML regex messages Signed-off-by: DL6ER --- src/database/message-table.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/database/message-table.c b/src/database/message-table.c index e6dab7de..81c4e3fa 100644 --- a/src/database/message-table.c +++ b/src/database/message-table.c @@ -549,7 +549,7 @@ static void format_regex_message(char *plain, const int sizeof_plain, char *html } if(snprintf(html, sizeof_html, "Encountered an error when processing regex %s filter with ID %d: 
    %s
    Error message: 
    %s
    ", - dbindex, type, dbindex, escaped_regex, escaped_warning)) + dbindex, type, dbindex, escaped_regex, escaped_warning) > sizeof_html) log_warn("format_regex_message(): Buffer too small to hold HTML message, warning truncated"); free(escaped_regex); From e05d9314120953635a00b9c3a7e297524f185c65 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 13 Jun 2024 06:31:44 +0200 Subject: [PATCH 161/339] Spellchecking Signed-off-by: DL6ER --- src/ntp/client.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ntp/client.c b/src/ntp/client.c index c6d53c25..1272d30c 100644 --- a/src/ntp/client.c +++ b/src/ntp/client.c @@ -365,7 +365,7 @@ static int getsock(const struct addrinfo *saddr) if(connect(s, saddr->ai_addr, saddr->ai_addrlen) != 0) { char errbuf[1024]; - strncpy(errbuf, "Canot connect to NTP server: ", sizeof(errbuf)); + strncpy(errbuf, "Cannot connect to NTP server: ", sizeof(errbuf)); strncat(errbuf, strerror(errno), sizeof(errbuf) - strlen(errbuf) - 1); errbuf[sizeof(errbuf) - 1] = '\0'; log_ntp_message(true, false, errbuf); From 09a1f6fe5e8b09cb5843116c9004e400179d6b3b Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 13 Jun 2024 07:35:47 +0200 Subject: [PATCH 162/339] Adjust CI tests due to modified NTP error message text Signed-off-by: DL6ER --- test/test_suite.bats | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_suite.bats b/test/test_suite.bats index 5969ba79..569bf002 100644 --- a/test/test_suite.bats +++ b/test/test_suite.bats @@ -1171,7 +1171,7 @@ @test "No ERROR messages in FTL.log (besides known/intended error)" { run bash -c 'grep "ERROR: " /var/log/pihole/FTL.log' printf "%s\n" "${lines[@]}" - run bash -c 'grep "ERROR: " /var/log/pihole/FTL.log | grep -c -v -E "(index\.html)|(Failed to create shared memory object)|(FTLCONF_debug_api is invalid)|(Failed to adjust time: Insufficient permissions)"' + run bash -c 'grep "ERROR: " /var/log/pihole/FTL.log | grep -c -v -E "(index\.html)|(Failed to create shared memory object)|(FTLCONF_debug_api is invalid)|(Failed to set|adjust time during NTP sync: Insufficient permissions)"' printf "count: %s\n" "${lines[@]}" [[ ${lines[0]} == "0" ]] } From 5a6a2129807fc5ee62e0ee51cae2b9fff2079c9f Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 13 Jun 2024 20:28:37 +0200 Subject: [PATCH 163/339] Add special handling for systems without password in the password checking short-circuiting Signed-off-by: DL6ER --- src/config/password.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/config/password.c b/src/config/password.c index 4ecf7169..7a60aaa1 100644 --- a/src/config/password.c +++ b/src/config/password.c @@ -601,8 +601,11 @@ int run_performance_test(void) bool set_and_check_password(struct conf_item *conf_item, const char *password) { - // Check if the newly set password is the same as the old one - if(verify_password(password, config.webserver.api.pwhash.v.s, false) == PASSWORD_CORRECT) + // Check if the user wants to set an empty password but the password is + // already empty, or if the newly set password is the same as the old + // one + if((strlen(password) == 0 && strlen(config.webserver.api.pwhash.v.s) == 0) || + verify_password(password, config.webserver.api.pwhash.v.s, false) == PASSWORD_CORRECT) { log_debug(DEBUG_CONFIG, "Password unchanged, not updating"); return true; From 950fc60ed363615a605d37edc22f40bc93cf71d1 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 14 Jun 2024 19:10:58 +0200 Subject: [PATCH 164/339] Limit app password permissions by default. Add new app_sudo mode for users to remove this new limitation if they really need to Signed-off-by: DL6ER --- src/api/api.c | 1 + src/api/auth.c | 1 + src/api/config.c | 10 ++++++++++ src/api/docs/content/specs/config.yaml | 3 +++ src/config/config.c | 6 ++++++ src/config/config.h | 1 + src/webserver/http-common.h | 4 ++++ test/pihole.toml | 10 +++++++++- 8 files changed, 35 insertions(+), 1 deletion(-) diff --git a/src/api/api.c b/src/api/api.c index d99b2b3d..042554a8 100644 --- a/src/api/api.c +++ b/src/api/api.c @@ -118,6 +118,7 @@ int api_handler(struct mg_connection *conn, void *ignored) double_time(), { false, NULL, NULL, NULL, 0u }, { false }, + NULL, { API_FLAG_NONE, 0 } }; diff --git a/src/api/auth.c b/src/api/auth.c index 227b2906..a8fb7e66 100644 --- a/src/api/auth.c +++ b/src/api/auth.c @@ -282,6 +282,7 @@ int check_client_auth(struct ftl_conn *api, const bool is_api) } api->user_id = user_id; + api->session = &auth_data[user_id]; api->message = "correct password"; return user_id; diff --git a/src/api/config.c b/src/api/config.c index 00105327..7ba024f6 100644 --- a/src/api/config.c +++ b/src/api/config.c @@ -1037,6 +1037,16 @@ int api_config(struct ftl_conn *api) if(api->method == HTTP_GET) return api_config_get(api); + // Check if this is an app session and reject the request if app sudo + // mode is disabled + if(api->session != NULL && api->session->app && !config.webserver.api.app_sudo.v.b) + { + return send_json_error(api, 403, + "forbidden", + "config read-only", + "app session but webserver.api.app_sudo is false"); + } + // POST: Create a new config (not supported) // PATCH: Replace parts of the the config with the provided one // PUT: Replaces the entire config with the provided one (not supported diff --git a/src/api/docs/content/specs/config.yaml b/src/api/docs/content/specs/config.yaml index 08d5dd1b..e4324dc5 100644 --- a/src/api/docs/content/specs/config.yaml +++ b/src/api/docs/content/specs/config.yaml @@ -413,6 +413,8 @@ components: type: string app_pwhash: type: string + app_sudo: + type: boolean excludeClients: type: array items: @@ -697,6 +699,7 @@ components: pwhash: '' totp_secret: '' app_pwhash: '' + app_sudo: false excludeClients: [ '1\.2\.3\.4', 'localhost', 'fe80::345' ] excludeDomains: [ 'google\\.de', 'pi-hole\.net' ] maxHistory: 86400 diff --git a/src/config/config.c b/src/config/config.c index 3c025fc7..5c8a1e23 100644 --- a/src/config/config.c +++ b/src/config/config.c @@ -1018,6 +1018,12 @@ void initConfig(struct config *conf) conf->webserver.api.app_pwhash.d.s = (char*)""; conf->webserver.api.app_pwhash.c = validate_stub; // Only type-based checking + conf->webserver.api.app_sudo.k = "webserver.api.app_sudo"; + conf->webserver.api.app_sudo.h = "Should the application password be allowed to modify Pi-hole config settings?\n Note that this setting is only relevant if the application password is set. Setting this to true allows third-party applications to modify advanced settings, e.g., the DNS server, DHCP server, or change passwords.\n Be aware that this setting is a security risk and should only be enabled if you trust the application and its developer."; + conf->webserver.api.app_sudo.t = CONF_BOOL; + conf->webserver.api.app_sudo.d.b = false; + conf->webserver.api.app_sudo.c = validate_stub; // Only type-based checking + conf->webserver.api.excludeClients.k = "webserver.api.excludeClients"; conf->webserver.api.excludeClients.h = "Array of clients to be excluded from certain API responses (regex):\n - Query Log (/api/queries)\n - Top Clients (/api/stats/top_clients)\n This setting accepts both IP addresses (IPv4 and IPv6) as well as hostnames.\n Note that backslashes \"\\\" need to be escaped, i.e. \"\\\\\" in this setting\n\n Example: [ \"^192\\\\.168\\\\.2\\\\.56$\", \"^fe80::341:[0-9a-f]*$\", \"^localhost$\" ]"; conf->webserver.api.excludeClients.a = cJSON_CreateStringReference("array of regular expressions describing clients"); diff --git a/src/config/config.h b/src/config/config.h index 1a0587cd..d666c1f5 100644 --- a/src/config/config.h +++ b/src/config/config.h @@ -237,6 +237,7 @@ struct config { struct conf_item password; // This is a pseudo-item struct conf_item totp_secret; // This is a write-only item struct conf_item app_pwhash; + struct conf_item app_sudo; struct conf_item excludeClients; struct conf_item excludeDomains; struct conf_item maxHistory; diff --git a/src/webserver/http-common.h b/src/webserver/http-common.h index edc979eb..5ff01564 100644 --- a/src/webserver/http-common.h +++ b/src/webserver/http-common.h @@ -21,6 +21,9 @@ // strlen() #include +// struct session +#include "api/auth.h" + // API-internal definitions // Maximum size of received and processed payload: 64 KB @@ -50,6 +53,7 @@ struct ftl_conn { struct { bool restart; } ftl; + struct session *session; struct api_options opts; }; diff --git a/test/pihole.toml b/test/pihole.toml index e1d72c8b..b2e5f587 100644 --- a/test/pihole.toml +++ b/test/pihole.toml @@ -702,6 +702,14 @@ # app_pwhash = "" + # Should the application password be allowed to modify Pi-hole config settings? + # Note that this setting is only relevant if the application password is set. Setting + # this to true allows third-party applications to modify advanced settings, e.g., the + # DNS server, DHCP server, or change passwords. + # Be aware that this setting is a security risk and should only be enabled if you + # trust the application and its developer. + app_sudo = false + # Array of clients to be excluded from certain API responses (regex): # - Query Log (/api/queries) # - Top Clients (/api/stats/top_clients) @@ -1039,7 +1047,7 @@ all = true ### CHANGED, default = false # Configuration statistics: -# 136 total entries out of which 82 entries are default +# 137 total entries out of which 83 entries are default # --> 54 entries are modified # 2 entries are forced through environment: # - misc.nice From d39480884de6545db651ff9723216f8e8d72289d Mon Sep 17 00:00:00 2001 From: Dominik Date: Sat, 15 Jun 2024 10:42:26 +0200 Subject: [PATCH 165/339] Apply code review Co-authored-by: RD WebDesign Signed-off-by: Dominik --- src/api/config.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/api/config.c b/src/api/config.c index 7ba024f6..fc345852 100644 --- a/src/api/config.c +++ b/src/api/config.c @@ -1043,8 +1043,8 @@ int api_config(struct ftl_conn *api) { return send_json_error(api, 403, "forbidden", - "config read-only", - "app session but webserver.api.app_sudo is false"); + "Unable to change configuration (read-only)", + "The current app session is not allowed to modify Pi-hole config settings (webserver.api.app_sudo is false)"); } // POST: Create a new config (not supported) From 4f601342005996dd13ebeea36302bf5944336104 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 15 Jun 2024 11:11:12 +0200 Subject: [PATCH 166/339] Improve config description of webserver.api.app_sudo Signed-off-by: DL6ER --- src/config/config.c | 2 +- test/pihole.toml | 15 +++++++-------- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/src/config/config.c b/src/config/config.c index 5c8a1e23..f49f2af5 100644 --- a/src/config/config.c +++ b/src/config/config.c @@ -1019,7 +1019,7 @@ void initConfig(struct config *conf) conf->webserver.api.app_pwhash.c = validate_stub; // Only type-based checking conf->webserver.api.app_sudo.k = "webserver.api.app_sudo"; - conf->webserver.api.app_sudo.h = "Should the application password be allowed to modify Pi-hole config settings?\n Note that this setting is only relevant if the application password is set. Setting this to true allows third-party applications to modify advanced settings, e.g., the DNS server, DHCP server, or change passwords.\n Be aware that this setting is a security risk and should only be enabled if you trust the application and its developer."; + conf->webserver.api.app_sudo.h = "Should application password API sessions be allowed to modify config settings?\n Setting this to true allows third-party applications using the application password to modify advanced settings, e.g., the upstream DNS servers, DHCP server settings, or changing passwords. This setting should only be enabled if really needed and only if you trust the applications using the application password."; conf->webserver.api.app_sudo.t = CONF_BOOL; conf->webserver.api.app_sudo.d.b = false; conf->webserver.api.app_sudo.c = validate_stub; // Only type-based checking diff --git a/test/pihole.toml b/test/pihole.toml index b2e5f587..be5e3ab0 100644 --- a/test/pihole.toml +++ b/test/pihole.toml @@ -1,7 +1,7 @@ -# Pi-hole configuration file (v5.25.2-1891-g7ff016f2-dirty) +# Pi-hole configuration file (v5.25.2-1921-gd3948088-dirty) # Encoding: UTF-8 # This file is managed by pihole-FTL -# Last updated on 2024-05-30 11:37:59 +# Last updated on 2024-06-15 09:10:13 UTC [dns] # Array of upstream DNS servers used by Pi-hole @@ -702,12 +702,11 @@ # app_pwhash = "" - # Should the application password be allowed to modify Pi-hole config settings? - # Note that this setting is only relevant if the application password is set. Setting - # this to true allows third-party applications to modify advanced settings, e.g., the - # DNS server, DHCP server, or change passwords. - # Be aware that this setting is a security risk and should only be enabled if you - # trust the application and its developer. + # Should application password API sessions be allowed to modify config settings? + # Setting this to true allows third-party applications using the application password + # to modify advanced settings, e.g., the upstream DNS servers, DHCP server settings, + # or changing passwords. This setting should only be enabled if really needed and only + # if you trust the applications using the application password. app_sudo = false # Array of clients to be excluded from certain API responses (regex): From bf476509b547ef42a5bc072b210ffbb200e84bbe Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 15 Jun 2024 10:14:29 +0000 Subject: [PATCH 167/339] Bump actions/checkout in the github_action-dependencies group Bumps the github_action-dependencies group with 1 update: [actions/checkout](https://github.com/actions/checkout). Updates `actions/checkout` from 4.1.6 to 4.1.7 - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v4.1.6...v4.1.7) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-patch dependency-group: github_action-dependencies ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 6 +++--- .github/workflows/codespell.yml | 2 +- .github/workflows/openapi-validator.yml | 2 +- .github/workflows/stale.yml | 2 +- .github/workflows/sync-back-to-dev.yml | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 370d88ef..e3d69c3b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -31,7 +31,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v4.1.6 + uses: actions/checkout@v4.1.7 - name: "Calculate required variables" id: variables @@ -81,7 +81,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v4.1.6 + uses: actions/checkout@v4.1.7 - name: Build and test and deploy FTL uses: ./.github/actions/build-and-test @@ -120,7 +120,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v4.1.6 + uses: actions/checkout@v4.1.7 - name: Build and test and deploy FTL uses: ./.github/actions/build-and-test diff --git a/.github/workflows/codespell.yml b/.github/workflows/codespell.yml index 1fda2c3f..8ce5dfe2 100644 --- a/.github/workflows/codespell.yml +++ b/.github/workflows/codespell.yml @@ -13,7 +13,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4.1.6 + uses: actions/checkout@v4.1.7 - name: Spell-Checking uses: codespell-project/actions-codespell@master diff --git a/.github/workflows/openapi-validator.yml b/.github/workflows/openapi-validator.yml index 0070f8fe..8827d074 100644 --- a/.github/workflows/openapi-validator.yml +++ b/.github/workflows/openapi-validator.yml @@ -12,7 +12,7 @@ jobs: steps: - name: Clone repository - uses: actions/checkout@v4.1.6 + uses: actions/checkout@v4.1.7 - name: Set Node.js version uses: actions/setup-node@v4 diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index c2699158..41e1793a 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -40,7 +40,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4.1.6 + uses: actions/checkout@v4.1.7 - name: Remove 'stale' label run: gh issue edit ${{ github.event.issue.number }} --remove-label ${{ env.stale_label }} env: diff --git a/.github/workflows/sync-back-to-dev.yml b/.github/workflows/sync-back-to-dev.yml index 0592cf35..e15d1049 100644 --- a/.github/workflows/sync-back-to-dev.yml +++ b/.github/workflows/sync-back-to-dev.yml @@ -11,7 +11,7 @@ jobs: name: Syncing branches steps: - name: Checkout - uses: actions/checkout@v4.1.6 + uses: actions/checkout@v4.1.7 - name: Opening pull request run: gh pr create -B development -H master --title 'Sync master back into development' --body 'Created by Github action' --label 'internal' env: From b3182568a54baa300690b9830a557e76573ceefb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sat, 15 Jun 2024 13:16:37 +0200 Subject: [PATCH 168/339] Fix API hint MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- src/api/config.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/api/config.c b/src/api/config.c index 00105327..b49e5886 100644 --- a/src/api/config.c +++ b/src/api/config.c @@ -844,9 +844,9 @@ static int api_config_put_delete(struct ftl_conn *api) const char *hint = NULL, *message = NULL; if(api->method == HTTP_PUT) - hint = "Use, e.g., PUT /api/config/dnsmasq/upstreams/127.0.0.1 to add \"127.0.0.1\" to config.dns.upstreams"; + hint = "Use, e.g., PUT /api/config/dns/upstreams/127.0.0.1 to add \"127.0.0.1\" to config.dns.upstreams"; else - hint = "Use, e.g., DELETE /api/config/dnsmasq/upstreams/127.0.0.1 to remove \"127.0.0.1\" from config.dns.upstreams"; + hint = "Use, e.g., DELETE /api/config/dns/upstreams/127.0.0.1 to remove \"127.0.0.1\" from config.dns.upstreams"; if(min_level < 2) { From 0ed7f6dde19465bedfd3fee535b59e3f5ef748bc Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 16 Jun 2024 07:47:44 +0200 Subject: [PATCH 169/339] Fix app status not being backed up correctly during FTL restarts Signed-off-by: DL6ER --- src/database/session-table.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/database/session-table.c b/src/database/session-table.c index 0147263a..7c630a44 100644 --- a/src/database/session-table.c +++ b/src/database/session-table.c @@ -151,14 +151,14 @@ bool backup_db_sessions(struct session *sessions, const uint16_t max_sessions) return false; } // 8: tls_mixed - if(sqlite3_bind_int(stmt, 8, sess->tls.mixed ? 1: 0) != SQLITE_OK) + if(sqlite3_bind_int(stmt, 8, sess->tls.mixed ? 1 : 0) != SQLITE_OK) { log_err("Cannot bind tls_mixed = %d in backup_db_sessions(): %s (%d)", sess->tls.mixed ? 1 : 0, sqlite3_errmsg(db), sqlite3_errcode(db)); return false; } // 9: app - if(sqlite3_bind_int(stmt, 8, sess->app ? 1: 0) != SQLITE_OK) + if(sqlite3_bind_int(stmt, 9, sess->app ? 1 : 0) != SQLITE_OK) { log_err("Cannot bind app = %d in backup_db_sessions(): %s (%d)", sess->app ? 1 : 0, sqlite3_errmsg(db), sqlite3_errcode(db)); From 87dd091535736e824e2033e36ef0dc420ba9c316 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 16 Jun 2024 21:28:27 +0200 Subject: [PATCH 170/339] Fix crash caused by double free() corruption encountered with rev-server addresses with prefix lengths != {8,16,24,32} Signed-off-by: DL6ER --- src/config/dnsmasq_config.c | 6 ++++++ src/dnsmasq/option.c | 9 ++++----- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/config/dnsmasq_config.c b/src/config/dnsmasq_config.c index 8c2e2686..ad9fb93d 100644 --- a/src/config/dnsmasq_config.c +++ b/src/config/dnsmasq_config.c @@ -723,6 +723,12 @@ bool __attribute__((const)) write_dnsmasq_config(struct config *conf, bool test_ { log_warn("New dnsmasq configuration is not valid (%s), config remains unchanged", errbuf); + if(debug_flags[DEBUG_ANY]) + { + log_debug(DEBUG_ANY, "Temporary dnsmasq config file left in place for debugging purposes"); + return false; + } + // Remove temporary config file if(remove(DNSMASQ_TEMP_CONF) != 0) { diff --git a/src/dnsmasq/option.c b/src/dnsmasq/option.c index 8e7377dc..d075ad0a 100644 --- a/src/dnsmasq/option.c +++ b/src/dnsmasq/option.c @@ -1193,10 +1193,10 @@ static char *domain_rev4(int from_file, char *server, struct in_addr *addr4, int return _("error"); } - if (sdetails.orig_hostinfo) - freeaddrinfo(sdetails.orig_hostinfo); } } + if (sdetails.orig_hostinfo) + freeaddrinfo(sdetails.orig_hostinfo); return NULL; } @@ -1280,11 +1280,10 @@ static char *domain_rev6(int from_file, char *server, struct in6_addr *addr6, in if (!add_update_server(flags, &serv_addr, &source_addr, interface, domain, NULL)) return _("error"); } - - if (sdetails.orig_hostinfo) - freeaddrinfo(sdetails.orig_hostinfo); } } + if (sdetails.orig_hostinfo) + freeaddrinfo(sdetails.orig_hostinfo); return NULL; } From 784e11989240c3458a9c94809354be5260aa54d5 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 19 Jun 2024 22:01:13 +0200 Subject: [PATCH 171/339] Add CLI password generation Signed-off-by: DL6ER --- src/api/2fa.c | 4 +- src/api/auth.c | 8 +- src/api/auth.h | 1 + src/api/config.c | 9 ++ src/api/docs/content/specs/auth.yaml | 4 + src/api/docs/content/specs/config.yaml | 3 + src/config/config.c | 6 ++ src/config/config.h | 1 + src/config/password.c | 126 ++++++++++++++++++++++--- src/config/password.h | 7 +- src/webserver/webserver.c | 8 ++ test/pihole.toml | 10 +- 12 files changed, 168 insertions(+), 19 deletions(-) diff --git a/src/api/2fa.c b/src/api/2fa.c index 4e255df2..164f7000 100644 --- a/src/api/2fa.c +++ b/src/api/2fa.c @@ -15,7 +15,7 @@ #include "config/config.h" // getrandom() #include "daemon.h" -// generate_app_password() +// generate_password() #include "config/password.h" // TOTP+HMAC @@ -313,7 +313,7 @@ int generateAppPw(struct ftl_conn *api) { // Generate and set app password char *password = NULL, *pwhash = NULL; - if(!generate_app_password(&password, &pwhash)) + if(!generate_password(&password, &pwhash)) { return send_json_error(api, 500, diff --git a/src/api/auth.c b/src/api/auth.c index a8fb7e66..8212a03d 100644 --- a/src/api/auth.c +++ b/src/api/auth.c @@ -310,6 +310,7 @@ static int get_all_sessions(struct ftl_conn *api, cJSON *json) JSON_REF_STR_IN_OBJECT(session, "remote_addr", auth_data[i].remote_addr); JSON_REF_STR_IN_OBJECT(session, "user_agent", auth_data[i].user_agent); JSON_ADD_BOOL_TO_OBJECT(session, "app", auth_data[i].app); + JSON_ADD_BOOL_TO_OBJECT(session, "cli", auth_data[i].cli); JSON_ADD_ITEM_TO_ARRAY(sessions, session); } JSON_ADD_ITEM_TO_OBJECT(json, "sessions", sessions); @@ -537,7 +538,9 @@ int api_auth(struct ftl_conn *api) else result = verify_login(password); - if(result == PASSWORD_CORRECT || result == APPPASSWORD_CORRECT) + if(result == PASSWORD_CORRECT || + result == APPPASSWORD_CORRECT || + result ==CLIPASSWORD_CORRECT) { // Accepted @@ -548,7 +551,7 @@ int api_auth(struct ftl_conn *api) // Check possible 2FA token // Successful login with empty password does not require 2FA - if(strlen(config.webserver.api.totp_secret.v.s) > 0 && result != APPPASSWORD_CORRECT) + if(strlen(config.webserver.api.totp_secret.v.s) > 0 && result == PASSWORD_CORRECT) { // Get 2FA token from payload cJSON *json_totp; @@ -619,6 +622,7 @@ int api_auth(struct ftl_conn *api) auth_data[i].tls.login = api->request->is_ssl; auth_data[i].tls.mixed = false; auth_data[i].app = result == APPPASSWORD_CORRECT; + auth_data[i].cli = result == CLIPASSWORD_CORRECT; // Generate new SID and CSRF token generateSID(auth_data[i].sid); diff --git a/src/api/auth.h b/src/api/auth.h index 5028b6c8..28af5b00 100644 --- a/src/api/auth.h +++ b/src/api/auth.h @@ -48,6 +48,7 @@ struct session { bool used; bool app; + bool cli; struct { bool login; bool mixed; diff --git a/src/api/config.c b/src/api/config.c index fc345852..31183c89 100644 --- a/src/api/config.c +++ b/src/api/config.c @@ -1047,6 +1047,15 @@ int api_config(struct ftl_conn *api) "The current app session is not allowed to modify Pi-hole config settings (webserver.api.app_sudo is false)"); } + // Check if this is a CLI session and reject the request + if(api->session != NULL && api->session->cli) + { + return send_json_error(api, 403, + "forbidden", + "Unable to change configuration (read-only)", + "The current CLI session is not allowed to modify Pi-hole config settings"); + } + // POST: Create a new config (not supported) // PATCH: Replace parts of the the config with the provided one // PUT: Replaces the entire config with the provided one (not supported diff --git a/src/api/docs/content/specs/auth.yaml b/src/api/docs/content/specs/auth.yaml index 0a739957..9b206517 100644 --- a/src/api/docs/content/specs/auth.yaml +++ b/src/api/docs/content/specs/auth.yaml @@ -345,6 +345,9 @@ components: app: type: boolean description: Indicator if this session was initiated using an application password + cli: + type: boolean + description: Indicator if this session was initiated using the command-line interface (CLI) login_at: type: integer description: Timestamp of login (seconds since epoch) @@ -368,6 +371,7 @@ components: login: true mixed: false app: false + cli: false login_at: 1580000000 last_active: 1580000000 valid_until: 1580000300 diff --git a/src/api/docs/content/specs/config.yaml b/src/api/docs/content/specs/config.yaml index e4324dc5..1de6ecff 100644 --- a/src/api/docs/content/specs/config.yaml +++ b/src/api/docs/content/specs/config.yaml @@ -415,6 +415,8 @@ components: type: string app_sudo: type: boolean + cli_pw: + type: boolean excludeClients: type: array items: @@ -700,6 +702,7 @@ components: totp_secret: '' app_pwhash: '' app_sudo: false + cli_pw: true excludeClients: [ '1\.2\.3\.4', 'localhost', 'fe80::345' ] excludeDomains: [ 'google\\.de', 'pi-hole\.net' ] maxHistory: 86400 diff --git a/src/config/config.c b/src/config/config.c index f49f2af5..12f4bcdc 100644 --- a/src/config/config.c +++ b/src/config/config.c @@ -1024,6 +1024,12 @@ void initConfig(struct config *conf) conf->webserver.api.app_sudo.d.b = false; conf->webserver.api.app_sudo.c = validate_stub; // Only type-based checking + conf->webserver.api.cli_pw.k = "webserver.api.cli_pw"; + conf->webserver.api.cli_pw.h = "Should FTL create a temporary CLI password? This password is stored in clear in /etc/pihole and can be used by the CLI (pihole ... commands) to authenticate against the API. Note that the password is only valid for the current session and regenerated on each FTL restart. Sessions initiated with this password cannot modify the Pi-hole configuration (change passwords, etc.) for security reasons but can still use the API to query data and manage lists."; + conf->webserver.api.cli_pw.t = CONF_BOOL; + conf->webserver.api.cli_pw.d.b = true; + conf->webserver.api.cli_pw.c = validate_stub; // Only type-based checking + conf->webserver.api.excludeClients.k = "webserver.api.excludeClients"; conf->webserver.api.excludeClients.h = "Array of clients to be excluded from certain API responses (regex):\n - Query Log (/api/queries)\n - Top Clients (/api/stats/top_clients)\n This setting accepts both IP addresses (IPv4 and IPv6) as well as hostnames.\n Note that backslashes \"\\\" need to be escaped, i.e. \"\\\\\" in this setting\n\n Example: [ \"^192\\\\.168\\\\.2\\\\.56$\", \"^fe80::341:[0-9a-f]*$\", \"^localhost$\" ]"; conf->webserver.api.excludeClients.a = cJSON_CreateStringReference("array of regular expressions describing clients"); diff --git a/src/config/config.h b/src/config/config.h index d666c1f5..408c72c3 100644 --- a/src/config/config.h +++ b/src/config/config.h @@ -238,6 +238,7 @@ struct config { struct conf_item totp_secret; // This is a write-only item struct conf_item app_pwhash; struct conf_item app_sudo; + struct conf_item cli_pw; struct conf_item excludeClients; struct conf_item excludeDomains; struct conf_item maxHistory; diff --git a/src/config/password.c b/src/config/password.c index 5c32411c..4178b810 100644 --- a/src/config/password.c +++ b/src/config/password.c @@ -39,6 +39,20 @@ // 2023, using 128 bits should be sufficient for the foreseeable future. #define SALT_LEN 16 // 16 bytes = 128 bits +// App password length +// The app password is a 256 bit password. This is a good balance between +// security and usability. It is long enough to be secure. +#define APPPW_LEN 32 // 32 bytes = 256 bits + +// CLI password file and memory +// We store the password in plain memory. This is not a security issue as the +// memory is only accessible to the user running the FTL process. Anyone with +// sufficient access to the memory (ptrace, swapfile) would also have access to +// the password file. Leaking the password after exit is not a concern as a new +// password is generated on every start. +#define CLI_PW_FILE "/etc/pihole/cli_pw" +static char *cli_password = NULL; + // Convert RAW data into hex representation // Two hexadecimal digits are generated for each input byte. void sha256_raw_to_hex(uint8_t *data, char *buffer) @@ -314,6 +328,13 @@ char * __attribute__((malloc)) create_password(const char *password) enum password_result verify_login(const char *password) { + // Check if this is the CLI password + if(config.webserver.api.cli_pw.v.b && cli_password != NULL) + { + if(strcmp(cli_password, password) == 0) + return CLIPASSWORD_CORRECT; + } + enum password_result pw = verify_password(password, config.webserver.api.pwhash.v.s, true); log_debug(DEBUG_API, "Password %s correct", pw == PASSWORD_CORRECT ? "" : "not"); @@ -627,28 +648,40 @@ bool set_and_check_password(struct conf_item *conf_item, const char *password) return true; } -bool generate_app_password(char **password, char **pwhash) +bool generate_password(char **password, char **pwhash) { - // Generate a 128 bit random salt - // genrandom() returns cryptographically secure random data - uint8_t salt[SALT_LEN] = { 0 }; - if(getrandom(salt, sizeof(salt), 0) < 0) - { - log_err("getrandom() failed in generate_app_password()"); - return false; - } - // Generate a 256 bit random password - uint8_t password_raw[256/8] = { 0 }; + // genrandom() returns cryptographically secure random data + uint8_t password_raw[APPPW_LEN] = { 0 }; if(getrandom(password_raw, sizeof(password_raw), 0) < 0) { - log_err("getrandom() failed in generate_app_password()"); + log_err("getrandom() failed in generate_password()"); return false; } // Encode password as base64 *password = base64_encode(password_raw, sizeof(password_raw)); + if(*password == NULL) + { + log_err("Error while encoding password as base64"); + return false; + } + + if(pwhash == NULL) + { + // No password hash requested + return true; + } + + // Generate a 128 bit random salt + uint8_t salt[SALT_LEN] = { 0 }; + if(getrandom(salt, sizeof(salt), 0) < 0) + { + log_err("getrandom() failed in generate_password()"); + return false; + } + // Generate balloon PHC-encoded password hash *pwhash = balloon_password(*password, salt, true); @@ -663,3 +696,72 @@ bool generate_app_password(char **password, char **pwhash) return true; } + +bool create_cli_password(void) +{ + // Check if the CLI password is enabled + if(!config.webserver.api.cli_pw.v.b) + { + log_debug(DEBUG_API, "CLI password is not set"); + return true; + } + + // Generate a new CLI password hash + if(!generate_password(&cli_password, NULL)) + { + log_err("Failed to generate CLI password hash!"); + return false; + } + + // Store the CLI password in the corresponding file + FILE *file = fopen(CLI_PW_FILE, "w"); + if(file == NULL) + { + log_err("Failed to open CLI password file for writing: %s", strerror(errno)); + free(cli_password); + return false; + } + + // Write password + if(fputs(cli_password, file) == EOF) + { + log_err("Failed to write CLI password to file: %s", strerror(errno)); + fclose(file); + free(cli_password); + return false; + } + + // Close file + fclose(file); + + // Set file permissions to 0640 + if(chmod(CLI_PW_FILE, S_IRUSR | S_IWUSR | S_IRGRP) < 0) + { + log_err("Failed to set permissions on CLI password file: %s", strerror(errno)); + free(cli_password); + return false; + } + + log_debug(DEBUG_API, "CLI password set and stored in file"); + return true; +} + +bool remove_cli_password(void) +{ + // Empty the CLI password file + FILE *file = fopen(CLI_PW_FILE, "w"); + if(file == NULL) + { + log_err("Failed to open CLI password file for writing: %s", strerror(errno)); + return false; + } + + // Close file + fclose(file); + + // Remove the CLI password from memory + free(cli_password); + + log_debug(DEBUG_API, "CLI password removed"); + return true; +} diff --git a/src/config/password.h b/src/config/password.h index 063e5dcf..fd6fd332 100644 --- a/src/config/password.h +++ b/src/config/password.h @@ -20,13 +20,16 @@ enum password_result verify_login(const char *password); enum password_result verify_password(const char *password, const char *pwhash, const bool rate_limiting); int run_performance_test(void); bool set_and_check_password(struct conf_item *conf_item, const char *password); -bool generate_app_password(char **password, char **pwhash); +bool generate_password(char **password, char **pwhash); +bool create_cli_password(void); +bool remove_cli_password(void); enum password_result { PASSWORD_INCORRECT = 0, PASSWORD_CORRECT = 1, APPPASSWORD_CORRECT = 2, - NO_PASSWORD_SET = 3, + CLIPASSWORD_CORRECT = 3, + NO_PASSWORD_SET = 4, PASSWORD_RATE_LIMITED = -1 } __attribute__((packed)); diff --git a/src/webserver/webserver.c b/src/webserver/webserver.c index 33d3ffcc..e16b6329 100644 --- a/src/webserver/webserver.c +++ b/src/webserver/webserver.c @@ -28,6 +28,8 @@ #include "webserver/lua_web.h" // log_certificate_domain_mismatch() #include "database/message-table.h" +// create_cli_password() +#include "config/password.h" // Server context handle static struct mg_context *ctx = NULL; @@ -542,6 +544,9 @@ void http_init(void) // Restore sessions from database init_api(); + + // Create CLI password (if enabled) + create_cli_password(); } static char *append_to_path(char *path, const char *append) @@ -635,6 +640,9 @@ void http_terminate(void) // Free Lua-related resources free_lua(); + // Remove CLI password + remove_cli_password(); + // Free error_pages path if(error_pages != NULL) { diff --git a/test/pihole.toml b/test/pihole.toml index be5e3ab0..bc13e912 100644 --- a/test/pihole.toml +++ b/test/pihole.toml @@ -709,6 +709,14 @@ # if you trust the applications using the application password. app_sudo = false + # Should FTL create a temporary CLI password? This password is stored in clear in + # /etc/pihole and can be used by the CLI (pihole ... commands) to authenticate + # against the API. Note that the password is only valid for the current session and + # regenerated on each FTL restart. Sessions initiated with this password cannot modify + # the Pi-hole configuration (change passwords, etc.) for security reasons but can + # still use the API to query data and manage lists. + cli_pw = true + # Array of clients to be excluded from certain API responses (regex): # - Query Log (/api/queries) # - Top Clients (/api/stats/top_clients) @@ -1046,7 +1054,7 @@ all = true ### CHANGED, default = false # Configuration statistics: -# 137 total entries out of which 83 entries are default +# 138 total entries out of which 84 entries are default # --> 54 entries are modified # 2 entries are forced through environment: # - misc.nice From b74696fc7747dd627e211fede348d14eb4c48ff5 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 19 Jun 2024 22:05:49 +0200 Subject: [PATCH 172/339] Remove webserver.api.searchAPIauth option Signed-off-by: DL6ER --- src/api/api.c | 17 +---------------- src/api/docs/content/specs/config.yaml | 3 --- src/api/docs/content/specs/search.yaml | 1 - src/config/config.c | 6 ------ src/config/config.h | 1 - test/pihole.toml | 7 +------ 6 files changed, 2 insertions(+), 33 deletions(-) diff --git a/src/api/api.c b/src/api/api.c index 042554a8..406ba7c1 100644 --- a/src/api/api.c +++ b/src/api/api.c @@ -172,22 +172,7 @@ int api_handler(struct mg_connection *conn, void *ignored) } // Verify requesting client is allowed to see this resource - if(api_request[i].func == api_search) - { - // Handle /api/search special as it may be allowed for local users due to webserver.api.searchAPIauth - if(!config.webserver.api.searchAPIauth.v.b && is_local_api_user(api.request->remote_addr)) - { - // Local users does not need to authenticate when searchAPIauth is false - ; - } - else if(api_request[i].require_auth && check_client_auth(&api, true) == API_AUTH_UNAUTHORIZED) - { - // Users need to authenticate but authentication failed - unauthorized = true; - break; - } - } - else if(api_request[i].require_auth && check_client_auth(&api, true) == API_AUTH_UNAUTHORIZED) + if(api_request[i].require_auth && check_client_auth(&api, true) == API_AUTH_UNAUTHORIZED) { unauthorized = true; break; diff --git a/src/api/docs/content/specs/config.yaml b/src/api/docs/content/specs/config.yaml index 1de6ecff..c6368c14 100644 --- a/src/api/docs/content/specs/config.yaml +++ b/src/api/docs/content/specs/config.yaml @@ -397,8 +397,6 @@ components: properties: localAPIauth: type: boolean - searchAPIauth: - type: boolean max_sessions: type: integer prettyJSON: @@ -694,7 +692,6 @@ components: theme: "default-darker" api: localAPIauth: false - searchAPIauth: false max_sessions: 16 prettyJSON: false password: "********" diff --git a/src/api/docs/content/specs/search.yaml b/src/api/docs/content/specs/search.yaml index b5ad6fc9..94a068be 100644 --- a/src/api/docs/content/specs/search.yaml +++ b/src/api/docs/content/specs/search.yaml @@ -18,7 +18,6 @@ components: The optional parameters `N` and `partial` limit the maximum number of returned records and whether partial matches should be returned, respectively. There is a hard upper limit of `N` defined in FTL (currently set to 10,000) to ensure that the response is not too large. ABP matches are not returned when partial matching is requested. - Depending on the value of the config option webserver.api.searchAPIauth, local clients may not need to authenticate for this endpoint. International domains names (IDNs) are internally converted to punycode before matching. responses: '200': diff --git a/src/config/config.c b/src/config/config.c index 12f4bcdc..0c3da544 100644 --- a/src/config/config.c +++ b/src/config/config.c @@ -961,12 +961,6 @@ void initConfig(struct config *conf) conf->webserver.interface.theme.c = validate_stub; // Only type-based checking // sub-struct api - conf->webserver.api.searchAPIauth.k = "webserver.api.searchAPIauth"; - conf->webserver.api.searchAPIauth.h = "Do local clients need to authenticate to access the search API? This settings allows local clients to use pihole -q ... without authentication. Note that \"local\" in the sense of the option means only 127.0.0.1 and [::1]"; - conf->webserver.api.searchAPIauth.t = CONF_BOOL; - conf->webserver.api.searchAPIauth.d.b = false; - conf->webserver.api.searchAPIauth.c = validate_stub; // Only type-based checking - conf->webserver.api.localAPIauth.k = "webserver.api.localAPIauth"; conf->webserver.api.localAPIauth.h = "Do local clients need to authenticate to access the API? This settings allows local clients to use the API without authentication."; conf->webserver.api.localAPIauth.t = CONF_BOOL; diff --git a/src/config/config.h b/src/config/config.h index 408c72c3..0af11040 100644 --- a/src/config/config.h +++ b/src/config/config.h @@ -230,7 +230,6 @@ struct config { } interface; struct { struct conf_item localAPIauth; - struct conf_item searchAPIauth; struct conf_item max_sessions; struct conf_item prettyJSON; struct conf_item pwhash; diff --git a/test/pihole.toml b/test/pihole.toml index bc13e912..1ddc6f7b 100644 --- a/test/pihole.toml +++ b/test/pihole.toml @@ -660,11 +660,6 @@ # clients to use the API without authentication. localAPIauth = true - # Do local clients need to authenticate to access the search API? This settings allows - # local clients to use pihole -q ... without authentication. Note that "local" in the - # sense of the option means only 127.0.0.1 and [::1] - searchAPIauth = false - # Number of concurrent sessions allowed for the API. If the number of sessions exceeds # this value, no new sessions will be allowed until the number of sessions drops due # to session expiration or logout. Note that the number of concurrent sessions is @@ -1054,7 +1049,7 @@ all = true ### CHANGED, default = false # Configuration statistics: -# 138 total entries out of which 84 entries are default +# 137 total entries out of which 83 entries are default # --> 54 entries are modified # 2 entries are forced through environment: # - misc.nice From 069cc309b02c7549799f0612013895709d9713a3 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 19 Jun 2024 22:07:18 +0200 Subject: [PATCH 173/339] Remove webserver.api.localAPIauth option Signed-off-by: DL6ER --- src/api/auth.c | 19 +------------------ src/api/docs/content/specs/config.yaml | 3 --- src/config/config.c | 6 ------ src/config/config.h | 1 - src/config/legacy_reader.c | 5 ----- src/enums.h | 3 +-- src/lua/ftl_lua.c | 17 ++--------------- test/pihole.toml | 6 +----- 8 files changed, 5 insertions(+), 55 deletions(-) diff --git a/src/api/auth.c b/src/api/auth.c index 8212a03d..b0abf590 100644 --- a/src/api/auth.c +++ b/src/api/auth.c @@ -78,15 +78,6 @@ bool __attribute__((pure)) is_local_api_user(const char *remote_addr) // Returns >= 0 for any valid authentication int check_client_auth(struct ftl_conn *api, const bool is_api) { - // Is the user requesting from localhost? - // This may be allowed without authentication depending on the configuration - if(!config.webserver.api.localAPIauth.v.b && is_local_api_user(api->request->remote_addr)) - { - api->message = "no auth required for local user"; - add_request_info(api, NULL); - return API_AUTH_LOCALHOST; - } - // When the pwhash is unset, authentication is disabled if(config.webserver.api.pwhash.v.s[0] == '\0') { @@ -322,7 +313,7 @@ static int get_session_object(struct ftl_conn *api, cJSON *json, const int user_ cJSON *session = JSON_NEW_OBJECT(); // Authentication not needed - if(user_id == API_AUTH_LOCALHOST || user_id == API_AUTH_EMPTYPASS) + if(user_id == API_AUTH_EMPTYPASS) { JSON_ADD_BOOL_TO_OBJECT(session, "valid", true); JSON_ADD_BOOL_TO_OBJECT(session, "totp", strlen(config.webserver.api.totp_secret.v.s) > 0); @@ -418,14 +409,6 @@ static int send_api_auth_status(struct ftl_conn *api, const int user_id, const t JSON_SEND_OBJECT_CODE(json, 401); // 401 Unauthorized } } - else if(user_id == API_AUTH_LOCALHOST) - { - log_debug(DEBUG_API, "API Auth status: OK (localhost does not need auth)"); - - cJSON *json = JSON_NEW_OBJECT(); - get_session_object(api, json, user_id, now); - JSON_SEND_OBJECT(json); - } else if(user_id == API_AUTH_EMPTYPASS) { log_debug(DEBUG_API, "API Auth status: OK (empty password)"); diff --git a/src/api/docs/content/specs/config.yaml b/src/api/docs/content/specs/config.yaml index c6368c14..d2574172 100644 --- a/src/api/docs/content/specs/config.yaml +++ b/src/api/docs/content/specs/config.yaml @@ -395,8 +395,6 @@ components: api: type: object properties: - localAPIauth: - type: boolean max_sessions: type: integer prettyJSON: @@ -691,7 +689,6 @@ components: boxed: true theme: "default-darker" api: - localAPIauth: false max_sessions: 16 prettyJSON: false password: "********" diff --git a/src/config/config.c b/src/config/config.c index 0c3da544..335bbbd2 100644 --- a/src/config/config.c +++ b/src/config/config.c @@ -961,12 +961,6 @@ void initConfig(struct config *conf) conf->webserver.interface.theme.c = validate_stub; // Only type-based checking // sub-struct api - conf->webserver.api.localAPIauth.k = "webserver.api.localAPIauth"; - conf->webserver.api.localAPIauth.h = "Do local clients need to authenticate to access the API? This settings allows local clients to use the API without authentication."; - conf->webserver.api.localAPIauth.t = CONF_BOOL; - conf->webserver.api.localAPIauth.d.b = true; - conf->webserver.api.localAPIauth.c = validate_stub; // Only type-based checking - conf->webserver.api.max_sessions.k = "webserver.api.max_sessions"; conf->webserver.api.max_sessions.h = "Number of concurrent sessions allowed for the API. If the number of sessions exceeds this value, no new sessions will be allowed until the number of sessions drops due to session expiration or logout. Note that the number of concurrent sessions is irrelevant if authentication is disabled as no sessions are used in this case."; conf->webserver.api.max_sessions.t = CONF_UINT16; diff --git a/src/config/config.h b/src/config/config.h index 0af11040..e22ea88c 100644 --- a/src/config/config.h +++ b/src/config/config.h @@ -229,7 +229,6 @@ struct config { struct conf_item theme; } interface; struct { - struct conf_item localAPIauth; struct conf_item max_sessions; struct conf_item prettyJSON; struct conf_item pwhash; diff --git a/src/config/legacy_reader.c b/src/config/legacy_reader.c index 75e9118a..8e64cfed 100644 --- a/src/config/legacy_reader.c +++ b/src/config/legacy_reader.c @@ -309,11 +309,6 @@ const char *readFTLlegacy(struct config *conf) if(buffer != NULL) conf->webserver.acl.v.s = strdup(buffer); - // API_AUTH_FOR_LOCALHOST - // defaults to: true - buffer = parseFTLconf(fp, "API_AUTH_FOR_LOCALHOST"); - parseBool(buffer, &conf->webserver.api.localAPIauth.v.b); - // API_SESSION_TIMEOUT // How long should a session be considered valid after login? // defaults to: 300 seconds diff --git a/src/enums.h b/src/enums.h index 09769a9c..640d2285 100644 --- a/src/enums.h +++ b/src/enums.h @@ -228,8 +228,7 @@ enum refresh_hostnames { enum api_auth_status { API_AUTH_UNAUTHORIZED = -1, - API_AUTH_LOCALHOST = -2, - API_AUTH_EMPTYPASS = -3, + API_AUTH_EMPTYPASS = -2, } __attribute__ ((packed)); enum db_result { diff --git a/src/lua/ftl_lua.c b/src/lua/ftl_lua.c index c5a40066..5c8e12e6 100644 --- a/src/lua/ftl_lua.c +++ b/src/lua/ftl_lua.c @@ -230,26 +230,13 @@ static int pihole_boxedlayout(lua_State *L) { return 1; // number of results } -// pihole.needLogin(remote_addr:str) +// pihole.needLogin() static int pihole_needLogin(lua_State *L) { - // Get remote_addr (first argument to LUA function) - const char *remote_addr = luaL_checkstring(L, 1); - // Check if password is set const bool has_password = config.webserver.api.pwhash.v.s != NULL && config.webserver.api.pwhash.v.s[0] != '\0'; - // Check if address is loopback - const bool is_loopback = strcmp(remote_addr, LOCALHOSTv4) == 0 || - strcmp(remote_addr, LOCALHOSTv6) == 0; - - // Check if local API authentication is enabled - const bool localAPIauth = config.webserver.api.localAPIauth.v.b; - - // Check if login is required - const bool need_login = has_password || (is_loopback && !localAPIauth); - - lua_pushboolean(L, need_login); + lua_pushboolean(L, has_password); return 1; // number of results } diff --git a/test/pihole.toml b/test/pihole.toml index 1ddc6f7b..4a881f3a 100644 --- a/test/pihole.toml +++ b/test/pihole.toml @@ -656,10 +656,6 @@ theme = "default-auto" [webserver.api] - # Do local clients need to authenticate to access the API? This settings allows local - # clients to use the API without authentication. - localAPIauth = true - # Number of concurrent sessions allowed for the API. If the number of sessions exceeds # this value, no new sessions will be allowed until the number of sessions drops due # to session expiration or logout. Note that the number of concurrent sessions is @@ -1049,7 +1045,7 @@ all = true ### CHANGED, default = false # Configuration statistics: -# 137 total entries out of which 83 entries are default +# 136 total entries out of which 82 entries are default # --> 54 entries are modified # 2 entries are forced through environment: # - misc.nice From a5569d54aec1253015f465bc6528491afd0ce64f Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 19 Jun 2024 22:14:52 +0200 Subject: [PATCH 174/339] Add CI test for generation and permissions of CLI password file Signed-off-by: DL6ER --- test/test_suite.bats | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/test/test_suite.bats b/test/test_suite.bats index e020846c..bebdbc13 100644 --- a/test/test_suite.bats +++ b/test/test_suite.bats @@ -1541,6 +1541,28 @@ [[ ${lines[0]} == "true" ]] } +@test "CLI password file is as expected" { + # Check the file is non-empty + run bash -c 'cat /etc/pihole/cli_pw' + printf "%s\n" "${lines[@]}" + [[ ${#lines[0]} -gt 0 ]] + + # Check if file has exactly one line + [[ ${#lines[@]} -eq 1 ]] + + # Check if this line does NOT have a newline character at the end + [[ ${lines[0]} != *$'\n' ]] + + # Check the file content is valid base64 + run bash -c 'echo ${0} | base64 -d > /dev/null' "${lines[0]}" + [[ $status == 0 ]] + + # Check permission set on the file is 640 + run bash -c 'stat -c "%a" /etc/pihole/cli_pw' + printf "%s\n" "${lines[@]}" + [[ ${lines[0]} == "640" ]] +} + @test "API authorization: Setting password" { # Password: ABC run bash -c 'curl -s -X PATCH http://127.0.0.1/api/config/webserver/api/password -d "{\"config\":{\"webserver\":{\"api\":{\"password\":\"ABC\"}}}}"' From 9e329be3ef072d16300cb4892bf4d62766ee6d5e Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 21 Jun 2024 19:15:45 +0200 Subject: [PATCH 175/339] Fix DNS-SD query analysis Signed-off-by: DL6ER --- src/dnsmasq_interface.c | 46 ++++++++++++++++++++++++++++++++--------- 1 file changed, 36 insertions(+), 10 deletions(-) diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c index b6ee3bb0..366195bc 100644 --- a/src/dnsmasq_interface.c +++ b/src/dnsmasq_interface.c @@ -61,7 +61,7 @@ // Private prototypes static void print_flags(const unsigned int flags); -#define query_set_reply(flags, type, addr, query, response) _query_set_reply(flags, type, addr, query, response, __FILE__, __LINE__) +#define query_set_reply(flags, reply, addr, query, response) _query_set_reply(flags, reply, addr, query, response, __FILE__, __LINE__) static void _query_set_reply(const unsigned int flags, const enum reply_type reply, const union all_addr *addr, queriesData* query, const struct timeval response, const char *file, const int line); #define FTL_check_blocking(queryID, domainID, clientID) _FTL_check_blocking(queryID, domainID, clientID, __FILE__, __LINE__) @@ -69,7 +69,7 @@ static bool _FTL_check_blocking(int queryID, int domainID, int clientID, const c static enum query_status detect_blocked_IP(const unsigned short flags, const union all_addr *addr, const queriesData *query, const domainsData *domain); static void query_blocked(queriesData* query, domainsData* domain, clientsData* client, const enum query_status new_status); static void FTL_forwarded(const unsigned int flags, const char *name, const union all_addr *addr, unsigned short port, const int id, const char* file, const int line); -static void FTL_reply(const unsigned int flags, const char *name, const union all_addr *addr, const char* arg, const int id, const char* file, const int line); +static void FTL_reply(const unsigned int flags, const char *name, const union all_addr *addr, const char* arg, unsigned short type, const int id, const char* file, const int line); static void FTL_upstream_error(const union all_addr *addr, const unsigned int flags, const int id, const char* file, const int line); static void FTL_dnssec(const char *result, const union all_addr *addr, const int id, const char* file, const int line); static void mysockaddr_extract_ip_port(const union mysockaddr *server, char ip[ADDRSTRLEN+1], in_port_t *port); @@ -180,7 +180,7 @@ void FTL_hook(unsigned int flags, const char *name, union all_addr *addr, char * // otherwise, flags will be F_UPSTREAM and the type is not set // (== 0) else - FTL_reply(flags, name, addr, arg, id, path, line); + FTL_reply(flags, name, addr, arg, type, id, path, line); } // This is inspired by make_local_answer() @@ -1901,7 +1901,7 @@ static void update_upstream(queriesData *query, const int id) } static void FTL_reply(const unsigned int flags, const char *name, const union all_addr *addr, - const char *arg, const int id, const char* file, const int line) + const char *arg, unsigned short type, const int id, const char* file, const int line) { const double now = double_time(); // If domain is "pi.hole", we skip this query @@ -2000,6 +2000,16 @@ static void FTL_reply(const unsigned int flags, const char *name, const union al if(!name || strlen(name) == 0) dispname = "."; + // Swap display name with answer if this is a reverse query + // Check for reverse query by looking at the query type not only + // the flag as some PTR queries are not flagged (DNS-SD) + if(flags & F_REVERSE || type == T_PTR) + { + const char *tmp = dispname; + dispname = answer; + answer = tmp; + } + if(cached || last_server.sa.sa_family == 0) { // Log cache or upstream reply from unknown source @@ -2204,19 +2214,34 @@ static void FTL_reply(const unsigned int flags, const char *name, const union al // Mark query for updating in the database query->flags.database.changed = true; } - else if(flags & F_REVERSE) + else if(flags & F_REVERSE || type == T_PTR) { // isExactMatch is not used here as the PTR is special. // Example: - // Question: PTR 8.8.8.8 + // Question: PTR -x 8.8.8.8 // will lead to: // domain->domain = 8.8.8.8.in-addr.arpa - // and will return - // name = google-public-dns-a.google.com + // name = 8.8.8.8 (derived above from addr) + // answer = dns.google // Hence, isExactMatch is always false + // DNS-SD example: + // Question: PTR _http._tcp.local + // will lead to: + // domain->domain = obs.cr + // name = (null) + // answer = obs.cr + + // if flags does not contain F_REVERSE, it is not a reverse + // query, e.g. DNS-SD + unsigned int pflags = flags; + if(!(flags & F_REVERSE)) + pflags |= F_RRNAME; // Save reply type and update individual reply counters - query_set_reply(flags, 0, addr, query, response); + query_set_reply(pflags, 0, addr, query, response); + + // Hereby, this query is now fully determined + query->flags.complete = true; // Mark query for updating in the database query->flags.database.changed = true; @@ -2227,7 +2252,8 @@ static void FTL_reply(const unsigned int flags, const char *name, const union al } else if(config.debug.flags.v.b) { - log_warn("Unknown upstream REPLY"); + log_warn("Unknown upstream REPLY, exact: %s, type: %u", + isExactMatch ? "true" : "false", type); } if(query && option_bool(OPT_DNSSEC_PROXY)) From b45695c3bded7a036a3fcf1571d519c26ced1b63 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 25 Jun 2024 10:01:53 +0200 Subject: [PATCH 176/339] Restart FTL if system time has been updated by more than one hour using the internal NTP synchronization method. This ensures FTL can import the real most recent 24 hours data of history after a restart on a system lacking a real hardware clock Signed-off-by: DL6ER --- src/ntp/client.c | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/src/ntp/client.c b/src/ntp/client.c index 1272d30c..c312c8ac 100644 --- a/src/ntp/client.c +++ b/src/ntp/client.c @@ -484,8 +484,8 @@ bool ntp_client(const char *server, const bool settime, const bool print) theta_stdev = sqrt(theta_stdev / valid); delta_stdev = sqrt(delta_stdev / valid); - log_info("Average time offset: (%e +/- %e s)", theta_avg, theta_stdev); - log_info("Average round-trip delay: (%e +/- %e s)", delta_avg, delta_stdev); + log_debug(DEBUG_NTP, "Average time offset: (%e +/- %e) s", theta_avg, theta_stdev); + log_debug(DEBUG_NTP, "Average round-trip delay: (%e +/- %e) s", delta_avg, delta_stdev); // Reject synchronization if the standard deviation of the time offset // or round-trip delay is larger than 1 second @@ -529,8 +529,8 @@ bool ntp_client(const char *server, const bool settime, const bool print) theta_trim /= trim; delta_trim /= trim; - log_info("Trimmed mean time offset: %e s (excluded %u outliers)", theta_trim, count - trim); - log_info("Trimmed mean round-trip delay: %e s (excluded %u outliers)", delta_trim, count - trim); + log_info("Time offset: %e ms (excluded %u outliers)", 1e3*theta_trim, count - trim); + log_info("Round-trip delay: %e ms (excluded %u outliers)", 1e3*delta_trim, count - trim); // Set time if requested if(settime) @@ -585,9 +585,31 @@ static void *ntp_client_thread(void *arg) // Run NTP client while(!killed) { + + // Get time before NTP sync + const time_t before = time(NULL); + // Run NTP client ntp_client(config.ntp.sync.server.v.s, true, false); + // Get time after NTP sync + const time_t after = time(NULL); + + // If the time was updated by more than one hour, restart FTL to + // import recent data. This is relevant when the system time was + // set to an incorrect value (e.g., due to a dead CMOS battery + // or overall missing RTC) and the time was off. + if(after - before > 3600) + { + log_info("System time was updated by more than one hour, restarting FTL to import recent data"); + // Set the restart flag to true + exit_code = RESTART_FTL_CODE; + // Send SIGTERM to FTL + kill(main_pid(), SIGTERM); + // Kill the NTP thread + killed = true; + } + // Intermediate cancellation-point BREAK_IF_KILLED(); From f863c058d18830da868154b971b0337ba317746e Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 25 Jun 2024 16:12:15 +0200 Subject: [PATCH 177/339] Update embedded LUA engine to 5.4.7 Signed-off-by: DL6ER --- src/lua/lapi.c | 4 +- src/lua/lauxlib.c | 28 ++++-- src/lua/lcode.c | 35 ++++---- src/lua/lcode.h | 3 - src/lua/ldebug.c | 218 ++++++++++++++++++++++++++------------------- src/lua/ldebug.h | 1 + src/lua/ldo.c | 10 ++- src/lua/ldo.h | 1 - src/lua/lgc.c | 20 +++-- src/lua/liolib.c | 27 ++++-- src/lua/lmathlib.c | 31 +++++-- src/lua/loadlib.c | 9 -- src/lua/lobject.c | 2 +- src/lua/lobject.h | 18 ++-- src/lua/lopcodes.h | 8 +- src/lua/loslib.c | 2 + src/lua/lparser.c | 12 +-- src/lua/lstate.c | 6 +- src/lua/lstate.h | 3 +- src/lua/lstring.c | 13 +-- src/lua/ltable.c | 39 +++++--- src/lua/ltable.h | 2 - src/lua/ltm.h | 5 +- src/lua/lua.c | 17 +++- src/lua/lua.h | 8 +- src/lua/luaconf.h | 9 ++ src/lua/lundump.c | 4 +- src/lua/lundump.h | 3 +- src/lua/lvm.c | 78 ++++++++-------- 29 files changed, 362 insertions(+), 254 deletions(-) diff --git a/src/lua/lapi.c b/src/lua/lapi.c index 34e64af1..332e97d1 100644 --- a/src/lua/lapi.c +++ b/src/lua/lapi.c @@ -417,9 +417,9 @@ LUA_API const char *lua_tolstring (lua_State *L, int idx, size_t *len) { o = index2value(L, idx); /* previous call may reallocate the stack */ } if (len != NULL) - *len = vslen(o); + *len = tsslen(tsvalue(o)); lua_unlock(L); - return svalue(o); + return getstr(tsvalue(o)); } diff --git a/src/lua/lauxlib.c b/src/lua/lauxlib.c index 4ca6c654..923105ed 100644 --- a/src/lua/lauxlib.c +++ b/src/lua/lauxlib.c @@ -80,6 +80,7 @@ static int pushglobalfuncname (lua_State *L, lua_Debug *ar) { int top = lua_gettop(L); lua_getinfo(L, "f", ar); /* push function */ lua_getfield(L, LUA_REGISTRYINDEX, LUA_LOADED_TABLE); + luaL_checkstack(L, 6, "not enough stack"); /* slots for 'findfield' */ if (findfield(L, top + 1, 2)) { const char *name = lua_tostring(L, -1); if (strncmp(name, LUA_GNAME ".", 3) == 0) { /* name start with '_G.'? */ @@ -249,11 +250,13 @@ LUALIB_API int luaL_fileresult (lua_State *L, int stat, const char *fname) { return 1; } else { + const char *msg; luaL_pushfail(L); + msg = (en != 0) ? strerror(en) : "(no extra info)"; if (fname) - lua_pushfstring(L, "%s: %s", fname, strerror(en)); + lua_pushfstring(L, "%s: %s", fname, msg); else - lua_pushstring(L, strerror(en)); + lua_pushstring(L, msg); lua_pushinteger(L, en); return 3; } @@ -732,9 +735,12 @@ static const char *getF (lua_State *L, void *ud, size_t *size) { static int errfile (lua_State *L, const char *what, int fnameindex) { - const char *serr = strerror(errno); + int err = errno; const char *filename = lua_tostring(L, fnameindex) + 1; - lua_pushfstring(L, "cannot %s %s: %s", what, filename, serr); + if (err != 0) + lua_pushfstring(L, "cannot %s %s: %s", what, filename, strerror(err)); + else + lua_pushfstring(L, "cannot %s %s", what, filename); lua_remove(L, fnameindex); return LUA_ERRFILE; } @@ -787,6 +793,7 @@ LUALIB_API int luaL_loadfilex (lua_State *L, const char *filename, } else { lua_pushfstring(L, "@%s", filename); + errno = 0; lf.f = fopen(filename, "r"); if (lf.f == NULL) return errfile(L, "open", fnameindex); } @@ -796,6 +803,7 @@ LUALIB_API int luaL_loadfilex (lua_State *L, const char *filename, if (c == LUA_SIGNATURE[0]) { /* binary file? */ lf.n = 0; /* remove possible newline */ if (filename) { /* "real" file? */ + errno = 0; lf.f = freopen(filename, "rb", lf.f); /* reopen in binary mode */ if (lf.f == NULL) return errfile(L, "reopen", fnameindex); skipcomment(lf.f, &c); /* re-read initial portion */ @@ -803,6 +811,7 @@ LUALIB_API int luaL_loadfilex (lua_State *L, const char *filename, } if (c != EOF) lf.buff[lf.n++] = c; /* 'c' is the first character of the stream */ + errno = 0; status = lua_load(L, getF, &lf, lua_tostring(L, -1), mode); readstatus = ferror(lf.f); if (filename) fclose(lf.f); /* close file (even in case of errors) */ @@ -933,7 +942,7 @@ LUALIB_API const char *luaL_tolstring (lua_State *L, int idx, size_t *len) { LUALIB_API void luaL_setfuncs (lua_State *L, const luaL_Reg *l, int nup) { luaL_checkstack(L, nup, "too many upvalues"); for (; l->name != NULL; l++) { /* fill the table with given functions */ - if (l->func == NULL) /* place holder? */ + if (l->func == NULL) /* placeholder? */ lua_pushboolean(L, 0); else { int i; @@ -1025,9 +1034,14 @@ static void *l_alloc (void *ud, void *ptr, size_t osize, size_t nsize) { } +/* +** Standard panic funcion just prints an error message. The test +** with 'lua_type' avoids possible memory errors in 'lua_tostring'. +*/ static int panic (lua_State *L) { - const char *msg = lua_tostring(L, -1); - if (msg == NULL) msg = "error object is not a string"; + const char *msg = (lua_type(L, -1) == LUA_TSTRING) + ? lua_tostring(L, -1) + : "error object is not a string"; lua_writestringerror("PANIC: unprotected error in call to Lua API (%s)\n", msg); return 0; /* return to Lua to abort */ diff --git a/src/lua/lcode.c b/src/lua/lcode.c index 1a371ca9..87616140 100644 --- a/src/lua/lcode.c +++ b/src/lua/lcode.c @@ -415,7 +415,7 @@ int luaK_codeABx (FuncState *fs, OpCode o, int a, unsigned int bc) { /* ** Format and emit an 'iAsBx' instruction. */ -int luaK_codeAsBx (FuncState *fs, OpCode o, int a, int bc) { +static int codeAsBx (FuncState *fs, OpCode o, int a, int bc) { unsigned int b = bc + OFFSET_sBx; lua_assert(getOpMode(o) == iAsBx); lua_assert(a <= MAXARG_A && b <= MAXARG_Bx); @@ -671,7 +671,7 @@ static int fitsBx (lua_Integer i) { void luaK_int (FuncState *fs, int reg, lua_Integer i) { if (fitsBx(i)) - luaK_codeAsBx(fs, OP_LOADI, reg, cast_int(i)); + codeAsBx(fs, OP_LOADI, reg, cast_int(i)); else luaK_codek(fs, reg, luaK_intK(fs, i)); } @@ -680,7 +680,7 @@ void luaK_int (FuncState *fs, int reg, lua_Integer i) { static void luaK_float (FuncState *fs, int reg, lua_Number f) { lua_Integer fi; if (luaV_flttointeger(f, &fi, F2Ieq) && fitsBx(fi)) - luaK_codeAsBx(fs, OP_LOADF, reg, cast_int(fi)); + codeAsBx(fs, OP_LOADF, reg, cast_int(fi)); else luaK_codek(fs, reg, luaK_numberK(fs, f)); } @@ -776,7 +776,8 @@ void luaK_dischargevars (FuncState *fs, expdesc *e) { break; } case VLOCAL: { /* already in a register */ - e->u.info = e->u.var.ridx; + int temp = e->u.var.ridx; + e->u.info = temp; /* (can't do a direct assignment; values overlap) */ e->k = VNONRELOC; /* becomes a non-relocatable value */ break; } @@ -1025,7 +1026,7 @@ static int luaK_exp2K (FuncState *fs, expdesc *e) { ** in the range of R/K indices). ** Returns 1 iff expression is K. */ -int luaK_exp2RK (FuncState *fs, expdesc *e) { +static int exp2RK (FuncState *fs, expdesc *e) { if (luaK_exp2K(fs, e)) return 1; else { /* not a constant in the right range: put it in a register */ @@ -1037,7 +1038,7 @@ int luaK_exp2RK (FuncState *fs, expdesc *e) { static void codeABRK (FuncState *fs, OpCode o, int a, int b, expdesc *ec) { - int k = luaK_exp2RK(fs, ec); + int k = exp2RK(fs, ec); luaK_codeABCk(fs, o, a, b, ec->u.info, k); } @@ -1215,7 +1216,7 @@ static void codenot (FuncState *fs, expdesc *e) { /* -** Check whether expression 'e' is a small literal string +** Check whether expression 'e' is a short literal string */ static int isKstr (FuncState *fs, expdesc *e) { return (e->k == VK && !hasjumps(e) && e->u.info <= MAXARG_B && @@ -1225,7 +1226,7 @@ static int isKstr (FuncState *fs, expdesc *e) { /* ** Check whether expression 'e' is a literal integer. */ -int luaK_isKint (expdesc *e) { +static int isKint (expdesc *e) { return (e->k == VKINT && !hasjumps(e)); } @@ -1235,7 +1236,7 @@ int luaK_isKint (expdesc *e) { ** proper range to fit in register C */ static int isCint (expdesc *e) { - return luaK_isKint(e) && (l_castS2U(e->u.ival) <= l_castS2U(MAXARG_C)); + return isKint(e) && (l_castS2U(e->u.ival) <= l_castS2U(MAXARG_C)); } @@ -1244,7 +1245,7 @@ static int isCint (expdesc *e) { ** proper range to fit in register sC */ static int isSCint (expdesc *e) { - return luaK_isKint(e) && fitsC(e->u.ival); + return isKint(e) && fitsC(e->u.ival); } @@ -1283,15 +1284,17 @@ void luaK_indexed (FuncState *fs, expdesc *t, expdesc *k) { if (t->k == VUPVAL && !isKstr(fs, k)) /* upvalue indexed by non 'Kstr'? */ luaK_exp2anyreg(fs, t); /* put it in a register */ if (t->k == VUPVAL) { - t->u.ind.t = t->u.info; /* upvalue index */ - t->u.ind.idx = k->u.info; /* literal string */ + int temp = t->u.info; /* upvalue index */ + lua_assert(isKstr(fs, k)); + t->u.ind.t = temp; /* (can't do a direct assignment; values overlap) */ + t->u.ind.idx = k->u.info; /* literal short string */ t->k = VINDEXUP; } else { /* register index of the table */ t->u.ind.t = (t->k == VLOCAL) ? t->u.var.ridx: t->u.info; if (isKstr(fs, k)) { - t->u.ind.idx = k->u.info; /* literal string */ + t->u.ind.idx = k->u.info; /* literal short string */ t->k = VINDEXSTR; } else if (isCint(k)) { @@ -1459,7 +1462,7 @@ static void codebinK (FuncState *fs, BinOpr opr, */ static int finishbinexpneg (FuncState *fs, expdesc *e1, expdesc *e2, OpCode op, int line, TMS event) { - if (!luaK_isKint(e2)) + if (!isKint(e2)) return 0; /* not an integer constant */ else { lua_Integer i2 = e2->u.ival; @@ -1592,7 +1595,7 @@ static void codeeq (FuncState *fs, BinOpr opr, expdesc *e1, expdesc *e2) { op = OP_EQI; r2 = im; /* immediate operand */ } - else if (luaK_exp2RK(fs, e2)) { /* 2nd expression is constant? */ + else if (exp2RK(fs, e2)) { /* 2nd expression is constant? */ op = OP_EQK; r2 = e2->u.info; /* constant index */ } @@ -1658,7 +1661,7 @@ void luaK_infix (FuncState *fs, BinOpr op, expdesc *v) { } case OPR_EQ: case OPR_NE: { if (!tonumeral(v, NULL)) - luaK_exp2RK(fs, v); + exp2RK(fs, v); /* else keep numeral, which may be an immediate operand */ break; } diff --git a/src/lua/lcode.h b/src/lua/lcode.h index 32658244..0b971fc4 100644 --- a/src/lua/lcode.h +++ b/src/lua/lcode.h @@ -61,10 +61,8 @@ typedef enum UnOpr { OPR_MINUS, OPR_BNOT, OPR_NOT, OPR_LEN, OPR_NOUNOPR } UnOpr; LUAI_FUNC int luaK_code (FuncState *fs, Instruction i); LUAI_FUNC int luaK_codeABx (FuncState *fs, OpCode o, int A, unsigned int Bx); -LUAI_FUNC int luaK_codeAsBx (FuncState *fs, OpCode o, int A, int Bx); LUAI_FUNC int luaK_codeABCk (FuncState *fs, OpCode o, int A, int B, int C, int k); -LUAI_FUNC int luaK_isKint (expdesc *e); LUAI_FUNC int luaK_exp2const (FuncState *fs, const expdesc *e, TValue *v); LUAI_FUNC void luaK_fixline (FuncState *fs, int line); LUAI_FUNC void luaK_nil (FuncState *fs, int from, int n); @@ -76,7 +74,6 @@ LUAI_FUNC int luaK_exp2anyreg (FuncState *fs, expdesc *e); LUAI_FUNC void luaK_exp2anyregup (FuncState *fs, expdesc *e); LUAI_FUNC void luaK_exp2nextreg (FuncState *fs, expdesc *e); LUAI_FUNC void luaK_exp2val (FuncState *fs, expdesc *e); -LUAI_FUNC int luaK_exp2RK (FuncState *fs, expdesc *e); LUAI_FUNC void luaK_self (FuncState *fs, expdesc *e, expdesc *key); LUAI_FUNC void luaK_indexed (FuncState *fs, expdesc *t, expdesc *k); LUAI_FUNC void luaK_goiftrue (FuncState *fs, expdesc *e); diff --git a/src/lua/ldebug.c b/src/lua/ldebug.c index 28b1caab..591b3528 100644 --- a/src/lua/ldebug.c +++ b/src/lua/ldebug.c @@ -31,7 +31,7 @@ -#define noLuaClosure(f) ((f) == NULL || (f)->c.tt == LUA_VCCL) +#define LuaClosure(f) ((f) != NULL && (f)->c.tt == LUA_VLCL) static const char *funcnamefromcall (lua_State *L, CallInfo *ci, @@ -254,7 +254,7 @@ LUA_API const char *lua_setlocal (lua_State *L, const lua_Debug *ar, int n) { static void funcinfo (lua_Debug *ar, Closure *cl) { - if (noLuaClosure(cl)) { + if (!LuaClosure(cl)) { ar->source = "=[C]"; ar->srclen = LL("=[C]"); ar->linedefined = -1; @@ -288,29 +288,31 @@ static int nextline (const Proto *p, int currentline, int pc) { static void collectvalidlines (lua_State *L, Closure *f) { - if (noLuaClosure(f)) { + if (!LuaClosure(f)) { setnilvalue(s2v(L->top.p)); api_incr_top(L); } else { - int i; - TValue v; const Proto *p = f->l.p; int currentline = p->linedefined; Table *t = luaH_new(L); /* new table to store active lines */ sethvalue2s(L, L->top.p, t); /* push it on stack */ api_incr_top(L); - setbtvalue(&v); /* boolean 'true' to be the value of all indices */ - if (!p->is_vararg) /* regular function? */ - i = 0; /* consider all instructions */ - else { /* vararg function */ - lua_assert(GET_OPCODE(p->code[0]) == OP_VARARGPREP); - currentline = nextline(p, currentline, 0); - i = 1; /* skip first instruction (OP_VARARGPREP) */ - } - for (; i < p->sizelineinfo; i++) { /* for each instruction */ - currentline = nextline(p, currentline, i); /* get its line */ - luaH_setint(L, t, currentline, &v); /* table[line] = true */ + if (p->lineinfo != NULL) { /* proto with debug information? */ + int i; + TValue v; + setbtvalue(&v); /* boolean 'true' to be the value of all indices */ + if (!p->is_vararg) /* regular function? */ + i = 0; /* consider all instructions */ + else { /* vararg function */ + lua_assert(GET_OPCODE(p->code[0]) == OP_VARARGPREP); + currentline = nextline(p, currentline, 0); + i = 1; /* skip first instruction (OP_VARARGPREP) */ + } + for (; i < p->sizelineinfo; i++) { /* for each instruction */ + currentline = nextline(p, currentline, i); /* get its line */ + luaH_setint(L, t, currentline, &v); /* table[line] = true */ + } } } } @@ -339,7 +341,7 @@ static int auxgetinfo (lua_State *L, const char *what, lua_Debug *ar, } case 'u': { ar->nups = (f == NULL) ? 0 : f->c.nupvalues; - if (noLuaClosure(f)) { + if (!LuaClosure(f)) { ar->isvararg = 1; ar->nparams = 0; } @@ -417,40 +419,6 @@ LUA_API int lua_getinfo (lua_State *L, const char *what, lua_Debug *ar) { ** ======================================================= */ -static const char *getobjname (const Proto *p, int lastpc, int reg, - const char **name); - - -/* -** Find a "name" for the constant 'c'. -*/ -static void kname (const Proto *p, int c, const char **name) { - TValue *kvalue = &p->k[c]; - *name = (ttisstring(kvalue)) ? svalue(kvalue) : "?"; -} - - -/* -** Find a "name" for the register 'c'. -*/ -static void rname (const Proto *p, int pc, int c, const char **name) { - const char *what = getobjname(p, pc, c, name); /* search for 'c' */ - if (!(what && *what == 'c')) /* did not find a constant name? */ - *name = "?"; -} - - -/* -** Find a "name" for a 'C' value in an RK instruction. -*/ -static void rkname (const Proto *p, int pc, Instruction i, const char **name) { - int c = GETARG_C(i); /* key index */ - if (GETARG_k(i)) /* is 'c' a constant? */ - kname(p, c, name); - else /* 'c' is a register */ - rname(p, pc, c, name); -} - static int filterpc (int pc, int jmptarget) { if (pc < jmptarget) /* is code conditional (inside a jump)? */ @@ -509,28 +477,29 @@ static int findsetreg (const Proto *p, int lastpc, int reg) { /* -** Check whether table being indexed by instruction 'i' is the -** environment '_ENV' +** Find a "name" for the constant 'c'. */ -static const char *gxf (const Proto *p, int pc, Instruction i, int isup) { - int t = GETARG_B(i); /* table index */ - const char *name; /* name of indexed variable */ - if (isup) /* is an upvalue? */ - name = upvalname(p, t); - else - getobjname(p, pc, t, &name); - return (name && strcmp(name, LUA_ENV) == 0) ? "global" : "field"; +static const char *kname (const Proto *p, int index, const char **name) { + TValue *kvalue = &p->k[index]; + if (ttisstring(kvalue)) { + *name = getstr(tsvalue(kvalue)); + return "constant"; + } + else { + *name = "?"; + return NULL; + } } -static const char *getobjname (const Proto *p, int lastpc, int reg, - const char **name) { - int pc; - *name = luaF_getlocalname(p, reg + 1, lastpc); +static const char *basicgetobjname (const Proto *p, int *ppc, int reg, + const char **name) { + int pc = *ppc; + *name = luaF_getlocalname(p, reg + 1, pc); if (*name) /* is a local? */ return "local"; /* else try symbolic execution */ - pc = findsetreg(p, lastpc, reg); + *ppc = pc = findsetreg(p, pc, reg); if (pc != -1) { /* could find instruction? */ Instruction i = p->code[pc]; OpCode op = GET_OPCODE(i); @@ -538,18 +507,80 @@ static const char *getobjname (const Proto *p, int lastpc, int reg, case OP_MOVE: { int b = GETARG_B(i); /* move from 'b' to 'a' */ if (b < GETARG_A(i)) - return getobjname(p, pc, b, name); /* get name for 'b' */ + return basicgetobjname(p, ppc, b, name); /* get name for 'b' */ break; } + case OP_GETUPVAL: { + *name = upvalname(p, GETARG_B(i)); + return "upvalue"; + } + case OP_LOADK: return kname(p, GETARG_Bx(i), name); + case OP_LOADKX: return kname(p, GETARG_Ax(p->code[pc + 1]), name); + default: break; + } + } + return NULL; /* could not find reasonable name */ +} + + +/* +** Find a "name" for the register 'c'. +*/ +static void rname (const Proto *p, int pc, int c, const char **name) { + const char *what = basicgetobjname(p, &pc, c, name); /* search for 'c' */ + if (!(what && *what == 'c')) /* did not find a constant name? */ + *name = "?"; +} + + +/* +** Find a "name" for a 'C' value in an RK instruction. +*/ +static void rkname (const Proto *p, int pc, Instruction i, const char **name) { + int c = GETARG_C(i); /* key index */ + if (GETARG_k(i)) /* is 'c' a constant? */ + kname(p, c, name); + else /* 'c' is a register */ + rname(p, pc, c, name); +} + + +/* +** Check whether table being indexed by instruction 'i' is the +** environment '_ENV' +*/ +static const char *isEnv (const Proto *p, int pc, Instruction i, int isup) { + int t = GETARG_B(i); /* table index */ + const char *name; /* name of indexed variable */ + if (isup) /* is 't' an upvalue? */ + name = upvalname(p, t); + else /* 't' is a register */ + basicgetobjname(p, &pc, t, &name); + return (name && strcmp(name, LUA_ENV) == 0) ? "global" : "field"; +} + + +/* +** Extend 'basicgetobjname' to handle table accesses +*/ +static const char *getobjname (const Proto *p, int lastpc, int reg, + const char **name) { + const char *kind = basicgetobjname(p, &lastpc, reg, name); + if (kind != NULL) + return kind; + else if (lastpc != -1) { /* could find instruction? */ + Instruction i = p->code[lastpc]; + OpCode op = GET_OPCODE(i); + switch (op) { case OP_GETTABUP: { int k = GETARG_C(i); /* key index */ kname(p, k, name); - return gxf(p, pc, i, 1); + return isEnv(p, lastpc, i, 1); } case OP_GETTABLE: { int k = GETARG_C(i); /* key index */ - rname(p, pc, k, name); - return gxf(p, pc, i, 0); + rname(p, lastpc, k, name); + return isEnv(p, lastpc, i, 0); } case OP_GETI: { *name = "integer index"; @@ -558,24 +589,10 @@ static const char *getobjname (const Proto *p, int lastpc, int reg, case OP_GETFIELD: { int k = GETARG_C(i); /* key index */ kname(p, k, name); - return gxf(p, pc, i, 0); - } - case OP_GETUPVAL: { - *name = upvalname(p, GETARG_B(i)); - return "upvalue"; - } - case OP_LOADK: - case OP_LOADKX: { - int b = (op == OP_LOADK) ? GETARG_Bx(i) - : GETARG_Ax(p->code[pc + 1]); - if (ttisstring(&p->k[b])) { - *name = svalue(&p->k[b]); - return "constant"; - } - break; + return isEnv(p, lastpc, i, 0); } case OP_SELF: { - rkname(p, pc, i, name); + rkname(p, lastpc, i, name); return "method"; } default: break; /* go through to return NULL */ @@ -627,7 +644,7 @@ static const char *funcnamefromcode (lua_State *L, const Proto *p, default: return NULL; /* cannot find a reasonable name */ } - *name = getstr(G(L)->tmname[tm]) + 2; + *name = getshrstr(G(L)->tmname[tm]) + 2; return "metamethod"; } @@ -865,6 +882,28 @@ static int changedline (const Proto *p, int oldpc, int newpc) { } +/* +** Traces Lua calls. If code is running the first instruction of a function, +** and function is not vararg, and it is not coming from an yield, +** calls 'luaD_hookcall'. (Vararg functions will call 'luaD_hookcall' +** after adjusting its variable arguments; otherwise, they could call +** a line/count hook before the call hook. Functions coming from +** an yield already called 'luaD_hookcall' before yielding.) +*/ +int luaG_tracecall (lua_State *L) { + CallInfo *ci = L->ci; + Proto *p = ci_func(ci)->p; + ci->u.l.trap = 1; /* ensure hooks will be checked */ + if (ci->u.l.savedpc == p->code) { /* first instruction (not resuming)? */ + if (p->is_vararg) + return 0; /* hooks will start at VARARGPREP instruction */ + else if (!(ci->callstatus & CIST_HOOKYIELD)) /* not yieded? */ + luaD_hookcall(L, ci); /* check 'call' hook */ + } + return 1; /* keep 'trap' on */ +} + + /* ** Traces the execution of a Lua function. Called before the execution ** of each opcode, when debug is on. 'L->oldpc' stores the last @@ -888,12 +927,12 @@ int luaG_traceexec (lua_State *L, const Instruction *pc) { } pc++; /* reference is always next instruction */ ci->u.l.savedpc = pc; /* save 'pc' */ - counthook = (--L->hookcount == 0 && (mask & LUA_MASKCOUNT)); + counthook = (mask & LUA_MASKCOUNT) && (--L->hookcount == 0); if (counthook) resethookcount(L); /* reset count */ else if (!(mask & LUA_MASKLINE)) return 1; /* no line hook and count != 0; nothing to be done now */ - if (ci->callstatus & CIST_HOOKYIELD) { /* called hook last time? */ + if (ci->callstatus & CIST_HOOKYIELD) { /* hook yielded last time? */ ci->callstatus &= ~CIST_HOOKYIELD; /* erase mark */ return 1; /* do not call hook again (VM yielded, so it did not move) */ } @@ -915,7 +954,6 @@ int luaG_traceexec (lua_State *L, const Instruction *pc) { if (L->status == LUA_YIELD) { /* did hook yield? */ if (counthook) L->hookcount = 1; /* undo decrement to zero */ - ci->u.l.savedpc--; /* undo increment (resume will increment it again) */ ci->callstatus |= CIST_HOOKYIELD; /* mark that it yielded */ luaD_throw(L, LUA_YIELD); } diff --git a/src/lua/ldebug.h b/src/lua/ldebug.h index 2c3074c6..2bfce3cb 100644 --- a/src/lua/ldebug.h +++ b/src/lua/ldebug.h @@ -58,6 +58,7 @@ LUAI_FUNC const char *luaG_addinfo (lua_State *L, const char *msg, TString *src, int line); LUAI_FUNC l_noret luaG_errormsg (lua_State *L); LUAI_FUNC int luaG_traceexec (lua_State *L, const Instruction *pc); +LUAI_FUNC int luaG_tracecall (lua_State *L); #endif diff --git a/src/lua/ldo.c b/src/lua/ldo.c index 2a0017ca..ea052950 100644 --- a/src/lua/ldo.c +++ b/src/lua/ldo.c @@ -409,7 +409,7 @@ static void rethook (lua_State *L, CallInfo *ci, int nres) { ** stack, below original 'func', so that 'luaD_precall' can call it. Raise ** an error if there is no '__call' metafield. */ -StkId luaD_tryfuncTM (lua_State *L, StkId func) { +static StkId tryfuncTM (lua_State *L, StkId func) { const TValue *tm; StkId p; checkstackGCp(L, 1, func); /* space for metamethod */ @@ -568,7 +568,7 @@ int luaD_pretailcall (lua_State *L, CallInfo *ci, StkId func, return -1; } default: { /* not a function */ - func = luaD_tryfuncTM(L, func); /* try to get '__call' metamethod */ + func = tryfuncTM(L, func); /* try to get '__call' metamethod */ /* return luaD_pretailcall(L, ci, func, narg1 + 1, delta); */ narg1++; goto retry; /* try again */ @@ -609,7 +609,7 @@ CallInfo *luaD_precall (lua_State *L, StkId func, int nresults) { return ci; } default: { /* not a function */ - func = luaD_tryfuncTM(L, func); /* try to get '__call' metamethod */ + func = tryfuncTM(L, func); /* try to get '__call' metamethod */ /* return luaD_precall(L, func, nresults); */ goto retry; /* try again with metamethod */ } @@ -792,6 +792,10 @@ static void resume (lua_State *L, void *ud) { lua_assert(L->status == LUA_YIELD); L->status = LUA_OK; /* mark that it is running (again) */ if (isLua(ci)) { /* yielded inside a hook? */ + /* undo increment made by 'luaG_traceexec': instruction was not + executed yet */ + lua_assert(ci->callstatus & CIST_HOOKYIELD); + ci->u.l.savedpc--; L->top.p = firstArg; /* discard arguments */ luaV_execute(L, ci); /* just continue running Lua code */ } diff --git a/src/lua/ldo.h b/src/lua/ldo.h index 1aa446ad..56008ab3 100644 --- a/src/lua/ldo.h +++ b/src/lua/ldo.h @@ -71,7 +71,6 @@ LUAI_FUNC int luaD_pretailcall (lua_State *L, CallInfo *ci, StkId func, LUAI_FUNC CallInfo *luaD_precall (lua_State *L, StkId func, int nResults); LUAI_FUNC void luaD_call (lua_State *L, StkId func, int nResults); LUAI_FUNC void luaD_callnoyield (lua_State *L, StkId func, int nResults); -LUAI_FUNC StkId luaD_tryfuncTM (lua_State *L, StkId func); LUAI_FUNC int luaD_closeprotected (lua_State *L, ptrdiff_t level, int status); LUAI_FUNC int luaD_pcall (lua_State *L, Pfunc func, void *u, ptrdiff_t oldtop, ptrdiff_t ef); diff --git a/src/lua/lgc.c b/src/lua/lgc.c index a3094ff5..5817f9ee 100644 --- a/src/lua/lgc.c +++ b/src/lua/lgc.c @@ -542,10 +542,12 @@ static void traversestrongtable (global_State *g, Table *h) { static lu_mem traversetable (global_State *g, Table *h) { const char *weakkey, *weakvalue; const TValue *mode = gfasttm(g, h->metatable, TM_MODE); + TString *smode; markobjectN(g, h->metatable); - if (mode && ttisstring(mode) && /* is there a weak mode? */ - (cast_void(weakkey = strchr(svalue(mode), 'k')), - cast_void(weakvalue = strchr(svalue(mode), 'v')), + if (mode && ttisshrstring(mode) && /* is there a weak mode? */ + (cast_void(smode = tsvalue(mode)), + cast_void(weakkey = strchr(getshrstr(smode), 'k')), + cast_void(weakvalue = strchr(getshrstr(smode), 'v')), (weakkey || weakvalue))) { /* is really weak? */ if (!weakkey) /* strong keys? */ traverseweakvalue(g, h); @@ -638,7 +640,9 @@ static int traversethread (global_State *g, lua_State *th) { for (uv = th->openupval; uv != NULL; uv = uv->u.open.next) markobject(g, uv); /* open upvalues cannot be collected */ if (g->gcstate == GCSatomic) { /* final traversal? */ - for (; o < th->stack_last.p + EXTRA_STACK; o++) + if (!g->gcemergency) + luaD_shrinkstack(th); /* do not change stack in emergency cycle */ + for (o = th->top.p; o < th->stack_last.p + EXTRA_STACK; o++) setnilvalue(s2v(o)); /* clear dead stack slice */ /* 'remarkupvals' may have removed thread from 'twups' list */ if (!isintwups(th) && th->openupval != NULL) { @@ -646,8 +650,6 @@ static int traversethread (global_State *g, lua_State *th) { g->twups = th; } } - else if (!g->gcemergency) - luaD_shrinkstack(th); /* do not change stack in emergency cycle */ return 1 + stacksize(th); } @@ -1409,7 +1411,7 @@ static void stepgenfull (lua_State *L, global_State *g) { setminordebt(g); } else { /* another bad collection; stay in incremental mode */ - g->GCestimate = gettotalbytes(g); /* first estimate */; + g->GCestimate = gettotalbytes(g); /* first estimate */ entersweep(L); luaC_runtilstate(L, bitmask(GCSpause)); /* finish collection */ setpause(g); @@ -1604,7 +1606,7 @@ static lu_mem singlestep (lua_State *L) { case GCSenteratomic: { work = atomic(L); /* work is what was traversed by 'atomic' */ entersweep(L); - g->GCestimate = gettotalbytes(g); /* first estimate */; + g->GCestimate = gettotalbytes(g); /* first estimate */ break; } case GCSswpallgc: { /* sweep "regular" objects */ @@ -1710,6 +1712,8 @@ static void fullinc (lua_State *L, global_State *g) { entersweep(L); /* sweep everything to turn them back to white */ /* finish any pending sweep phase to start a new cycle */ luaC_runtilstate(L, bitmask(GCSpause)); + luaC_runtilstate(L, bitmask(GCSpropagate)); /* start new cycle */ + g->gcstate = GCSenteratomic; /* go straight to atomic phase */ luaC_runtilstate(L, bitmask(GCScallfin)); /* run up to finalizers */ /* estimate must be correct after a full GC cycle */ lua_assert(g->GCestimate == gettotalbytes(g)); diff --git a/src/lua/liolib.c b/src/lua/liolib.c index b08397da..c5075f3e 100644 --- a/src/lua/liolib.c +++ b/src/lua/liolib.c @@ -245,8 +245,8 @@ static int f_gc (lua_State *L) { */ static int io_fclose (lua_State *L) { LStream *p = tolstream(L); - int res = fclose(p->f); - return luaL_fileresult(L, (res == 0), NULL); + errno = 0; + return luaL_fileresult(L, (fclose(p->f) == 0), NULL); } @@ -272,6 +272,7 @@ static int io_open (lua_State *L) { LStream *p = newfile(L); const char *md = mode; /* to traverse/check mode */ luaL_argcheck(L, l_checkmode(md), 2, "invalid mode"); + errno = 0; p->f = fopen(filename, mode); return (p->f == NULL) ? luaL_fileresult(L, 0, filename) : 1; } @@ -292,6 +293,7 @@ static int io_popen (lua_State *L) { const char *mode = luaL_optstring(L, 2, "r"); LStream *p = newprefile(L); luaL_argcheck(L, l_checkmodep(mode), 2, "invalid mode"); + errno = 0; p->f = l_popen(L, filename, mode); p->closef = &io_pclose; return (p->f == NULL) ? luaL_fileresult(L, 0, filename) : 1; @@ -300,6 +302,7 @@ static int io_popen (lua_State *L) { static int io_tmpfile (lua_State *L) { LStream *p = newfile(L); + errno = 0; p->f = tmpfile(); return (p->f == NULL) ? luaL_fileresult(L, 0, NULL) : 1; } @@ -567,6 +570,7 @@ static int g_read (lua_State *L, FILE *f, int first) { int nargs = lua_gettop(L) - 1; int n, success; clearerr(f); + errno = 0; if (nargs == 0) { /* no arguments? */ success = read_line(L, f, 1); n = first + 1; /* to return 1 result */ @@ -660,6 +664,7 @@ static int io_readline (lua_State *L) { static int g_write (lua_State *L, FILE *f, int arg) { int nargs = lua_gettop(L) - arg; int status = 1; + errno = 0; for (; nargs--; arg++) { if (lua_type(L, arg) == LUA_TNUMBER) { /* optimization: could be done exactly as for strings */ @@ -678,7 +683,8 @@ static int g_write (lua_State *L, FILE *f, int arg) { } if (l_likely(status)) return 1; /* file handle already on stack top */ - else return luaL_fileresult(L, status, NULL); + else + return luaL_fileresult(L, status, NULL); } @@ -703,6 +709,7 @@ static int f_seek (lua_State *L) { l_seeknum offset = (l_seeknum)p3; luaL_argcheck(L, (lua_Integer)offset == p3, 3, "not an integer in proper range"); + errno = 0; op = l_fseek(f, offset, mode[op]); if (l_unlikely(op)) return luaL_fileresult(L, 0, NULL); /* error */ @@ -719,19 +726,25 @@ static int f_setvbuf (lua_State *L) { FILE *f = tofile(L); int op = luaL_checkoption(L, 2, NULL, modenames); lua_Integer sz = luaL_optinteger(L, 3, LUAL_BUFFERSIZE); - int res = setvbuf(f, NULL, mode[op], (size_t)sz); + int res; + errno = 0; + res = setvbuf(f, NULL, mode[op], (size_t)sz); return luaL_fileresult(L, res == 0, NULL); } static int io_flush (lua_State *L) { - return luaL_fileresult(L, fflush(getiofile(L, IO_OUTPUT)) == 0, NULL); + FILE *f = getiofile(L, IO_OUTPUT); + errno = 0; + return luaL_fileresult(L, fflush(f) == 0, NULL); } static int f_flush (lua_State *L) { - return luaL_fileresult(L, fflush(tofile(L)) == 0, NULL); + FILE *f = tofile(L); + errno = 0; + return luaL_fileresult(L, fflush(f) == 0, NULL); } @@ -773,7 +786,7 @@ static const luaL_Reg meth[] = { ** metamethods for file handles */ static const luaL_Reg metameth[] = { - {"__index", NULL}, /* place holder */ + {"__index", NULL}, /* placeholder */ {"__gc", f_gc}, {"__close", f_gc}, {"__tostring", f_tostring}, diff --git a/src/lua/lmathlib.c b/src/lua/lmathlib.c index d0b1e1e5..43810634 100644 --- a/src/lua/lmathlib.c +++ b/src/lua/lmathlib.c @@ -249,6 +249,15 @@ static int math_type (lua_State *L) { ** =================================================================== */ +/* +** This code uses lots of shifts. ANSI C does not allow shifts greater +** than or equal to the width of the type being shifted, so some shifts +** are written in convoluted ways to match that restriction. For +** preprocessor tests, it assumes a width of 32 bits, so the maximum +** shift there is 31 bits. +*/ + + /* number of binary digits in the mantissa of a float */ #define FIGS l_floatatt(MANT_DIG) @@ -271,16 +280,19 @@ static int math_type (lua_State *L) { /* 'long' has at least 64 bits */ #define Rand64 unsigned long +#define SRand64 long #elif !defined(LUA_USE_C89) && defined(LLONG_MAX) /* there is a 'long long' type (which must have at least 64 bits) */ #define Rand64 unsigned long long +#define SRand64 long long #elif ((LUA_MAXUNSIGNED >> 31) >> 31) >= 3 /* 'lua_Unsigned' has at least 64 bits */ #define Rand64 lua_Unsigned +#define SRand64 lua_Integer #endif @@ -319,23 +331,30 @@ static Rand64 nextrand (Rand64 *state) { } -/* must take care to not shift stuff by more than 63 slots */ - - /* ** Convert bits from a random integer into a float in the ** interval [0,1), getting the higher FIG bits from the ** random unsigned integer and converting that to a float. +** Some old Microsoft compilers cannot cast an unsigned long +** to a floating-point number, so we use a signed long as an +** intermediary. When lua_Number is float or double, the shift ensures +** that 'sx' is non negative; in that case, a good compiler will remove +** the correction. */ /* must throw out the extra (64 - FIGS) bits */ #define shift64_FIG (64 - FIGS) -/* to scale to [0, 1), multiply by scaleFIG = 2^(-FIGS) */ +/* 2^(-FIGS) == 2^-1 / 2^(FIGS-1) */ #define scaleFIG (l_mathop(0.5) / ((Rand64)1 << (FIGS - 1))) static lua_Number I2d (Rand64 x) { - return (lua_Number)(trim64(x) >> shift64_FIG) * scaleFIG; + SRand64 sx = (SRand64)(trim64(x) >> shift64_FIG); + lua_Number res = (lua_Number)(sx) * scaleFIG; + if (sx < 0) + res += l_mathop(1.0); /* correct the two's complement if negative */ + lua_assert(0 <= res && res < 1); + return res; } /* convert a 'Rand64' to a 'lua_Unsigned' */ @@ -471,8 +490,6 @@ static lua_Number I2d (Rand64 x) { #else /* 32 < FIGS <= 64 */ -/* must take care to not shift stuff by more than 31 slots */ - /* 2^(-FIGS) = 1.0 / 2^30 / 2^3 / 2^(FIGS-33) */ #define scaleFIG \ (l_mathop(1.0) / (UONE << 30) / l_mathop(8.0) / (UONE << (FIGS - 33))) diff --git a/src/lua/loadlib.c b/src/lua/loadlib.c index d792dffa..6d289fce 100644 --- a/src/lua/loadlib.c +++ b/src/lua/loadlib.c @@ -24,15 +24,6 @@ #include "lualib.h" -/* -** LUA_IGMARK is a mark to ignore all before it when building the -** luaopen_ function name. -*/ -#if !defined (LUA_IGMARK) -#define LUA_IGMARK "-" -#endif - - /* ** LUA_CSUBSEP is the character that replaces dots in submodule names ** when searching for a C loader. diff --git a/src/lua/lobject.c b/src/lua/lobject.c index f73ffc6d..9cfa5227 100644 --- a/src/lua/lobject.c +++ b/src/lua/lobject.c @@ -542,7 +542,7 @@ const char *luaO_pushvfstring (lua_State *L, const char *fmt, va_list argp) { addstr2buff(&buff, fmt, strlen(fmt)); /* rest of 'fmt' */ clearbuff(&buff); /* empty buffer into the stack */ lua_assert(buff.pushed == 1); - return svalue(s2v(L->top.p - 1)); + return getstr(tsvalue(s2v(L->top.p - 1))); } diff --git a/src/lua/lobject.h b/src/lua/lobject.h index 556608e4..980e42f8 100644 --- a/src/lua/lobject.h +++ b/src/lua/lobject.h @@ -386,7 +386,7 @@ typedef struct GCObject { typedef struct TString { CommonHeader; lu_byte extra; /* reserved words for short strings; "has hash" for longs */ - lu_byte shrlen; /* length for short strings */ + lu_byte shrlen; /* length for short strings, 0xFF for long strings */ unsigned int hash; union { size_t lnglen; /* length for long strings */ @@ -398,19 +398,17 @@ typedef struct TString { /* -** Get the actual string (array of bytes) from a 'TString'. +** Get the actual string (array of bytes) from a 'TString'. (Generic +** version and specialized versions for long and short strings.) */ -#define getstr(ts) ((ts)->contents) +#define getstr(ts) ((ts)->contents) +#define getlngstr(ts) check_exp((ts)->shrlen == 0xFF, (ts)->contents) +#define getshrstr(ts) check_exp((ts)->shrlen != 0xFF, (ts)->contents) -/* get the actual string (array of bytes) from a Lua value */ -#define svalue(o) getstr(tsvalue(o)) - /* get string length from 'TString *s' */ -#define tsslen(s) ((s)->tt == LUA_VSHRSTR ? (s)->shrlen : (s)->u.lnglen) - -/* get string length from 'TValue *o' */ -#define vslen(o) tsslen(tsvalue(o)) +#define tsslen(s) \ + ((s)->shrlen != 0xFF ? (s)->shrlen : (s)->u.lnglen) /* }================================================================== */ diff --git a/src/lua/lopcodes.h b/src/lua/lopcodes.h index 4c551453..46911cac 100644 --- a/src/lua/lopcodes.h +++ b/src/lua/lopcodes.h @@ -210,15 +210,15 @@ OP_LOADNIL,/* A B R[A], R[A+1], ..., R[A+B] := nil */ OP_GETUPVAL,/* A B R[A] := UpValue[B] */ OP_SETUPVAL,/* A B UpValue[B] := R[A] */ -OP_GETTABUP,/* A B C R[A] := UpValue[B][K[C]:string] */ +OP_GETTABUP,/* A B C R[A] := UpValue[B][K[C]:shortstring] */ OP_GETTABLE,/* A B C R[A] := R[B][R[C]] */ OP_GETI,/* A B C R[A] := R[B][C] */ -OP_GETFIELD,/* A B C R[A] := R[B][K[C]:string] */ +OP_GETFIELD,/* A B C R[A] := R[B][K[C]:shortstring] */ -OP_SETTABUP,/* A B C UpValue[A][K[B]:string] := RK(C) */ +OP_SETTABUP,/* A B C UpValue[A][K[B]:shortstring] := RK(C) */ OP_SETTABLE,/* A B C R[A][R[B]] := RK(C) */ OP_SETI,/* A B C R[A][B] := RK(C) */ -OP_SETFIELD,/* A B C R[A][K[B]:string] := RK(C) */ +OP_SETFIELD,/* A B C R[A][K[B]:shortstring] := RK(C) */ OP_NEWTABLE,/* A B C k R[A] := {} */ diff --git a/src/lua/loslib.c b/src/lua/loslib.c index ad5a9276..ba80d72c 100644 --- a/src/lua/loslib.c +++ b/src/lua/loslib.c @@ -155,6 +155,7 @@ static int os_execute (lua_State *L) { static int os_remove (lua_State *L) { const char *filename = luaL_checkstring(L, 1); + errno = 0; return luaL_fileresult(L, remove(filename) == 0, filename); } @@ -162,6 +163,7 @@ static int os_remove (lua_State *L) { static int os_rename (lua_State *L) { const char *fromname = luaL_checkstring(L, 1); const char *toname = luaL_checkstring(L, 2); + errno = 0; return luaL_fileresult(L, rename(fromname, toname) == 0, NULL); } diff --git a/src/lua/lparser.c b/src/lua/lparser.c index b745f236..2b888c7c 100644 --- a/src/lua/lparser.c +++ b/src/lua/lparser.c @@ -1022,10 +1022,11 @@ static int explist (LexState *ls, expdesc *v) { } -static void funcargs (LexState *ls, expdesc *f, int line) { +static void funcargs (LexState *ls, expdesc *f) { FuncState *fs = ls->fs; expdesc args; int base, nparams; + int line = ls->linenumber; switch (ls->t.token) { case '(': { /* funcargs -> '(' [ explist ] ')' */ luaX_next(ls); @@ -1063,8 +1064,8 @@ static void funcargs (LexState *ls, expdesc *f, int line) { } init_exp(f, VCALL, luaK_codeABC(fs, OP_CALL, base, nparams+1, 2)); luaK_fixline(fs, line); - fs->freereg = base+1; /* call remove function and arguments and leaves - (unless changed) one result */ + fs->freereg = base+1; /* call removes function and arguments and leaves + one result (unless changed later) */ } @@ -1103,7 +1104,6 @@ static void suffixedexp (LexState *ls, expdesc *v) { /* suffixedexp -> primaryexp { '.' NAME | '[' exp ']' | ':' NAME funcargs | funcargs } */ FuncState *fs = ls->fs; - int line = ls->linenumber; primaryexp(ls, v); for (;;) { switch (ls->t.token) { @@ -1123,12 +1123,12 @@ static void suffixedexp (LexState *ls, expdesc *v) { luaX_next(ls); codename(ls, &key); luaK_self(fs, v, &key); - funcargs(ls, v, line); + funcargs(ls, v); break; } case '(': case TK_STRING: case '{': { /* funcargs */ luaK_exp2nextreg(fs, v); - funcargs(ls, v, line); + funcargs(ls, v); break; } default: return; diff --git a/src/lua/lstate.c b/src/lua/lstate.c index 1e925e5a..7fefacba 100644 --- a/src/lua/lstate.c +++ b/src/lua/lstate.c @@ -119,7 +119,7 @@ CallInfo *luaE_extendCI (lua_State *L) { /* ** free all CallInfo structures not in use by a thread */ -void luaE_freeCI (lua_State *L) { +static void freeCI (lua_State *L) { CallInfo *ci = L->ci; CallInfo *next = ci->next; ci->next = NULL; @@ -204,7 +204,7 @@ static void freestack (lua_State *L) { if (L->stack.p == NULL) return; /* stack not completely built yet */ L->ci = &L->base_ci; /* free the entire 'ci' list */ - luaE_freeCI(L); + freeCI(L); lua_assert(L->nci == 0); luaM_freearray(L, L->stack.p, stacksize(L) + EXTRA_STACK); /* free stack */ } @@ -433,7 +433,7 @@ void luaE_warning (lua_State *L, const char *msg, int tocont) { void luaE_warnerror (lua_State *L, const char *where) { TValue *errobj = s2v(L->top.p - 1); /* error object */ const char *msg = (ttisstring(errobj)) - ? svalue(errobj) + ? getstr(tsvalue(errobj)) : "error object is not a string"; /* produce warning "error in %s (%s)" (where, msg) */ luaE_warning(L, "error in ", 1); diff --git a/src/lua/lstate.h b/src/lua/lstate.h index 8bf6600e..007704c8 100644 --- a/src/lua/lstate.h +++ b/src/lua/lstate.h @@ -181,7 +181,7 @@ struct CallInfo { union { struct { /* only for Lua functions */ const Instruction *savedpc; - volatile l_signalT trap; + volatile l_signalT trap; /* function is tracing lines/counts */ int nextraargs; /* # of extra arguments in vararg functions */ } l; struct { /* only for C functions */ @@ -396,7 +396,6 @@ union GCUnion { LUAI_FUNC void luaE_setdebt (global_State *g, l_mem debt); LUAI_FUNC void luaE_freethread (lua_State *L, lua_State *L1); LUAI_FUNC CallInfo *luaE_extendCI (lua_State *L); -LUAI_FUNC void luaE_freeCI (lua_State *L); LUAI_FUNC void luaE_shrinkCI (lua_State *L); LUAI_FUNC void luaE_checkcstack (lua_State *L); LUAI_FUNC void luaE_incCstack (lua_State *L); diff --git a/src/lua/lstring.c b/src/lua/lstring.c index 13dcaf42..97757355 100644 --- a/src/lua/lstring.c +++ b/src/lua/lstring.c @@ -36,7 +36,7 @@ int luaS_eqlngstr (TString *a, TString *b) { lua_assert(a->tt == LUA_VLNGSTR && b->tt == LUA_VLNGSTR); return (a == b) || /* same instance or... */ ((len == b->u.lnglen) && /* equal length and ... */ - (memcmp(getstr(a), getstr(b), len) == 0)); /* equal contents */ + (memcmp(getlngstr(a), getlngstr(b), len) == 0)); /* equal contents */ } @@ -52,7 +52,7 @@ unsigned int luaS_hashlongstr (TString *ts) { lua_assert(ts->tt == LUA_VLNGSTR); if (ts->extra == 0) { /* no hash? */ size_t len = ts->u.lnglen; - ts->hash = luaS_hash(getstr(ts), len, ts->hash); + ts->hash = luaS_hash(getlngstr(ts), len, ts->hash); ts->extra = 1; /* now it has its hash */ } return ts->hash; @@ -157,6 +157,7 @@ static TString *createstrobj (lua_State *L, size_t l, int tag, unsigned int h) { TString *luaS_createlngstrobj (lua_State *L, size_t l) { TString *ts = createstrobj(L, l, LUA_VLNGSTR, G(L)->seed); ts->u.lnglen = l; + ts->shrlen = 0xFF; /* signals that it is a long string */ return ts; } @@ -193,7 +194,7 @@ static TString *internshrstr (lua_State *L, const char *str, size_t l) { TString **list = &tb->hash[lmod(h, tb->size)]; lua_assert(str != NULL); /* otherwise 'memcmp'/'memcpy' are undefined */ for (ts = *list; ts != NULL; ts = ts->u.hnext) { - if (l == ts->shrlen && (memcmp(str, getstr(ts), l * sizeof(char)) == 0)) { + if (l == ts->shrlen && (memcmp(str, getshrstr(ts), l * sizeof(char)) == 0)) { /* found! */ if (isdead(g, ts)) /* dead (but not collected yet)? */ changewhite(ts); /* resurrect it */ @@ -206,8 +207,8 @@ static TString *internshrstr (lua_State *L, const char *str, size_t l) { list = &tb->hash[lmod(h, tb->size)]; /* rehash with new size */ } ts = createstrobj(L, l, LUA_VSHRSTR, h); - memcpy(getstr(ts), str, l * sizeof(char)); ts->shrlen = cast_byte(l); + memcpy(getshrstr(ts), str, l * sizeof(char)); ts->u.hnext = *list; *list = ts; tb->nuse++; @@ -223,10 +224,10 @@ TString *luaS_newlstr (lua_State *L, const char *str, size_t l) { return internshrstr(L, str, l); else { TString *ts; - if (l_unlikely(l >= (MAX_SIZE - sizeof(TString))/sizeof(char))) + if (l_unlikely(l * sizeof(char) >= (MAX_SIZE - sizeof(TString)))) luaM_toobig(L); ts = luaS_createlngstrobj(L, l); - memcpy(getstr(ts), str, l * sizeof(char)); + memcpy(getlngstr(ts), str, l * sizeof(char)); return ts; } } diff --git a/src/lua/ltable.c b/src/lua/ltable.c index 3c690c5f..3353c047 100644 --- a/src/lua/ltable.c +++ b/src/lua/ltable.c @@ -252,7 +252,7 @@ LUAI_FUNC unsigned int luaH_realasize (const Table *t) { return t->alimit; /* this is the size */ else { unsigned int size = t->alimit; - /* compute the smallest power of 2 not smaller than 'n' */ + /* compute the smallest power of 2 not smaller than 'size' */ size |= (size >> 1); size |= (size >> 2); size |= (size >> 4); @@ -662,7 +662,8 @@ static Node *getfreepos (Table *t) { ** put new key in its main position; otherwise (colliding node is in its main ** position), new key goes to an empty position. */ -void luaH_newkey (lua_State *L, Table *t, const TValue *key, TValue *value) { +static void luaH_newkey (lua_State *L, Table *t, const TValue *key, + TValue *value) { Node *mp; TValue aux; if (l_unlikely(ttisnil(key))) @@ -721,22 +722,36 @@ void luaH_newkey (lua_State *L, Table *t, const TValue *key, TValue *value) { /* ** Search function for integers. If integer is inside 'alimit', get it -** directly from the array part. Otherwise, if 'alimit' is not equal to -** the real size of the array, key still can be in the array part. In -** this case, try to avoid a call to 'luaH_realasize' when key is just -** one more than the limit (so that it can be incremented without -** changing the real size of the array). +** directly from the array part. Otherwise, if 'alimit' is not +** the real size of the array, the key still can be in the array part. +** In this case, do the "Xmilia trick" to check whether 'key-1' is +** smaller than the real size. +** The trick works as follow: let 'p' be an integer such that +** '2^(p+1) >= alimit > 2^p', or '2^(p+1) > alimit-1 >= 2^p'. +** That is, 2^(p+1) is the real size of the array, and 'p' is the highest +** bit on in 'alimit-1'. What we have to check becomes 'key-1 < 2^(p+1)'. +** We compute '(key-1) & ~(alimit-1)', which we call 'res'; it will +** have the 'p' bit cleared. If the key is outside the array, that is, +** 'key-1 >= 2^(p+1)', then 'res' will have some bit on higher than 'p', +** therefore it will be larger or equal to 'alimit', and the check +** will fail. If 'key-1 < 2^(p+1)', then 'res' has no bit on higher than +** 'p', and as the bit 'p' itself was cleared, 'res' will be smaller +** than 2^p, therefore smaller than 'alimit', and the check succeeds. +** As special cases, when 'alimit' is 0 the condition is trivially false, +** and when 'alimit' is 1 the condition simplifies to 'key-1 < alimit'. +** If key is 0 or negative, 'res' will have its higher bit on, so that +** if cannot be smaller than alimit. */ const TValue *luaH_getint (Table *t, lua_Integer key) { - if (l_castS2U(key) - 1u < t->alimit) /* 'key' in [1, t->alimit]? */ + lua_Unsigned alimit = t->alimit; + if (l_castS2U(key) - 1u < alimit) /* 'key' in [1, t->alimit]? */ return &t->array[key - 1]; - else if (!limitequalsasize(t) && /* key still may be in the array part? */ - (l_castS2U(key) == t->alimit + 1 || - l_castS2U(key) - 1u < luaH_realasize(t))) { + else if (!isrealasize(t) && /* key still may be in the array part? */ + (((l_castS2U(key) - 1u) & ~(alimit - 1u)) < alimit)) { t->alimit = cast_uint(key); /* probably '#t' is here now */ return &t->array[key - 1]; } - else { + else { /* key is not in the array part; check the hash */ Node *n = hashint(t, key); for (;;) { /* check whether 'key' is somewhere in the chain */ if (keyisinteger(n) && keyival(n) == key) diff --git a/src/lua/ltable.h b/src/lua/ltable.h index 75dd9e26..8e689034 100644 --- a/src/lua/ltable.h +++ b/src/lua/ltable.h @@ -41,8 +41,6 @@ LUAI_FUNC void luaH_setint (lua_State *L, Table *t, lua_Integer key, LUAI_FUNC const TValue *luaH_getshortstr (Table *t, TString *key); LUAI_FUNC const TValue *luaH_getstr (Table *t, TString *key); LUAI_FUNC const TValue *luaH_get (Table *t, const TValue *key); -LUAI_FUNC void luaH_newkey (lua_State *L, Table *t, const TValue *key, - TValue *value); LUAI_FUNC void luaH_set (lua_State *L, Table *t, const TValue *key, TValue *value); LUAI_FUNC void luaH_finishset (lua_State *L, Table *t, const TValue *key, diff --git a/src/lua/ltm.h b/src/lua/ltm.h index c309e2ae..73b833c6 100644 --- a/src/lua/ltm.h +++ b/src/lua/ltm.h @@ -9,7 +9,6 @@ #include "lobject.h" -#include "lstate.h" /* @@ -96,8 +95,8 @@ LUAI_FUNC int luaT_callorderiTM (lua_State *L, const TValue *p1, int v2, int inv, int isfloat, TMS event); LUAI_FUNC void luaT_adjustvarargs (lua_State *L, int nfixparams, - CallInfo *ci, const Proto *p); -LUAI_FUNC void luaT_getvarargs (lua_State *L, CallInfo *ci, + struct CallInfo *ci, const Proto *p); +LUAI_FUNC void luaT_getvarargs (lua_State *L, struct CallInfo *ci, StkId where, int wanted); diff --git a/src/lua/lua.c b/src/lua/lua.c index f269c997..35fb281d 100644 --- a/src/lua/lua.c +++ b/src/lua/lua.c @@ -119,12 +119,13 @@ static void l_message (const char *pname, const char *msg) { /* ** Check whether 'status' is not OK and, if so, prints the error -** message on the top of the stack. It assumes that the error object -** is a string, as it was either generated by Lua or by 'msghandler'. +** message on the top of the stack. */ static int report (lua_State *L, int status) { if (status != LUA_OK) { const char *msg = lua_tostring(L, -1); + if (msg == NULL) + msg = "(error message not a string)"; l_message(progname, msg); lua_pop(L, 1); /* remove message */ } @@ -214,14 +215,19 @@ static int dostring (lua_State *L, const char *s, const char *name) { /* ** Receives 'globname[=modname]' and runs 'globname = require(modname)'. +** If there is no explicit modname and globname contains a '-', cut +** the suffix after '-' (the "version") to make the global name. */ /************** Pi-hole modification ***************/ int dolibrary (lua_State *L, char *globname) { /***************************************************/ int status; + char *suffix = NULL; char *modname = strchr(globname, '='); - if (modname == NULL) /* no explicit name? */ + if (modname == NULL) { /* no explicit name? */ modname = globname; /* module name is equal to global name */ + suffix = strchr(modname, *LUA_IGMARK); /* look for a suffix mark */ + } else { *modname = '\0'; /* global name ends here */ modname++; /* module name starts after the '=' */ @@ -229,8 +235,11 @@ int dolibrary (lua_State *L, char *globname) { lua_getglobal(L, "require"); lua_pushstring(L, modname); status = docall(L, 1, 1); /* call 'require(modname)' */ - if (status == LUA_OK) + if (status == LUA_OK) { + if (suffix != NULL) /* is there a suffix mark? */ + *suffix = '\0'; /* remove suffix from global name */ lua_setglobal(L, globname); /* globname = require(modname) */ + } return report(L, status); } diff --git a/src/lua/lua.h b/src/lua/lua.h index fd16cf80..f050dac0 100644 --- a/src/lua/lua.h +++ b/src/lua/lua.h @@ -18,14 +18,14 @@ #define LUA_VERSION_MAJOR "5" #define LUA_VERSION_MINOR "4" -#define LUA_VERSION_RELEASE "6" +#define LUA_VERSION_RELEASE "7" #define LUA_VERSION_NUM 504 -#define LUA_VERSION_RELEASE_NUM (LUA_VERSION_NUM * 100 + 6) +#define LUA_VERSION_RELEASE_NUM (LUA_VERSION_NUM * 100 + 7) #define LUA_VERSION "Lua " LUA_VERSION_MAJOR "." LUA_VERSION_MINOR #define LUA_RELEASE LUA_VERSION "." LUA_VERSION_RELEASE -#define LUA_COPYRIGHT LUA_RELEASE " Copyright (C) 1994-2023 Lua.org, PUC-Rio" +#define LUA_COPYRIGHT LUA_RELEASE " Copyright (C) 1994-2024 Lua.org, PUC-Rio" #define LUA_AUTHORS "R. Ierusalimschy, L. H. de Figueiredo, W. Celes" @@ -497,7 +497,7 @@ struct lua_Debug { /****************************************************************************** -* Copyright (C) 1994-2023 Lua.org, PUC-Rio. +* Copyright (C) 1994-2024 Lua.org, PUC-Rio. * * Permission is hereby granted, free of charge, to any person obtaining * a copy of this software and associated documentation files (the diff --git a/src/lua/luaconf.h b/src/lua/luaconf.h index 137103ed..33bb580d 100644 --- a/src/lua/luaconf.h +++ b/src/lua/luaconf.h @@ -257,6 +257,15 @@ #endif + +/* +** LUA_IGMARK is a mark to ignore all after it when building the +** module name (e.g., used to build the luaopen_ function name). +** Typically, the suffix after the mark is the module version, +** as in "mod-v1.2.so". +*/ +#define LUA_IGMARK "-" + /* }================================================================== */ diff --git a/src/lua/lundump.c b/src/lua/lundump.c index 02aed64f..e8d92a85 100644 --- a/src/lua/lundump.c +++ b/src/lua/lundump.c @@ -81,7 +81,7 @@ static size_t loadUnsigned (LoadState *S, size_t limit) { static size_t loadSize (LoadState *S) { - return loadUnsigned(S, ~(size_t)0); + return loadUnsigned(S, MAX_SIZET); } @@ -122,7 +122,7 @@ static TString *loadStringN (LoadState *S, Proto *p) { ts = luaS_createlngstrobj(L, size); /* create string */ setsvalue2s(L, L->top.p, ts); /* anchor it ('loadVector' can GC) */ luaD_inctop(L); - loadVector(S, getstr(ts), size); /* load directly in final place */ + loadVector(S, getlngstr(ts), size); /* load directly in final place */ L->top.p--; /* pop string */ } luaC_objbarrier(L, p, ts); diff --git a/src/lua/lundump.h b/src/lua/lundump.h index f3748a99..a97676ca 100644 --- a/src/lua/lundump.h +++ b/src/lua/lundump.h @@ -21,8 +21,7 @@ /* ** Encode major-minor version in one byte, one nibble for each */ -#define MYINT(s) (s[0]-'0') /* assume one-digit numerals */ -#define LUAC_VERSION (MYINT(LUA_VERSION_MAJOR)*16+MYINT(LUA_VERSION_MINOR)) +#define LUAC_VERSION (((LUA_VERSION_NUM / 100) * 16) + LUA_VERSION_NUM % 100) #define LUAC_FORMAT 0 /* this is the official format */ diff --git a/src/lua/lvm.c b/src/lua/lvm.c index 8493a770..fcd24e11 100644 --- a/src/lua/lvm.c +++ b/src/lua/lvm.c @@ -91,8 +91,10 @@ static int l_strton (const TValue *obj, TValue *result) { lua_assert(obj != result); if (!cvt2num(obj)) /* is object not a string? */ return 0; - else - return (luaO_str2num(svalue(obj), result) == vslen(obj) + 1); + else { + TString *st = tsvalue(obj); + return (luaO_str2num(getstr(st), result) == tsslen(st) + 1); + } } @@ -366,30 +368,32 @@ void luaV_finishset (lua_State *L, const TValue *t, TValue *key, /* -** Compare two strings 'ls' x 'rs', returning an integer less-equal- -** -greater than zero if 'ls' is less-equal-greater than 'rs'. +** Compare two strings 'ts1' x 'ts2', returning an integer less-equal- +** -greater than zero if 'ts1' is less-equal-greater than 'ts2'. ** The code is a little tricky because it allows '\0' in the strings -** and it uses 'strcoll' (to respect locales) for each segments -** of the strings. +** and it uses 'strcoll' (to respect locales) for each segment +** of the strings. Note that segments can compare equal but still +** have different lengths. */ -static int l_strcmp (const TString *ls, const TString *rs) { - const char *l = getstr(ls); - size_t ll = tsslen(ls); - const char *r = getstr(rs); - size_t lr = tsslen(rs); +static int l_strcmp (const TString *ts1, const TString *ts2) { + const char *s1 = getstr(ts1); + size_t rl1 = tsslen(ts1); /* real length */ + const char *s2 = getstr(ts2); + size_t rl2 = tsslen(ts2); for (;;) { /* for each segment */ - int temp = strcoll(l, r); + int temp = strcoll(s1, s2); if (temp != 0) /* not equal? */ return temp; /* done */ else { /* strings are equal up to a '\0' */ - size_t len = strlen(l); /* index of first '\0' in both strings */ - if (len == lr) /* 'rs' is finished? */ - return (len == ll) ? 0 : 1; /* check 'ls' */ - else if (len == ll) /* 'ls' is finished? */ - return -1; /* 'ls' is less than 'rs' ('rs' is not finished) */ - /* both strings longer than 'len'; go on comparing after the '\0' */ - len++; - l += len; ll -= len; r += len; lr -= len; + size_t zl1 = strlen(s1); /* index of first '\0' in 's1' */ + size_t zl2 = strlen(s2); /* index of first '\0' in 's2' */ + if (zl2 == rl2) /* 's2' is finished? */ + return (zl1 == rl1) ? 0 : 1; /* check 's1' */ + else if (zl1 == rl1) /* 's1' is finished? */ + return -1; /* 's1' is less than 's2' ('s2' is not finished) */ + /* both strings longer than 'zl'; go on comparing after the '\0' */ + zl1++; zl2++; + s1 += zl1; rl1 -= zl1; s2 += zl2; rl2 -= zl2; } } } @@ -624,8 +628,9 @@ int luaV_equalobj (lua_State *L, const TValue *t1, const TValue *t2) { static void copy2buff (StkId top, int n, char *buff) { size_t tl = 0; /* size already copied */ do { - size_t l = vslen(s2v(top - n)); /* length of string being copied */ - memcpy(buff + tl, svalue(s2v(top - n)), l * sizeof(char)); + TString *st = tsvalue(s2v(top - n)); + size_t l = tsslen(st); /* length of string being copied */ + memcpy(buff + tl, getstr(st), l * sizeof(char)); tl += l; } while (--n > 0); } @@ -651,12 +656,12 @@ void luaV_concat (lua_State *L, int total) { } else { /* at least two non-empty string values; get as many as possible */ - size_t tl = vslen(s2v(top - 1)); + size_t tl = tsslen(tsvalue(s2v(top - 1))); TString *ts; /* collect total length and number of strings */ for (n = 1; n < total && tostring(L, s2v(top - n - 1)); n++) { - size_t l = vslen(s2v(top - n - 1)); - if (l_unlikely(l >= (MAX_SIZE/sizeof(char)) - tl)) { + size_t l = tsslen(tsvalue(s2v(top - n - 1))); + if (l_unlikely(l >= MAX_SIZE - sizeof(TString) - tl)) { L->top.p = top - total; /* pop strings to avoid wasting stack */ luaG_runerror(L, "string length overflow"); } @@ -669,7 +674,7 @@ void luaV_concat (lua_State *L, int total) { } else { /* long string; copy strings directly to final result */ ts = luaS_createlngstrobj(L, tl); - copy2buff(top, n, getstr(ts)); + copy2buff(top, n, getlngstr(ts)); } setsvalue2s(L, top - n, ts); /* create result */ } @@ -1155,18 +1160,11 @@ void luaV_execute (lua_State *L, CallInfo *ci) { startfunc: trap = L->hookmask; returning: /* trap already set */ - cl = clLvalue(s2v(ci->func.p)); + cl = ci_func(ci); k = cl->p->k; pc = ci->u.l.savedpc; - if (l_unlikely(trap)) { - if (pc == cl->p->code) { /* first instruction (not resuming)? */ - if (cl->p->is_vararg) - trap = 0; /* hooks will start after VARARGPREP instruction */ - else /* check 'call' hook */ - luaD_hookcall(L, ci); - } - ci->u.l.trap = 1; /* assume trap is on, for now */ - } + if (l_unlikely(trap)) + trap = luaG_tracecall(L); base = ci->func.p + 1; /* main loop of interpreter */ for (;;) { @@ -1253,7 +1251,7 @@ void luaV_execute (lua_State *L, CallInfo *ci) { const TValue *slot; TValue *upval = cl->upvals[GETARG_B(i)]->v.p; TValue *rc = KC(i); - TString *key = tsvalue(rc); /* key must be a string */ + TString *key = tsvalue(rc); /* key must be a short string */ if (luaV_fastget(L, upval, key, slot, luaH_getshortstr)) { setobj2s(L, ra, slot); } @@ -1296,7 +1294,7 @@ void luaV_execute (lua_State *L, CallInfo *ci) { const TValue *slot; TValue *rb = vRB(i); TValue *rc = KC(i); - TString *key = tsvalue(rc); /* key must be a string */ + TString *key = tsvalue(rc); /* key must be a short string */ if (luaV_fastget(L, rb, key, slot, luaH_getshortstr)) { setobj2s(L, ra, slot); } @@ -1309,7 +1307,7 @@ void luaV_execute (lua_State *L, CallInfo *ci) { TValue *upval = cl->upvals[GETARG_A(i)]->v.p; TValue *rb = KB(i); TValue *rc = RKC(i); - TString *key = tsvalue(rb); /* key must be a string */ + TString *key = tsvalue(rb); /* key must be a short string */ if (luaV_fastget(L, upval, key, slot, luaH_getshortstr)) { luaV_finishfastset(L, upval, slot, rc); } @@ -1352,7 +1350,7 @@ void luaV_execute (lua_State *L, CallInfo *ci) { const TValue *slot; TValue *rb = KB(i); TValue *rc = RKC(i); - TString *key = tsvalue(rb); /* key must be a string */ + TString *key = tsvalue(rb); /* key must be a short string */ if (luaV_fastget(L, s2v(ra), key, slot, luaH_getshortstr)) { luaV_finishfastset(L, s2v(ra), slot, rc); } From 23ddd85837f720887a791061204f0631271adb03 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 26 Jun 2024 08:49:31 +0200 Subject: [PATCH 178/339] Check if database is actually writable when we request this Signed-off-by: DL6ER --- src/database/common.c | 9 ++++++ src/database/message-table.c | 59 +++++++++++------------------------- 2 files changed, 26 insertions(+), 42 deletions(-) diff --git a/src/database/common.c b/src/database/common.c index f209a50f..bcf7d61d 100644 --- a/src/database/common.c +++ b/src/database/common.c @@ -103,6 +103,15 @@ sqlite3* _dbopen(const bool readonly, const bool create, const char *func, const return NULL; } + // If the database is opened in read-write mode, actually check if it is + // writable. If it is not, close the database and return an error + if(!readonly && sqlite3_db_readonly(db, NULL)) + { + log_err("Cannot open database in read-write mode"); + dbclose(&db); + return NULL; + } + // Explicitly set busy handler to value defined in FTL.h rc = sqlite3_busy_timeout(db, DATABASE_BUSY_TIMEOUT); if( rc != SQLITE_OK ) diff --git a/src/database/message-table.c b/src/database/message-table.c index 81c4e3fa..031d7fb8 100644 --- a/src/database/message-table.c +++ b/src/database/message-table.c @@ -332,10 +332,8 @@ static int _add_message(const enum message_type type, sqlite3 *db; // Open database connection if((db = dbopen(false, false)) == NULL) - { - log_err("add_message() - Failed to open DB"); + // Reason for failure is logged in dbopen() return -1; - } // Ensure there are no duplicates when adding messages sqlite3_stmt* stmt = NULL; @@ -1244,9 +1242,7 @@ void logg_regex_warning(const char *type, const char *warning, const int dbindex return; // Add to database - const int rowid = add_message(REGEX_MESSAGE, regex, type, warning, dbindex); - if(rowid == -1) - log_err("logg_regex_warning(): Failed to add message to database"); + add_message(REGEX_MESSAGE, regex, type, warning, dbindex); } void logg_subnet_warning(const char *ip, const int matching_count, const char *matching_ids, @@ -1264,10 +1260,8 @@ void logg_subnet_warning(const char *ip, const int matching_count, const char *m log_warn("%s", buf); // Log to database - const int rowid = add_message(SUBNET_MESSAGE, ip, matching_count, names, matching_ids, chosen_match_text, chosen_match_id); + add_message(SUBNET_MESSAGE, ip, matching_count, names, matching_ids, chosen_match_text, chosen_match_id); - if(rowid == -1) - log_err("logg_subnet_warning(): Failed to add message to database"); free(names); } @@ -1286,10 +1280,8 @@ void logg_hostname_warning(const char *ip, const char *name, const unsigned int log_warn("%s", buf); // Log to database - const int rowid = add_message(HOSTNAME_MESSAGE, ip, name, (const int)pos); + add_message(HOSTNAME_MESSAGE, ip, name, (const int)pos); - if(rowid == -1) - log_err("logg_hostname_warning(): Failed to add message to database"); } void logg_fatal_dnsmasq_message(const char *message) @@ -1302,10 +1294,8 @@ void logg_fatal_dnsmasq_message(const char *message) log_crit("%s", buf); // Log to database - const int rowid = add_message_no_args(DNSMASQ_CONFIG_MESSAGE, message); + add_message_no_args(DNSMASQ_CONFIG_MESSAGE, message); - if(rowid == -1) - log_err("logg_fatal_dnsmasq_message(): Failed to add message to database"); } void logg_rate_limit_message(const char *clientIP, const unsigned int rate_limit_count) @@ -1320,10 +1310,8 @@ void logg_rate_limit_message(const char *clientIP, const unsigned int rate_limit log_info("%s", buf); // Log to database - const int rowid = add_message(RATE_LIMIT_MESSAGE, clientIP, config.dns.rateLimit.count.v.ui, config.dns.rateLimit.interval.v.ui, turnaround); + add_message(RATE_LIMIT_MESSAGE, clientIP, config.dns.rateLimit.count.v.ui, config.dns.rateLimit.interval.v.ui, turnaround); - if(rowid == -1) - log_err("logg_rate_limit_message(): Failed to add message to database"); } void logg_warn_dnsmasq_message(char *message) @@ -1336,10 +1324,8 @@ void logg_warn_dnsmasq_message(char *message) log_warn("%s", buf); // Log to database - const int rowid = add_message_no_args(DNSMASQ_WARN_MESSAGE, message); + add_message_no_args(DNSMASQ_WARN_MESSAGE, message); - if(rowid == -1) - log_err("logg_warn_dnsmasq_message(): Failed to add message to database"); } void log_resource_shortage(const double load, const int nprocs, const int shmem, const int disk, const char *path, const char *msg) @@ -1355,10 +1341,9 @@ void log_resource_shortage(const double load, const int nprocs, const int shmem, log_warn("%s", buf); // Log to database - const int rowid = add_message(LOAD_MESSAGE, "excessive load", load, nprocs); + add_message(LOAD_MESSAGE, "excessive load", load, nprocs); + - if(rowid == -1) - log_err("log_resource_shortage(): Failed to add message to database"); } else if(shmem > -1) { @@ -1368,10 +1353,9 @@ void log_resource_shortage(const double load, const int nprocs, const int shmem, log_warn("%s", buf); // Log to database - const int rowid = add_message(SHMEM_MESSAGE, path, shmem, msg); + add_message(SHMEM_MESSAGE, path, shmem, msg); + - if(rowid == -1) - log_err("log_resource_shortage(): Failed to add message to database"); } else if(disk > -1) { @@ -1405,12 +1389,11 @@ void log_resource_shortage(const double load, const int nprocs, const int shmem, log_warn("%s", buf); // Log to database - const int rowid = fsdetails != NULL ? + fsdetails != NULL ? add_message(DISK_MESSAGE_EXTENDED, path, disk, msg, fsdetails->mnt_type, fsdetails->mnt_dir) : add_message(DISK_MESSAGE, path, disk, msg); - if(rowid == -1) - log_err("log_resource_shortage(): Failed to add message to database"); + } } @@ -1424,10 +1407,8 @@ void logg_inaccessible_adlist(const int dbindex, const char *address) log_warn("%s", buf); // Log to database - const int rowid = add_message(INACCESSIBLE_ADLIST_MESSAGE, address, dbindex); + add_message(INACCESSIBLE_ADLIST_MESSAGE, address, dbindex); - if(rowid == -1) - log_err("logg_inaccessible_adlist(): Failed to add message to database"); } void log_certificate_domain_mismatch(const char *certfile, const char *domain) @@ -1440,10 +1421,8 @@ void log_certificate_domain_mismatch(const char *certfile, const char *domain) log_warn("%s", buf); // Log to database - const int rowid = add_message(CERTIFICATE_DOMAIN_MISMATCH_MESSAGE, certfile, domain); + add_message(CERTIFICATE_DOMAIN_MISMATCH_MESSAGE, certfile, domain); - if(rowid == -1) - log_err("log_certificate_domain_mismatch(): Failed to add message to database"); } void log_connection_error(const char *server, const char *reason, const char *error) @@ -1456,10 +1435,8 @@ void log_connection_error(const char *server, const char *reason, const char *er log_warn("%s", buf); // Log to database - const int rowid = add_message(CONNECTION_ERROR_MESSAGE, server, reason, error); + add_message(CONNECTION_ERROR_MESSAGE, server, reason, error); - if(rowid == -1) - log_err("logg_connection_error(): Failed to add message to database"); } void log_ntp_message(const bool error, const bool server, const char *message) @@ -1478,8 +1455,6 @@ void log_ntp_message(const bool error, const bool server, const char *message) log_warn("%s", buf); // Log to database - const int rowid = add_message(NTP_MESSAGE, message, level, who); + add_message(NTP_MESSAGE, message, level, who); - if(rowid == -1) - log_err("log_ntp_message(): Failed to add message to database"); } From 5c2da0d074c684c128f5e297e8240524e904e6a2 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 26 Jun 2024 09:04:18 +0200 Subject: [PATCH 179/339] Only use valid replies. Before, invalid replies would have contributed to the mean with a value of 0.0ms deviation, i.e., reducing any existing real time difference Signed-off-by: DL6ER --- src/ntp/client.c | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/src/ntp/client.c b/src/ntp/client.c index c312c8ac..365e76d0 100644 --- a/src/ntp/client.c +++ b/src/ntp/client.c @@ -41,6 +41,7 @@ #include "database/message-table.h" struct ntp_sync { + bool valid; uint64_t org; uint64_t xmt; double theta; @@ -50,7 +51,7 @@ struct ntp_sync // Create minimal NTP request, see server implementation for details about the // packet structure -static bool request(int fd, struct ntp_sync *ntp) +static bool request(int fd, const char *server, struct ntp_sync *ntp) { // NTP Packet buffer unsigned char buf[48] = {0}; @@ -73,8 +74,8 @@ static bool request(int fd, struct ntp_sync *ntp) // Send request if(send(fd, buf, 48, 0) != 48) { - log_err("Failed to send data to NTP server: %s", - errno == EAGAIN ? "Timeout" : strerror(errno)); + log_err("Failed to send data to NTP server %s: %s", + server, errno == EAGAIN ? "Timeout" : strerror(errno)); return false; } @@ -209,7 +210,7 @@ static bool settime_skew(const double offset) return true; } -static bool reply(int fd, struct ntp_sync *ntp, const bool verbose) +static bool reply(int fd, const char *server, struct ntp_sync *ntp, const bool verbose) { // NTP Packet buffer unsigned char buf[48]; @@ -217,8 +218,8 @@ static bool reply(int fd, struct ntp_sync *ntp, const bool verbose) // Receive reply if(recv(fd, buf, 48, 0) < 48) { - log_err("Failed to receive data from NTP server: %s", - errno == EAGAIN ? "Timeout" : strerror(errno)); + log_err("Failed to receive data from NTP server %s: %s", + server, errno == EAGAIN ? "Timeout" : strerror(errno)); return false; } @@ -296,6 +297,9 @@ static bool reply(int fd, struct ntp_sync *ntp, const bool verbose) // technologies are highly variable ntp->delta = ( T4 - T1 ) - ( T3 - T2 ); + // This reply is valid + ntp->valid = true; + // In some scenarios where the initial frequency offset of the client is // relatively large and the actual propagation time small, it is // possible for the delay computation to become negative. For instance, @@ -416,7 +420,7 @@ bool ntp_client(const char *server, const bool settime, const bool print) continue; // Send request - if(!request(s, &ntp[i])) + if(!request(s, server, &ntp[i])) { close(s); free(ntp); @@ -424,7 +428,7 @@ bool ntp_client(const char *server, const bool settime, const bool print) return false; } // Get reply - if(!reply(s, &ntp[i], false)) + if(!reply(s, server, &ntp[i], false)) { close(s); continue; @@ -453,7 +457,8 @@ bool ntp_client(const char *server, const bool settime, const bool print) { // Skip invalid values if(fabs(ntp[i].theta) < ntp[i].precision || - fabs(ntp[i].delta) < ntp[i].precision) + fabs(ntp[i].delta) < ntp[i].precision || + !ntp[i].valid) continue; theta_avg += ntp[i].theta; @@ -475,7 +480,8 @@ bool ntp_client(const char *server, const bool settime, const bool print) { // Skip invalid values if(fabs(ntp[i].theta) < ntp[i].precision || - fabs(ntp[i].delta) < ntp[i].precision) + fabs(ntp[i].delta) < ntp[i].precision || + !ntp[i].valid) continue; theta_stdev += pow(ntp[i].theta - theta_avg, 2); @@ -503,7 +509,8 @@ bool ntp_client(const char *server, const bool settime, const bool print) { // Skip invalid values if(fabs(ntp[i].theta) < ntp[i].precision || - fabs(ntp[i].delta) < ntp[i].precision) + fabs(ntp[i].delta) < ntp[i].precision || + !ntp[i].valid) continue; // Skip outliers From b3d72cb7b60cdc26c06873dc02066035f6913663 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 26 Jun 2024 09:35:13 +0200 Subject: [PATCH 180/339] Load queries only after first NTP synchronization (if enabled) Signed-off-by: DL6ER --- src/database/query-table.c | 24 ++++++++++++++++++++++++ src/database/query-table.h | 1 + src/dnsmasq_interface.c | 7 ------- src/ntp/client.c | 14 ++++++++++++++ 4 files changed, 39 insertions(+), 7 deletions(-) diff --git a/src/database/query-table.c b/src/database/query-table.c index 342730a4..d5e1ccfc 100644 --- a/src/database/query-table.c +++ b/src/database/query-table.c @@ -21,8 +21,11 @@ #include "overTime.h" #include "database/common.h" #include "timers.h" +// runGC() +#include "gc.h" static sqlite3 *_memdb = NULL; +static bool store_in_database = false; static double new_last_timestamp = 0; static unsigned int new_total = 0, new_blocked = 0; static unsigned long last_mem_db_idx = 0, last_disk_db_idx = 0; @@ -1367,6 +1370,11 @@ bool queries_to_database(void) log_debug(DEBUG_DATABASE, "Not storing query in database as there are none"); return true; } + if(!store_in_database) + { + log_debug(DEBUG_DATABASE, "Not storing query in database as this is disabled"); + return true; + } // Loop over recent queries and store new or changed ones in the // in-memory database @@ -1626,3 +1634,19 @@ bool queries_to_database(void) return true; } + +void load_queries_from_disk(void) +{ + // Compensate for possible jumps in time + runGC(time(NULL), NULL, false); + + // Skip if we are not supposed to load queries from disk + if(!config.database.DBimport.v.b) + return; + + // Try to import queries from long-term database if available + import_queries_from_disk(); + DB_read_queries(); + + store_in_database = true; +} \ No newline at end of file diff --git a/src/database/query-table.h b/src/database/query-table.h index 9fecb9ea..71458e28 100644 --- a/src/database/query-table.h +++ b/src/database/query-table.h @@ -119,6 +119,7 @@ bool add_additional_info_column(sqlite3 *db); void DB_read_queries(void); void update_disk_db_idx(void); bool queries_to_database(void); +void load_queries_from_disk(void); bool optimize_queries_table(sqlite3 *db); bool create_addinfo_table(sqlite3 *db); diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c index ad21e847..0c8132db 100644 --- a/src/dnsmasq_interface.c +++ b/src/dnsmasq_interface.c @@ -2904,13 +2904,6 @@ void FTL_fork_and_bind_sockets(struct passwd *ent_pw, bool dnsmasq_start) // Flush messages stored in the long-term database flush_message_table(); - // Try to import queries from long-term database if available - if(config.database.DBimport.v.b) - { - import_queries_from_disk(); - DB_read_queries(); - } - // Initialize in-memory database starting index update_disk_db_idx(); diff --git a/src/ntp/client.c b/src/ntp/client.c index 365e76d0..f1fb042f 100644 --- a/src/ntp/client.c +++ b/src/ntp/client.c @@ -39,6 +39,8 @@ #include // log_ntp_message() #include "database/message-table.h" +// load_queries_from_disk() +#include "database/query-table.h" struct ntp_sync { bool valid; @@ -590,6 +592,7 @@ static void *ntp_client_thread(void *arg) prctl(PR_SET_NAME, thread_names[NTP], 0, 0, 0); // Run NTP client + bool first_run = true; while(!killed) { @@ -599,6 +602,13 @@ static void *ntp_client_thread(void *arg) // Run NTP client ntp_client(config.ntp.sync.server.v.s, true, false); + // Load queries from database after first NTP synchronization + if(first_run) + { + load_queries_from_disk(); + first_run = false; + } + // Get time after NTP sync const time_t after = time(NULL); @@ -636,12 +646,16 @@ bool ntp_start_sync_thread(pthread_attr_t *attr) if(config.ntp.sync.server.v.s == NULL || strlen(config.ntp.sync.server.v.s) == 0 || config.ntp.sync.interval.v.ui == 0) + { + load_queries_from_disk(); return false; + } // Create thread if(pthread_create(&threads[NTP], attr, ntp_client_thread, NULL) != 0) { log_err("Cannot create NTP client thread"); + load_queries_from_disk(); return false; } From 6d164a352937448b95756a65c75a30545048b089 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 26 Jun 2024 09:37:01 +0200 Subject: [PATCH 181/339] Remove restarting step as queries are now loaded *after* NTP time synchronization Signed-off-by: DL6ER --- src/ntp/client.c | 24 +----------------------- 1 file changed, 1 insertion(+), 23 deletions(-) diff --git a/src/ntp/client.c b/src/ntp/client.c index f1fb042f..532689a2 100644 --- a/src/ntp/client.c +++ b/src/ntp/client.c @@ -8,7 +8,7 @@ * This file is copyright under the latest version of the EUPL. * Please see LICENSE file for your rights under this license. */ -#include "ntp/ntp.h" +#include "ntp.h" // close() #include // clock_gettime() @@ -595,10 +595,6 @@ static void *ntp_client_thread(void *arg) bool first_run = true; while(!killed) { - - // Get time before NTP sync - const time_t before = time(NULL); - // Run NTP client ntp_client(config.ntp.sync.server.v.s, true, false); @@ -609,24 +605,6 @@ static void *ntp_client_thread(void *arg) first_run = false; } - // Get time after NTP sync - const time_t after = time(NULL); - - // If the time was updated by more than one hour, restart FTL to - // import recent data. This is relevant when the system time was - // set to an incorrect value (e.g., due to a dead CMOS battery - // or overall missing RTC) and the time was off. - if(after - before > 3600) - { - log_info("System time was updated by more than one hour, restarting FTL to import recent data"); - // Set the restart flag to true - exit_code = RESTART_FTL_CODE; - // Send SIGTERM to FTL - kill(main_pid(), SIGTERM); - // Kill the NTP thread - killed = true; - } - // Intermediate cancellation-point BREAK_IF_KILLED(); From 14e716729c0501b6c69a05a87dbaa03ee9579b2c Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 26 Jun 2024 10:21:12 +0200 Subject: [PATCH 182/339] Styling Signed-off-by: DL6ER --- src/database/query-table.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/database/query-table.c b/src/database/query-table.c index d5e1ccfc..5c707dc2 100644 --- a/src/database/query-table.c +++ b/src/database/query-table.c @@ -1649,4 +1649,4 @@ void load_queries_from_disk(void) DB_read_queries(); store_in_database = true; -} \ No newline at end of file +} From 6eff0296b3e74ce37f1258550bdf3ef22711169c Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 27 Jun 2024 16:33:16 +0200 Subject: [PATCH 183/339] Check availablity of CAP_SYS_TIME when NTP client is invoked from CLI Signed-off-by: DL6ER --- src/args.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/args.c b/src/args.c index f6e252a1..dcd625b4 100644 --- a/src/args.c +++ b/src/args.c @@ -68,6 +68,8 @@ #include "resolve.h" // ntp_client() #include "ntp/ntp.h" +// check_capability() +#include "capabilities.h" // defined in dnsmasq.c extern void print_dnsmasq_version(const char *yellow, const char *green, const char *bold, const char *normal); @@ -310,6 +312,18 @@ void parse_args(int argc, char* argv[]) // Create test NTP client if((argc > 1 && argc < 5) && strcmp(argv[1], "ntp") == 0) { + // Ensure we have the necessary capabilities + if(!check_capability(CAP_SYS_TIME)) + { + puts("Insufficient capabilities to run NTP client"); + const char *bold = cli_bold(); + const char *normal = cli_normal(); + printf("Try: %ssudo%s ", bold, normal); + for(int i = 0; i < argc; i++) + printf("%s ", argv[i]); + puts(""); + exit(EXIT_FAILURE); + } // Enable stdout printing cli_mode = true; log_ctrl(false, true); From 4e5526854422f2d929ed15d880dd0ba52c04335a Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 27 Jun 2024 16:34:54 +0200 Subject: [PATCH 184/339] Clarify which server is used when invoked via CLI Signed-off-by: DL6ER --- src/args.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/args.c b/src/args.c index dcd625b4..dbb74476 100644 --- a/src/args.c +++ b/src/args.c @@ -333,6 +333,7 @@ void parse_args(int argc, char* argv[]) const char *server = "127.0.0.1"; if(argc > 2 && strcmp(argv[2], "--update") != 0) server = argv[2]; + printf("Using NTP server: %s\n", server); exit(ntp_client(server, update, true) ? EXIT_SUCCESS : EXIT_FAILURE); } From ec0e0c98bf72ff38315847abdd5ab66ce398ca8d Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 27 Jun 2024 16:36:02 +0200 Subject: [PATCH 185/339] Print database statistics after historic queries have been loaded from disk Signed-off-by: DL6ER --- src/database/query-table.c | 3 +++ src/dnsmasq_interface.c | 3 --- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/database/query-table.c b/src/database/query-table.c index 5c707dc2..435f560a 100644 --- a/src/database/query-table.c +++ b/src/database/query-table.c @@ -1648,5 +1648,8 @@ void load_queries_from_disk(void) import_queries_from_disk(); DB_read_queries(); + // Log some information about the imported queries (if any) + log_counter_info(); + store_in_database = true; } diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c index 0c8132db..72bb41ce 100644 --- a/src/dnsmasq_interface.c +++ b/src/dnsmasq_interface.c @@ -2907,9 +2907,6 @@ void FTL_fork_and_bind_sockets(struct passwd *ent_pw, bool dnsmasq_start) // Initialize in-memory database starting index update_disk_db_idx(); - // Log some information about the imported queries (if any) - log_counter_info(); - // Handle real-time signals in this process (and its children) // Helper processes are already split from the main instance // so they will not listen to real-time signals From 8e63dd99f9360bf736008c0bedaab58dd92b277d Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 27 Jun 2024 17:35:56 +0200 Subject: [PATCH 186/339] Adjust tests Signed-off-by: DL6ER --- test/test_suite.bats | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/test/test_suite.bats b/test/test_suite.bats index 569bf002..a740a024 100644 --- a/test/test_suite.bats +++ b/test/test_suite.bats @@ -22,12 +22,6 @@ [[ ${lines[1]} == "" ]] } -@test "Starting tests without prior history" { - run bash -c 'grep -c "Total DNS queries: 0" /var/log/pihole/FTL.log' - printf "%s\n" "${lines[@]}" - [[ ${lines[0]} == "1" ]] -} - @test "Initial blocking status is enabled" { run bash -c 'grep -c "Blocking status is enabled" /var/log/pihole/FTL.log' printf "%s\n" "${lines[@]}" @@ -40,7 +34,7 @@ [[ ${lines[0]} == *"Compiled 2 allow and 11 deny regex"* ]] } -@test "denied domain is blocked" { +@test "Denied domain is blocked" { run bash -c "dig denied.ftl @127.0.0.1 +short" printf "%s\n" "${lines[@]}" [[ ${lines[0]} == "0.0.0.0" ]] From 3f7d317b8ad11cc9daf8ef0e9166d030eb5853a2 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 27 Jun 2024 18:42:56 +0200 Subject: [PATCH 187/339] Check capabilities only when user requested updating time time Signed-off-by: DL6ER --- src/args.c | 18 +++++++++++------- test/test_suite.bats | 2 +- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/src/args.c b/src/args.c index dbb74476..a261d3be 100644 --- a/src/args.c +++ b/src/args.c @@ -312,8 +312,15 @@ void parse_args(int argc, char* argv[]) // Create test NTP client if((argc > 1 && argc < 5) && strcmp(argv[1], "ntp") == 0) { + // Parse arguments + const bool update = (argc > 2 && strcmp(argv[2], "--update") == 0) || + (argc > 3 && strcmp(argv[3], "--update") == 0); + const char *server = "127.0.0.1"; + if(argc > 2 && strcmp(argv[2], "--update") != 0) + server = argv[2]; + // Ensure we have the necessary capabilities - if(!check_capability(CAP_SYS_TIME)) + if(update && !check_capability(CAP_SYS_TIME)) { puts("Insufficient capabilities to run NTP client"); const char *bold = cli_bold(); @@ -324,16 +331,13 @@ void parse_args(int argc, char* argv[]) puts(""); exit(EXIT_FAILURE); } + + printf("Using NTP server: %s\n", server); + // Enable stdout printing cli_mode = true; log_ctrl(false, true); readFTLconf(&config, false); - const bool update = (argc > 2 && strcmp(argv[2], "--update") == 0) || - (argc > 3 && strcmp(argv[3], "--update") == 0); - const char *server = "127.0.0.1"; - if(argc > 2 && strcmp(argv[2], "--update") != 0) - server = argv[2]; - printf("Using NTP server: %s\n", server); exit(ntp_client(server, update, true) ? EXIT_SUCCESS : EXIT_FAILURE); } diff --git a/test/test_suite.bats b/test/test_suite.bats index a740a024..c850a709 100644 --- a/test/test_suite.bats +++ b/test/test_suite.bats @@ -1354,7 +1354,7 @@ } @test "Check NTP server is broadcasting correct time" { - run bash -c './pihole-FTL ntp 127.0.0.1' + run bash -c './pihole-FTL ntp 127.0.0.1 --dry-run' printf "%s\n" "${lines[@]}" [[ $status == 0 ]] } From 0278cc1593b5846ea152589a84d53d57edf23a34 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 29 Jun 2024 09:01:38 +0200 Subject: [PATCH 188/339] Update condig item description as suggested during code review Signed-off-by: DL6ER --- src/config/config.c | 2 +- test/pihole.toml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/config/config.c b/src/config/config.c index f49f2af5..896f3a68 100644 --- a/src/config/config.c +++ b/src/config/config.c @@ -1019,7 +1019,7 @@ void initConfig(struct config *conf) conf->webserver.api.app_pwhash.c = validate_stub; // Only type-based checking conf->webserver.api.app_sudo.k = "webserver.api.app_sudo"; - conf->webserver.api.app_sudo.h = "Should application password API sessions be allowed to modify config settings?\n Setting this to true allows third-party applications using the application password to modify advanced settings, e.g., the upstream DNS servers, DHCP server settings, or changing passwords. This setting should only be enabled if really needed and only if you trust the applications using the application password."; + conf->webserver.api.app_sudo.h = "Should application password API sessions be allowed to modify config settings?\n Setting this to true allows third-party applications using the application password to modify settings, e.g., the upstream DNS servers, DHCP server settings, or changing passwords. This setting should only be enabled if really needed and only if you trust the applications using the application password."; conf->webserver.api.app_sudo.t = CONF_BOOL; conf->webserver.api.app_sudo.d.b = false; conf->webserver.api.app_sudo.c = validate_stub; // Only type-based checking diff --git a/test/pihole.toml b/test/pihole.toml index be5e3ab0..d3372ce6 100644 --- a/test/pihole.toml +++ b/test/pihole.toml @@ -704,9 +704,9 @@ # Should application password API sessions be allowed to modify config settings? # Setting this to true allows third-party applications using the application password - # to modify advanced settings, e.g., the upstream DNS servers, DHCP server settings, - # or changing passwords. This setting should only be enabled if really needed and only - # if you trust the applications using the application password. + # to modify settings, e.g., the upstream DNS servers, DHCP server settings, or + # changing passwords. This setting should only be enabled if really needed and only if + # you trust the applications using the application password. app_sudo = false # Array of clients to be excluded from certain API responses (regex): From 62acbce2cce2926f7deb0ae8b5ad8f1533f7e57a Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 29 Jun 2024 22:03:42 +0200 Subject: [PATCH 189/339] Backup and restore CLI session property Signed-off-by: DL6ER --- src/database/common.c | 15 +++++++++++++++ src/database/session-table.c | 33 ++++++++++++++++++++++++++++++++- src/database/session-table.h | 1 + 3 files changed, 48 insertions(+), 1 deletion(-) diff --git a/src/database/common.c b/src/database/common.c index bcf7d61d..b9f891cf 100644 --- a/src/database/common.c +++ b/src/database/common.c @@ -575,6 +575,21 @@ void db_init(void) dbversion = db_get_int(db, DB_VERSION); } + // Update to version 18 if lower + if(dbversion < 18) + { + // Update to version 18: Add cli column to session table + log_info("Updating long-term database to version 18"); + if(!add_session_cli_column(db)) + { + log_info("Session table cannot be updated, database not available"); + dbclose(&db); + return; + } + // Get updated version + dbversion = db_get_int(db, DB_VERSION); + } + // Last check after all migrations, if this happens, it will cause the // CI to fail the tests if(dbversion != MEMDB_VERSION) diff --git a/src/database/session-table.c b/src/database/session-table.c index 7c630a44..43e00844 100644 --- a/src/database/session-table.c +++ b/src/database/session-table.c @@ -65,6 +65,27 @@ bool add_session_app_column(sqlite3 *db) return true; } +bool add_session_cli_column(sqlite3 *db) +{ + // Start transaction of database update + SQL_bool(db, "BEGIN TRANSACTION;"); + + // Create session table + SQL_bool(db, "ALTER TABLE session ADD COLUMN cli BOOL;"); + + // Update database version to 18 + if(!db_set_FTL_property(db, DB_VERSION, 18)) + { + log_err("add_session_cli_column(): Failed to update database version!"); + return false; + } + + // Finish transaction + SQL_bool(db, "COMMIT"); + + return true; +} + // Store all session in database bool backup_db_sessions(struct session *sessions, const uint16_t max_sessions) { @@ -83,7 +104,7 @@ bool backup_db_sessions(struct session *sessions, const uint16_t max_sessions) // Insert session into database sqlite3_stmt *stmt = NULL; - if(sqlite3_prepare_v2(db, "INSERT INTO session (login_at, valid_until, remote_addr, user_agent, sid, csrf, tls_login, tls_mixed, app) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?);", -1, &stmt, 0) != SQLITE_OK) + if(sqlite3_prepare_v2(db, "INSERT INTO session (login_at, valid_until, remote_addr, user_agent, sid, csrf, tls_login, tls_mixed, app, cli) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?);", -1, &stmt, 0) != SQLITE_OK) { log_err("SQL error in backup_db_sessions(): %s (%d)", sqlite3_errmsg(db), sqlite3_errcode(db)); @@ -164,6 +185,13 @@ bool backup_db_sessions(struct session *sessions, const uint16_t max_sessions) sess->app ? 1 : 0, sqlite3_errmsg(db), sqlite3_errcode(db)); return false; } + // 10: cli + if(sqlite3_bind_int(stmt, 10, sess->cli ? 1 : 0) != SQLITE_OK) + { + log_err("Cannot bind cli = %d in backup_db_sessions(): %s (%d)", + sess->cli ? 1 : 0, sqlite3_errmsg(db), sqlite3_errcode(db)); + return false; + } // Execute statement if(sqlite3_step(stmt) != SQLITE_DONE) @@ -287,6 +315,9 @@ bool restore_db_sessions(struct session *sessions, const uint16_t max_sessions) // 8: app sess->app = sqlite3_column_int(stmt, 8) == 1 ? true : false; + // 9: app + sess->cli = sqlite3_column_int(stmt, 9) == 1 ? true : false; + // Mark session as used sess->used = true; diff --git a/src/database/session-table.h b/src/database/session-table.h index 2e9ff296..fbdaea41 100644 --- a/src/database/session-table.h +++ b/src/database/session-table.h @@ -16,6 +16,7 @@ bool create_session_table(sqlite3 *db); bool add_session_app_column(sqlite3 *db); +bool add_session_cli_column(sqlite3 *db); bool backup_db_sessions(struct session *sessions, const uint16_t max_sessions); bool restore_db_sessions(struct session *sessions, const uint16_t max_sessions); From 9de43368588941f8a2b8484dba954a412c490f55 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 30 Jun 2024 08:14:35 +0200 Subject: [PATCH 190/339] Update expected database schema in CI tests Signed-off-by: DL6ER --- src/database/common.c | 8 ++++++++ src/database/query-table.h | 2 +- test/pihole.toml | 2 +- test/test_suite.bats | 4 ++-- 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/database/common.c b/src/database/common.c index b9f891cf..85ee0dbc 100644 --- a/src/database/common.c +++ b/src/database/common.c @@ -590,6 +590,14 @@ void db_init(void) dbversion = db_get_int(db, DB_VERSION); } + /* * * * * * * * * * * * * IMPORTANT * * * * * * * * * * * * * + * If you add a new database version, check if the in-memory + * schema needs to be update as well (always recreated from + * scratch on every FTL (re)start). Also, ensure to update the + * MEMDB_VERSION in src/database/query-table.h as well as the + * expected database schema in the CI tests. + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + // Last check after all migrations, if this happens, it will cause the // CI to fail the tests if(dbversion != MEMDB_VERSION) diff --git a/src/database/query-table.h b/src/database/query-table.h index 71458e28..df0a8dd0 100644 --- a/src/database/query-table.h +++ b/src/database/query-table.h @@ -23,7 +23,7 @@ "client TEXT NOT NULL, " \ "forward TEXT );" -#define MEMDB_VERSION 17 +#define MEMDB_VERSION 18 #define CREATE_QUERY_STORAGE_TABLE "CREATE TABLE query_storage ( id INTEGER PRIMARY KEY AUTOINCREMENT, " \ "timestamp INTEGER NOT NULL, " \ "type INTEGER NOT NULL, " \ diff --git a/test/pihole.toml b/test/pihole.toml index 676893e9..0b433266 100644 --- a/test/pihole.toml +++ b/test/pihole.toml @@ -1102,7 +1102,7 @@ all = true ### CHANGED, default = false # Configuration statistics: -# 149 total entries out of which 94 entries are default +# 148 total entries out of which 93 entries are default # --> 55 entries are modified # 2 entries are forced through environment: # - misc.nice diff --git a/test/test_suite.bats b/test/test_suite.bats index c59551eb..d747e8dd 100644 --- a/test/test_suite.bats +++ b/test/test_suite.bats @@ -440,7 +440,7 @@ [[ "${lines[@]}" == *"CREATE TABLE IF NOT EXISTS \"network\" (id INTEGER PRIMARY KEY NOT NULL, hwaddr TEXT UNIQUE NOT NULL, interface TEXT NOT NULL, firstSeen INTEGER NOT NULL, lastQuery INTEGER NOT NULL, numQueries INTEGER NOT NULL, macVendor TEXT, aliasclient_id INTEGER);"* ]] [[ "${lines[@]}" == *"CREATE TABLE IF NOT EXISTS \"network_addresses\" (network_id INTEGER NOT NULL, ip TEXT UNIQUE NOT NULL, lastSeen INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), name TEXT, nameUpdated INTEGER, FOREIGN KEY(network_id) REFERENCES network(id));"* ]] [[ "${lines[@]}" == *"CREATE TABLE aliasclient (id INTEGER PRIMARY KEY NOT NULL, name TEXT NOT NULL, comment TEXT);"* ]] - [[ "${lines[@]}" == *"INSERT INTO ftl VALUES(0,17,'Database version');"* ]] + [[ "${lines[@]}" == *"INSERT INTO ftl VALUES(0,18,'Database version');"* ]] # vvv This has been added in version 10 vvv [[ "${lines[@]}" == *"CREATE VIEW queries AS SELECT id, timestamp, type, status, CASE typeof(domain) WHEN 'integer' THEN (SELECT domain FROM domain_by_id d WHERE d.id = q.domain) ELSE domain END domain,CASE typeof(client) WHEN 'integer' THEN (SELECT ip FROM client_by_id c WHERE c.id = q.client) ELSE client END client,CASE typeof(forward) WHEN 'integer' THEN (SELECT forward FROM forward_by_id f WHERE f.id = q.forward) ELSE forward END forward,CASE typeof(additional_info) WHEN 'integer' THEN (SELECT content FROM addinfo_by_id a WHERE a.id = q.additional_info) ELSE additional_info END additional_info, reply_type, reply_time, dnssec, list_id FROM query_storage q;"* ]] [[ "${lines[@]}" == *"CREATE TABLE domain_by_id (id INTEGER PRIMARY KEY, domain TEXT NOT NULL);"* ]] @@ -452,7 +452,7 @@ [[ "${lines[@]}" == *"CREATE TABLE addinfo_by_id (id INTEGER PRIMARY KEY, type INTEGER NOT NULL, content NOT NULL);"* ]] [[ "${lines[@]}" == *"CREATE UNIQUE INDEX addinfo_by_id_idx ON addinfo_by_id(type,content);"* ]] # vvv This has been added in version 15 vvv - [[ "${lines[@]}" == *"CREATE TABLE session (id INTEGER PRIMARY KEY, login_at TIMESTAMP NOT NULL, valid_until TIMESTAMP NOT NULL, remote_addr TEXT NOT NULL, user_agent TEXT, sid TEXT NOT NULL, csrf TEXT NOT NULL, tls_login BOOL, tls_mixed BOOL, app BOOL);"* ]] + [[ "${lines[@]}" == *"CREATE TABLE session (id INTEGER PRIMARY KEY, login_at TIMESTAMP NOT NULL, valid_until TIMESTAMP NOT NULL, remote_addr TEXT NOT NULL, user_agent TEXT, sid TEXT NOT NULL, csrf TEXT NOT NULL, tls_login BOOL, tls_mixed BOOL, app BOOL, cli BOOL);"* ]] } @test "Ownership, permissions and type of pihole-FTL.db correct" { From 10491f0b079019a68f726bd8cf9362a890529704 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 30 Jun 2024 09:07:55 +0200 Subject: [PATCH 191/339] Query CLI property from database table Signed-off-by: DL6ER --- src/database/session-table.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/database/session-table.c b/src/database/session-table.c index 43e00844..04654088 100644 --- a/src/database/session-table.c +++ b/src/database/session-table.c @@ -253,7 +253,7 @@ bool restore_db_sessions(struct session *sessions, const uint16_t max_sessions) // Get all sessions from database sqlite3_stmt *stmt = NULL; - if(sqlite3_prepare_v2(memdb, "SELECT login_at, valid_until, remote_addr, user_agent, sid, csrf, tls_login, tls_mixed, app FROM disk.session;", -1, &stmt, 0) != SQLITE_OK) + if(sqlite3_prepare_v2(memdb, "SELECT login_at, valid_until, remote_addr, user_agent, sid, csrf, tls_login, tls_mixed, app, cli FROM disk.session;", -1, &stmt, 0) != SQLITE_OK) { log_err("SQL error in restore_db_sessions(): %s (%d)", sqlite3_errmsg(memdb), sqlite3_errcode(memdb)); @@ -315,7 +315,7 @@ bool restore_db_sessions(struct session *sessions, const uint16_t max_sessions) // 8: app sess->app = sqlite3_column_int(stmt, 8) == 1 ? true : false; - // 9: app + // 9: cli sess->cli = sqlite3_column_int(stmt, 9) == 1 ? true : false; // Mark session as used From 8e032db78c92747f5d84158988ef04df05a74a22 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 1 Jul 2024 16:19:38 +0200 Subject: [PATCH 192/339] Start NTP server only after first successful NTP synchronization Signed-off-by: DL6ER --- src/dnsmasq_interface.c | 5 ----- src/ntp/client.c | 15 +++++++++++++-- src/ntp/ntp.h | 2 +- src/ntp/server.c | 6 +++--- 4 files changed, 17 insertions(+), 11 deletions(-) diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c index 72bb41ce..42caa431 100644 --- a/src/dnsmasq_interface.c +++ b/src/dnsmasq_interface.c @@ -2912,17 +2912,12 @@ void FTL_fork_and_bind_sockets(struct passwd *ent_pw, bool dnsmasq_start) // so they will not listen to real-time signals handle_realtime_signals(); - // We will use the attributes object later to start all threads in - // detached mode pthread_attr_t attr; // Initialize thread attributes object with default attribute values // Do NOT detach threads as we want to join them during shutdown with a // fixed timeout to give them time to clean up and finish their work pthread_attr_init(&attr); - // Initialize NTP server - ntp_server_start(&attr); - // Start NTP sync thread ntp_start_sync_thread(&attr); diff --git a/src/ntp/client.c b/src/ntp/client.c index 532689a2..0b689a6f 100644 --- a/src/ntp/client.c +++ b/src/ntp/client.c @@ -593,18 +593,28 @@ static void *ntp_client_thread(void *arg) // Run NTP client bool first_run = true; + bool ntp_server_started = false; while(!killed) { // Run NTP client - ntp_client(config.ntp.sync.server.v.s, true, false); + const bool success = ntp_client(config.ntp.sync.server.v.s, true, false); // Load queries from database after first NTP synchronization if(first_run) { load_queries_from_disk(); + first_run = false; } + if(success && !ntp_server_started) + { + // Initialize NTP server only after first NTP + // synchronization to ensure that the time is set + // correctly + ntp_server_started = ntp_server_start(); + } + // Intermediate cancellation-point BREAK_IF_KILLED(); @@ -625,6 +635,7 @@ bool ntp_start_sync_thread(pthread_attr_t *attr) strlen(config.ntp.sync.server.v.s) == 0 || config.ntp.sync.interval.v.ui == 0) { + log_info("NTP sync is disabled - NTP server will not be available"); load_queries_from_disk(); return false; } @@ -632,7 +643,7 @@ bool ntp_start_sync_thread(pthread_attr_t *attr) // Create thread if(pthread_create(&threads[NTP], attr, ntp_client_thread, NULL) != 0) { - log_err("Cannot create NTP client thread"); + log_err("Cannot create NTP client thread - NTP server will not be available"); load_queries_from_disk(); return false; } diff --git a/src/ntp/ntp.h b/src/ntp/ntp.h index 72445423..7adbe8ad 100644 --- a/src/ntp/ntp.h +++ b/src/ntp/ntp.h @@ -27,7 +27,7 @@ uint64_t gettime64(void); void print_debug_time(const char *label, const uint32_t *u32p, const uint64_t ntp_time); // Start NTP server -bool ntp_server_start(pthread_attr_t *attr); +bool ntp_server_start(void); // Start NTP client bool ntp_client(const char *server, const bool settime, const bool print); diff --git a/src/ntp/server.c b/src/ntp/server.c index 652c9525..94a259f5 100644 --- a/src/ntp/server.c +++ b/src/ntp/server.c @@ -373,7 +373,7 @@ static void *ntp_bind_and_listen(void *param) } // Start the NTP server -bool ntp_server_start(pthread_attr_t *attr) +bool ntp_server_start(void) { // Spawn two pthreads, one for IPv4 and one for IPv6 @@ -382,7 +382,7 @@ bool ntp_server_start(pthread_attr_t *attr) { // Create a thread for the IPv4 NTP server pthread_t thread; - if (pthread_create(&thread, attr, ntp_bind_and_listen, (void *)0) != 0) + if (pthread_create(&thread, NULL, ntp_bind_and_listen, (void *)0) != 0) { log_ntp_message(true, true, "Cannot create NTP server thread for IPv4"); return false; @@ -394,7 +394,7 @@ bool ntp_server_start(pthread_attr_t *attr) { // Create a thread for the IPv6 NTP server pthread_t thread; - if (pthread_create(&thread, attr, ntp_bind_and_listen, (void *)1) != 0) + if (pthread_create(&thread, NULL, ntp_bind_and_listen, (void *)1) != 0) { log_ntp_message(true, true, "Cannot create NTP server thread for IPv6"); return false; From a04303ae2373ba5e68181e6341ab22669df393f2 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 1 Jul 2024 16:22:18 +0200 Subject: [PATCH 193/339] Ensure NTP servers are properly terminated when FTL is shutting down and remove obsolete variable thread_running[] (it is only ever set but never read) Signed-off-by: DL6ER --- src/database/database-thread.c | 2 -- src/enums.h | 4 +++- src/gc.c | 2 -- src/ntp/client.c | 8 +++----- src/ntp/server.c | 17 +++++++++++------ src/resolve.c | 3 --- src/signals.c | 5 +++-- src/signals.h | 1 - src/timers.c | 2 -- 9 files changed, 20 insertions(+), 24 deletions(-) diff --git a/src/database/database-thread.c b/src/database/database-thread.c index f8768902..3a83e425 100644 --- a/src/database/database-thread.c +++ b/src/database/database-thread.c @@ -83,7 +83,6 @@ static bool analyze_database(sqlite3 *db) void *DB_thread(void *val) { // Set thread name - thread_running[DB] = true; prctl(PR_SET_NAME, thread_names[DB], 0, 0, 0); // Save timestamp as we do not want to store immediately @@ -241,6 +240,5 @@ void *DB_thread(void *val) dbclose(&db); log_info("Terminating database thread"); - thread_running[DB] = false; return NULL; } diff --git a/src/enums.h b/src/enums.h index 2df1a0eb..66ba915a 100644 --- a/src/enums.h +++ b/src/enums.h @@ -250,7 +250,9 @@ enum thread_types { GC, DNSclient, TIMER, - NTP, + NTP_CLIENT, + NTP_SERVER4, + NTP_SERVER6, THREADS_MAX } __attribute__ ((packed)); diff --git a/src/gc.c b/src/gc.c index 3d4f0962..4b6a42f5 100644 --- a/src/gc.c +++ b/src/gc.c @@ -481,7 +481,6 @@ static bool check_files_on_same_device(const char *path1, const char *path2) void *GC_thread(void *val) { // Set thread name - thread_running[GC] = true; prctl(PR_SET_NAME, thread_names[GC], 0, 0, 0); // Remember when we last ran the actions @@ -567,6 +566,5 @@ void *GC_thread(void *val) watch_config(false); log_info("Terminating GC thread"); - thread_running[GC] = false; return NULL; } diff --git a/src/ntp/client.c b/src/ntp/client.c index 0b689a6f..066d3162 100644 --- a/src/ntp/client.c +++ b/src/ntp/client.c @@ -588,8 +588,7 @@ bool ntp_client(const char *server, const bool settime, const bool print) static void *ntp_client_thread(void *arg) { // Set thread name - thread_running[NTP] = true; - prctl(PR_SET_NAME, thread_names[NTP], 0, 0, 0); + prctl(PR_SET_NAME, thread_names[NTP_CLIENT], 0, 0, 0); // Run NTP client bool first_run = true; @@ -619,11 +618,10 @@ static void *ntp_client_thread(void *arg) BREAK_IF_KILLED(); // Sleep before retrying - thread_sleepms(NTP, 1000 * config.ntp.sync.interval.v.ui); + thread_sleepms(NTP_CLIENT, 1000 * config.ntp.sync.interval.v.ui); } log_info("Terminating NTP thread"); - thread_running[NTP] = false; return NULL; } @@ -641,7 +639,7 @@ bool ntp_start_sync_thread(pthread_attr_t *attr) } // Create thread - if(pthread_create(&threads[NTP], attr, ntp_client_thread, NULL) != 0) + if(pthread_create(&threads[NTP_CLIENT], attr, ntp_client_thread, NULL) != 0) { log_err("Cannot create NTP client thread - NTP server will not be available"); load_queries_from_disk(); diff --git a/src/ntp/server.c b/src/ntp/server.c index 94a259f5..ae9eed9c 100644 --- a/src/ntp/server.c +++ b/src/ntp/server.c @@ -39,6 +39,10 @@ #include // log_ntp_message() #include "database/message-table.h" +// NTP_SERVER_IPV4,6 +#include "enums.h" +// threads +#include "signals.h" uint64_t ntp_last_sync = 0u; uint32_t ntp_root_delay = 0u; @@ -224,7 +228,7 @@ static bool ntp_reply(const int socket_fd, const struct sockaddr *saddr_p, const } // Process incoming NTP requests -static void request_process_loop(int fd, const char *ipstr, const int protocol) +static void request_process_loop(const int fd, const char *ipstr, const int protocol) { log_info("NTP server listening on %s:123 (%s)", ipstr, protocol == AF_INET ? "IPv4" : "IPv6"); while (true) @@ -280,9 +284,12 @@ static void request_process_loop(int fd, const char *ipstr, const int protocol) // Start the NTP server static void *ntp_bind_and_listen(void *param) { - const int protocol = param == 0 ? AF_INET : AF_INET6; + // Set thread name + const unsigned int thread_id = param == 0 ? NTP_SERVER4 : NTP_SERVER6; + prctl(PR_SET_NAME, thread_names[thread_id], 0, 0, 0); // Create a socket + const int protocol = param == 0 ? AF_INET : AF_INET6; errno = 0; const int s = socket(protocol, SOCK_DGRAM, IPPROTO_UDP); if(s == -1) @@ -301,8 +308,7 @@ static void *ntp_bind_and_listen(void *param) memset(ipstr, 0, sizeof(ipstr)); if(protocol == AF_INET) { - // IPv4 - set thread name - prctl(PR_SET_NAME, "NTP (IPv4)", 0, 0, 0); + // IPv4 NTP server // Prepare the bind address struct sockaddr_in bind_addr; @@ -327,8 +333,7 @@ static void *ntp_bind_and_listen(void *param) } else { - // IPv6 - set thread name - prctl(PR_SET_NAME, "NTP (IPv6)", 0, 0, 0); + // IPv6 NTP server // Set socket options to allow IPv6 only, otherwise it will bind // to both IPv4 and IPv6 and show IPv4 addresses as diff --git a/src/resolve.c b/src/resolve.c index 453e477e..a99c93ac 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -1053,14 +1053,12 @@ static void resolveUpstreams(const bool onlynew) void *DNSclient_thread(void *val) { // Set thread name - thread_running[DNSclient] = true; prctl(PR_SET_NAME, thread_names[DNSclient], 0, 0, 0); // Test struct sizes if(!check_struct_sizes()) { log_err("Struct sizes do not match expected sizes, aborting resolver thread"); - thread_running[DNSclient] = false; return NULL; } @@ -1124,6 +1122,5 @@ void *DNSclient_thread(void *val) } log_info("Terminating resolver thread"); - thread_running[DNSclient] = false; return NULL; } diff --git a/src/signals.c b/src/signals.c index 8e14cfe2..34f6b52c 100644 --- a/src/signals.c +++ b/src/signals.c @@ -34,13 +34,14 @@ static time_t FTLstarttime = 0; volatile int exit_code = EXIT_SUCCESS; volatile sig_atomic_t thread_cancellable[THREADS_MAX] = { false }; -volatile sig_atomic_t thread_running[THREADS_MAX] = { false }; const char * const thread_names[THREADS_MAX] = { "database", "housekeeper", "dns-client", "timer", - "ntp-client" + "ntp-client", + "ntp-server4", + "ntp-server6", }; // Return the (null-terminated) name of the calling thread diff --git a/src/signals.h b/src/signals.h index 76664887..4e388dd9 100644 --- a/src/signals.h +++ b/src/signals.h @@ -29,7 +29,6 @@ extern volatile sig_atomic_t want_to_reimport_aliasclients; extern volatile sig_atomic_t want_to_reload_lists; extern volatile sig_atomic_t thread_cancellable[THREADS_MAX]; -extern volatile sig_atomic_t thread_running[THREADS_MAX]; extern const char * const thread_names[THREADS_MAX]; #define BREAK_IF_KILLED() { if(killed) break; } diff --git a/src/timers.c b/src/timers.c index 17b16704..b841d3be 100644 --- a/src/timers.c +++ b/src/timers.c @@ -84,7 +84,6 @@ void get_blockingmode_timer(double *delay, bool *target_status) void *timer(void *val) { // Set thread name - thread_running[GC] = true; prctl(PR_SET_NAME, thread_names[TIMER], 0, 0, 0); // Save timestamp as we do not want to store immediately @@ -110,7 +109,6 @@ void *timer(void *val) } log_info("Terminating timer thread"); - thread_running[GC] = false; return NULL; } From 1abf15843107218a3feb3102986e0d3f2bbb41c1 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 1 Jul 2024 16:24:58 +0200 Subject: [PATCH 194/339] Fix compile problem with Pogoplug, reported on Discourse Signed-off-by: DL6ER --- src/ntp/client.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/ntp/client.c b/src/ntp/client.c index 532689a2..894d9690 100644 --- a/src/ntp/client.c +++ b/src/ntp/client.c @@ -93,10 +93,10 @@ static void format_NTP_time(char time_str[TIMESTR_SIZE], const uint64_t ntp_time client_time.tv_sec = NTPtoSEC(ntp_time); client_time.tv_usec = NTPtoUSEC(ntp_time); struct tm *client_tm = localtime(&client_time.tv_sec); - snprintf(time_str, TIMESTR_SIZE, "%04i-%02i-%02i %02i:%02i:%02i.%06"PRIi64" %s", + snprintf(time_str, TIMESTR_SIZE, "%04i-%02i-%02i %02i:%02i:%02i.%06li %s", client_tm->tm_year + 1900, client_tm->tm_mon + 1, client_tm->tm_mday, - client_tm->tm_hour, client_tm->tm_min, client_tm->tm_sec, client_time.tv_usec, - client_tm->tm_zone); + client_tm->tm_hour, client_tm->tm_min, client_tm->tm_sec, + (long int)client_time.tv_usec, client_tm->tm_zone); time_str[TIMESTR_SIZE - 1] = '\0'; } From a0372769c17acdfb78b0a8834f1234dda32f6264 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 1 Jul 2024 16:40:45 +0200 Subject: [PATCH 195/339] DNSSEC signatures are only valid for specified time windows, and should be rejected outside those windows. This generates an interesting chicken-and-egg problem for machines which don't have a hardware real time clock. For these machines to determine the correct time typically requires use of NTP and therefore DNS, but validating DNS requires that the correct time is already known. Resolve this by setting dnssec-no-timecheck removing the time-window checks (but not other DNSSEC validation.) only until NTP sync finishes (or if we realize the user doesn't want it) We do not use the overloaded SIGINT (as dnsmasq) but SIGUSR7 to avoid killing the process when in debug mode (this is a fundamental drawback of the dnsmasq implementation) Signed-off-by: DL6ER --- src/config/dnsmasq_config.c | 6 +++++- src/dnsmasq/dnsmasq.c | 3 +++ src/ntp/client.c | 9 +++++++++ src/signals.c | 10 +++++++--- src/signals.h | 1 + 5 files changed, 25 insertions(+), 4 deletions(-) diff --git a/src/config/dnsmasq_config.c b/src/config/dnsmasq_config.c index 6b8cbc82..8fbd5fdf 100644 --- a/src/config/dnsmasq_config.c +++ b/src/config/dnsmasq_config.c @@ -400,7 +400,11 @@ bool __attribute__((const)) write_dnsmasq_config(struct config *conf, bool test_ fputs("# 2017-02-02 root zone trust anchor\n", pihole_conf); fputs("trust-anchor=.,20326,8,2,E06D44B80B8F1D39A95C0B0D7C65D08458E880409BBC683457104237C7F8EC8D\n", pihole_conf); - fputs("\n", pihole_conf); + + // Prevent DNSSEC timestamp checks until either NTP synchronization has succeeded or + // the user has disabled the NTP client + fputs("# Do not check DNSSEC timestamps until NTP synchronization has succeeded\n", pihole_conf); + fputs("dnssec-no-timecheck\n\n", pihole_conf); } if(strlen(conf->dns.hostRecord.v.s) > 0) diff --git a/src/dnsmasq/dnsmasq.c b/src/dnsmasq/dnsmasq.c index 7990ed61..1a817046 100644 --- a/src/dnsmasq/dnsmasq.c +++ b/src/dnsmasq/dnsmasq.c @@ -97,6 +97,7 @@ int main_dnsmasq (int argc, char **argv) sigaction(SIGUSR2, &sigact, NULL); sigaction(SIGHUP, &sigact, NULL); sigaction(SIGUSR6, &sigact, NULL); // Pi-hole modification + sigaction(SIGUSR7, &sigact, NULL); // Pi-hole modification sigaction(SIGALRM, &sigact, NULL); sigaction(SIGCHLD, &sigact, NULL); sigaction(SIGINT, &sigact, NULL); @@ -1358,6 +1359,8 @@ static void sig_handler(int sig) else event = EVENT_TIME; } + else if (sig == SIGUSR7) // Pi-hole modified + event = EVENT_TIME; else return; diff --git a/src/ntp/client.c b/src/ntp/client.c index 066d3162..a34d8504 100644 --- a/src/ntp/client.c +++ b/src/ntp/client.c @@ -603,6 +603,12 @@ static void *ntp_client_thread(void *arg) { load_queries_from_disk(); + // If this was the first run and NTP time synchronization was + // successful, we send SIGUSR7 to the embedded dnsmasq instance + // to signal time is now guaranteed to be correct + if(success) + kill(main_pid(), SIGUSR7); + first_run = false; } @@ -634,6 +640,9 @@ bool ntp_start_sync_thread(pthread_attr_t *attr) config.ntp.sync.interval.v.ui == 0) { log_info("NTP sync is disabled - NTP server will not be available"); + // Send SIGUSR7 to embedded dnsmasq instance to signal time is + // assumed to be correct + kill(main_pid(), SIGUSR7); load_queries_from_disk(); return false; } diff --git a/src/signals.c b/src/signals.c index 34f6b52c..63270cc2 100644 --- a/src/signals.c +++ b/src/signals.c @@ -322,6 +322,11 @@ static void SIGRT_handler(int signum, siginfo_t *si, void *unused) // { // // Signal internally used to signal dnsmasq it has to stop // } + // else if(rtsig == 7) + // { + // // Signal internally used to signal dnsmasq it should do + // // DNSSEC timestamp checks + // } // Restore errno before returning back to previous context errno = _errno; @@ -448,9 +453,8 @@ void handle_realtime_signals(void) // Catch all real-time signals for(int signum = SIGRTMIN; signum <= SIGRTMAX; signum++) { - if(signum == SIGUSR6) - // Skip SIGUSR6 as it is used internally to signify - // dnsmasq to stop + if(signum == SIGUSR6 || signum == SIGUSR7) + // Skip SIGUSR6 as it is used internally by dnsmasq continue; struct sigaction SIGACTION = { 0 }; diff --git a/src/signals.h b/src/signals.h index 4e388dd9..cc5d8f1b 100644 --- a/src/signals.h +++ b/src/signals.h @@ -13,6 +13,7 @@ #include "enums.h" #define SIGUSR6 (SIGRTMIN + 6) +#define SIGUSR7 (SIGRTMIN + 7) // defined in dnsmasq/dnsmasq.h extern volatile char FTL_terminate; From 4d32ffe13f8999fd6cda80060987985fe1aa87fc Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 1 Jul 2024 17:44:54 +0200 Subject: [PATCH 196/339] Run the NTP test later in the test suite to ensure the NTP server has been started Signed-off-by: DL6ER --- test/test_suite.bats | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/test/test_suite.bats b/test/test_suite.bats index d747e8dd..3f7fa23c 100644 --- a/test/test_suite.bats +++ b/test/test_suite.bats @@ -1353,12 +1353,6 @@ [[ ${lines[0]} == '{"error":{"key":"bad_request","message":"Config items set via environment variables cannot be changed via the API","hint":"misc.nice"},"took":'*'}' ]] } -@test "Check NTP server is broadcasting correct time" { - run bash -c './pihole-FTL ntp 127.0.0.1 --dry-run' - printf "%s\n" "${lines[@]}" - [[ $status == 0 ]] -} - # We cannot easily test IPv6 as it may not be available in docker (CI) @test "API domain search: Non-existing domain returns expected JSON" { @@ -1817,3 +1811,11 @@ printf "%s\n" "${lines[@]}" [[ ${lines[0]} == "3" ]] } + +@test "Check NTP server is broadcasting correct time" { + # Run this test at the very end of the test suite + # to ensure the NTP server has been started + run bash -c './pihole-FTL ntp 127.0.0.1' + printf "%s\n" "${lines[@]}" + [[ $status == 0 ]] +} From c846b21876741430c467eee3a9b8f9c10aab95c1 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 1 Jul 2024 20:22:03 +0200 Subject: [PATCH 197/339] Add new ntp.sync.active boolean to ease disabling of the NTP client. Also move all the RTC properties inside ntp.sync because this is where they apply and where RTC sync can be disabled Signed-off-by: DL6ER --- src/api/docs/content/specs/config.yaml | 29 ++++++++++--------- src/config/config.c | 39 +++++++++++++++----------- src/config/config.h | 11 ++++---- src/ntp/client.c | 5 ++-- src/ntp/rtc.c | 28 +++++++++--------- test/pihole.toml | 25 +++++++++-------- 6 files changed, 76 insertions(+), 61 deletions(-) diff --git a/src/api/docs/content/specs/config.yaml b/src/api/docs/content/specs/config.yaml index af0edbfb..cdab29c6 100644 --- a/src/api/docs/content/specs/config.yaml +++ b/src/api/docs/content/specs/config.yaml @@ -348,21 +348,23 @@ components: sync: type: object properties: + active: + type: boolean server: type: string interval: type: integer count: type: integer - rtc: - type: object - properties: - set: - type: boolean - device: - type: string - utc: - type: boolean + rtc: + type: object + properties: + set: + type: boolean + device: + type: string + utc: + type: boolean resolver: type: object properties: @@ -708,13 +710,14 @@ components: active: true address: "" sync: + active: true server: "pool.ntp.org" interval: 3600 count: 8 - rtc: - set: true - device: "" - utc: true + rtc: + set: true + device: "" + utc: true resolver: resolveIPv4: true resolveIPv6: true diff --git a/src/config/config.c b/src/config/config.c index 8054e83d..eee5a8c0 100644 --- a/src/config/config.c +++ b/src/config/config.c @@ -814,6 +814,13 @@ void initConfig(struct config *conf) memset(&conf->ntp.ipv6.address.d.in6_addr, 0, sizeof(struct in6_addr)); conf->ntp.ipv6.address.c = validate_stub; // Only type-based checking + conf->ntp.sync.active.k = "ntp.sync.active"; + conf->ntp.sync.active.h = "Should FTL try to synchronize the system time with an upstream NTP server?"; + conf->ntp.sync.active.t = CONF_BOOL; + conf->ntp.sync.active.f = FLAG_RESTART_FTL; + conf->ntp.sync.active.d.b = true; + conf->ntp.sync.active.c = validate_stub; // Only type-based checking + conf->ntp.sync.server.k = "ntp.sync.server"; conf->ntp.sync.server.h = "NTP upstream server to sync with, e.g., \"pool.ntp.org\". Note that the NTP server should be located as close as possible to you in order to minimize the time offset possibly introduced by different routing paths."; conf->ntp.sync.server.a = cJSON_CreateStringReference("valid NTP upstream server"); @@ -833,24 +840,24 @@ void initConfig(struct config *conf) conf->ntp.sync.count.d.ui = 8; conf->ntp.sync.count.c = validate_stub; // Only type-based checking - conf->ntp.rtc.set.k = "ntp.rtc.set"; - conf->ntp.rtc.set.h = "Should FTL update a real-time clock (RTC) if available?"; - conf->ntp.rtc.set.t = CONF_BOOL; - conf->ntp.rtc.set.d.b = true; - conf->ntp.rtc.set.c = validate_stub; // Only type-based checking + conf->ntp.sync.rtc.set.k = "ntp.sync.rtc.set"; + conf->ntp.sync.rtc.set.h = "Should FTL update a real-time clock (RTC) if available?"; + conf->ntp.sync.rtc.set.t = CONF_BOOL; + conf->ntp.sync.rtc.set.d.b = true; + conf->ntp.sync.rtc.set.c = validate_stub; // Only type-based checking - conf->ntp.rtc.device.k = "ntp.rtc.device"; - conf->ntp.rtc.device.h = "Path to the RTC device to update. Leave empty for auto-discovery"; - conf->ntp.rtc.device.a = cJSON_CreateStringReference("Path to the RTC device, e.g., \"/dev/rtc0\""); - conf->ntp.rtc.device.t = CONF_STRING; - conf->ntp.rtc.device.d.s = (char*)""; - conf->ntp.rtc.device.c = validate_stub; // Only type-based checking + conf->ntp.sync.rtc.device.k = "ntp.sync.rtc.device"; + conf->ntp.sync.rtc.device.h = "Path to the RTC device to update. Leave empty for auto-discovery"; + conf->ntp.sync.rtc.device.a = cJSON_CreateStringReference("Path to the RTC device, e.g., \"/dev/rtc0\""); + conf->ntp.sync.rtc.device.t = CONF_STRING; + conf->ntp.sync.rtc.device.d.s = (char*)""; + conf->ntp.sync.rtc.device.c = validate_stub; // Only type-based checking - conf->ntp.rtc.utc.k = "ntp.rtc.utc"; - conf->ntp.rtc.utc.h = "Should the RTC be set to UTC?"; - conf->ntp.rtc.utc.t = CONF_BOOL; - conf->ntp.rtc.utc.d.b = true; - conf->ntp.rtc.utc.c = validate_stub; // Only type-based checking + conf->ntp.sync.rtc.utc.k = "ntp.sync.rtc.utc"; + conf->ntp.sync.rtc.utc.h = "Should the RTC be set to UTC?"; + conf->ntp.sync.rtc.utc.t = CONF_BOOL; + conf->ntp.sync.rtc.utc.d.b = true; + conf->ntp.sync.rtc.utc.c = validate_stub; // Only type-based checking // struct resolver diff --git a/src/config/config.h b/src/config/config.h index 069217d3..81d4a15f 100644 --- a/src/config/config.h +++ b/src/config/config.h @@ -201,15 +201,16 @@ struct config { struct conf_item address; } ipv6; struct { + struct conf_item active; struct conf_item server; struct conf_item interval; struct conf_item count; + struct { + struct conf_item set; + struct conf_item device; + struct conf_item utc; + } rtc; } sync; - struct { - struct conf_item set; - struct conf_item device; - struct conf_item utc; - } rtc; } ntp; struct { diff --git a/src/ntp/client.c b/src/ntp/client.c index a34d8504..f5c9929a 100644 --- a/src/ntp/client.c +++ b/src/ntp/client.c @@ -576,7 +576,7 @@ bool ntp_client(const char *server, const bool settime, const bool print) ntp_root_dispersion = D2FP(theta_stdev); // Finally, adjust RTC if configured - if(config.ntp.rtc.set.v.b) + if(config.ntp.sync.rtc.set.v.b) ntp_sync_rtc(); } @@ -635,7 +635,8 @@ static void *ntp_client_thread(void *arg) bool ntp_start_sync_thread(pthread_attr_t *attr) { // Return early if NTP client is disabled - if(config.ntp.sync.server.v.s == NULL || + if(config.ntp.sync.active.v.b == false || + config.ntp.sync.server.v.s == NULL || strlen(config.ntp.sync.server.v.s) == 0 || config.ntp.sync.interval.v.ui == 0) { diff --git a/src/ntp/rtc.c b/src/ntp/rtc.c index abeb457e..3fe8f93b 100644 --- a/src/ntp/rtc.c +++ b/src/ntp/rtc.c @@ -51,15 +51,15 @@ static int open_rtc(void) const gid_t gid = getgid(); // If the user has specified an RTC device, try to open it - if(config.ntp.rtc.device.v.s != NULL && - strlen(config.ntp.rtc.device.v.s) > 0) + if(config.ntp.sync.rtc.device.v.s != NULL && + strlen(config.ntp.sync.rtc.device.v.s) > 0) { // Open the RTC device - rtc_fd = open(config.ntp.rtc.device.v.s, O_RDONLY); + rtc_fd = open(config.ntp.sync.rtc.device.v.s, O_RDONLY); if (rtc_fd != -1) { log_debug(DEBUG_NTP, "Successfully opened RTC at \"%s\"", - config.ntp.rtc.device.v.s); + config.ntp.sync.rtc.device.v.s); return rtc_fd; } @@ -72,32 +72,32 @@ static int open_rtc(void) { // Get current owner of the device struct stat st = { 0 }; - if(stat(config.ntp.rtc.device.v.s, &st) == -1) + if(stat(config.ntp.sync.rtc.device.v.s, &st) == -1) { log_debug(DEBUG_NTP, "stat(\"%s\") failed: %s", - config.ntp.rtc.device.v.s, strerror(errno)); + config.ntp.sync.rtc.device.v.s, strerror(errno)); return -1; } - if(chown(config.ntp.rtc.device.v.s, uid, gid) == -1) + if(chown(config.ntp.sync.rtc.device.v.s, uid, gid) == -1) { log_debug(DEBUG_NTP, "chown(\"%s\", %u, %u) failed: %s", - config.ntp.rtc.device.v.s, uid, gid, strerror(errno)); + config.ntp.sync.rtc.device.v.s, uid, gid, strerror(errno)); return -1; } - rtc_fd = open(config.ntp.rtc.device.v.s, O_RDONLY); + rtc_fd = open(config.ntp.sync.rtc.device.v.s, O_RDONLY); if (rtc_fd != -1) { log_debug(DEBUG_NTP, "Successfully opened RTC at \"%s\"", - config.ntp.rtc.device.v.s); + config.ntp.sync.rtc.device.v.s); } // Chown the device back to the original owner - if(chown(config.ntp.rtc.device.v.s, st.st_uid, st.st_gid) == -1) + if(chown(config.ntp.sync.rtc.device.v.s, st.st_uid, st.st_gid) == -1) { log_debug(DEBUG_NTP, "chown(\"%s\", %u, %u) failed: %s", - config.ntp.rtc.device.v.s, st.st_uid, st.st_gid, strerror(errno)); + config.ntp.sync.rtc.device.v.s, st.st_uid, st.st_gid, strerror(errno)); return -1; } @@ -106,7 +106,7 @@ static int open_rtc(void) } log_debug(DEBUG_NTP, "Failed to open RTC at \"%s\": %s", - config.ntp.rtc.device.v.s, strerror(errno)); + config.ntp.sync.rtc.device.v.s, strerror(errno)); return -1; } @@ -255,7 +255,7 @@ bool ntp_sync_rtc(void) // Time to which we will set Hardware Clock, in broken down format struct tm new_time = { 0 }; const time_t newtime = time(NULL); - if(config.ntp.rtc.utc.v.b) + if(config.ntp.sync.rtc.utc.v.b) // UTC gmtime_r(&newtime, &new_time); else diff --git a/test/pihole.toml b/test/pihole.toml index 0b433266..4dbe9889 100644 --- a/test/pihole.toml +++ b/test/pihole.toml @@ -481,6 +481,9 @@ address = "" [ntp.sync] + # Should FTL try to synchronize the system time with an upstream NTP server? + active = true + # NTP upstream server to sync with, e.g., "pool.ntp.org". Note that the NTP server # should be located as close as possible to you in order to minimize the time offset # possibly introduced by different routing paths. @@ -495,18 +498,18 @@ # Number of NTP syncs to perform and average before updating the system time count = 8 - [ntp.rtc] - # Should FTL update a real-time clock (RTC) if available? - set = true + [ntp.sync.rtc] + # Should FTL update a real-time clock (RTC) if available? + set = true - # Path to the RTC device to update. Leave empty for auto-discovery - # - # Possible values are: - # Path to the RTC device, e.g., "/dev/rtc0" - device = "" + # Path to the RTC device to update. Leave empty for auto-discovery + # + # Possible values are: + # Path to the RTC device, e.g., "/dev/rtc0" + device = "" - # Should the RTC be set to UTC? - utc = true + # Should the RTC be set to UTC? + utc = true [resolver] # Should FTL try to resolve IPv4 addresses to hostnames? @@ -1102,7 +1105,7 @@ all = true ### CHANGED, default = false # Configuration statistics: -# 148 total entries out of which 93 entries are default +# 149 total entries out of which 94 entries are default # --> 55 entries are modified # 2 entries are forced through environment: # - misc.nice From a3d2d469a5071a5dff4a4dc36df0941f3269c7b9 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 1 Jul 2024 20:23:35 +0200 Subject: [PATCH 198/339] Start NTP server also when NTP client is disabled - the system may get accurate time from elsewhere - it isn't intuitive that the server cannot be started without the client Signed-off-by: DL6ER --- src/ntp/client.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/ntp/client.c b/src/ntp/client.c index f5c9929a..913fc2f8 100644 --- a/src/ntp/client.c +++ b/src/ntp/client.c @@ -640,19 +640,24 @@ bool ntp_start_sync_thread(pthread_attr_t *attr) strlen(config.ntp.sync.server.v.s) == 0 || config.ntp.sync.interval.v.ui == 0) { - log_info("NTP sync is disabled - NTP server will not be available"); + log_info("NTP sync is disabled"); // Send SIGUSR7 to embedded dnsmasq instance to signal time is // assumed to be correct kill(main_pid(), SIGUSR7); load_queries_from_disk(); + ntp_server_start(); return false; } // Create thread if(pthread_create(&threads[NTP_CLIENT], attr, ntp_client_thread, NULL) != 0) { - log_err("Cannot create NTP client thread - NTP server will not be available"); + log_err("Cannot create NTP client thread"); + // Send SIGUSR7 to embedded dnsmasq instance to signal time is + // assumed to be correct - at least we cannot synchronize it + kill(main_pid(), SIGUSR7); load_queries_from_disk(); + ntp_server_start(); return false; } From 4059586ed1d394616561c4b520df24d033b3dde2 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 1 Jul 2024 20:28:19 +0200 Subject: [PATCH 199/339] Do not even try to start NTP client thread if CAP_SYS_TIME is not available Signed-off-by: DL6ER --- src/ntp/client.c | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/src/ntp/client.c b/src/ntp/client.c index 913fc2f8..43cce286 100644 --- a/src/ntp/client.c +++ b/src/ntp/client.c @@ -41,6 +41,8 @@ #include "database/message-table.h" // load_queries_from_disk() #include "database/query-table.h" +// check_capability() +#include "capabilities.h" struct ntp_sync { bool valid; @@ -649,12 +651,27 @@ bool ntp_start_sync_thread(pthread_attr_t *attr) return false; } + // Check if we have the ambient capabilities to set the system time. + // Without CAP_SYS_TIME, we cannot set the system time and the NTP + // client will not be able to synchronize the time so there is no point + // in starting the thread. + if(!check_capability(CAP_SYS_TIME)) + { + log_warn("Insufficient permissions to set system time, NTP client not available"); + // Send SIGUSR7 to embedded dnsmasq instance to signal time is + // assumed to be correct + kill(main_pid(), SIGUSR7); + load_queries_from_disk(); + ntp_server_start(); + return false; + } + // Create thread if(pthread_create(&threads[NTP_CLIENT], attr, ntp_client_thread, NULL) != 0) { log_err("Cannot create NTP client thread"); // Send SIGUSR7 to embedded dnsmasq instance to signal time is - // assumed to be correct - at least we cannot synchronize it + // assumed to be correct kill(main_pid(), SIGUSR7); load_queries_from_disk(); ntp_server_start(); From 0b82825b53dbc128f858c964a15237564e7c70ca Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 1 Jul 2024 20:52:57 +0200 Subject: [PATCH 200/339] The CI containers may not be able to set the host's time - this is okay Signed-off-by: DL6ER --- test/test_suite.bats | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_suite.bats b/test/test_suite.bats index 3f7fa23c..c62c9c1d 100644 --- a/test/test_suite.bats +++ b/test/test_suite.bats @@ -481,7 +481,7 @@ } @test "No WARNING messages in FTL.log (besides known warnings)" { - run bash -c 'grep "WARNING:" /var/log/pihole/FTL.log | grep -v -E "CAP_NET_ADMIN|CAP_NET_RAW|CAP_SYS_NICE|CAP_IPC_LOCK|CAP_CHOWN|CAP_NET_BIND_SERVICE|CAP_SYS_TIME|(Cannot set process priority)|FTLCONF_"' + run bash -c 'grep "WARNING:" /var/log/pihole/FTL.log | grep -v -E "CAP_NET_ADMIN|CAP_NET_RAW|CAP_SYS_NICE|CAP_IPC_LOCK|CAP_CHOWN|CAP_NET_BIND_SERVICE|CAP_SYS_TIME|(Cannot set process priority)|FTLCONF_|(Insufficient permissions to set system time, NTP client not available)"' printf "%s\n" "${lines[@]}" [[ "${lines[@]}" == "" ]] } From 73d5827e901ce28f534c7182a053cbe20429d8e4 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 1 Jul 2024 05:27:45 +0200 Subject: [PATCH 201/339] Log API warnings caused by send_json_error() only if debug.api is true Signed-off-by: DL6ER --- src/api/config.c | 8 ++++---- src/api/list.c | 2 +- src/api/teleporter.c | 2 +- src/webserver/http-common.c | 26 ++++++++++++++++---------- src/webserver/http-common.h | 2 +- src/webserver/lua_web.c | 3 +-- 6 files changed, 24 insertions(+), 19 deletions(-) diff --git a/src/api/config.c b/src/api/config.c index 5b30fbbc..91ad30c3 100644 --- a/src/api/config.c +++ b/src/api/config.c @@ -713,7 +713,7 @@ static int api_config_patch(struct ftl_conn *api) return send_json_error_free(api, 400, "bad_request", "This config option can only be set in pihole.toml, not via the API", - key, true); + key, true, true); } // Check if this is a write-only config item with the placeholder value @@ -744,7 +744,7 @@ static int api_config_patch(struct ftl_conn *api) return send_json_error_free(api, 400, "bad_request", "Config item is invalid", - hint, true); + hint, true, true); } // Get pointer to memory location of this conf_item (global) @@ -759,7 +759,7 @@ static int api_config_patch(struct ftl_conn *api) return send_json_error_free(api, 400, "bad_request", "Config items set via environment variables cannot be changed via the API", - key, true); + key, true, true); } // Skip processing if value didn't change compared to current value @@ -922,7 +922,7 @@ static int api_config_put_delete(struct ftl_conn *api) return send_json_error_free(api, 400, "bad_request", "Config items set via environment variables cannot be changed via the API", - key, true); + key, true, true); } // Check if this entry exists in the array diff --git a/src/api/list.c b/src/api/list.c index e6fcc685..5a4563d6 100644 --- a/src/api/list.c +++ b/src/api/list.c @@ -461,7 +461,7 @@ static int api_list_write(struct ftl_conn *api, return send_json_error_free(api, 400, // 400 Bad Request "regex_error", "Regex validation failed", - regex_msg, true); + regex_msg, true, true); } // Try to add item(s) to table diff --git a/src/api/teleporter.c b/src/api/teleporter.c index 68c870ae..b202cf5f 100644 --- a/src/api/teleporter.c +++ b/src/api/teleporter.c @@ -316,7 +316,7 @@ static int process_received_zip(struct ftl_conn *api, struct upload_data *data) return send_json_error_free(api, 400, "bad_request", "Invalid request", - msg, true); + msg, true, true); } // Free allocated memory diff --git a/src/webserver/http-common.c b/src/webserver/http-common.c index 2b11057d..2154f0f6 100644 --- a/src/webserver/http-common.c +++ b/src/webserver/http-common.c @@ -68,27 +68,33 @@ int send_http_code(struct ftl_conn *api, const char *mime_type, int send_json_unauthorized(struct ftl_conn *api) { - return send_json_error(api, 401, - "unauthorized", - "Unauthorized", - NULL); + // Log API warnings only if debug.api is true + return send_json_error_free(api, 401, + "unauthorized", + "Unauthorized", + NULL, false, + config.debug.api.v.b); } int send_json_error(struct ftl_conn *api, const int code, const char *key, const char* message, const char *hint) { - return send_json_error_free(api, code, key, message, (char*)hint, false); + return send_json_error_free(api, code, key, message, + (char*)hint, false, true); } int send_json_error_free(struct ftl_conn *api, const int code, const char *key, const char* message, - char *hint, bool free_hint) + char *hint, const bool free_hint, const bool log) { - if(hint != NULL) - log_warn("API: %s (%s)", message, hint); - else - log_warn("API: %s", message); + if(log) + { + if(hint != NULL) + log_warn("API: %s (%s)", message, hint); + else + log_warn("API: %s", message); + } cJSON *error = JSON_NEW_OBJECT(); JSON_REF_STR_IN_OBJECT(error, "key", key); diff --git a/src/webserver/http-common.h b/src/webserver/http-common.h index 5ff01564..d8bfe1af 100644 --- a/src/webserver/http-common.h +++ b/src/webserver/http-common.h @@ -70,7 +70,7 @@ int send_json_error(struct ftl_conn *api, const int code, const char *hint); int send_json_error_free(struct ftl_conn *api, const int code, const char *key, const char* message, - char *hint, bool free_hint); + char *hint, bool free_hint, const bool log); int send_json_success(struct ftl_conn *api); const char *get_http_method_str(const enum http_method method) __attribute__((const)); diff --git a/src/webserver/lua_web.c b/src/webserver/lua_web.c index 58e36004..bef0cab1 100644 --- a/src/webserver/lua_web.c +++ b/src/webserver/lua_web.c @@ -95,8 +95,7 @@ int request_handler(struct mg_connection *conn, void *cbdata) return send_json_error_free(&api, 400, "bad_request", "Bad request", - hint, - true); + hint, true, true); } // Check if last part of the URI contains a dot (is a file) From 4781c8d134d8c1383f8f00dad5dd0e66028e32bf Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 2 Jul 2024 03:46:26 +0200 Subject: [PATCH 202/339] Do not add errors encountered seen in CLI mode to the message table Signed-off-by: DL6ER --- src/database/message-table.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/database/message-table.c b/src/database/message-table.c index 031d7fb8..409d151b 100644 --- a/src/database/message-table.c +++ b/src/database/message-table.c @@ -293,6 +293,10 @@ static int _add_message(const enum message_type type, static int _add_message(const enum message_type type, const char *message, const size_t count,...) { + // Log to database only if not in CLI mode + if(cli_mode) + return -1; + int rowid = -1; // Return early if database is known to be broken if(FTLDBerror()) @@ -1237,10 +1241,6 @@ void logg_regex_warning(const char *type, const char *warning, const int dbindex // Log to FTL.log log_warn("%s", buf); - // Log to database only if not in CLI mode - if(cli_mode) - return; - // Add to database add_message(REGEX_MESSAGE, regex, type, warning, dbindex); } From cdeb5e34272bcd6edaa78e1a8e85ebeb89a73aee Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 2 Jul 2024 03:48:12 +0200 Subject: [PATCH 203/339] Add which capability is missing in warnings (if applicable). Also reduce chown() code duplication by using a single function for all chown()-activities Signed-off-by: DL6ER --- src/config/toml_helper.c | 24 ++++-------------------- src/daemon.c | 33 ++++++++++++++++++++++++++------- src/dnsmasq_interface.c | 30 +++++++----------------------- src/files.c | 40 ++++++++++++++++++++++------------------ src/files.h | 3 +++ src/ntp/client.c | 2 +- src/ntp/rtc.c | 12 ++++++++---- src/shmem.c | 11 +++++++---- test/test_suite.bats | 2 +- 9 files changed, 79 insertions(+), 78 deletions(-) diff --git a/src/config/toml_helper.c b/src/config/toml_helper.c index 26c6b1dd..22b85dd0 100644 --- a/src/config/toml_helper.c +++ b/src/config/toml_helper.c @@ -23,6 +23,8 @@ #include // escape_json() #include "webserver/http-common.h" +// chown_pihole() +#include "files.h" // Open the TOML file for reading or writing FILE * __attribute((malloc)) __attribute((nonnull(1))) openFTLtoml(const char *mode, const unsigned int version) @@ -96,26 +98,8 @@ void closeFTLtoml(FILE *fp) // Chown file if we are root if(geteuid() == 0) - { - // Get UID and GID of user with name "pihole" - struct passwd *pwd = getpwnam("pihole"); - if(pwd == NULL) - { - log_warn("Cannot get UID and GID of user pihole: %s", strerror(errno)); - } - else - { - const uid_t pihole_uid = pwd->pw_uid; - const gid_t pihole_gid = pwd->pw_gid; - // Chown file to pihole user - if(chown(GLOBALTOMLPATH, pihole_uid, pihole_gid) != 0) - log_warn("Cannot chown "GLOBALTOMLPATH" to pihole:pihole (%u:%u): %s", - (unsigned int)pihole_uid, (unsigned int)pihole_gid, strerror(errno)); - else - log_debug(DEBUG_CONFIG, "Chown-ed "GLOBALTOMLPATH" to pihole:pihole (%u:%u)", - (unsigned int)pihole_uid, (unsigned int)pihole_gid); - } - } + chown_pihole(GLOBALTOMLPATH, NULL); + return; } diff --git a/src/daemon.c b/src/daemon.c index 5e7cf2db..a85b0ed0 100644 --- a/src/daemon.c +++ b/src/daemon.c @@ -330,13 +330,32 @@ void set_nice(void) // Set nice value const int ret = setpriority(which, pid, config.misc.nice.v.i); if(ret == -1) - // ERROR EPERM: The calling process attempted to increase its priority - // by supplying a negative value but has insufficient privileges. - // On Linux, the RLIMIT_NICE resource limit can be used to define a limit to - // which an unprivileged process's nice value can be raised. We are not - // affected by this limit when pihole-FTL is running with CAP_SYS_NICE - log_warn("Cannot set process priority to %d: %s. Process priority remains at %d", - config.misc.nice.v.i, strerror(errno), priority); + { + if(errno == EACCES || errno == EPERM) + { + // from man 2 setpriority: + // + // ERRORS + // [...] + // EACCES The caller attempted to set a lower nice value (i.e., a higher + // process priority), but did not have the required privilege (on + // Linux: did not have the CAP_SYS_NICE capability). + // + // EPERM A process was located, but its effective user ID did not match + // either the effective or the real user ID of the caller, and was + // not privileged (on Linux: did not have the CAP_SYS_NICE capabil‐ + // ity). + // [...] + log_warn("Insufficient permissions to set process priority to %d (CAP_SYS_NICE required), process priority remains at %d", + config.misc.nice.v.i, priority); + } + else + { + // Other error + log_warn("Cannot set process priority to %d: %s. Process priority remains at %d", + config.misc.nice.v.i, strerror(errno), priority); + } + } } } diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c index 42caa431..cce51b45 100644 --- a/src/dnsmasq_interface.c +++ b/src/dnsmasq_interface.c @@ -2963,26 +2963,16 @@ void FTL_fork_and_bind_sockets(struct passwd *ent_pw, bool dnsmasq_start) // we're actually dropping root (user/group may be set to root) if(ent_pw != NULL && ent_pw->pw_uid != 0) { - log_info("FTL is going to drop from root to user %s (UID %u)", - ent_pw->pw_name, ent_pw->pw_uid); + log_info("FTL is going to drop from root to user pihole"); // Change ownership of shared memory objects chown_all_shmem(ent_pw); // Configured FTL log file - if(chown(config.files.log.ftl.v.s, ent_pw->pw_uid, ent_pw->pw_gid) == -1) - { - log_warn("Setting ownership (%u:%u) of %s failed: %s (%i)", - ent_pw->pw_uid, ent_pw->pw_gid, config.files.log.ftl.v.s, strerror(errno), errno); - } + chown_pihole(config.files.log.ftl.v.s, ent_pw); // Configured FTL database file - if(chown(config.files.database.v.s, ent_pw->pw_uid, ent_pw->pw_gid) == -1) - { - log_warn("Setting ownership (%u:%u) of %s failed: %s (%i)", - ent_pw->pw_uid, ent_pw->pw_gid, config.files.database.v.s, strerror(errno), errno); - - } + chown_pihole(config.files.database.v.s, ent_pw); // Check if auxiliary files exist and change ownership char *extrafile = calloc(strlen(config.files.database.v.s) + 5, sizeof(char)); @@ -2995,20 +2985,14 @@ void FTL_fork_and_bind_sockets(struct passwd *ent_pw, bool dnsmasq_start) // Check -wal file (write-ahead log) strcpy(extrafile, config.files.database.v.s); strcat(extrafile, "-wal"); - if(file_exists(extrafile) && chown(extrafile, ent_pw->pw_uid, ent_pw->pw_gid) == -1) - { - log_warn("Setting ownership (%u:%u) of %s failed: %s (%i)", - ent_pw->pw_uid, ent_pw->pw_gid, extrafile, strerror(errno), errno); - } + if(file_exists(extrafile)) + chown_pihole(extrafile, ent_pw); // Check -shm file (mmapped shared memory) strcpy(extrafile, config.files.database.v.s); strcat(extrafile, "-shm"); - if(file_exists(extrafile) && chown(extrafile, ent_pw->pw_uid, ent_pw->pw_gid) == -1) - { - log_warn("Setting ownership (%u:%u) of %s failed: %s (%i)", - ent_pw->pw_uid, ent_pw->pw_gid, extrafile, strerror(errno), errno); - } + if(file_exists(extrafile)) + chown_pihole(extrafile, ent_pw); // Free allocated memory free(extrafile); diff --git a/src/files.c b/src/files.c index cb5799fd..8158f60a 100644 --- a/src/files.c +++ b/src/files.c @@ -16,8 +16,6 @@ // opendir(), readdir() #include -// getpwuid() -#include // getgrgid() #include // NAME_MAX @@ -434,29 +432,35 @@ static int copy_file(const char *source, const char *destination) } // Change ownership of file to pihole user -static bool chown_pihole(const char *path) +bool chown_pihole(const char *path, struct passwd *pwd) { - // Get pihole user's uid and gid - struct passwd *pwd = getpwnam("pihole"); + // Get pihole user's UID and GID if not provided if(pwd == NULL) { - log_warn("chown_pihole(): Failed to get pihole user's uid: %s", strerror(errno)); - return false; + pwd = getpwnam("pihole"); + if(pwd == NULL) + { + log_warn("chown_pihole(): Failed to get pihole user's UID/GID: %s", strerror(errno)); + return false; + } } - struct group *grp = getgrnam("pihole"); - if(grp == NULL) + + // Get group name + struct group *grp = getgrgid(pwd->pw_gid); + const char *grp_name = grp != NULL ? grp->gr_name : ""; + + // Change ownership of file to pihole user + if(chown(path, pwd->pw_uid, pwd->pw_gid) < 0) { - log_warn("chown_pihole(): Failed to get pihole user's gid: %s", strerror(errno)); + log_warn("Failed to change ownership of \"%s\" to %s:%s (%u:%u): %s", + path, pwd->pw_name, grp_name, pwd->pw_uid, pwd->pw_gid, + errno == EPERM ? "Insufficient permissions (CAP_CHOWN required)" : strerror(errno)); + return false; } - // Change ownership of file to pihole user - if(chown(path, pwd->pw_uid, grp->gr_gid) < 0) - { - log_warn("chown_pihole(): Failed to change ownership of \"%s\" to %u:%u: %s", - path, pwd->pw_uid, grp->gr_gid, strerror(errno)); - return false; - } + log_debug(DEBUG_INOTIFY, "Changed ownership of \"%s\" to %s:%s (%u:%u)", + path, pwd->pw_name, grp_name, pwd->pw_uid, pwd->pw_gid); return true; } @@ -533,7 +537,7 @@ void rotate_files(const char *path, char **first_file) } // Change ownership of file to pihole user - chown_pihole(new_path); + chown_pihole(new_path, NULL); } // Free memory diff --git a/src/files.h b/src/files.h index 742e8d9a..fc10ad23 100644 --- a/src/files.h +++ b/src/files.h @@ -16,6 +16,8 @@ #include // SHA256_DIGEST_SIZE #include +// getpwuid() +#include #define MAX_ROTATIONS 15 #define BACKUP_DIR "/etc/pihole/config_backups" @@ -31,6 +33,7 @@ void ls_dir(const char* path); unsigned int get_path_usage(const char *path, char buffer[64]); struct mntent *get_filesystem_details(const char *path); bool directory_exists(const char *path); +bool chown_pihole(const char *path, struct passwd *pwd); void rotate_files(const char *path, char **first_file); bool files_different(const char *pathA, const char* pathB, unsigned int from); bool sha256sum(const char *path, uint8_t checksum[SHA256_DIGEST_SIZE]); diff --git a/src/ntp/client.c b/src/ntp/client.c index 43cce286..2359d152 100644 --- a/src/ntp/client.c +++ b/src/ntp/client.c @@ -657,7 +657,7 @@ bool ntp_start_sync_thread(pthread_attr_t *attr) // in starting the thread. if(!check_capability(CAP_SYS_TIME)) { - log_warn("Insufficient permissions to set system time, NTP client not available"); + log_warn("Insufficient permissions to set system time (CAP_SYS_TIME required), NTP client not available"); // Send SIGUSR7 to embedded dnsmasq instance to signal time is // assumed to be correct kill(main_pid(), SIGUSR7); diff --git a/src/ntp/rtc.c b/src/ntp/rtc.c index 3fe8f93b..07232711 100644 --- a/src/ntp/rtc.c +++ b/src/ntp/rtc.c @@ -82,7 +82,8 @@ static int open_rtc(void) if(chown(config.ntp.sync.rtc.device.v.s, uid, gid) == -1) { log_debug(DEBUG_NTP, "chown(\"%s\", %u, %u) failed: %s", - config.ntp.sync.rtc.device.v.s, uid, gid, strerror(errno)); + config.ntp.sync.rtc.device.v.s, uid, gid, + errno == EPERM ? "Insufficient permissions (CAP_CHOWN required)" : strerror(errno)); return -1; } @@ -97,7 +98,8 @@ static int open_rtc(void) if(chown(config.ntp.sync.rtc.device.v.s, st.st_uid, st.st_gid) == -1) { log_debug(DEBUG_NTP, "chown(\"%s\", %u, %u) failed: %s", - config.ntp.sync.rtc.device.v.s, st.st_uid, st.st_gid, strerror(errno)); + config.ntp.sync.rtc.device.v.s, st.st_uid, st.st_gid, + errno == EPERM ? "Insufficient permissions (CAP_CHOWN required)" : strerror(errno)); return -1; } @@ -139,7 +141,8 @@ static int open_rtc(void) if(chown(rtc_devices[i], uid, gid) == -1) { log_debug(DEBUG_NTP, "chown(\"%s\", %u, %u) failed: %s", - rtc_devices[i], uid, gid, strerror(errno)); + rtc_devices[i], uid, gid, + errno == EPERM ? "Insufficient permissions (CAP_CHOWN required)" : strerror(errno)); return -1; } @@ -154,7 +157,8 @@ static int open_rtc(void) if(chown(rtc_devices[i], st.st_uid, st.st_gid) == -1) { log_debug(DEBUG_NTP, "chown(\"%s\", %u, %u) failed: %s", - rtc_devices[i], st.st_uid, st.st_gid, strerror(errno)); + rtc_devices[i], st.st_uid, st.st_gid, + errno == EPERM ? "Insufficient permissions (CAP_CHOWN required)" : strerror(errno)); return -1; } diff --git a/src/shmem.c b/src/shmem.c index 44f72eb8..83e7da5c 100644 --- a/src/shmem.c +++ b/src/shmem.c @@ -190,17 +190,20 @@ static bool chown_shmem(SharedMemory *sharedMemory, struct passwd *ent_pw) // Open shared memory object const int fd = shm_open(sharedMemory->name, O_RDWR, S_IRUSR | S_IWUSR); log_debug(DEBUG_SHMEM, "Changing %s (%d) to %u:%u", sharedMemory->name, fd, ent_pw->pw_uid, ent_pw->pw_gid); + if(fd == -1) { - log_crit("chown_shmem(): Failed to open shared memory object \"%s\": %s", + log_crit("Failed to open shared memory object \"%s\" for chown: %s", sharedMemory->name, strerror(errno)); exit(EXIT_FAILURE); } + if(fchown(fd, ent_pw->pw_uid, ent_pw->pw_gid) == -1) { - log_warn("chown_shmem(%d, %u, %u): failed for %s: %s (%d)", - fd, ent_pw->pw_uid, ent_pw->pw_gid, sharedMemory->name, - strerror(errno), errno); + log_crit("Failed to change ownership of shared memory object \"%s\": %s", + sharedMemory->name, + errno == EPERM ? "Insufficient permissions (CAP_CHOWN required)" : strerror(errno)); + return false; } diff --git a/test/test_suite.bats b/test/test_suite.bats index c62c9c1d..2e0db7e5 100644 --- a/test/test_suite.bats +++ b/test/test_suite.bats @@ -481,7 +481,7 @@ } @test "No WARNING messages in FTL.log (besides known warnings)" { - run bash -c 'grep "WARNING:" /var/log/pihole/FTL.log | grep -v -E "CAP_NET_ADMIN|CAP_NET_RAW|CAP_SYS_NICE|CAP_IPC_LOCK|CAP_CHOWN|CAP_NET_BIND_SERVICE|CAP_SYS_TIME|(Cannot set process priority)|FTLCONF_|(Insufficient permissions to set system time, NTP client not available)"' + run bash -c 'grep "WARNING:" /var/log/pihole/FTL.log | grep -v -E "CAP_NET_ADMIN|CAP_NET_RAW|CAP_SYS_NICE|CAP_IPC_LOCK|CAP_CHOWN|CAP_NET_BIND_SERVICE|CAP_SYS_TIME|FTLCONF_"' printf "%s\n" "${lines[@]}" [[ "${lines[@]}" == "" ]] } From 1e251fbae859779203ba01c6cb8f3f3f4cad8145 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 5 Jul 2024 16:18:16 +0200 Subject: [PATCH 204/339] Remove hard-coded signal name from reloading string Signed-off-by: DL6ER --- src/dnsmasq_interface.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c index cce51b45..d74ce18d 100644 --- a/src/dnsmasq_interface.c +++ b/src/dnsmasq_interface.c @@ -1803,7 +1803,7 @@ void FTL_dnsmasq_reload(void) // This function is called by the dnsmasq code on receive of SIGHUP // *before* clearing the cache and re-reading the lists if(reload++ > 0) - log_info("Received SIGHUP, flushing cache and re-reading config"); + log_info("Flushing cache and re-reading config"); // Gravity database updates // - (Re-)open gravity database connection From bfd242d6a8d5cda2b2b4226b88d83bbd86cf3e74 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 5 Jul 2024 16:45:45 +0200 Subject: [PATCH 205/339] Importmetatables already before delayed importing of queries during startup Signed-off-by: DL6ER --- src/database/query-table.c | 46 ++++++++++++++++++++++++++++---------- 1 file changed, 34 insertions(+), 12 deletions(-) diff --git a/src/database/query-table.c b/src/database/query-table.c index 435f560a..799811e5 100644 --- a/src/database/query-table.c +++ b/src/database/query-table.c @@ -41,6 +41,9 @@ static sqlite3_stmt **stmts[] = { &query_stmt, &forward_stmt, &addinfo_stmt }; +// Private prototypes +static bool import_linked_tables_from_disk(void); + // Return the maximum ID of the in-memory database unsigned long __attribute__((pure)) get_max_db_idx(void) { @@ -220,6 +223,9 @@ bool init_memory_database(void) return false; } + // Import linked-tables from disk database (domains, clients, ...) + import_linked_tables_from_disk(); + // Everything went well return true; } @@ -535,6 +541,24 @@ bool import_queries_from_disk(void) // Finalize statement sqlite3_finalize(stmt); + // End transaction + if((rc = sqlite3_exec(memdb, "END TRANSACTION", NULL, NULL, NULL)) != SQLITE_OK) + { + log_err("import_queries_from_disk(): Cannot end transaction: %s", sqlite3_errstr(rc)); + return false; + } + + // Get number of queries on disk before detaching + disk_db_num = get_number_of_queries_in_DB(memdb, "disk.query_storage"); + mem_db_num = get_number_of_queries_in_DB(memdb, "query_storage"); + + log_info("Imported %u queries from the on-disk database (it has %u rows)", mem_db_num, disk_db_num); + + return okay; +} + +static bool import_linked_tables_from_disk(void) +{ // Import linking tables and current AUTOINCREMENT values from the disk database const char *subtable_names[] = { "domain_by_id", @@ -551,6 +575,15 @@ bool import_queries_from_disk(void) "INSERT OR REPLACE INTO sqlite_sequence SELECT * FROM disk.sqlite_sequence" }; + // Begin transaction + int rc; + sqlite3 *memdb = get_memdb(); + if((rc = sqlite3_exec(memdb, "BEGIN TRANSACTION", NULL, NULL, NULL)) != SQLITE_OK) + { + log_err("import_queries_from_disk(): Cannot start transaction: %s", sqlite3_errstr(rc)); + return false; + } + // Import linking tables for(unsigned int i = 0; i < ArraySize(subtable_sql); i++) { @@ -567,13 +600,7 @@ bool import_queries_from_disk(void) return false; } - // Get number of queries on disk before detaching - disk_db_num = get_number_of_queries_in_DB(memdb, "disk.query_storage"); - mem_db_num = get_number_of_queries_in_DB(memdb, "query_storage"); - - log_info("Imported %u queries from the on-disk database (it has %u rows)", mem_db_num, disk_db_num); - - return okay; + return true; } // Export in-memory queries to disk - either due to periodic dumping (final = @@ -1370,11 +1397,6 @@ bool queries_to_database(void) log_debug(DEBUG_DATABASE, "Not storing query in database as there are none"); return true; } - if(!store_in_database) - { - log_debug(DEBUG_DATABASE, "Not storing query in database as this is disabled"); - return true; - } // Loop over recent queries and store new or changed ones in the // in-memory database From 93ca236ad9f04db3ab18d711672e20f7b4f1bd15 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 6 Jul 2024 21:18:36 +0200 Subject: [PATCH 206/339] Revert "Importmetatables already before delayed importing of queries during startup" This reverts commit bfd242d6a8d5cda2b2b4226b88d83bbd86cf3e74. --- src/database/query-table.c | 46 ++++++++++---------------------------- 1 file changed, 12 insertions(+), 34 deletions(-) diff --git a/src/database/query-table.c b/src/database/query-table.c index 799811e5..435f560a 100644 --- a/src/database/query-table.c +++ b/src/database/query-table.c @@ -41,9 +41,6 @@ static sqlite3_stmt **stmts[] = { &query_stmt, &forward_stmt, &addinfo_stmt }; -// Private prototypes -static bool import_linked_tables_from_disk(void); - // Return the maximum ID of the in-memory database unsigned long __attribute__((pure)) get_max_db_idx(void) { @@ -223,9 +220,6 @@ bool init_memory_database(void) return false; } - // Import linked-tables from disk database (domains, clients, ...) - import_linked_tables_from_disk(); - // Everything went well return true; } @@ -541,24 +535,6 @@ bool import_queries_from_disk(void) // Finalize statement sqlite3_finalize(stmt); - // End transaction - if((rc = sqlite3_exec(memdb, "END TRANSACTION", NULL, NULL, NULL)) != SQLITE_OK) - { - log_err("import_queries_from_disk(): Cannot end transaction: %s", sqlite3_errstr(rc)); - return false; - } - - // Get number of queries on disk before detaching - disk_db_num = get_number_of_queries_in_DB(memdb, "disk.query_storage"); - mem_db_num = get_number_of_queries_in_DB(memdb, "query_storage"); - - log_info("Imported %u queries from the on-disk database (it has %u rows)", mem_db_num, disk_db_num); - - return okay; -} - -static bool import_linked_tables_from_disk(void) -{ // Import linking tables and current AUTOINCREMENT values from the disk database const char *subtable_names[] = { "domain_by_id", @@ -575,15 +551,6 @@ static bool import_linked_tables_from_disk(void) "INSERT OR REPLACE INTO sqlite_sequence SELECT * FROM disk.sqlite_sequence" }; - // Begin transaction - int rc; - sqlite3 *memdb = get_memdb(); - if((rc = sqlite3_exec(memdb, "BEGIN TRANSACTION", NULL, NULL, NULL)) != SQLITE_OK) - { - log_err("import_queries_from_disk(): Cannot start transaction: %s", sqlite3_errstr(rc)); - return false; - } - // Import linking tables for(unsigned int i = 0; i < ArraySize(subtable_sql); i++) { @@ -600,7 +567,13 @@ static bool import_linked_tables_from_disk(void) return false; } - return true; + // Get number of queries on disk before detaching + disk_db_num = get_number_of_queries_in_DB(memdb, "disk.query_storage"); + mem_db_num = get_number_of_queries_in_DB(memdb, "query_storage"); + + log_info("Imported %u queries from the on-disk database (it has %u rows)", mem_db_num, disk_db_num); + + return okay; } // Export in-memory queries to disk - either due to periodic dumping (final = @@ -1397,6 +1370,11 @@ bool queries_to_database(void) log_debug(DEBUG_DATABASE, "Not storing query in database as there are none"); return true; } + if(!store_in_database) + { + log_debug(DEBUG_DATABASE, "Not storing query in database as this is disabled"); + return true; + } // Loop over recent queries and store new or changed ones in the // in-memory database From f2f8c24fde0898f3a9ab26c5570b7aa9349842ad Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 6 Jul 2024 21:21:45 +0200 Subject: [PATCH 207/339] Revert "DNSSEC signatures are only valid for specified time windows, and should be rejected outside those windows. This generates an interesting chicken-and-egg problem for machines which don't have a hardware real time clock. For these machines to determine the correct time typically requires use of NTP and therefore DNS, but validating DNS requires that the correct time is already known. Resolve this by setting dnssec-no-timecheck removing the time-window checks (but not other DNSSEC validation.) only until NTP sync finishes (or if we realize the user doesn't want it)" This reverts commit a0372769c17acdfb78b0a8834f1234dda32f6264. Signed-off-by: DL6ER --- src/config/dnsmasq_config.c | 6 +----- src/dnsmasq/dnsmasq.c | 3 --- src/ntp/client.c | 11 +---------- src/signals.c | 10 +++------- src/signals.h | 1 - 5 files changed, 5 insertions(+), 26 deletions(-) diff --git a/src/config/dnsmasq_config.c b/src/config/dnsmasq_config.c index 8fbd5fdf..6b8cbc82 100644 --- a/src/config/dnsmasq_config.c +++ b/src/config/dnsmasq_config.c @@ -400,11 +400,7 @@ bool __attribute__((const)) write_dnsmasq_config(struct config *conf, bool test_ fputs("# 2017-02-02 root zone trust anchor\n", pihole_conf); fputs("trust-anchor=.,20326,8,2,E06D44B80B8F1D39A95C0B0D7C65D08458E880409BBC683457104237C7F8EC8D\n", pihole_conf); - - // Prevent DNSSEC timestamp checks until either NTP synchronization has succeeded or - // the user has disabled the NTP client - fputs("# Do not check DNSSEC timestamps until NTP synchronization has succeeded\n", pihole_conf); - fputs("dnssec-no-timecheck\n\n", pihole_conf); + fputs("\n", pihole_conf); } if(strlen(conf->dns.hostRecord.v.s) > 0) diff --git a/src/dnsmasq/dnsmasq.c b/src/dnsmasq/dnsmasq.c index 1a817046..7990ed61 100644 --- a/src/dnsmasq/dnsmasq.c +++ b/src/dnsmasq/dnsmasq.c @@ -97,7 +97,6 @@ int main_dnsmasq (int argc, char **argv) sigaction(SIGUSR2, &sigact, NULL); sigaction(SIGHUP, &sigact, NULL); sigaction(SIGUSR6, &sigact, NULL); // Pi-hole modification - sigaction(SIGUSR7, &sigact, NULL); // Pi-hole modification sigaction(SIGALRM, &sigact, NULL); sigaction(SIGCHLD, &sigact, NULL); sigaction(SIGINT, &sigact, NULL); @@ -1359,8 +1358,6 @@ static void sig_handler(int sig) else event = EVENT_TIME; } - else if (sig == SIGUSR7) // Pi-hole modified - event = EVENT_TIME; else return; diff --git a/src/ntp/client.c b/src/ntp/client.c index 2359d152..aeb729ec 100644 --- a/src/ntp/client.c +++ b/src/ntp/client.c @@ -605,12 +605,6 @@ static void *ntp_client_thread(void *arg) { load_queries_from_disk(); - // If this was the first run and NTP time synchronization was - // successful, we send SIGUSR7 to the embedded dnsmasq instance - // to signal time is now guaranteed to be correct - if(success) - kill(main_pid(), SIGUSR7); - first_run = false; } @@ -642,10 +636,7 @@ bool ntp_start_sync_thread(pthread_attr_t *attr) strlen(config.ntp.sync.server.v.s) == 0 || config.ntp.sync.interval.v.ui == 0) { - log_info("NTP sync is disabled"); - // Send SIGUSR7 to embedded dnsmasq instance to signal time is - // assumed to be correct - kill(main_pid(), SIGUSR7); + log_info("NTP sync is disabled - NTP server will not be available"); load_queries_from_disk(); ntp_server_start(); return false; diff --git a/src/signals.c b/src/signals.c index 63270cc2..34f6b52c 100644 --- a/src/signals.c +++ b/src/signals.c @@ -322,11 +322,6 @@ static void SIGRT_handler(int signum, siginfo_t *si, void *unused) // { // // Signal internally used to signal dnsmasq it has to stop // } - // else if(rtsig == 7) - // { - // // Signal internally used to signal dnsmasq it should do - // // DNSSEC timestamp checks - // } // Restore errno before returning back to previous context errno = _errno; @@ -453,8 +448,9 @@ void handle_realtime_signals(void) // Catch all real-time signals for(int signum = SIGRTMIN; signum <= SIGRTMAX; signum++) { - if(signum == SIGUSR6 || signum == SIGUSR7) - // Skip SIGUSR6 as it is used internally by dnsmasq + if(signum == SIGUSR6) + // Skip SIGUSR6 as it is used internally to signify + // dnsmasq to stop continue; struct sigaction SIGACTION = { 0 }; diff --git a/src/signals.h b/src/signals.h index cc5d8f1b..4e388dd9 100644 --- a/src/signals.h +++ b/src/signals.h @@ -13,7 +13,6 @@ #include "enums.h" #define SIGUSR6 (SIGRTMIN + 6) -#define SIGUSR7 (SIGRTMIN + 7) // defined in dnsmasq/dnsmasq.h extern volatile char FTL_terminate; From 7f070e61982eef5963a16736c9423b41983c12f0 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 6 Jul 2024 21:25:16 +0200 Subject: [PATCH 208/339] Load queries during initialization of FTL Signed-off-by: DL6ER --- src/database/query-table.c | 7 ++++++- src/database/query-table.h | 1 - src/ntp/client.c | 23 ++--------------------- 3 files changed, 8 insertions(+), 23 deletions(-) diff --git a/src/database/query-table.c b/src/database/query-table.c index 435f560a..a2a98a00 100644 --- a/src/database/query-table.c +++ b/src/database/query-table.c @@ -41,6 +41,9 @@ static sqlite3_stmt **stmts[] = { &query_stmt, &forward_stmt, &addinfo_stmt }; +// Private prototypes +static void load_queries_from_disk(void); + // Return the maximum ID of the in-memory database unsigned long __attribute__((pure)) get_max_db_idx(void) { @@ -220,6 +223,8 @@ bool init_memory_database(void) return false; } + load_queries_from_disk(); + // Everything went well return true; } @@ -1635,7 +1640,7 @@ bool queries_to_database(void) return true; } -void load_queries_from_disk(void) +static void load_queries_from_disk(void) { // Compensate for possible jumps in time runGC(time(NULL), NULL, false); diff --git a/src/database/query-table.h b/src/database/query-table.h index df0a8dd0..1a5b8b5a 100644 --- a/src/database/query-table.h +++ b/src/database/query-table.h @@ -119,7 +119,6 @@ bool add_additional_info_column(sqlite3 *db); void DB_read_queries(void); void update_disk_db_idx(void); bool queries_to_database(void); -void load_queries_from_disk(void); bool optimize_queries_table(sqlite3 *db); bool create_addinfo_table(sqlite3 *db); diff --git a/src/ntp/client.c b/src/ntp/client.c index aeb729ec..ac4c1eff 100644 --- a/src/ntp/client.c +++ b/src/ntp/client.c @@ -39,10 +39,9 @@ #include // log_ntp_message() #include "database/message-table.h" -// load_queries_from_disk() -#include "database/query-table.h" // check_capability() #include "capabilities.h" + struct ntp_sync { bool valid; @@ -593,21 +592,12 @@ static void *ntp_client_thread(void *arg) prctl(PR_SET_NAME, thread_names[NTP_CLIENT], 0, 0, 0); // Run NTP client - bool first_run = true; bool ntp_server_started = false; while(!killed) { // Run NTP client const bool success = ntp_client(config.ntp.sync.server.v.s, true, false); - // Load queries from database after first NTP synchronization - if(first_run) - { - load_queries_from_disk(); - - first_run = false; - } - if(success && !ntp_server_started) { // Initialize NTP server only after first NTP @@ -636,8 +626,7 @@ bool ntp_start_sync_thread(pthread_attr_t *attr) strlen(config.ntp.sync.server.v.s) == 0 || config.ntp.sync.interval.v.ui == 0) { - log_info("NTP sync is disabled - NTP server will not be available"); - load_queries_from_disk(); + log_info("NTP sync is disabled"); ntp_server_start(); return false; } @@ -649,10 +638,6 @@ bool ntp_start_sync_thread(pthread_attr_t *attr) if(!check_capability(CAP_SYS_TIME)) { log_warn("Insufficient permissions to set system time (CAP_SYS_TIME required), NTP client not available"); - // Send SIGUSR7 to embedded dnsmasq instance to signal time is - // assumed to be correct - kill(main_pid(), SIGUSR7); - load_queries_from_disk(); ntp_server_start(); return false; } @@ -661,10 +646,6 @@ bool ntp_start_sync_thread(pthread_attr_t *attr) if(pthread_create(&threads[NTP_CLIENT], attr, ntp_client_thread, NULL) != 0) { log_err("Cannot create NTP client thread"); - // Send SIGUSR7 to embedded dnsmasq instance to signal time is - // assumed to be correct - kill(main_pid(), SIGUSR7); - load_queries_from_disk(); ntp_server_start(); return false; } From e07858fe92670aa76ea7f0973da2dc0f7850ab1c Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 6 Jul 2024 21:27:36 +0200 Subject: [PATCH 209/339] Restart FTL if system time has been updated by more than one hour using the internal NTP synchronization method. This ensures FTL can import the real most recent 24 hours data of history after a restart on a system lacking a real hardware clock Signed-off-by: DL6ER --- src/ntp/client.c | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/ntp/client.c b/src/ntp/client.c index ac4c1eff..7b4cfb51 100644 --- a/src/ntp/client.c +++ b/src/ntp/client.c @@ -593,11 +593,34 @@ static void *ntp_client_thread(void *arg) // Run NTP client bool ntp_server_started = false; + bool first_run = true; while(!killed) { + // Get time before NTP sync + const time_t before = time(NULL); + // Run NTP client const bool success = ntp_client(config.ntp.sync.server.v.s, true, false); + // Get time after NTP sync + const time_t after = time(NULL); + + // If the time was updated by more than one hour, restart FTL to + // import recent data. This is relevant when the system time was + // set to an incorrect value (e.g., due to a dead CMOS battery + // or overall missing RTC) and the time was off. + if(first_run && after - before > 3600) + { + log_info("System time was updated by more than one hour, restarting FTL to import recent data"); + // Set the restart flag to true + exit_code = RESTART_FTL_CODE; + // Send SIGTERM to FTL + kill(main_pid(), SIGTERM); + } + + // Set first run to false + first_run = false; + if(success && !ntp_server_started) { // Initialize NTP server only after first NTP From 3aa3e84ac6a1fb950640de5530a584fbb6656b2b Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 8 Jul 2024 18:39:52 +0200 Subject: [PATCH 210/339] Use abs(time_delta) to ensure we also restart if coming from the future. Use GCinterval instead of hard-coding one hour as interval Signed-off-by: DL6ER --- src/ntp/client.c | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/ntp/client.c b/src/ntp/client.c index 7b4cfb51..bd8090f4 100644 --- a/src/ntp/client.c +++ b/src/ntp/client.c @@ -605,13 +605,16 @@ static void *ntp_client_thread(void *arg) // Get time after NTP sync const time_t after = time(NULL); - // If the time was updated by more than one hour, restart FTL to - // import recent data. This is relevant when the system time was - // set to an incorrect value (e.g., due to a dead CMOS battery - // or overall missing RTC) and the time was off. - if(first_run && after - before > 3600) + // If the time was updated by more than a certain amount, + // restart FTL to import recent data. This is relevant when the + // system time was set to an incorrect value (e.g., due to a + // dead CMOS battery or overall missing RTC) and the time was + // off. + double time_delta; + if(first_run && (time_delta = fabs((double)after - before)) > GCinterval) { - log_info("System time was updated by more than one hour, restarting FTL to import recent data"); + log_info("System time was updated by %.1f seconds, restarting FTL to import recent data", + time_delta); // Set the restart flag to true exit_code = RESTART_FTL_CODE; // Send SIGTERM to FTL From 3e7bfd36d413870b0f8ce46cbf97617a414325c5 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 8 Jul 2024 20:26:41 +0200 Subject: [PATCH 211/339] Usw double time calculation Signed-off-by: DL6ER --- src/ntp/client.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/ntp/client.c b/src/ntp/client.c index bd8090f4..91b4ec76 100644 --- a/src/ntp/client.c +++ b/src/ntp/client.c @@ -597,21 +597,21 @@ static void *ntp_client_thread(void *arg) while(!killed) { // Get time before NTP sync - const time_t before = time(NULL); + const double before = double_time(); // Run NTP client const bool success = ntp_client(config.ntp.sync.server.v.s, true, false); // Get time after NTP sync - const time_t after = time(NULL); + const double after = double_time(); // If the time was updated by more than a certain amount, // restart FTL to import recent data. This is relevant when the // system time was set to an incorrect value (e.g., due to a // dead CMOS battery or overall missing RTC) and the time was // off. - double time_delta; - if(first_run && (time_delta = fabs((double)after - before)) > GCinterval) + double time_delta = fabs(after - before); + if(first_run && time_delta > GCinterval) { log_info("System time was updated by %.1f seconds, restarting FTL to import recent data", time_delta); From 172eaa52b9db6fde90a614d62456c11b9bf1479e Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 8 Jul 2024 22:48:12 +0200 Subject: [PATCH 212/339] Parse additional kernel info to get more IPv6 address details and laso parse IPv6 routing table Signed-off-by: DL6ER --- src/api/api.c | 1 + src/api/api.h | 1 + src/api/docs/content/specs/main.yaml | 3 + src/api/docs/content/specs/network.yaml | 241 +++++++++- src/api/network.c | 589 +++++++++++++++++++++--- 5 files changed, 749 insertions(+), 86 deletions(-) diff --git a/src/api/api.c b/src/api/api.c index 406ba7c1..8d21de2f 100644 --- a/src/api/api.c +++ b/src/api/api.c @@ -90,6 +90,7 @@ static struct { { "/api/config", "/{element}", api_config, { API_PARSE_JSON, 0 }, true, HTTP_GET }, { "/api/config", "/{element}/{value}", api_config, { API_PARSE_JSON, 0 }, true, HTTP_DELETE | HTTP_PUT }, { "/api/network/gateway", "", api_network_gateway, { API_PARSE_JSON, 0 }, true, HTTP_GET }, + { "/api/network/routes", "", api_network_routes, { API_PARSE_JSON, 0 }, true, HTTP_GET }, { "/api/network/interfaces", "", api_network_interfaces, { API_PARSE_JSON, 0 }, true, HTTP_GET }, { "/api/network/devices", "", api_network_devices, { API_PARSE_JSON, 0 }, true, HTTP_GET }, { "/api/network/devices", "/{device_id}", api_network_devices, { API_PARSE_JSON, 0 }, true, HTTP_DELETE }, diff --git a/src/api/api.h b/src/api/api.h index e8e57964..5c9bf8c8 100644 --- a/src/api/api.h +++ b/src/api/api.h @@ -74,6 +74,7 @@ int api_logs(struct ftl_conn *api); // Network methods int api_network_gateway(struct ftl_conn *api); +int api_network_routes(struct ftl_conn *api); int api_network_interfaces(struct ftl_conn *api); int api_network_devices(struct ftl_conn *api); int api_client_suggestions(struct ftl_conn *api); diff --git a/src/api/docs/content/specs/main.yaml b/src/api/docs/content/specs/main.yaml index 1762aed4..6aa7dc79 100644 --- a/src/api/docs/content/specs/main.yaml +++ b/src/api/docs/content/specs/main.yaml @@ -241,6 +241,9 @@ paths: /network/gateway: $ref: 'network.yaml#/components/paths/gateway' + /network/routes: + $ref: 'network.yaml#/components/paths/routes' + /network/interfaces: $ref: 'network.yaml#/components/paths/interfaces' diff --git a/src/api/docs/content/specs/network.yaml b/src/api/docs/content/specs/network.yaml index 1c497a65..ddc7efcd 100644 --- a/src/api/docs/content/specs/network.yaml +++ b/src/api/docs/content/specs/network.yaml @@ -27,6 +27,31 @@ components: allOf: - $ref: 'common.yaml#/components/errors/unauthorized' - $ref: 'common.yaml#/components/schemas/took' + routes: + get: + summary: Get info about the routes of your Pi-hole + tags: + - "Network information" + operationId: "get_routes" + description: | + This API hook returns infos about the networking routes of your Pi-hole. + responses: + '200': + description: OK + content: + application/json: + schema: + allOf: + - $ref: 'network.yaml#/components/schemas/routes' + - $ref: 'common.yaml#/components/schemas/took' + '401': + description: Unauthorized + content: + application/json: + schema: + allOf: + - $ref: 'common.yaml#/components/errors/unauthorized' + - $ref: 'common.yaml#/components/schemas/took' interfaces: get: summary: Get info about the interfaces of your Pi-hole @@ -119,14 +144,158 @@ components: gateway: type: object properties: - address: - type: string - description: Address of the gateway - example: "192.168.0.1" - interface: - type: string - description: Interface of your Pi-hole connected to the gateway - example: "eth0" + ipv4: + type: object + description: IPv4 gateway information + properties: + interface: + type: string + description: Interface + example: "eth0" + address: + type: string + description: Address of the gateway + example: "192.168.0.1" + ipv6: + type: object + description: IPv6 gateway information + properties: + interface: + type: string + description: Interface + example: "eth0" + address: + type: string + description: Address of the gateway + example: "fe80::3587:2fff:f11a:4321" + routes: + type: object + properties: + routes: + type: object + description: Routing table + properties: + ipv4: + type: array + description: Array of IPv4 routes + items: + type: object + properties: + destination: + type: string + description: Destination of the route + gateway: + type: string + description: Gateway of the route + metric: + type: integer + description: Metric of the route + flags: + type: array + description: Array of flags of the route + items: + type: string + interface: + type: string + description: Interface of the route + example: + - destination: "0.0.0.0" + gateway: "192.168.1.1" + metric: 0 + flags: [ "UP", "GATEWAY" ] + interface: "eth0" + - destination: "10.100.0.0" + gateway: "0.0.0.0" + metric: 0 + flags: [ "UP" ] + interface: "wg0" + ipv6: + type: array + description: Array of IPv6 routes + items: + type: object + properties: + destination: + type: object + description: Destination of the route + properties: + address: + type: string + description: IPv6 address + prefix: + type: integer + description: Prefix of the IPv6 address + type: + type: string + enum: [ "LL", "GUA", "ULA", "UNSPEC" ] + description: Type of the IPv6 address + source: + type: object + description: Source of the route + properties: + address: + type: string + description: IPv6 address + prefix: + type: integer + description: Prefix of the IPv6 address + type: + type: string + enum: [ "LL", "GUA", "ULA", "UNSPEC" ] + description: Type of the IPv6 address + gateway: + type: object + description: Gateway of the route + properties: + address: + type: string + description: IPv6 address + type: + type: string + enum: [ "LL", "GUA", "ULA", "UNSPEC" ] + description: Type of the IPv6 address + metric: + type: integer + description: Metric of the route + ref: + type: integer + description: Reference count of the route + use: + type: integer + description: Use count of the route + flags: + type: array + description: Array of flags of the route + items: + type: string + interface: + type: string + description: Interface of the route + example: + - destination: { "address": "2001:db8::", "prefix": 32, "type": "GUA" } + source: { "address": "2001:db8::1", "prefix": 128, "type": "GUA" } + gateway: { "address": "fe80::1", "type": "LL" } + metric: 0 + ref: 0 + use: 0 + flags: [ "UP", "GATEWAY" ] + interface: "eth0" + - destination: { "address": "::1", prefix: 128, "type": "UNSPEC" } + source: { "address": "::", prefix: 0, "type": "UNSPEC" } + gateway: { "address": "::", "type": "UNSPEC" } + metric: 256 + ref: 2 + use: 0 + flags: [ "UP" ] + interface: "lo" + - destination: { "address": "fd00:4711:0", prefix: 64, "type": "ULA" } + source: { "address": "::", prefix: 0, "type": "UNSPEC" } + gateway: { "address": "::", "type": "UNSPEC" } + metric: 256 + ref: 1 + use: 0 + flags: [ "UP" ] + interface: "wg0" interfaces: type: object properties: @@ -140,9 +309,6 @@ components: type: string nullable: true description: Interface name - default: - type: boolean - description: If the interface is the default gateway carrier: type: boolean description: If the interface is connected @@ -167,21 +333,55 @@ components: unit: type: string description: Unit of received data since boot + flags: + type: array + description: Array of interface flags + items: + type: string ipv4: type: array nullable: true description: Array of associated IPv4 addresses items: - type: string + type: object + properties: + address: + type: string + description: IPv4 address + netmask: + type: string + description: Netmask of the IPv4 address ipv6: type: array nullable: true description: Array of associated IPv6 addresses items: - type: string + type: object + properties: + address: + type: string + description: IPv6 address + netmask: + type: string + description: Netmask of the IPv4 address + type: + type: string + enum: [ "LL", "GUA", "ULA", "UNSPEC" ] + description: Type of the IPv6 address + prefix: + type: integer + description: Prefix of the IPv6 address + scope: + type: string + enum: [ "LINK", "GLOBAL", "HOST", "SITE", "COMPATv4", "UNKNOWN" ] + description: Scope of the IPv6 address + flags: + type: array + description: Array of flags of the IPv6 address + items: + type: string example: - name: "eth0" - default: true carrier: true speed: 1000 tx: @@ -190,10 +390,10 @@ components: rx: num: 8.1 unit: "MB" - ipv4: ["192.168.0.123"] - ipv6: ["fe80::1234:5678:9abc:def0", "2001:db8::1234:5678:9abc:def0"] + flags: [ "UP", "BROADCAST", "RUNNING", "MULTICAST" ] + ipv4: [ { "address": "192.168.0.123", "netmask": "255.255.255.0" } ] + ipv6: [ { "address": "fe80::1234:5678:9abc:def0", "netmask": "ffff:ffff:ffff:ffff::", "type": "LL", "prefix": 64, "scope": "LINK", "flags": ["PERMANENT"] }, { "address": "2001:db8::1234:5678:9abc:def0", "netmask": "ffff:ffff:ffff:ffff::", "type": "GUA", "prefix": 64, "scope": "GLOBAL", "flags": [] } ] - name: "wlan0" - default: false carrier: false speed: -1 tx: @@ -202,10 +402,10 @@ components: rx: num: 0 unit: "B" + flags: [] ipv4: [] ipv6: [] - name: "wg0" - default: false carrier: true speed: -1 tx: @@ -214,8 +414,9 @@ components: rx: num: 222.3 unit: "kB" - ipv4: ["10.1.0.1"] - ipv6: ["fd00:4711::1"] + flags: [ "UP", "POINTOPOINT", "RUNNING", "NOARP" ] + ipv4: [ { "address": "10.1.0.1", "netmask": "255.255.255.0" } ] + ipv6: [ { "address": "fd00:4711::1", "netmask": "ffff:ffff:ffff:ffff::", "type": "ULA", "prefix": 64, "scope": "GLOBAL", "flags": [ "PERMANENT" ] } ] devices: type: object properties: diff --git a/src/api/network.c b/src/api/network.c index 532a985e..d3d7832e 100644 --- a/src/api/network.c +++ b/src/api/network.c @@ -24,68 +24,484 @@ #include "database/query-table.h" // config struct #include "config/config.h" +// PRIx64 +#include +#include +// IFA_LINK and friends +#include -static bool getDefaultInterface(char iface[IF_NAMESIZE], in_addr_t *gw) + +struct flag_names { + uint32_t flag; + const char *name; +}; + +static struct flag_names iff_flags[] = { + { IFF_UP, "UP" }, + { IFF_BROADCAST, "BROADCAST" }, + { IFF_DEBUG, "DEBUG" }, + { IFF_LOOPBACK, "LOOPBACK" }, + { IFF_POINTOPOINT, "POINTOPOINT" }, + { IFF_NOTRAILERS, "NOTRAILERS" }, + { IFF_RUNNING, "RUNNING" }, + { IFF_NOARP, "NOARP" }, + { IFF_PROMISC, "PROMISC" }, + { IFF_ALLMULTI, "ALLMULTI" }, + { IFF_MASTER, "MASTER" }, + { IFF_SLAVE, "SLAVE" }, + { IFF_MULTICAST, "MULTICAST" }, + { IFF_PORTSEL, "PORTSEL" }, + { IFF_AUTOMEDIA, "AUTOMEDIA" }, + { IFF_DYNAMIC, "DYNAMIC" }, +#ifdef IFF_LOWER_UP + { IFF_LOWER_UP, "LOWER_UP" }, +#endif +#ifdef IFF_DORMANT + { IFF_DORMANT, "DORMANT" }, +#endif +#ifdef IFF_ECHO + { IFF_ECHO, "ECHO" }, +#endif +}; + +static struct flag_names ifaf_flags[] = { + { IFA_F_TEMPORARY, "TEMPORARY" }, + { IFA_F_NODAD, "NODAD" }, + { IFA_F_OPTIMISTIC, "OPTIMISTIC" }, + { IFA_F_DADFAILED, "DADFAILED" }, + { IFA_F_HOMEADDRESS, "HOMEADDRESS" }, + { IFA_F_DEPRECATED, "DEPRECATED" }, + { IFA_F_TENTATIVE, "TENTATIVE" }, + { IFA_F_PERMANENT, "PERMANENT" }, + { IFA_F_MANAGETEMPADDR, "MANAGETEMPADDR" }, + { IFA_F_NOPREFIXROUTE, "NOPREFIXROUTE" }, + { IFA_F_MCAUTOJOIN, "MCAUTOJOIN" }, + { IFA_F_STABLE_PRIVACY, "STABLE_PRIVACY" }, +}; + +static struct flag_names ripv4[] = { + { RTF_UP, "UP" }, + { RTF_GATEWAY, "GATEWAY" }, + { RTF_HOST, "HOST" }, + { RTF_REINSTATE, "REINSTATE" }, + { RTF_DYNAMIC, "DYNAMIC" }, + { RTF_MODIFIED, "MODIFIED" }, + { RTF_MTU, "MTU" }, + { RTF_MSS, "MSS" }, + { RTF_WINDOW, "WINDOW" }, + { RTF_IRTT, "IRTT" }, + { RTF_REJECT, "REJECT" }, + { RTF_STATIC, "STATIC" }, + { RTF_XRESOLVE, "XRESOLVE" }, + { RTF_NOFORWARD, "NOFORWARD" }, + { RTF_THROW, "THROW" }, + { RTF_NOPMTUDISC, "NOPMTUDISC" }, +}; + +static struct flag_names ripv6[] = { + { RTF_DEFAULT, "DEFAULT" }, + { RTF_ALLONLINK, "ALLONLINK" }, + { RTF_ADDRCONF, "ADDRCONF" }, + { RTF_LINKRT, "LINKRT" }, + { RTF_NONEXTHOP, "NONEXTHOP" }, + { RTF_CACHE, "CACHE" }, + { RTF_FLOW, "FLOW" }, + { RTF_POLICY, "POLICY" }, + { RTF_LOCAL, "LOCAL" }, + { RTF_INTERFACE, "INTERFACE" }, + { RTF_MULTICAST, "MULTICAST" }, + { RTF_BROADCAST, "BROADCAST" }, + { RTF_NAT, "NAT" }, + { RTF_ADDRCLASSMASK, "ADDRCLASSMASK" }, +}; + +// Manually taken from kernel source code in include/net/ipv6.h +#define IFA_GLOBAL 0x0000U +#define IFA_HOST 0x0010U +#define IFA_LINK 0x0020U +#define IFA_SITE 0x0040U +#define IFA_COMPATv4 0x0080U + + +static struct flag_names scopes[] = { + { IFA_GLOBAL, "GLOBAL" }, + { IFA_HOST, "HOST" }, + { IFA_LINK, "LINK" }, + { IFA_SITE, "SITE" }, + { IFA_COMPATv4, "COMPATv4" }, +}; + +static bool ipv6_hex_to_human(const char oct[33], char human[INET6_ADDRSTRLEN], const char **addr_type) { - // Get IPv4 default route gateway and associated interface - unsigned long dest_r = 0, gw_r = 0; - unsigned int flags = 0u; - int metric = 0, minmetric = __INT_MAX__; + strncpy(human, oct, 32); + // Insert ":" into address string + for(size_t i = 1; i < 8; i++) + { + const size_t m = 4*i + i - 1; + memmove(&human[m + 1], &human[m], INET6_ADDRSTRLEN - m); + human[m] = ':'; + } + // Add trailing null byte + human[INET6_ADDRSTRLEN - 1] = '\0'; + // Format address into most-compact form, e.g. + // "fe80:0000:0000:0000:0042:3dff:feb1:d93d" -> "fe80::42:3dff:feb1:d93d" + // If conversion fails, return false and keep the non-compact form + struct in6_addr addr6 = { 0 }; + if(inet_pton(AF_INET6, human, &addr6)) + { + if(addr_type != NULL) + { + // Extract first byte + // We do not directly access the underlying union as + // MUSL defines it differently than GNU C + uint8_t bytes[2]; + memcpy(&bytes, &addr6, 2); + // Global Unicast Address (2000::/3, RFC 4291) + if((bytes[0] & 0x70) == 0x20) + *addr_type = "GUA"; + // Unique Local Address (fc00::/7, RFC 4193) + if((bytes[0] & 0xfe) == 0xfc) + *addr_type = "ULA"; + // Link Local Address (fe80::/10, RFC 4291) + if((bytes[0] & 0xff) == 0xfe && (bytes[1] & 0x30) == 0) + *addr_type = "LL"; + } + + return inet_ntop(AF_INET6, &addr6, human, INET6_ADDRSTRLEN); + } + + return false; +} + +static bool read_proc_net_if_inet6(cJSON *addresses) +{ + // 4.1. if_inet6 + // + // Type: One line per address containing multiple values + // + // Here all configured IPv6 addresses are shown in a special format. The + // example displays for loopback interface only. The meaning is shown + // below (see "net/ipv6/addrconf.c" for more). + // + // # cat /proc/net/if_inet6 + // 00000000000000000000000000000001 01 80 10 80 lo + // +------------------------------+ ++ ++ ++ ++ ++ + // | | | | | | + // 1 2 3 4 5 6 + // + // 1. IPv6 address displayed in 32 hexadecimal chars without colons as separator + // 2. Netlink device number (interface index) in hexadecimal (see "ip addr" , too) + // 3. Prefix length in hexadecimal + // 4. Scope value (see kernel source " include/net/ipv6.h" and "net/ipv6/addrconf.c" for more) + // 5. Interface flags (see "include/linux/rtnetlink.h" and "net/ipv6/addrconf.c" for more) + // 6. Device name + + // Open /proc/net/if_inet6 + FILE *file; + if((file = fopen("/proc/net/if_inet6", "r"))) + { + // Parse /proc/net/if_inet6 - the kernel's IPv6 address table + char buf[1024] = { 0 }; + while(fgets(buf, sizeof(buf), file)) + { + char oct[33] = { 0 }; + unsigned int ifaceid = 0; + char iface[IF_NAMESIZE] = { 0 }; + unsigned int prefix = 0; + unsigned int scope = 0; + unsigned int flags = 0; + + // Parse address information + if(sscanf(buf, "%32s %x %x %x %x %15s", oct, &ifaceid, &prefix, &scope, &flags, iface) != 6) + continue; + + char addr_str[INET6_ADDRSTRLEN] = { 0 }; + const char *addr_type = "UNKNOWN"; + ipv6_hex_to_human(oct, addr_str, &addr_type); + + // Format flags into human-readable array of strings + cJSON *flag_array = cJSON_CreateArray(); + for(size_t i = 0; i < sizeof(ifaf_flags) / sizeof(ifaf_flags[0]); i++) + if(flags & ifaf_flags[i].flag) + cJSON_AddItemToArray(flag_array, cJSON_CreateStringReference(ifaf_flags[i].name)); + + // Create new address record + cJSON *address = cJSON_CreateObject(); + cJSON_AddStringToObject(address, "address", addr_str); + cJSON_AddItemReferenceToObject(address, "type", cJSON_CreateStringReference(addr_type)); + cJSON_AddStringToObject(address, "interface", iface); + cJSON_AddNumberToObject(address, "prefix", prefix); + const char *scope_str = "UNSPEC"; + for(size_t i = 0; i < sizeof(scopes) / sizeof(scopes[0]); i++) + if(scope == scopes[i].flag) + scope_str = scopes[i].name; + cJSON_AddItemToObject(address, "scope", cJSON_CreateStringReference(scope_str)); + cJSON_AddItemToObject(address, "flags", flag_array); + + // Add address to JSON array + cJSON_AddItemToArray(addresses, address); + } + + fclose(file); + } + else + { + log_err("Cannot read /proc/net/if_inet6: %s", strerror(errno)); + return false; + } + + return true; +} + +static bool read_proc_net_route(cJSON *routes) +{ + // Open /proc/net/route FILE *file; if((file = fopen("/proc/net/route", "r"))) { // Parse /proc/net/route - the kernel's IPv4 routing table + cJSON *ipv4 = cJSON_CreateArray(); char buf[1024] = { 0 }; while(fgets(buf, sizeof(buf), file)) { - char iface_r[IF_NAMESIZE] = { 0 }; - if(sscanf(buf, "%15s %lx %lx %x %*i %*i %i", iface_r, &dest_r, &gw_r, &flags, &metric) != 5) + char iface[IF_NAMESIZE] = { 0 }; + unsigned long dest = 0, gw = 0; + unsigned int flags = 0; + int metric = 0; + + // Parse route information + if(sscanf(buf, "%15s %lx %lx %x %*i %*i %i", iface, &dest, &gw, &flags, &metric) != 5) continue; - // Only analyze routes which are UP and whose - // destinations are a gateway - if(!(flags & RTF_UP) || !(flags & RTF_GATEWAY)) - continue; + cJSON *entry = cJSON_CreateObject(); - // Only analyze "catch all" routes (destination 0.0.0.0) - if(dest_r != 0) - continue; + // Format destination and gateway addresses + char dest_addr[INET_ADDRSTRLEN] = { 0 }; + char gw_addr[INET_ADDRSTRLEN] = { 0 }; + inet_ntop(AF_INET, &dest, dest_addr, sizeof(dest_addr)); + inet_ntop(AF_INET, &gw, gw_addr, sizeof(gw_addr)); - // Store default gateway, overwrite if we find a route with - // a lower metric - if(metric < minmetric) - { - minmetric = metric; - *gw = gw_r; - strcpy(iface, iface_r); + // Format flags into human-readable array of strings + cJSON *flag_array = cJSON_CreateArray(); + for(size_t i = 0; i < sizeof(ripv4) / sizeof(ripv4[0]); i++) + if(flags & ripv4[i].flag) + cJSON_AddItemToArray(flag_array, cJSON_CreateStringReference(ripv4[i].name)); - log_debug(DEBUG_API, "Reading interfaces: flags: %u, addr: %s, iface: %s, metric: %i, minmetric: %i", - flags, inet_ntoa(*(struct in_addr *) gw), iface, metric, minmetric); - } + // Add route information to JSON object + cJSON_AddStringToObject(entry, "destination", dest_addr); + cJSON_AddStringToObject(entry, "gateway", gw_addr); + cJSON_AddNumberToObject(entry, "metric", metric); + cJSON_AddItemToObject(entry, "flags", flag_array); + cJSON_AddStringToObject(entry, "interface", iface); + + // Add route information to JSON array + cJSON_AddItemToArray(ipv4, entry); } + + fclose(file); + + // Add IPv4 routes to JSON object + cJSON_AddItemToObject(routes, "ipv4", ipv4); + } + return true; +} + +static bool read_proc_net_ipv6_route(cJSON *routes) +{ + // Open /proc/net/route + FILE *file; + + // Open /proc/net/ipv6_route + if((file = fopen("/proc/net/ipv6_route", "r"))) + { + // Parse /proc/net/ipv6_route - the kernel's IPv6 routing table + // 4.2. ipv6_route + // + // Type: One line per route containing multiple values + // + // Here all configured IPv6 routes are shown in a special + // format. The example displays for loopback interface only. The + // meaning is shown below (see ”net/ipv6/route.c” for more). + // + // # cat /proc/net/ipv6_route + // 00000000000000000000000000000000 00 00000000000000000000000000000000 00 00000000000000000000000000000000 ffffffff 00000001 00000001 00200200 lo + // +------------------------------+ ++ +------------------------------+ ++ +------------------------------+ +------+ +------+ +------+ +------+ ++ + // | | | | | | | | | | + // 1 2 3 4 5 6 7 8 9 10 + // + // 1. IPv6 destination network displayed in 32 hexadecimal chars without colons as separator + // 2. IPv6 destination prefix length in hexadecimal + // 3. IPv6 source network displayed in 32 hexadecimal chars without colons as separator + // 4. IPv6 source prefix length in hexadecimal + // 5. IPv6 next hop displayed in 32 hexadecimal chars without colons as separator + // 6. Metric in hexadecimal + // 7. Reference counter + // 8. Use counter + // 9. Flags + // 10. Device name + + cJSON *ipv6 = cJSON_CreateArray(); + + char buf[1024] = { 0 }; + while(fgets(buf, sizeof(buf), file)) + { + char iface[IF_NAMESIZE] = { 0 }; + char dest[33] = { 0 }; + char src[33] = { 0 }; + char gw[33] = { 0 }; + unsigned int prefix_dest = 0; + unsigned int prefix_src = 0; + unsigned int metric = 0; + unsigned int ref = 0; + unsigned int use = 0; + unsigned int flags = 0; + + // Parse route information + if(sscanf(buf, "%32s %x %32s %x %32s %x %x %x %x %15s", + dest, &prefix_dest, src, &prefix_src, gw, &metric, &ref, &use, &flags, iface) != 10) + continue; + + // Format flags into human-readable array of strings + cJSON *flag_array = cJSON_CreateArray(); + for(size_t i = 0; i < sizeof(ripv4) / sizeof(ripv4[0]); i++) + if(flags & ripv4[i].flag) + cJSON_AddItemToArray(flag_array, cJSON_CreateStringReference(ripv4[i].name)); + for(size_t i = 0; i < sizeof(ripv6) / sizeof(ripv6[0]); i++) + if(flags & ripv6[i].flag) + cJSON_AddItemToArray(flag_array, cJSON_CreateStringReference(ripv6[i].name)); + + // Format destination, source, and gateway addresses + char dest_addr[INET6_ADDRSTRLEN] = { 0 }; + const char *dest_addr_type = "UNSPEC"; + char src_addr[INET6_ADDRSTRLEN] = { 0 }; + const char *src_addr_type = "UNSPEC"; + char gw_addr[INET6_ADDRSTRLEN] = { 0 }; + const char *gw_addr_type = "UNSPEC"; + ipv6_hex_to_human(dest, dest_addr, &dest_addr_type); + ipv6_hex_to_human(src, src_addr, &src_addr_type); + ipv6_hex_to_human(gw, gw_addr, &gw_addr_type); + + // Create new route record + cJSON *entry = cJSON_CreateObject(); + + cJSON *destination = cJSON_CreateObject(); + cJSON_AddStringToObject(destination, "address", dest_addr); + cJSON_AddNumberToObject(destination, "prefix", prefix_dest); + cJSON_AddItemToObject(destination, "type", cJSON_CreateStringReference(dest_addr_type)); + cJSON_AddItemToObject(entry, "destination", destination); + + cJSON *source = cJSON_CreateObject(); + cJSON_AddStringToObject(source, "address", src_addr); + cJSON_AddNumberToObject(source, "prefix", prefix_src); + cJSON_AddItemToObject(source, "type", cJSON_CreateStringReference(src_addr_type)); + cJSON_AddItemToObject(entry, "source", source); + + cJSON *gateway = cJSON_CreateObject(); + cJSON_AddStringToObject(gateway, "address", gw_addr); + cJSON_AddItemToObject(gateway, "type", cJSON_CreateStringReference(gw_addr_type)); + cJSON_AddItemToObject(entry, "gateway", gateway); + + cJSON_AddNumberToObject(entry, "metric", metric); + cJSON_AddNumberToObject(entry, "ref", ref); + cJSON_AddNumberToObject(entry, "use", use); + cJSON_AddItemToObject(entry, "flags", flag_array); + cJSON_AddStringToObject(entry, "interface", iface); + + // Add route information to JSON array + cJSON_AddItemToArray(ipv6, entry); + } + + // Add IPv6 routes to JSON object + cJSON_AddItemToObject(routes, "ipv6", ipv6); + fclose(file); } - else - log_err("Cannot read /proc/net/route: %s", strerror(errno)); - // Return success based on having found the default gateway's address - return gw != 0; + return true; } int api_network_gateway(struct ftl_conn *api) { - in_addr_t gw = 0; - char iface[IF_NAMESIZE] = { 0 }; - - // Get default interface - getDefaultInterface(iface, &gw); - - // Generate JSON response cJSON *json = JSON_NEW_OBJECT(); - const char *gwaddr = inet_ntoa(*(struct in_addr *) &gw); - JSON_COPY_STR_TO_OBJECT(json, "address", gwaddr); - JSON_REF_STR_IN_OBJECT(json, "interface", iface); + + // Get JSON routes + cJSON *routes = cJSON_CreateObject(); + read_proc_net_route(routes); + read_proc_net_ipv6_route(routes); + + // Search for route with GATEWAY flag set + cJSON *ipv4 = cJSON_GetObjectItem(routes, "ipv4"); + cJSON *r_ipv4 = cJSON_CreateObject(); + cJSON *route = NULL; + cJSON_ArrayForEach(route, ipv4) + { + cJSON *flags = cJSON_GetObjectItem(route, "flags"); + if(cJSON_IsArray(flags)) + { + cJSON *flag = NULL; + cJSON_ArrayForEach(flag, flags) + { + if(strcmp(cJSON_GetStringValue(flag), "GATEWAY") == 0) + { + // Extract interface name + const char *iface_name = cJSON_GetStringValue(cJSON_GetObjectItem(route, "interface")); + JSON_COPY_STR_TO_OBJECT(r_ipv4, "interface", iface_name); + + // Extract gateway address + const char *gw_addr = cJSON_GetStringValue(cJSON_GetObjectItem(route, "gateway")); + JSON_COPY_STR_TO_OBJECT(r_ipv4, "address", gw_addr); + + break; + } + } + } + } + + // else: Search ipv6 routes + cJSON *ipv6 = cJSON_GetObjectItem(routes, "ipv6"); + cJSON *r_ipv6 = cJSON_CreateObject(); + cJSON_ArrayForEach(route, ipv6) + { + cJSON *flags = cJSON_GetObjectItem(route, "flags"); + if(cJSON_IsArray(flags)) + { + cJSON *flag = NULL; + cJSON_ArrayForEach(flag, flags) + { + if(strcmp(cJSON_GetStringValue(flag), "GATEWAY") == 0) + { + // Extract interface name + const char *iface_name = cJSON_GetStringValue(cJSON_GetObjectItem(route, "interface")); + JSON_COPY_STR_TO_OBJECT(r_ipv6, "interface", iface_name); + + // Extract gateway address + const char *gw_addr = cJSON_GetStringValue(cJSON_GetObjectItem(cJSON_GetObjectItem(route, "gateway"), "address")); + + JSON_COPY_STR_TO_OBJECT(r_ipv6, "address", gw_addr); + break; + } + } + } + } + + // Add gateway information to JSON object + JSON_ADD_ITEM_TO_OBJECT(json, "ipv4", r_ipv4); + JSON_ADD_ITEM_TO_OBJECT(json, "ipv6", r_ipv6); + + cJSON_Delete(routes); + + JSON_SEND_OBJECT(json); +} + +int api_network_routes(struct ftl_conn *api) +{ + // Add routing information + cJSON *routes = JSON_NEW_OBJECT(); + read_proc_net_route(routes); + read_proc_net_ipv6_route(routes); + cJSON *json = JSON_NEW_OBJECT(); + JSON_ADD_ITEM_TO_OBJECT(json, "routes", routes); JSON_SEND_OBJECT(json); } @@ -93,11 +509,6 @@ int api_network_interfaces(struct ftl_conn *api) { cJSON *json = JSON_NEW_OBJECT(); - // Get interface with default route - in_addr_t gw = 0; - char default_iface[IF_NAMESIZE] = { 0 }; - getDefaultInterface(default_iface, &gw); - // Enumerate and list interfaces // Loop over interfaces and extract information DIR *dfd; @@ -119,6 +530,10 @@ int api_network_interfaces(struct ftl_conn *api) if(getifaddrs(&ifap) == -1) log_err("API: Cannot get interface addresses: %s", strerror(errno)); + // Parse IPv6 address details + cJSON *ipv6a = JSON_NEW_ARRAY(); + read_proc_net_if_inet6(ipv6a); + cJSON *interfaces = JSON_NEW_ARRAY(); // Walk /sys/class/net directory while ((dp = readdir(dfd)) != NULL) @@ -134,10 +549,6 @@ int api_network_interfaces(struct ftl_conn *api) const char *iface_name = dp->d_name; JSON_COPY_STR_TO_OBJECT(iface, "name", iface_name); - // Is this the default interface? - const bool is_default_iface = strcmp(iface_name, default_iface) == 0; - JSON_ADD_BOOL_TO_OBJECT(iface, "default", is_default_iface); - // Extract carrier status bool carrier = false; snprintf(fname, sizeof(fname)-1, "/sys/class/net/%s/carrier", iface_name); @@ -230,30 +641,74 @@ int api_network_interfaces(struct ftl_conn *api) // If we reach this point, we found the correct interface const sa_family_t family = ifa->ifa_addr->sa_family; char host[NI_MAXHOST] = { 0 }; - if(family == AF_INET || family == AF_INET6) + if(family != AF_INET && family != AF_INET6) + continue; + // Get IP address + const int s = getnameinfo(ifa->ifa_addr, + (family == AF_INET) ? + sizeof(struct sockaddr_in) : + sizeof(struct sockaddr_in6), + host, NI_MAXHOST, + NULL, 0, NI_NUMERICHOST); + if (s != 0) { - // Get IP address - const int s = getnameinfo(ifa->ifa_addr, - (family == AF_INET) ? - sizeof(struct sockaddr_in) : - sizeof(struct sockaddr_in6), - host, NI_MAXHOST, - NULL, 0, NI_NUMERICHOST); - if (s != 0) + log_warn("API: getnameinfo(1) failed: %s\n", gai_strerror(s)); + continue; + } + // Get netmask + char netmask[NI_MAXHOST] = { 0 }; + const int s2 = getnameinfo(ifa->ifa_netmask, + (family == AF_INET) ? + sizeof(struct sockaddr_in) : + sizeof(struct sockaddr_in6), + netmask, NI_MAXHOST, + NULL, 0, NI_NUMERICHOST); + if (s2 != 0) + { + log_warn("API: getnameinfo(2) failed: %s\n", gai_strerror(s2)); + continue; + } + + cJSON *new_addr = JSON_NEW_OBJECT(); + JSON_COPY_STR_TO_OBJECT(new_addr, "address", host); + JSON_COPY_STR_TO_OBJECT(new_addr, "netmask", netmask); + + if(family == AF_INET) + { + // Add IPv4 address to array + JSON_ADD_ITEM_TO_ARRAY(ipv4, new_addr); + } + else if(family == AF_INET6) + { + // Search address in ipv6a array and add further details + cJSON *ipv6_entry = NULL; + cJSON_ArrayForEach(ipv6_entry, ipv6a) { - log_warn("API: getnameinfo() failed: %s\n", gai_strerror(s)); - continue; + // Compare interface and address + const char *arr_name = cJSON_GetStringValue(cJSON_GetObjectItem(ipv6_entry, "interface")); + const char *arr_addr = cJSON_GetStringValue(cJSON_GetObjectItem(ipv6_entry, "address")); + // We compare only the first part of the address as the second part may be an interface specifier (%veth...) + if(strcmp(arr_name, iface_name) == 0 && strncmp(arr_addr, host, min(strlen(arr_addr), strlen(host))) == 0) + { + // Copy details from ipv6a array to new_addr (prefix, scope, flags) + JSON_ADD_ITEM_TO_OBJECT(new_addr, "type", cJSON_Duplicate(cJSON_GetObjectItem(ipv6_entry, "type"), true)); + JSON_ADD_NUMBER_TO_OBJECT(new_addr, "prefix", cJSON_GetNumberValue(cJSON_GetObjectItem(ipv6_entry, "prefix"))); + JSON_ADD_ITEM_TO_OBJECT(new_addr, "scope", cJSON_Duplicate(cJSON_GetObjectItem(ipv6_entry, "scope"), true)); + JSON_ADD_ITEM_TO_OBJECT(new_addr, "flags", cJSON_Duplicate(cJSON_GetObjectItem(ipv6_entry, "flags"), true)); + break; + } } - if(family == AF_INET) - { - JSON_COPY_STR_TO_ARRAY(ipv4, host); - } - else if(family == AF_INET6) - { - JSON_COPY_STR_TO_ARRAY(ipv6, host); - } + // Add IPv6 address to array + JSON_ADD_ITEM_TO_ARRAY(ipv6, new_addr); } + + // Format flags into human-readable array of strings + cJSON *flag_array = cJSON_CreateArray(); + for(size_t i = 0; i < sizeof(iff_flags) / sizeof(iff_flags[0]); i++) + if(ifa->ifa_flags & iff_flags[i].flag) + cJSON_AddItemToArray(flag_array, cJSON_CreateStringReference(iff_flags[i].name)); + JSON_ADD_ITEM_TO_OBJECT(iface, "flags", flag_array); } JSON_ADD_ITEM_TO_OBJECT(iface, "ipv4", ipv4); JSON_ADD_ITEM_TO_OBJECT(iface, "ipv6", ipv6); @@ -271,6 +726,8 @@ int api_network_interfaces(struct ftl_conn *api) freeifaddrs(ifap); closedir(dfd); + cJSON_Delete(ipv6a); + ipv6a = NULL; cJSON *sum = JSON_NEW_OBJECT(); JSON_COPY_STR_TO_OBJECT(sum, "name", "sum"); From 00ff114cabf091fbb108d77d57699ec6e2c0f72c Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 9 Jul 2024 18:58:55 +0200 Subject: [PATCH 213/339] Do not start threads in detached mode. Joining them may SEGFAULT with libmusl Signed-off-by: DL6ER --- src/dnsmasq_interface.c | 4 +--- src/tools/arp-scan.c | 2 -- src/tools/dhcp-discover.c | 2 -- 3 files changed, 1 insertion(+), 7 deletions(-) diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c index 72bb41ce..c3803f46 100644 --- a/src/dnsmasq_interface.c +++ b/src/dnsmasq_interface.c @@ -2912,12 +2912,10 @@ void FTL_fork_and_bind_sockets(struct passwd *ent_pw, bool dnsmasq_start) // so they will not listen to real-time signals handle_realtime_signals(); - // We will use the attributes object later to start all threads in - // detached mode - pthread_attr_t attr; // Initialize thread attributes object with default attribute values // Do NOT detach threads as we want to join them during shutdown with a // fixed timeout to give them time to clean up and finish their work + pthread_attr_t attr; pthread_attr_init(&attr); // Initialize NTP server diff --git a/src/tools/arp-scan.c b/src/tools/arp-scan.c index c5ac6ec3..923625a3 100644 --- a/src/tools/arp-scan.c +++ b/src/tools/arp-scan.c @@ -616,8 +616,6 @@ int run_arp_scan(const bool scan_all, const bool extreme_mode) pthread_attr_t attr; // Initialize thread attributes object with default attribute values pthread_attr_init(&attr); - // Set thread attributes to detached mode - pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); struct ifaddrs *addrs, *tmp; getifaddrs(&addrs); diff --git a/src/tools/dhcp-discover.c b/src/tools/dhcp-discover.c index 045c8c14..c68a74f9 100644 --- a/src/tools/dhcp-discover.c +++ b/src/tools/dhcp-discover.c @@ -725,8 +725,6 @@ int run_dhcp_discover(void) pthread_attr_t attr; // Initialize thread attributes object with default attribute values pthread_attr_init(&attr); - // Set thread attributes to detached mode - pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); // Create processing/printfing lock pthread_mutexattr_t lock_attr; From 6cf1a6774e83958b85186057541d6c51131cc9bf Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 9 Jul 2024 21:57:45 +0200 Subject: [PATCH 214/339] Add netlink implementation Signed-off-by: DL6ER --- src/api/docs/content/specs/network.yaml | 530 +++++++++------- src/api/network.c | 748 ++-------------------- src/dnsmasq/network.c | 43 ++ src/syscalls/CMakeLists.txt | 3 + src/syscalls/netlink.c | 796 ++++++++++++++++++++++++ src/syscalls/netlink.h | 53 ++ src/syscalls/netlink_consts.h | 598 ++++++++++++++++++ src/webserver/json_macros.h | 9 + 8 files changed, 1836 insertions(+), 944 deletions(-) create mode 100644 src/syscalls/netlink.c create mode 100644 src/syscalls/netlink.h create mode 100644 src/syscalls/netlink_consts.h diff --git a/src/api/docs/content/specs/network.yaml b/src/api/docs/content/specs/network.yaml index ddc7efcd..60a68f22 100644 --- a/src/api/docs/content/specs/network.yaml +++ b/src/api/docs/content/specs/network.yaml @@ -33,8 +33,12 @@ components: tags: - "Network information" operationId: "get_routes" + parameters: + - $ref: 'network.yaml#/components/parameters/devices/detailed' description: | - This API hook returns infos about the networking routes of your Pi-hole. + This API hook returns infos about the networking routes of your Pi-hole. Note that not all described fields are applicable to any routing type. Users must not rely on the presence of any field without checking the route type first. + + If the optional parameter `detailed` is set to `true`, the response will include more detailed information about the individual routes where the available information is dependent on the route type and state. responses: '200': description: OK @@ -58,8 +62,12 @@ components: tags: - "Network information" operationId: "get_interfaces" + parameters: + - $ref: 'network.yaml#/components/parameters/devices/detailed' description: | - This API hook returns infos about the networking interfaces of your Pi-hole. + This API hook returns infos about the networking interfaces of your Pi-hole. Note that not all described fields are applicable to any routing type. Users must not rely on the presence of any field without checking the route type first. + + If the optional parameter `detailed` is set to `true`, the response will include more detailed information about the individual interfaces where the available information is dependent on the interface type and state. responses: '200': description: OK @@ -144,158 +152,132 @@ components: gateway: type: object properties: - ipv4: - type: object - description: IPv4 gateway information - properties: - interface: - type: string - description: Interface - example: "eth0" - address: - type: string - description: Address of the gateway - example: "192.168.0.1" - ipv6: - type: object - description: IPv6 gateway information - properties: - interface: - type: string - description: Interface - example: "eth0" - address: - type: string - description: Address of the gateway - example: "fe80::3587:2fff:f11a:4321" + gateway: + type: array + items: + type: object + properties: + family: + type: string + description: Address family + interface: + type: string + description: Interface name + address: + type: string + description: Gateway address + example: + - family: "inet" + interface: "eth0" + address: "192.168.0.2" + - family: "inet6" + interface: "eth0" + address: "fe80::3587:2fff:f11a:4321" routes: type: object properties: routes: - type: object - description: Routing table - properties: - ipv4: - type: array - description: Array of IPv4 routes - items: - type: object - properties: - destination: - type: string - description: Destination of the route - gateway: - type: string - description: Gateway of the route - metric: - type: integer - description: Metric of the route - flags: - type: array - description: Array of flags of the route - items: - type: string - interface: - type: string - description: Interface of the route - example: - - destination: "0.0.0.0" - gateway: "192.168.1.1" - metric: 0 - flags: [ "UP", "GATEWAY" ] - interface: "eth0" - - destination: "10.100.0.0" - gateway: "0.0.0.0" - metric: 0 - flags: [ "UP" ] - interface: "wg0" - ipv6: - type: array - description: Array of IPv6 routes - items: - type: object - properties: - destination: - type: object - description: Destination of the route - properties: - address: - type: string - description: IPv6 address - prefix: - type: integer - description: Prefix of the IPv6 address - type: - type: string - enum: [ "LL", "GUA", "ULA", "UNSPEC" ] - description: Type of the IPv6 address - source: - type: object - description: Source of the route - properties: - address: - type: string - description: IPv6 address - prefix: - type: integer - description: Prefix of the IPv6 address - type: - type: string - enum: [ "LL", "GUA", "ULA", "UNSPEC" ] - description: Type of the IPv6 address - gateway: - type: object - description: Gateway of the route - properties: - address: - type: string - description: IPv6 address - type: - type: string - enum: [ "LL", "GUA", "ULA", "UNSPEC" ] - description: Type of the IPv6 address - metric: - type: integer - description: Metric of the route - ref: - type: integer - description: Reference count of the route - use: - type: integer - description: Use count of the route - flags: - type: array - description: Array of flags of the route - items: - type: string - interface: - type: string - description: Interface of the route - example: - - destination: { "address": "2001:db8::", "prefix": 32, "type": "GUA" } - source: { "address": "2001:db8::1", "prefix": 128, "type": "GUA" } - gateway: { "address": "fe80::1", "type": "LL" } - metric: 0 - ref: 0 - use: 0 - flags: [ "UP", "GATEWAY" ] - interface: "eth0" - - destination: { "address": "::1", prefix: 128, "type": "UNSPEC" } - source: { "address": "::", prefix: 0, "type": "UNSPEC" } - gateway: { "address": "::", "type": "UNSPEC" } - metric: 256 - ref: 2 - use: 0 - flags: [ "UP" ] - interface: "lo" - - destination: { "address": "fd00:4711:0", prefix: 64, "type": "ULA" } - source: { "address": "::", prefix: 0, "type": "UNSPEC" } - gateway: { "address": "::", "type": "UNSPEC" } - metric: 256 - ref: 1 - use: 0 - flags: [ "UP" ] - interface: "wg0" + type: array + description: Array of routes + items: + type: object + properties: + gateway: + type: string + description: Gateway address + family: + type: string + enum: [ "inet", "inet6", "link", "mpls", "bridge", "???" ] + description: Address family + table: + type: main + description: Routing table ID (0 = unspecified, 253 = default, 254 = local, 255 = local) + protocol: + type: string + description: Routing protocol + scope: + type: string + description: Routing scope + type: + type: string + description: Routing type + flags: + type: array + description: Array of route flags + items: + type: string + oif: + type: string + description: Outgoing interface + iif: + type: string + description: Incoming interface + dst: + type: string + description: Destination address (or "default" for the default route) + src: + type: string + description: Source address + prefsrc: + type: string + description: Preferred source address + priority: + type: string + description: Route priority + + example: + - family: "inet" + table: 254 + protocol: "static" + scope: "universe" + type: "unicast" + flags: [] + gateway: "192.168.0.1" + oif: "eth0" + - family: "inet" + table: 254 + protocol: "boot" + scope: "link" + type: "unicast" + flags: [] + dst: "10.1.0.0" + oif: "wg0" + - family: "inet" + table: 255 + protocol: "kernel" + scope: "host" + type: "local" + flags: [] + dst: "127.0.0.1" + prefsrc: "127.0.0.1" + oif: "lo" + - family: "inet6" + table: 255 + protocol: "kernel" + scope: "universe" + type: "local" + flags: [] + dst: "::1" + priority: "medium" + oif: "eth0" + - family: "inet6" + table: 254 + protocol: "static" + scope: "universe" + type: "unicast" + flags: [] + gateway: "fe80::3587:2fff:f11a:4321" + oif: "eth0" + - family: "inet6" + table: 255 + protocol: "kernel" + scope: "universe" + type: "multicast" + flags: [] + dst: "fd00:4711::" + priority: "unknown" + oif: "wg0" + interfaces: type: object properties: @@ -307,116 +289,184 @@ components: properties: name: type: string - nullable: true description: Interface name - carrier: - type: boolean - description: If the interface is connected speed: type: integer - description: Speed of the interface in Mbit/s (-1 if not applicable) - tx: - type: object - properties: - num: - type: number - description: Number of transmitted data since boot - unit: - type: string - description: Unit of transmitted data since boot - rx: - type: object - properties: - num: - type: number - description: Number of received data since boot - unit: - type: string - description: Unit of received data since boot + nullable: true + description: Speed of the interface in Mbit/s (`null` if not applicable) + type: + type: string + description: Type of the interface flags: type: array - description: Array of interface flags + description: Array of address flags items: type: string - ipv4: + state: + type: string + description: State of the interface + proto_down: + type: boolean + description: Whether the interface is administratively down + address: + type: string + description: Interface hardware address + broadcase: + type: string + description: Interface broadcast address + perm_address: + type: string + description: Interface permanent hardware address + addresses: type: array nullable: true - description: Array of associated IPv4 addresses + description: Array of associated IPv addresses items: type: object properties: address: type: string - description: IPv4 address - netmask: + description: Interface address + local: type: string - description: Netmask of the IPv4 address - ipv6: - type: array - nullable: true - description: Array of associated IPv6 addresses - items: - type: object - properties: - address: + description: Local address + family: type: string - description: IPv6 address - netmask: - type: string - description: Netmask of the IPv4 address - type: - type: string - enum: [ "LL", "GUA", "ULA", "UNSPEC" ] - description: Type of the IPv6 address - prefix: - type: integer - description: Prefix of the IPv6 address - scope: - type: string - enum: [ "LINK", "GLOBAL", "HOST", "SITE", "COMPATv4", "UNKNOWN" ] - description: Scope of the IPv6 address + enum: [ "inet", "inet6", "link", "mpls", "bridge", "???" ] + description: Address family flags: type: array - description: Array of flags of the IPv6 address + description: Array of address flags items: type: string + prefixlen: + type: integer + description: Prefix length of the interface address + scope: + type: string + description: Address scope + prefered: + type: integer + description: Preferred lifetime of the address (`4294967295` = forever) + valid: + type: integer + description: Valid lifetime of the address (`4294967295` = forever) + cstamp: + type: number + description: Creation timestamp of the address (relative to the system uptime) + tstamp: + type: number + description: Updated timestamp of the address (relative to the system uptime) example: + - name: "lo" + speed: null + type: "loopback" + flags: [ "up", "loopback", "running", "lower_up" ] + state: "unknown" + carrier: true + address: "00:00:00:00:00:00" + broadcast: "00:00:00:00:00:00" + addresses: + - address: "127.0.0.1" + local: "127.0.0.1" + family: "inet" + scope: "host" + flags: [ "permanent" ] + prefixlen: 8 + label: "lo" + prefered: 4294967295 + valid: 4294967295 + cstamp: 6.1 + tstamp: 6.1 + - address: "::1" + local: "::1" + family: "inet6" + scope: "host" + flags: [ "permanent" ] + prefixlen: 128 + label: "lo" + prefered: 4294967295 + valid: 4294967295 + cstamp: 6.1 + tstamp: 6.1 - name: "eth0" - carrier: true speed: 1000 - tx: - num: 10.4 - unit: "MB" - rx: - num: 8.1 - unit: "MB" - flags: [ "UP", "BROADCAST", "RUNNING", "MULTICAST" ] - ipv4: [ { "address": "192.168.0.123", "netmask": "255.255.255.0" } ] - ipv6: [ { "address": "fe80::1234:5678:9abc:def0", "netmask": "ffff:ffff:ffff:ffff::", "type": "LL", "prefix": 64, "scope": "LINK", "flags": ["PERMANENT"] }, { "address": "2001:db8::1234:5678:9abc:def0", "netmask": "ffff:ffff:ffff:ffff::", "type": "GUA", "prefix": 64, "scope": "GLOBAL", "flags": [] } ] - - name: "wlan0" - carrier: false - speed: -1 - tx: - num: 0 - unit: "B" - rx: - num: 0 - unit: "B" - flags: [] - ipv4: [] - ipv6: [] - - name: "wg0" + type: "ether" + flags: [ "up", "broadcast", "running", "multicast", "lower_up" ] + state: "up" carrier: true - speed: -1 - tx: - num: 170.3 - unit: "kB" - rx: - num: 222.3 - unit: "kB" - flags: [ "UP", "POINTOPOINT", "RUNNING", "NOARP" ] - ipv4: [ { "address": "10.1.0.1", "netmask": "255.255.255.0" } ] - ipv6: [ { "address": "fd00:4711::1", "netmask": "ffff:ffff:ffff:ffff::", "type": "ULA", "prefix": 64, "scope": "GLOBAL", "flags": [ "PERMANENT" ] } ] + address: "00:11:22:33:44:55" + broadcast: "ff:ff:ff:ff:ff:ff" + perm_address: "00:11:22:33:44:55" + addresses: + - address: "192.168.0.123" + local: "192.168.0.123" + family: "inet" + scope: "universe" + flags: [ "permanent" ] + prefixlen: 24 + label: "eth0" + prefered: 4294967295 + valid: 4294967295 + cstamp: 11.23 + tstamp: 11.23 + - address: "2001:db8::1234:5678:9abc:def0" + family: "inet6" + scope: "universe" + flags: [] + prefixlen: 64 + label: "eth0" + prefered: 3461 + valid: 7061 + cstamp: 2789057.25 + tstamp: 2789057.25 + - address: "fd29:db8::1234:5678:9abc:def0" + family: "inet6" + scope: "universe" + flags: [] + prefixlen: 64 + label: "eth0" + prefered: 3461 + valid: 7061 + cstamp: 12.5 + tstamp: 2827298.75 + - address: "fe80::1234:5678:9abc:def0" + family: "inet6" + scope: "link" + flags: [ "permanent" ] + prefixlen: 64 + label: "eth0" + prefered: 4294967295 + valid: 4294967295 + cstamp: 11.23 + tstamp: 11.23 + - name: "wg0" + speed: null + type: "none" + flags: [ "up", "pointopoint", "running", "noarp", "lower_up" ] + state: "unknown" + carrier: true + addresses: + - address: "10.1.0.1" + local: "10.1.0.1" + scope: "universe" + flags: [ "permanent" ] + prefixlen: 24 + label: "wg0" + prefered: 4294967295 + valid: 4294967295 + cstamp: 11.23 + tstamp: 11.23 + - address: "fd00:4711::1" + family: "inet6" + scope: "global" + flags: [ "permanent" ] + prefixlen: 64 + label: "wg0" + prefered: 4294967295 + valid: 4294967295 + cstamp: 11.23 + tstamp: 11.23 devices: type: object properties: @@ -505,3 +555,11 @@ components: type: integer required: true example: 1 + detailed: + in: query + description: (Optional) Detailed interface information + name: detailed + schema: + type: boolean + required: false + example: false diff --git a/src/api/network.c b/src/api/network.c index d3d7832e..1deeeeaa 100644 --- a/src/api/network.c +++ b/src/api/network.c @@ -29,477 +29,58 @@ #include // IFA_LINK and friends #include - - -struct flag_names { - uint32_t flag; - const char *name; -}; - -static struct flag_names iff_flags[] = { - { IFF_UP, "UP" }, - { IFF_BROADCAST, "BROADCAST" }, - { IFF_DEBUG, "DEBUG" }, - { IFF_LOOPBACK, "LOOPBACK" }, - { IFF_POINTOPOINT, "POINTOPOINT" }, - { IFF_NOTRAILERS, "NOTRAILERS" }, - { IFF_RUNNING, "RUNNING" }, - { IFF_NOARP, "NOARP" }, - { IFF_PROMISC, "PROMISC" }, - { IFF_ALLMULTI, "ALLMULTI" }, - { IFF_MASTER, "MASTER" }, - { IFF_SLAVE, "SLAVE" }, - { IFF_MULTICAST, "MULTICAST" }, - { IFF_PORTSEL, "PORTSEL" }, - { IFF_AUTOMEDIA, "AUTOMEDIA" }, - { IFF_DYNAMIC, "DYNAMIC" }, -#ifdef IFF_LOWER_UP - { IFF_LOWER_UP, "LOWER_UP" }, -#endif -#ifdef IFF_DORMANT - { IFF_DORMANT, "DORMANT" }, -#endif -#ifdef IFF_ECHO - { IFF_ECHO, "ECHO" }, -#endif -}; - -static struct flag_names ifaf_flags[] = { - { IFA_F_TEMPORARY, "TEMPORARY" }, - { IFA_F_NODAD, "NODAD" }, - { IFA_F_OPTIMISTIC, "OPTIMISTIC" }, - { IFA_F_DADFAILED, "DADFAILED" }, - { IFA_F_HOMEADDRESS, "HOMEADDRESS" }, - { IFA_F_DEPRECATED, "DEPRECATED" }, - { IFA_F_TENTATIVE, "TENTATIVE" }, - { IFA_F_PERMANENT, "PERMANENT" }, - { IFA_F_MANAGETEMPADDR, "MANAGETEMPADDR" }, - { IFA_F_NOPREFIXROUTE, "NOPREFIXROUTE" }, - { IFA_F_MCAUTOJOIN, "MCAUTOJOIN" }, - { IFA_F_STABLE_PRIVACY, "STABLE_PRIVACY" }, -}; - -static struct flag_names ripv4[] = { - { RTF_UP, "UP" }, - { RTF_GATEWAY, "GATEWAY" }, - { RTF_HOST, "HOST" }, - { RTF_REINSTATE, "REINSTATE" }, - { RTF_DYNAMIC, "DYNAMIC" }, - { RTF_MODIFIED, "MODIFIED" }, - { RTF_MTU, "MTU" }, - { RTF_MSS, "MSS" }, - { RTF_WINDOW, "WINDOW" }, - { RTF_IRTT, "IRTT" }, - { RTF_REJECT, "REJECT" }, - { RTF_STATIC, "STATIC" }, - { RTF_XRESOLVE, "XRESOLVE" }, - { RTF_NOFORWARD, "NOFORWARD" }, - { RTF_THROW, "THROW" }, - { RTF_NOPMTUDISC, "NOPMTUDISC" }, -}; - -static struct flag_names ripv6[] = { - { RTF_DEFAULT, "DEFAULT" }, - { RTF_ALLONLINK, "ALLONLINK" }, - { RTF_ADDRCONF, "ADDRCONF" }, - { RTF_LINKRT, "LINKRT" }, - { RTF_NONEXTHOP, "NONEXTHOP" }, - { RTF_CACHE, "CACHE" }, - { RTF_FLOW, "FLOW" }, - { RTF_POLICY, "POLICY" }, - { RTF_LOCAL, "LOCAL" }, - { RTF_INTERFACE, "INTERFACE" }, - { RTF_MULTICAST, "MULTICAST" }, - { RTF_BROADCAST, "BROADCAST" }, - { RTF_NAT, "NAT" }, - { RTF_ADDRCLASSMASK, "ADDRCLASSMASK" }, -}; - -// Manually taken from kernel source code in include/net/ipv6.h -#define IFA_GLOBAL 0x0000U -#define IFA_HOST 0x0010U -#define IFA_LINK 0x0020U -#define IFA_SITE 0x0040U -#define IFA_COMPATv4 0x0080U - - -static struct flag_names scopes[] = { - { IFA_GLOBAL, "GLOBAL" }, - { IFA_HOST, "HOST" }, - { IFA_LINK, "LINK" }, - { IFA_SITE, "SITE" }, - { IFA_COMPATv4, "COMPATv4" }, -}; - -static bool ipv6_hex_to_human(const char oct[33], char human[INET6_ADDRSTRLEN], const char **addr_type) -{ - strncpy(human, oct, 32); - // Insert ":" into address string - for(size_t i = 1; i < 8; i++) - { - const size_t m = 4*i + i - 1; - memmove(&human[m + 1], &human[m], INET6_ADDRSTRLEN - m); - human[m] = ':'; - } - // Add trailing null byte - human[INET6_ADDRSTRLEN - 1] = '\0'; - - // Format address into most-compact form, e.g. - // "fe80:0000:0000:0000:0042:3dff:feb1:d93d" -> "fe80::42:3dff:feb1:d93d" - // If conversion fails, return false and keep the non-compact form - struct in6_addr addr6 = { 0 }; - if(inet_pton(AF_INET6, human, &addr6)) - { - if(addr_type != NULL) - { - // Extract first byte - // We do not directly access the underlying union as - // MUSL defines it differently than GNU C - uint8_t bytes[2]; - memcpy(&bytes, &addr6, 2); - // Global Unicast Address (2000::/3, RFC 4291) - if((bytes[0] & 0x70) == 0x20) - *addr_type = "GUA"; - // Unique Local Address (fc00::/7, RFC 4193) - if((bytes[0] & 0xfe) == 0xfc) - *addr_type = "ULA"; - // Link Local Address (fe80::/10, RFC 4291) - if((bytes[0] & 0xff) == 0xfe && (bytes[1] & 0x30) == 0) - *addr_type = "LL"; - } - - return inet_ntop(AF_INET6, &addr6, human, INET6_ADDRSTRLEN); - } - - return false; -} - -static bool read_proc_net_if_inet6(cJSON *addresses) -{ - // 4.1. if_inet6 - // - // Type: One line per address containing multiple values - // - // Here all configured IPv6 addresses are shown in a special format. The - // example displays for loopback interface only. The meaning is shown - // below (see "net/ipv6/addrconf.c" for more). - // - // # cat /proc/net/if_inet6 - // 00000000000000000000000000000001 01 80 10 80 lo - // +------------------------------+ ++ ++ ++ ++ ++ - // | | | | | | - // 1 2 3 4 5 6 - // - // 1. IPv6 address displayed in 32 hexadecimal chars without colons as separator - // 2. Netlink device number (interface index) in hexadecimal (see "ip addr" , too) - // 3. Prefix length in hexadecimal - // 4. Scope value (see kernel source " include/net/ipv6.h" and "net/ipv6/addrconf.c" for more) - // 5. Interface flags (see "include/linux/rtnetlink.h" and "net/ipv6/addrconf.c" for more) - // 6. Device name - - // Open /proc/net/if_inet6 - FILE *file; - if((file = fopen("/proc/net/if_inet6", "r"))) - { - // Parse /proc/net/if_inet6 - the kernel's IPv6 address table - char buf[1024] = { 0 }; - while(fgets(buf, sizeof(buf), file)) - { - char oct[33] = { 0 }; - unsigned int ifaceid = 0; - char iface[IF_NAMESIZE] = { 0 }; - unsigned int prefix = 0; - unsigned int scope = 0; - unsigned int flags = 0; - - // Parse address information - if(sscanf(buf, "%32s %x %x %x %x %15s", oct, &ifaceid, &prefix, &scope, &flags, iface) != 6) - continue; - - char addr_str[INET6_ADDRSTRLEN] = { 0 }; - const char *addr_type = "UNKNOWN"; - ipv6_hex_to_human(oct, addr_str, &addr_type); - - // Format flags into human-readable array of strings - cJSON *flag_array = cJSON_CreateArray(); - for(size_t i = 0; i < sizeof(ifaf_flags) / sizeof(ifaf_flags[0]); i++) - if(flags & ifaf_flags[i].flag) - cJSON_AddItemToArray(flag_array, cJSON_CreateStringReference(ifaf_flags[i].name)); - - // Create new address record - cJSON *address = cJSON_CreateObject(); - cJSON_AddStringToObject(address, "address", addr_str); - cJSON_AddItemReferenceToObject(address, "type", cJSON_CreateStringReference(addr_type)); - cJSON_AddStringToObject(address, "interface", iface); - cJSON_AddNumberToObject(address, "prefix", prefix); - const char *scope_str = "UNSPEC"; - for(size_t i = 0; i < sizeof(scopes) / sizeof(scopes[0]); i++) - if(scope == scopes[i].flag) - scope_str = scopes[i].name; - cJSON_AddItemToObject(address, "scope", cJSON_CreateStringReference(scope_str)); - cJSON_AddItemToObject(address, "flags", flag_array); - - // Add address to JSON array - cJSON_AddItemToArray(addresses, address); - } - - fclose(file); - } - else - { - log_err("Cannot read /proc/net/if_inet6: %s", strerror(errno)); - return false; - } - - return true; -} - -static bool read_proc_net_route(cJSON *routes) -{ - // Open /proc/net/route - FILE *file; - if((file = fopen("/proc/net/route", "r"))) - { - // Parse /proc/net/route - the kernel's IPv4 routing table - cJSON *ipv4 = cJSON_CreateArray(); - char buf[1024] = { 0 }; - while(fgets(buf, sizeof(buf), file)) - { - char iface[IF_NAMESIZE] = { 0 }; - unsigned long dest = 0, gw = 0; - unsigned int flags = 0; - int metric = 0; - - // Parse route information - if(sscanf(buf, "%15s %lx %lx %x %*i %*i %i", iface, &dest, &gw, &flags, &metric) != 5) - continue; - - cJSON *entry = cJSON_CreateObject(); - - // Format destination and gateway addresses - char dest_addr[INET_ADDRSTRLEN] = { 0 }; - char gw_addr[INET_ADDRSTRLEN] = { 0 }; - inet_ntop(AF_INET, &dest, dest_addr, sizeof(dest_addr)); - inet_ntop(AF_INET, &gw, gw_addr, sizeof(gw_addr)); - - // Format flags into human-readable array of strings - cJSON *flag_array = cJSON_CreateArray(); - for(size_t i = 0; i < sizeof(ripv4) / sizeof(ripv4[0]); i++) - if(flags & ripv4[i].flag) - cJSON_AddItemToArray(flag_array, cJSON_CreateStringReference(ripv4[i].name)); - - // Add route information to JSON object - cJSON_AddStringToObject(entry, "destination", dest_addr); - cJSON_AddStringToObject(entry, "gateway", gw_addr); - cJSON_AddNumberToObject(entry, "metric", metric); - cJSON_AddItemToObject(entry, "flags", flag_array); - cJSON_AddStringToObject(entry, "interface", iface); - - // Add route information to JSON array - cJSON_AddItemToArray(ipv4, entry); - } - - fclose(file); - - // Add IPv4 routes to JSON object - cJSON_AddItemToObject(routes, "ipv4", ipv4); - } - return true; -} - -static bool read_proc_net_ipv6_route(cJSON *routes) -{ - // Open /proc/net/route - FILE *file; - - // Open /proc/net/ipv6_route - if((file = fopen("/proc/net/ipv6_route", "r"))) - { - // Parse /proc/net/ipv6_route - the kernel's IPv6 routing table - // 4.2. ipv6_route - // - // Type: One line per route containing multiple values - // - // Here all configured IPv6 routes are shown in a special - // format. The example displays for loopback interface only. The - // meaning is shown below (see ”net/ipv6/route.c” for more). - // - // # cat /proc/net/ipv6_route - // 00000000000000000000000000000000 00 00000000000000000000000000000000 00 00000000000000000000000000000000 ffffffff 00000001 00000001 00200200 lo - // +------------------------------+ ++ +------------------------------+ ++ +------------------------------+ +------+ +------+ +------+ +------+ ++ - // | | | | | | | | | | - // 1 2 3 4 5 6 7 8 9 10 - // - // 1. IPv6 destination network displayed in 32 hexadecimal chars without colons as separator - // 2. IPv6 destination prefix length in hexadecimal - // 3. IPv6 source network displayed in 32 hexadecimal chars without colons as separator - // 4. IPv6 source prefix length in hexadecimal - // 5. IPv6 next hop displayed in 32 hexadecimal chars without colons as separator - // 6. Metric in hexadecimal - // 7. Reference counter - // 8. Use counter - // 9. Flags - // 10. Device name - - cJSON *ipv6 = cJSON_CreateArray(); - - char buf[1024] = { 0 }; - while(fgets(buf, sizeof(buf), file)) - { - char iface[IF_NAMESIZE] = { 0 }; - char dest[33] = { 0 }; - char src[33] = { 0 }; - char gw[33] = { 0 }; - unsigned int prefix_dest = 0; - unsigned int prefix_src = 0; - unsigned int metric = 0; - unsigned int ref = 0; - unsigned int use = 0; - unsigned int flags = 0; - - // Parse route information - if(sscanf(buf, "%32s %x %32s %x %32s %x %x %x %x %15s", - dest, &prefix_dest, src, &prefix_src, gw, &metric, &ref, &use, &flags, iface) != 10) - continue; - - // Format flags into human-readable array of strings - cJSON *flag_array = cJSON_CreateArray(); - for(size_t i = 0; i < sizeof(ripv4) / sizeof(ripv4[0]); i++) - if(flags & ripv4[i].flag) - cJSON_AddItemToArray(flag_array, cJSON_CreateStringReference(ripv4[i].name)); - for(size_t i = 0; i < sizeof(ripv6) / sizeof(ripv6[0]); i++) - if(flags & ripv6[i].flag) - cJSON_AddItemToArray(flag_array, cJSON_CreateStringReference(ripv6[i].name)); - - // Format destination, source, and gateway addresses - char dest_addr[INET6_ADDRSTRLEN] = { 0 }; - const char *dest_addr_type = "UNSPEC"; - char src_addr[INET6_ADDRSTRLEN] = { 0 }; - const char *src_addr_type = "UNSPEC"; - char gw_addr[INET6_ADDRSTRLEN] = { 0 }; - const char *gw_addr_type = "UNSPEC"; - ipv6_hex_to_human(dest, dest_addr, &dest_addr_type); - ipv6_hex_to_human(src, src_addr, &src_addr_type); - ipv6_hex_to_human(gw, gw_addr, &gw_addr_type); - - // Create new route record - cJSON *entry = cJSON_CreateObject(); - - cJSON *destination = cJSON_CreateObject(); - cJSON_AddStringToObject(destination, "address", dest_addr); - cJSON_AddNumberToObject(destination, "prefix", prefix_dest); - cJSON_AddItemToObject(destination, "type", cJSON_CreateStringReference(dest_addr_type)); - cJSON_AddItemToObject(entry, "destination", destination); - - cJSON *source = cJSON_CreateObject(); - cJSON_AddStringToObject(source, "address", src_addr); - cJSON_AddNumberToObject(source, "prefix", prefix_src); - cJSON_AddItemToObject(source, "type", cJSON_CreateStringReference(src_addr_type)); - cJSON_AddItemToObject(entry, "source", source); - - cJSON *gateway = cJSON_CreateObject(); - cJSON_AddStringToObject(gateway, "address", gw_addr); - cJSON_AddItemToObject(gateway, "type", cJSON_CreateStringReference(gw_addr_type)); - cJSON_AddItemToObject(entry, "gateway", gateway); - - cJSON_AddNumberToObject(entry, "metric", metric); - cJSON_AddNumberToObject(entry, "ref", ref); - cJSON_AddNumberToObject(entry, "use", use); - cJSON_AddItemToObject(entry, "flags", flag_array); - cJSON_AddStringToObject(entry, "interface", iface); - - // Add route information to JSON array - cJSON_AddItemToArray(ipv6, entry); - } - - // Add IPv6 routes to JSON object - cJSON_AddItemToObject(routes, "ipv6", ipv6); - - fclose(file); - } - - return true; -} +// nlroutes(), nladdrs(), nllinks() +#include "syscalls/netlink.h" int api_network_gateway(struct ftl_conn *api) { - cJSON *json = JSON_NEW_OBJECT(); - // Get JSON routes - cJSON *routes = cJSON_CreateObject(); - read_proc_net_route(routes); - read_proc_net_ipv6_route(routes); + // Add routing information + cJSON *routes = JSON_NEW_ARRAY(); + nlroutes(routes, false); + cJSON *gateway = JSON_NEW_ARRAY(); - // Search for route with GATEWAY flag set - cJSON *ipv4 = cJSON_GetObjectItem(routes, "ipv4"); - cJSON *r_ipv4 = cJSON_CreateObject(); + // Search through routes for the default gateway + // They are the ones with "dst" == "default" cJSON *route = NULL; - cJSON_ArrayForEach(route, ipv4) + cJSON_ArrayForEach(route, routes) { - cJSON *flags = cJSON_GetObjectItem(route, "flags"); - if(cJSON_IsArray(flags)) + cJSON *dst = cJSON_GetObjectItem(route, "dst"); + if(dst != NULL && cJSON_IsString(dst) && strcmp(cJSON_GetStringValue(dst), "default") == 0) { - cJSON *flag = NULL; - cJSON_ArrayForEach(flag, flags) - { - if(strcmp(cJSON_GetStringValue(flag), "GATEWAY") == 0) - { - // Extract interface name - const char *iface_name = cJSON_GetStringValue(cJSON_GetObjectItem(route, "interface")); - JSON_COPY_STR_TO_OBJECT(r_ipv4, "interface", iface_name); + cJSON *gwobj = JSON_NEW_OBJECT(); - // Extract gateway address - const char *gw_addr = cJSON_GetStringValue(cJSON_GetObjectItem(route, "gateway")); - JSON_COPY_STR_TO_OBJECT(r_ipv4, "address", gw_addr); + // Extract and add family + const int family = cJSON_GetNumberValue(cJSON_GetObjectItem(route, "family")); + JSON_ADD_NUMBER_TO_OBJECT(gwobj, "family", family); - break; - } - } + // Extract and add interface name + const char *iface_name = cJSON_GetStringValue(cJSON_GetObjectItem(route, "oif")); + JSON_COPY_STR_TO_OBJECT(gwobj, "interface", iface_name); + + // Extract and add gateway address + const char *gw_addr = cJSON_GetStringValue(cJSON_GetObjectItem(route, "gateway")); + JSON_COPY_STR_TO_OBJECT(gwobj, "address", gw_addr); + + cJSON_AddItemToArray(gateway, gwobj); } } - - // else: Search ipv6 routes - cJSON *ipv6 = cJSON_GetObjectItem(routes, "ipv6"); - cJSON *r_ipv6 = cJSON_CreateObject(); - cJSON_ArrayForEach(route, ipv6) - { - cJSON *flags = cJSON_GetObjectItem(route, "flags"); - if(cJSON_IsArray(flags)) - { - cJSON *flag = NULL; - cJSON_ArrayForEach(flag, flags) - { - if(strcmp(cJSON_GetStringValue(flag), "GATEWAY") == 0) - { - // Extract interface name - const char *iface_name = cJSON_GetStringValue(cJSON_GetObjectItem(route, "interface")); - JSON_COPY_STR_TO_OBJECT(r_ipv6, "interface", iface_name); - - // Extract gateway address - const char *gw_addr = cJSON_GetStringValue(cJSON_GetObjectItem(cJSON_GetObjectItem(route, "gateway"), "address")); - - JSON_COPY_STR_TO_OBJECT(r_ipv6, "address", gw_addr); - break; - } - } - } - } - - // Add gateway information to JSON object - JSON_ADD_ITEM_TO_OBJECT(json, "ipv4", r_ipv4); - JSON_ADD_ITEM_TO_OBJECT(json, "ipv6", r_ipv6); - cJSON_Delete(routes); + cJSON *json = JSON_NEW_OBJECT(); + JSON_ADD_ITEM_TO_OBJECT(json, "gateway", gateway); JSON_SEND_OBJECT(json); } int api_network_routes(struct ftl_conn *api) { + // Get ?detailed parameter + bool detailed = false; + get_bool_var(api->request->query_string, "detailed", &detailed); + // Add routing information - cJSON *routes = JSON_NEW_OBJECT(); - read_proc_net_route(routes); - read_proc_net_ipv6_route(routes); + cJSON *routes = JSON_NEW_ARRAY(); + nlroutes(routes, detailed); cJSON *json = JSON_NEW_OBJECT(); JSON_ADD_ITEM_TO_OBJECT(json, "routes", routes); JSON_SEND_OBJECT(json); @@ -507,266 +88,17 @@ int api_network_routes(struct ftl_conn *api) int api_network_interfaces(struct ftl_conn *api) { - cJSON *json = JSON_NEW_OBJECT(); - - // Enumerate and list interfaces - // Loop over interfaces and extract information - DIR *dfd; - FILE *f; - struct dirent *dp; - size_t tx_sum = 0, rx_sum = 0; - char fname[64 + IF_NAMESIZE] = { 0 }; - char readbuffer[1024] = { 0 }; - - // Open /sys/class/net directory - if ((dfd = opendir("/sys/class/net")) == NULL) - { - log_err("API: Cannot access /sys/class/net"); - return 500; - } - - // Get IP addresses of all interfaces on this machine - struct ifaddrs *ifap = NULL; - if(getifaddrs(&ifap) == -1) - log_err("API: Cannot get interface addresses: %s", strerror(errno)); - - // Parse IPv6 address details - cJSON *ipv6a = JSON_NEW_ARRAY(); - read_proc_net_if_inet6(ipv6a); + // Get ?detailed parameter + bool detailed = false; + get_bool_var(api->request->query_string, "detailed", &detailed); cJSON *interfaces = JSON_NEW_ARRAY(); - // Walk /sys/class/net directory - while ((dp = readdir(dfd)) != NULL) - { - // Skip "." and ".." - if(strcmp(dp->d_name, ".") == 0 || strcmp(dp->d_name, "..") == 0) - continue; + // Get links ... + nllinks(interfaces, detailed); + // ... and enrich them with addresses + nladdrs(interfaces, detailed); - // Create new interface record - cJSON *iface = JSON_NEW_OBJECT(); - - // Extract interface name - const char *iface_name = dp->d_name; - JSON_COPY_STR_TO_OBJECT(iface, "name", iface_name); - - // Extract carrier status - bool carrier = false; - snprintf(fname, sizeof(fname)-1, "/sys/class/net/%s/carrier", iface_name); - if((f = fopen(fname, "r")) != NULL) - { - if(fgets(readbuffer, sizeof(readbuffer)-1, f) != NULL) - carrier = readbuffer[0] == '1'; - fclose(f); - } - else - log_err("Cannot read %s: %s", fname, strerror(errno)); - JSON_ADD_BOOL_TO_OBJECT(iface, "carrier", carrier); - - // Extract link speed (may not be possible, e.g., for WiFi devices with dynamic link speeds) - int speed = -1; - snprintf(fname, sizeof(fname)-1, "/sys/class/net/%s/speed", iface_name); - if((f = fopen(fname, "r")) != NULL) - { - if(fscanf(f, "%i", &(speed)) != 1) - speed = -1; - fclose(f); - } - else - log_err("Cannot read %s: %s", fname, strerror(errno)); - JSON_ADD_NUMBER_TO_OBJECT(iface, "speed", speed); - - // Get total transmitted bytes - ssize_t tx_bytes = -1; - snprintf(fname, sizeof(fname)-1, "/sys/class/net/%s/statistics/tx_bytes", iface_name); - if((f = fopen(fname, "r")) != NULL) - { - if(fscanf(f, "%zi", &(tx_bytes)) != 1) - tx_bytes = -1; - fclose(f); - } - else - log_err("Cannot read %s: %s", fname, strerror(errno)); - - // Format transmitted bytes - double tx = 0.0; - char tx_unit[3] = { 0 }; - format_memory_size(tx_unit, tx_bytes, &tx); - if(tx_unit[0] != '\0') - tx_unit[1] = 'B'; - - // Add transmitted bytes to interface record - cJSON *tx_json = JSON_NEW_OBJECT(); - JSON_ADD_NUMBER_TO_OBJECT(tx_json, "num", tx); - JSON_COPY_STR_TO_OBJECT(tx_json, "unit", tx_unit); - JSON_ADD_ITEM_TO_OBJECT(iface, "tx", tx_json); - - // Get total received bytes - ssize_t rx_bytes = -1; - snprintf(fname, sizeof(fname)-1, "/sys/class/net/%s/statistics/rx_bytes", iface_name); - if((f = fopen(fname, "r")) != NULL) - { - if(fscanf(f, "%zi", &(rx_bytes)) != 1) - rx_bytes = -1; - fclose(f); - } - else - log_err("Cannot read %s: %s", fname, strerror(errno)); - - // Format received bytes - double rx = 0.0; - char rx_unit[3] = { 0 }; - format_memory_size(rx_unit, rx_bytes, &rx); - if(rx_unit[0] != '\0') - rx_unit[1] = 'B'; - - // Add received bytes to JSON object - cJSON *rx_json = JSON_NEW_OBJECT(); - JSON_ADD_NUMBER_TO_OBJECT(rx_json, "num", rx); - JSON_COPY_STR_TO_OBJECT(rx_json, "unit", rx_unit); - JSON_ADD_ITEM_TO_OBJECT(iface, "rx", rx_json); - - // Get IP address(es) of this interface - if(ifap) - { - // Walk through linked list of interface addresses - cJSON *ipv4 = JSON_NEW_ARRAY(); - cJSON *ipv6 = JSON_NEW_ARRAY(); - for(struct ifaddrs *ifa = ifap; ifa != NULL; ifa = ifa->ifa_next) - { - // Skip interfaces without an address and those - // not matching the current interface - if(ifa->ifa_addr == NULL || strcmp(ifa->ifa_name, iface_name) != 0) - continue; - - // If we reach this point, we found the correct interface - const sa_family_t family = ifa->ifa_addr->sa_family; - char host[NI_MAXHOST] = { 0 }; - if(family != AF_INET && family != AF_INET6) - continue; - // Get IP address - const int s = getnameinfo(ifa->ifa_addr, - (family == AF_INET) ? - sizeof(struct sockaddr_in) : - sizeof(struct sockaddr_in6), - host, NI_MAXHOST, - NULL, 0, NI_NUMERICHOST); - if (s != 0) - { - log_warn("API: getnameinfo(1) failed: %s\n", gai_strerror(s)); - continue; - } - // Get netmask - char netmask[NI_MAXHOST] = { 0 }; - const int s2 = getnameinfo(ifa->ifa_netmask, - (family == AF_INET) ? - sizeof(struct sockaddr_in) : - sizeof(struct sockaddr_in6), - netmask, NI_MAXHOST, - NULL, 0, NI_NUMERICHOST); - if (s2 != 0) - { - log_warn("API: getnameinfo(2) failed: %s\n", gai_strerror(s2)); - continue; - } - - cJSON *new_addr = JSON_NEW_OBJECT(); - JSON_COPY_STR_TO_OBJECT(new_addr, "address", host); - JSON_COPY_STR_TO_OBJECT(new_addr, "netmask", netmask); - - if(family == AF_INET) - { - // Add IPv4 address to array - JSON_ADD_ITEM_TO_ARRAY(ipv4, new_addr); - } - else if(family == AF_INET6) - { - // Search address in ipv6a array and add further details - cJSON *ipv6_entry = NULL; - cJSON_ArrayForEach(ipv6_entry, ipv6a) - { - // Compare interface and address - const char *arr_name = cJSON_GetStringValue(cJSON_GetObjectItem(ipv6_entry, "interface")); - const char *arr_addr = cJSON_GetStringValue(cJSON_GetObjectItem(ipv6_entry, "address")); - // We compare only the first part of the address as the second part may be an interface specifier (%veth...) - if(strcmp(arr_name, iface_name) == 0 && strncmp(arr_addr, host, min(strlen(arr_addr), strlen(host))) == 0) - { - // Copy details from ipv6a array to new_addr (prefix, scope, flags) - JSON_ADD_ITEM_TO_OBJECT(new_addr, "type", cJSON_Duplicate(cJSON_GetObjectItem(ipv6_entry, "type"), true)); - JSON_ADD_NUMBER_TO_OBJECT(new_addr, "prefix", cJSON_GetNumberValue(cJSON_GetObjectItem(ipv6_entry, "prefix"))); - JSON_ADD_ITEM_TO_OBJECT(new_addr, "scope", cJSON_Duplicate(cJSON_GetObjectItem(ipv6_entry, "scope"), true)); - JSON_ADD_ITEM_TO_OBJECT(new_addr, "flags", cJSON_Duplicate(cJSON_GetObjectItem(ipv6_entry, "flags"), true)); - break; - } - } - - // Add IPv6 address to array - JSON_ADD_ITEM_TO_ARRAY(ipv6, new_addr); - } - - // Format flags into human-readable array of strings - cJSON *flag_array = cJSON_CreateArray(); - for(size_t i = 0; i < sizeof(iff_flags) / sizeof(iff_flags[0]); i++) - if(ifa->ifa_flags & iff_flags[i].flag) - cJSON_AddItemToArray(flag_array, cJSON_CreateStringReference(iff_flags[i].name)); - JSON_ADD_ITEM_TO_OBJECT(iface, "flags", flag_array); - } - JSON_ADD_ITEM_TO_OBJECT(iface, "ipv4", ipv4); - JSON_ADD_ITEM_TO_OBJECT(iface, "ipv6", ipv6); - } - - // Sum up transmitted and received bytes - if(tx_bytes > 0) - tx_sum += tx_bytes; - if(rx_bytes > 0) - rx_sum += rx_bytes; - - // Add interface to array - JSON_ADD_ITEM_TO_ARRAY(interfaces, iface); - } - - freeifaddrs(ifap); - closedir(dfd); - cJSON_Delete(ipv6a); - ipv6a = NULL; - - cJSON *sum = JSON_NEW_OBJECT(); - JSON_COPY_STR_TO_OBJECT(sum, "name", "sum"); - JSON_ADD_BOOL_TO_OBJECT(sum, "carrier", true); - JSON_ADD_NUMBER_TO_OBJECT(sum, "speed", 0); - - // Format transmitted bytes - double tx = 0.0; - char tx_unit[3] = { 0 }; - format_memory_size(tx_unit, tx_sum, &tx); - if(tx_unit[0] != '\0') - tx_unit[1] = 'B'; - - // Add transmitted bytes to interface record - cJSON *tx_json = JSON_NEW_OBJECT(); - JSON_ADD_NUMBER_TO_OBJECT(tx_json, "num", tx); - JSON_COPY_STR_TO_OBJECT(tx_json, "unit", tx_unit); - JSON_ADD_ITEM_TO_OBJECT(sum, "tx", tx_json); - - // Format received bytes - double rx = 0.0; - char rx_unit[3] = { 0 }; - format_memory_size(rx_unit, rx_sum, &rx); - if(rx_unit[0] != '\0') - rx_unit[1] = 'B'; - - // Add received bytes to JSON object - cJSON *rx_json = JSON_NEW_OBJECT(); - JSON_ADD_NUMBER_TO_OBJECT(rx_json, "num", rx); - JSON_COPY_STR_TO_OBJECT(rx_json, "unit", rx_unit); - JSON_ADD_ITEM_TO_OBJECT(sum, "rx", rx_json); - - cJSON *ipv4 = JSON_NEW_ARRAY(); - cJSON *ipv6 = JSON_NEW_ARRAY(); - JSON_ADD_ITEM_TO_OBJECT(sum, "ipv4", ipv4); - JSON_ADD_ITEM_TO_OBJECT(sum, "ipv6", ipv6); - - // Add interface to array - JSON_ADD_ITEM_TO_ARRAY(interfaces, sum); + cJSON *json = JSON_NEW_OBJECT(); JSON_ADD_ITEM_TO_OBJECT(json, "interfaces", interfaces); JSON_SEND_OBJECT(json); } diff --git a/src/dnsmasq/network.c b/src/dnsmasq/network.c index 9e009f77..4d35478b 100644 --- a/src/dnsmasq/network.c +++ b/src/dnsmasq/network.c @@ -1861,3 +1861,46 @@ void newaddress(time_t now) relay->iface_index = 0; #endif } + + +static int callback_v4(struct in_addr local, int if_index, char *label, + struct in_addr netmask, struct in_addr broadcast, void *vparam) + { + log_info("callback_v4"); + // Log the interface information + log_info("Interface: %s", label); + log_info("IP Address: %s", inet_ntoa(local)); + log_info("Netmask: %s", inet_ntoa(netmask)); + log_info("Broadcast: %s", inet_ntoa(broadcast)); + log_info("Interface Index: %d", if_index); + return 1; + } + + +static int callback_v6(struct in6_addr *local, int prefix, + int scope, int if_index, int flags, + int preferred, int valid, void *vparam) + { + log_info("callback_v6"); + // Log the interface information + char ip[INET6_ADDRSTRLEN]; + inet_ntop(AF_INET6, local, ip, INET6_ADDRSTRLEN); + log_info("IP Address: %s", ip); + log_info("Prefix: %d", prefix); + log_info("Scope: %d", scope); + log_info("Interface Index: %d", if_index); + log_info("Flags: %d", flags); + log_info("Preferred: %d", preferred); + log_info("Valid: %d", valid); + return 1; + } + +extern int iface_enumerate(int family, void *parm, int (*callback)()); +void test_enumerate(void) +{ + log_info("test_enumerate 4"); + iface_enumerate(AF_INET, NULL, callback_v4); + log_info("test_enumerate 6"); + iface_enumerate(AF_INET6, NULL, callback_v6); + log_info("test_enumerate done"); +}; diff --git a/src/syscalls/CMakeLists.txt b/src/syscalls/CMakeLists.txt index 7ba43aa4..951b1bbe 100644 --- a/src/syscalls/CMakeLists.txt +++ b/src/syscalls/CMakeLists.txt @@ -13,6 +13,9 @@ set(sources asprintf.c calloc.c ftlallocate.c + netlink_consts.h + netlink.c + netlink.h fopen.c fprintf.c free.c diff --git a/src/syscalls/netlink.c b/src/syscalls/netlink.c new file mode 100644 index 00000000..e5597a1a --- /dev/null +++ b/src/syscalls/netlink.c @@ -0,0 +1,796 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2024 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* Network implementation for netlink +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ + +#include "FTL.h" +#include "netlink.h" +#include "netlink_consts.h" +#include "log.h" +#include +#include +#include +#include + +static bool nlrequest(int fd, struct sockaddr_nl *sa, int nlmsg_type) +{ + char buf[BUFLEN] = { 0 }; + // Assemble the message according to the netlink protocol + struct nlmsghdr *nl; + nl = (struct nlmsghdr*)(void*)buf; + nl->nlmsg_flags = NLM_F_REQUEST | NLM_F_ROOT; + + if(nlmsg_type == RTM_GETADDR) + { + // Request address information + nl->nlmsg_len = NLMSG_LENGTH(sizeof(struct ifaddrmsg)); + + struct ifaddrmsg *ifa; + ifa = (struct ifaddrmsg*)NLMSG_DATA(nl); + ifa->ifa_family = AF_LOCAL; + } + else if(nlmsg_type == RTM_GETROUTE) + { + // Request route information + nl->nlmsg_len = NLMSG_LENGTH(sizeof(struct rtmsg)); + + struct rtmsg *rt; + rt = (struct rtmsg*)NLMSG_DATA(nl); + rt->rtm_family = AF_LOCAL; + } + else if(nlmsg_type == RTM_GETLINK) + { + // Request link information + nl->nlmsg_len = NLMSG_LENGTH(sizeof(struct rtmsg)); + + struct ifinfomsg *link; + link = (struct ifinfomsg*)NLMSG_DATA(nl); + link->ifi_family = AF_UNSPEC; + } + nl->nlmsg_type = nlmsg_type; + + // Prepare struct msghdr for sending + struct iovec iov = { nl, nl->nlmsg_len }; + struct msghdr msg = { sa, sizeof(*sa), &iov, 1, NULL, 0, 0 }; + + // Send netlink message to kernel + return sendmsg(fd, &msg, 0) >= 0; +} + +static ssize_t nlgetmsg(int fd, struct sockaddr_nl *sa, void *buf, size_t len) +{ + struct iovec iov; + struct msghdr msg; + iov.iov_base = buf; + iov.iov_len = len; + + memset(&msg, 0, sizeof(msg)); + msg.msg_name = sa; + msg.msg_namelen = sizeof(*sa); + msg.msg_iov = &iov; + msg.msg_iovlen = 1; + + return recvmsg(fd, &msg, 0); +} + +static int nlparsemsg_route(struct rtmsg *rt, void *buf, size_t len, cJSON *routes, const bool detailed) +{ + char ifname[IF_NAMESIZE]; + cJSON *route = cJSON_CreateObject(); + cJSON_AddNumberToObject(route, "family", rt->rtm_family); + cJSON_AddNumberToObject(route, "table", rt->rtm_table); + + // Print human-readable protocol + for(unsigned int i = 0; i < sizeof(rtprots)/sizeof(rtprots[0]); i++) + if (rtprots[i].flag == rt->rtm_protocol) + { + cJSON_AddStringReferenceToObject(route, "protocol", rtprots[i].name); + break; + } + // If the protocol is not found, add it as a number + if (cJSON_GetObjectItem(route, "protocol") == NULL) { + cJSON_AddNumberToObject(route, "protocol", rt->rtm_protocol); + } + + // Print human-readable scope + for(unsigned int i = 0; i < sizeof(rtscopes)/sizeof(rtscopes[0]); i++) + if (rtscopes[i].flag == rt->rtm_scope) + { + cJSON_AddStringReferenceToObject(route, "scope", rtscopes[i].name); + break; + } + // If the scope is not found, add it as a number + if (cJSON_GetObjectItem(route, "scope") == NULL) + cJSON_AddNumberToObject(route, "scope", rt->rtm_scope); + + // Print human-readable type + for(unsigned int i = 0; i < sizeof(rttypes)/sizeof(rttypes[0]); i++) + if (rttypes[i].flag == rt->rtm_type) + { + cJSON_AddStringReferenceToObject(route, "type", rttypes[i].name); + break; + } + // If the type is not found, add it as a number + if (cJSON_GetObjectItem(route, "type") == NULL) + cJSON_AddNumberToObject(route, "type", rt->rtm_type); + + // Add array of human-readable flags + cJSON *flags = cJSON_CreateArray(); + for(unsigned int i = 0; i < sizeof(rtmflags)/sizeof(rtmflags[0]); i++) + if (rtmflags[i].flag & rt->rtm_flags) + cJSON_AddStringReferenceToArray(flags, rtmflags[i].name); + for(unsigned int i = 0; i < sizeof(rtnhflags)/sizeof(rtnhflags[0]); i++) + if (rtnhflags[i].flag & rt->rtm_flags) + cJSON_AddStringReferenceToArray(flags, rtnhflags[i].name); + cJSON_AddItemToObject(route, "flags", flags); + if(detailed) + cJSON_AddNumberToObject(route, "iflags", rt->rtm_flags); + + // Parse the route attributes + struct rtattr *rta = NULL; + static char ip[INET6_ADDRSTRLEN]; + for_each_rattr(rta, buf, len) + { + switch (rta->rta_type) + { + case RTA_DST: // route destination address + case RTA_SRC: // route source address + case RTA_GATEWAY: // gateway of the route + case RTA_PREFSRC: // preferred source address + case RTA_NEWDST: // change package destination address + inet_ntop(rt->rtm_family, RTA_DATA(rta), ip, INET6_ADDRSTRLEN); + cJSON_AddStringToObject(route, rtaTypeToString(rta->rta_type), ip); + break; + + case RTA_IIF: // incoming interface + case RTA_OIF: // outgoing interface + const uint32_t ifidx = *(uint32_t*)RTA_DATA(rta); + if_indextoname(ifidx, ifname); + cJSON_AddStringToObject(route, rtaTypeToString(rta->rta_type), ifname); + break; + + case RTA_FLOW: // route realm + case RTA_METRICS: // route metric + case RTA_TABLE: // routing table id + case RTA_MARK: // route mark + case RTA_EXPIRES: // route expires (in seconds) + case RTA_UID: // user id + case RTA_TTL_PROPAGATE: // propagate TTL + case RTA_IP_PROTO: // IP protocol + case RTA_SPORT: + case RTA_DPORT: + case RTA_NH_ID: + if(!detailed) + break; + const uint32_t number = *(uint32_t*)RTA_DATA(rta); + cJSON_AddNumberToObject(route, rtaTypeToString(rta->rta_type), number); + break; + + case RTA_PRIORITY: // royute priority + const uint32_t prio = *(uint32_t*)RTA_DATA(rta); + cJSON_AddNumberToObject(route, rtaTypeToString(rta->rta_type), prio); + break; + + case RTA_PREF: // route preference + const uint32_t pref = *(uint32_t*)RTA_DATA(rta); + cJSON_AddNumberToObject(route, rtaTypeToString(rta->rta_type), pref); + break; + + case RTA_MULTIPATH: // multipath route + if(!detailed) + break; + struct rtnexthop *rtnh = (struct rtnexthop *) RTA_DATA (rta); + cJSON *multipath = cJSON_CreateObject(); + cJSON_AddNumberToObject(multipath, "len", rtnh->rtnh_len); // Length of struct + length of RTAs + + // Add array of human-readable nexthop flags + cJSON *nhflags = cJSON_CreateArray(); + for(unsigned int i = 0; i < sizeof(rtnhflags)/sizeof(rtnhflags[0]); i++) + if (rtnhflags[i].flag & rtnh->rtnh_flags) + cJSON_AddStringReferenceToArray(nhflags, rtnhflags[i].name); + cJSON_AddItemToObject(route, "mflags", nhflags); + if(detailed) + cJSON_AddNumberToObject(route, "imflags", rtnh->rtnh_flags); + + cJSON_AddNumberToObject(multipath, "hops", rtnh->rtnh_hops); // Nexthop priority + if_indextoname(rtnh->rtnh_ifindex, ifname); + cJSON_AddStringToObject(multipath, "if", ifname); // Interface for this nexthop + cJSON_AddItemToObject(route, rtaTypeToString(rta->rta_type), multipath); + break; + + case RTA_VIA: // next hop address + struct rtvia *via = (struct rtvia*)RTA_DATA(rta); + inet_ntop(via->rtvia_family, &via->rtvia_addr, ip, INET6_ADDRSTRLEN); + cJSON_AddStringToObject(route, rtaTypeToString(rta->rta_type), ip); + break; + + case RTA_MFC_STATS: // multicast forwarding cache statistics + if(!detailed) + break; + struct rta_mfc_stats *mfc = (struct rta_mfc_stats*)RTA_DATA(rta); + cJSON_AddNumberToObject(route, "mfcs_packets", mfc->mfcs_packets); + cJSON_AddNumberToObject(route, "mfcs_bytes", mfc->mfcs_bytes); + cJSON_AddNumberToObject(route, "mfcs_wrong_if", mfc->mfcs_wrong_if); + break; + + case RTA_CACHEINFO: + if(!detailed) + break; + struct rta_cacheinfo *ci = (struct rta_cacheinfo*)RTA_DATA(rta); + cJSON_AddNumberToObject(route, "cstamp", ci->rta_clntref); + cJSON_AddNumberToObject(route, "tstamp", ci->rta_lastuse); + cJSON_AddNumberToObject(route, "expires", ci->rta_expires); + cJSON_AddNumberToObject(route, "error", ci->rta_error); + cJSON_AddNumberToObject(route, "used", ci->rta_used); + break; + + default: + // Unknown rta_type + // Add the rta_type as a number to an array of + // unknown types if in detailed mode + if(!detailed) + break; + + cJSON *unknown = cJSON_GetObjectItem(route, "unknown"); + if(unknown == NULL) + { + unknown = cJSON_CreateArray(); + cJSON_AddItemToObject(route, "unknown", unknown); + } + cJSON_AddNumberToArray(unknown, rta->rta_type); + break; + } + } + + // The default route is the one which does not have a "dst" attribute + if(cJSON_GetObjectItem(route, "dst") == NULL) + cJSON_AddStringToObject(route, "dst", "default"); + + cJSON_AddItemToArray(routes, route); + return 0; +} + +static int nlparsemsg_address(struct ifaddrmsg *ifa, void *buf, size_t len, cJSON *links, const bool detailed) +{ + cJSON *addr = cJSON_CreateObject(); + + // Add interface ID + if(detailed) + cJSON_AddNumberToObject(addr, "index", ifa->ifa_index); + + // Add family + cJSON_AddStringReferenceToObject(addr, "family", family_name(ifa->ifa_family)); + + // Print human-readable scope + for(unsigned int i = 0; i < sizeof(rtscopes)/sizeof(rtscopes[0]); i++) + if (rtscopes[i].flag == ifa->ifa_scope) + { + cJSON_AddStringReferenceToObject(addr, "scope", rtscopes[i].name); + break; + } + // If the scope is not found, add it as a number + if (cJSON_GetObjectItem(addr, "scope") == NULL) + cJSON_AddNumberToObject(addr, "scope", ifa->ifa_scope); + + // Add array of human-readable flags + cJSON *flags = cJSON_CreateArray(); + for(unsigned int i = 0; i < sizeof(ifaf_flags)/sizeof(ifaf_flags[0]); i++) + if (ifaf_flags[i].flag & ifa->ifa_flags) + cJSON_AddStringReferenceToArray(flags, ifaf_flags[i].name); + cJSON_AddItemToObject(addr, "flags", flags); + + // Add prefix length + cJSON_AddNumberToObject(addr, "prefixlen", ifa->ifa_prefixlen); + + // Parse the address attributes + struct rtattr *rta = NULL; + char ifname[IF_NAMESIZE] = { 0 }; + for_each_rattr(rta, buf, len){ + switch(rta->rta_type) + { + case IFA_ADDRESS: + case IFA_LOCAL: + case IFA_BROADCAST: + case IFA_ANYCAST: + char ip[INET6_ADDRSTRLEN] = { 0 }; + inet_ntop(ifa->ifa_family, RTA_DATA(rta), ip, INET6_ADDRSTRLEN); + cJSON_AddStringToObject(addr, ifaTypeToString(rta->rta_type), ip); + break; + + case IFA_LABEL: + strncpy(ifname, (char*)RTA_DATA(rta), IF_NAMESIZE); + cJSON_AddStringToObject(addr, ifaTypeToString(rta->rta_type), (char*)RTA_DATA(rta)); + break; + + case IFA_CACHEINFO: + struct ifa_cacheinfo *ci = (struct ifa_cacheinfo*)RTA_DATA(rta); + cJSON_AddNumberToObject(addr, "prefered", ci->ifa_prefered); + cJSON_AddNumberToObject(addr, "valid", ci->ifa_valid); + cJSON_AddNumberToObject(addr, "cstamp", 0.01*ci->cstamp); // created timestamp + cJSON_AddNumberToObject(addr, "tstamp", 0.01*ci->tstamp); // updated timestamp + break; + + case IFA_FLAGS: + cJSON *iflags = cJSON_CreateArray(); + for(unsigned int i = 0; i < sizeof(ifaf_flags)/sizeof(ifaf_flags[0]); i++) + if (ifaf_flags[i].flag & ifa->ifa_flags) + cJSON_AddStringReferenceToArray(iflags, ifaf_flags[i].name); + cJSON_AddItemToObject(addr, "flags", iflags); + break; + + case IFA_RT_PRIORITY: + if(!detailed) + break; + const uint32_t prio = *(uint32_t*)RTA_DATA(rta); + cJSON_AddStringToObject(addr, rtaTypeToString(rta->rta_type), rt_priority(prio)); + break; + + case IFA_TARGET_NETNSID: + if(!detailed) + break; + const uint32_t number = *(uint32_t*)RTA_DATA(rta); + cJSON_AddNumberToObject(addr, ifaTypeToString(rta->rta_type), number); + break; + + default: + // Unknown rta_type + // Add the rta_type as a number to an array of + // unknown types if in detailed mode + if(!detailed) + break; + + cJSON *unknown = cJSON_GetObjectItem(addr, "unknown"); + if(unknown == NULL) + { + unknown = cJSON_CreateArray(); + cJSON_AddItemToObject(addr, "unknown", unknown); + } + cJSON_AddNumberToArray(unknown, rta->rta_type); + break; + } + } + + // Get the interface name if it is not already set + if(!ifname[0]) + if_indextoname(ifa->ifa_index, ifname); + + // Return early if the interface is not in the list of known interfaces + cJSON *ifobj = cJSON_GetObjectItem(links, ifname); + if(ifobj == NULL) + { + cJSON_Delete(addr); + return 0; + } + + // Ensure there is an addresses object for the interface + if(cJSON_GetObjectItem(ifobj, "addresses") == NULL) + cJSON_AddItemToObject(ifobj, "addresses", cJSON_CreateArray()); + + // Get the addresses object + cJSON *addrsobj = cJSON_GetObjectItem(ifobj, "addresses"); + + // Add the address to the object + cJSON_AddItemToArray(addrsobj, addr); + return 0; +} + +static int nlparsemsg_link(struct ifinfomsg *ifi, void *buf, size_t len, cJSON *links, const bool detailed) +{ + cJSON *link = cJSON_CreateObject(); + + // Add ifname at the top of the JSON object + char ifname[IF_NAMESIZE] = { 0 }; + if_indextoname(ifi->ifi_index, ifname); + cJSON_AddStringToObject(link, "name", ifname); + + // Add interface ID and family if detailed + if(detailed) + { + cJSON_AddNumberToObject(link, "index", ifi->ifi_index); + cJSON_AddStringReferenceToObject(link, "family", family_name(ifi->ifi_family)); + } + + // Get link speed (not available through netlink) + // (may not be possible, e.g., for WiFi devices with dynamic link speeds) + int speed = -1; + char fname[64]; + snprintf(fname, sizeof(fname)-1, "/sys/class/net/%s/speed", ifname); + FILE *f = fopen(fname, "r"); + if(f != NULL) + { + if(fscanf(f, "%i", &(speed)) != 1) + speed = -1; + fclose(f); + } + if(speed > -1) + cJSON_AddNumberToObject(link, "speed", speed); + else + cJSON_AddNullToObject(link, "speed"); + + // Add human-readable type + for(unsigned int i = 0; i < sizeof(iflatypes)/sizeof(iflatypes[0]); i++) + if (iflatypes[i].flag == ifi->ifi_type) + { + cJSON_AddStringReferenceToObject(link, "type", iflatypes[i].name); + break; + } + + // Add interface flags + cJSON *flags = cJSON_CreateArray(); + for(unsigned int i = 0; i < sizeof(iff_flags)/sizeof(iff_flags[0]); i++) + if (iff_flags[i].flag & ifi->ifi_flags) + cJSON_AddStringReferenceToArray(flags, iff_flags[i].name); + cJSON_AddItemToObject(link, "flags", flags); + + // Parse the link attributes + struct rtattr *rta = NULL; + for_each_rattr(rta, buf, len){ + switch(rta->rta_type) + { + case IFLA_ADDRESS: + case IFLA_BROADCAST: + case IFLA_PERM_ADDRESS: + char mac[18]; + const unsigned char *addr = RTA_DATA(rta); + snprintf(mac, sizeof(mac), "%02x:%02x:%02x:%02x:%02x:%02x", + addr[0], addr[1], addr[2], addr[3], addr[4], addr[5]); + + // Addresses may be empty, so only add them if they are not + cJSON_AddStringToObject(link, iflaTypeToString(rta->rta_type), mac); + break; + + case IFLA_IFNAME: + // Only add it != ifname + if(strcmp(ifname, (char*)RTA_DATA(rta)) != 0) + cJSON_AddStringToObject(link, iflaTypeToString(rta->rta_type), (char*)RTA_DATA(rta)); + break; + case IFLA_ALT_IFNAME: + case IFLA_PHYS_PORT_NAME: + case IFLA_QDISC: + case IFLA_PARENT_DEV_NAME: + case IFLA_PARENT_DEV_BUS_NAME: + if(!detailed) + break; + cJSON_AddStringToObject(link, iflaTypeToString(rta->rta_type), (char*)RTA_DATA(rta)); + break; + + case IFLA_CARRIER: + case IFLA_PROTO_DOWN: + const uint8_t carrier = *(uint8_t*)RTA_DATA(rta); + cJSON_AddBoolToObject(link, iflaTypeToString(rta->rta_type), carrier == 0 ? false : true); + break; + + case IFLA_OPERSTATE: + for(unsigned int i = 0; i < sizeof(ifstates)/sizeof(ifstates[0]); i++) + if (ifstates[i].flag == *(unsigned int*)RTA_DATA(rta)) + { + cJSON_AddStringReferenceToObject(link, "state", ifstates[i].name); + break; + } + break; + + case IFLA_LINK: // Interface index + case IFLA_PHYS_PORT_ID: + case IFLA_PHYS_SWITCH_ID: + case IFLA_CARRIER_CHANGES: + case IFLA_MTU: + case IFLA_MASTER: + case IFLA_TXQLEN: + case IFLA_MAP: + case IFLA_WEIGHT: + case IFLA_LINKMODE: + case IFLA_COST: + case IFLA_PRIORITY: + case IFLA_GROUP: + case IFLA_NET_NS_PID: + case IFLA_NET_NS_FD: + case IFLA_EXT_MASK: + case IFLA_PROMISCUITY: + case IFLA_NUM_TX_QUEUES: + case IFLA_NUM_RX_QUEUES: + case IFLA_CARRIER_UP_COUNT: + case IFLA_CARRIER_DOWN_COUNT: + case IFLA_GSO_MAX_SEGS: + case IFLA_GSO_MAX_SIZE: + case IFLA_NEW_NETNSID: + case IFLA_MIN_MTU: + case IFLA_MAX_MTU: + if(!detailed) + break; + const uint32_t number = *(uint32_t*)RTA_DATA(rta); + cJSON_AddNumberToObject(link, iflaTypeToString(rta->rta_type), number); + break; + + case IFLA_STATS: + if(!detailed) + break; + struct rtnl_link_stats *stats = (struct rtnl_link_stats*)RTA_DATA(rta); + cJSON_AddNumberToObject(link, "rx_packets", stats->rx_packets); + cJSON_AddNumberToObject(link, "tx_packets", stats->tx_packets); + cJSON_AddNumberToObject(link, "rx_bytes", stats->rx_bytes); + cJSON_AddNumberToObject(link, "tx_bytes", stats->tx_bytes); + cJSON_AddNumberToObject(link, "rx_errors", stats->rx_errors); + cJSON_AddNumberToObject(link, "tx_errors", stats->tx_errors); + cJSON_AddNumberToObject(link, "rx_dropped", stats->rx_dropped); + cJSON_AddNumberToObject(link, "tx_dropped", stats->tx_dropped); + cJSON_AddNumberToObject(link, "multicast", stats->multicast); + cJSON_AddNumberToObject(link, "collisions", stats->collisions); + cJSON_AddNumberToObject(link, "rx_length_errors", stats->rx_length_errors); + cJSON_AddNumberToObject(link, "rx_over_errors", stats->rx_over_errors); + cJSON_AddNumberToObject(link, "rx_crc_errors", stats->rx_crc_errors); + cJSON_AddNumberToObject(link, "rx_frame_errors", stats->rx_frame_errors); + cJSON_AddNumberToObject(link, "rx_fifo_errors", stats->rx_fifo_errors); + cJSON_AddNumberToObject(link, "rx_missed_errors", stats->rx_missed_errors); + cJSON_AddNumberToObject(link, "tx_aborted_errors", stats->tx_aborted_errors); + cJSON_AddNumberToObject(link, "tx_carrier_errors", stats->tx_carrier_errors); + cJSON_AddNumberToObject(link, "tx_fifo_errors", stats->tx_fifo_errors); + cJSON_AddNumberToObject(link, "tx_heartbeat_errors", stats->tx_heartbeat_errors); + cJSON_AddNumberToObject(link, "tx_window_errors", stats->tx_window_errors); + cJSON_AddNumberToObject(link, "rx_compressed", stats->rx_compressed); + cJSON_AddNumberToObject(link, "tx_compressed", stats->tx_compressed); + break; + + case IFLA_STATS64: + if(!detailed) + break; + struct rtnl_link_stats64 *stats64 = (struct rtnl_link_stats64*)RTA_DATA(rta); + cJSON_AddNumberToObject(link, "rx_packets", stats64->rx_packets); + cJSON_AddNumberToObject(link, "tx_packets", stats64->tx_packets); + cJSON_AddNumberToObject(link, "rx_bytes", stats64->rx_bytes); + cJSON_AddNumberToObject(link, "tx_bytes", stats64->tx_bytes); + cJSON_AddNumberToObject(link, "rx_errors", stats64->rx_errors); + cJSON_AddNumberToObject(link, "tx_errors", stats64->tx_errors); + cJSON_AddNumberToObject(link, "rx_dropped", stats64->rx_dropped); + cJSON_AddNumberToObject(link, "tx_dropped", stats64->tx_dropped); + cJSON_AddNumberToObject(link, "multicast", stats64->multicast); + cJSON_AddNumberToObject(link, "collisions", stats64->collisions); + cJSON_AddNumberToObject(link, "rx_length_errors", stats64->rx_length_errors); + cJSON_AddNumberToObject(link, "rx_over_errors", stats64->rx_over_errors); + cJSON_AddNumberToObject(link, "rx_crc_errors", stats64->rx_crc_errors); + cJSON_AddNumberToObject(link, "rx_frame_errors", stats64->rx_frame_errors); + cJSON_AddNumberToObject(link, "rx_fifo_errors", stats64->rx_fifo_errors); + cJSON_AddNumberToObject(link, "rx_missed_errors", stats64->rx_missed_errors); + cJSON_AddNumberToObject(link, "tx_aborted_errors", stats64->tx_aborted_errors); + cJSON_AddNumberToObject(link, "tx_carrier_errors", stats64->tx_carrier_errors); + cJSON_AddNumberToObject(link, "tx_fifo_errors", stats64->tx_fifo_errors); + cJSON_AddNumberToObject(link, "tx_heartbeat_errors", stats64->tx_heartbeat_errors); + cJSON_AddNumberToObject(link, "tx_window_errors", stats64->tx_window_errors); + cJSON_AddNumberToObject(link, "rx_compressed", stats64->rx_compressed); + cJSON_AddNumberToObject(link, "tx_compressed", stats64->tx_compressed); + break; + + case IFLA_LINKINFO: + if(!detailed) + break; + struct rtattr *nlinkinfo = NULL; + size_t nlen = RTA_PAYLOAD(rta); + void *ndata = RTA_DATA(rta); + for_each_rattr(nlinkinfo, ndata, nlen){ + switch(nlinkinfo->rta_type) + { + case IFLA_INFO_KIND: + cJSON_AddStringToObject(link, "link_kind", (char*)RTA_DATA(nlinkinfo)); + break; + default: + // Unknown rta_type + cJSON *unknown = cJSON_GetObjectItem(link, "linkinfo_unknown"); + if(unknown == NULL) + { + unknown = cJSON_CreateArray(); + cJSON_AddItemToObject(link, "linkinfo_unknown", unknown); + } + cJSON_AddNumberToArray(unknown, nlinkinfo->rta_type); + break; + } + } + break; + + case IFLA_VFINFO_LIST: + if(!detailed) + break; + struct rtattr *vfinfo = RTA_DATA(rta); + if (vfinfo->rta_type != IFLA_VF_INFO) + break; + + struct ifla_vf_mac *vf_mac; + struct ifla_vf_broadcast *vf_broadcast; + struct ifla_vf_tx_rate *vf_tx_rate; + struct rtattr *vf[IFLA_VF_MAX + 1] = {}; + + parse_rtattr_nested(vf, IFLA_VF_MAX, vfinfo); + + vf_mac = RTA_DATA(vf[IFLA_VF_MAC]); + vf_broadcast = RTA_DATA(vf[IFLA_VF_BROADCAST]); + vf_tx_rate = RTA_DATA(vf[IFLA_VF_TX_RATE]); + + if (vf[IFLA_VF_BROADCAST]) + { + snprintf(mac, sizeof(mac), "%02x:%02x:%02x:%02x:%02x:%02x", + vf_broadcast->broadcast[0], vf_broadcast->broadcast[1], + vf_broadcast->broadcast[2], vf_broadcast->broadcast[3], + vf_broadcast->broadcast[4], vf_broadcast->broadcast[5]); + cJSON_AddStringToObject(link, "vf_broadcast", mac); + } + if(vf[IFLA_VF_MAC]) + { + snprintf(mac, sizeof(mac), "%02x:%02x:%02x:%02x:%02x:%02x", + vf_mac->mac[0], vf_mac->mac[1], vf_mac->mac[2], + vf_mac->mac[3], vf_mac->mac[4], vf_mac->mac[5]); + cJSON_AddStringToObject(link, "vf_mac", mac); + } + if(vf[IFLA_VF_TX_RATE]) + { + cJSON_AddNumberToObject(link, "vf_tx_rate", vf_tx_rate->rate); + } + if(vf[IFLA_VF_LINK_STATE]) + { + const uint32_t link_state = *(uint32_t*)RTA_DATA(vf[IFLA_VF_LINK_STATE]); + cJSON_AddNumberToObject(link, "vf_link_state", link_state); + } + + break; + + case IFLA_EVENT: + if(!detailed) + break; + const uint32_t event = *(uint32_t*)RTA_DATA(rta); + for(unsigned int i = 0; i < sizeof(link_events)/sizeof(link_events[0]); i++) + if (link_events[i].flag == event) + { + cJSON_AddStringReferenceToObject(link, "event", link_events[i].name); + break; + } + if(cJSON_GetObjectItem(link, "event") == NULL) + cJSON_AddNumberToObject(link, "event", event); + break; + + case IFLA_AF_SPEC: + if(!detailed) + break; + struct rtattr *af_spec = RTA_DATA(rta); + struct rtattr *inet6_attr = parse_rtattr_one_nested(AF_INET6, af_spec); + if(!inet6_attr) + break; + + struct rtattr *tb[IFLA_INET6_MAX + 1]; + parse_rtattr_nested(tb, IFLA_INET6_MAX, inet6_attr); + + if(tb[IFLA_INET6_ADDR_GEN_MODE]) + { + const uint8_t mode = *(uint8_t*)RTA_DATA(tb[IFLA_INET6_ADDR_GEN_MODE]); + for(unsigned int i = 0; i < sizeof(addr_gen_modes)/sizeof(addr_gen_modes[0]); i++) + if (addr_gen_modes[i].flag == mode) + { + cJSON_AddStringReferenceToObject(link, "addr_gen_mode", addr_gen_modes[i].name); + break; + } + if(cJSON_GetObjectItem(link, "addr_gen_mode") == NULL) + cJSON_AddNumberToObject(link, "addr_gen_mode", mode); + } + cJSON *af_specs = cJSON_CreateArray(); + for(unsigned int i = 0; i < __IFLA_INET6_MAX; i++) + if(tb[i]) + { + cJSON *jaf_spec = cJSON_CreateObject(); + cJSON_AddNumberToObject(jaf_spec, "type", i); + cJSON_AddNumberToObject(jaf_spec, "len", RTA_PAYLOAD(tb[i])); + cJSON_AddItemToArray(af_specs, jaf_spec); + } + cJSON_AddItemToObject(link, "af_specs", af_specs); + break; + + default: + // Unknown rta_type + // Add the rta_type as a number to an array of + // unknown types if in detailed mode + if(!detailed) + break; + + cJSON *unknown = cJSON_GetObjectItem(link, "unknown"); + if(unknown == NULL) + { + unknown = cJSON_CreateArray(); + cJSON_AddItemToObject(link, "unknown", unknown); + } + cJSON_AddNumberToArray(unknown, rta->rta_type); + break; + } + } + + // Add the link to the object + cJSON_AddItemToObject(links, ifname, link); + + return 0; +} + +static uint32_t parse_nl_msg(void *buf, size_t len, cJSON *json, const bool detailed) +{ + struct nlmsghdr *nl = NULL; + for_each_nlmsg(nl, buf, len) + { + if (nl->nlmsg_type == NLMSG_ERROR) + { + log_info("error"); + return -1; + } + else if (nl->nlmsg_type == RTM_NEWROUTE) + { + struct rtmsg *rt; + rt = (struct rtmsg*)NLMSG_DATA(nl); + nlparsemsg_route(rt, RTM_RTA(rt), RTM_PAYLOAD(nl), json, detailed); + continue; + } + else if (nl->nlmsg_type == RTM_NEWADDR) + { + struct ifaddrmsg *ifa; + ifa = (struct ifaddrmsg*)NLMSG_DATA(nl); + nlparsemsg_address(ifa, IFA_RTA(ifa), IFA_PAYLOAD(nl), json, detailed); + continue; + } + else if (nl->nlmsg_type == RTM_NEWLINK) + { + struct ifinfomsg *ifi; + ifi = (struct ifinfomsg*)NLMSG_DATA(nl); + nlparsemsg_link(ifi, IFLA_RTA(ifi), IFLA_PAYLOAD(nl), json, detailed); + continue; + } + else + { + log_err("unknown nlmsg_type: %d", nl->nlmsg_type); + } + + } + return nl->nlmsg_type; +} + +static int nlquery(const int type, cJSON *json, const bool detailed) +{ + // First of all, we need to create a socket with the AF_NETLINK domain + const int fd = socket(AF_NETLINK, SOCK_RAW, NETLINK_ROUTE); + if(fd < 0) + { + log_info("socket error: %s", strerror(errno)); + return -1; + } + + struct sockaddr_nl sa; + memset(&sa, 0, sizeof(sa)); + sa.nl_family = AF_NETLINK; + + ssize_t len = nlrequest(fd, &sa, type); + if(len < 0) + { + log_info("nlrequest error: %s", strerror(errno)); + return -1; + } + + char buf[BUFLEN]; + uint32_t nl_msg_type; + do { + len = nlgetmsg(fd, &sa, buf, BUFLEN); + nl_msg_type = parse_nl_msg(buf, len, json, detailed); + } while (nl_msg_type != NLMSG_DONE && nl_msg_type != NLMSG_ERROR); + + return 0; + +} + +bool nlroutes(cJSON *routes, const bool detailed) +{ + return nlquery(RTM_GETROUTE, routes, detailed); +} + +bool nladdrs(cJSON *interfaces, const bool detailed) +{ + return nlquery(RTM_GETADDR, interfaces, detailed); +} + +bool nllinks(cJSON *interfaces, const bool detailed) +{ + return nlquery(RTM_GETLINK, interfaces, detailed); +} diff --git a/src/syscalls/netlink.h b/src/syscalls/netlink.h new file mode 100644 index 00000000..1d94f241 --- /dev/null +++ b/src/syscalls/netlink.h @@ -0,0 +1,53 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2024 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* Netlink prototypes +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ +#ifndef NETLINK_H +#define NETLINK_H + +#include +#include "webserver/cJSON/cJSON.h" +#include "webserver/json_macros.h" + +// ICMPV6_PREF_LOW, etc. +#include +#include +// IFF_UP, etc. +#include +#include +#include +#include + +bool nlroutes(cJSON *routes, const bool detailed); +bool nladdrs(cJSON *interfaces, const bool detailed); +bool nllinks(cJSON *interfaces, const bool detailed); + + +#define BUFLEN 4096 + +#define for_each_nlmsg(n, buf, len) \ + for (n = (struct nlmsghdr*)buf; \ + NLMSG_OK(n, (uint32_t)len) && n->nlmsg_type != NLMSG_DONE; \ + n = NLMSG_NEXT(n, len)) + +#define for_each_rattr(n, buf, len) \ + for (n = (struct rtattr*)buf; RTA_OK(n, len); n = RTA_NEXT(n, len)) + +struct flag_names { + uint32_t flag; + const char *name; +}; + +// Manually taken from kernel source code in include/net/ipv6.h +#define IFA_GLOBAL 0x0000U +#define IFA_HOST 0x0010U +#define IFA_LINK 0x0020U +#define IFA_SITE 0x0040U +#define IFA_COMPATv4 0x0080U + +#endif // NETLINK_H diff --git a/src/syscalls/netlink_consts.h b/src/syscalls/netlink_consts.h new file mode 100644 index 00000000..867e3290 --- /dev/null +++ b/src/syscalls/netlink_consts.h @@ -0,0 +1,598 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2024 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* Netlink constants +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ + +#include "netlink.h" + +static struct flag_names iflatypes[] = +{ + { ARPHRD_NETROM, "netrom" }, + { ARPHRD_ETHER, "ether" }, + { ARPHRD_EETHER, "eether" }, + { ARPHRD_AX25, "ax25" }, + { ARPHRD_PRONET, "pronet" }, + { ARPHRD_CHAOS, "chaos" }, + { ARPHRD_IEEE802, "ieee802" }, + { ARPHRD_ARCNET, "arcnet" }, + { ARPHRD_APPLETLK, "appletlk" }, + { ARPHRD_DLCI, "dlci" }, + { ARPHRD_ATM, "atm" }, + { ARPHRD_METRICOM, "metricom" }, + { ARPHRD_IEEE1394, "ieee1394" }, + { ARPHRD_EUI64, "eui64" }, + { ARPHRD_INFINIBAND, "infiniband" }, + { ARPHRD_SLIP, "slip" }, + { ARPHRD_CSLIP, "cslip" }, + { ARPHRD_SLIP6, "slip6" }, + { ARPHRD_CSLIP6, "cslip6" }, + { ARPHRD_RSRVD, "rsrvd" }, + { ARPHRD_ADAPT, "adapt" }, + { ARPHRD_ROSE, "rose" }, + { ARPHRD_X25, "x25" }, + { ARPHRD_HWX25, "hwx25" }, + { ARPHRD_CAN, "can" }, + { ARPHRD_MCTP, "mctp" }, + { ARPHRD_PPP, "ppp" }, + { ARPHRD_CISCO, "cisco" }, + { ARPHRD_HDLC, "hdlc" }, + { ARPHRD_CISCO, "cisco" }, + { ARPHRD_LAPB, "lapb" }, + { ARPHRD_DDCMP, "ddcmp" }, + { ARPHRD_RAWHDLC, "rawhdlc" }, + { ARPHRD_RAWIP, "rawip" }, + { ARPHRD_TUNNEL, "tunnel" }, + { ARPHRD_TUNNEL6, "tunnel6" }, + { ARPHRD_FRAD, "frad" }, + { ARPHRD_SKIP, "skip" }, + { ARPHRD_LOOPBACK, "loopback" }, + { ARPHRD_LOCALTLK, "localtlk" }, + { ARPHRD_FDDI, "fddi" }, + { ARPHRD_BIF, "bif" }, + { ARPHRD_SIT, "sit" }, + { ARPHRD_IPDDP, "ipddp" }, + { ARPHRD_IPGRE, "ipgre" }, + { ARPHRD_PIMREG, "pimreg" }, + { ARPHRD_HIPPI, "hippi" }, + { ARPHRD_ASH, "ash" }, + { ARPHRD_ECONET, "econet" }, + { ARPHRD_IRDA, "irda" }, + { ARPHRD_FCPP, "fcpp" }, + { ARPHRD_FCAL, "fcal" }, + { ARPHRD_FCPL, "fcpl" }, + { ARPHRD_FCFABRIC, "fcfabric" }, + { ARPHRD_IEEE802_TR, "ieee802_tr" }, + { ARPHRD_IEEE80211, "ieee80211" }, + { ARPHRD_IEEE80211_PRISM, "ieee80211_prism" }, + { ARPHRD_IEEE80211_RADIOTAP, "ieee80211_radiotap" }, + { ARPHRD_IEEE802154, "ieee802154" }, + { ARPHRD_IEEE802154_MONITOR, "ieee802154_monitor" }, + { ARPHRD_PHONET, "phonet" }, + { ARPHRD_PHONET_PIPE, "phonet_pipe" }, + { ARPHRD_CAIF, "caif" }, + { ARPHRD_IP6GRE, "ip6gre" }, + { ARPHRD_NETLINK, "netlink" }, + { ARPHRD_6LOWPAN, "6lowpan" }, + { ARPHRD_VSOCKMON, "vsockmon" }, + { ARPHRD_VOID, "void" }, + { ARPHRD_NONE, "none" }, +}; + +static struct flag_names ifaf_flags[] = { + { IFA_F_SECONDARY, "secondary" }, + { IFA_F_TEMPORARY, "temporary" }, + { IFA_F_NODAD, "nodad" }, + { IFA_F_OPTIMISTIC, "optimistic" }, + { IFA_F_DADFAILED, "dadfailed" }, + { IFA_F_HOMEADDRESS, "homeaddress" }, + { IFA_F_DEPRECATED, "deprecated" }, + { IFA_F_TENTATIVE, "tentative" }, + { IFA_F_PERMANENT, "permanent" }, + { IFA_F_MANAGETEMPADDR, "managetempaddr" }, + { IFA_F_NOPREFIXROUTE, "noprefixroute" }, + { IFA_F_MCAUTOJOIN, "mcautojoin" }, + { IFA_F_STABLE_PRIVACY, "stable_privacy" }, +}; + +static struct flag_names iff_flags[] = { + { IFF_UP, "up" }, + { IFF_BROADCAST, "broadcast" }, + { IFF_DEBUG, "debug" }, + { IFF_LOOPBACK, "loopback" }, + { IFF_POINTOPOINT, "pointopoint" }, + { IFF_NOTRAILERS, "notrailers" }, + { IFF_RUNNING, "running" }, + { IFF_NOARP, "noarp" }, + { IFF_PROMISC, "promisc" }, + { IFF_ALLMULTI, "allmulti" }, + { IFF_MASTER, "master" }, + { IFF_SLAVE, "slave" }, + { IFF_MULTICAST, "multicast" }, + { IFF_PORTSEL, "portsel" }, + { IFF_AUTOMEDIA, "automedia" }, + { IFF_DYNAMIC, "dynamic" }, +#ifdef IFF_LOWER_UP + { IFF_LOWER_UP, "lower_up" }, +#endif +#ifdef IFF_DORMANT + { IFF_DORMANT, "dormant" }, +#endif +#ifdef IFF_ECHO + { IFF_ECHO, "echo" }, +#endif +}; + +static struct flag_names rtprots[] = { + { RTPROT_UNSPEC, "unspec" }, + { RTPROT_REDIRECT, "redirect" }, + { RTPROT_KERNEL, "kernel" }, + { RTPROT_BOOT, "boot" }, + { RTPROT_STATIC, "static" }, + { RTPROT_GATED, "gated" }, + { RTPROT_RA, "ra" }, + { RTPROT_MRT, "mrt" }, + { RTPROT_ZEBRA, "zebra" }, + { RTPROT_BIRD, "bird" }, + { RTPROT_DNROUTED, "dnrouted" }, + { RTPROT_XORP, "xorp" }, + { RTPROT_NTK, "ntk" }, + { RTPROT_DHCP, "dhcp" }, + { RTPROT_MROUTED, "mrouted" }, + { RTPROT_KEEPALIVED, "keepalived" }, + { RTPROT_BABEL, "babel" }, + { RTPROT_OPENR, "openr" }, + { RTPROT_BGP, "bgp" }, + { RTPROT_ISIS, "isis" }, + { RTPROT_OSPF, "ospf" }, + { RTPROT_RIP, "rip" }, + { RTPROT_EIGRP, "eigrp" }, +}; + +static struct flag_names rtscopes[] = { + { RT_SCOPE_UNIVERSE, "universe" }, + { RT_SCOPE_SITE, "site" }, + { RT_SCOPE_LINK, "link" }, + { RT_SCOPE_HOST, "host" }, + { RT_SCOPE_NOWHERE, "nowhere" }, +}; + +static struct flag_names rttypes[] = { + { RTN_UNSPEC, "unspec" }, + { RTN_UNICAST, "unicast" }, + { RTN_LOCAL, "local" }, + { RTN_BROADCAST, "broadcast" }, + { RTN_ANYCAST, "anycast" }, + { RTN_MULTICAST, "multicast" }, + { RTN_BLACKHOLE, "blackhole" }, + { RTN_UNREACHABLE, "unreachable" }, + { RTN_PROHIBIT, "prohibit" }, + { RTN_THROW, "throw" }, + { RTN_NAT, "nat" }, + { RTN_XRESOLVE, "xresolve" }, +}; + +static struct flag_names rtmflags[] = { + { RTM_F_NOTIFY, "notify" }, + { RTM_F_CLONED, "cloned" }, + { RTM_F_EQUALIZE, "equalize" }, + { RTM_F_PREFIX, "prefix" }, + { RTM_F_LOOKUP_TABLE, "lookup_table" }, + { RTM_F_FIB_MATCH, "fib_match" }, + { RTM_F_OFFLOAD, "offload" }, + { RTM_F_TRAP, "trap" }, + { RTM_F_OFFLOAD_FAILED, "offload_failed" }, +}; + +static struct flag_names rtnhflags[] = { + { RTNH_F_DEAD, "dead" }, + { RTNH_F_PERVASIVE, "pervasive" }, + { RTNH_F_ONLINK, "onlink" }, + { RTNH_F_OFFLOAD, "offload" }, + { RTNH_F_LINKDOWN, "linkdown" }, + { RTNH_F_UNRESOLVED, "unresolved" }, + { RTNH_F_TRAP, "trap" }, +}; + +static struct flag_names ifstates[] = { + { IF_OPER_UNKNOWN, "unknown" }, + { IF_OPER_NOTPRESENT, "notpresent" }, + { IF_OPER_DOWN, "down" }, + { IF_OPER_LOWERLAYERDOWN, "lower_layer_down" }, + { IF_OPER_TESTING, "testing" }, + { IF_OPER_DORMANT, "dormant" }, + { IF_OPER_UP, "up" }, +}; + +static struct flag_names link_events[] = { + { IFLA_EVENT_NONE, "none" }, + { IFLA_EVENT_REBOOT, "reboot" }, + { IFLA_EVENT_FEATURES, "feature change" }, + { IFLA_EVENT_BONDING_FAILOVER, "bonding failover" }, + { IFLA_EVENT_NOTIFY_PEERS, "notify peers" }, + { IFLA_EVENT_IGMP_RESEND, "resend igmp" }, + { IFLA_EVENT_BONDING_OPTIONS, "bonding option" }, +}; + +static struct flag_names addr_gen_modes[] = { + { IN6_ADDR_GEN_MODE_EUI64, "eui64" }, + { IN6_ADDR_GEN_MODE_NONE, "none" }, + { IN6_ADDR_GEN_MODE_STABLE_PRIVACY, "stable_secret" }, + { IN6_ADDR_GEN_MODE_RANDOM, "random" }, +}; + +static const char *__attribute__ ((const)) rtaTypeToString(const int rta_type) +{ + switch (rta_type) { + case RTA_UNSPEC: + return "unspec"; + case RTA_DST: + return "dst"; + case RTA_SRC: + return "src"; + case RTA_IIF: + return "iif"; + case RTA_OIF: + return "oif"; + case RTA_GATEWAY: + return "gateway"; + case RTA_PRIORITY: + return "priority"; + case RTA_PREFSRC: + return "prefsrc"; + case RTA_METRICS: + return "metrics"; + case RTA_MULTIPATH: + return "multipath"; + case RTA_PROTOINFO: + return "protoinfo"; + case RTA_FLOW: + return "flow"; + case RTA_CACHEINFO: + return "cacheinfo"; + case RTA_SESSION: + return "session"; + case RTA_MP_ALGO: + return "mp_algo"; + case RTA_TABLE: + return "table"; + case RTA_MARK: + return "mark"; + case RTA_MFC_STATS: + return "mfc_stats"; + case RTA_VIA: + return "via"; + case RTA_NEWDST: + return "newdst"; + case RTA_PREF: + return "pref"; + case RTA_ENCAP_TYPE: + return "encap_type"; + case RTA_ENCAP: + return "encap"; + case RTA_EXPIRES: + return "expires"; + case RTA_PAD: + return "pad"; + case RTA_UID: + return "uid"; + case RTA_TTL_PROPAGATE: + return "ttl_propagate"; + case RTA_IP_PROTO: + return "ip_proto"; + case RTA_SPORT: + return "sport"; + case RTA_DPORT: + return "dport"; + case RTA_NH_ID: + return "nh_id"; + default: + return "unknown"; + } +} + +static const char *__attribute__ ((const)) ifaTypeToString(const int ifa_type) +{ + switch (ifa_type) { + case IFA_ADDRESS: + return "address"; + case IFA_LOCAL: + return "local"; + case IFA_LABEL: + return "label"; + case IFA_BROADCAST: + return "broadcast"; + case IFA_ANYCAST: + return "anycast"; + case IFA_CACHEINFO: + return "cacheinfo"; + case IFA_MULTICAST: + return "multicast"; + case IFA_FLAGS: + return "flags"; + case IFA_RT_PRIORITY: + return "rt_priority"; + case IFA_TARGET_NETNSID: + return "target_netnsid"; + default: + return "unknown"; + } +} + +static const char *__attribute__ ((const)) iflaTypeToString(const int ifla_type) +{ + switch (ifla_type) + { + case IFLA_UNSPEC: + return "unspec"; + case IFLA_ADDRESS: + return "address"; + case IFLA_BROADCAST: + return "broadcast"; + case IFLA_IFNAME: + return "ifname"; + case IFLA_MTU: + return "mtu"; + case IFLA_LINK: + return "link"; + case IFLA_QDISC: + return "qdisc"; + case IFLA_STATS: + return "stats"; + case IFLA_COST: + return "cost"; + case IFLA_PRIORITY: + return "priority"; + case IFLA_MASTER: + return "master"; + case IFLA_WIRELESS: + return "wireless"; + case IFLA_PROTINFO: + return "protinfo"; + case IFLA_TXQLEN: + return "txqlen"; + case IFLA_MAP: + return "map"; + case IFLA_WEIGHT: + return "weight"; + case IFLA_OPERSTATE: + return "operstate"; + case IFLA_LINKMODE: + return "linkmode"; + case IFLA_LINKINFO: + return "linkinfo"; + case IFLA_NET_NS_FD: + return "net_ns_fd"; + case IFLA_IFALIAS: + return "ifalias"; + case IFLA_NUM_VF: + return "num_vf"; + case IFLA_VFINFO_LIST: + return "vfinfo_list"; + case IFLA_STATS64: + return "stats64"; + case IFLA_VF_PORTS: + return "vf_ports"; + case IFLA_PORT_SELF: + return "port_self"; + case IFLA_AF_SPEC: + return "af_spec"; + case IFLA_GROUP: + return "group"; + case IFLA_NET_NS_PID: + return "net_ns_pid"; + case IFLA_EXT_MASK: + return "ext_mask"; + case IFLA_PROMISCUITY: + return "promiscuity"; + case IFLA_NUM_TX_QUEUES: + return "num_tx_queues"; + case IFLA_NUM_RX_QUEUES: + return "num_rx_queues"; + case IFLA_CARRIER: + return "carrier"; + case IFLA_PHYS_PORT_ID: + return "phys_port_id"; + case IFLA_CARRIER_CHANGES: + return "carrier_changes"; + case IFLA_PHYS_SWITCH_ID: + return "phys_switch_id"; + case IFLA_LINK_NETNSID: + return "link_netnsid"; + case IFLA_PHYS_PORT_NAME: + return "phys_port_name"; + case IFLA_PROTO_DOWN: + return "proto_down"; + case IFLA_GSO_MAX_SEGS: + return "gso_max_segs"; + case IFLA_GSO_MAX_SIZE: + return "gso_max_size"; + case IFLA_PAD: + return "pad"; + case IFLA_XDP: + return "xdp"; + case IFLA_EVENT: + return "event"; + case IFLA_NEW_NETNSID: + return "new_netnsid"; + case IFLA_IF_NETNSID: + return "if_netnsid"; + case IFLA_CARRIER_UP_COUNT: + return "carrier_up_count"; + case IFLA_CARRIER_DOWN_COUNT: + return "carrier_down_count"; + case IFLA_NEW_IFINDEX: + return "new_ifindex"; + case IFLA_MIN_MTU: + return "min_mtu"; + case IFLA_MAX_MTU: + return "max_mtu"; + case IFLA_PROP_LIST: + return "prop_list"; + case IFLA_ALT_IFNAME: + return "alt_ifname"; + case IFLA_PERM_ADDRESS: + return "perm_address"; + case IFLA_PROTO_DOWN_REASON: + return "proto_down_reason"; + case IFLA_PARENT_DEV_NAME: + return "parent_dev_name"; + case IFLA_PARENT_DEV_BUS_NAME: + return "parent_dev_bus_name"; + default: + return "unknown"; + } +} + +static const char *__attribute__ ((const)) rt_priority(const uint32_t pref) +{ + switch (pref) { + case ICMPV6_ROUTER_PREF_HIGH: + return "high"; + case ICMPV6_ROUTER_PREF_MEDIUM: + return "medium"; + case ICMPV6_ROUTER_PREF_LOW: + return "low"; + case ICMPV6_ROUTER_PREF_INVALID: + return "invalid"; + default: + return "unknown"; + } +} + +static const char *__attribute__ ((const)) family_name(int family) +{ + switch(family) + { + case PF_UNSPEC: + return "unspec"; + case PF_LOCAL: + return "local"; + case PF_INET: + return "inet"; + case PF_AX25: + return "ax25"; + case PF_IPX: + return "ipx"; + case PF_APPLETALK: + return "appletalk"; + case PF_NETROM: + return "netrom"; + case PF_BRIDGE: + return "bridge"; + case PF_ATMPVC: + return "atmpvc"; + case PF_X25: + return "x25"; + case PF_INET6: + return "inet6"; + case PF_ROSE: + return "rose"; + case PF_DECnet: + return "decnet"; + case PF_NETBEUI: + return "netbeui"; + case PF_SECURITY: + return "security"; + case PF_KEY: + return "key"; + case PF_NETLINK: + return "netlink"; + case PF_PACKET: + return "packet"; + case PF_ASH: + return "ash"; + case PF_ECONET: + return "econet"; + case PF_ATMSVC: + return "atmsvc"; + case PF_RDS: + return "rds"; + case PF_SNA: + return "sna"; + case PF_IRDA: + return "irda"; + case PF_PPPOX: + return "pppox"; + case PF_WANPIPE: + return "wanpipe"; + case PF_LLC: + return "llc"; + case PF_IB: + return "ib"; + case PF_MPLS: + return "mpls"; + case PF_CAN: + return "can"; + case PF_TIPC: + return "tipc"; + case PF_BLUETOOTH: + return "bluetooth"; + case PF_IUCV: + return "iucv"; + case PF_RXRPC: + return "rxrpc"; + case PF_ISDN: + return "isdn"; + case PF_PHONET: + return "phonet"; + case PF_IEEE802154: + return "ieee802154"; + case PF_CAIF: + return "caif"; + case PF_ALG: + return "alg"; + case PF_NFC: + return "nfc"; + case PF_VSOCK: + return "vsock"; + case PF_KCM: + return "kcm"; + case PF_QIPCRTR: + return "qipcrtr"; + case PF_SMC: + return "smc"; + case PF_XDP: + return "xdp"; + case PF_MCTP: + return "mctp"; + default: + return "unknown"; + } +} + +// Taken from https://github.com/Gandi/packet-journey/blob/master/lib/libnetlink/netlink.c +#define parse_rtattr_nested(tb, max, rta) \ + (parse_rtattr_flags((tb), (max), RTA_DATA(rta), RTA_PAYLOAD(rta), 0)) +#define parse_rtattr_one_nested(type, rta) \ + (parse_rtattr_one(type, RTA_DATA(rta), RTA_PAYLOAD(rta))) + +static int parse_rtattr_flags(struct rtattr *tb[], int max, + struct rtattr *rta, int len, + unsigned short flags) +{ + unsigned short type; + + memset(tb, 0, sizeof(struct rtattr *) * (max + 1)); + while (RTA_OK(rta, len)) { + type = rta->rta_type & ~flags; + if ((type <= max) && (!tb[type])) + tb[type] = rta; + rta = RTA_NEXT(rta, len); + } + return 0; +} + +static struct rtattr * __attribute__((pure)) parse_rtattr_one(int type, struct rtattr *rta, int len) +{ + while (RTA_OK(rta, len)) { + if (rta->rta_type == type) + return rta; + rta = RTA_NEXT(rta, len); + } + return NULL; +} diff --git a/src/webserver/json_macros.h b/src/webserver/json_macros.h index 1ad9b09d..2644e075 100644 --- a/src/webserver/json_macros.h +++ b/src/webserver/json_macros.h @@ -260,3 +260,12 @@ cJSON *elem = cJSON_GetObjectItemCaseSensitive(obj, key); \ elem != NULL ? cJSON_IsTrue(elem) : false; \ }) + +#define cJSON_AddStringReferenceToObject(object, key, string) \ + cJSON_AddItemToObject(object, key, cJSON_CreateStringReference((const char*)(string))) + +#define cJSON_AddStringReferenceToArray(array, string) \ + cJSON_AddItemToArray(array, cJSON_CreateStringReference((const char*)(string))) + +#define cJSON_AddNumberToArray(array, num) \ + cJSON_AddItemToArray(array, cJSON_CreateNumber(num)) From 35166fd00c9018d3d2d9a31c2bf28a5b4bf59083 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 13 Jul 2024 08:23:43 +0200 Subject: [PATCH 215/339] Fix a small bug in the API response verifier and ensure we always favor using 64 bit interface statistics if available. The reason is that the legacy statistics use 32 bit conters which overflow every 4 GB of interface traffic Signed-off-by: DL6ER --- .devcontainer/devcontainer.json | 4 +- .github/.codespellignore | 2 + src/api/docs/content/specs/network.yaml | 68 ++++- src/api/network.c | 13 +- src/api/stats.c | 21 +- src/syscalls/netlink.c | 379 ++++++++++++++++++++---- src/syscalls/netlink_consts.h | 3 + test/api/libs/responseVerifyer.py | 1 - 8 files changed, 398 insertions(+), 93 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index bb8c5892..646a1a08 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -8,7 +8,9 @@ "jetmartin.bats", "ms-vscode.cpptools", "ms-vscode.cmake-tools", - "eamodio.gitlens" + "eamodio.gitlens", + "github.copilot", + "ms-python.python" ] } }, diff --git a/.github/.codespellignore b/.github/.codespellignore index 0dd61bef..5ca20bf8 100644 --- a/.github/.codespellignore +++ b/.github/.codespellignore @@ -9,3 +9,5 @@ punycode bitap mmapped dnsmasq +iif +prefered diff --git a/src/api/docs/content/specs/network.yaml b/src/api/docs/content/specs/network.yaml index 60a68f22..904fb063 100644 --- a/src/api/docs/content/specs/network.yaml +++ b/src/api/docs/content/specs/network.yaml @@ -190,8 +190,8 @@ components: enum: [ "inet", "inet6", "link", "mpls", "bridge", "???" ] description: Address family table: - type: main - description: Routing table ID (0 = unspecified, 253 = default, 254 = local, 255 = local) + type: integer + description: Routing table ID (0 = unspecified, 253 = default, 254 = local, 255 = local, other = user-defined) protocol: type: string description: Routing protocol @@ -222,8 +222,11 @@ components: type: string description: Preferred source address priority: - type: string + type: integer description: Route priority + pref: + type: integer + description: Route preference example: - family: "inet" @@ -258,7 +261,7 @@ components: type: "local" flags: [] dst: "::1" - priority: "medium" + priority: 0 oif: "eth0" - family: "inet6" table: 254 @@ -275,7 +278,7 @@ components: type: "multicast" flags: [] dst: "fd00:4711::" - priority: "unknown" + priority: 5 oif: "wg0" interfaces: @@ -294,6 +297,9 @@ components: type: integer nullable: true description: Speed of the interface in Mbit/s (`null` if not applicable) + carrier: + type: boolean + description: Whether the interface is connected type: type: string description: Type of the interface @@ -311,12 +317,35 @@ components: address: type: string description: Interface hardware address - broadcase: + broadcast: type: string description: Interface broadcast address perm_address: type: string description: Interface permanent hardware address + stats: + type: object + properties: + rx_bytes: + type: object + description: Interface received bytes + properties: + value: + type: number + description: Number of received bytes + unit: + type: string + description: Unit of the received bytes + tx_bytes: + type: object + description: Interface transmitted bytes + properties: + value: + type: number + description: Number of transmitted bytes + unit: + type: string + description: Unit of the transmitted bytes addresses: type: array nullable: true @@ -327,9 +356,15 @@ components: address: type: string description: Interface address + broadcast: + type: string + description: Interface broadcast address local: type: string description: Local address + label: + type: string + description: Interface label family: type: string enum: [ "inet", "inet6", "link", "mpls", "bridge", "???" ] @@ -366,6 +401,13 @@ components: carrier: true address: "00:00:00:00:00:00" broadcast: "00:00:00:00:00:00" + stats: + rx_bytes: + value: 81.6571641 + unit: "MB" + tx_bytes: + value: 648.818 + unit: "MB" addresses: - address: "127.0.0.1" local: "127.0.0.1" @@ -398,6 +440,13 @@ components: address: "00:11:22:33:44:55" broadcast: "ff:ff:ff:ff:ff:ff" perm_address: "00:11:22:33:44:55" + stats: + rx_bytes: + value: 15.5585 + unit: "GB" + tx_bytes: + value: 1.55858 + unit: "GB" addresses: - address: "192.168.0.123" local: "192.168.0.123" @@ -446,6 +495,13 @@ components: flags: [ "up", "pointopoint", "running", "noarp", "lower_up" ] state: "unknown" carrier: true + stats: + rx_bytes: + value: 458.44598 + unit: "MB" + tx_bytes: + value: 5.5895 + unit: "MB" addresses: - address: "10.1.0.1" local: "10.1.0.1" diff --git a/src/api/network.c b/src/api/network.c index 1deeeeaa..dce73292 100644 --- a/src/api/network.c +++ b/src/api/network.c @@ -38,21 +38,23 @@ int api_network_gateway(struct ftl_conn *api) // Add routing information cJSON *routes = JSON_NEW_ARRAY(); nlroutes(routes, false); - cJSON *gateway = JSON_NEW_ARRAY(); + cJSON *gateway = JSON_NEW_ARRAY(); // Search through routes for the default gateway // They are the ones with "dst" == "default" cJSON *route = NULL; cJSON_ArrayForEach(route, routes) { cJSON *dst = cJSON_GetObjectItem(route, "dst"); - if(dst != NULL && cJSON_IsString(dst) && strcmp(cJSON_GetStringValue(dst), "default") == 0) + if(dst != NULL && + cJSON_IsString(dst) && + strcmp(cJSON_GetStringValue(dst), "default") == 0) { cJSON *gwobj = JSON_NEW_OBJECT(); // Extract and add family - const int family = cJSON_GetNumberValue(cJSON_GetObjectItem(route, "family")); - JSON_ADD_NUMBER_TO_OBJECT(gwobj, "family", family); + const char *family = cJSON_GetStringValue(cJSON_GetObjectItem(route, "family")); + JSON_REF_STR_IN_OBJECT(gwobj, "family", family); // Extract and add interface name const char *iface_name = cJSON_GetStringValue(cJSON_GetObjectItem(route, "oif")); @@ -65,8 +67,11 @@ int api_network_gateway(struct ftl_conn *api) cJSON_AddItemToArray(gateway, gwobj); } } + + // Free routes array cJSON_Delete(routes); + // Send gateway information cJSON *json = JSON_NEW_OBJECT(); JSON_ADD_ITEM_TO_OBJECT(json, "gateway", gateway); JSON_SEND_OBJECT(json); diff --git a/src/api/stats.c b/src/api/stats.c index b272e768..1569fb6b 100644 --- a/src/api/stats.c +++ b/src/api/stats.c @@ -196,23 +196,6 @@ int api_stats_top_domains(struct ftl_conn *api) // Sort temporary array qsort(temparray, added_domains, sizeof(int[2]), cmpdesc); - // Get filter - const char* log_show = read_setupVarsconf("API_QUERY_LOG_SHOW"); - bool showpermitted = true, showblocked = true; - if(log_show != NULL) - { - if((strcmp(log_show, "permittedonly")) == 0) - showblocked = false; - else if((strcmp(log_show, "blockedonly")) == 0) - showpermitted = false; - else if((strcmp(log_show, "nothing")) == 0) - { - showpermitted = false; - showblocked = false; - } - } - clearSetupVarsArray(); - // Get domains which the user doesn't want to see regex_t *regex_domains = NULL; unsigned int N_regex_domains = 0; @@ -259,12 +242,12 @@ int api_stats_top_domains(struct ftl_conn *api) continue; int domain_count = -1; - if(blocked && showblocked && domain->blockedcount > 0) + if(blocked && domain->blockedcount > 0) { domain_count = domain->blockedcount; n++; } - else if(!blocked && showpermitted && (domain->count - domain->blockedcount) > 0) + else if(!blocked && (domain->count - domain->blockedcount) > 0) { domain_count = domain->count - domain->blockedcount; n++; diff --git a/src/syscalls/netlink.c b/src/syscalls/netlink.c index e5597a1a..d8e91c38 100644 --- a/src/syscalls/netlink.c +++ b/src/syscalls/netlink.c @@ -56,7 +56,11 @@ static bool nlrequest(int fd, struct sockaddr_nl *sa, int nlmsg_type) // Prepare struct msghdr for sending struct iovec iov = { nl, nl->nlmsg_len }; - struct msghdr msg = { sa, sizeof(*sa), &iov, 1, NULL, 0, 0 }; + struct msghdr msg = { 0 }; + msg.msg_name = sa; + msg.msg_namelen = sizeof(*sa); + msg.msg_iov = &iov; + msg.msg_iovlen = 1; // Send netlink message to kernel return sendmsg(fd, &msg, 0) >= 0; @@ -82,8 +86,8 @@ static int nlparsemsg_route(struct rtmsg *rt, void *buf, size_t len, cJSON *rout { char ifname[IF_NAMESIZE]; cJSON *route = cJSON_CreateObject(); - cJSON_AddNumberToObject(route, "family", rt->rtm_family); cJSON_AddNumberToObject(route, "table", rt->rtm_table); + cJSON_AddStringReferenceToObject(route, "family", family_name(rt->rtm_family)); // Print human-readable protocol for(unsigned int i = 0; i < sizeof(rtprots)/sizeof(rtprots[0]); i++) @@ -149,10 +153,12 @@ static int nlparsemsg_route(struct rtmsg *rt, void *buf, size_t len, cJSON *rout case RTA_IIF: // incoming interface case RTA_OIF: // outgoing interface + { const uint32_t ifidx = *(uint32_t*)RTA_DATA(rta); if_indextoname(ifidx, ifname); cJSON_AddStringToObject(route, rtaTypeToString(rta->rta_type), ifname); break; + } case RTA_FLOW: // route realm case RTA_METRICS: // route metric @@ -165,23 +171,24 @@ static int nlparsemsg_route(struct rtmsg *rt, void *buf, size_t len, cJSON *rout case RTA_SPORT: case RTA_DPORT: case RTA_NH_ID: + { if(!detailed) break; const uint32_t number = *(uint32_t*)RTA_DATA(rta); cJSON_AddNumberToObject(route, rtaTypeToString(rta->rta_type), number); break; + } - case RTA_PRIORITY: // royute priority - const uint32_t prio = *(uint32_t*)RTA_DATA(rta); - cJSON_AddNumberToObject(route, rtaTypeToString(rta->rta_type), prio); - break; - + case RTA_PRIORITY: // route priority case RTA_PREF: // route preference - const uint32_t pref = *(uint32_t*)RTA_DATA(rta); - cJSON_AddNumberToObject(route, rtaTypeToString(rta->rta_type), pref); + { + const uint32_t num = *(uint32_t*)RTA_DATA(rta); + cJSON_AddNumberToObject(route, rtaTypeToString(rta->rta_type), num); break; + } case RTA_MULTIPATH: // multipath route + { if(!detailed) break; struct rtnexthop *rtnh = (struct rtnexthop *) RTA_DATA (rta); @@ -202,14 +209,18 @@ static int nlparsemsg_route(struct rtmsg *rt, void *buf, size_t len, cJSON *rout cJSON_AddStringToObject(multipath, "if", ifname); // Interface for this nexthop cJSON_AddItemToObject(route, rtaTypeToString(rta->rta_type), multipath); break; + } case RTA_VIA: // next hop address + { struct rtvia *via = (struct rtvia*)RTA_DATA(rta); inet_ntop(via->rtvia_family, &via->rtvia_addr, ip, INET6_ADDRSTRLEN); cJSON_AddStringToObject(route, rtaTypeToString(rta->rta_type), ip); break; + } case RTA_MFC_STATS: // multicast forwarding cache statistics + { if(!detailed) break; struct rta_mfc_stats *mfc = (struct rta_mfc_stats*)RTA_DATA(rta); @@ -217,8 +228,10 @@ static int nlparsemsg_route(struct rtmsg *rt, void *buf, size_t len, cJSON *rout cJSON_AddNumberToObject(route, "mfcs_bytes", mfc->mfcs_bytes); cJSON_AddNumberToObject(route, "mfcs_wrong_if", mfc->mfcs_wrong_if); break; + } case RTA_CACHEINFO: + { if(!detailed) break; struct rta_cacheinfo *ci = (struct rta_cacheinfo*)RTA_DATA(rta); @@ -228,8 +241,10 @@ static int nlparsemsg_route(struct rtmsg *rt, void *buf, size_t len, cJSON *rout cJSON_AddNumberToObject(route, "error", ci->rta_error); cJSON_AddNumberToObject(route, "used", ci->rta_used); break; + } default: + { // Unknown rta_type // Add the rta_type as a number to an array of // unknown types if in detailed mode @@ -244,6 +259,7 @@ static int nlparsemsg_route(struct rtmsg *rt, void *buf, size_t len, cJSON *rout } cJSON_AddNumberToArray(unknown, rta->rta_type); break; + } } } @@ -297,10 +313,12 @@ static int nlparsemsg_address(struct ifaddrmsg *ifa, void *buf, size_t len, cJSO case IFA_LOCAL: case IFA_BROADCAST: case IFA_ANYCAST: + { char ip[INET6_ADDRSTRLEN] = { 0 }; inet_ntop(ifa->ifa_family, RTA_DATA(rta), ip, INET6_ADDRSTRLEN); cJSON_AddStringToObject(addr, ifaTypeToString(rta->rta_type), ip); break; + } case IFA_LABEL: strncpy(ifname, (char*)RTA_DATA(rta), IF_NAMESIZE); @@ -308,36 +326,45 @@ static int nlparsemsg_address(struct ifaddrmsg *ifa, void *buf, size_t len, cJSO break; case IFA_CACHEINFO: + { struct ifa_cacheinfo *ci = (struct ifa_cacheinfo*)RTA_DATA(rta); cJSON_AddNumberToObject(addr, "prefered", ci->ifa_prefered); cJSON_AddNumberToObject(addr, "valid", ci->ifa_valid); cJSON_AddNumberToObject(addr, "cstamp", 0.01*ci->cstamp); // created timestamp cJSON_AddNumberToObject(addr, "tstamp", 0.01*ci->tstamp); // updated timestamp break; + } case IFA_FLAGS: + { cJSON *iflags = cJSON_CreateArray(); for(unsigned int i = 0; i < sizeof(ifaf_flags)/sizeof(ifaf_flags[0]); i++) if (ifaf_flags[i].flag & ifa->ifa_flags) cJSON_AddStringReferenceToArray(iflags, ifaf_flags[i].name); cJSON_AddItemToObject(addr, "flags", iflags); break; + } case IFA_RT_PRIORITY: + { if(!detailed) break; const uint32_t prio = *(uint32_t*)RTA_DATA(rta); cJSON_AddStringToObject(addr, rtaTypeToString(rta->rta_type), rt_priority(prio)); break; + } case IFA_TARGET_NETNSID: + { if(!detailed) break; const uint32_t number = *(uint32_t*)RTA_DATA(rta); cJSON_AddNumberToObject(addr, ifaTypeToString(rta->rta_type), number); break; + } default: + { // Unknown rta_type // Add the rta_type as a number to an array of // unknown types if in detailed mode @@ -352,6 +379,7 @@ static int nlparsemsg_address(struct ifaddrmsg *ifa, void *buf, size_t len, cJSO } cJSON_AddNumberToArray(unknown, rta->rta_type); break; + } } } @@ -429,12 +457,14 @@ static int nlparsemsg_link(struct ifinfomsg *ifi, void *buf, size_t len, cJSON * // Parse the link attributes struct rtattr *rta = NULL; + cJSON *jstats = NULL, *jstats64 = NULL; for_each_rattr(rta, buf, len){ switch(rta->rta_type) { case IFLA_ADDRESS: case IFLA_BROADCAST: case IFLA_PERM_ADDRESS: + { char mac[18]; const unsigned char *addr = RTA_DATA(rta); snprintf(mac, sizeof(mac), "%02x:%02x:%02x:%02x:%02x:%02x", @@ -443,27 +473,29 @@ static int nlparsemsg_link(struct ifinfomsg *ifi, void *buf, size_t len, cJSON * // Addresses may be empty, so only add them if they are not cJSON_AddStringToObject(link, iflaTypeToString(rta->rta_type), mac); break; + } case IFLA_IFNAME: - // Only add it != ifname - if(strcmp(ifname, (char*)RTA_DATA(rta)) != 0) - cJSON_AddStringToObject(link, iflaTypeToString(rta->rta_type), (char*)RTA_DATA(rta)); - break; case IFLA_ALT_IFNAME: case IFLA_PHYS_PORT_NAME: case IFLA_QDISC: case IFLA_PARENT_DEV_NAME: case IFLA_PARENT_DEV_BUS_NAME: + { if(!detailed) break; - cJSON_AddStringToObject(link, iflaTypeToString(rta->rta_type), (char*)RTA_DATA(rta)); + const char *string = (char*)RTA_DATA(rta); + cJSON_AddStringToObject(link, iflaTypeToString(rta->rta_type), string); break; + } case IFLA_CARRIER: case IFLA_PROTO_DOWN: + { const uint8_t carrier = *(uint8_t*)RTA_DATA(rta); cJSON_AddBoolToObject(link, iflaTypeToString(rta->rta_type), carrier == 0 ? false : true); break; + } case IFLA_OPERSTATE: for(unsigned int i = 0; i < sizeof(ifstates)/sizeof(ifstates[0]); i++) @@ -500,71 +532,267 @@ static int nlparsemsg_link(struct ifinfomsg *ifi, void *buf, size_t len, cJSON * case IFLA_NEW_NETNSID: case IFLA_MIN_MTU: case IFLA_MAX_MTU: + { if(!detailed) break; const uint32_t number = *(uint32_t*)RTA_DATA(rta); cJSON_AddNumberToObject(link, iflaTypeToString(rta->rta_type), number); break; + } case IFLA_STATS: + { + // See description of the individual statistics + // below in the IFLA_STATS64 case + jstats = JSON_NEW_OBJECT(); + struct rtnl_link_stats *stats = (struct rtnl_link_stats*)RTA_DATA(rta); + { + // Warning: May be overflown if the interface has been up for a long time + // and has transferred a lot of data as 32 bits are used for the counters + // resulting in a maximum of 4 GiB. It is recommended to use the 64 bit + // counters if available. + char prefix[2] = { 0 }; + double formatted_size; + format_memory_size(prefix, stats->rx_bytes, &formatted_size); + cJSON *rx_bytes = cJSON_CreateObject(); + cJSON_AddNumberToObject(rx_bytes, "value", formatted_size); + cJSON_AddStringToObject(rx_bytes, "unit", prefix); + cJSON_AddItemToObject(jstats, "rx_bytes", rx_bytes); + } + { + // Warning: May be overflown if the interface has been up for a long time + // and has transferred a lot of data as 32 bits are used for the counters + // resulting in a maximum of 4 GiB. It is recommended to use the 64 bit + // counters if available. + char prefix[2] = { 0 }; + double formatted_size; + format_memory_size(prefix, stats->tx_bytes, &formatted_size); + cJSON *tx_bytes = cJSON_CreateObject(); + cJSON_AddNumberToObject(tx_bytes, "value", formatted_size); + cJSON_AddStringToObject(tx_bytes, "unit", prefix); + cJSON_AddItemToObject(jstats, "tx_bytes", tx_bytes); + } + cJSON_AddNumberToObject(jstats, "bits", 32); if(!detailed) break; - struct rtnl_link_stats *stats = (struct rtnl_link_stats*)RTA_DATA(rta); - cJSON_AddNumberToObject(link, "rx_packets", stats->rx_packets); - cJSON_AddNumberToObject(link, "tx_packets", stats->tx_packets); - cJSON_AddNumberToObject(link, "rx_bytes", stats->rx_bytes); - cJSON_AddNumberToObject(link, "tx_bytes", stats->tx_bytes); - cJSON_AddNumberToObject(link, "rx_errors", stats->rx_errors); - cJSON_AddNumberToObject(link, "tx_errors", stats->tx_errors); - cJSON_AddNumberToObject(link, "rx_dropped", stats->rx_dropped); - cJSON_AddNumberToObject(link, "tx_dropped", stats->tx_dropped); - cJSON_AddNumberToObject(link, "multicast", stats->multicast); - cJSON_AddNumberToObject(link, "collisions", stats->collisions); - cJSON_AddNumberToObject(link, "rx_length_errors", stats->rx_length_errors); - cJSON_AddNumberToObject(link, "rx_over_errors", stats->rx_over_errors); - cJSON_AddNumberToObject(link, "rx_crc_errors", stats->rx_crc_errors); - cJSON_AddNumberToObject(link, "rx_frame_errors", stats->rx_frame_errors); - cJSON_AddNumberToObject(link, "rx_fifo_errors", stats->rx_fifo_errors); - cJSON_AddNumberToObject(link, "rx_missed_errors", stats->rx_missed_errors); - cJSON_AddNumberToObject(link, "tx_aborted_errors", stats->tx_aborted_errors); - cJSON_AddNumberToObject(link, "tx_carrier_errors", stats->tx_carrier_errors); - cJSON_AddNumberToObject(link, "tx_fifo_errors", stats->tx_fifo_errors); - cJSON_AddNumberToObject(link, "tx_heartbeat_errors", stats->tx_heartbeat_errors); - cJSON_AddNumberToObject(link, "tx_window_errors", stats->tx_window_errors); - cJSON_AddNumberToObject(link, "rx_compressed", stats->rx_compressed); - cJSON_AddNumberToObject(link, "tx_compressed", stats->tx_compressed); + cJSON_AddNumberToObject(jstats, "rx_packets", stats->rx_packets); + cJSON_AddNumberToObject(jstats, "tx_packets", stats->tx_packets); + cJSON_AddNumberToObject(jstats, "rx_errors", stats->rx_errors); + cJSON_AddNumberToObject(jstats, "tx_errors", stats->tx_errors); + cJSON_AddNumberToObject(jstats, "rx_dropped", stats->rx_dropped); + cJSON_AddNumberToObject(jstats, "tx_dropped", stats->tx_dropped); + cJSON_AddNumberToObject(jstats, "multicast", stats->multicast); + cJSON_AddNumberToObject(jstats, "collisions", stats->collisions); + cJSON_AddNumberToObject(jstats, "rx_length_errors", stats->rx_length_errors); + cJSON_AddNumberToObject(jstats, "rx_over_errors", stats->rx_over_errors); + cJSON_AddNumberToObject(jstats, "rx_crc_errors", stats->rx_crc_errors); + cJSON_AddNumberToObject(jstats, "rx_frame_errors", stats->rx_frame_errors); + cJSON_AddNumberToObject(jstats, "rx_fifo_errors", stats->rx_fifo_errors); + cJSON_AddNumberToObject(jstats, "rx_missed_errors", stats->rx_missed_errors); + cJSON_AddNumberToObject(jstats, "tx_aborted_errors", stats->tx_aborted_errors); + cJSON_AddNumberToObject(jstats, "tx_carrier_errors", stats->tx_carrier_errors); + cJSON_AddNumberToObject(jstats, "tx_fifo_errors", stats->tx_fifo_errors); + cJSON_AddNumberToObject(jstats, "tx_heartbeat_errors", stats->tx_heartbeat_errors); + cJSON_AddNumberToObject(jstats, "tx_window_errors", stats->tx_window_errors); + cJSON_AddNumberToObject(jstats, "rx_compressed", stats->rx_compressed); + cJSON_AddNumberToObject(jstats, "tx_compressed", stats->tx_compressed); + cJSON_AddNumberToObject(jstats, "rx_nohandler", stats->rx_nohandler); break; + } case IFLA_STATS64: + { + jstats64 = JSON_NEW_OBJECT(); + struct rtnl_link_stats64 *stats64 = (struct rtnl_link_stats64*)RTA_DATA(rta); + { + char prefix[2] = { 0 }; + double formatted_size; + format_memory_size(prefix, stats64->rx_bytes, &formatted_size); + cJSON *rx_bytes = cJSON_CreateObject(); + cJSON_AddNumberToObject(rx_bytes, "value", formatted_size); + cJSON_AddStringToObject(rx_bytes, "unit", prefix); + // @rx_bytes: Number of good received + // bytes, corresponding to @rx_packets. + cJSON_AddItemToObject(jstats64, "rx_bytes", rx_bytes); + } + { + char prefix[2] = { 0 }; + double formatted_size; + format_memory_size(prefix, stats64->tx_bytes, &formatted_size); + cJSON *tx_bytes = cJSON_CreateObject(); + cJSON_AddNumberToObject(tx_bytes, "value", formatted_size); + cJSON_AddStringToObject(tx_bytes, "unit", prefix); + // @tx_bytes: Number of transmitted bytes, + // corresponding to @tx_packets. + cJSON_AddItemToObject(jstats64, "tx_bytes", tx_bytes); + } + cJSON_AddNumberToObject(jstats64, "bits", 64); if(!detailed) break; - struct rtnl_link_stats64 *stats64 = (struct rtnl_link_stats64*)RTA_DATA(rta); - cJSON_AddNumberToObject(link, "rx_packets", stats64->rx_packets); - cJSON_AddNumberToObject(link, "tx_packets", stats64->tx_packets); - cJSON_AddNumberToObject(link, "rx_bytes", stats64->rx_bytes); - cJSON_AddNumberToObject(link, "tx_bytes", stats64->tx_bytes); - cJSON_AddNumberToObject(link, "rx_errors", stats64->rx_errors); - cJSON_AddNumberToObject(link, "tx_errors", stats64->tx_errors); - cJSON_AddNumberToObject(link, "rx_dropped", stats64->rx_dropped); - cJSON_AddNumberToObject(link, "tx_dropped", stats64->tx_dropped); - cJSON_AddNumberToObject(link, "multicast", stats64->multicast); - cJSON_AddNumberToObject(link, "collisions", stats64->collisions); - cJSON_AddNumberToObject(link, "rx_length_errors", stats64->rx_length_errors); - cJSON_AddNumberToObject(link, "rx_over_errors", stats64->rx_over_errors); - cJSON_AddNumberToObject(link, "rx_crc_errors", stats64->rx_crc_errors); - cJSON_AddNumberToObject(link, "rx_frame_errors", stats64->rx_frame_errors); - cJSON_AddNumberToObject(link, "rx_fifo_errors", stats64->rx_fifo_errors); - cJSON_AddNumberToObject(link, "rx_missed_errors", stats64->rx_missed_errors); - cJSON_AddNumberToObject(link, "tx_aborted_errors", stats64->tx_aborted_errors); - cJSON_AddNumberToObject(link, "tx_carrier_errors", stats64->tx_carrier_errors); - cJSON_AddNumberToObject(link, "tx_fifo_errors", stats64->tx_fifo_errors); - cJSON_AddNumberToObject(link, "tx_heartbeat_errors", stats64->tx_heartbeat_errors); - cJSON_AddNumberToObject(link, "tx_window_errors", stats64->tx_window_errors); - cJSON_AddNumberToObject(link, "rx_compressed", stats64->rx_compressed); - cJSON_AddNumberToObject(link, "tx_compressed", stats64->tx_compressed); + // @rx_packets: Number of good packets received + // by the interface. For hardware interfaces + // counts all good packets received from the + // device by the host, including packets which + // host had to drop at various stages of + // processing (even in the driver). + cJSON_AddNumberToObject(jstats64, "rx_packets", stats64->rx_packets); + // @tx_packets: Number of packets successfully + // transmitted. For hardware interfaces counts + // packets which host was able to successfully + // hand over to the device, which does not + // necessarily mean that packets had been + // successfully transmitted out of the device, + // only that device acknowledged it copied them + // out of host memory. + cJSON_AddNumberToObject(jstats64, "tx_packets", stats64->tx_packets); + // @rx_errors: Total number of bad packets + // received on this network device. This counter + // must include events counted by + // @rx_length_errors, @rx_crc_errors, + // @rx_frame_errors and other errors not + // otherwise counted. + cJSON_AddNumberToObject(jstats64, "rx_errors", stats64->rx_errors); + // @tx_errors: Total number of transmit + // problems. This counter must include events + // counter by @tx_aborted_errors, + // @tx_carrier_errors, @tx_fifo_errors, + // @tx_heartbeat_errors, + // @tx_window_errors and other errors not + // otherwise counted. + cJSON_AddNumberToObject(jstats64, "tx_errors", stats64->tx_errors); + // @rx_dropped: Number of packets received but + // not processed, e.g. due to lack of resources + // or unsupported protocol. For hardware + // interfaces this counter may include packets + // discarded due to L2 address filtering but + // should not include packets dropped by the + // device due to buffer exhaustion which are + // counted separately in + // @rx_missed_errors (since procfs folds those + // two counters together). + cJSON_AddNumberToObject(jstats64, "rx_dropped", stats64->rx_dropped); + // @tx_dropped: Number of packets dropped on + // their way to transmission, e.g. due to lack + // of resources. + cJSON_AddNumberToObject(jstats64, "tx_dropped", stats64->tx_dropped); + // @multicast: Multicast packets received. For + // hardware interfaces this statistic is + // commonly calculated at the device level + // (unlike @rx_packets) and therefore may + // include packets which did not reach the host. + cJSON_AddNumberToObject(jstats64, "multicast", stats64->multicast); + // @collisions: Number of collisions during + // packet transmissions. + cJSON_AddNumberToObject(jstats64, "collisions", stats64->collisions); + // @rx_length_errors: Number of packets dropped + // due to invalid length. Part of aggregate + // "frame" errors in `/proc/net/dev`. + cJSON_AddNumberToObject(jstats64, "rx_length_errors", stats64->rx_length_errors); + // @rx_over_errors: Receiver FIFO overflow event + // counter. Historically the count of overflow + // events. Such events may be reported in the + // receive descriptors or via interrupts, and + // may not correspond one-to-one with dropped + // packets. + // + // The recommended interpretation for high speed + // interfaces is - number of packets dropped + // because they did not fit into buffers + // provided by the host, e.g. packets larger + // than MTU or next buffer in the ring was not + // available for a scatter transfer. + // + // Part of aggregate "frame" errors in `/proc/net/dev`. + // + // This statistics was historically used + // interchangeably with @rx_fifo_errors. + // + // This statistic corresponds to hardware events + // and is not commonly used on software devices. + cJSON_AddNumberToObject(jstats64, "rx_over_errors", stats64->rx_over_errors); + // @rx_crc_errors: Number of packets received + // with a CRC error. Part of aggregate "frame" + // errors in `/proc/net/dev`. + cJSON_AddNumberToObject(jstats64, "rx_crc_errors", stats64->rx_crc_errors); + // @rx_frame_errors: Receiver frame alignment + // errors. Part of aggregate "frame" errors in + // `/proc/net/dev`. + cJSON_AddNumberToObject(jstats64, "rx_frame_errors", stats64->rx_frame_errors); + // @rx_fifo_errors: Receiver FIFO error counter. + // + // Historically the count of overflow events. + // Those events may be reported in the receive + // descriptors or via interrupts, and may not + // correspond one-to-one with dropped packets. + // + // This statistics was used interchangeably with + // @rx_over_errors. Not recommended for use in + // drivers for high speed interfaces. + // + // This statistic is used on software devices, + // e.g. to count software packet queue overflow + // (can) or sequencing errors (GRE). + cJSON_AddNumberToObject(jstats64, "rx_fifo_errors", stats64->rx_fifo_errors); + // @rx_missed_errors: Count of packets missed by + // the host. Folded into the "drop" counter in + // `/proc/net/dev`. + // + // Counts number of packets dropped by the device due to lack + // of buffer space. This usually indicates that the host interface + // is slower than the network interface, or host is not keeping up + // with the receive packet rate. + // + // This statistic corresponds to hardware events and is not used + // on software devices. + cJSON_AddNumberToObject(jstats64, "rx_missed_errors", stats64->rx_missed_errors); + // @tx_aborted_errors: Part of aggregate + // "carrier" errors in `/proc/net/dev`. + cJSON_AddNumberToObject(jstats64, "tx_aborted_errors", stats64->tx_aborted_errors); + // @tx_carrier_errors: Number of frame + // transmission errors due to loss of carrier + // during transmission. Part of aggregate + // "carrier" errors in `/proc/net/dev`. + cJSON_AddNumberToObject(jstats64, "tx_carrier_errors", stats64->tx_carrier_errors); + // @tx_fifo_errors: Number of frame transmission + // errors due to device FIFO underrun / + // underflow. This condition occurs when the + // device begins transmission of a frame but is + // unable to deliver the entire frame to the + // transmitter in time for transmission. Part of + // aggregate "carrier" errors in + // `/proc/net/dev`. + cJSON_AddNumberToObject(jstats64, "tx_fifo_errors", stats64->tx_fifo_errors); + // @tx_heartbeat_errors: Number of Heartbeat / + // SQE Test errors for old half-duplex Ethernet. + // Part of aggregate "carrier" errors in + // `/proc/net/dev`. + cJSON_AddNumberToObject(jstats64, "tx_heartbeat_errors", stats64->tx_heartbeat_errors); + // @tx_window_errors: Number of frame + // transmission errors due to late collisions + // (for Ethernet - after the first 64B of + // transmission). Part of aggregate "carrier" + // errors in `/proc/net/dev`. + cJSON_AddNumberToObject(jstats64, "tx_window_errors", stats64->tx_window_errors); + // @rx_compressed: Number of received compressed + // packets. This counters is only meaningful for + // interfaces which support packet compression + // (e.g. CSLIP, PPP). + cJSON_AddNumberToObject(jstats64, "rx_compressed", stats64->rx_compressed); + // @tx_compressed: Number of transmitted + // compressed packets. This counters is only + // meaningful for interfaces which support + // packet compression (e.g. CSLIP, PPP). + cJSON_AddNumberToObject(jstats64, "tx_compressed", stats64->tx_compressed); + // @rx_nohandler: Number of packets received on + // the interface but dropped by the networking + // stack because the device is not designated to + // receive packets (e.g. backup link in a bond). + cJSON_AddNumberToObject(jstats64, "rx_nohandler", stats64->rx_nohandler); break; + } case IFLA_LINKINFO: + { if(!detailed) break; struct rtattr *nlinkinfo = NULL; @@ -577,6 +805,7 @@ static int nlparsemsg_link(struct ifinfomsg *ifi, void *buf, size_t len, cJSON * cJSON_AddStringToObject(link, "link_kind", (char*)RTA_DATA(nlinkinfo)); break; default: + { // Unknown rta_type cJSON *unknown = cJSON_GetObjectItem(link, "linkinfo_unknown"); if(unknown == NULL) @@ -586,11 +815,14 @@ static int nlparsemsg_link(struct ifinfomsg *ifi, void *buf, size_t len, cJSON * } cJSON_AddNumberToArray(unknown, nlinkinfo->rta_type); break; + } } } break; + } case IFLA_VFINFO_LIST: + { if(!detailed) break; struct rtattr *vfinfo = RTA_DATA(rta); @@ -610,6 +842,7 @@ static int nlparsemsg_link(struct ifinfomsg *ifi, void *buf, size_t len, cJSON * if (vf[IFLA_VF_BROADCAST]) { + char mac[18]; snprintf(mac, sizeof(mac), "%02x:%02x:%02x:%02x:%02x:%02x", vf_broadcast->broadcast[0], vf_broadcast->broadcast[1], vf_broadcast->broadcast[2], vf_broadcast->broadcast[3], @@ -618,6 +851,7 @@ static int nlparsemsg_link(struct ifinfomsg *ifi, void *buf, size_t len, cJSON * } if(vf[IFLA_VF_MAC]) { + char mac[18]; snprintf(mac, sizeof(mac), "%02x:%02x:%02x:%02x:%02x:%02x", vf_mac->mac[0], vf_mac->mac[1], vf_mac->mac[2], vf_mac->mac[3], vf_mac->mac[4], vf_mac->mac[5]); @@ -634,8 +868,10 @@ static int nlparsemsg_link(struct ifinfomsg *ifi, void *buf, size_t len, cJSON * } break; + } case IFLA_EVENT: + { if(!detailed) break; const uint32_t event = *(uint32_t*)RTA_DATA(rta); @@ -648,8 +884,10 @@ static int nlparsemsg_link(struct ifinfomsg *ifi, void *buf, size_t len, cJSON * if(cJSON_GetObjectItem(link, "event") == NULL) cJSON_AddNumberToObject(link, "event", event); break; + } case IFLA_AF_SPEC: + { if(!detailed) break; struct rtattr *af_spec = RTA_DATA(rta); @@ -683,8 +921,10 @@ static int nlparsemsg_link(struct ifinfomsg *ifi, void *buf, size_t len, cJSON * } cJSON_AddItemToObject(link, "af_specs", af_specs); break; + } default: + { // Unknown rta_type // Add the rta_type as a number to an array of // unknown types if in detailed mode @@ -699,9 +939,24 @@ static int nlparsemsg_link(struct ifinfomsg *ifi, void *buf, size_t len, cJSON * } cJSON_AddNumberToArray(unknown, rta->rta_type); break; + } } } + // Add 64 bit statistics if available and delete the 32 bit statistics + if(jstats64) + { + cJSON_AddItemToObject(link, "stats", jstats64); + if(jstats) + { + cJSON_Delete(jstats); + jstats = NULL; + } + } + // otherwise add the 32 bit statistics (64 has never been allocated) + else if(jstats) + cJSON_AddItemToObject(link, "stats", jstats); + // Add the link to the object cJSON_AddItemToObject(links, ifname, link); diff --git a/src/syscalls/netlink_consts.h b/src/syscalls/netlink_consts.h index 867e3290..f6a5e95f 100644 --- a/src/syscalls/netlink_consts.h +++ b/src/syscalls/netlink_consts.h @@ -558,8 +558,11 @@ static const char *__attribute__ ((const)) family_name(int family) return "smc"; case PF_XDP: return "xdp"; +#ifdef PF_MCTP + // 2024-July: defined by glibc but not musl case PF_MCTP: return "mctp"; +#endif default: return "unknown"; } diff --git a/test/api/libs/responseVerifyer.py b/test/api/libs/responseVerifyer.py index 238ca24c..27558907 100644 --- a/test/api/libs/responseVerifyer.py +++ b/test/api/libs/responseVerifyer.py @@ -292,7 +292,6 @@ class ResponseVerifyer(): # Check if the property is defined in the API specs (unless we know there are "any-key" items here) if props[-1] not in YAMLprops: self.errors.append("Property '" + flat_path + "' missing in the API specs (2)") - print(YAMLprop) return False YAMLprop = YAMLprops[props[-1]] From c284d6447e9288c5ad2327758f1634e1dd88ac82 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 14 Jul 2024 15:51:33 +0200 Subject: [PATCH 216/339] Move netlink under tools/ Signed-off-by: DL6ER --- src/api/network.c | 2 +- src/syscalls/CMakeLists.txt | 3 --- src/tools/CMakeLists.txt | 3 +++ src/{syscalls => tools}/netlink.c | 4 ++-- src/{syscalls => tools}/netlink.h | 0 src/{syscalls => tools}/netlink_consts.h | 0 6 files changed, 6 insertions(+), 6 deletions(-) rename src/{syscalls => tools}/netlink.c (99%) rename src/{syscalls => tools}/netlink.h (100%) rename src/{syscalls => tools}/netlink_consts.h (100%) diff --git a/src/api/network.c b/src/api/network.c index dce73292..31df4584 100644 --- a/src/api/network.c +++ b/src/api/network.c @@ -30,7 +30,7 @@ // IFA_LINK and friends #include // nlroutes(), nladdrs(), nllinks() -#include "syscalls/netlink.h" +#include "tools/netlink.h" int api_network_gateway(struct ftl_conn *api) { diff --git a/src/syscalls/CMakeLists.txt b/src/syscalls/CMakeLists.txt index 951b1bbe..7ba43aa4 100644 --- a/src/syscalls/CMakeLists.txt +++ b/src/syscalls/CMakeLists.txt @@ -13,9 +13,6 @@ set(sources asprintf.c calloc.c ftlallocate.c - netlink_consts.h - netlink.c - netlink.h fopen.c fprintf.c free.c diff --git a/src/tools/CMakeLists.txt b/src/tools/CMakeLists.txt index 47b9e5e1..ce5d4244 100644 --- a/src/tools/CMakeLists.txt +++ b/src/tools/CMakeLists.txt @@ -15,6 +15,9 @@ set(tools_sources dhcp-discover.h gravity-parseList.c gravity-parseList.h + netlink_consts.h + netlink.c + netlink.h ) add_library(tools OBJECT ${tools_sources}) diff --git a/src/syscalls/netlink.c b/src/tools/netlink.c similarity index 99% rename from src/syscalls/netlink.c rename to src/tools/netlink.c index d8e91c38..02d2cafe 100644 --- a/src/syscalls/netlink.c +++ b/src/tools/netlink.c @@ -685,8 +685,8 @@ static int nlparsemsg_link(struct ifinfomsg *ifi, void *buf, size_t len, cJSON * // packet transmissions. cJSON_AddNumberToObject(jstats64, "collisions", stats64->collisions); // @rx_length_errors: Number of packets dropped - // due to invalid length. Part of aggregate - // "frame" errors in `/proc/net/dev`. + // due to invalid length. Part of aggregate + // "frame" errors in `/proc/net/dev`. cJSON_AddNumberToObject(jstats64, "rx_length_errors", stats64->rx_length_errors); // @rx_over_errors: Receiver FIFO overflow event // counter. Historically the count of overflow diff --git a/src/syscalls/netlink.h b/src/tools/netlink.h similarity index 100% rename from src/syscalls/netlink.h rename to src/tools/netlink.h diff --git a/src/syscalls/netlink_consts.h b/src/tools/netlink_consts.h similarity index 100% rename from src/syscalls/netlink_consts.h rename to src/tools/netlink_consts.h From c342c60c71d89d179975f4fd63e30aa52c5cc15f Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 14 Jul 2024 15:54:19 +0200 Subject: [PATCH 217/339] fail*.dnssec.works changed its configuration from being BOGUS to ABANDONNED and cannot be used any longer for BOGUS testing Signed-off-by: DL6ER --- test/test_suite.bats | 6 ------ 1 file changed, 6 deletions(-) diff --git a/test/test_suite.bats b/test/test_suite.bats index d747e8dd..ed73f557 100644 --- a/test/test_suite.bats +++ b/test/test_suite.bats @@ -401,12 +401,6 @@ [[ ${lines[@]} == *"status: NOERROR"* ]] } -@test "DNSSEC: BOGUS domain is rejected" { - run bash -c "dig A fail01.dnssec.works @127.0.0.1" - printf "%s\n" "${lines[@]}" - [[ ${lines[@]} == *"status: SERVFAIL"* ]] -} - @test "Special domain: NXDOMAIN is returned" { run bash -c "dig A mask.icloud.com @127.0.0.1" printf "%s\n" "${lines[@]}" From 7c42a785cc6b99f89dc5545c391b98469419f413 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 14 Jul 2024 22:44:25 +0200 Subject: [PATCH 218/339] Report cstamp/tstamp in wall_clock time Signed-off-by: DL6ER --- src/api/docs/content/specs/network.yaml | 31 ++++++++++---- src/api/network.c | 57 ++++++++++++++++++++++--- src/tools/netlink.c | 20 +++++++-- 3 files changed, 90 insertions(+), 18 deletions(-) diff --git a/src/api/docs/content/specs/network.yaml b/src/api/docs/content/specs/network.yaml index 904fb063..62f0b66f 100644 --- a/src/api/docs/content/specs/network.yaml +++ b/src/api/docs/content/specs/network.yaml @@ -10,6 +10,10 @@ components: operationId: "get_gateway" description: | This API hook returns infos about the gateway of your Pi-hole. + + If the optional parameter `detailed` is set to `true`, the response will include detailed information about the individual interfaces and routes. Note that the available information is dependent on the interface type and state. + parameters: + - $ref: 'network.yaml#/components/parameters/devices/detailed' responses: '200': description: OK @@ -166,13 +170,22 @@ components: address: type: string description: Gateway address + local: + type: array + description: Local interface addresses + items: + type: string example: - family: "inet" interface: "eth0" - address: "192.168.0.2" + address: "192.168.0.1" + local: + - "192.168.0.22" - family: "inet6" interface: "eth0" - address: "fe80::3587:2fff:f11a:4321" + address: "fe80::3587:2fff:f11a:1" + local: + - "fe80::3587:2fff:f11a:4321" routes: type: object properties: @@ -388,10 +401,10 @@ components: description: Valid lifetime of the address (`4294967295` = forever) cstamp: type: number - description: Creation timestamp of the address (relative to the system uptime) + description: Creation timestamp of the address tstamp: type: number - description: Updated timestamp of the address (relative to the system uptime) + description: Updated timestamp of the address example: - name: "lo" speed: null @@ -418,8 +431,8 @@ components: label: "lo" prefered: 4294967295 valid: 4294967295 - cstamp: 6.1 - tstamp: 6.1 + cstamp: 1720989931 + tstamp: 1720989931 - address: "::1" local: "::1" family: "inet6" @@ -429,8 +442,8 @@ components: label: "lo" prefered: 4294967295 valid: 4294967295 - cstamp: 6.1 - tstamp: 6.1 + cstamp: 1720989931.1 + tstamp: 1720989931.1 - name: "eth0" speed: 1000 type: "ether" @@ -613,7 +626,7 @@ components: example: 1 detailed: in: query - description: (Optional) Detailed interface information + description: (Optional) Detailed interface/routing information name: detailed schema: type: boolean diff --git a/src/api/network.c b/src/api/network.c index 31df4584..6ff36a62 100644 --- a/src/api/network.c +++ b/src/api/network.c @@ -34,10 +34,19 @@ int api_network_gateway(struct ftl_conn *api) { + // Get ?detailed parameter + bool detailed = false; + get_bool_var(api->request->query_string, "detailed", &detailed); - // Add routing information + // Get routing information cJSON *routes = JSON_NEW_ARRAY(); - nlroutes(routes, false); + nlroutes(routes, detailed); + + // Get interface information ... + cJSON *interfaces = JSON_NEW_ARRAY(); + nllinks(interfaces, detailed); + // ... and enrich them with addresses + nladdrs(interfaces, detailed); cJSON *gateway = JSON_NEW_ARRAY(); // Search through routes for the default gateway @@ -64,16 +73,54 @@ int api_network_gateway(struct ftl_conn *api) const char *gw_addr = cJSON_GetStringValue(cJSON_GetObjectItem(route, "gateway")); JSON_COPY_STR_TO_OBJECT(gwobj, "address", gw_addr); + // Extract and add local interface address + cJSON *local = JSON_NEW_ARRAY(); + cJSON *iface = NULL; + cJSON_ArrayForEach(iface, interfaces) + { + const char *ifname = cJSON_GetStringValue(cJSON_GetObjectItem(iface, "name")); + if(ifname != NULL && strcmp(ifname, iface_name) == 0) + { + cJSON *addr = NULL; + cJSON *addrs = cJSON_GetObjectItem(iface, "addresses"); + cJSON_ArrayForEach(addr, addrs) + { + // Skip addresses belonging to another address family + const char *ifamily = cJSON_GetStringValue(cJSON_GetObjectItem(addr, "family")); + if(ifamily == NULL || strcmp(ifamily, family) != 0) + continue; + + const char *addr_str = cJSON_GetStringValue(cJSON_GetObjectItem(addr, "address")); + if(addr_str != NULL) + JSON_COPY_STR_TO_ARRAY(local, addr_str); + } + break; + } + } + + // Add local addresses array to gateway object + JSON_ADD_ITEM_TO_OBJECT(gwobj, "local", local); + cJSON_AddItemToArray(gateway, gwobj); } } - // Free routes array - cJSON_Delete(routes); - // Send gateway information cJSON *json = JSON_NEW_OBJECT(); JSON_ADD_ITEM_TO_OBJECT(json, "gateway", gateway); + + if(detailed) + { + JSON_ADD_ITEM_TO_OBJECT(json, "routes", routes); + JSON_ADD_ITEM_TO_OBJECT(json, "interfaces", interfaces); + } + else + { + // Free arrays + cJSON_Delete(routes); + cJSON_Delete(interfaces); + } + JSON_SEND_OBJECT(json); } diff --git a/src/tools/netlink.c b/src/tools/netlink.c index 02d2cafe..57813984 100644 --- a/src/tools/netlink.c +++ b/src/tools/netlink.c @@ -235,8 +235,14 @@ static int nlparsemsg_route(struct rtmsg *rt, void *buf, size_t len, cJSON *rout if(!detailed) break; struct rta_cacheinfo *ci = (struct rta_cacheinfo*)RTA_DATA(rta); - cJSON_AddNumberToObject(route, "cstamp", ci->rta_clntref); - cJSON_AddNumberToObject(route, "tstamp", ci->rta_lastuse); + // Get seconds the system is already up ("uptime") + struct timespec wall_clock; + clock_gettime(CLOCK_REALTIME, &wall_clock); + struct timespec boot_clock; + clock_gettime(CLOCK_BOOTTIME, &boot_clock); + const time_t delta_time = wall_clock.tv_sec - boot_clock.tv_sec; + cJSON_AddNumberToObject(route, "cstamp", delta_time + ci->rta_clntref); + cJSON_AddNumberToObject(route, "tstamp", delta_time + ci->rta_lastuse); cJSON_AddNumberToObject(route, "expires", ci->rta_expires); cJSON_AddNumberToObject(route, "error", ci->rta_error); cJSON_AddNumberToObject(route, "used", ci->rta_used); @@ -330,8 +336,14 @@ static int nlparsemsg_address(struct ifaddrmsg *ifa, void *buf, size_t len, cJSO struct ifa_cacheinfo *ci = (struct ifa_cacheinfo*)RTA_DATA(rta); cJSON_AddNumberToObject(addr, "prefered", ci->ifa_prefered); cJSON_AddNumberToObject(addr, "valid", ci->ifa_valid); - cJSON_AddNumberToObject(addr, "cstamp", 0.01*ci->cstamp); // created timestamp - cJSON_AddNumberToObject(addr, "tstamp", 0.01*ci->tstamp); // updated timestamp + // Get seconds the system is already up ("uptime") + struct timespec wall_clock; + clock_gettime(CLOCK_REALTIME, &wall_clock); + struct timespec boot_clock; + clock_gettime(CLOCK_BOOTTIME, &boot_clock); + const time_t delta_time = wall_clock.tv_sec - boot_clock.tv_sec; + cJSON_AddNumberToObject(addr, "cstamp", delta_time + 0.01*ci->cstamp); // created timestamp + cJSON_AddNumberToObject(addr, "tstamp", delta_time + 0.01*ci->tstamp); // updated timestamp break; } From 66b7fdf1cc729ef2e0090a1bdcbab7fd37824729 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 15 Jul 2024 07:23:57 +0200 Subject: [PATCH 219/339] Add link-netnsid and address types Signed-off-by: DL6ER --- src/api/docs/content/specs/network.yaml | 42 +++++++--- src/dnsmasq_interface.c | 2 +- src/tools/netlink.c | 105 ++++++++++++++++++++++-- 3 files changed, 130 insertions(+), 19 deletions(-) diff --git a/src/api/docs/content/specs/network.yaml b/src/api/docs/content/specs/network.yaml index 62f0b66f..079e8238 100644 --- a/src/api/docs/content/specs/network.yaml +++ b/src/api/docs/content/specs/network.yaml @@ -369,12 +369,21 @@ components: address: type: string description: Interface address + address_type: + type: string + description: Type of the interface address broadcast: type: string description: Interface broadcast address + broadcast_type: + type: string + description: Type of the broadcast address local: type: string description: Local address + local_type: + type: string + description: Type of the local address label: type: string description: Interface label @@ -423,7 +432,9 @@ components: unit: "MB" addresses: - address: "127.0.0.1" + address_type: "loopback" local: "127.0.0.1" + local_type: "loopback" family: "inet" scope: "host" flags: [ "permanent" ] @@ -434,7 +445,9 @@ components: cstamp: 1720989931 tstamp: 1720989931 - address: "::1" + address_type: "loopback" local: "::1" + local_type: "loopback" family: "inet6" scope: "host" flags: [ "permanent" ] @@ -462,7 +475,9 @@ components: unit: "GB" addresses: - address: "192.168.0.123" + address_type: "private" local: "192.168.0.123" + local_type: "private" family: "inet" scope: "universe" flags: [ "permanent" ] @@ -470,9 +485,10 @@ components: label: "eth0" prefered: 4294967295 valid: 4294967295 - cstamp: 11.23 - tstamp: 11.23 + cstamp: 1720989931.1 + tstamp: 1720989931.1 - address: "2001:db8::1234:5678:9abc:def0" + address_type: "global (GUA)" family: "inet6" scope: "universe" flags: [] @@ -483,6 +499,7 @@ components: cstamp: 2789057.25 tstamp: 2789057.25 - address: "fd29:db8::1234:5678:9abc:def0" + address_type: "site-local (ULA)" family: "inet6" scope: "universe" flags: [] @@ -490,9 +507,10 @@ components: label: "eth0" prefered: 3461 valid: 7061 - cstamp: 12.5 - tstamp: 2827298.75 + cstamp: 1720989931.1 + tstamp: 1720989931.1 - address: "fe80::1234:5678:9abc:def0" + address_type: "link-local (LL)" family: "inet6" scope: "link" flags: [ "permanent" ] @@ -500,8 +518,8 @@ components: label: "eth0" prefered: 4294967295 valid: 4294967295 - cstamp: 11.23 - tstamp: 11.23 + cstamp: 1720989931.1 + tstamp: 1720989931.1 - name: "wg0" speed: null type: "none" @@ -517,16 +535,20 @@ components: unit: "MB" addresses: - address: "10.1.0.1" + address_type: "private" local: "10.1.0.1" + local_type: "private" + family: "inet" scope: "universe" flags: [ "permanent" ] prefixlen: 24 label: "wg0" prefered: 4294967295 valid: 4294967295 - cstamp: 11.23 - tstamp: 11.23 + cstamp: 1720989931.1 + tstamp: 1720989931.1 - address: "fd00:4711::1" + address_type: "site-local (ULA)" family: "inet6" scope: "global" flags: [ "permanent" ] @@ -534,8 +556,8 @@ components: label: "wg0" prefered: 4294967295 valid: 4294967295 - cstamp: 11.23 - tstamp: 11.23 + cstamp: 1720989931.1 + tstamp: 1720989931.1 devices: type: object properties: diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c index 72bb41ce..200e37e9 100644 --- a/src/dnsmasq_interface.c +++ b/src/dnsmasq_interface.c @@ -1001,7 +1001,7 @@ void _FTL_iface(struct irec *recviface, const union all_addr *addr, const sa_fam // MUSL defines it differently than GNU C uint8_t bytes[2]; memcpy(&bytes, &iface->addr.in6.sin6_addr, 2); - // Global Unicast Address (2000::/3, RFC 4291) + // Global Unicast Address (2000::/3, RFC 4291) isGUA = (bytes[0] & 0x70) == 0x20; // Unique Local Address (fc00::/7, RFC 4193) isULA = (bytes[0] & 0xfe) == 0xfc; diff --git a/src/tools/netlink.c b/src/tools/netlink.c index 57813984..1de4d67e 100644 --- a/src/tools/netlink.c +++ b/src/tools/netlink.c @@ -17,6 +17,9 @@ #include #include +// defined in src/dnsmasq/rfc1035.c +extern int private_net(struct in_addr addr, int ban_localhost); + static bool nlrequest(int fd, struct sockaddr_nl *sa, int nlmsg_type) { char buf[BUFLEN] = { 0 }; @@ -162,7 +165,6 @@ static int nlparsemsg_route(struct rtmsg *rt, void *buf, size_t len, cJSON *rout case RTA_FLOW: // route realm case RTA_METRICS: // route metric - case RTA_TABLE: // routing table id case RTA_MARK: // route mark case RTA_EXPIRES: // route expires (in seconds) case RTA_UID: // user id @@ -179,6 +181,10 @@ static int nlparsemsg_route(struct rtmsg *rt, void *buf, size_t len, cJSON *rout break; } + case RTA_TABLE: // routing table id + // Already added above + break; + case RTA_PRIORITY: // route priority case RTA_PREF: // route preference { @@ -323,6 +329,79 @@ static int nlparsemsg_address(struct ifaddrmsg *ifa, void *buf, size_t len, cJSO char ip[INET6_ADDRSTRLEN] = { 0 }; inet_ntop(ifa->ifa_family, RTA_DATA(rta), ip, INET6_ADDRSTRLEN); cJSON_AddStringToObject(addr, ifaTypeToString(rta->rta_type), ip); + + // Determine and add address type (GUA, ULA, LL, ...) + const char *type_str = "unknown"; + if(rta->rta_type == IFA_ADDRESS) + type_str = "address_type"; + else if(rta->rta_type == IFA_LOCAL) + type_str = "local_type"; + else if(rta->rta_type == IFA_BROADCAST) + type_str = "broadcast_type"; + else if(rta->rta_type == IFA_ANYCAST) + type_str = "anycast_type"; + + if(ifa->ifa_family == AF_INET6) + { + const struct in6_addr *in6 = (struct in6_addr*)RTA_DATA(rta); + if(IN6_IS_ADDR_UNSPECIFIED(in6)) + cJSON_AddStringToObject(addr, type_str, "unspecified"); + else if(IN6_IS_ADDR_LOOPBACK(in6)) + cJSON_AddStringToObject(addr, type_str, "loopback"); + else if(IN6_IS_ADDR_MULTICAST(in6)) + cJSON_AddStringToObject(addr, type_str, "multicast"); + else if(IN6_IS_ADDR_LINKLOCAL(in6)) + cJSON_AddStringToObject(addr, type_str, "link-local (LL)"); + else if(IN6_IS_ADDR_SITELOCAL(in6)) + cJSON_AddStringToObject(addr, type_str, "site-local (ULA)"); + else if(IN6_IS_ADDR_V4MAPPED(in6)) + cJSON_AddStringToObject(addr, type_str, "IPv4-mapped"); + else if(IN6_IS_ADDR_V4COMPAT(in6)) + cJSON_AddStringToObject(addr, type_str, "IPv4-compatible"); + else if(IN6_IS_ADDR_MC_NODELOCAL(in6)) + cJSON_AddStringToObject(addr, type_str, "node-local"); + else if(IN6_IS_ADDR_MC_LINKLOCAL(in6)) + cJSON_AddStringToObject(addr, type_str, "link-local (LL)"); + else if(IN6_IS_ADDR_MC_SITELOCAL(in6)) + cJSON_AddStringToObject(addr, type_str, "site-local (ULA)"); + else if(IN6_IS_ADDR_MC_ORGLOCAL(in6)) + cJSON_AddStringToObject(addr, type_str, "organization-local"); + else if(IN6_IS_ADDR_MC_GLOBAL(in6)) + cJSON_AddStringToObject(addr, type_str, "global (GUA)"); + else + { + uint8_t bytes[2]; + memcpy(&bytes, in6, 2); + // Global Unicast Address (2000::/3, RFC 4291) + if((bytes[0] & 0x70) == 0x20) + cJSON_AddStringToObject(addr, type_str, "global (GUA)"); + // Unique Local Address (fc00::/7, RFC 4193) + else if((bytes[0] & 0xfe) == 0xfc) + cJSON_AddStringToObject(addr, type_str, "site-local (ULA)"); + // Link Local Address (fe80::/10, RFC 4291) + else if((bytes[0] & 0xff) == 0xfe && (bytes[1] & 0x30) == 0) + cJSON_AddStringToObject(addr, type_str, "link-local (LL)"); + else + cJSON_AddStringToObject(addr, type_str, "unknown"); + } + } + else if(ifa->ifa_family == AF_INET) + { + const struct in_addr *in = (struct in_addr*)RTA_DATA(rta); + if(in->s_addr == INADDR_ANY) + cJSON_AddStringToObject(addr, type_str, "unspecified"); + else if(in->s_addr == INADDR_LOOPBACK || + (in->s_addr & htonl(0xff000000)) == htonl(0x7f000000)) + cJSON_AddStringToObject(addr, type_str, "loopback"); + else if((in->s_addr & htonl(0xf0000000)) == htonl(0xe0000000)) + cJSON_AddStringToObject(addr, type_str, "multicast"); + else if(private_net(*in, false)) + cJSON_AddStringToObject(addr, type_str, "private"); + else + cJSON_AddStringToObject(addr, type_str, "public"); + } + else + cJSON_AddStringToObject(addr, type_str, "unknown"); break; } @@ -348,14 +427,8 @@ static int nlparsemsg_address(struct ifaddrmsg *ifa, void *buf, size_t len, cJSO } case IFA_FLAGS: - { - cJSON *iflags = cJSON_CreateArray(); - for(unsigned int i = 0; i < sizeof(ifaf_flags)/sizeof(ifaf_flags[0]); i++) - if (ifaf_flags[i].flag & ifa->ifa_flags) - cJSON_AddStringReferenceToArray(iflags, ifaf_flags[i].name); - cJSON_AddItemToObject(addr, "flags", iflags); + // Already added above, ignore this duplicate break; - } case IFA_RT_PRIORITY: { @@ -544,6 +617,7 @@ static int nlparsemsg_link(struct ifinfomsg *ifi, void *buf, size_t len, cJSON * case IFLA_NEW_NETNSID: case IFLA_MIN_MTU: case IFLA_MAX_MTU: + case IFLA_LINK_NETNSID: { if(!detailed) break; @@ -816,6 +890,15 @@ static int nlparsemsg_link(struct ifinfomsg *ifi, void *buf, size_t len, cJSON * case IFLA_INFO_KIND: cJSON_AddStringToObject(link, "link_kind", (char*)RTA_DATA(nlinkinfo)); break; + case IFLA_INFO_SLAVE_KIND: + cJSON_AddStringToObject(link, "slave_kind", (char*)RTA_DATA(nlinkinfo)); + break; + case IFLA_INFO_DATA: + case IFLA_INFO_SLAVE_DATA: + // Needs a very complex + // disassembler, out of + // scope here + break; default: { // Unknown rta_type @@ -935,6 +1018,12 @@ static int nlparsemsg_link(struct ifinfomsg *ifi, void *buf, size_t len, cJSON * break; } + case IFLA_XDP: + // Parsing XDP needs a full BPF program + // disassembler which is clearly out of scope + // here + break; + default: { // Unknown rta_type From c91ac40bd2795d1041b4f7da5897dd4312b7bff1 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 14 Jul 2024 15:54:19 +0200 Subject: [PATCH 220/339] fail*.dnssec.works changed its configuration from being BOGUS to ABANDONNED and cannot be used any longer for BOGUS testing Signed-off-by: DL6ER --- test/test_suite.bats | 6 ------ 1 file changed, 6 deletions(-) diff --git a/test/test_suite.bats b/test/test_suite.bats index d747e8dd..ed73f557 100644 --- a/test/test_suite.bats +++ b/test/test_suite.bats @@ -401,12 +401,6 @@ [[ ${lines[@]} == *"status: NOERROR"* ]] } -@test "DNSSEC: BOGUS domain is rejected" { - run bash -c "dig A fail01.dnssec.works @127.0.0.1" - printf "%s\n" "${lines[@]}" - [[ ${lines[@]} == *"status: SERVFAIL"* ]] -} - @test "Special domain: NXDOMAIN is returned" { run bash -c "dig A mask.icloud.com @127.0.0.1" printf "%s\n" "${lines[@]}" From d39703b9a19b30bc5d70c64bf9d4b79b4b38197e Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 16 Jul 2024 11:05:03 +0200 Subject: [PATCH 221/339] Restart timer if set even if no blocking mode change was requested. This allows users to extend a temporary state, e.g. by running another "pihole disable 5m" shortly before the previous cacall reaches timeout. Previously, calling "pihoe disable 1m" would make the change permanent which seems undesirable Signed-off-by: DL6ER --- src/api/dns.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/api/dns.c b/src/api/dns.c index 10e90605..489ed7df 100644 --- a/src/api/dns.c +++ b/src/api/dns.c @@ -100,7 +100,7 @@ static int set_blocking(struct ftl_conn *api) // The blocking status does not need to be changed // Delete a possibly running timer - set_blockingmode_timer(-1.0, true); + set_blockingmode_timer(timer, true); log_debug(DEBUG_API, "No change in blocking mode, resetting timer"); } From af7ea105eb0987aec4ebbe58189f063d93ea2ae6 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 25 Jun 2024 09:02:29 +0200 Subject: [PATCH 222/339] Implement actual TOP suggestions for the Query Log Signed-off-by: DL6ER --- src/api/queries.c | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/src/api/queries.c b/src/api/queries.c index eb16529f..1728048f 100644 --- a/src/api/queries.c +++ b/src/api/queries.c @@ -47,7 +47,7 @@ static int add_strings_to_array(struct ftl_conn *api, cJSON *array, const char * // Loop through returned rows int counter = 0; while((rc = sqlite3_step(stmt)) == SQLITE_ROW && - (max_count < 0 || ++counter < max_count)) + (max_count < 0 || ++counter <= max_count)) JSON_COPY_STR_TO_ARRAY(array, (const char*)sqlite3_column_text(stmt, 0)); // Acceptable return codes are either @@ -77,19 +77,24 @@ int api_queries_suggestions(struct ftl_conn *api) // Get domains cJSON *domain = JSON_NEW_ARRAY(); - rc = add_strings_to_array(api, domain, "SELECT domain FROM domain_by_id", count); + log_debug(DEBUG_API, "Reading top domains from database"); + rc = add_strings_to_array(api, domain, "WITH CTE AS (SELECT COUNT(*) cnt, domain FROM query_storage GROUP BY domain ORDER BY cnt DESC)"\ + "SELECT d.domain FROM CTE JOIN domain_by_id d ON CTE.domain = d.id", count); if(rc != 0) { log_err("Cannot read domains from database"); cJSON_Delete(domain); return rc; } + log_debug(DEBUG_API, "Read %d domains from database", cJSON_GetArraySize(domain)); // Get clients, both by IP and names // We have to call DISTINCT() here as multiple IPs can map to and name and // vice versa cJSON *client_ip = JSON_NEW_ARRAY(); - rc = add_strings_to_array(api, client_ip, "SELECT DISTINCT(ip) FROM client_by_id", count); + log_debug(DEBUG_API, "Reading top client IPs from database"); + rc = add_strings_to_array(api, client_ip, "WITH CTE AS (SELECT COUNT(*) cnt, client FROM query_storage GROUP BY client ORDER BY cnt DESC)"\ + "SELECT c.ip FROM CTE JOIN client_by_id c ON CTE.client = c.id", count); if(rc != 0) { log_err("Cannot read client IPs from database"); @@ -97,8 +102,12 @@ int api_queries_suggestions(struct ftl_conn *api) cJSON_Delete(client_ip); return rc; } + log_debug(DEBUG_API, "Read %d client IPs from database", cJSON_GetArraySize(client_ip)); + cJSON *client_name = JSON_NEW_ARRAY(); - rc = add_strings_to_array(api, client_name, "SELECT DISTINCT(name) FROM client_by_id", count); + log_debug(DEBUG_API, "Reading top client names from database"); + rc = add_strings_to_array(api, client_name, "WITH CTE AS (SELECT COUNT(*) cnt, client FROM query_storage GROUP BY client ORDER BY cnt DESC)"\ + "SELECT c.name FROM CTE JOIN client_by_id c ON CTE.client = c.id WHERE c.name IS NOT NULL", count); if(rc != 0) { log_err("Cannot read client names from database"); @@ -107,10 +116,13 @@ int api_queries_suggestions(struct ftl_conn *api) cJSON_Delete(client_name); return rc; } + log_debug(DEBUG_API, "Read %d client names from database", cJSON_GetArraySize(client_name)); // Get upstreams cJSON *upstream = JSON_NEW_ARRAY(); - rc = add_strings_to_array(api, upstream, "SELECT forward FROM forward_by_id", count); + log_debug(DEBUG_API, "Reading top upstreams from database"); + rc = add_strings_to_array(api, upstream, "WITH CTE AS (SELECT COUNT(*) cnt, forward FROM query_storage GROUP BY forward ORDER BY cnt DESC)"\ + "SELECT f.forward FROM CTE JOIN forward_by_id f ON CTE.forward = f.id", count); if(rc != 0) { log_err("Cannot read forward from database"); @@ -120,6 +132,7 @@ int api_queries_suggestions(struct ftl_conn *api) cJSON_Delete(upstream); return rc; } + log_debug(DEBUG_API, "Read %d upstreams from database", cJSON_GetArraySize(upstream)); // Get types cJSON *type = JSON_NEW_ARRAY(); From d01809a1a76a23ee2ff93993d9dc1f52e048db1c Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 25 Jun 2024 09:12:23 +0200 Subject: [PATCH 223/339] Sort only once for TOP clients Signed-off-by: DL6ER --- src/api/queries.c | 39 +++++++++++++++------------------------ 1 file changed, 15 insertions(+), 24 deletions(-) diff --git a/src/api/queries.c b/src/api/queries.c index 1728048f..e7d71f68 100644 --- a/src/api/queries.c +++ b/src/api/queries.c @@ -22,7 +22,7 @@ // dbopen(false, ), dbclose() #include "database/common.h" -static int add_strings_to_array(struct ftl_conn *api, cJSON *array, const char *querystr, const int max_count) +static int add_strings_to_array(struct ftl_conn *api, cJSON *array1, cJSON *array2, const char *querystr, const int max_count) { sqlite3 *memdb = get_memdb(); @@ -48,7 +48,11 @@ static int add_strings_to_array(struct ftl_conn *api, cJSON *array, const char * int counter = 0; while((rc = sqlite3_step(stmt)) == SQLITE_ROW && (max_count < 0 || ++counter <= max_count)) - JSON_COPY_STR_TO_ARRAY(array, (const char*)sqlite3_column_text(stmt, 0)); + { + JSON_COPY_STR_TO_ARRAY(array1, (const char*)sqlite3_column_text(stmt, 0)); + if(array2 != NULL) + JSON_COPY_STR_TO_ARRAY(array2, (const char*)sqlite3_column_text(stmt, 1)); + } // Acceptable return codes are either // - SQLITE_DONE: We read all lines, or @@ -78,8 +82,8 @@ int api_queries_suggestions(struct ftl_conn *api) // Get domains cJSON *domain = JSON_NEW_ARRAY(); log_debug(DEBUG_API, "Reading top domains from database"); - rc = add_strings_to_array(api, domain, "WITH CTE AS (SELECT COUNT(*) cnt, domain FROM query_storage GROUP BY domain ORDER BY cnt DESC)"\ - "SELECT d.domain FROM CTE JOIN domain_by_id d ON CTE.domain = d.id", count); + rc = add_strings_to_array(api, domain, NULL, "WITH CTE AS (SELECT COUNT(*) cnt, domain FROM query_storage GROUP BY domain ORDER BY cnt DESC)"\ + "SELECT d.domain FROM CTE JOIN domain_by_id d ON CTE.domain = d.id", count); if(rc != 0) { log_err("Cannot read domains from database"); @@ -92,9 +96,10 @@ int api_queries_suggestions(struct ftl_conn *api) // We have to call DISTINCT() here as multiple IPs can map to and name and // vice versa cJSON *client_ip = JSON_NEW_ARRAY(); - log_debug(DEBUG_API, "Reading top client IPs from database"); - rc = add_strings_to_array(api, client_ip, "WITH CTE AS (SELECT COUNT(*) cnt, client FROM query_storage GROUP BY client ORDER BY cnt DESC)"\ - "SELECT c.ip FROM CTE JOIN client_by_id c ON CTE.client = c.id", count); + cJSON *client_name = JSON_NEW_ARRAY(); + log_debug(DEBUG_API, "Reading top client IPs and names from database"); + rc = add_strings_to_array(api, client_ip, client_name, "WITH CTE AS (SELECT COUNT(*) cnt, client FROM query_storage GROUP BY client ORDER BY cnt DESC)"\ + "SELECT c.ip,c.name FROM CTE JOIN client_by_id c ON CTE.client = c.id", count); if(rc != 0) { log_err("Cannot read client IPs from database"); @@ -102,27 +107,13 @@ int api_queries_suggestions(struct ftl_conn *api) cJSON_Delete(client_ip); return rc; } - log_debug(DEBUG_API, "Read %d client IPs from database", cJSON_GetArraySize(client_ip)); - - cJSON *client_name = JSON_NEW_ARRAY(); - log_debug(DEBUG_API, "Reading top client names from database"); - rc = add_strings_to_array(api, client_name, "WITH CTE AS (SELECT COUNT(*) cnt, client FROM query_storage GROUP BY client ORDER BY cnt DESC)"\ - "SELECT c.name FROM CTE JOIN client_by_id c ON CTE.client = c.id WHERE c.name IS NOT NULL", count); - if(rc != 0) - { - log_err("Cannot read client names from database"); - cJSON_Delete(domain); - cJSON_Delete(client_ip); - cJSON_Delete(client_name); - return rc; - } - log_debug(DEBUG_API, "Read %d client names from database", cJSON_GetArraySize(client_name)); + log_debug(DEBUG_API, "Read %d client IPs and %d client names from database", cJSON_GetArraySize(client_ip), cJSON_GetArraySize(client_name)); // Get upstreams cJSON *upstream = JSON_NEW_ARRAY(); log_debug(DEBUG_API, "Reading top upstreams from database"); - rc = add_strings_to_array(api, upstream, "WITH CTE AS (SELECT COUNT(*) cnt, forward FROM query_storage GROUP BY forward ORDER BY cnt DESC)"\ - "SELECT f.forward FROM CTE JOIN forward_by_id f ON CTE.forward = f.id", count); + rc = add_strings_to_array(api, upstream, NULL, "WITH CTE AS (SELECT COUNT(*) cnt, forward FROM query_storage GROUP BY forward ORDER BY cnt DESC)"\ + "SELECT f.forward FROM CTE JOIN forward_by_id f ON CTE.forward = f.id", count); if(rc != 0) { log_err("Cannot read forward from database"); From 94dceae4198618f1ef023f51188174d604d2eaff Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 25 Jun 2024 09:14:42 +0200 Subject: [PATCH 224/339] Only add non-empty TOP replies Signed-off-by: DL6ER --- src/api/queries.c | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/api/queries.c b/src/api/queries.c index e7d71f68..7874c7d6 100644 --- a/src/api/queries.c +++ b/src/api/queries.c @@ -44,14 +44,23 @@ static int add_strings_to_array(struct ftl_conn *api, cJSON *array1, cJSON *arra sqlite3_errstr(rc)); } - // Loop through returned rows + // Loop through returned rows and add them to the array int counter = 0; while((rc = sqlite3_step(stmt)) == SQLITE_ROW && (max_count < 0 || ++counter <= max_count)) { - JSON_COPY_STR_TO_ARRAY(array1, (const char*)sqlite3_column_text(stmt, 0)); + const char *array1_str = (const char*)sqlite3_column_text(stmt, 0); + if(array1_str != NULL && array1_str[0] != '\0') + // Only add non-empty strings + JSON_COPY_STR_TO_ARRAY(array1, array1_str); if(array2 != NULL) - JSON_COPY_STR_TO_ARRAY(array2, (const char*)sqlite3_column_text(stmt, 1)); + { + // We have a second array to fill (second column in the query) + const char *array2_str = (const char*)sqlite3_column_text(stmt, 1); + if(array2_str != NULL && array2_str[0] != '\0') + // Only add non-empty strings + JSON_COPY_STR_TO_ARRAY(array2, array2_str); + } } // Acceptable return codes are either From 5d2d74a3e331a36d4547b018530f7f2dcdfc1ca3 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 25 Jun 2024 09:22:35 +0200 Subject: [PATCH 225/339] Fix LegacyKeyValueFormat warning during CI builds Signed-off-by: DL6ER --- .github/Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/Dockerfile b/.github/Dockerfile index 046e45f6..40d7a614 100644 --- a/.github/Dockerfile +++ b/.github/Dockerfile @@ -5,13 +5,13 @@ WORKDIR /app COPY . /app ARG CI_ARCH="linux/amd64" -ENV CI_ARCH ${CI_ARCH} +ENV CI_ARCH=${CI_ARCH} ARG GIT_BRANCH="test" -ENV GIT_BRANCH ${GIT_BRANCH} +ENV GIT_BRANCH=${GIT_BRANCH} ARG GIT_TAG="test" -ENV GIT_TAG ${GIT_TAG} +ENV GIT_TAG=${GIT_TAG} ARG BUILD_OPTS="" -ENV BUILD_OPTS ${BUILD_OPTS} +ENV BUILD_OPTS=${BUILD_OPTS} # Build FTL # Remove possible old build files From bf1355c8ce43fcd7f94e522849fc05890279deda Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 16 Jul 2024 08:20:30 +0200 Subject: [PATCH 226/339] Increase speed of the /api/stats/* endpoints by reducing the use of shared memory locks. This is possible at the costs of a few bytes extra memory (at most a few KB) as we collect all data only once and don't need to get object poiters twice. Signed-off-by: DL6ER --- src/api/stats.c | 335 ++++++++++++++++++++++++++---------------------- 1 file changed, 185 insertions(+), 150 deletions(-) diff --git a/src/api/stats.c b/src/api/stats.c index b272e768..09f71fc6 100644 --- a/src/api/stats.c +++ b/src/api/stats.c @@ -14,8 +14,6 @@ #include "api/api.h" #include "shmem.h" #include "datastructure.h" -// read_setupVarsconf() -#include "config/setupVars.h" // logging routines #include "log.h" // config struct @@ -27,6 +25,17 @@ // sqrt() #include +struct top_entries { + int count; + unsigned int responses; + in_port_t port; + size_t namepos; + size_t ippos; + double rtime; + double rtuncertainty; + +}; + /* qsort comparison function (count field), sort ASC static int __attribute__((pure)) cmpasc(const void *a, const void *b) { @@ -55,6 +64,20 @@ int __attribute__((pure)) cmpdesc(const void *a, const void *b) return 0; } +// qsort subroutine, sort DESC +static int __attribute__((pure)) cmpdesc_te(const void *a, const void *b) +{ + const struct top_entries *elem1 = (struct top_entries*)a; + const struct top_entries *elem2 = (struct top_entries*)b; + + if (elem1->count > elem2->count) + return -1; + else if (elem1->count < elem2->count) + return 1; + else + return 0; +} + static int get_query_types_obj(struct ftl_conn *api, cJSON *types) { for(unsigned int i = TYPE_A; i < TYPE_MAX; i++) @@ -128,11 +151,14 @@ int api_stats_summary(struct ftl_conn *api) cJSON *gravity = JSON_NEW_OBJECT(); JSON_ADD_NUMBER_TO_OBJECT(gravity, "domains_being_blocked", counters->database.gravity); + // Unlock shared memory + unlock_shm(); + cJSON *json = JSON_NEW_OBJECT(); JSON_ADD_ITEM_TO_OBJECT(json, "queries", queries); JSON_ADD_ITEM_TO_OBJECT(json, "clients", clients); JSON_ADD_ITEM_TO_OBJECT(json, "gravity", gravity); - JSON_SEND_OBJECT_UNLOCK(json); + JSON_SEND_OBJECT(json); } int api_stats_top_domains(struct ftl_conn *api) @@ -151,18 +177,6 @@ int api_stats_top_domains(struct ftl_conn *api) JSON_SEND_OBJECT(json); } - // Lock shared memory - lock_shm(); - - // Allocate memory - const int domains = counters->domains; - int *temparray = calloc(2*domains, sizeof(int)); - if(temparray == NULL) - { - log_err("Memory allocation failed in %s()", __FUNCTION__); - return 0; - } - bool blocked = false; // Can be overwritten by query string int count = 10; // /api/stats/top_domains?blocked=true @@ -176,6 +190,26 @@ int api_stats_top_domains(struct ftl_conn *api) get_int_var(api->request->query_string, "count", &count); } + // Get domains which the user doesn't want to see + regex_t *regex_domains = NULL; + unsigned int N_regex_domains = 0; + compile_filter_regex(api, "webserver.api.excludeDomains", + config.webserver.api.excludeDomains.v.json, + ®ex_domains, &N_regex_domains); + + // Lock shared memory + lock_shm(); + + const int domains = counters->domains; + const int total_queries = counters->queries; + const int blocked_count = get_blocked_count(); + struct top_entries *top_domains = calloc(domains, sizeof(struct top_entries)); + if(top_domains == NULL) + { + log_err("Memory allocation failed in %s()", __FUNCTION__); + return 0; + } + unsigned int added_domains = 0u; for(int domainID = 0; domainID < domains; domainID++) { @@ -184,60 +218,42 @@ int api_stats_top_domains(struct ftl_conn *api) if(domain == NULL) continue; - // Add domain ID - temparray[2*added_domains + 0] = domainID; - - // Use either blocked or total count based on request string - temparray[2*added_domains + 1] = blocked ? domain->blockedcount : domain->count - domain->blockedcount; - - added_domains++; - } - - // Sort temporary array - qsort(temparray, added_domains, sizeof(int[2]), cmpdesc); - - // Get filter - const char* log_show = read_setupVarsconf("API_QUERY_LOG_SHOW"); - bool showpermitted = true, showblocked = true; - if(log_show != NULL) - { - if((strcmp(log_show, "permittedonly")) == 0) - showblocked = false; - else if((strcmp(log_show, "blockedonly")) == 0) - showpermitted = false; - else if((strcmp(log_show, "nothing")) == 0) - { - showpermitted = false; - showblocked = false; - } - } - clearSetupVarsArray(); - - // Get domains which the user doesn't want to see - regex_t *regex_domains = NULL; - unsigned int N_regex_domains = 0; - compile_filter_regex(api, "webserver.api.excludeDomains", - config.webserver.api.excludeDomains.v.json, - ®ex_domains, &N_regex_domains); - - int n = 0; - cJSON *top_domains = JSON_NEW_ARRAY(); - for(unsigned int i = 0; i < added_domains; i++) - { - // Get sorted index - const int domainID = temparray[2*i + 0]; - // Get domain pointer - const domainsData* domain = getDomain(domainID, true); - if(domain == NULL) - continue; - - // Get domain name const char *domain_name = getstr(domain->domainpos); // Hidden domain, probably due to privacy level. Skip this in the top lists if(strcmp(domain_name, HIDDEN_DOMAIN) == 0) continue; + // Use either blocked or total count based on request string + top_domains[added_domains].count = blocked ? domain->blockedcount : domain->count - domain->blockedcount; + + // Get domain name + top_domains[added_domains].namepos = domain->domainpos; + + // Increment counter + added_domains++; + } + + // Unlock shared memory + unlock_shm(); + + // Sort temporary array + qsort(top_domains, added_domains, sizeof(*top_domains), cmpdesc_te); + + int n = 0; + cJSON *jtop_domains = JSON_NEW_ARRAY(); + + // Lock shared memory + lock_shm(); + + for(unsigned int i = 0; i < added_domains; i++) + { + // Skip e.g. recycled domains + if(top_domains[i].namepos == 0) + continue; + + const char *domain = getstr(top_domains[i].namepos); + // Skip this client if there is a filter on it bool skip_domain = false; if(N_regex_domains > 0) @@ -246,7 +262,7 @@ int api_stats_top_domains(struct ftl_conn *api) for(unsigned int j = 0; j < N_regex_domains; j++) { // Check if the domain matches the regex - if(regexec(®ex_domains[j], domain_name, 0, NULL, 0) == 0) + if(regexec(®ex_domains[j], domain, 0, NULL, 0) == 0) { // Domain matches skip_domain = true; @@ -258,30 +274,25 @@ int api_stats_top_domains(struct ftl_conn *api) if(skip_domain) continue; - int domain_count = -1; - if(blocked && showblocked && domain->blockedcount > 0) - { - domain_count = domain->blockedcount; - n++; - } - else if(!blocked && showpermitted && (domain->count - domain->blockedcount) > 0) - { - domain_count = domain->count - domain->blockedcount; - n++; - } - if(domain_count > -1) + if(top_domains[i].count > 0) { cJSON *domain_item = JSON_NEW_OBJECT(); - JSON_REF_STR_IN_OBJECT(domain_item, "domain", domain_name); - JSON_ADD_NUMBER_TO_OBJECT(domain_item, "count", domain_count); - JSON_ADD_ITEM_TO_ARRAY(top_domains, domain_item); + JSON_COPY_STR_TO_OBJECT(domain_item, "domain", domain); + JSON_ADD_NUMBER_TO_OBJECT(domain_item, "count", top_domains[i].count); + JSON_ADD_ITEM_TO_ARRAY(jtop_domains, domain_item); + n++; } // Only count entries that are actually sent and return when we have send enough data if(n >= count) break; } - free(temparray); + + // Unlock shared memory + unlock_shm(); + + // Free temporary array + free(top_domains); // Free regexes if(N_regex_domains > 0) @@ -295,13 +306,12 @@ int api_stats_top_domains(struct ftl_conn *api) } cJSON *json = JSON_NEW_OBJECT(); - JSON_ADD_ITEM_TO_OBJECT(json, "domains", top_domains); + JSON_ADD_ITEM_TO_OBJECT(json, "domains", jtop_domains); - const int blocked_count = get_blocked_count(); - JSON_ADD_NUMBER_TO_OBJECT(json, "total_queries", counters->queries); + JSON_ADD_NUMBER_TO_OBJECT(json, "total_queries", total_queries); JSON_ADD_NUMBER_TO_OBJECT(json, "blocked_queries", blocked_count); - JSON_SEND_OBJECT_UNLOCK(json); + JSON_SEND_OBJECT(json); } int api_stats_top_clients(struct ftl_conn *api) @@ -337,10 +347,12 @@ int api_stats_top_clients(struct ftl_conn *api) lock_shm(); int clients = counters->clients; - int *temparray = calloc(2*clients, sizeof(int)); - if(temparray == NULL) + const int total_queries = counters->queries; + const int blocked_count = get_blocked_count(); + struct top_entries *top_clients = calloc(clients, sizeof(struct top_entries)); + if(top_clients == NULL) { - log_err("Memory allocation failed in api_stats_top_clients()"); + log_err("Memory allocation failed in %s()", __FUNCTION__); return 0; } @@ -354,15 +366,26 @@ int api_stats_top_clients(struct ftl_conn *api) if(client == NULL || (!client->flags.aliasclient && client->aliasclient_id >= 0)) continue; - temparray[2*added_clients + 0] = clientID; + const char *client_ip = getstr(client->ippos); + // Hidden client, probably due to privacy level. Skip this in the top lists + if(strcmp(client_ip, HIDDEN_CLIENT) == 0) + continue; + // Use either blocked or total count based on request string - temparray[2*added_clients + 1] = blocked ? client->blockedcount : client->count; + top_clients[added_clients].count = blocked ? client->blockedcount : client->count; + + // Get client name and IP + top_clients[added_clients].ippos = client->ippos; + top_clients[added_clients].namepos = client->namepos; added_clients++; } + // Unlock shared memory + unlock_shm(); + // Sort temporary array - qsort(temparray, added_clients, sizeof(int[2]), cmpdesc); + qsort(top_clients, added_clients, sizeof(*top_clients), cmpdesc_te); // Get clients which the user doesn't want to see regex_t *regex_clients = NULL; @@ -372,24 +395,19 @@ int api_stats_top_clients(struct ftl_conn *api) ®ex_clients, &N_regex_clients); int n = 0; - cJSON *top_clients = JSON_NEW_ARRAY(); + cJSON *jtop_clients = JSON_NEW_ARRAY(); + + // Lock shared memory + lock_shm(); + for(unsigned int i = 0; i < added_clients; i++) { - // Get sorted indices and counter values (may be either total or blocked count) - const int clientID = temparray[2*i + 0]; - const int client_count = temparray[2*i + 1]; - // Get client pointer - const clientsData* client = getClient(clientID, true); - if(client == NULL) + // Skip e.g. recycled clients + if(top_clients[i].namepos == 0) continue; - // Get IP and host name of client - const char *client_ip = getstr(client->ippos); - const char *client_name = getstr(client->namepos); - - // Hidden client, probably due to privacy level. Skip this in the top lists - if(strcmp(client_ip, HIDDEN_CLIENT) == 0) - continue; + const char *client_ip = getstr(top_clients[i].ippos); + const char *client_name = getstr(top_clients[i].namepos); // Skip this client if there is a filter on it bool skip_client = false; @@ -419,21 +437,25 @@ int api_stats_top_clients(struct ftl_conn *api) // Return this client if the client made at least one query // within the most recent 24 hours - if(client_count > 0) + if(top_clients[i].count > 0) { cJSON *client_item = JSON_NEW_OBJECT(); - JSON_REF_STR_IN_OBJECT(client_item, "name", client_name); - JSON_REF_STR_IN_OBJECT(client_item, "ip", client_ip); - JSON_ADD_NUMBER_TO_OBJECT(client_item, "count", client_count); - JSON_ADD_ITEM_TO_ARRAY(top_clients, client_item); + JSON_COPY_STR_TO_OBJECT(client_item, "name", client_name); + JSON_COPY_STR_TO_OBJECT(client_item, "ip", client_ip); + JSON_ADD_NUMBER_TO_OBJECT(client_item, "count", top_clients[i].count); + JSON_ADD_ITEM_TO_ARRAY(jtop_clients, client_item); n++; } if(n == count) break; } + + // Unlock shared memory + unlock_shm(); + // Free temporary array - free(temparray); + free(top_clients); // Free regexes if(N_regex_clients > 0) @@ -447,20 +469,21 @@ int api_stats_top_clients(struct ftl_conn *api) } cJSON *json = JSON_NEW_OBJECT(); - JSON_ADD_ITEM_TO_OBJECT(json, "clients", top_clients); + JSON_ADD_ITEM_TO_OBJECT(json, "clients", jtop_clients); - const int blocked_count = get_blocked_count(); JSON_ADD_NUMBER_TO_OBJECT(json, "blocked_queries", blocked_count); - JSON_ADD_NUMBER_TO_OBJECT(json, "total_queries", counters->queries); - JSON_SEND_OBJECT_UNLOCK(json); + JSON_ADD_NUMBER_TO_OBJECT(json, "total_queries", total_queries); + JSON_SEND_OBJECT(json); } int api_stats_upstreams(struct ftl_conn *api) { const int upstreams = counters->upstreams; - int *temparray = calloc(2*upstreams, sizeof(int)); - if(temparray == NULL) + const int forwarded_count = get_forwarded_count(); + const int total_queries = counters->queries; + struct top_entries *top_upstreams = calloc(upstreams, sizeof(struct top_entries)); + if(top_upstreams == NULL) { log_err("Memory allocation failed in api_stats_upstreams()"); return 0; @@ -477,22 +500,34 @@ int api_stats_upstreams(struct ftl_conn *api) if(upstream == NULL) continue; - temparray[2*added_upstreams + 0] = upstreamID; - temparray[2*added_upstreams + 1] = upstream->count; + top_upstreams[added_upstreams].count = upstream->count; + top_upstreams[added_upstreams].ippos = upstream->ippos; + top_upstreams[added_upstreams].namepos = upstream->namepos; + top_upstreams[added_upstreams].port = upstream->port; + top_upstreams[added_upstreams].responses = upstream->responses; + top_upstreams[added_upstreams].rtime = upstream->rtime; + top_upstreams[added_upstreams].rtuncertainty = upstream->rtuncertainty; added_upstreams++; } + // Unlock shared memory + unlock_shm(); + // Sort temporary array in descending order - qsort(temparray, upstreams, sizeof(int[2]), cmpdesc); + qsort(top_upstreams, added_upstreams, sizeof(*top_upstreams), cmpdesc); // Loop over available forward destinations - cJSON *top_upstreams = JSON_NEW_ARRAY(); + cJSON *jtop_upstreams = JSON_NEW_ARRAY(); + + // Lock shared memory + lock_shm(); + for(int i = -2; i < (int)added_upstreams; i++) { int count = 0; const char* ip, *name; - int port = -1; + in_port_t port = -1; double responsetime = 0.0, uncertainty = 0.0; if(i == -2) @@ -512,33 +547,22 @@ int api_stats_upstreams(struct ftl_conn *api) else { // Regular upstream destination - // Get sorted indices - const int upstreamID = temparray[2*i + 0]; - - // Get upstream pointer - const upstreamsData *upstream = getUpstream(upstreamID, true); - if(upstream == NULL) - continue; - - // Get IP and host name of upstream destination if available - ip = getstr(upstream->ippos); - name = getstr(upstream->namepos); - port = upstream->port; - - // Get percentage - count = upstream->count; + ip = getstr(top_upstreams[i].ippos); + name = getstr(top_upstreams[i].namepos); + port = top_upstreams[i].port; + count = top_upstreams[i].count; // Compute average response time and uncertainty (unit: seconds) - if(upstream->responses > 0) + if(top_upstreams[i].responses > 0) { // Simple average of the response times - responsetime = upstream->rtime / upstream->responses; + responsetime = top_upstreams[i].rtime / top_upstreams[i].responses; } - if(upstream->responses > 1) + if(top_upstreams[i].responses > 1) { // The actual value will be somewhere in a neighborhood around the mean value. // This neighborhood of values is the uncertainty in the mean. - uncertainty = sqrt(upstream->rtuncertainty / upstream->responses / (upstream->responses-1)); + uncertainty = sqrt(top_upstreams[i].rtuncertainty / top_upstreams[i].responses / (top_upstreams[i].responses-1)); } } @@ -548,31 +572,36 @@ int api_stats_upstreams(struct ftl_conn *api) if(count > 0 || i < 0) { cJSON *upstream = JSON_NEW_OBJECT(); - JSON_REF_STR_IN_OBJECT(upstream, "ip", ip); - JSON_REF_STR_IN_OBJECT(upstream, "name", name); + JSON_COPY_STR_TO_OBJECT(upstream, "ip", ip); + JSON_COPY_STR_TO_OBJECT(upstream, "name", name); JSON_ADD_NUMBER_TO_OBJECT(upstream, "port", port); JSON_ADD_NUMBER_TO_OBJECT(upstream, "count", count); cJSON *statistics = JSON_NEW_OBJECT(); JSON_ADD_NUMBER_TO_OBJECT(statistics, "response", responsetime); JSON_ADD_NUMBER_TO_OBJECT(statistics, "variance", uncertainty); JSON_ADD_ITEM_TO_OBJECT(upstream, "statistics", statistics); - JSON_ADD_ITEM_TO_ARRAY(top_upstreams, upstream); + JSON_ADD_ITEM_TO_ARRAY(jtop_upstreams, upstream); } } + // Unlock shared memory + unlock_shm(); + // Free temporary array - free(temparray); + free(top_upstreams); cJSON *json = JSON_NEW_OBJECT(); - JSON_ADD_ITEM_TO_OBJECT(json, "upstreams", top_upstreams); - const int forwarded_count = get_forwarded_count(); + JSON_ADD_ITEM_TO_OBJECT(json, "upstreams", jtop_upstreams); + JSON_ADD_NUMBER_TO_OBJECT(json, "forwarded_queries", forwarded_count); - JSON_ADD_NUMBER_TO_OBJECT(json, "total_queries", counters->queries); - JSON_SEND_OBJECT_UNLOCK(json); + JSON_ADD_NUMBER_TO_OBJECT(json, "total_queries", total_queries); + + JSON_SEND_OBJECT(json); } int api_stats_query_types(struct ftl_conn *api) { + // Lock shared memory lock_shm(); cJSON *types = JSON_NEW_OBJECT(); @@ -583,11 +612,14 @@ int api_stats_query_types(struct ftl_conn *api) return ret; } + // Unlock shared memory + unlock_shm(); + cJSON *json = JSON_NEW_OBJECT(); JSON_ADD_ITEM_TO_OBJECT(json, "types", types); // Send response - JSON_SEND_OBJECT_UNLOCK(json); + JSON_SEND_OBJECT(json); } int api_stats_recentblocked(struct ftl_conn *api) @@ -641,7 +673,10 @@ int api_stats_recentblocked(struct ftl_conn *api) break; } + // Unlock shared memory + unlock_shm(); + cJSON *json = JSON_NEW_OBJECT(); JSON_ADD_ITEM_TO_OBJECT(json, "blocked", blocked); - JSON_SEND_OBJECT_UNLOCK(json); + JSON_SEND_OBJECT(json); } From 5fdfeb9ef62a4923c0db35b97f4b479f9dd62abd Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 16 Jul 2024 09:45:59 +0200 Subject: [PATCH 227/339] Add blocked domains in Query Log suggestions Signed-off-by: DL6ER --- src/api/api.h | 6 + src/api/queries.c | 65 +++++------ src/api/stats.c | 218 +++++++++++++++++++++++------------- src/webserver/json_macros.h | 25 +++++ 4 files changed, 196 insertions(+), 118 deletions(-) diff --git a/src/api/api.h b/src/api/api.h index e8e57964..e9db5409 100644 --- a/src/api/api.h +++ b/src/api/api.h @@ -33,6 +33,12 @@ int api_stats_upstreams(struct ftl_conn *api); int api_stats_top_domains(struct ftl_conn *api); int api_stats_top_clients(struct ftl_conn *api); int api_stats_recentblocked(struct ftl_conn *api); +cJSON *get_top_domains(struct ftl_conn *api, const int count, + const bool blocked, const bool domains_only); +cJSON *get_top_clients(struct ftl_conn *api, const int count, + const bool blocked, const bool clients_only, + const bool names_only); +cJSON *get_top_upstreams(struct ftl_conn *api, const bool upstreams_only); // History methods int api_history(struct ftl_conn *api); diff --git a/src/api/queries.c b/src/api/queries.c index 7874c7d6..77409438 100644 --- a/src/api/queries.c +++ b/src/api/queries.c @@ -22,6 +22,7 @@ // dbopen(false, ), dbclose() #include "database/common.h" +#if 0 static int add_strings_to_array(struct ftl_conn *api, cJSON *array1, cJSON *array2, const char *querystr, const int max_count) { @@ -80,60 +81,44 @@ static int add_strings_to_array(struct ftl_conn *api, cJSON *array1, cJSON *arra return 0; } +#endif int api_queries_suggestions(struct ftl_conn *api) { - int rc; // Does the user request a custom number of records to be included? int count = 30; get_int_var(api->request->query_string, "count", &count); // Get domains - cJSON *domain = JSON_NEW_ARRAY(); - log_debug(DEBUG_API, "Reading top domains from database"); - rc = add_strings_to_array(api, domain, NULL, "WITH CTE AS (SELECT COUNT(*) cnt, domain FROM query_storage GROUP BY domain ORDER BY cnt DESC)"\ - "SELECT d.domain FROM CTE JOIN domain_by_id d ON CTE.domain = d.id", count); - if(rc != 0) + cJSON *domain = get_top_domains(api, count, false, true); + cJSON *blocked = get_top_domains(api, count, true, true); + // Add domains from both arrays, avoiding duplicates + cJSON *entry = NULL; + cJSON_ArrayForEach(entry, blocked) { - log_err("Cannot read domains from database"); - cJSON_Delete(domain); - return rc; + // Check if the domain is already in the list + bool found = false; + cJSON *entry2 = NULL; + cJSON_ArrayForEach(entry2, domain) + { + if(strcmp(cJSON_GetStringValue(entry), cJSON_GetStringValue(entry2)) == 0) + { + found = true; + break; + } + } + if(!found) + JSON_ADD_ITEM_TO_ARRAY(domain, cJSON_Duplicate(entry, true)); } - log_debug(DEBUG_API, "Read %d domains from database", cJSON_GetArraySize(domain)); + // Free the blocked list + cJSON_Delete(blocked); // Get clients, both by IP and names - // We have to call DISTINCT() here as multiple IPs can map to and name and - // vice versa - cJSON *client_ip = JSON_NEW_ARRAY(); - cJSON *client_name = JSON_NEW_ARRAY(); - log_debug(DEBUG_API, "Reading top client IPs and names from database"); - rc = add_strings_to_array(api, client_ip, client_name, "WITH CTE AS (SELECT COUNT(*) cnt, client FROM query_storage GROUP BY client ORDER BY cnt DESC)"\ - "SELECT c.ip,c.name FROM CTE JOIN client_by_id c ON CTE.client = c.id", count); - if(rc != 0) - { - log_err("Cannot read client IPs from database"); - cJSON_Delete(domain); - cJSON_Delete(client_ip); - return rc; - } - log_debug(DEBUG_API, "Read %d client IPs and %d client names from database", cJSON_GetArraySize(client_ip), cJSON_GetArraySize(client_name)); + cJSON *client_ip = get_top_clients(api, count, false, true, false); + cJSON *client_name = get_top_clients(api, count, false, true, true); // Get upstreams - cJSON *upstream = JSON_NEW_ARRAY(); - log_debug(DEBUG_API, "Reading top upstreams from database"); - rc = add_strings_to_array(api, upstream, NULL, "WITH CTE AS (SELECT COUNT(*) cnt, forward FROM query_storage GROUP BY forward ORDER BY cnt DESC)"\ - "SELECT f.forward FROM CTE JOIN forward_by_id f ON CTE.forward = f.id", count); - if(rc != 0) - { - log_err("Cannot read forward from database"); - cJSON_Delete(domain); - cJSON_Delete(client_ip); - cJSON_Delete(client_name); - cJSON_Delete(upstream); - return rc; - } - log_debug(DEBUG_API, "Read %d upstreams from database", cJSON_GetArraySize(upstream)); - + cJSON *upstream = get_top_upstreams(api, true); // Get types cJSON *type = JSON_NEW_ARRAY(); queriesData query = { 0 }; diff --git a/src/api/stats.c b/src/api/stats.c index 09f71fc6..18227e3c 100644 --- a/src/api/stats.c +++ b/src/api/stats.c @@ -161,7 +161,8 @@ int api_stats_summary(struct ftl_conn *api) JSON_SEND_OBJECT(json); } -int api_stats_top_domains(struct ftl_conn *api) +cJSON *get_top_domains(struct ftl_conn *api, const int count, + const bool blocked, const bool domains_only) { // Exit before processing any data if requested via config setting if(config.misc.privacylevel.v.privacy_level >= PRIVACY_HIDE_DOMAINS) @@ -171,23 +172,14 @@ int api_stats_top_domains(struct ftl_conn *api) // Minimum structure is // {"top_domains":[]} - cJSON *json = JSON_NEW_OBJECT(); - cJSON *top_domains = JSON_NEW_ARRAY(); - JSON_ADD_ITEM_TO_OBJECT(json, "top_domains", top_domains); - JSON_SEND_OBJECT(json); - } + if(domains_only) + return cJSON_CreateArray(); - bool blocked = false; // Can be overwritten by query string - int count = 10; - // /api/stats/top_domains?blocked=true - if(api->request->query_string != NULL) - { - // Should blocked domains be shown? - get_bool_var(api->request->query_string, "blocked", &blocked); - - // Does the user request a non-default number of replies? - // Note: We do not accept zero query requests here - get_int_var(api->request->query_string, "count", &count); + cJSON *json = cJSON_CreateObject(); + cJSON_AddItemToObject(json, "domains", cJSON_CreateArray()); + cJSON_AddNumberToObject(json, "total_queries", -1); + cJSON_AddNumberToObject(json, "blocked_queries", -1); + return json; } // Get domains which the user doesn't want to see @@ -207,7 +199,7 @@ int api_stats_top_domains(struct ftl_conn *api) if(top_domains == NULL) { log_err("Memory allocation failed in %s()", __FUNCTION__); - return 0; + return NULL; } unsigned int added_domains = 0u; @@ -241,7 +233,7 @@ int api_stats_top_domains(struct ftl_conn *api) qsort(top_domains, added_domains, sizeof(*top_domains), cmpdesc_te); int n = 0; - cJSON *jtop_domains = JSON_NEW_ARRAY(); + cJSON *jtop_domains = cJSON_CreateArray(); // Lock shared memory lock_shm(); @@ -271,20 +263,23 @@ int api_stats_top_domains(struct ftl_conn *api) } } - if(skip_domain) + if(skip_domain || top_domains[i].count < 1) continue; - if(top_domains[i].count > 0) + if(domains_only) { - cJSON *domain_item = JSON_NEW_OBJECT(); - JSON_COPY_STR_TO_OBJECT(domain_item, "domain", domain); - JSON_ADD_NUMBER_TO_OBJECT(domain_item, "count", top_domains[i].count); - JSON_ADD_ITEM_TO_ARRAY(jtop_domains, domain_item); - n++; + cJSON_AddStringToArray(jtop_domains, domain); + } + else + { + cJSON *domain_item = cJSON_CreateObject(); + cJSON_AddStringToObject(domain_item, "domain", domain); + cJSON_AddNumberToObject(domain_item, "count", top_domains[i].count); + cJSON_AddItemToArray(jtop_domains, domain_item); } // Only count entries that are actually sent and return when we have send enough data - if(n >= count) + if(++n >= count) break; } @@ -305,19 +300,43 @@ int api_stats_top_domains(struct ftl_conn *api) free(regex_domains); } - cJSON *json = JSON_NEW_OBJECT(); - JSON_ADD_ITEM_TO_OBJECT(json, "domains", jtop_domains); + if(domains_only) + { + // Return the array of domains only + return jtop_domains; + } - JSON_ADD_NUMBER_TO_OBJECT(json, "total_queries", total_queries); - JSON_ADD_NUMBER_TO_OBJECT(json, "blocked_queries", blocked_count); + // else: Build and return full object + cJSON *json = cJSON_CreateObject(); + cJSON_AddItemToObject(json, "domains", jtop_domains); + cJSON_AddNumberToObject(json, "total_queries", total_queries); + cJSON_AddNumberToObject(json, "blocked_queries", blocked_count); + return json; +} +int api_stats_top_domains(struct ftl_conn *api) +{ + bool blocked = false; // Can be overwritten by query string + int count = 10; + // /api/stats/top_domains?blocked=true + if(api->request->query_string != NULL) + { + // Should blocked domains be shown? + get_bool_var(api->request->query_string, "blocked", &blocked); + + // Does the user request a non-default number of replies? + // Note: We do not accept zero query requests here + get_int_var(api->request->query_string, "count", &count); + } + + cJSON *json = get_top_domains(api, count, blocked, false); JSON_SEND_OBJECT(json); } -int api_stats_top_clients(struct ftl_conn *api) +cJSON *get_top_clients(struct ftl_conn *api, const int count, + const bool blocked, const bool clients_only, + const bool names_only) { - int count = 10; - // Exit before processing any data if requested via config setting if(config.misc.privacylevel.v.privacy_level >= PRIVACY_HIDE_DOMAINS_CLIENTS) { @@ -326,21 +345,14 @@ int api_stats_top_clients(struct ftl_conn *api) // Minimum structure is // {"top_clients":[]} - cJSON *json = JSON_NEW_OBJECT(); - cJSON *top_clients = JSON_NEW_ARRAY(); - JSON_ADD_ITEM_TO_OBJECT(json, "top_clients", top_clients); - JSON_SEND_OBJECT(json); - } + if(clients_only) + return cJSON_CreateArray(); - bool blocked = false; // /api/stats/top_clients?blocked=true - if(api->request->query_string != NULL) - { - // Should blocked clients be shown? - get_bool_var(api->request->query_string, "blocked", &blocked); - - // Does the user request a non-default number of replies? - // Note: We do not accept zero query requests here - get_int_var(api->request->query_string, "count", &count); + cJSON *json = cJSON_CreateObject(); + cJSON_AddItemToObject(json, "clients", cJSON_CreateArray()); + cJSON_AddNumberToObject(json, "total_queries", -1); + cJSON_AddNumberToObject(json, "blocked_queries", -1); + return json; } // Lock shared memory @@ -432,22 +444,29 @@ int api_stats_top_clients(struct ftl_conn *api) } } - if(skip_client) + if(skip_client || top_clients[i].count < 1) continue; - // Return this client if the client made at least one query - // within the most recent 24 hours - if(top_clients[i].count > 0) + if(clients_only) { - cJSON *client_item = JSON_NEW_OBJECT(); - JSON_COPY_STR_TO_OBJECT(client_item, "name", client_name); - JSON_COPY_STR_TO_OBJECT(client_item, "ip", client_ip); - JSON_ADD_NUMBER_TO_OBJECT(client_item, "count", top_clients[i].count); - JSON_ADD_ITEM_TO_ARRAY(jtop_clients, client_item); - n++; + if(names_only) + { + if(strlen(client_name) > 0) + cJSON_AddStringToArray(jtop_clients, client_name); + } + else + cJSON_AddStringToArray(jtop_clients, client_ip); + } + else + { + cJSON *client_item = cJSON_CreateObject(); + cJSON_AddStringToObject(client_item, "name", client_name); + cJSON_AddStringToObject(client_item, "ip", client_ip); + cJSON_AddNumberToObject(client_item, "count", top_clients[i].count); + cJSON_AddItemToArray(jtop_clients, client_item); } - if(n == count) + if(++n == count) break; } @@ -468,16 +487,40 @@ int api_stats_top_clients(struct ftl_conn *api) free(regex_clients); } - cJSON *json = JSON_NEW_OBJECT(); - JSON_ADD_ITEM_TO_OBJECT(json, "clients", jtop_clients); + if(clients_only) + { + // Return the array of clients only + return jtop_clients; + } - JSON_ADD_NUMBER_TO_OBJECT(json, "blocked_queries", blocked_count); - JSON_ADD_NUMBER_TO_OBJECT(json, "total_queries", total_queries); + // else: Build and return full object + cJSON *json = cJSON_CreateObject(); + cJSON_AddItemToObject(json, "clients", jtop_clients); + cJSON_AddNumberToObject(json, "total_queries", total_queries); + cJSON_AddNumberToObject(json, "blocked_queries", blocked_count); + return json; +} + +int api_stats_top_clients(struct ftl_conn *api) +{ + bool blocked = false; // Can be overwritten by query string + int count = 10; + // /api/stats/top_clients?blocked=true + if(api->request->query_string != NULL) + { + // Should blocked clients be shown? + get_bool_var(api->request->query_string, "blocked", &blocked); + + // Does the user request a non-default number of replies? + // Note: We do not accept zero query requests here + get_int_var(api->request->query_string, "count", &count); + } + + cJSON *json = get_top_clients(api, count, blocked, false, false); JSON_SEND_OBJECT(json); } - -int api_stats_upstreams(struct ftl_conn *api) +cJSON *get_top_upstreams(struct ftl_conn *api, const bool upstreams_only) { const int upstreams = counters->upstreams; const int forwarded_count = get_forwarded_count(); @@ -569,18 +612,25 @@ int api_stats_upstreams(struct ftl_conn *api) // Send data: // - always if i < 0 (special upstreams: blocklist and cache) // - only if there are any queries for all others (i > 0) - if(count > 0 || i < 0) + if(count < 1 && i >= 0) + continue; + + if(upstreams_only) + { + cJSON_AddStringToArray(jtop_upstreams, name); + } + else { cJSON *upstream = JSON_NEW_OBJECT(); - JSON_COPY_STR_TO_OBJECT(upstream, "ip", ip); - JSON_COPY_STR_TO_OBJECT(upstream, "name", name); - JSON_ADD_NUMBER_TO_OBJECT(upstream, "port", port); - JSON_ADD_NUMBER_TO_OBJECT(upstream, "count", count); + cJSON_AddStringToObject(upstream, "ip", ip); + cJSON_AddStringToObject(upstream, "name", name); + cJSON_AddNumberToObject(upstream, "port", port); + cJSON_AddNumberToObject(upstream, "count", count); cJSON *statistics = JSON_NEW_OBJECT(); - JSON_ADD_NUMBER_TO_OBJECT(statistics, "response", responsetime); - JSON_ADD_NUMBER_TO_OBJECT(statistics, "variance", uncertainty); - JSON_ADD_ITEM_TO_OBJECT(upstream, "statistics", statistics); - JSON_ADD_ITEM_TO_ARRAY(jtop_upstreams, upstream); + cJSON_AddNumberToObject(statistics, "response", responsetime); + cJSON_AddNumberToObject(statistics, "variance", uncertainty); + cJSON_AddItemToObject(upstream, "statistics", statistics); + cJSON_AddItemToArray(jtop_upstreams, upstream); } } @@ -590,12 +640,24 @@ int api_stats_upstreams(struct ftl_conn *api) // Free temporary array free(top_upstreams); - cJSON *json = JSON_NEW_OBJECT(); - JSON_ADD_ITEM_TO_OBJECT(json, "upstreams", jtop_upstreams); + if(upstreams_only) + { + // Return the array of upstreams only + return jtop_upstreams; + } - JSON_ADD_NUMBER_TO_OBJECT(json, "forwarded_queries", forwarded_count); - JSON_ADD_NUMBER_TO_OBJECT(json, "total_queries", total_queries); + // else: Build and return full object + cJSON *json = cJSON_CreateObject(); + cJSON_AddItemToObject(json, "upstreams", jtop_upstreams); + cJSON_AddNumberToObject(json, "total_queries", total_queries); + cJSON_AddNumberToObject(json, "forwarded_queries", forwarded_count); + return json; +} + +int api_stats_upstreams(struct ftl_conn *api) +{ + cJSON *json = get_top_upstreams(api, false); JSON_SEND_OBJECT(json); } diff --git a/src/webserver/json_macros.h b/src/webserver/json_macros.h index 1ad9b09d..5961c3ac 100644 --- a/src/webserver/json_macros.h +++ b/src/webserver/json_macros.h @@ -57,6 +57,29 @@ cJSON_AddItemToObject(object, key, string_item); \ }) +// Hand over allocated string to cJSON - it will thereafter take care of freeing +// it when the cJSON object is deleted +#define JSON_GIVE_STR_TO_OBJECT(object, key, string)({ \ + cJSON *string_item = NULL; \ + if(string != NULL) \ + { \ + string_item = cJSON_CreateStringReference((const char*)(string)); \ + string_item->type &= ~cJSON_IsReference; \ + } \ + else \ + { \ + string_item = cJSON_CreateNull(); \ + } \ + if(string_item == NULL) \ + { \ + cJSON_Delete(object); \ + send_http_internal_error(api); \ + log_err("JSON_GIVE_STR_TO_OBJECT FAILED (key: \"%s\", string: \"%s\")!", key, string); \ + return 500; \ + } \ + cJSON_AddItemToObject(object, key, string_item); \ +}) + #define JSON_ADD_NUMBER_TO_OBJECT(object, key, num)({ \ const double number = num; \ if(cJSON_AddNumberToObject(object, key, number) == NULL) \ @@ -260,3 +283,5 @@ cJSON *elem = cJSON_GetObjectItemCaseSensitive(obj, key); \ elem != NULL ? cJSON_IsTrue(elem) : false; \ }) + +#define cJSON_AddStringToArray(array, string) cJSON_AddItemToArray(array, cJSON_CreateString(string)) From 0a971d39b86cdaa3eedbcfabac5ff791c7232ae4 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 18 Jul 2024 16:55:17 +0200 Subject: [PATCH 228/339] Remove duplicates from client name suggestions Signed-off-by: DL6ER --- src/api/queries.c | 3 +++ src/webserver/http-common.c | 43 +++++++++++++++++++++++++++++++++++++ src/webserver/http-common.h | 1 + test/pdns/setup.sh | 6 ++++++ 4 files changed, 53 insertions(+) diff --git a/src/api/queries.c b/src/api/queries.c index 77409438..bc8dac1d 100644 --- a/src/api/queries.c +++ b/src/api/queries.c @@ -117,6 +117,9 @@ int api_queries_suggestions(struct ftl_conn *api) cJSON *client_ip = get_top_clients(api, count, false, true, false); cJSON *client_name = get_top_clients(api, count, false, true, true); + // Delete duplicate entries from client_name + cJSON_unique_array(client_name); + // Get upstreams cJSON *upstream = get_top_upstreams(api, true); // Get types diff --git a/src/webserver/http-common.c b/src/webserver/http-common.c index 2154f0f6..75e687ba 100644 --- a/src/webserver/http-common.c +++ b/src/webserver/http-common.c @@ -662,3 +662,46 @@ char *__attribute__((malloc)) escape_json(const char *string) // Return the JSON escaped string return namep; } + +// Remove duplicates from a cJSON array +// This function uses the less efficient cJSON_GetArraySize() function compared +// to cJSON_ArrayForEach() as we are going to modify the array in-place while +// iterating over it +void cJSON_unique_array(cJSON *array) +{ + // Check if the array is an array + if(!cJSON_IsArray(array)) + return; + + for(int oi = 0; oi < cJSON_GetArraySize(array); oi++) + { + // Get the outer item + cJSON *outer_item = cJSON_GetArrayItem(array, oi); + // Check if the item is a string + if (!cJSON_IsString(outer_item)) + continue; + + // Check for duplicates in the remainder of the array + for(int ii = oi + 1; ii < cJSON_GetArraySize(array); ii++) + { + // Get the inner item + cJSON *inner_item = cJSON_GetArrayItem(array, ii); + // Check if the inner item is a string + if (!cJSON_IsString(inner_item)) + continue; + + // Compare the two strings + if(strcmp(outer_item->valuestring, inner_item->valuestring) == 0) + { + // Remove the duplicate item, this is safe as we are + // at least one item ahead of the outer item + cJSON_DeleteItemFromArray(array, ii); + // Compensate for removed item (the for loop + // will increment ii for the next step, thus + // we need to decrement it here) + ii--; + continue; + } + } + } +} diff --git a/src/webserver/http-common.h b/src/webserver/http-common.h index d8bfe1af..bde500e5 100644 --- a/src/webserver/http-common.h +++ b/src/webserver/http-common.h @@ -103,5 +103,6 @@ char * __attribute__((malloc)) escape_html(const char *string); int check_json_payload(struct ftl_conn *api); int parse_groupIDs(struct ftl_conn *api, tablerow *table, cJSON *row); char * __attribute__((malloc)) escape_json(const char *string); +void cJSON_unique_array(cJSON *array); #endif // HTTP_H diff --git a/test/pdns/setup.sh b/test/pdns/setup.sh index 715d5571..7a860fef 100644 --- a/test/pdns/setup.sh +++ b/test/pdns/setup.sh @@ -124,6 +124,12 @@ pdnsutil add-record arpa. 2.1.168.192.in-addr PTR a.ftl. pdnsutil add-record arpa. 1.0.c.1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.8.e.f.ip6 PTR ftl. pdnsutil add-record arpa. 2.0.c.1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.8.e.f.ip6 PTR aaaa.ftl. +# Add DNSSEC zone +pdnsutil create-zone dnssec ns1.ftl + +# Create trust anchor +pdnsutil add-zone-key dnssec KSK active + # Calculates the ‘ordername’ and ‘auth’ fields for all zones so they comply with # DNSSEC settings. Can be used to fix up migrated data. Can always safely be # run, it does no harm. From e128d866bf96d5529f0361998b3553b854d8b928 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 18 Jul 2024 17:55:25 +0200 Subject: [PATCH 229/339] Undo unintended test change Signed-off-by: DL6ER --- test/pdns/setup.sh | 6 ------ 1 file changed, 6 deletions(-) diff --git a/test/pdns/setup.sh b/test/pdns/setup.sh index 7a860fef..715d5571 100644 --- a/test/pdns/setup.sh +++ b/test/pdns/setup.sh @@ -124,12 +124,6 @@ pdnsutil add-record arpa. 2.1.168.192.in-addr PTR a.ftl. pdnsutil add-record arpa. 1.0.c.1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.8.e.f.ip6 PTR ftl. pdnsutil add-record arpa. 2.0.c.1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.8.e.f.ip6 PTR aaaa.ftl. -# Add DNSSEC zone -pdnsutil create-zone dnssec ns1.ftl - -# Create trust anchor -pdnsutil add-zone-key dnssec KSK active - # Calculates the ‘ordername’ and ‘auth’ fields for all zones so they comply with # DNSSEC settings. Can be used to fix up migrated data. Can always safely be # run, it does no harm. From 260456c29a858c819ee609b336c576f72ddc7dcc Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 22 Jul 2024 14:01:36 +0200 Subject: [PATCH 230/339] Implement fully self-contained VALID/BOGUS DNSSEC zones for our CI tests Signed-off-by: DL6ER --- test/pdns/pdns.conf | 3 +++ test/pdns/recursor.conf | 4 ++-- test/pdns/setup.sh | 22 ++++++++++++++++++++++ test/test_suite.bats | 8 +++++++- 4 files changed, 34 insertions(+), 3 deletions(-) diff --git a/test/pdns/pdns.conf b/test/pdns/pdns.conf index 2449edf5..8cf7854c 100644 --- a/test/pdns/pdns.conf +++ b/test/pdns/pdns.conf @@ -19,5 +19,8 @@ launch=gsqlite3 # Database location gsqlite3-database=/var/lib/powerdns/pdns.sqlite3 +# Enable DNSSEC in the backend +gsqlite3-dnssec=yes + # Used when creating a new zone default-soa-content=ns1.@ hostmaster.@ 0 10800 3600 604800 3600 diff --git a/test/pdns/recursor.conf b/test/pdns/recursor.conf index 06f40946..0694e52b 100644 --- a/test/pdns/recursor.conf +++ b/test/pdns/recursor.conf @@ -10,8 +10,8 @@ # Local DNS address and port local-address=127.0.0.1:5555 -# Use authoritative server for ftl. and arpa. zones -forward-zones=ftl=127.0.0.1:5554,168.192.in-addr.arpa=127.0.0.1:5554,ip6.arpa=127.0.0.1:5554 +# Use authoritative server for ftl., dnssec. and arpa. zones +forward-zones=ftl=127.0.0.1:5554,168.192.in-addr.arpa=127.0.0.1:5554,ip6.arpa=127.0.0.1:5554,dnssec=127.0.0.1:5554,bogus=127.0.0.1:5554 # In this mode the Recursor acts as a “security aware, non-validating” # nameserver, meaning it will set the DO-bit on outgoing queries and will diff --git a/test/pdns/setup.sh b/test/pdns/setup.sh index 715d5571..7eb30126 100644 --- a/test/pdns/setup.sh +++ b/test/pdns/setup.sh @@ -117,6 +117,28 @@ pdnsutil add-record ftl. regex-notMultiple AAAA fe80::3f41 # TXT pdnsutil add-record ftl. any TXT "\"Some example text\"" +# Create valid internal DNSSEC zone +pdnsutil create-zone dnssec ns1.ftl +pdnsutil add-record dnssec. a A 192.168.4.1 +pdnsutil add-record dnssec. aaaa AAAA fe80::4c01 +pdnsutil secure-zone dnssec +# Export zone DS records and convert to dnsmasq trust-anchor format +# Example: +# dnssec. IN DS 42206 8 2 6d2007e292483fa061db37011676d9592649d1600e5b2ece1326f792ebedd412 ; ( SHA256 digest ) +# ---> +# trust-anchor=dnssec.,42206,8,2,6d2007e292483fa061db37011676d9592649d1600e5b2ece1326f792ebedd412 +pdnsutil export-zone-ds dnssec. | head -n1 | awk '{FS=" "; OFS=""; print "trust-anchor=",$1,",",$4,",",$5,",",$6,",",$7}' > /etc/dnsmasq.d/02-trust-anchor.conf + +# Create intentionally broken DNSSEC (BOGUS) zone +# The only difference to above is that this zone is signed with a key that is +# not in the trust chain +# It will cause the DNSSEC validation to fail with error message: +# unsupported DS digest +pdnsutil create-zone bogus ns1.ftl +pdnsutil add-record bogus. a A 192.168.5.1 +pdnsutil add-record bogus. aaaa AAAA fe80::5c01 +pdnsutil secure-zone bogus + # Create reverse lookup zone pdnsutil create-zone arpa ns1.ftl pdnsutil add-record arpa. 1.1.168.192.in-addr PTR ftl. diff --git a/test/test_suite.bats b/test/test_suite.bats index ca07dbf9..85791d8b 100644 --- a/test/test_suite.bats +++ b/test/test_suite.bats @@ -396,11 +396,17 @@ } @test "DNSSEC: SECURE domain is resolved" { - run bash -c "dig A dnssec.works @127.0.0.1" + run bash -c "dig A a.dnssec @127.0.0.1" printf "%s\n" "${lines[@]}" [[ ${lines[@]} == *"status: NOERROR"* ]] } +@test "DNSSEC: BOGUS domain is rejected" { + run bash -c "dig A a.bogus @127.0.0.1" + printf "%s\n" "${lines[@]}" + [[ ${lines[@]} == *"status: SERVFAIL"* ]] +} + @test "Special domain: NXDOMAIN is returned" { run bash -c "dig A mask.icloud.com @127.0.0.1" printf "%s\n" "${lines[@]}" From d6bf943be2715ceb4768c478f32293443644fbc1 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 22 Jul 2024 19:14:17 +0200 Subject: [PATCH 231/339] Remove webserver.tls.rev_server config option Signed-off-by: DL6ER --- src/api/docs/content/specs/config.yaml | 3 --- src/config/config.c | 6 ------ src/config/config.h | 1 - src/lua/ftl_lua.c | 7 ------- test/pihole.toml | 10 +--------- 5 files changed, 1 insertion(+), 26 deletions(-) diff --git a/src/api/docs/content/specs/config.yaml b/src/api/docs/content/specs/config.yaml index cdab29c6..b8894f74 100644 --- a/src/api/docs/content/specs/config.yaml +++ b/src/api/docs/content/specs/config.yaml @@ -413,8 +413,6 @@ components: tls: type: object properties: - rev_proxy: - type: boolean cert: type: string paths: @@ -739,7 +737,6 @@ components: timeout: 300 restore: true tls: - rev_proxy: false cert: "/etc/pihole/tls.pem" paths: webroot: "/var/www/html" diff --git a/src/config/config.c b/src/config/config.c index eee5a8c0..42d1f7c7 100644 --- a/src/config/config.c +++ b/src/config/config.c @@ -972,12 +972,6 @@ void initConfig(struct config *conf) conf->webserver.port.d.s = (char*)"80,[::]:80,443s,[::]:443s"; conf->webserver.port.c = validate_stub; // Type-based checking + civetweb syntax checking - conf->webserver.tls.rev_proxy.k = "webserver.tls.rev_proxy"; - conf->webserver.tls.rev_proxy.h = "Is Pi-hole running behind a reverse proxy? If yes, Pi-hole will not consider HTTP-only connections being insecure. This is useful if you are running Pi-hole in a trusted environment, for example, in a local network, and you are using a reverse proxy to provide TLS encryption, e.g., by using Traefik (docker). If you are using a reverse proxy, you can alternatively set webserver.tls.cert to the path of the TLS certificate file and let Pi-hole handle true end-to-end encryption."; - conf->webserver.tls.rev_proxy.t = CONF_BOOL; - conf->webserver.tls.rev_proxy.d.b = false; - conf->webserver.tls.rev_proxy.c = validate_stub; // Only type-based checking - conf->webserver.tls.cert.k = "webserver.tls.cert"; conf->webserver.tls.cert.h = "Path to the TLS (SSL) certificate file. This option is only required when at least one of webserver.port is TLS. The file must be in PEM format, and it must have both, private key and certificate (the *.pem file created must contain a 'CERTIFICATE' section as well as a 'RSA PRIVATE KEY' section).\n The *.pem file can be created using\n cp server.crt server.pem\n cat server.key >> server.pem\n if you have these files instead"; conf->webserver.tls.cert.a = cJSON_CreateStringReference(""); diff --git a/src/config/config.h b/src/config/config.h index 81d4a15f..0839c1b0 100644 --- a/src/config/config.h +++ b/src/config/config.h @@ -240,7 +240,6 @@ struct config { struct conf_item restore; } session; struct { - struct conf_item rev_proxy; struct conf_item cert; } tls; struct { diff --git a/src/lua/ftl_lua.c b/src/lua/ftl_lua.c index 5c8e12e6..5da8202d 100644 --- a/src/lua/ftl_lua.c +++ b/src/lua/ftl_lua.c @@ -240,12 +240,6 @@ static int pihole_needLogin(lua_State *L) { return 1; // number of results } -// pihole.rev_proxy() -static int pihole_rev_proxy(lua_State *L) { - lua_pushboolean(L, config.webserver.tls.rev_proxy.v.b); - return 1; // number of results -} - static const luaL_Reg piholelib[] = { {"ftl_version", pihole_ftl_version}, {"hostname", pihole_hostname}, @@ -255,7 +249,6 @@ static const luaL_Reg piholelib[] = { {"include", pihole_include}, {"boxedlayout", pihole_boxedlayout}, {"needLogin", pihole_needLogin}, - {"rev_proxy", pihole_rev_proxy}, {NULL, NULL} }; diff --git a/test/pihole.toml b/test/pihole.toml index 4dbe9889..c6514e6f 100644 --- a/test/pihole.toml +++ b/test/pihole.toml @@ -649,14 +649,6 @@ restore = true [webserver.tls] - # Is Pi-hole running behind a reverse proxy? If yes, Pi-hole will not consider - # HTTP-only connections being insecure. This is useful if you are running Pi-hole in a - # trusted environment, for example, in a local network, and you are using a reverse - # proxy to provide TLS encryption, e.g., by using Traefik (docker). If you are using a - # reverse proxy, you can alternatively set webserver.tls.cert to the path of the TLS - # certificate file and let Pi-hole handle true end-to-end encryption. - rev_proxy = false - # Path to the TLS (SSL) certificate file. This option is only required when at least # one of webserver.port is TLS. The file must be in PEM format, and it must have both, # private key and certificate (the *.pem file created must contain a 'CERTIFICATE' @@ -1105,7 +1097,7 @@ all = true ### CHANGED, default = false # Configuration statistics: -# 149 total entries out of which 94 entries are default +# 148 total entries out of which 93 entries are default # --> 55 entries are modified # 2 entries are forced through environment: # - misc.nice From 64144a960caeed715f838231df795378af09096b Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 23 Jul 2024 16:29:47 +0200 Subject: [PATCH 232/339] Add session.x_forwarded_for property to API sessions Signed-off-by: DL6ER --- src/api/auth.c | 24 ++++++++-- src/api/auth.h | 1 + src/api/docs/content/specs/auth.yaml | 8 +++- src/database/common.c | 15 ++++++ src/database/query-table.h | 2 +- src/database/session-table.c | 70 +++++++++++++++++++++------- src/database/session-table.h | 1 + test/test_suite.bats | 4 +- 8 files changed, 101 insertions(+), 24 deletions(-) diff --git a/src/api/auth.c b/src/api/auth.c index b0abf590..2aea675e 100644 --- a/src/api/auth.c +++ b/src/api/auth.c @@ -299,7 +299,14 @@ static int get_all_sessions(struct ftl_conn *api, cJSON *json) JSON_ADD_NUMBER_TO_OBJECT(session, "last_active", auth_data[i].valid_until - config.webserver.session.timeout.v.ui); JSON_ADD_NUMBER_TO_OBJECT(session, "valid_until", auth_data[i].valid_until); JSON_REF_STR_IN_OBJECT(session, "remote_addr", auth_data[i].remote_addr); - JSON_REF_STR_IN_OBJECT(session, "user_agent", auth_data[i].user_agent); + if(auth_data[i].user_agent[0] != '\0') + JSON_REF_STR_IN_OBJECT(session, "user_agent", auth_data[i].user_agent); + else + JSON_ADD_NULL_TO_OBJECT(session, "user_agent"); + if(auth_data[i].x_forwarded_for[0] != '\0') + JSON_REF_STR_IN_OBJECT(session, "x_forwarded_for", auth_data[i].x_forwarded_for); + else + JSON_ADD_NULL_TO_OBJECT(session, "x_forwarded_for"); JSON_ADD_BOOL_TO_OBJECT(session, "app", auth_data[i].app); JSON_ADD_BOOL_TO_OBJECT(session, "cli", auth_data[i].cli); JSON_ADD_ITEM_TO_ARRAY(sessions, session); @@ -523,7 +530,7 @@ int api_auth(struct ftl_conn *api) if(result == PASSWORD_CORRECT || result == APPPASSWORD_CORRECT || - result ==CLIPASSWORD_CORRECT) + result == CLIPASSWORD_CORRECT) { // Accepted @@ -575,7 +582,7 @@ int api_auth(struct ftl_conn *api) auth_data[i].valid_until < now) { log_debug(DEBUG_API, "API: Session of client %u (%s) expired, freeing...", - i, auth_data[i].remote_addr); + i, auth_data[i].remote_addr); delete_session(i); } @@ -601,6 +608,17 @@ int api_auth(struct ftl_conn *api) { auth_data[i].user_agent[0] = '\0'; } + // Store X-Forwarded-For (if available) + const char *x_forwarded_for = mg_get_header(api->conn, "X-Forwarded-For"); + if(x_forwarded_for != NULL) + { + strncpy(auth_data[i].x_forwarded_for, x_forwarded_for, sizeof(auth_data[i].x_forwarded_for)); + auth_data[i].x_forwarded_for[sizeof(auth_data[i].x_forwarded_for)-1] = '\0'; + } + else + { + auth_data[i].x_forwarded_for[0] = '\0'; + } auth_data[i].tls.login = api->request->is_ssl; auth_data[i].tls.mixed = false; diff --git a/src/api/auth.h b/src/api/auth.h index 28af5b00..9f5cc9b5 100644 --- a/src/api/auth.h +++ b/src/api/auth.h @@ -57,6 +57,7 @@ struct session { time_t valid_until; char remote_addr[48]; // Large enough for IPv4 and IPv6 addresses, hard-coded in civetweb.h as mg_request_info.remote_addr char user_agent[128]; + char x_forwarded_for[48]; // see remote_addr note char sid[SID_SIZE]; char csrf[SID_SIZE]; }; diff --git a/src/api/docs/content/specs/auth.yaml b/src/api/docs/content/specs/auth.yaml index 9b206517..4d75549c 100644 --- a/src/api/docs/content/specs/auth.yaml +++ b/src/api/docs/content/specs/auth.yaml @@ -362,7 +362,12 @@ components: description: IP address of the client user_agent: type: string - description: User agent of the client + nullable: true + description: User agent of the client (optional) + x_forwarded_for: + type: string + nullable: true + description: IP address of the client (if behind a proxy, optional) example: - id: 1 current_session: true @@ -377,6 +382,7 @@ components: valid_until: 1580000300 remote_addr: "192.168.0.34" user_agent: "Mozilla/5.0 (X11; Linux x86_64; rv:107.0) Gecko/20100101 Firefox/107.0" + x_forwarded_for: null totp: type: object description: TOTP secret suggestion diff --git a/src/database/common.c b/src/database/common.c index 85ee0dbc..08880f18 100644 --- a/src/database/common.c +++ b/src/database/common.c @@ -590,6 +590,21 @@ void db_init(void) dbversion = db_get_int(db, DB_VERSION); } + // Update to version 19 if lower + if(dbversion < 19) + { + // Update to version 19: Add x_forwarded_for column to session table + log_info("Updating long-term database to version 19"); + if(!add_session_x_forwarded_for_column(db)) + { + log_info("Session table cannot be updated, database not available"); + dbclose(&db); + return; + } + // Get updated version + dbversion = db_get_int(db, DB_VERSION); + } + /* * * * * * * * * * * * * IMPORTANT * * * * * * * * * * * * * * If you add a new database version, check if the in-memory * schema needs to be update as well (always recreated from diff --git a/src/database/query-table.h b/src/database/query-table.h index 1a5b8b5a..50d824f6 100644 --- a/src/database/query-table.h +++ b/src/database/query-table.h @@ -23,7 +23,7 @@ "client TEXT NOT NULL, " \ "forward TEXT );" -#define MEMDB_VERSION 18 +#define MEMDB_VERSION 19 #define CREATE_QUERY_STORAGE_TABLE "CREATE TABLE query_storage ( id INTEGER PRIMARY KEY AUTOINCREMENT, " \ "timestamp INTEGER NOT NULL, " \ "type INTEGER NOT NULL, " \ diff --git a/src/database/session-table.c b/src/database/session-table.c index 04654088..0ee28e3c 100644 --- a/src/database/session-table.c +++ b/src/database/session-table.c @@ -86,6 +86,27 @@ bool add_session_cli_column(sqlite3 *db) return true; } +bool add_session_x_forwarded_for_column(sqlite3 *db) +{ + // Start transaction of database update + SQL_bool(db, "BEGIN TRANSACTION;"); + + // Create session table + SQL_bool(db, "ALTER TABLE session ADD COLUMN x_forwarded_for TEXT;"); + + // Update database version to 18 + if(!db_set_FTL_property(db, DB_VERSION, 19)) + { + log_err("add_session_x_forwarded_for_column(): Failed to update database version!"); + return false; + } + + // Finish transaction + SQL_bool(db, "COMMIT"); + + return true; +} + // Store all session in database bool backup_db_sessions(struct session *sessions, const uint16_t max_sessions) { @@ -104,7 +125,7 @@ bool backup_db_sessions(struct session *sessions, const uint16_t max_sessions) // Insert session into database sqlite3_stmt *stmt = NULL; - if(sqlite3_prepare_v2(db, "INSERT INTO session (login_at, valid_until, remote_addr, user_agent, sid, csrf, tls_login, tls_mixed, app, cli) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?);", -1, &stmt, 0) != SQLITE_OK) + if(sqlite3_prepare_v2(db, "INSERT INTO session (login_at, valid_until, remote_addr, user_agent, sid, csrf, tls_login, tls_mixed, app, cli, x_forwarded_for) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);", -1, &stmt, 0) != SQLITE_OK) { log_err("SQL error in backup_db_sessions(): %s (%d)", sqlite3_errmsg(db), sqlite3_errcode(db)); @@ -126,70 +147,77 @@ bool backup_db_sessions(struct session *sessions, const uint16_t max_sessions) if(sqlite3_bind_int64(stmt, 1, sess->login_at) != SQLITE_OK) { log_err("Cannot bind login_at = %ld in backup_db_sessions(): %s (%d)", - (long int)sess->login_at, sqlite3_errmsg(db), sqlite3_errcode(db)); + (long int)sess->login_at, sqlite3_errmsg(db), sqlite3_errcode(db)); return false; } // 2: valid_until if(sqlite3_bind_int64(stmt, 2, sess->valid_until) != SQLITE_OK) { log_err("Cannot bind valid_until = %ld in backup_db_sessions(): %s (%d)", - (long int)sess->valid_until, sqlite3_errmsg(db), sqlite3_errcode(db)); + (long int)sess->valid_until, sqlite3_errmsg(db), sqlite3_errcode(db)); return false; } // 3: remote_addr if(sqlite3_bind_text(stmt, 3, sess->remote_addr, -1, SQLITE_STATIC) != SQLITE_OK) { log_err("Cannot bind remote_addr = %s in backup_db_sessions(): %s (%d)", - sess->remote_addr, sqlite3_errmsg(db), sqlite3_errcode(db)); + sess->remote_addr, sqlite3_errmsg(db), sqlite3_errcode(db)); return false; } // 4: user_agent if(sqlite3_bind_text(stmt, 4, sess->user_agent, -1, SQLITE_STATIC) != SQLITE_OK) { log_err("Cannot bind user_agent = %s in backup_db_sessions(): %s (%d)", - sess->user_agent, sqlite3_errmsg(db), sqlite3_errcode(db)); + sess->user_agent, sqlite3_errmsg(db), sqlite3_errcode(db)); return false; } // 5: sid if(sqlite3_bind_text(stmt, 5, sess->sid, -1, SQLITE_STATIC) != SQLITE_OK) { log_err("Cannot bind sid = %s in backup_db_sessions(): %s (%d)", - sess->sid, sqlite3_errmsg(db), sqlite3_errcode(db)); + sess->sid, sqlite3_errmsg(db), sqlite3_errcode(db)); return false; } // 6: csrf if(sqlite3_bind_text(stmt, 6, sess->csrf, -1, SQLITE_STATIC) != SQLITE_OK) { log_err("Cannot bind csrf = %s in backup_db_sessions(): %s (%d)", - sess->csrf, sqlite3_errmsg(db), sqlite3_errcode(db)); + sess->csrf, sqlite3_errmsg(db), sqlite3_errcode(db)); return false; } // 7: tls_login if(sqlite3_bind_int(stmt, 7, sess->tls.login ? 1 : 0) != SQLITE_OK) { log_err("Cannot bind tls_login = %d in backup_db_sessions(): %s (%d)", - sess->tls.login ? 1 : 0, sqlite3_errmsg(db), sqlite3_errcode(db)); + sess->tls.login ? 1 : 0, sqlite3_errmsg(db), sqlite3_errcode(db)); return false; } // 8: tls_mixed if(sqlite3_bind_int(stmt, 8, sess->tls.mixed ? 1 : 0) != SQLITE_OK) { log_err("Cannot bind tls_mixed = %d in backup_db_sessions(): %s (%d)", - sess->tls.mixed ? 1 : 0, sqlite3_errmsg(db), sqlite3_errcode(db)); + sess->tls.mixed ? 1 : 0, sqlite3_errmsg(db), sqlite3_errcode(db)); return false; } // 9: app if(sqlite3_bind_int(stmt, 9, sess->app ? 1 : 0) != SQLITE_OK) { log_err("Cannot bind app = %d in backup_db_sessions(): %s (%d)", - sess->app ? 1 : 0, sqlite3_errmsg(db), sqlite3_errcode(db)); + sess->app ? 1 : 0, sqlite3_errmsg(db), sqlite3_errcode(db)); return false; } // 10: cli if(sqlite3_bind_int(stmt, 10, sess->cli ? 1 : 0) != SQLITE_OK) { log_err("Cannot bind cli = %d in backup_db_sessions(): %s (%d)", - sess->cli ? 1 : 0, sqlite3_errmsg(db), sqlite3_errcode(db)); + sess->cli ? 1 : 0, sqlite3_errmsg(db), sqlite3_errcode(db)); + return false; + } + // 11: x_forwarded_for + if(sqlite3_bind_text(stmt, 11, sess->x_forwarded_for, -1, SQLITE_STATIC) != SQLITE_OK) + { + log_err("Cannot bind x_forwarded_for = %s in backup_db_sessions(): %s (%d)", + sess->x_forwarded_for, sqlite3_errmsg(db), sqlite3_errcode(db)); return false; } @@ -197,7 +225,7 @@ bool backup_db_sessions(struct session *sessions, const uint16_t max_sessions) if(sqlite3_step(stmt) != SQLITE_DONE) { log_err("SQL error in backup_db_sessions(): %s (%d)", - sqlite3_errmsg(db), sqlite3_errcode(db)); + sqlite3_errmsg(db), sqlite3_errcode(db)); return false; } @@ -205,7 +233,7 @@ bool backup_db_sessions(struct session *sessions, const uint16_t max_sessions) if(sqlite3_clear_bindings(stmt) != SQLITE_OK) { log_err("SQL error in backup_db_sessions(): %s (%d)", - sqlite3_errmsg(db), sqlite3_errcode(db)); + sqlite3_errmsg(db), sqlite3_errcode(db)); return false; } @@ -213,7 +241,7 @@ bool backup_db_sessions(struct session *sessions, const uint16_t max_sessions) if(sqlite3_reset(stmt) != SQLITE_OK) { log_err("SQL error in backup_db_sessions(): %s (%d)", - sqlite3_errmsg(db), sqlite3_errcode(db)); + sqlite3_errmsg(db), sqlite3_errcode(db)); return false; } @@ -253,7 +281,7 @@ bool restore_db_sessions(struct session *sessions, const uint16_t max_sessions) // Get all sessions from database sqlite3_stmt *stmt = NULL; - if(sqlite3_prepare_v2(memdb, "SELECT login_at, valid_until, remote_addr, user_agent, sid, csrf, tls_login, tls_mixed, app, cli FROM disk.session;", -1, &stmt, 0) != SQLITE_OK) + if(sqlite3_prepare_v2(memdb, "SELECT login_at, valid_until, remote_addr, user_agent, sid, csrf, tls_login, tls_mixed, app, cli, x_forwarded_for FROM disk.session;", -1, &stmt, 0) != SQLITE_OK) { log_err("SQL error in restore_db_sessions(): %s (%d)", sqlite3_errmsg(memdb), sqlite3_errcode(memdb)); @@ -312,12 +340,20 @@ bool restore_db_sessions(struct session *sessions, const uint16_t max_sessions) // 8: tls_mixed sess->tls.mixed = sqlite3_column_int(stmt, 7) == 1 ? true : false; - // 8: app + // 9: app sess->app = sqlite3_column_int(stmt, 8) == 1 ? true : false; - // 9: cli + // 10: cli sess->cli = sqlite3_column_int(stmt, 9) == 1 ? true : false; + // 11: x_forwarded_for + const char *x_forwarded_for = (const char *)sqlite3_column_text(stmt, 10); + if(x_forwarded_for != NULL) + { + strncpy(sess->x_forwarded_for, x_forwarded_for, sizeof(sess->x_forwarded_for)-1); + sess->x_forwarded_for[sizeof(sess->x_forwarded_for)-1] = '\0'; + } + // Mark session as used sess->used = true; diff --git a/src/database/session-table.h b/src/database/session-table.h index fbdaea41..0aa79b19 100644 --- a/src/database/session-table.h +++ b/src/database/session-table.h @@ -17,6 +17,7 @@ bool create_session_table(sqlite3 *db); bool add_session_app_column(sqlite3 *db); bool add_session_cli_column(sqlite3 *db); +bool add_session_x_forwarded_for_column(sqlite3 *db); bool backup_db_sessions(struct session *sessions, const uint16_t max_sessions); bool restore_db_sessions(struct session *sessions, const uint16_t max_sessions); diff --git a/test/test_suite.bats b/test/test_suite.bats index 85791d8b..b7fcf867 100644 --- a/test/test_suite.bats +++ b/test/test_suite.bats @@ -440,7 +440,7 @@ [[ "${lines[@]}" == *"CREATE TABLE IF NOT EXISTS \"network\" (id INTEGER PRIMARY KEY NOT NULL, hwaddr TEXT UNIQUE NOT NULL, interface TEXT NOT NULL, firstSeen INTEGER NOT NULL, lastQuery INTEGER NOT NULL, numQueries INTEGER NOT NULL, macVendor TEXT, aliasclient_id INTEGER);"* ]] [[ "${lines[@]}" == *"CREATE TABLE IF NOT EXISTS \"network_addresses\" (network_id INTEGER NOT NULL, ip TEXT UNIQUE NOT NULL, lastSeen INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), name TEXT, nameUpdated INTEGER, FOREIGN KEY(network_id) REFERENCES network(id));"* ]] [[ "${lines[@]}" == *"CREATE TABLE aliasclient (id INTEGER PRIMARY KEY NOT NULL, name TEXT NOT NULL, comment TEXT);"* ]] - [[ "${lines[@]}" == *"INSERT INTO ftl VALUES(0,18,'Database version');"* ]] + [[ "${lines[@]}" == *"INSERT INTO ftl VALUES(0,19,'Database version');"* ]] # vvv This has been added in version 10 vvv [[ "${lines[@]}" == *"CREATE VIEW queries AS SELECT id, timestamp, type, status, CASE typeof(domain) WHEN 'integer' THEN (SELECT domain FROM domain_by_id d WHERE d.id = q.domain) ELSE domain END domain,CASE typeof(client) WHEN 'integer' THEN (SELECT ip FROM client_by_id c WHERE c.id = q.client) ELSE client END client,CASE typeof(forward) WHEN 'integer' THEN (SELECT forward FROM forward_by_id f WHERE f.id = q.forward) ELSE forward END forward,CASE typeof(additional_info) WHEN 'integer' THEN (SELECT content FROM addinfo_by_id a WHERE a.id = q.additional_info) ELSE additional_info END additional_info, reply_type, reply_time, dnssec, list_id FROM query_storage q;"* ]] [[ "${lines[@]}" == *"CREATE TABLE domain_by_id (id INTEGER PRIMARY KEY, domain TEXT NOT NULL);"* ]] @@ -452,7 +452,7 @@ [[ "${lines[@]}" == *"CREATE TABLE addinfo_by_id (id INTEGER PRIMARY KEY, type INTEGER NOT NULL, content NOT NULL);"* ]] [[ "${lines[@]}" == *"CREATE UNIQUE INDEX addinfo_by_id_idx ON addinfo_by_id(type,content);"* ]] # vvv This has been added in version 15 vvv - [[ "${lines[@]}" == *"CREATE TABLE session (id INTEGER PRIMARY KEY, login_at TIMESTAMP NOT NULL, valid_until TIMESTAMP NOT NULL, remote_addr TEXT NOT NULL, user_agent TEXT, sid TEXT NOT NULL, csrf TEXT NOT NULL, tls_login BOOL, tls_mixed BOOL, app BOOL, cli BOOL);"* ]] + [[ "${lines[@]}" == *"CREATE TABLE session (id INTEGER PRIMARY KEY, login_at TIMESTAMP NOT NULL, valid_until TIMESTAMP NOT NULL, remote_addr TEXT NOT NULL, user_agent TEXT, sid TEXT NOT NULL, csrf TEXT NOT NULL, tls_login BOOL, tls_mixed BOOL, app BOOL, cli BOOL, x_forwarded_for TEXT);"* ]] } @test "Ownership, permissions and type of pihole-FTL.db correct" { From d3e4f108d6bf4c916672410a5f99f3596965cf74 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 22 Jul 2024 11:01:02 +0200 Subject: [PATCH 233/339] Fix "Conditional jump or move depends on uninitialised value(s)" in dnssec_validate_ds() when debug.queries == true Signed-off-by: DL6ER --- src/dnsmasq_interface.c | 2 +- src/signals.c | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c index 8a5f15af..79e3c275 100644 --- a/src/dnsmasq_interface.c +++ b/src/dnsmasq_interface.c @@ -1961,7 +1961,7 @@ static void FTL_reply(const unsigned int flags, const char *name, const union al const char *answer = arg; // Determine returned address (if applicable) char dest[ADDRSTRLEN]; dest[0] = '\0'; - if(addr) + if(addr && flags & (F_IPV4 | F_IPV6)) { inet_ntop((flags & F_IPV4) ? AF_INET : AF_INET6, addr, dest, ADDRSTRLEN); answer = dest; // Overwrite answer with human-readable IP address diff --git a/src/signals.c b/src/signals.c index 34f6b52c..d4dc2356 100644 --- a/src/signals.c +++ b/src/signals.c @@ -323,6 +323,8 @@ static void SIGRT_handler(int signum, siginfo_t *si, void *unused) // // Signal internally used to signal dnsmasq it has to stop // } + // SIGRT32: Used internally by valgrind, do not use + // Restore errno before returning back to previous context errno = _errno; } From 092a68dedb5763d7ed558e56f755a369058ff010 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 22 Jul 2024 11:37:22 +0200 Subject: [PATCH 234/339] Remove minor (fre bytes) memory-leaks in resolver code. Ensure to finalize Sqlite3 vector statements during shutdown so valgrind doesn't consider them as "possibly lost" memory Signed-off-by: DL6ER --- src/FTL.h | 2 +- src/database/gravity-db.c | 3 ++- src/resolve.c | 11 ++++++++++- src/signals.c | 4 ++++ src/signals.h | 1 + src/vector.c | 10 ++++++++++ 6 files changed, 28 insertions(+), 3 deletions(-) diff --git a/src/FTL.h b/src/FTL.h index 69b88b5d..5a76e979 100644 --- a/src/FTL.h +++ b/src/FTL.h @@ -48,7 +48,7 @@ // MIN(x,y) is already defined in dnsmasq.h // Number of elements in an array -#define ArraySize(X) (sizeof(X)/sizeof(X[0])) +#define ArraySize(X) (sizeof(X)/sizeof(*X)) // Constant socket buffer length #define SOCKETBUFFERLEN 1024 diff --git a/src/database/gravity-db.c b/src/database/gravity-db.c index f0ac7398..4d2d5705 100644 --- a/src/database/gravity-db.c +++ b/src/database/gravity-db.c @@ -955,6 +955,7 @@ void gravityDB_close(void) free_sqlite3_stmt_vec(&antigravity_stmt); // Close table + log_debug(DEBUG_ANY, "Closing gravity database"); sqlite3_close(gravity_db); gravity_db = NULL; gravityDB_opened = false; @@ -1046,7 +1047,7 @@ inline const char* gravityDB_getDomain(int *rowid) // Finalize statement of a gravity database transaction void gravityDB_finalizeTable(void) { - if(!gravityDB_opened) + if(!gravityDB_opened || table_stmt == NULL) return; // Finalize statement diff --git a/src/resolve.c b/src/resolve.c index a99c93ac..0b2a9864 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -395,7 +395,7 @@ static char *__attribute__((malloc)) ngethostbyname(const int sock, const bool t // Start reading answers uint16_t stop = 0; char *name = NULL; - for(uint16_t i = 0; i < ntohs(dns->ans_count); i++) + for(uint16_t i = 0; i < min(ntohs(dns->ans_count), ArraySize(answers)); i++) { answers[i].name = name_fromDNS(reader, buf, &stop); reader = reader + stop; @@ -433,6 +433,15 @@ static char *__attribute__((malloc)) ngethostbyname(const int sock, const bool t } } + // Free memory + for(uint16_t i = 0; i < min(ntohs(dns->ans_count), ArraySize(answers)); i++) + { + if(answers[i].name != NULL) + free(answers[i].name); + if(answers[i].rdata != NULL && (char*)answers[i].rdata != name) + free(answers[i].rdata); + } + if(name != NULL) { // We have a valid hostname, return it diff --git a/src/signals.c b/src/signals.c index d4dc2356..1b469988 100644 --- a/src/signals.c +++ b/src/signals.c @@ -454,6 +454,10 @@ void handle_realtime_signals(void) // Skip SIGUSR6 as it is used internally to signify // dnsmasq to stop continue; + if(signum == SIGUSR32) + // Skip SIGUSR32 as it is used internally by valgrind + // and should not be used + continue; struct sigaction SIGACTION = { 0 }; SIGACTION.sa_flags = SA_SIGINFO; diff --git a/src/signals.h b/src/signals.h index 4e388dd9..eea8046b 100644 --- a/src/signals.h +++ b/src/signals.h @@ -13,6 +13,7 @@ #include "enums.h" #define SIGUSR6 (SIGRTMIN + 6) +#define SIGUSR32 (SIGRTMIN + 32) // defined in dnsmasq/dnsmasq.h extern volatile char FTL_terminate; diff --git a/src/vector.c b/src/vector.c index 2abb617f..431498ef 100644 --- a/src/vector.c +++ b/src/vector.c @@ -118,6 +118,16 @@ void free_sqlite3_stmt_vec(sqlite3_stmt_vec **v) if(v == NULL || *v == NULL || (*v)->items == NULL) return; + // Run sqlite3_finalize on all statements in the vector + for(unsigned int i = 0; i < (*v)->capacity; i++) + { + if((*v)->items[i] != NULL) + { + log_debug(DEBUG_VECTORS, "Finalizing sqlite3_stmt** %p[%u] --> %p", *v, i, (*v)->items[i]); + sqlite3_finalize((*v)->items[i]); + } + } + // Free elements of the vector... free((*v)->items); // ...and then the vector itself From ca1c870e6d4bf8a618abaaabd6f16d3306c1555e Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 22 Jul 2024 11:54:44 +0200 Subject: [PATCH 235/339] Fix possible memory leak when handling special domains Signed-off-by: DL6ER --- src/database/gravity-db.c | 5 ++++- src/database/gravity-db.h | 1 - src/datastructure.c | 2 +- src/dnsmasq_interface.c | 1 + 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/database/gravity-db.c b/src/database/gravity-db.c index 4d2d5705..509907e6 100644 --- a/src/database/gravity-db.c +++ b/src/database/gravity-db.c @@ -50,6 +50,9 @@ static sqlite3_stmt* table_stmt = NULL; bool gravityDB_opened = false; static bool gravity_abp_format = false; +// Private prototypes +static bool gravityDB_open(void); + // Table names corresponding to the enum defined in gravity-db.h static const char* tablename[] = { "vw_gravity", "vw_blacklist", "vw_whitelist", "vw_regex_blacklist", "vw_regex_whitelist" , "client", "group", "adlist", "denied_domains", "allowed_domains", "" }; @@ -133,7 +136,7 @@ static void gravity_check_ABP_format(void) } // Open gravity database -bool gravityDB_open(void) +static bool gravityDB_open(void) { struct stat st; if(stat(config.files.gravity.v.s, &st) != 0) diff --git a/src/database/gravity-db.h b/src/database/gravity-db.h index 6ce75fd5..7bded00c 100644 --- a/src/database/gravity-db.h +++ b/src/database/gravity-db.h @@ -39,7 +39,6 @@ typedef struct { time_t date_updated; } tablerow; -bool gravityDB_open(void); bool gravityDB_reopen(void); void gravityDB_forked(void); void gravityDB_reload_groups(clientsData* client); diff --git a/src/datastructure.c b/src/datastructure.c index 6a48c9a5..28a7be4b 100644 --- a/src/datastructure.c +++ b/src/datastructure.c @@ -594,7 +594,7 @@ void FTL_reload_all_domainlists(void) counters->database.domains.denied = gravityDB_count(ALLOWED_DOMAINS_TABLE); // Read and compile possible regex filters - // only after having called gravityDB_open() + // only after having called gravityDB_reopen() read_regex_from_database(); // Check for inaccessible adlist URLs diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c index 79e3c275..8a46e5bd 100644 --- a/src/dnsmasq_interface.c +++ b/src/dnsmasq_interface.c @@ -1459,6 +1459,7 @@ static bool _FTL_check_blocking(int queryID, int domainID, int clientID, const c // Debug output log_debug(DEBUG_QUERIES, "Special domain: %s is %s", domainstr, blockingreason); + free(domainstr); return true; } From 966650c1697325f6cd24f28ba38536742e249a3d Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 22 Jul 2024 12:24:28 +0200 Subject: [PATCH 236/339] Memorize parent memory pointers when reopening gravity database in forks to avoid false-positive memory leak reports in valgrind(memcheck) Signed-off-by: DL6ER --- src/database/gravity-db.c | 13 +++++++++++++ src/timers.c | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/database/gravity-db.c b/src/database/gravity-db.c index 509907e6..f6744b0b 100644 --- a/src/database/gravity-db.c +++ b/src/database/gravity-db.c @@ -50,6 +50,14 @@ static sqlite3_stmt* table_stmt = NULL; bool gravityDB_opened = false; static bool gravity_abp_format = false; +// Variables memorizing the parent gravity database connection and prepared +// statements to avoid valgrind warnings about memory leaks +static sqlite3 *parent_gravity_db = NULL; +sqlite3_stmt_vec *parent_whitelist_stmt = NULL; +sqlite3_stmt_vec *parent_gravity_stmt = NULL; +sqlite3_stmt_vec *parent_antigravity_stmt = NULL; +sqlite3_stmt_vec *parent_blacklist_stmt = NULL; + // Private prototypes static bool gravityDB_open(void); @@ -89,12 +97,17 @@ void gravityDB_forked(void) // is clear that this in not what we want to do as this is a slow // process and many TCP queries could lead to a DoS attack. gravityDB_opened = false; + parent_gravity_db = gravity_db; gravity_db = NULL; // Also pretend we have not yet prepared the list statements + parent_whitelist_stmt = whitelist_stmt; whitelist_stmt = NULL; + parent_blacklist_stmt = blacklist_stmt; blacklist_stmt = NULL; + parent_gravity_stmt = gravity_stmt; gravity_stmt = NULL; + parent_antigravity_stmt = antigravity_stmt; antigravity_stmt = NULL; // Open the database diff --git a/src/timers.c b/src/timers.c index b841d3be..d153ccdc 100644 --- a/src/timers.c +++ b/src/timers.c @@ -66,7 +66,7 @@ void sleepms(const int milliseconds) } static double timer_delay = -1.0; -static bool timer_target_status; +static bool timer_target_status = true; void set_blockingmode_timer(double delay, bool target_status) { From f32b68ab3a8ca1e49a32b2ed3d422e9a95ed68be Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 18 Jul 2024 17:01:33 +0200 Subject: [PATCH 237/339] Add undocumented sigtest feature Signed-off-by: DL6ER --- src/args.c | 4 ++++ src/signals.c | 32 ++++++++++++++++++++++++++++++++ src/signals.h | 1 + 3 files changed, 37 insertions(+) diff --git a/src/args.c b/src/args.c index a261d3be..2a6c2b8b 100644 --- a/src/args.c +++ b/src/args.c @@ -201,6 +201,10 @@ void parse_args(int argc, char* argv[]) if(strEndsWith(argv[0], "luac")) exit(run_luac(argc, argv)); + // Special (undocumented) mode to test kernel signal handling + if(argc == 2 && strcmp(argv[1], "sigtest") == 0) + exit(sigtest()); + // If the binary name is "sqlite3" (e.g., symlink /usr/bin/sqlite3 -> /usr/bin/pihole-FTL), // we operate in drop-in mode and consume all arguments for the embedded SQLite3 engine // Also, we do this if the first argument is a file with ".db" ending diff --git a/src/signals.c b/src/signals.c index 1b469988..ee413d9a 100644 --- a/src/signals.c +++ b/src/signals.c @@ -487,3 +487,35 @@ void thread_sleepms(const enum thread_types thread, const int milliseconds) sleepms(milliseconds); thread_cancellable[thread] = false; } + +static void print_signal(int signum, siginfo_t *si, void *unused) +{ + printf("Received signal %d: \"%s\"\n", signum, strsignal(signum)); + fflush(stdin); + if(signum == SIGTERM) + exit(EXIT_SUCCESS); +} + +// Register handler that catches *all* signals and displays them +int sigtest(void) +{ + printf("PID: %d\n", getpid()); + // Catch all real-time signals + for(int signum = 0; signum <= SIGRTMAX; signum++) + { + struct sigaction SIGACTION = { 0 }; + SIGACTION.sa_flags = SA_SIGINFO; + sigemptyset(&SIGACTION.sa_mask); + SIGACTION.sa_sigaction = &print_signal; + sigaction(signum, &SIGACTION, NULL); + } + + printf("Waiting (30sec)...\n"); + fflush(stdin); + + // Sleep here for 30 seconds + sleepms(30000); + + // Exit successfully + return EXIT_SUCCESS; +} diff --git a/src/signals.h b/src/signals.h index eea8046b..7201fbf3 100644 --- a/src/signals.h +++ b/src/signals.h @@ -23,6 +23,7 @@ void handle_realtime_signals(void); pid_t main_pid(void); void thread_sleepms(const enum thread_types thread, const int milliseconds); void generate_backtrace(void); +int sigtest(void); extern volatile int exit_code; extern volatile sig_atomic_t killed; From 8b566e37f063a6fc983ee1408e10f87735bc3bb1 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 22 Jul 2024 10:01:09 +0200 Subject: [PATCH 238/339] Add extra debug logging and reduce code duplication in signal handling (no functional change) Signed-off-by: DL6ER --- src/dnsmasq/dnsmasq.c | 3 +++ src/signals.c | 22 +++++++++------------- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/src/dnsmasq/dnsmasq.c b/src/dnsmasq/dnsmasq.c index 7990ed61..97d2d878 100644 --- a/src/dnsmasq/dnsmasq.c +++ b/src/dnsmasq/dnsmasq.c @@ -28,6 +28,8 @@ #include "signals.h" // FTL_fork_and_bind_sockets() #include "main.h" +// log_debug() +#include "log.h" struct daemon *daemon; @@ -1313,6 +1315,7 @@ int main_dnsmasq (int argc, char **argv) static void sig_handler(int sig) { + log_debug(DEBUG_ANY, "dnsmasq received signal %d", sig); if (pid == 0) { /* ignore anything other than TERM during startup diff --git a/src/signals.c b/src/signals.c index ee413d9a..8003d47f 100644 --- a/src/signals.c +++ b/src/signals.c @@ -333,7 +333,11 @@ static void SIGTERM_handler(int signum, siginfo_t *si, void *unused) { // Ignore SIGTERM outside of the main process (TCP forks) if(mpid != getpid()) + { + log_debug(DEBUG_ANY, "Ignoring SIGTERM in TCP worker"); return; + } + log_debug(DEBUG_ANY, "Received SIGTERM"); // Get PID and UID of the process that sent the terminating signal const pid_t kill_pid = si->si_pid; @@ -401,10 +405,10 @@ static void SIGTERM_handler(int signum, siginfo_t *si, void *unused) // Log who sent the signal log_info("Asked to terminate by \"%s\" (PID %ld, user %s UID %ld)", - kill_name, (long int)kill_pid, - kill_user, (long int)kill_uid); + kill_name, (long int)kill_pid, kill_user, (long int)kill_uid); // Terminate dnsmasq to stop DNS service + log_debug(DEBUG_ANY, "Sending SIGUSR6 to dnsmasq to stop DNS service"); raise(SIGUSR6); } @@ -413,29 +417,21 @@ void handle_signals(void) { struct sigaction old_action; - const int signals[] = { SIGSEGV, SIGBUS, SIGILL, SIGFPE }; + const int signals[] = { SIGSEGV, SIGBUS, SIGILL, SIGFPE, SIGTERM }; for(unsigned int i = 0; i < ArraySize(signals); i++) { // Catch this signal sigaction (signals[i], NULL, &old_action); if(old_action.sa_handler != SIG_IGN) { - struct sigaction SIGaction; - memset(&SIGaction, 0, sizeof(struct sigaction)); + struct sigaction SIGaction = { 0 }; SIGaction.sa_flags = SA_SIGINFO; sigemptyset(&SIGaction.sa_mask); - SIGaction.sa_sigaction = &signal_handler; + SIGaction.sa_sigaction = signals[i] != SIGTERM ? &signal_handler : &SIGTERM_handler; sigaction(signals[i], &SIGaction, NULL); } } - // Also catch SIGTERM - struct sigaction SIGaction = { 0 }; - SIGaction.sa_flags = SA_SIGINFO; - sigemptyset(&SIGaction.sa_mask); - SIGaction.sa_sigaction = &SIGTERM_handler; - sigaction(SIGTERM, &SIGaction, NULL); - // Log start time of FTL FTLstarttime = time(NULL); } From d54f0a22059da28cf88e2741f3863d78aecae889 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 25 Jul 2024 18:27:07 +0200 Subject: [PATCH 239/339] Handle SIGTERM in FTL if dnsmasq failed to start. Otherwise, we may end up in a situation where nobody feels responsible for handling SIGTERM events and FTL ends up in an un-terminate-able state Signed-off-by: DL6ER --- src/database/common.c | 4 ---- src/dnsmasq/dnsmasq.c | 9 ++------- src/dnsmasq/log.c | 2 +- src/dnsmasq_interface.c | 2 ++ src/main.c | 5 +++-- src/main.h | 1 + src/signals.c | 12 ++++++++++-- src/signals.h | 3 --- 8 files changed, 19 insertions(+), 19 deletions(-) diff --git a/src/database/common.c b/src/database/common.c index 85ee0dbc..cad80229 100644 --- a/src/database/common.c +++ b/src/database/common.c @@ -255,10 +255,6 @@ void SQLite3LogCallback(void *pArg, int iErrCode, const char *zMsg) return; } - // Log backtrace if any debug flag is set - if(config.debug.extra.v.b) - generate_backtrace(); - if(iErrCode == SQLITE_WARNING) log_warn("SQLite3: %s (%d)", zMsg, iErrCode); else if(iErrCode == SQLITE_NOTICE || iErrCode == SQLITE_SCHEMA) diff --git a/src/dnsmasq/dnsmasq.c b/src/dnsmasq/dnsmasq.c index 97d2d878..cfcb38da 100644 --- a/src/dnsmasq/dnsmasq.c +++ b/src/dnsmasq/dnsmasq.c @@ -35,7 +35,6 @@ struct daemon *daemon; static volatile pid_t pid = 0; static volatile int pipewrite; -volatile char FTL_terminate = 0; static void set_dns_listeners(void); static void set_tftp_listeners(void); @@ -1067,12 +1066,8 @@ int main_dnsmasq (int argc, char **argv) /* Using inotify, have to select a resolv file at startup */ poll_resolv(1, 0, now); #endif - - /*** Pi-hole modification ***/ - FTL_terminate = killed; - /****************************/ - while (!FTL_terminate) + while (!killed) { int timeout = fast_retry(now); @@ -1669,7 +1664,7 @@ static void async_event(int pipe, time_t now) flush_log(); /*** Pi-hole modification ***/ // exit(EC_GOOD); - FTL_terminate = 1; + killed = 1; /*** Pi-hole modification ***/ } } diff --git a/src/dnsmasq/log.c b/src/dnsmasq/log.c index eb0f2cef..f040163f 100644 --- a/src/dnsmasq/log.c +++ b/src/dnsmasq/log.c @@ -90,7 +90,7 @@ int log_start(struct passwd *ent_pw, int errfd) if (!log_reopen(daemon->log_file)) { send_event(errfd, EVENT_LOG_ERR, errno, daemon->log_file ? daemon->log_file : ""); - _exit(0); + die(_("failed to open log file: %s"), strerror(errno), 1); // Pi-hole modification } /* if queuing is inhibited, make sure we allocate diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c index 8a46e5bd..b53790a5 100644 --- a/src/dnsmasq_interface.c +++ b/src/dnsmasq_interface.c @@ -3019,6 +3019,8 @@ void FTL_fork_and_bind_sockets(struct passwd *ent_pw, bool dnsmasq_start) // Initialize Pi-hole PTR pointer init_pihole_PTR(); + + forked = true; } static char *get_ptrname(struct in_addr *addr) diff --git a/src/main.c b/src/main.c index 61e7b405..68b22027 100644 --- a/src/main.c +++ b/src/main.c @@ -32,6 +32,7 @@ char *username; bool needGC = false; bool needDBGC = false; bool startup = true; +bool forked = false; jmp_buf exit_jmp; int main (int argc, char *argv[]) @@ -124,7 +125,7 @@ int main (int argc, char *argv[]) log_debug(DEBUG_ANY, "Jumped back to main() from dnsmasq/die()"); dnsmasq_failed = true; - if(!resolver_ready) + if(!forked) { // If dnsmasq never finished initializing, we need to // launch the threads @@ -132,7 +133,7 @@ int main (int argc, char *argv[]) } // Loop here to keep the webserver running unless requested to restart - while(!FTL_terminate) + while(!killed) sleepms(100); } diff --git a/src/main.h b/src/main.h index 7a40c894..7918c889 100644 --- a/src/main.h +++ b/src/main.h @@ -20,6 +20,7 @@ void FTL_fork_and_bind_sockets(struct passwd *ent_pw, bool dnsmasq_start); extern char *username; extern bool startup; +extern bool forked; extern jmp_buf exit_jmp; #endif //MAIN_H diff --git a/src/signals.c b/src/signals.c index 8003d47f..392aff3a 100644 --- a/src/signals.c +++ b/src/signals.c @@ -408,8 +408,16 @@ static void SIGTERM_handler(int signum, siginfo_t *si, void *unused) kill_name, (long int)kill_pid, kill_user, (long int)kill_uid); // Terminate dnsmasq to stop DNS service - log_debug(DEBUG_ANY, "Sending SIGUSR6 to dnsmasq to stop DNS service"); - raise(SIGUSR6); + if(!dnsmasq_failed) + { + log_debug(DEBUG_ANY, "Sending SIGUSR6 to dnsmasq to stop DNS service"); + raise(SIGUSR6); + } + else + { + log_debug(DEBUG_ANY, "Embedded dnsmasq failed, exiting on request"); + killed = true; + } } // Register ordinary signals handler diff --git a/src/signals.h b/src/signals.h index 7201fbf3..1f397b26 100644 --- a/src/signals.h +++ b/src/signals.h @@ -15,9 +15,6 @@ #define SIGUSR6 (SIGRTMIN + 6) #define SIGUSR32 (SIGRTMIN + 32) -// defined in dnsmasq/dnsmasq.h -extern volatile char FTL_terminate; - void handle_signals(void); void handle_realtime_signals(void); pid_t main_pid(void); From ae1415f29ba64763ba8c9afcfe2ade9d85ed9e4a Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 26 Jul 2024 17:51:55 +0200 Subject: [PATCH 240/339] Add new 2024 DNS root trust anchor published today on www.iana.org Signed-off-by: DL6ER --- src/config/dnsmasq_config.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/config/dnsmasq_config.c b/src/config/dnsmasq_config.c index c50223b1..abf076ca 100644 --- a/src/config/dnsmasq_config.c +++ b/src/config/dnsmasq_config.c @@ -398,8 +398,13 @@ bool __attribute__((const)) write_dnsmasq_config(struct config *conf, bool test_ fputs("# Use DNNSEC\n", pihole_conf); fputs("dnssec\n", pihole_conf); fputs("# 2017-02-02 root zone trust anchor\n", pihole_conf); + fputs("# https://www.iana.org/reports/2017/root-ksk-2017.pdf\n", pihole_conf); fputs("trust-anchor=.,20326,8,2,E06D44B80B8F1D39A95C0B0D7C65D08458E880409BBC683457104237C7F8EC8D\n", pihole_conf); + fputs("# 2024-07-26 root zone trust anchor\n", pihole_conf); + fputs("# https://www.iana.org/reports/2024/root-ksk-2024.pdf\n", pihole_conf); + fputs("trust-anchor=.,38696,8,2,683D2D0ACB8C9B712A1948B27F741219298D0A450D612C483AF444A4C0FB2B16\n", + pihole_conf); fputs("\n", pihole_conf); } From e942301897840c5b9df19196d72c62ef87f17645 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 30 Jul 2024 20:20:55 +0200 Subject: [PATCH 241/339] Improve automatic PTR handling code, fix a small memory leak and remove a bit of duplicated code Signed-off-by: DL6ER --- src/database/common.c | 4 +- src/database/network-table.c | 1 + src/database/query-table.c | 8 --- src/dnsmasq/dhcp.c | 2 +- src/dnsmasq/forward.c | 4 +- src/dnsmasq/rfc1035.c | 4 +- src/dnsmasq_interface.c | 105 ++++++++++++++++++----------------- 7 files changed, 62 insertions(+), 66 deletions(-) diff --git a/src/database/common.c b/src/database/common.c index f9c83383..c29d889f 100644 --- a/src/database/common.c +++ b/src/database/common.c @@ -248,7 +248,7 @@ void SQLite3LogCallback(void *pArg, int iErrCode, const char *zMsg) // Note: pArg is NULL and not used // See https://sqlite.org/rescode.html#extrc for details // concerning the return codes returned here - if(strncmp(zMsg, "file renamed while open: ", sizeof("file renamed while open: ")-1) == 0) + if(zMsg != NULL && strncmp(zMsg, "file renamed while open: ", sizeof("file renamed while open: ")-1) == 0) { // This happens when gravity.db is replaced while FTL is running // We can safely ignore this warning @@ -612,7 +612,7 @@ void db_init(void) // Last check after all migrations, if this happens, it will cause the // CI to fail the tests if(dbversion != MEMDB_VERSION) - log_err("Database version %i does not match MEMDB_VERSION %i", dbversion, MEMDB_VERSION); + log_err("Expected query database version %d but found %d", MEMDB_VERSION, dbversion); lock_shm(); import_aliasclients(db); diff --git a/src/database/network-table.c b/src/database/network-table.c index 9592b627..754d1e93 100644 --- a/src/database/network-table.c +++ b/src/database/network-table.c @@ -1298,6 +1298,7 @@ void parse_neighbor_cache(sqlite3* db) if((arpfp = popen(cmd, "r")) == NULL) { log_warn("Command \"%s\" failed: %s", cmd, strerror(errno)); + free(client_status); return; } diff --git a/src/database/query-table.c b/src/database/query-table.c index a2a98a00..8aa16790 100644 --- a/src/database/query-table.c +++ b/src/database/query-table.c @@ -481,14 +481,6 @@ int get_number_of_queries_in_DB(sqlite3 *db, const char *tablename) rc = sqlite3_step(stmt); if( rc == SQLITE_ROW ) num = sqlite3_column_int(stmt, 0); - else - { - log_err("get_number_of_queries_in_DB(%s): Step error: %s", - tablename, sqlite3_errstr(rc)); - free(querystr); - sqlite3_finalize(stmt); - return false; - } sqlite3_finalize(stmt); free(querystr); diff --git a/src/dnsmasq/dhcp.c b/src/dnsmasq/dhcp.c index b65facd8..e70e011c 100644 --- a/src/dnsmasq/dhcp.c +++ b/src/dnsmasq/dhcp.c @@ -162,7 +162,7 @@ void dhcp_packet(time_t now, int pxe_fd) #elif defined(HAVE_BSD_NETWORK) char control[CMSG_SPACE(sizeof(struct sockaddr_dl))]; #endif - } control_u; + } control_u = { 0 }; struct dhcp_bridge *bridge, *alias; msg.msg_controllen = sizeof(control_u); diff --git a/src/dnsmasq/forward.c b/src/dnsmasq/forward.c index 15713ec2..b505d354 100644 --- a/src/dnsmasq/forward.c +++ b/src/dnsmasq/forward.c @@ -36,7 +36,7 @@ int send_from(int fd, int nowild, char *packet, size_t len, union mysockaddr *to, union all_addr *source, unsigned int iface) { - struct msghdr msg; + struct msghdr msg = { 0 }; struct iovec iov[1]; union { struct cmsghdr align; /* this ensures alignment */ @@ -46,7 +46,7 @@ int send_from(int fd, int nowild, char *packet, size_t len, char control[CMSG_SPACE(sizeof(struct in_addr))]; #endif char control6[CMSG_SPACE(sizeof(struct in6_pktinfo))]; - } control_u; + } control_u = { 0 }; iov[0].iov_base = packet; iov[0].iov_len = len; diff --git a/src/dnsmasq/rfc1035.c b/src/dnsmasq/rfc1035.c index ef618ff0..1446908d 100644 --- a/src/dnsmasq/rfc1035.c +++ b/src/dnsmasq/rfc1035.c @@ -868,7 +868,7 @@ int extract_addresses(struct dns_header *header, size_t qlen, char *name, time_t return 2; // ****************************** Pi-hole modification ****************************** - const char *src = cpp != NULL ? cpp->flags & F_BIGNAME ? cpp->name.bname->name : cpp->name.sname : NULL; + const char *src = cpp != NULL ? cache_get_name(cpp) : NULL; if(FTL_CNAME(name, src, daemon->log_display_id)) { // Found while processing a reply from upstream. We prevent cache insertion here @@ -2047,7 +2047,7 @@ size_t answer_request(struct dns_header *header, char *limit, size_t qlen, log_query(stale_flag | (crecp->flags & ~F_REVERSE), name, &crecp->addr, record_source(crecp->uid), 0); // ****************************** Pi-hole modification ****************************** - const char *src = crecp != NULL ? crecp->flags & F_BIGNAME ? crecp->name.bname->name : crecp->name.sname : NULL; + const char *src = crecp != NULL ? cache_get_name(crecp) : NULL; if(FTL_CNAME(name, src, daemon->log_display_id)) { // Served from cache. This can happen if a domain hidden in the CNAME path diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c index b53790a5..d5506eaf 100644 --- a/src/dnsmasq_interface.c +++ b/src/dnsmasq_interface.c @@ -87,7 +87,6 @@ static bool adbit = false; static const char *blockingreason = ""; static enum reply_type force_next_DNS_reply = REPLY_UNKNOWN; static int last_regex_idx = -1; -static struct ptr_record *pihole_ptr = NULL; static char *pihole_suffix = NULL; static char *hostname_suffix = NULL; static char *cname_target = NULL; @@ -1058,9 +1057,22 @@ void _FTL_iface(struct irec *recviface, const union all_addr *addr, const sa_fam static void check_pihole_PTR(char *domain) { - // Return early if Pi-hole PTR is not available - if(pihole_ptr == NULL) - return; + // Iterate through the already configured PTR entries in dnsmasq's + // structure and check if we already have a PTR record for this address + // This avoids adding work into defining PTR records that have already + // been added but also overwriting PTR records manually added by users + // using custom dnsmasq config lines like "ptr-record=," + for(struct ptr_record *ptr = daemon->ptr; ptr; ptr = ptr->next) + { + log_debug(DEBUG_EXTRA, "Known PTR record %p: %s -> %s (next = %p)", ptr, ptr->name, ptr->ptr, ptr->next); + + if(ptr->name != NULL && strcmp(ptr->name, domain) == 0) + { + // We already have a PTR record for this address + log_debug(DEBUG_QUERIES, "PTR record for %s exists", domain); + return; + } + } // Convert PTR request into numeric form union all_addr addr = {{ 0 }}; @@ -1082,30 +1094,46 @@ static void check_pihole_PTR(char *domain) for (struct irec *iface = daemon->interfaces; iface != NULL; iface = iface->next) { const sa_family_t family = iface->addr.sa.sa_family; - if((family == AF_INET && flags == F_IPV4 && iface->addr.in.sin_addr.s_addr == addr.addr4.s_addr) || - (family == AF_INET6 && flags == F_IPV6 && IN6_ARE_ADDR_EQUAL(&iface->addr.in6.sin6_addr, &addr.addr6))) + // If the family matches but the address doesn't, we skip this address + if(!(family == AF_INET && flags == F_IPV4 && iface->addr.in.sin_addr.s_addr == addr.addr4.s_addr) && + !(family == AF_INET6 && flags == F_IPV6 && IN6_ARE_ADDR_EQUAL(&iface->addr.in6.sin6_addr, &addr.addr6))) + continue; + + // If we reached this point, we have a match between the address the client + struct ptr_record *pihole_ptr = calloc(1, sizeof(struct ptr_record)); + pihole_ptr->name = strdup(domain); + if(family == AF_INET) { - // The last PTR record in daemon->ptr is reserved for Pi-hole - free(pihole_ptr->name); - pihole_ptr->name = strdup(domain); - if(family == AF_INET) - { - // IPv4 supports conditional domains - struct in_addr addrv4 = { 0 }; - addrv4.s_addr = iface->addr.in.sin_addr.s_addr; - pihole_ptr->ptr = get_ptrname(&addrv4); - } - else - { - // IPv6 does not support conditional domains - pihole_ptr->ptr = get_ptrname(NULL); - } - - // Debug logging - log_debug(DEBUG_QUERIES, "Generating PTR response: %s -> %s", pihole_ptr->name, pihole_ptr->ptr); - - return; + // IPv4 supports conditional domains + pihole_ptr->ptr = get_ptrname(&iface->addr.in.sin_addr); } + else + { + // IPv6 does not support conditional domains + pihole_ptr->ptr = get_ptrname(NULL); + } + + // If we have a PTR record, we add it to the list + if(daemon->ptr != NULL) + { + // Iterate to the last PTR entry in dnsmasq's structure + struct ptr_record *ptr; + for(ptr = daemon->ptr; ptr && ptr->next; ptr = ptr->next); + + // Add our record after the last existing ptr-record + ptr->next = pihole_ptr; + } + else + { + // We do not have any PTR records yet, so we add our + // record as the first one + daemon->ptr = pihole_ptr; + } + + // Debug logging + log_debug(DEBUG_QUERIES, "Generating PTR record (%p): %s -> %s", pihole_ptr, pihole_ptr->name, pihole_ptr->ptr); + + return; } } @@ -2854,32 +2882,7 @@ static void init_pihole_PTR(void) // Fallback to "" on memory error ptrname = (char*)hostname(); } - } break; - } - - // Obtain PTR record used for Pi-hole PTR injection (if enabled) - if(config.dns.piholePTR.v.ptr_type != PTR_NONE) - { - // Add PTR record for pi.hole, the address will be injected later - pihole_ptr = calloc(1, sizeof(struct ptr_record)); - pihole_ptr->name = strdup("x.x.x.x.in-addr.arpa"); - pihole_ptr->ptr = (char*)""; - pihole_ptr->next = NULL; - // Add our PTR record to the end of the linked list - if(daemon->ptr != NULL) - { - // Iterate to the last PTR entry in dnsmasq's structure - struct ptr_record *ptr; - for(ptr = daemon->ptr; ptr && ptr->next; ptr = ptr->next); - - // Add our record after the last existing ptr-record - ptr->next = pihole_ptr; - } - else - { - // Ours is the only record for daemon->ptr - daemon->ptr = pihole_ptr; } } } From 4c4489a539ca902aa55e8a01c8836047a8becc77 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 31 Jul 2024 18:57:15 +0200 Subject: [PATCH 242/339] Use pipe to main process instead of printing directly from the signal handler to avoid futex wait blocking Signed-off-by: DL6ER --- src/dnsmasq/dnsmasq.c | 11 ++++++++++- src/dnsmasq/dnsmasq.h | 3 +++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/dnsmasq/dnsmasq.c b/src/dnsmasq/dnsmasq.c index cfcb38da..e051d3e2 100644 --- a/src/dnsmasq/dnsmasq.c +++ b/src/dnsmasq/dnsmasq.c @@ -1310,7 +1310,10 @@ int main_dnsmasq (int argc, char **argv) static void sig_handler(int sig) { - log_debug(DEBUG_ANY, "dnsmasq received signal %d", sig); + /**** Pi-hole modification ****/ + send_event(pipewrite, EVENT_SIGNAL, sig, NULL); + /******************************/ + if (pid == 0) { /* ignore anything other than TERM during startup @@ -1570,6 +1573,12 @@ static void async_event(int pipe, time_t now) my_syslog(LOG_WARNING, _("script process exited with status %d"), ev.data); break; + /**** Pi-hole modification ****/ + case EVENT_SIGNAL: + log_debug(DEBUG_ANY, "dnsmasq received signal %d", ev.data); + break; + /**************************** */ + case EVENT_EXEC_ERR: my_syslog(LOG_ERR, _("failed to execute %s: %s"), daemon->lease_change_command, strerror(ev.data)); diff --git a/src/dnsmasq/dnsmasq.h b/src/dnsmasq/dnsmasq.h index a16c83b4..ce8d1d09 100644 --- a/src/dnsmasq/dnsmasq.h +++ b/src/dnsmasq/dnsmasq.h @@ -200,6 +200,9 @@ struct event_desc { #define EVENT_SCRIPT_LOG 25 #define EVENT_TIME 26 +// Pi-hole +#define EVENT_SIGNAL 255 + /* Exit codes. */ #define EC_GOOD 0 #define EC_BADCONF 1 From 1c2e44d519182fe3e3681cd46620407daede1f4d Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 1 Aug 2024 09:19:31 +0200 Subject: [PATCH 243/339] Use client->firstSeen for new cients for the netDB if available Signed-off-by: DL6ER --- src/database/network-table.c | 43 ++++++++++++++++++++++-------------- 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/src/database/network-table.c b/src/database/network-table.c index 754d1e93..ec464f2e 100644 --- a/src/database/network-table.c +++ b/src/database/network-table.c @@ -510,8 +510,8 @@ static int add_netDB_network_address(sqlite3 *db, const int network_id, const ch } // Insert a new record into the network table -static int insert_netDB_device(sqlite3 *db, const char *hwaddr, time_t now, time_t lastQuery, - unsigned int numQueriesARP, const char *macVendor) +static int insert_netDB_device(sqlite3 *db, const char *hwaddr, const time_t firstSeen, const time_t lastQuery, + const unsigned int numQueriesARP, const char *macVendor) { // Return early if database is known to be broken if(FTLDBerror()) @@ -526,29 +526,29 @@ static int insert_netDB_device(sqlite3 *db, const char *hwaddr, time_t now, time if(rc != SQLITE_OK) { log_err("insert_netDB_device(\"%s\", %lu, %lu, %u, \"%s\") - SQL error prepare (%i): %s", - hwaddr, (unsigned long)now, (unsigned long)lastQuery, numQueriesARP, macVendor, rc, sqlite3_errstr(rc)); + hwaddr, (unsigned long)firstSeen, (unsigned long)lastQuery, numQueriesARP, macVendor, rc, sqlite3_errstr(rc)); checkFTLDBrc(rc); return rc; } log_debug(DEBUG_DATABASE, "dbquery: \"%s\" with arguments ?1-?5 = (\"%s\", %lu, %lu, %u, \"%s\")", - querystr, hwaddr, (unsigned long)now, (unsigned long)lastQuery, numQueriesARP, macVendor); + querystr, hwaddr, (unsigned long)firstSeen, (unsigned long)lastQuery, numQueriesARP, macVendor); // Bind hwaddr to prepared statement (1st argument) if((rc = sqlite3_bind_text(query_stmt, 1, hwaddr, -1, SQLITE_STATIC)) != SQLITE_OK) { log_err("insert_netDB_device(\"%s\", %lu, %lu, %u, \"%s\"): Failed to bind hwaddr (error %d): %s", - hwaddr, (unsigned long)now, (unsigned long)lastQuery, numQueriesARP, macVendor, rc, sqlite3_errstr(rc)); + hwaddr, (unsigned long)firstSeen, (unsigned long)lastQuery, numQueriesARP, macVendor, rc, sqlite3_errstr(rc)); sqlite3_reset(query_stmt); checkFTLDBrc(rc); return rc; } - // Bind now to prepared statement (2nd argument) - if((rc = sqlite3_bind_int(query_stmt, 2, now)) != SQLITE_OK) + // Bind firstSeen to prepared statement (2nd argument) + if((rc = sqlite3_bind_int(query_stmt, 2, firstSeen)) != SQLITE_OK) { - log_err("insert_netDB_device(\"%s\",%lu, %lu, %u, \"%s\"): Failed to bind now (error %d): %s", - hwaddr, (unsigned long)now, (unsigned long)lastQuery, numQueriesARP, macVendor, rc, sqlite3_errstr(rc)); + log_err("insert_netDB_device(\"%s\",%lu, %lu, %u, \"%s\"): Failed to bind firstSeen (error %d): %s", + hwaddr, (unsigned long)firstSeen, (unsigned long)lastQuery, numQueriesARP, macVendor, rc, sqlite3_errstr(rc)); sqlite3_reset(query_stmt); checkFTLDBrc(rc); return rc; @@ -558,7 +558,7 @@ static int insert_netDB_device(sqlite3 *db, const char *hwaddr, time_t now, time if((rc = sqlite3_bind_int(query_stmt, 3, lastQuery)) != SQLITE_OK) { log_err("insert_netDB_device(\"%s\",%lu, %lu, %u, \"%s\"): Failed to bind lastQuery (error %d): %s", - hwaddr, (unsigned long)now, (unsigned long)lastQuery, numQueriesARP, macVendor, rc, sqlite3_errstr(rc)); + hwaddr, (unsigned long)firstSeen, (unsigned long)lastQuery, numQueriesARP, macVendor, rc, sqlite3_errstr(rc)); sqlite3_reset(query_stmt); checkFTLDBrc(rc); return rc; @@ -568,7 +568,7 @@ static int insert_netDB_device(sqlite3 *db, const char *hwaddr, time_t now, time if((rc = sqlite3_bind_int(query_stmt, 4, numQueriesARP)) != SQLITE_OK) { log_err("insert_netDB_device(\"%s\",%lu, %lu, %u, \"%s\"): Failed to bind numQueriesARP (error %d): %s", - hwaddr, (unsigned long)now, (unsigned long)lastQuery, numQueriesARP, macVendor, rc, sqlite3_errstr(rc)); + hwaddr, (unsigned long)firstSeen, (unsigned long)lastQuery, numQueriesARP, macVendor, rc, sqlite3_errstr(rc)); sqlite3_reset(query_stmt); checkFTLDBrc(rc); return rc; @@ -578,7 +578,7 @@ static int insert_netDB_device(sqlite3 *db, const char *hwaddr, time_t now, time if((rc = sqlite3_bind_text(query_stmt, 5, macVendor, -1, SQLITE_STATIC)) != SQLITE_OK) { log_err("insert_netDB_device(\"%s\",%lu, %lu, %u, \"%s\"): Failed to bind macVendor (error %d): %s", - hwaddr, (unsigned long)now, (unsigned long)lastQuery, numQueriesARP, macVendor, rc, sqlite3_errstr(rc)); + hwaddr, (unsigned long)firstSeen, (unsigned long)lastQuery, numQueriesARP, macVendor, rc, sqlite3_errstr(rc)); sqlite3_reset(query_stmt); checkFTLDBrc(rc); return rc; @@ -588,7 +588,7 @@ static int insert_netDB_device(sqlite3 *db, const char *hwaddr, time_t now, time if ((rc = sqlite3_step(query_stmt)) != SQLITE_DONE) { log_err("insert_netDB_device(\"%s\",%lu, %lu, %u, \"%s\"): Failed to step (error %d): %s", - hwaddr, (unsigned long)now, (unsigned long)lastQuery, numQueriesARP, macVendor, rc, sqlite3_errstr(rc)); + hwaddr, (unsigned long)firstSeen, (unsigned long)lastQuery, numQueriesARP, macVendor, rc, sqlite3_errstr(rc)); sqlite3_reset(query_stmt); checkFTLDBrc(rc); return rc; @@ -598,7 +598,7 @@ static int insert_netDB_device(sqlite3 *db, const char *hwaddr, time_t now, time if ((rc = sqlite3_finalize(query_stmt)) != SQLITE_OK) { log_err("insert_netDB_device(\"%s\",%lu, %lu, %u, \"%s\"): Failed to finalize (error %d): %s", - hwaddr, (unsigned long)now, (unsigned long)lastQuery, numQueriesARP, macVendor, rc, sqlite3_errstr(rc)); + hwaddr, (unsigned long)firstSeen, (unsigned long)lastQuery, numQueriesARP, macVendor, rc, sqlite3_errstr(rc)); sqlite3_reset(query_stmt); checkFTLDBrc(rc); return rc; @@ -901,9 +901,10 @@ static bool add_FTL_clients_to_network_table(sqlite3 *db, const enum arp_status // Add new device to database const time_t lastQuery = client->lastQuery; + const time_t firstSeen = client->firstSeen; const unsigned int numQueriesARP = client->numQueriesARP; unlock_shm(); - insert_netDB_device(db, hwaddr, now, lastQuery, numQueriesARP, macVendor); + insert_netDB_device(db, hwaddr, firstSeen, lastQuery, numQueriesARP, macVendor); lock_shm(); // Reacquire client pointer (if may have changed when unlocking above) @@ -1370,10 +1371,12 @@ void parse_neighbor_cache(sqlite3* db) lock_shm(); int clientID = findClientID(ip, false, false); - // Get hostname of this client if the client is known + // Set default values for a new device, may be updated + // below if the client is known to pihole-FTL char *hostname = NULL; bool client_valid = false; time_t lastQuery = 0; + time_t firstSeen = now; unsigned int numQueries = 0; // This client is known (by its IP address) to pihole-FTL if @@ -1384,14 +1387,20 @@ void parse_neighbor_cache(sqlite3* db) if(!client) continue; + // Client is known to Pi-hole, update properties + // with their real values client_valid = true; hostname = strdup(getstr(client->namepos)); + firstSeen = client->firstSeen; lastQuery = client->lastQuery; numQueries = client->numQueriesARP; client_status[clientID] = CLIENT_ARP_COMPLETE; } else { + // Client is not known to Pi-hole, create a + // mock-device with the default values set above + // and an empty hostname hostname = strdup(""); } unlock_shm(); @@ -1413,7 +1422,7 @@ void parse_neighbor_cache(sqlite3* db) hwaddr, ip, hostname, macVendor); // Create new record (INSERT) - insert_netDB_device(db, hwaddr, now, lastQuery, numQueries, macVendor); + insert_netDB_device(db, hwaddr, firstSeen, lastQuery, numQueries, macVendor); lock_shm(); clientsData *client = getClient(clientID, true); From 73634762ed04e5b6101de465a3baee35dad40d46 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 1 Aug 2024 09:36:54 +0200 Subject: [PATCH 244/339] Remove deprecated asprintf() calls from network-table code Signed-off-by: DL6ER --- src/database/common.c | 92 +++++++++++++++++++++++ src/database/common.h | 2 + src/database/network-table.c | 142 +++++++++++++---------------------- 3 files changed, 145 insertions(+), 91 deletions(-) diff --git a/src/database/common.c b/src/database/common.c index c29d889f..dee1019c 100644 --- a/src/database/common.c +++ b/src/database/common.c @@ -758,6 +758,98 @@ int db_query_int(sqlite3 *db, const char* querystr) return result; } +int db_query_int_int(sqlite3 *db, const char* querystr, const int arg) +{ + log_debug(DEBUG_DATABASE, "db_query_int_arg: \"%s\"", querystr); + + sqlite3_stmt* stmt; + int rc = sqlite3_prepare_v2(db, querystr, -1, &stmt, NULL); + if( rc != SQLITE_OK ) + { + if( rc != SQLITE_BUSY ) + log_err("Encountered prepare error in db_query_int(\"%s\"): %s", + querystr, sqlite3_errstr(rc)); + return DB_FAILED; + } + + // Bind argument to prepared statement + if((rc = sqlite3_bind_int(stmt, 1, arg)) != SQLITE_OK) + { + log_err("Encountered bind error in db_query_int(\"%s\"): %s", + querystr, sqlite3_errstr(rc)); + } + + rc = sqlite3_step(stmt); + int result; + + if( rc == SQLITE_ROW ) + { + result = sqlite3_column_int(stmt, 0); + log_debug(DEBUG_DATABASE, " ---> Result %i (int)", result); + } + else if( rc == SQLITE_DONE ) + { + // No rows available + result = DB_NODATA; + log_debug(DEBUG_DATABASE, " ---> No data"); + } + else + { + log_err("Encountered step error in db_query_int(\"%s\"): %s", + querystr, sqlite3_errstr(rc)); + return DB_FAILED; + } + + sqlite3_finalize(stmt); + return result; +} + +int db_query_int_str(sqlite3 *db, const char* querystr, const char *arg) +{ + log_debug(DEBUG_DATABASE, "db_query_int_str: \"%s\"", querystr); + + sqlite3_stmt* stmt; + int rc = sqlite3_prepare_v2(db, querystr, -1, &stmt, NULL); + if( rc != SQLITE_OK ) + { + if( rc != SQLITE_BUSY ) + log_err("Encountered prepare error in db_query_int(\"%s\"): %s", + querystr, sqlite3_errstr(rc)); + return DB_FAILED; + } + + // Bind argument to prepared statement + if((rc = sqlite3_bind_text(stmt, 1, arg, -1, SQLITE_STATIC)) != SQLITE_OK) + { + log_err("Encountered bind error in db_query_int(\"%s\"): %s", + querystr, sqlite3_errstr(rc)); + } + + rc = sqlite3_step(stmt); + int result; + + if( rc == SQLITE_ROW ) + { + result = sqlite3_column_int(stmt, 0); + log_debug(DEBUG_DATABASE, " ---> Result %i (int)", result); + } + else if( rc == SQLITE_DONE ) + { + // No rows available + result = DB_NODATA; + log_debug(DEBUG_DATABASE, " ---> No data"); + } + else + { + log_err("Encountered step error in db_query_int(\"%s\"): %s", + querystr, sqlite3_errstr(rc)); + return DB_FAILED; + } + + sqlite3_finalize(stmt); + return result; +} + double db_query_double(sqlite3 *db, const char* querystr) { log_debug(DEBUG_DATABASE, "dbquery: \"%s\"", querystr); diff --git a/src/database/common.h b/src/database/common.h index 5dd0c875..b5673f91 100644 --- a/src/database/common.h +++ b/src/database/common.h @@ -45,6 +45,8 @@ void _dbclose(sqlite3 **db, const char *func, const int line, const char *file); void piholeFTLDB_reopen(void); int db_query_int(sqlite3 *db, const char *querystr); +int db_query_int_int(sqlite3 *db, const char* querystr, const int arg); +int db_query_int_str(sqlite3 *db, const char* querystr, const char *arg); double db_query_double(sqlite3 *db, const char *querystr); int db_query_int_from_until(sqlite3 *db, const char* querystr, const double from, const double until); int db_query_int_from_until_type(sqlite3 *db, const char* querystr, const double from, const double until, const int type); diff --git a/src/database/network-table.c b/src/database/network-table.c index ec464f2e..f33279a1 100644 --- a/src/database/network-table.c +++ b/src/database/network-table.c @@ -210,23 +210,13 @@ static int find_device_by_recent_ip(sqlite3 *db, const char *ipaddr) if(FTLDBerror()) return -1; - char *querystr = NULL; - int ret = asprintf(&querystr, - "SELECT network_id FROM network_addresses " - "WHERE ip = \'%s\' AND " - "lastSeen > (cast(strftime('%%s', 'now') as int)-86400) " - "ORDER BY lastSeen DESC LIMIT 1;", ipaddr); - if(querystr == NULL || ret < 0) - { - log_warn("Memory allocation failed in find_device_by_recent_ip(\"%s\"): %i", - ipaddr, ret); - return -1; - } + const char *querystr = "SELECT network_id FROM network_addresses " + "WHERE ip = ?1 AND " + "lastSeen > (cast(strftime('%%s', 'now') as int)-86400) " + "ORDER BY lastSeen DESC LIMIT 1;"; // Perform SQL query - int network_id = db_query_int(db, querystr); - free(querystr); - querystr = NULL; + int network_id = db_query_int_str(db, querystr, ipaddr); if(network_id == DB_FAILED) { @@ -252,20 +242,10 @@ static int find_device_by_mock_hwaddr(sqlite3 *db, const char *ipaddr) if(FTLDBerror()) return DB_FAILED; - char *querystr = NULL; - int ret = asprintf(&querystr, "SELECT id FROM network WHERE hwaddr = \'ip-%s\';", ipaddr); - if(querystr == NULL || ret < 0) - { - log_warn("Memory allocation failed in find_device_by_mock_hwaddr(\"%s\"): %i", - ipaddr, ret); - return -1; - } + const char *querystr = "SELECT id FROM network WHERE hwaddr = concat('ip-',?1)"; // Perform SQL query - int network_id = db_query_int(db, querystr); - free(querystr); - - return network_id; + return db_query_int_str(db, querystr, ipaddr); } // Try to find device by hardware address @@ -275,20 +255,10 @@ static int find_device_by_hwaddr(sqlite3 *db, const char hwaddr[]) if(FTLDBerror()) return DB_FAILED; - char *querystr = NULL; - int ret = asprintf(&querystr, "SELECT id FROM network WHERE hwaddr = \'%s\' COLLATE NOCASE;", hwaddr); - if(querystr == NULL || ret < 0) - { - log_warn("Memory allocation failed in find_device_by_hwaddr(\"%s\"): %i", - hwaddr, ret); - return -1; - } + const char *querystr = "SELECT id FROM network WHERE hwaddr = ?1 COLLATE NOCASE;"; // Perform SQL query - int network_id = db_query_int(db, querystr); - free(querystr); - - return network_id; + return db_query_int_str(db, querystr, hwaddr); } // Try to find device by RECENT mock hardware address (generated from IP address) @@ -298,24 +268,12 @@ static int find_recent_device_by_mock_hwaddr(sqlite3 *db, const char *ipaddr) if(FTLDBerror()) return DB_FAILED; - char *querystr = NULL; - int ret = asprintf(&querystr, - "SELECT id FROM network WHERE " - "hwaddr = \'ip-%s\' AND " - "firstSeen > (cast(strftime('%%s', 'now') as int)-3600);", - ipaddr); - if(querystr == NULL || ret < 0) - { - log_warn("Memory allocation failed in find_device_by_recent_mock_hwaddr(\"%s\"): %i", - ipaddr, ret); - return -1; - } + const char *querystr = "SELECT id FROM network WHERE " + "hwaddr = concat('ip-',?1) AND " + "firstSeen > (cast(strftime('%%s', 'now') as int)-3600)"; // Perform SQL query - int network_id = db_query_int(db, querystr); - free(querystr); - - return network_id; + return db_query_int_str(db, querystr, ipaddr); } // Store hostname of device identified by dbID @@ -1137,30 +1095,9 @@ static bool add_local_interfaces_to_network_table(sqlite3 *db, time_t now, unsig int lastQuery = 0, firstSeen = now, numQueries = 0; if(mockID >= 0) { - char *querystr = NULL; - if(asprintf(&querystr, "SELECT lastQuery from network where id = %i", mockID) < 10) - { - free(macVendor); - return false; - } - lastQuery = db_query_int(db, querystr); - free(querystr); - - if(asprintf(&querystr, "SELECT firstSeen from network where id = %i", mockID) < 10) - { - free(macVendor); - return false; - } - firstSeen = db_query_int(db, querystr); - free(querystr); - - if(asprintf(&querystr, "SELECT numQueries from network where id = %i", mockID) < 10) - { - free(macVendor); - return false; - } - numQueries = db_query_int(db, querystr); - free(querystr); + lastQuery = db_query_int_int(db, "SELECT lastQuery from network where id = ?1", mockID); + firstSeen = db_query_int_int(db, "SELECT firstSeen from network where id = ?1", mockID); + numQueries = db_query_int_int(db, "SELECT numQueries from network where id = ?1", mockID); } // Add new device to database @@ -1800,33 +1737,56 @@ void updateMACVendorRecords(sqlite3 *db) // Get vendor for MAC char *vendor = getMACVendor(hwaddr); + + // Free allocated memory free(hwaddr); hwaddr = NULL; - // Prepare UPDATE statement - char *updatestr = NULL; - if(asprintf(&updatestr, "UPDATE network SET macVendor = \'%s\' WHERE id = %i", vendor, id) < 1) + // Prepare statement + sqlite3_stmt *stmt2 = NULL; + const char *updatestr = "UPDATE network SET macVendor = ?1 WHERE id = ?2"; + rc = sqlite3_prepare_v2(db, updatestr, -1, &stmt2, NULL); + if(rc != SQLITE_OK) { - log_err("updateMACVendorRecords() - Allocation error"); + log_err("updateMACVendorRecords() - SQL error prepare \"%s\": %s", updatestr, sqlite3_errstr(rc)); + checkFTLDBrc(rc); free(vendor); break; } - // Execute prepared statement - char *zErrMsg = NULL; - rc = sqlite3_exec(db, updatestr, NULL, NULL, &zErrMsg); - if(rc != SQLITE_OK) + // Bind vendor to prepared statement + if((rc = sqlite3_bind_text(stmt2, 1, vendor, -1, SQLITE_STATIC)) != SQLITE_OK) { - log_err("updateMACVendorRecords() - SQL exec error: \"%s\": %s", updatestr, zErrMsg); + log_err("updateMACVendorRecords() - Failed to bind vendor: %s", sqlite3_errstr(rc)); + sqlite3_reset(stmt2); + sqlite3_finalize(stmt2); + free(vendor); + break; + } + + // Bind id to prepared statement + if((rc = sqlite3_bind_int(stmt2, 2, id)) != SQLITE_OK) + { + log_err("updateMACVendorRecords() - Failed to bind id: %s", sqlite3_errstr(rc)); + sqlite3_reset(stmt2); + sqlite3_finalize(stmt2); + free(vendor); + break; + } + + // Execute statement + rc = sqlite3_step(stmt2); + if(rc != SQLITE_DONE) + { + log_err("updateMACVendorRecords() - SQL error step: %s", sqlite3_errstr(rc)); checkFTLDBrc(rc); - sqlite3_free(zErrMsg); - free(updatestr); + sqlite3_reset(stmt2); + sqlite3_finalize(stmt2); free(vendor); break; } // Free allocated memory - free(updatestr); free(vendor); } if(rc != SQLITE_DONE) From 04f255927a825848433f23056a404980d56be46d Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 1 Aug 2024 14:08:50 +0200 Subject: [PATCH 245/339] Update firstSeen on importing clients from the database. Before, their firstSeen was the timestamp of their creation, i.e., the time FTL imported the database. Now the timestamp is set to the first query we import for this client from the database Signed-off-by: DL6ER --- src/database/aliasclients.c | 3 ++- src/database/network-table.c | 14 +++++++++----- src/database/query-table.c | 2 +- src/datastructure.c | 5 +++-- src/datastructure.h | 6 +++--- src/dnsmasq_interface.c | 2 +- 6 files changed, 19 insertions(+), 13 deletions(-) diff --git a/src/database/aliasclients.c b/src/database/aliasclients.c index 7e8fddeb..e258f5c6 100644 --- a/src/database/aliasclients.c +++ b/src/database/aliasclients.c @@ -109,6 +109,7 @@ bool import_aliasclients(sqlite3 *db) // Loop until no further data is available int imported = 0; + const double now = double_time(); while((rc = sqlite3_step(stmt)) != SQLITE_DONE) { // Check if we ran into an error @@ -132,7 +133,7 @@ bool import_aliasclients(sqlite3 *db) } // Try to open existing client - const int clientID = findClientID(aliasclient_str, false, true); + const int clientID = findClientID(aliasclient_str, false, true, now); clientsData *client = getClient(clientID, true); if(client == NULL) diff --git a/src/database/network-table.c b/src/database/network-table.c index f33279a1..c92fd70a 100644 --- a/src/database/network-table.c +++ b/src/database/network-table.c @@ -1189,7 +1189,7 @@ void parse_neighbor_cache(sqlite3* db) char *linebuffer = NULL; size_t linebuffersize = 0u; unsigned int entries = 0u, additional_entries = 0u; - time_t now = time(NULL); + const time_t now = time(NULL); // Start ARP timer if(config.debug.arp.v.b) @@ -1269,8 +1269,11 @@ void parse_neighbor_cache(sqlite3* db) { // This line is incomplete, remember this to skip // mock-device creation after ARP processing + // both false = do not create a new record if the client + // is unknown (only DNS requesting clients + // do this), the now value is ignored lock_shm(); - int clientID = findClientID(ip, false, false); + int clientID = findClientID(ip, false, false, 0.0); unlock_shm(); if(clientID >= 0 && clientID < clients) client_status[clientID] = CLIENT_ARP_INCOMPLETE; @@ -1303,10 +1306,11 @@ void parse_neighbor_cache(sqlite3* db) // If we reach this point, we can check if this client // is known to pihole-FTL - // false = do not create a new record if the client is - // unknown (only DNS requesting clients do this) + // both false = do not create a new record if the client + // is unknown (only DNS requesting clients + // do this), the now value is ignored lock_shm(); - int clientID = findClientID(ip, false, false); + int clientID = findClientID(ip, false, false, 0.0); // Set default values for a new device, may be updated // below if the client is known to pihole-FTL diff --git a/src/database/query-table.c b/src/database/query-table.c index 8aa16790..1b661e77 100644 --- a/src/database/query-table.c +++ b/src/database/query-table.c @@ -1148,7 +1148,7 @@ void DB_read_queries(void) // Obtain IDs only after filtering which queries we want to keep const int timeidx = getOverTimeID(queryTimeStamp); const int domainID = findDomainID(domainname, true); - const int clientID = findClientID(clientIP, true, false); + const int clientID = findClientID(clientIP, true, false, queryTimeStamp); // Set index for this query const int queryIndex = counters->queries; diff --git a/src/datastructure.c b/src/datastructure.c index 28a7be4b..72376a9f 100644 --- a/src/datastructure.c +++ b/src/datastructure.c @@ -239,7 +239,8 @@ static int get_next_free_clientID(void) return counters->clients; } -int _findClientID(const char *clientIP, const bool count, const bool aliasclient, int line, const char *func, const char *file) +int _findClientID(const char *clientIP, const bool count, const bool aliasclient, + const double now, int line, const char *func, const char *file) { // Compare content of client against known client IP addresses for(int clientID=0; clientID < counters->clients; clientID++) @@ -308,7 +309,7 @@ int _findClientID(const char *clientIP, const bool count, const bool aliasclient // some time after adding a client to ensure we pick up possible // group configuration though hostname, MAC address or interface client->reread_groups = 0u; - client->firstSeen = time(NULL); + client->firstSeen = now; // Interface is not yet known client->ifacepos = 0; // Set all MAC address bytes to zero diff --git a/src/datastructure.h b/src/datastructure.h index 43b2a5c1..b2b66772 100644 --- a/src/datastructure.h +++ b/src/datastructure.h @@ -93,7 +93,7 @@ typedef struct { size_t ippos; size_t namepos; size_t ifacepos; - time_t firstSeen; + double firstSeen; double lastQuery; } clientsData; @@ -124,8 +124,8 @@ int findQueryID(const int id); int _findUpstreamID(const char *upstream, const in_port_t port, int line, const char *func, const char *file); #define findDomainID(domain, count) _findDomainID(domain, count, __LINE__, __FUNCTION__, __FILE__) int _findDomainID(const char *domain, const bool count, int line, const char *func, const char *file); -#define findClientID(client, count, aliasclient) _findClientID(client, count, aliasclient, __LINE__, __FUNCTION__, __FILE__) -int _findClientID(const char *client, const bool count, const bool aliasclient, int line, const char *func, const char *file); +#define findClientID(client, count, aliasclient, now) _findClientID(client, count, aliasclient, now, __LINE__, __FUNCTION__, __FILE__) +int _findClientID(const char *client, const bool count, const bool aliasclient, const double now, int line, const char *func, const char *file); #define findCacheID(domainID, clientID, query_type, create_new) _findCacheID(domainID, clientID, query_type, create_new, __FUNCTION__, __LINE__, __FILE__) int _findCacheID(const int domainID, const int clientID, const enum query_type query_type, const bool create_new, const char *func, const int line, const char *file); bool isValidIPv4(const char *addr); diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c index d5506eaf..fa9ca8d1 100644 --- a/src/dnsmasq_interface.c +++ b/src/dnsmasq_interface.c @@ -647,7 +647,7 @@ bool _FTL_new_query(const unsigned int flags, const char *name, const int queryID = counters->queries; // Find client IP - const int clientID = findClientID(clientIP, true, false); + const int clientID = findClientID(clientIP, true, false, querytimestamp); // Get client pointer clientsData* client = getClient(clientID, true); From 4d71d88e7f704adfb7e3483e825fd90405a7c23f Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 2 Aug 2024 14:20:06 +0200 Subject: [PATCH 246/339] Add exception for the case where the device is not yet in the database: Use total count of queries as the number of queries for the new device instead of the special ARP cache counter to add also the number of queries in the DNS history imported from the long-term database. Signed-off-by: DL6ER --- src/database/network-table.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/database/network-table.c b/src/database/network-table.c index c92fd70a..9ca8fdf5 100644 --- a/src/database/network-table.c +++ b/src/database/network-table.c @@ -1318,7 +1318,7 @@ void parse_neighbor_cache(sqlite3* db) bool client_valid = false; time_t lastQuery = 0; time_t firstSeen = now; - unsigned int numQueries = 0; + unsigned int numQueries = 0, totalQueries = 0; // This client is known (by its IP address) to pihole-FTL if // findClientID() returned a non-negative index @@ -1335,6 +1335,7 @@ void parse_neighbor_cache(sqlite3* db) firstSeen = client->firstSeen; lastQuery = client->lastQuery; numQueries = client->numQueriesARP; + totalQueries = client->count; client_status[clientID] = CLIENT_ARP_COMPLETE; } else @@ -1356,6 +1357,15 @@ void parse_neighbor_cache(sqlite3* db) // and the ARP entry just came a bit delayed (reported by at least one user) dbID = find_recent_device_by_mock_hwaddr(db, ip); + // Exception for the case where the device is + // not yet in the database: Use total count of + // queries as the number of queries for the new + // device instead of the special ARP cache + // counter to add also the number of queries in + // the DNS history imported from the long-term + // database + numQueries = totalQueries; + if(dbID == DB_NODATA) { // Device not known AND no recent mock-device found ---> create new device record From 1fb9df910be4efd07f7e178ca4181d077f7250fd Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 3 Aug 2024 14:36:10 +0200 Subject: [PATCH 247/339] 127.0.0.1 is not in the ARP table and handled specially Signed-off-by: DL6ER --- src/database/network-table.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/database/network-table.c b/src/database/network-table.c index 9ca8fdf5..686e1ed4 100644 --- a/src/database/network-table.c +++ b/src/database/network-table.c @@ -860,9 +860,9 @@ static bool add_FTL_clients_to_network_table(sqlite3 *db, const enum arp_status // Add new device to database const time_t lastQuery = client->lastQuery; const time_t firstSeen = client->firstSeen; - const unsigned int numQueriesARP = client->numQueriesARP; + const unsigned int numQueries = client->count; unlock_shm(); - insert_netDB_device(db, hwaddr, firstSeen, lastQuery, numQueriesARP, macVendor); + insert_netDB_device(db, hwaddr, firstSeen, lastQuery, numQueries, macVendor); lock_shm(); // Reacquire client pointer (if may have changed when unlocking above) From 1b2ce2945483283a9e3f5d8561bedcfde2559704 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 3 Aug 2024 22:40:02 +0200 Subject: [PATCH 248/339] Fix value overflow in get_top_upstreams() due to in_port_t being an unsigned 16 bit integer where we need a signed data type. This is a regression of #2001 Signed-off-by: DL6ER --- src/api/stats.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/api/stats.c b/src/api/stats.c index 18227e3c..fcfa02a6 100644 --- a/src/api/stats.c +++ b/src/api/stats.c @@ -570,7 +570,7 @@ cJSON *get_top_upstreams(struct ftl_conn *api, const bool upstreams_only) { int count = 0; const char* ip, *name; - in_port_t port = -1; + int port = -1; // Need signed data type here as -1 means: no port applicable double responsetime = 0.0, uncertainty = 0.0; if(i == -2) From 23c26071654f6ca80600f2f899610cee6faf6af4 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 5 Aug 2024 19:05:13 +0200 Subject: [PATCH 249/339] Add new GET /api/padd endpoint Signed-off-by: DL6ER --- .github/.codespellignore | 1 + src/api/CMakeLists.txt | 1 + src/api/api.c | 1 + src/api/api.h | 15 +- src/api/config.c | 40 ++-- src/api/dns.c | 17 +- src/api/docs/CMakeLists.txt | 1 + src/api/docs/content/specs/main.yaml | 5 + src/api/docs/content/specs/padd.yaml | 248 +++++++++++++++++++++ src/api/docs/docs.h | 5 + src/api/info.c | 35 ++- src/api/network.c | 18 +- src/api/padd.c | 308 +++++++++++++++++++++++++++ src/api/queries.c | 4 +- src/api/stats.c | 79 ++++--- src/datastructure.c | 16 ++ src/datastructure.h | 1 + 17 files changed, 717 insertions(+), 78 deletions(-) create mode 100644 src/api/docs/content/specs/padd.yaml create mode 100644 src/api/padd.c diff --git a/.github/.codespellignore b/.github/.codespellignore index 5ca20bf8..6ffa6265 100644 --- a/.github/.codespellignore +++ b/.github/.codespellignore @@ -11,3 +11,4 @@ mmapped dnsmasq iif prefered +padd diff --git a/src/api/CMakeLists.txt b/src/api/CMakeLists.txt index e42a2505..a96a0f30 100644 --- a/src/api/CMakeLists.txt +++ b/src/api/CMakeLists.txt @@ -20,6 +20,7 @@ set(sources dhcp.c dns.c network.c + padd.c history.c info.c list.c diff --git a/src/api/api.c b/src/api/api.c index 8d21de2f..32acbb61 100644 --- a/src/api/api.c +++ b/src/api/api.c @@ -102,6 +102,7 @@ static struct { { "/api/action/restartdns", "", api_action_restartDNS, { API_PARSE_JSON, 0 }, true, HTTP_POST }, { "/api/action/flush/logs", "", api_action_flush_logs, { API_PARSE_JSON, 0 }, true, HTTP_POST }, { "/api/action/flush/arp", "", api_action_flush_arp, { API_PARSE_JSON, 0 }, true, HTTP_POST }, + { "/api/padd", "", api_padd, { API_PARSE_JSON, 0 }, true, HTTP_GET }, { "/api/docs", "", api_docs, { API_PARSE_JSON, 0 }, false, HTTP_GET }, }; diff --git a/src/api/api.h b/src/api/api.h index 793da3c6..b72a1e92 100644 --- a/src/api/api.h +++ b/src/api/api.h @@ -17,6 +17,8 @@ #include "webserver/http-common.h" // regex_t #include "regex_r.h" +// enum conf_type +#include "config/config.h" // Common definitions #define LOCALHOSTv4 "127.0.0.1" @@ -27,6 +29,7 @@ int api_handler(struct mg_connection *conn, void *ignored); // Statistic methods int __attribute__((pure)) cmpdesc(const void *a, const void *b); +unsigned int get_active_clients(void); int api_stats_summary(struct ftl_conn *api); int api_stats_query_types(struct ftl_conn *api); int api_stats_upstreams(struct ftl_conn *api); @@ -37,7 +40,7 @@ cJSON *get_top_domains(struct ftl_conn *api, const int count, const bool blocked, const bool domains_only); cJSON *get_top_clients(struct ftl_conn *api, const int count, const bool blocked, const bool clients_only, - const bool names_only); + const bool names_only, const bool ip_if_no_name); cJSON *get_top_upstreams(struct ftl_conn *api, const bool upstreams_only); // History methods @@ -71,9 +74,15 @@ int api_info_messages_count(struct ftl_conn *api); int api_info_messages(struct ftl_conn *api); int api_info_metrics(struct ftl_conn *api); int api_info_login(struct ftl_conn *api); +cJSON *read_sys_property(const char *path); +int get_system_obj(struct ftl_conn *api, cJSON *system); +int get_sensors_obj(struct ftl_conn *api, cJSON *sensors, const bool add_list); +int get_version_obj(struct ftl_conn *api, cJSON *version); // Config methods int api_config(struct ftl_conn *api); +int get_json_config(struct ftl_conn *api, cJSON *json, const bool detailed); +cJSON *addJSONConfValue(const enum conf_type conf_type, union conf_value *val); // Log methods int api_logs(struct ftl_conn *api); @@ -84,6 +93,7 @@ int api_network_routes(struct ftl_conn *api); int api_network_interfaces(struct ftl_conn *api); int api_network_devices(struct ftl_conn *api); int api_client_suggestions(struct ftl_conn *api); +int get_gateway(struct ftl_conn *api, cJSON * json, const bool detailed); // DNS methods int api_dns_blocking(struct ftl_conn *api); @@ -132,4 +142,7 @@ int api_search(struct ftl_conn *api); int api_dhcp_leases_GET(struct ftl_conn *api); int api_dhcp_leases_DELETE(struct ftl_conn *api); +// PADD methods +int api_padd(struct ftl_conn *api); + #endif // ROUTES_H diff --git a/src/api/config.c b/src/api/config.c index 91ad30c3..c5d005bb 100644 --- a/src/api/config.c +++ b/src/api/config.c @@ -92,7 +92,7 @@ static cJSON *get_or_create_object(cJSON *parent, const char *path_element) // This function is used to add a property to the JSON output using the // appropriate type of the config item to add. -static cJSON *addJSONvalue(const enum conf_type conf_type, union conf_value *val) +cJSON *addJSONConfValue(const enum conf_type conf_type, union conf_value *val) { switch(conf_type) { @@ -464,17 +464,9 @@ static const char *getJSONvalue(struct conf_item *conf_item, cJSON *elem, struct return NULL; } -static int api_config_get(struct ftl_conn *api) +int get_json_config(struct ftl_conn *api, cJSON *json, const bool detailed) { - // Parse query string parameters - bool detailed = false; - if(api->request->query_string != NULL) - { - // Check if we should return detailed config information - get_bool_var(api->request->query_string, "detailed", &detailed); - } - - // Create root JSON object + // Create root config object cJSON *config_j = JSON_NEW_OBJECT(); // Does the user request only a subset of /config? @@ -552,7 +544,7 @@ static int api_config_get(struct ftl_conn *api) else { // Add current value - cJSON *val = addJSONvalue(conf_item->t, &conf_item->v); + cJSON *val = addJSONConfValue(conf_item->t, &conf_item->v); if(val == NULL) { log_warn("Cannot format config item type %s of type %i", @@ -563,7 +555,7 @@ static int api_config_get(struct ftl_conn *api) } // Add default value - cJSON *dval = addJSONvalue(conf_item->t, &conf_item->d); + cJSON *dval = addJSONConfValue(conf_item->t, &conf_item->d); if(dval == NULL) { log_warn("Cannot format config item type %s of type %i", @@ -592,7 +584,7 @@ static int api_config_get(struct ftl_conn *api) else { // Create the config item leaf object - cJSON *leaf = addJSONvalue(conf_item->t, &conf_item->v); + cJSON *leaf = addJSONConfValue(conf_item->t, &conf_item->v); if(leaf == NULL) { log_warn("Cannot format config item type %s of type %i", @@ -607,8 +599,6 @@ static int api_config_get(struct ftl_conn *api) // Release allocated memory free_config_path(requested_path); - cJSON *json = JSON_NEW_OBJECT(); - // Add topics and DNS server suggestions if in detailed mode if(detailed) { @@ -650,6 +640,24 @@ static int api_config_get(struct ftl_conn *api) // Build and return JSON response JSON_ADD_ITEM_TO_OBJECT(json, "config", config_j); + + return 0; +} + +static int api_config_get(struct ftl_conn *api) +{ + // Parse query string parameters + bool detailed = false; + if(api->request->query_string != NULL) + { + // Check if we should return detailed config information + get_bool_var(api->request->query_string, "detailed", &detailed); + } + + cJSON *json = JSON_NEW_OBJECT(); + get_json_config(api, json, detailed); + + // Build and return JSON response JSON_SEND_OBJECT(json); } diff --git a/src/api/dns.c b/src/api/dns.c index 489ed7df..a7747868 100644 --- a/src/api/dns.c +++ b/src/api/dns.c @@ -31,21 +31,8 @@ static int get_blocking(struct ftl_conn *api) // Return current status cJSON *json = JSON_NEW_OBJECT(); const enum blocking_status blocking = get_blockingstatus(); - switch(blocking) - { - case BLOCKING_ENABLED: - JSON_REF_STR_IN_OBJECT(json, "blocking", "enabled"); - break; - case BLOCKING_DISABLED: - JSON_REF_STR_IN_OBJECT(json, "blocking", "disabled"); - break; - case DNS_FAILED: - JSON_REF_STR_IN_OBJECT(json, "blocking", "failure"); - break; - case BLOCKING_UNKNOWN: - JSON_REF_STR_IN_OBJECT(json, "blocking", "unknown"); - break; - } + const char *status = get_blocking_status_str(blocking); + JSON_REF_STR_IN_OBJECT(json, "blocking", status); // Get timer information (if applicable) double delay; diff --git a/src/api/docs/CMakeLists.txt b/src/api/docs/CMakeLists.txt index 8df3045a..8e22c45d 100644 --- a/src/api/docs/CMakeLists.txt +++ b/src/api/docs/CMakeLists.txt @@ -34,6 +34,7 @@ set(sources hex/specs/logs.yaml hex/specs/main.yaml hex/specs/network.yaml + hex/specs/padd.yaml hex/specs/queries.yaml hex/specs/search.yaml hex/specs/stats.yaml diff --git a/src/api/docs/content/specs/main.yaml b/src/api/docs/content/specs/main.yaml index 6aa7dc79..5f1e35ea 100644 --- a/src/api/docs/content/specs/main.yaml +++ b/src/api/docs/content/specs/main.yaml @@ -63,6 +63,8 @@ tags: description: Methods used to gather advanced information about your network - name: "Actions" description: Methods used to trigger certain actions on your Pi-hole + - name: "PADD" + description: Methods used to query Pi-hole from PADD @@ -274,6 +276,9 @@ paths: /docs: $ref: 'docs.yaml#/components/paths/docs' + /padd: + $ref: 'padd.yaml#/components/paths/padd' + components: securitySchemes: query_sid: diff --git a/src/api/docs/content/specs/padd.yaml b/src/api/docs/content/specs/padd.yaml new file mode 100644 index 00000000..ea7ead7b --- /dev/null +++ b/src/api/docs/content/specs/padd.yaml @@ -0,0 +1,248 @@ +openapi: 3.0.2 +components: + paths: + padd: + get: + summary: Get summarized data for PADD + tags: + - "PADD" + operationId: "get_padd" + parameters: + - in: query + description: (Optional) Return full data + name: full + schema: + type: boolean + required: false + example: true + responses: + '200': + description: OK + content: + application/json: + schema: + allOf: + - $ref: 'padd.yaml#/components/schemas/padd' + - $ref: 'info.yaml#/components/schemas/system' + - $ref: 'info.yaml#/components/schemas/version' + - $ref: 'common.yaml#/components/schemas/took' + '401': + description: Unauthorized + content: + application/json: + schema: + allOf: + - $ref: 'common.yaml#/components/errors/unauthorized' + - $ref: 'common.yaml#/components/schemas/took' + + schemas: + padd: + type: object + properties: + recent_blocked: + type: string + description: "Most recent blocked domain" + nullable: true + example: "bad.example.com" + top_domain: + type: string + description: "Most requested domain" + nullable: true + example: "good.example.com" + top_blocked: + type: string + description: "Most blocked domain" + nullable: true + example: "bad.example.com" + top_client: + type: string + description: "Most active client" + nullable: true + example: "localhost" + active_clients: + type: integer + description: "Number of active clients" + example: 22 + gravity_size: + type: integer + description: "Gravity list size" + example: 225382 + blocking: + type: string + description: "Blocking status" + example: "enabled" + queries: + type: object + properties: + total: + type: integer + description: "Total number of queries within the last 24 hours" + example: 92258 + blocked: + type: integer + description: "Number of blocked queries" + example: 4784 + percent_blocked: + type: number + description: "Percentage of blocked queries" + example: 5.18 + cache: + type: object + properties: + size: + type: integer + description: "Total cache size" + example: 10000 + inserted: + type: integer + description: "Number of inserted cache entries" + example: 233 + evicted: + type: integer + description: "Number of evicted cache entries" + example: 0 + iface: + type: object + description: "Default interfaces" + properties: + v4: + type: object + description: "IPv4 interface" + properties: + addr: + type: string + description: "Primary address" + nullable: true # there may be no IPv4 address + example: "192.168.2.11" + rx_bytes: + type: object + description: "Received bytes" + properties: + value: + type: number + example: 76.46 + unit: + type: string + example: "G" + tx_bytes: + type: object + description: "Transmitted bytes" + properties: + value: + type: number + example: 68.58 + unit: + type: string + example: "G" + num_addrs: + type: integer + description: "Number of addresses on the interface" + example: 1 + name: + type: string + description: "Interface name" + example: "eth0" + gw_addr: + type: string + description: "Gateway address" + nullable: true # there may be no IPv4 gateway + example: "192.168.2.1" + v6: + type: object + description: "IPv6 interface" + properties: + addr: + type: string + description: "Primary address" + nullable: true # there may be no IPv6 address + example: "fe80::b0e4:1b1e:7b7d:5855" + num_addrs: + type: integer + description: "Number of addresses on the interface" + example: 3 + name: + type: string + description: "Interface name" + example: "eth0" + gw_addr: + type: string + description: "Gateway address" + nullable: true # there may be no IPv6 gateway + example: "fe80::b0e4:1b1e:7b7d:1b1e" + node_name: + type: string + description: "Pi-hole host's name" + example: "pihole" + host_model: + type: string + description: "Host model" + example: "Raspberry Pi 3 Model B Plus Rev 1.3" + nullable: true + config: + type: object + description: "Pi-hole configuration (excerpt)" + properties: + dhcp_active: + type: boolean + description: "DHCP server status" + example: true + dhcp_start: + type: string + description: "DHCP start address" + example: "192.168.0.1" + dhcp_end: + type: string + description: "DHCP end address" + example: "192.168.0.254" + dhcp_ipv6: + type: boolean + description: "DHCPv6 server status" + example: false + dns_domain: + type: string + description: "DNS domain" + example: "lan" + dns_port: + type: integer + description: "DNS port" + example: 53 + dns_num_upstreams: + type: integer + description: "Number of upstream DNS servers" + example: 1 + dns_dnssec: + type: boolean + description: "DNSSEC status" + example: true + dns_revServer_active: + type: boolean + description: "Reverse DNS server status" + example: false + "%cpu": + type: number + description: "CPU usage" + example: 0.0 + "%mem": + type: number + description: "Memory usage" + example: 1.5 + pid: + type: integer + description: "FTL's process ID" + example: 1639 + sensors: + type: object + properties: + cpu_temp: + type: number + description: "CPU temperature" + nullable: true + example: 45.0 + hot_limit: + type: number + description: "Temperature limit" + example: 80.0 + unit: + type: string + description: "Temperature unit" + example: "C" diff --git a/src/api/docs/docs.h b/src/api/docs/docs.h index 2884c88d..66c3ad70 100644 --- a/src/api/docs/docs.h +++ b/src/api/docs/docs.h @@ -132,6 +132,10 @@ static const unsigned char specs_action_yaml[] = { #include "hex/specs/action.yaml" }; +static const unsigned char specs_padd_yaml[] = { +#include "hex/specs/padd.yaml" +}; + struct { const char *path; const char *mime_type; @@ -168,6 +172,7 @@ struct { {"specs/stats.yaml", "text/plain", (const char*)specs_stats_yaml, sizeof(specs_stats_yaml)}, {"specs/teleporter.yaml", "text/plain", (const char*)specs_teleporter_yaml, sizeof(specs_teleporter_yaml)}, {"specs/action.yaml", "text/plain", (const char*)specs_action_yaml, sizeof(specs_action_yaml)}, + {"specs/padd.yaml", "text/plain", (const char*)specs_padd_yaml, sizeof(specs_padd_yaml)}, }; #endif // API_DOCS_H diff --git a/src/api/info.c b/src/api/info.c index 581fc0b7..6256d6a0 100644 --- a/src/api/info.c +++ b/src/api/info.c @@ -157,7 +157,7 @@ int api_info_database(struct ftl_conn *api) JSON_SEND_OBJECT(json); } -static int get_system_obj(struct ftl_conn *api, cJSON *system) +int get_system_obj(struct ftl_conn *api, cJSON *system) { const int nprocs = get_nprocs(); struct sysinfo info; @@ -465,7 +465,7 @@ static int get_hwmon_sensors(struct ftl_conn *api, cJSON *sensors) return 0; } -static cJSON *read_sys_property(const char *path) +cJSON *read_sys_property(const char *path) { if(!file_exists(path)) return cJSON_CreateNull(); @@ -641,16 +641,15 @@ int api_info_host(struct ftl_conn *api) JSON_SEND_OBJECT(json); } -int api_info_sensors(struct ftl_conn *api) +int get_sensors_obj(struct ftl_conn *api, cJSON *sensors, const bool add_list) { - cJSON *sensors = JSON_NEW_OBJECT(); - // Get sensors array cJSON *list = JSON_NEW_ARRAY(); int ret = get_hwmon_sensors(api, list); if (ret != 0) return ret; - JSON_ADD_ITEM_TO_OBJECT(sensors, "list", list); + if(add_list) + JSON_ADD_ITEM_TO_OBJECT(sensors, "list", list); // Loop over available sensors and try to identify the most suitable CPU temperature sensor int cpu_temp_sensor = -1; @@ -708,12 +707,25 @@ int api_info_sensors(struct ftl_conn *api) unit = "K"; JSON_REF_STR_IN_OBJECT(sensors, "unit", unit); + if(!add_list) + cJSON_Delete(list); + + return 0; +} + +int api_info_sensors(struct ftl_conn *api) +{ + cJSON *sensors = JSON_NEW_OBJECT(); + int ret = get_sensors_obj(api, sensors, true); + if (ret != 0) + return ret; + cJSON *json = JSON_NEW_OBJECT(); JSON_ADD_ITEM_TO_OBJECT(json, "sensors", sensors); JSON_SEND_OBJECT(json); } -int api_info_version(struct ftl_conn *api) +int get_version_obj(struct ftl_conn *api, cJSON *version) { char *line = NULL; size_t len = 0; @@ -802,8 +814,6 @@ int api_info_version(struct ftl_conn *api) JSON_REF_STR_IN_OBJECT(ftl_local, "version", get_FTL_version()); JSON_REF_STR_IN_OBJECT(ftl_local, "date", GIT_DATE); - cJSON *version = JSON_NEW_OBJECT(); - cJSON *core = JSON_NEW_OBJECT(); JSON_ADD_NULL_IF_NOT_EXISTS(core_local, "branch"); JSON_ADD_NULL_IF_NOT_EXISTS(core_local, "version"); @@ -839,7 +849,14 @@ int api_info_version(struct ftl_conn *api) JSON_ADD_NULL_IF_NOT_EXISTS(docker, "remote"); JSON_ADD_ITEM_TO_OBJECT(version, "docker", docker); + return 0; +} + +int api_info_version(struct ftl_conn *api) +{ // Send reply + cJSON *version = JSON_NEW_OBJECT(); + get_version_obj(api, version); cJSON *json = JSON_NEW_OBJECT(); JSON_ADD_ITEM_TO_OBJECT(json, "version", version); JSON_SEND_OBJECT(json); diff --git a/src/api/network.c b/src/api/network.c index 6ff36a62..67d80c57 100644 --- a/src/api/network.c +++ b/src/api/network.c @@ -32,11 +32,8 @@ // nlroutes(), nladdrs(), nllinks() #include "tools/netlink.h" -int api_network_gateway(struct ftl_conn *api) +int get_gateway(struct ftl_conn *api, cJSON * json, const bool detailed) { - // Get ?detailed parameter - bool detailed = false; - get_bool_var(api->request->query_string, "detailed", &detailed); // Get routing information cJSON *routes = JSON_NEW_ARRAY(); @@ -106,7 +103,6 @@ int api_network_gateway(struct ftl_conn *api) } // Send gateway information - cJSON *json = JSON_NEW_OBJECT(); JSON_ADD_ITEM_TO_OBJECT(json, "gateway", gateway); if(detailed) @@ -121,6 +117,18 @@ int api_network_gateway(struct ftl_conn *api) cJSON_Delete(interfaces); } + return 0; +} + +int api_network_gateway(struct ftl_conn *api) +{ + // Get ?detailed parameter + bool detailed = false; + get_bool_var(api->request->query_string, "detailed", &detailed); + + cJSON *json = JSON_NEW_OBJECT(); + get_gateway(api, json, detailed); + JSON_SEND_OBJECT(json); } diff --git a/src/api/padd.c b/src/api/padd.c new file mode 100644 index 00000000..a3db08b9 --- /dev/null +++ b/src/api/padd.c @@ -0,0 +1,308 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2019 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* API Implementation /api/dns +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ + +#include "FTL.h" +#include "webserver/http-common.h" +#include "webserver/json_macros.h" +#include "api.h" +// lock_shm() and unlock_shm() +#include "shmem.h" +// counters +#include "datastructure.h" +// get_dnsmasq_metrics(&metrics) +#include "metrics.h" +// get_blockingstatus() +#include "config/config.h" +// uname() +#include +// nlroutes(), nladdrs(), nllinks() +#include "tools/netlink.h" +// struct proc_mem, getProcessMemory() +#include "procps.h" +// getcpu_percentage() +#include "daemon.h" + +int api_padd(struct ftl_conn *api) +{ + // Parse parameters + bool full = true; + if(api->request->query_string != NULL) + get_bool_var(api->request->query_string, "full", &full); + + cJSON *json = JSON_NEW_OBJECT(); + // Lock shared memory + lock_shm(); + + const int total = counters->queries; + const int blocked = get_blocked_count(); + const unsigned int active_clients = get_active_clients(); + const int num_gravity = counters->database.gravity; + + // If privacy level is set to hide domains, do not return the most + // recent blocked domain + if(config.misc.privacylevel.v.privacy_level < PRIVACY_HIDE_DOMAINS) + { + // Find most recently blocked query + for(int queryID = counters->queries - 1; queryID > 0 ; queryID--) + { + const queriesData* query = getQuery(queryID, true); + if(query == NULL) + continue; + + if(query->flags.blocked) + { + // Ask subroutine for domain. It may return "hidden" depending on + // the privacy settings at the time the query was made + const char *domain = getDomainString(query); + if(domain == NULL) + continue; + + JSON_COPY_STR_TO_OBJECT(json, "recent_blocked", domain); + break; + } + } + } + + // Unlock shared memory + unlock_shm(); + + // Add the number of active clients, the size of the gravity list + JSON_ADD_NUMBER_TO_OBJECT(json, "active_clients", active_clients); + JSON_ADD_NUMBER_TO_OBJECT(json, "gravity_size", num_gravity); + + cJSON *top_domains = get_top_domains(api, 1, false, true); + if(cJSON_GetArraySize(top_domains) == 0) + { + JSON_ADD_NULL_TO_OBJECT(json, "top_domain"); + } + else + { + cJSON *top_domain = cJSON_GetArrayItem(top_domains, 0); + const char *domain = cJSON_GetStringValue(top_domain); + JSON_COPY_STR_TO_OBJECT(json, "top_domain", domain); + } + cJSON_Delete(top_domains); + cJSON *top_blocked = get_top_domains(api, 1, true, true); + if(cJSON_GetArraySize(top_blocked) == 0) + { + JSON_ADD_NULL_TO_OBJECT(json, "top_blocked"); + } + else + { + cJSON *top_block = cJSON_GetArrayItem(top_blocked, 0); + const char *domain = cJSON_GetStringValue(top_block); + JSON_COPY_STR_TO_OBJECT(json, "top_blocked", domain); + } + cJSON *top_clients = get_top_clients(api, 1, false, true, false, true); + if(cJSON_GetArraySize(top_clients) == 0) + { + JSON_ADD_NULL_TO_OBJECT(json, "top_client"); + } + else + { + cJSON *top_client = cJSON_GetArrayItem(top_clients, 0); + const char *client = cJSON_GetStringValue(top_client); + JSON_COPY_STR_TO_OBJECT(json, "top_client", client); + } + + // Add a null entry if the domain is hidden or there is no recent + // blocked domain (e.g. when blocking is disabled) + JSON_ADD_NULL_IF_NOT_EXISTS(json, "recent_blocked"); + + // Calculate percentage of blocked queries + float percent_blocked = 0.0f; + // Avoid 1/0 condition + if(total > 0) + percent_blocked = 1e2f*blocked/total; + + // Add the blocking status + const char *blocking = get_blocking_status_str(get_blockingstatus()); + JSON_REF_STR_IN_OBJECT(json, "blocking", blocking); + + // Add query statistics + cJSON *queries = JSON_NEW_OBJECT(); + JSON_ADD_NUMBER_TO_OBJECT(queries, "total", total); + JSON_ADD_NUMBER_TO_OBJECT(queries, "blocked", blocked); + JSON_ADD_NUMBER_TO_OBJECT(queries, "percent_blocked", percent_blocked); + JSON_ADD_ITEM_TO_OBJECT(json, "queries", queries); + + // Add cache statistics + cJSON *cache = JSON_NEW_OBJECT(); + struct metrics metrics = { 0 }; + get_dnsmasq_metrics(&metrics); + JSON_ADD_NUMBER_TO_OBJECT(cache, "size", metrics.dns.cache.size); + JSON_ADD_NUMBER_TO_OBJECT(cache, "inserted", metrics.dns.cache.inserted); + JSON_ADD_NUMBER_TO_OBJECT(cache, "evicted", metrics.dns.cache.live_freed); + JSON_ADD_ITEM_TO_OBJECT(json, "cache", cache); + + // info/system + cJSON *system = JSON_NEW_OBJECT(); + get_system_obj(api, system); + JSON_ADD_ITEM_TO_OBJECT(json, "system", system); + + // info/host + struct utsname un = { 0 }; + uname(&un); + JSON_COPY_STR_TO_OBJECT(json, "node_name", un.nodename); + JSON_ADD_ITEM_TO_OBJECT(json, "host_model", read_sys_property("/sys/firmware/devicetree/base/model")); + + // Expensive calls, do only if full is requested + if(full) + { + // network/gateway + cJSON *gateway_ = JSON_NEW_OBJECT(); + get_gateway(api, gateway_, true); + + cJSON *gateway = cJSON_GetObjectItemCaseSensitive(gateway_, "gateway"); + cJSON *interfaces = cJSON_GetObjectItemCaseSensitive(gateway_, "interfaces"); + + // Loop over gateway and find first entry with family == "inet" + cJSON *entry = NULL; + const char *gw_v4_name = NULL, *gw_v6_name = NULL; + const char *gw_v4_addr = NULL, *gw_v6_addr = NULL; + cJSON_ArrayForEach(entry, gateway) + { + cJSON *family = cJSON_GetObjectItemCaseSensitive(entry, "family"); + if(gw_v4_name == NULL && strcmp(cJSON_GetStringValue(family), "inet") == 0) + { + gw_v4_name = cJSON_GetStringValue(cJSON_GetObjectItemCaseSensitive(entry, "interface")); + gw_v4_addr = cJSON_GetStringValue(cJSON_GetObjectItemCaseSensitive(entry, "address")); + } + if(gw_v6_name == NULL && strcmp(cJSON_GetStringValue(family), "inet6") == 0) + { + gw_v6_name = cJSON_GetStringValue(cJSON_GetObjectItemCaseSensitive(entry, "interface")); + gw_v6_addr = cJSON_GetStringValue(cJSON_GetObjectItemCaseSensitive(entry, "address")); + } + + // Break if both addresses are found + if(gw_v4_name && gw_v6_name) + break; + } + + // If no IPv6 gateway is found, use the IPv4 gateway + if(gw_v6_name == NULL) + gw_v6_name = gw_v4_name; + + // Iterate over all interfaces until we find the one associated + // with the IPv4 gateway + cJSON *iface_v4 = JSON_NEW_OBJECT(); + cJSON *iface_v6 = JSON_NEW_OBJECT(); + unsigned int v4_addrs = 0, v6_addrs = 0; + cJSON_ArrayForEach(entry, interfaces) + { + if(strcmp(cJSON_GetStringValue(cJSON_GetObjectItemCaseSensitive(entry, "name")), gw_v4_name) == 0) + { + // Add first interface address with family == inet + cJSON *addr = NULL; + cJSON *addrs = cJSON_GetObjectItemCaseSensitive(entry, "addresses"); + cJSON_ArrayForEach(addr, addrs) + { + cJSON *family = cJSON_GetObjectItemCaseSensitive(addr, "family"); + if(strcmp(cJSON_GetStringValue(family), "inet") == 0) + { + if(v4_addrs == 0) + { + cJSON *_addr = cJSON_GetObjectItemCaseSensitive(addr, "address"); + JSON_COPY_STR_TO_OBJECT(iface_v4, "addr", cJSON_GetStringValue(_addr)); + } + v4_addrs++; + } + } + + // Add NULL if no IPv4 address is found + if(v4_addrs == 0) + JSON_ADD_NULL_TO_OBJECT(iface_v4, "addr"); + + // Also add IPv4 interface statistics + cJSON *stats = cJSON_GetObjectItemCaseSensitive(entry, "stats"); + cJSON *rx_bytes = cJSON_GetObjectItemCaseSensitive(stats, "rx_bytes"); + JSON_ADD_ITEM_TO_OBJECT(iface_v4, "rx_bytes", cJSON_Duplicate(rx_bytes, true)); + cJSON *tx_bytes = cJSON_GetObjectItemCaseSensitive(stats, "tx_bytes"); + JSON_ADD_ITEM_TO_OBJECT(iface_v4, "tx_bytes", cJSON_Duplicate(tx_bytes, true)); + } + if(strcmp(cJSON_GetStringValue(cJSON_GetObjectItemCaseSensitive(entry, "name")), gw_v6_name) == 0) + { + // Add first interface address with family == inet + cJSON *addr = NULL; + cJSON *addrs = cJSON_GetObjectItemCaseSensitive(entry, "addresses"); + cJSON_ArrayForEach(addr, addrs) + { + cJSON *family = cJSON_GetObjectItemCaseSensitive(addr, "family"); + if(strcmp(cJSON_GetStringValue(family), "inet6") == 0) + { + if(v6_addrs == 0) + { + cJSON *_addr = cJSON_GetObjectItemCaseSensitive(addr, "address"); + JSON_COPY_STR_TO_OBJECT(iface_v6, "addr", cJSON_GetStringValue(_addr)); + } + v6_addrs++; + } + } + + // Add NULL if no IPv6 address is found + if(v6_addrs == 0) + JSON_ADD_NULL_TO_OBJECT(iface_v6, "addr"); + } + } + + // Add the number of addresses found + JSON_ADD_NUMBER_TO_OBJECT(iface_v4, "num_addrs", v4_addrs); + JSON_ADD_NUMBER_TO_OBJECT(iface_v6, "num_addrs", v6_addrs); + + // Add the interfaces to the gateway object + JSON_COPY_STR_TO_OBJECT(iface_v4, "name", gw_v4_name); + JSON_COPY_STR_TO_OBJECT(iface_v4, "gw_addr", gw_v4_addr); + JSON_COPY_STR_TO_OBJECT(iface_v6, "name", gw_v6_name); + JSON_COPY_STR_TO_OBJECT(iface_v6, "gw_addr", gw_v6_addr); + + // Create interface object + cJSON *iface = JSON_NEW_OBJECT(); + JSON_ADD_ITEM_TO_OBJECT(iface, "v4", iface_v4); + JSON_ADD_ITEM_TO_OBJECT(iface, "v6", iface_v6); + JSON_ADD_ITEM_TO_OBJECT(json, "iface", iface); + + // Free memory + cJSON_Delete(gateway_); + + // info/version + cJSON *version = JSON_NEW_OBJECT(); + get_version_obj(api, version); + JSON_ADD_ITEM_TO_OBJECT(json, "version", version); + } + + // subset of config + cJSON *jconfig = JSON_NEW_OBJECT(); + JSON_ADD_BOOL_TO_OBJECT(jconfig, "dhcp_active", config.dhcp.active.v.b); + JSON_ADD_ITEM_TO_OBJECT(jconfig, "dhcp_start", addJSONConfValue(config.dhcp.start.t, &config.dhcp.start.v)); + JSON_ADD_ITEM_TO_OBJECT(jconfig, "dhcp_end", addJSONConfValue(config.dhcp.end.t, &config.dhcp.end.v)); + JSON_ADD_BOOL_TO_OBJECT(jconfig, "dhcp_ipv6", config.dhcp.ipv6.v.b); + JSON_COPY_STR_TO_OBJECT(jconfig, "dns_domain", config.dns.domain.v.s); + JSON_ADD_NUMBER_TO_OBJECT(jconfig, "dns_port", config.dns.port.v.u16); + JSON_ADD_NUMBER_TO_OBJECT(jconfig, "dns_num_upstreams", cJSON_GetArraySize(config.dns.upstreams.v.json)); + JSON_ADD_BOOL_TO_OBJECT(jconfig, "dns_dnssec", config.dns.dnssec.v.b); + JSON_ADD_BOOL_TO_OBJECT(jconfig, "dns_revServer_active", cJSON_GetArraySize(config.dns.revServers.v.json) > 0); + JSON_ADD_ITEM_TO_OBJECT(json, "config", jconfig); + + // subset of info/ftl + struct proc_mem pmem = { 0 }; + struct proc_meminfo mem = { 0 }; + parse_proc_meminfo(&mem); + getProcessMemory(&pmem, mem.total); + JSON_ADD_NUMBER_TO_OBJECT(json, "%mem", pmem.VmRSS_percent); + JSON_ADD_NUMBER_TO_OBJECT(json, "%cpu", get_cpu_percentage()); + JSON_ADD_NUMBER_TO_OBJECT(json, "pid", getpid()); + + // info/sensors -> CPU temp sensor + cJSON *sensors = JSON_NEW_OBJECT(); + get_sensors_obj(api, sensors, false); + JSON_ADD_ITEM_TO_OBJECT(json, "sensors", sensors); + + JSON_SEND_OBJECT(json); +} diff --git a/src/api/queries.c b/src/api/queries.c index bc8dac1d..5b94d940 100644 --- a/src/api/queries.c +++ b/src/api/queries.c @@ -114,8 +114,8 @@ int api_queries_suggestions(struct ftl_conn *api) cJSON_Delete(blocked); // Get clients, both by IP and names - cJSON *client_ip = get_top_clients(api, count, false, true, false); - cJSON *client_name = get_top_clients(api, count, false, true, true); + cJSON *client_ip = get_top_clients(api, count, false, true, false, false); + cJSON *client_name = get_top_clients(api, count, false, true, true, false); // Delete duplicate entries from client_name cJSON_unique_array(client_name); diff --git a/src/api/stats.c b/src/api/stats.c index fcfa02a6..12024bfc 100644 --- a/src/api/stats.c +++ b/src/api/stats.c @@ -92,26 +92,54 @@ static int get_query_types_obj(struct ftl_conn *api, cJSON *types) return 0; } +// shmem needs to be locked while calling this function +unsigned int get_active_clients(void) +{ + unsigned int activeclients = 0; + for(int clientID=0; clientID < counters->clients; clientID++) + { + // Get client pointer + const clientsData* client = getClient(clientID, true); + if(client == NULL) + continue; + + if(client->count > 0) + activeclients++; + } + + return activeclients; +} + int api_stats_summary(struct ftl_conn *api) { - const int blocked = get_blocked_count(); - const int forwarded = get_forwarded_count(); - const int cached = get_cached_count(); - const int total = counters->queries; - float percent_blocked = 0.0f; + // Lock shared memory + lock_shm(); + const int blocked = get_blocked_count(); + const int forwarded = get_forwarded_count(); + const int cached = get_cached_count(); + const int total = counters->queries; + const int num_gravity = counters->database.gravity; + const int num_clients = counters->clients; + const int num_domains = counters->domains; + + // Count clients that have been active within the most recent 24 hours + unsigned int activeclients = get_active_clients(); + + // Unlock shared memory + unlock_shm(); + + // Calculate percentage of blocked queries + float percent_blocked = 0.0f; // Avoid 1/0 condition if(total > 0) percent_blocked = 1e2f*blocked/total; - // Lock shared memory - lock_shm(); - cJSON *queries = JSON_NEW_OBJECT(); JSON_ADD_NUMBER_TO_OBJECT(queries, "total", total); JSON_ADD_NUMBER_TO_OBJECT(queries, "blocked", blocked); JSON_ADD_NUMBER_TO_OBJECT(queries, "percent_blocked", percent_blocked); - JSON_ADD_NUMBER_TO_OBJECT(queries, "unique_domains", counters->domains); + JSON_ADD_NUMBER_TO_OBJECT(queries, "unique_domains", num_domains); JSON_ADD_NUMBER_TO_OBJECT(queries, "forwarded", forwarded); JSON_ADD_NUMBER_TO_OBJECT(queries, "cached", cached); @@ -131,28 +159,12 @@ int api_stats_summary(struct ftl_conn *api) JSON_ADD_NUMBER_TO_OBJECT(replies, get_query_reply_str(reply), counters->reply[reply]); JSON_ADD_ITEM_TO_OBJECT(queries, "replies", replies); - // Count clients that have been active within the most recent 24 hours - unsigned int activeclients = 0; - for(int clientID=0; clientID < counters->clients; clientID++) - { - // Get client pointer - const clientsData* client = getClient(clientID, true); - if(client == NULL) - continue; - - if(client->count > 0) - activeclients++; - } - cJSON *clients = JSON_NEW_OBJECT(); JSON_ADD_NUMBER_TO_OBJECT(clients, "active", activeclients); - JSON_ADD_NUMBER_TO_OBJECT(clients, "total", counters->clients); + JSON_ADD_NUMBER_TO_OBJECT(clients, "total", num_clients); cJSON *gravity = JSON_NEW_OBJECT(); - JSON_ADD_NUMBER_TO_OBJECT(gravity, "domains_being_blocked", counters->database.gravity); - - // Unlock shared memory - unlock_shm(); + JSON_ADD_NUMBER_TO_OBJECT(gravity, "domains_being_blocked", num_gravity); cJSON *json = JSON_NEW_OBJECT(); JSON_ADD_ITEM_TO_OBJECT(json, "queries", queries); @@ -335,7 +347,7 @@ int api_stats_top_domains(struct ftl_conn *api) cJSON *get_top_clients(struct ftl_conn *api, const int count, const bool blocked, const bool clients_only, - const bool names_only) + const bool names_only, const bool ip_if_no_name) { // Exit before processing any data if requested via config setting if(config.misc.privacylevel.v.privacy_level >= PRIVACY_HIDE_DOMAINS_CLIENTS) @@ -449,7 +461,14 @@ cJSON *get_top_clients(struct ftl_conn *api, const int count, if(clients_only) { - if(names_only) + if(ip_if_no_name) + { + if(strlen(client_name) > 0) + cJSON_AddStringToArray(jtop_clients, client_name); + else + cJSON_AddStringToArray(jtop_clients, client_ip); + } + else if(names_only) { if(strlen(client_name) > 0) cJSON_AddStringToArray(jtop_clients, client_name); @@ -516,7 +535,7 @@ int api_stats_top_clients(struct ftl_conn *api) get_int_var(api->request->query_string, "count", &count); } - cJSON *json = get_top_clients(api, count, blocked, false, false); + cJSON *json = get_top_clients(api, count, blocked, false, false, false); JSON_SEND_OBJECT(json); } diff --git a/src/datastructure.c b/src/datastructure.c index 72376a9f..7ac9a1e2 100644 --- a/src/datastructure.c +++ b/src/datastructure.c @@ -835,6 +835,22 @@ int __attribute__ ((pure)) get_blocking_mode_val(const char *blocking_mode) return -1; } +const char * __attribute__ ((const)) get_blocking_status_str(const enum blocking_status blocking) +{ + switch(blocking) + { + case BLOCKING_ENABLED: + return "enabled"; + case BLOCKING_DISABLED: + return "disabled"; + case DNS_FAILED: + return "failure"; + case BLOCKING_UNKNOWN: + default: + return "unknown"; + } +} + bool __attribute__ ((const)) is_blocked(const enum query_status status) { switch (status) diff --git a/src/datastructure.h b/src/datastructure.h index b2b66772..48450a94 100644 --- a/src/datastructure.h +++ b/src/datastructure.h @@ -159,6 +159,7 @@ const char *get_refresh_hostnames_str(const enum refresh_hostnames refresh) __at int get_refresh_hostnames_val(const char *refresh_hostnames) __attribute__ ((pure)); const char *get_blocking_mode_str(const enum blocking_mode mode) __attribute__ ((const)); int get_blocking_mode_val(const char *blocking_mode) __attribute__ ((pure)); +const char * __attribute__ ((const)) get_blocking_status_str(const enum blocking_status blocking); const char *get_ptr_type_str(const enum ptr_type piholePTR) __attribute__ ((const)); int get_ptr_type_val(const char *piholePTR) __attribute__ ((pure)); const char *get_busy_reply_str(const enum busy_reply replyWhenBusy) __attribute__ ((const)); From e5d901a80c738c7e6fdeae2372c0ef59401a7fe5 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 6 Aug 2024 18:18:45 +0200 Subject: [PATCH 250/339] Improve pihole-FTL process concurrency FTL is trying hard to prevent you from starting another instance of itself. It does so by creating a PID file in `/run/pihole-FTL.pid` and checking for its existence. If the file is present, FTL assumes that another instance is already running and exits. It also checks for the existence of the shared memory objects `/dev/shm/FTL-*` and exits if they are present. These checks can be fooled by manually removing the PID file or shared memory objects (needs `root` privileges). This can lead to multiple instances of FTL running at the same time, which can cause various issues. The most commonly seen issue is that, when the first process needs more memory, its original shared memory objects are already gone. Hence, it actually tries to resize the shared memory objects of the second instance which, on the other hand, doesn't expect this and crashes. As the first process was not able to allocate more memory, this process will eventually crash as well. This commit resolves this by: 1. Improve support for concurrent `pihole-FTL` instances. Even when this continues to be somewhat discouraged, it is now possible to run multiple instances of `pihole-FTL` at the same time. This is achieved by ensuring that we keep the shared memory object file descriptors open as long as the process is running. This way, the shared memory objects are not removed until the process exits. This also means that the shared memory objects are not *hard* removed when they are deleted from the outside of the process (in Linux, a file continues to exist as long as there is at least one open file descriptor pointing to it). A hypothetical second instance of `pihole-FTL` will now be able to create new shared memory objects without interfering with the first instance. This is a more robust solution than the previous one, as it doesn't rely on the existence of the PID file or shared memory objects which might have been removed by external influences. 2. Check for a potential second instance earlier in the code. We move the check to before even trying to create shared memory objects. This way, we can exit early if we detect that another instance is already running. This is a more efficient solution than the previous one, as we don't need to create shared memory objects just to find out that we can't use them. 3. Add an exclusive lock on the shared memory objects. Even when this is not strictly necessary, it is a good practice to prevent other processes from interfering with the shared memory objects. This is especially important on systems which isolate processes from each other and is a safety net in rare cases such aas multiple Docker containers (isolating processes from one another by default) with (incorrectly!) host-mounted shared memory folders. Once they remove the mounting of `dev/shm`, they will again be able to run multiple `pihole-FTL` across multiple containers. Signed-off-by: DL6ER --- src/dnsmasq/util.c | 9 ++++ src/main.c | 6 ++- src/procps.c | 14 +++--- src/procps.h | 2 +- src/shmem.c | 107 +++++++++++++++++-------------------------- src/shmem.h | 5 ++ test/test_suite.bats | 29 ++++-------- 7 files changed, 79 insertions(+), 93 deletions(-) diff --git a/src/dnsmasq/util.c b/src/dnsmasq/util.c index 0c7de444..a23759a6 100644 --- a/src/dnsmasq/util.c +++ b/src/dnsmasq/util.c @@ -34,6 +34,10 @@ #include #endif +/****** Pi-hole modification ******/ +extern int is_shm_fd(const int fd); +/**********************************/ + /* SURF random number generator */ static u32 seed[32]; @@ -815,6 +819,11 @@ void close_fds(long max_fd, int spare1, int spare2, int spare3) fd == spare1 || fd == spare2 || fd == spare3) continue; + /****** Pi-hole modification ******/ + if(is_shm_fd(fd)) + continue; + /**********************************/ + close(fd); } diff --git a/src/main.c b/src/main.c index 68b22027..518a64fa 100644 --- a/src/main.c +++ b/src/main.c @@ -72,6 +72,10 @@ int main (int argc, char *argv[]) if(readFTLconf(&config, true)) log_info("Parsed config file "GLOBALTOMLPATH" successfully"); + // Check if another FTL process is already running + if(another_FTL()) + return EXIT_FAILURE; + // Set process priority set_nice(); @@ -79,8 +83,6 @@ int main (int argc, char *argv[]) if(!init_shmem()) { log_crit("Initialization of shared memory failed."); - // Check if there is already a running FTL process - check_running_FTL(); return EXIT_FAILURE; } diff --git a/src/procps.c b/src/procps.c index 8e3656a4..7362b98c 100644 --- a/src/procps.c +++ b/src/procps.c @@ -117,7 +117,7 @@ static bool get_process_creation_time(const pid_t pid, char timestr[TIMESTR_SIZE // This function prints an info message about if another FTL process is already // running. It returns true if another FTL process is already running, false // otherwise. -bool check_running_FTL(void) +bool another_FTL(void) { DIR *dirPos; struct dirent *entry; @@ -144,7 +144,7 @@ bool check_running_FTL(void) { // Note: kill(pid, 0) does not send a // signal, but merely checks if the - // process exists If the process does + // process exists. If the process does // not exist, kill() returns -1 and sets // errno to ESRCH. However, if the // process exists, but security @@ -162,20 +162,22 @@ bool check_running_FTL(void) } else { - log_debug(DEBUG_SHMEM, "Failed to parse PID in PID file"); + log_debug(DEBUG_SHMEM, "Failed to parse PID in PID file: %s", + strerror(errno)); } fclose(pidFile); } else { - log_debug(DEBUG_SHMEM, "Failed to open PID file"); + log_debug(DEBUG_SHMEM, "Failed to open PID file \"%s\": %s", + config.files.pid.v.s, strerror(errno)); } } // If already_running is true, we are done if(already_running) { - log_info("%s is already running (PID %d)!", PROCESS_NAME, pid); + log_crit("%s is already running (PID %d)!", PROCESS_NAME, pid); return true; } @@ -238,7 +240,7 @@ bool check_running_FTL(void) if(!already_running) { already_running = true; - log_info("%s is already running!", PROCESS_NAME); + log_crit("%s is already running!", PROCESS_NAME); } if(last_pid != ppid) diff --git a/src/procps.h b/src/procps.h index e707ed6b..45568adf 100644 --- a/src/procps.h +++ b/src/procps.h @@ -10,7 +10,7 @@ #ifndef PROCPS_H #define PROCPS_H -bool check_running_FTL(void); +bool another_FTL(void); struct proc_mem { // Memory currently resident in RAM (in kB) diff --git a/src/shmem.c b/src/shmem.c index 83e7da5c..efa72428 100644 --- a/src/shmem.c +++ b/src/shmem.c @@ -32,8 +32,6 @@ #include "files.h" // log_resource_shortage() #include "database/message-table.h" -// check_running_FTL() -#include "procps.h" /// The version of shared memory used #define SHARED_MEMORY_VERSION 14 @@ -149,41 +147,6 @@ static int get_dev_shm_usage(char buffer[64]) return percentage; } -// Verify the PID stored during shared memory initialization is the same as ours -// (while we initialized the shared memory objects) -static void verify_shmem_pid(void) -{ - // Open shared memory settings object - const int settingsfd = shm_open(SHARED_SETTINGS_NAME, O_RDONLY, S_IRUSR | S_IWUSR); - if(settingsfd == -1) - { - log_crit("verify_shmem_pid(): Failed to open shared memory object \"%s\": %s", - SHARED_SETTINGS_NAME, strerror(errno)); - exit(EXIT_FAILURE); - } - - ShmSettings shms = { 0 }; - if(read(settingsfd, &shms, sizeof(shms)) != sizeof(shms)) - { - log_crit("verify_shmem_pid(): Failed to read %zu bytes from shared memory object \"%s\": %s", - sizeof(shms), SHARED_SETTINGS_NAME, strerror(errno)); - exit(EXIT_FAILURE); - } - - close(settingsfd); - - // Compare the SHM's PID to the one we had when creating the SHM objects - if(shms.pid == shmem_pid) - return; - - // If we reach here, we are in serious trouble. Terminating with error - // code is the most sensible thing we can do at this point - log_crit("Shared memory is owned by a different process (PID %d)", shms.pid); - check_running_FTL(); - log_crit("Exiting now!"); - exit(EXIT_FAILURE); -} - // chown_shmem() changes the file ownership of a given shared memory object static bool chown_shmem(SharedMemory *sharedMemory, struct passwd *ent_pw) { @@ -624,25 +587,41 @@ static bool create_shm(const char *name, SharedMemory *sharedMemory, const size_ // - O_CREAT: Create the shared memory object if it does not exist. // - O_EXCL: Return an error if a shared memory object with the given name already exists. errno = 0; - const int fd = shm_open(sharedMemory->name, O_RDWR | O_CREAT | O_EXCL, S_IRUSR | S_IWUSR); + sharedMemory->fd = shm_open(sharedMemory->name, O_RDWR | O_CREAT | O_EXCL, S_IRUSR | S_IWUSR); // Check for `shm_open` error - if(fd == -1) + if(sharedMemory->fd == -1) { log_err("create_shm(): Failed to create shared memory object \"%s\": %s", name, strerror(errno)); return sharedMemory; } + // Create exclusive file lock on shared memory object + // The lock will be automatically released when the file descriptor is closed + sharedMemory->lock.l_type = F_WRLCK; // write = exclusive lock + sharedMemory->lock.l_whence = SEEK_SET; + sharedMemory->lock.l_start = 0; // lock everything from the start ... + sharedMemory->lock.l_len = 0; // ... to the end of the file (magic 0 = EOF) + + // Try to lock the shared memory object + if(fcntl(sharedMemory->fd, F_SETLK, &sharedMemory->lock) == -1) + { + log_err("create_shm(): Failed to exclusively lock shared memory object \"%s\": %s", + name, strerror(errno)); + close(sharedMemory->fd); + return sharedMemory; + } + // Allocate shared memory object to specified size // Using f[tl]allocate() will ensure that there's actually space for // this file. Otherwise we end up with a sparse file that can give // SIGBUS if we run out of space while writing to it. - const int ret = ftlallocate(fd, 0U, size); + const int ret = ftlallocate(sharedMemory->fd, 0U, size); if(ret != 0) { log_err("create_shm(): Failed to resize \"%s\" (%i) to %zu: %s (%i)", - sharedMemory->name, fd, size, strerror(errno), ret); + sharedMemory->name, sharedMemory->fd, size, strerror(errno), ret); exit(EXIT_FAILURE); } @@ -651,23 +630,19 @@ static bool create_shm(const char *name, SharedMemory *sharedMemory, const size_ used_shmem += size; // Create shared memory mapping - void *shm = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + void *shm = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, sharedMemory->fd, 0); // Check for `mmap` error if(shm == MAP_FAILED) { log_err("create_shm(): Failed to map shared memory object \"%s\" (%i): %s", - sharedMemory->name, fd, strerror(errno)); + sharedMemory->name, sharedMemory->fd, strerror(errno)); return sharedMemory; } // Initialize shared memory object to zero memset(shm, 0, size); - // Close shared memory object file descriptor as it is no longer - // needed after having called mmap() - close(fd); - sharedMemory->ptr = shm; return sharedMemory; } @@ -762,34 +737,18 @@ static bool realloc_shm(SharedMemory *sharedMemory, const size_t size1, const si // TCP requests. if(resize) { - // Verify shared memory ownership - verify_shmem_pid(); - - // Open shared memory object - const int fd = shm_open(sharedMemory->name, O_RDWR, S_IRUSR | S_IWUSR); - if(fd == -1) - { - log_crit("realloc_shm(): Failed to open shared memory object \"%s\": %s", - sharedMemory->name, strerror(errno)); - exit(EXIT_FAILURE); - } - // Allocate shared memory object to specified size // Using f[tl]allocate() will ensure that there's actually space for // this file. Otherwise we end up with a sparse file that can give // SIGBUS if we run out of space while writing to it. - const int ret = ftlallocate(fd, 0U, size); + const int ret = ftlallocate(sharedMemory->fd, 0U, size); if(ret != 0) { log_crit("realloc_shm(): Failed to resize \"%s\" (%i) to %zu: %s (%i)", - sharedMemory->name, fd, size, strerror(ret), ret); + sharedMemory->name, sharedMemory->fd, size, strerror(ret), ret); exit(EXIT_FAILURE); } - // Close shared memory object file descriptor as it is no longer - // needed after having called f[tl]allocate() - close(fd); - // Update shm counters to indicate that at least one shared memory object changed shmSettings->global_shm_counter++; local_shm_counter++; @@ -848,6 +807,11 @@ static void delete_shm(SharedMemory *sharedMemory) // Set unmapped pointer to NULL sharedMemory->ptr = NULL; + // Close shared memory file descriptor + if(close(sharedMemory->fd) != 0) + log_warn("delete_shm(): close(%i) failed: %s", sharedMemory->fd, strerror(errno)); + sharedMemory->fd = -1; + // Now you can no longer `shm_open` the memory, and once all others // unlink, it will be destroyed. if(shm_unlink(sharedMemory->name) != 0) @@ -1229,3 +1193,16 @@ DNSCacheData* _getDNSCache(int cacheID, bool checkMagic, int line, const char *f return NULL; } + +// Return 1 if this fd is associated with any shared memory object to avoid +// dnsmasq closing it during initialization +int __attribute__((pure)) is_shm_fd(const int fd) +{ + // Check all shared memory objects + for(unsigned int i = 0; i < ArraySize(sharedMemories); i++) + if(sharedMemories[i]->fd == fd) + return 1; + + // Not found + return 0; +} diff --git a/src/shmem.h b/src/shmem.h index 8a6ed627..ad35ebee 100644 --- a/src/shmem.h +++ b/src/shmem.h @@ -22,6 +22,8 @@ typedef struct { const char *name; size_t size; void *ptr; + int fd; + struct flock lock; } SharedMemory; typedef struct { @@ -140,4 +142,7 @@ void reset_per_client_regex(const int clientID); bool get_per_client_regex(const int clientID, const int regexID); void set_per_client_regex(const int clientID, const int regexID, const bool value); +// Used in dnsmasq/utils.c +int is_shm_fd(const int fd); + #endif //SHARED_MEMORY_SERVER_H diff --git a/test/test_suite.bats b/test/test_suite.bats index b7fcf867..edbb3149 100644 --- a/test/test_suite.bats +++ b/test/test_suite.bats @@ -11,8 +11,7 @@ @test "Running a second instance is detected and prevented" { run bash -c 'su pihole -s /bin/sh -c "./pihole-FTL -f"' printf "%s\n" "${lines[@]}" - [[ "${lines[@]}" == *"CRIT: Initialization of shared memory failed."* ]] - [[ "${lines[@]}" == *"INFO: pihole-FTL is already running"* ]] + [[ "${lines[@]}" == *"CRIT: pihole-FTL is already running"* ]] } @test "dnsmasq options as expected" { @@ -486,8 +485,16 @@ [[ "${lines[@]}" == "" ]] } +@test "No ERROR messages in FTL.log (besides known/intended error)" { + run bash -c 'grep "ERROR: " /var/log/pihole/FTL.log' + printf "%s\n" "${lines[@]}" + run bash -c 'grep "ERROR: " /var/log/pihole/FTL.log | grep -c -v -E "(index\.html)|(Failed to create shared memory object)|(FTLCONF_debug_api is invalid)|(Failed to set|adjust time during NTP sync: Insufficient permissions)"' + printf "count: %s\n" "${lines[@]}" + [[ ${lines[0]} == "0" ]] +} + @test "No CRIT messages in FTL.log (besides error due to starting FTL more than once)" { - run bash -c 'grep "CRIT:" /var/log/pihole/FTL.log | grep -v "CRIT: Initialization of shared memory failed"' + run bash -c 'grep "CRIT:" /var/log/pihole/FTL.log | grep -v "CRIT: pihole-FTL is already running"' printf "%s\n" "${lines[@]}" [[ "${lines[@]}" == "" ]] } @@ -1162,22 +1169,6 @@ [[ "${lines[@]}" != *"ERROR"* ]] } -@test "No ERROR messages in FTL.log (besides known/intended error)" { - run bash -c 'grep "ERROR: " /var/log/pihole/FTL.log' - printf "%s\n" "${lines[@]}" - run bash -c 'grep "ERROR: " /var/log/pihole/FTL.log | grep -c -v -E "(index\.html)|(Failed to create shared memory object)|(FTLCONF_debug_api is invalid)|(Failed to set|adjust time during NTP sync: Insufficient permissions)"' - printf "count: %s\n" "${lines[@]}" - [[ ${lines[0]} == "0" ]] -} - -@test "No CRIT messages in FTL.log (besides error due to testing to start FTL more than once)" { - run bash -c 'grep "CRIT: " /var/log/pihole/FTL.log' - printf "%s\n" "${lines[@]}" - run bash -c 'grep "CRIT: " /var/log/pihole/FTL.log | grep -c -v "Initialization of shared memory failed."' - printf "count: %s\n" "${lines[@]}" - [[ ${lines[0]} == "0" ]] -} - @test "No missing config items in pihole.toml" { run bash -c 'grep "DEBUG_CONFIG: " /var/log/pihole/FTL.log' printf "%s\n" "${lines[@]}" From 56593936a8eea251cddd9181c65f09c3f9b1e215 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 7 Aug 2024 22:12:32 +0200 Subject: [PATCH 251/339] Be more explicit about errors appending to the main FTL log file Signed-off-by: DL6ER --- src/log.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/log.c b/src/log.c index 0fbbe447..e11e09a7 100644 --- a/src/log.c +++ b/src/log.c @@ -53,8 +53,9 @@ void init_FTL_log(const char *name) FILE *logfile = NULL; if((logfile = fopen(config.files.log.ftl.v.s, "a+")) == NULL) { + printf("ERROR: Opening of FTL log (%s) failed: %s\nUsing syslog instead!\n", + config.files.log.ftl.v.s, strerror(errno)); syslog(LOG_ERR, "Opening of FTL\'s log file failed, using syslog instead!"); - printf("ERROR: Opening of FTL log (%s) failed!\n",config.files.log.ftl.v.s); config.files.log.ftl.v.s = NULL; } From a2d1d16e610d564de136ce2c2d710a1b4e454c0c Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 9 Aug 2024 19:14:34 +0200 Subject: [PATCH 252/339] Do not read config.files.log.ftl from the TOML file if it has been set to NULL due to permissions issues during startup Signed-off-by: DL6ER --- src/config/toml_reader.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/config/toml_reader.c b/src/config/toml_reader.c index 6f3f98e3..de40703d 100644 --- a/src/config/toml_reader.c +++ b/src/config/toml_reader.c @@ -141,6 +141,12 @@ bool readFTLtoml(struct config *oldconf, struct config *newconf, struct conf_item *old_conf_item = oldconf != NULL ? get_conf_item(oldconf, i) : NULL; struct conf_item *new_conf_item = get_conf_item(newconf, i); + // Do not read config.files.log.ftl from the TOML file if it has been + // set to NULL due to permissions issues during startup + if(config.files.log.ftl.v.s == NULL && + new_conf_item == &newconf->files.log.ftl) + continue; + // First try to read this config option from an environment variable // Skip reading environment variables when importing from Teleporter // If this succeeds, skip searching the TOML file for this config item From e6f5bf7901b159ca1ba1147598e97da4e4bcff2c Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 9 Aug 2024 19:19:04 +0200 Subject: [PATCH 253/339] ALways log to syslog when writing to the logfile failed Signed-off-by: DL6ER --- src/config/toml_reader.c | 6 ------ src/log.c | 5 ++++- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/src/config/toml_reader.c b/src/config/toml_reader.c index de40703d..6f3f98e3 100644 --- a/src/config/toml_reader.c +++ b/src/config/toml_reader.c @@ -141,12 +141,6 @@ bool readFTLtoml(struct config *oldconf, struct config *newconf, struct conf_item *old_conf_item = oldconf != NULL ? get_conf_item(oldconf, i) : NULL; struct conf_item *new_conf_item = get_conf_item(newconf, i); - // Do not read config.files.log.ftl from the TOML file if it has been - // set to NULL due to permissions issues during startup - if(config.files.log.ftl.v.s == NULL && - new_conf_item == &newconf->files.log.ftl) - continue; - // First try to read this config option from an environment variable // Skip reading environment variables when importing from Teleporter // If this succeeds, skip searching the TOML file for this config item diff --git a/src/log.c b/src/log.c index e11e09a7..0d089e02 100644 --- a/src/log.c +++ b/src/log.c @@ -289,6 +289,7 @@ void __attribute__ ((format (printf, 3, 4))) _FTL_log(const int priority, const va_end(args); add_to_fifo_buffer(FIFO_FTL, buffer, prio, len > MAX_MSG_FIFO ? MAX_MSG_FIFO : len); + bool logged = false; if(config.files.log.ftl.v.s != NULL) { // Open log file @@ -310,6 +311,8 @@ void __attribute__ ((format (printf, 3, 4))) _FTL_log(const int priority, const // Close file after writing fclose(logfile); + + logged = true; } else if(!daemonmode) { @@ -317,7 +320,7 @@ void __attribute__ ((format (printf, 3, 4))) _FTL_log(const int priority, const syslog(LOG_ERR, "Writing to FTL\'s log file failed!"); } } - else + if(!logged) { // Syslog logging va_start(args, format); From 72e2a0b06813b03100ac5f9e4a9cd4ab3b8ab120 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 10 Aug 2024 06:57:43 +0200 Subject: [PATCH 254/339] Immediately continue after canceling a thread. It may crash FTL otherwise when trying to join afterwards (https://discourse.pi-hole.net/t/pihole-6-0-beta-and-sqlite-issue/71586/13) Signed-off-by: DL6ER --- src/daemon.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/daemon.c b/src/daemon.c index a85b0ed0..b2204b6e 100644 --- a/src/daemon.c +++ b/src/daemon.c @@ -285,6 +285,7 @@ static void terminate_threads(void) log_info("Thread %s (%d) is idle, terminating it.", thread_names[i], i); pthread_cancel(threads[i]); + continue; } // Cancel thread if we cannot set a timeout for joining From 6928ddfa4a9fdf5fd3ad63319c4ac9db6d228da3 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 10 Aug 2024 08:39:22 +0200 Subject: [PATCH 255/339] Directly specify NTP port instead of service by name. Having checked the actual implementation of getaddrinfo(), we see that this avoids iterating over /etc/services ensuring we don't get "Unknown service" errors on systems that - for any reason - either lack the file or somehow lack the particular NTP service line (see https://discourse.pi-hole.net/t/cannot-resolve-ntp-server-address-unrecognized-service/71653) Signed-off-by: DL6ER --- src/ntp/client.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ntp/client.c b/src/ntp/client.c index b0e634e3..08f92b67 100644 --- a/src/ntp/client.c +++ b/src/ntp/client.c @@ -389,7 +389,8 @@ bool ntp_client(const char *server, const bool settime, const bool print) // Resolve server address int eai; struct addrinfo *saddr; - if((eai = getaddrinfo(server, "ntp", NULL, &saddr)) != 0) + // Resolve server address, port 123 is used for NTP + if((eai = getaddrinfo(server, "123", NULL, &saddr)) != 0) { char errbuf[1024]; strncpy(errbuf, "Cannot resolve NTP server address: ", sizeof(errbuf)); From 828dd7b8069b13b8083b6836b69486612ab523c6 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 13 Aug 2024 16:34:14 +0200 Subject: [PATCH 256/339] Update embedded SQLite3 engine to 3.46.1 released today. None of the mentioned points in the changelog apply to how Pi-hole uses SQLite3 but we may still benefit from "other minor fixes" Signed-off-by: DL6ER --- ...int-FTL-version-in-interactive-shell.patch | 2 +- src/database/shell.c | 40 +++-- src/database/sqlite3.c | 154 +++++++++--------- src/database/sqlite3.h | 6 +- 4 files changed, 106 insertions(+), 96 deletions(-) diff --git a/patch/sqlite3/0001-print-FTL-version-in-interactive-shell.patch b/patch/sqlite3/0001-print-FTL-version-in-interactive-shell.patch index c7aaa292..1efac968 100644 --- a/patch/sqlite3/0001-print-FTL-version-in-interactive-shell.patch +++ b/patch/sqlite3/0001-print-FTL-version-in-interactive-shell.patch @@ -7,7 +7,7 @@ index 6280ebf6..a5e82f70 100644 #include #include +// print_FTL_version() -+#include "log.h" ++#include "../log.h" #if !defined(_WIN32) && !defined(WIN32) # include diff --git a/src/database/shell.c b/src/database/shell.c index fc6bf5e6..d573e8d4 100644 --- a/src/database/shell.c +++ b/src/database/shell.c @@ -606,11 +606,6 @@ zSkipValidUtf8(const char *z, int nAccept, long ccm); # define CIO_WIN_WC_XLATE 0 /* Not exposing translation routines at all */ #endif -#if CIO_WIN_WC_XLATE -/* Character used to represent a known-incomplete UTF-8 char group (�) */ -static WCHAR cBadGroup = 0xfffd; -#endif - #if CIO_WIN_WC_XLATE static HANDLE handleOfFile(FILE *pf){ int fileDesc = _fileno(pf); @@ -12549,7 +12544,7 @@ static int expertFilter( pCsr->pData = 0; if( rc==SQLITE_OK ){ rc = idxPrintfPrepareStmt(pExpert->db, &pCsr->pData, &pVtab->base.zErrMsg, - "SELECT * FROM main.%Q WHERE sample()", pVtab->pTab->zName + "SELECT * FROM main.%Q WHERE sqlite_expert_sample()", pVtab->pTab->zName ); } @@ -13423,7 +13418,7 @@ struct IdxRemCtx { }; /* -** Implementation of scalar function rem(). +** Implementation of scalar function sqlite_expert_rem(). */ static void idxRemFunc( sqlite3_context *pCtx, @@ -13436,7 +13431,7 @@ static void idxRemFunc( assert( argc==2 ); iSlot = sqlite3_value_int(argv[0]); - assert( iSlot<=p->nSlot ); + assert( iSlotnSlot ); pSlot = &p->aSlot[iSlot]; switch( pSlot->eType ){ @@ -13547,7 +13542,8 @@ static int idxPopulateOneStat1( const char *zName = (const char*)sqlite3_column_text(pIndexXInfo, 0); const char *zColl = (const char*)sqlite3_column_text(pIndexXInfo, 1); zCols = idxAppendText(&rc, zCols, - "%sx.%Q IS rem(%d, x.%Q) COLLATE %s", zComma, zName, nCol, zName, zColl + "%sx.%Q IS sqlite_expert_rem(%d, x.%Q) COLLATE %s", + zComma, zName, nCol, zName, zColl ); zOrder = idxAppendText(&rc, zOrder, "%s%d", zComma, ++nCol); } @@ -13680,13 +13676,13 @@ static int idxPopulateStat1(sqlite3expert *p, char **pzErr){ if( rc==SQLITE_OK ){ sqlite3 *dbrem = (p->iSample==100 ? p->db : p->dbv); - rc = sqlite3_create_function( - dbrem, "rem", 2, SQLITE_UTF8, (void*)pCtx, idxRemFunc, 0, 0 + rc = sqlite3_create_function(dbrem, "sqlite_expert_rem", + 2, SQLITE_UTF8, (void*)pCtx, idxRemFunc, 0, 0 ); } if( rc==SQLITE_OK ){ - rc = sqlite3_create_function( - p->db, "sample", 0, SQLITE_UTF8, (void*)&samplectx, idxSampleFunc, 0, 0 + rc = sqlite3_create_function(p->db, "sqlite_expert_sample", + 0, SQLITE_UTF8, (void*)&samplectx, idxSampleFunc, 0, 0 ); } @@ -13738,6 +13734,9 @@ static int idxPopulateStat1(sqlite3expert *p, char **pzErr){ rc = sqlite3_exec(p->dbm, "ANALYZE sqlite_schema", 0, 0, 0); } + sqlite3_create_function(p->db, "sqlite_expert_rem", 2, SQLITE_UTF8, 0,0,0,0); + sqlite3_create_function(p->db, "sqlite_expert_sample", 0,SQLITE_UTF8,0,0,0,0); + sqlite3_exec(p->db, "DROP TABLE IF EXISTS temp."UNIQUE_TABLE_NAME,0,0,0); return rc; } @@ -16838,8 +16837,8 @@ static int recoverError( va_start(ap, zFmt); if( zFmt ){ z = sqlite3_vmprintf(zFmt, ap); - va_end(ap); } + va_end(ap); sqlite3_free(p->zErrMsg); p->zErrMsg = z; p->errCode = errCode; @@ -27087,7 +27086,6 @@ static int do_meta_command(char *zLine, ShellState *p){ import_cleanup(&sCtx); shell_out_of_memory(); } - nByte = strlen(zSql); rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); sqlite3_free(zSql); zSql = 0; @@ -27106,16 +27104,21 @@ static int do_meta_command(char *zLine, ShellState *p){ sqlite3_finalize(pStmt); pStmt = 0; if( nCol==0 ) return 0; /* no columns, no error */ - zSql = sqlite3_malloc64( nByte*2 + 20 + nCol*2 ); + + nByte = 64 /* space for "INSERT INTO", "VALUES(", ")\0" */ + + (zSchema ? strlen(zSchema)*2 + 2: 0) /* Quoted schema name */ + + strlen(zTable)*2 + 2 /* Quoted table name */ + + nCol*2; /* Space for ",?" for each column */ + zSql = sqlite3_malloc64( nByte ); if( zSql==0 ){ import_cleanup(&sCtx); shell_out_of_memory(); } if( zSchema ){ - sqlite3_snprintf(nByte+20, zSql, "INSERT INTO \"%w\".\"%w\" VALUES(?", + sqlite3_snprintf(nByte, zSql, "INSERT INTO \"%w\".\"%w\" VALUES(?", zSchema, zTable); }else{ - sqlite3_snprintf(nByte+20, zSql, "INSERT INTO \"%w\" VALUES(?", zTable); + sqlite3_snprintf(nByte, zSql, "INSERT INTO \"%w\" VALUES(?", zTable); } j = strlen30(zSql); for(i=1; i=2 ){ oputf("Insert using: %s\n", zSql); } diff --git a/src/database/sqlite3.c b/src/database/sqlite3.c index 4458f270..98809f1b 100644 --- a/src/database/sqlite3.c +++ b/src/database/sqlite3.c @@ -1,6 +1,6 @@ /****************************************************************************** ** This file is an amalgamation of many separate C source files from SQLite -** version 3.46.0. By combining all the individual C code files into this +** version 3.46.1. By combining all the individual C code files into this ** single large file, the entire code can be compiled as a single translation ** unit. This allows many compilers to do optimizations that would not be ** possible if the files were compiled separately. Performance improvements @@ -18,7 +18,7 @@ ** separate file. This file contains only code for the core SQLite library. ** ** The content in this amalgamation comes from Fossil check-in -** 96c92aba00c8375bc32fafcdf12429c58bd8. +** c9c2ab54ba1f5f46360f1b4f35d849cd3f08. */ #define SQLITE_CORE 1 #define SQLITE_AMALGAMATION 1 @@ -459,9 +459,9 @@ extern "C" { ** [sqlite3_libversion_number()], [sqlite3_sourceid()], ** [sqlite_version()] and [sqlite_source_id()]. */ -#define SQLITE_VERSION "3.46.0" -#define SQLITE_VERSION_NUMBER 3046000 -#define SQLITE_SOURCE_ID "2024-05-23 13:25:27 96c92aba00c8375bc32fafcdf12429c58bd8aabfcadab6683e35bbb9cdebf19e" +#define SQLITE_VERSION "3.46.1" +#define SQLITE_VERSION_NUMBER 3046001 +#define SQLITE_SOURCE_ID "2024-08-13 09:16:08 c9c2ab54ba1f5f46360f1b4f35d849cd3f080e6fc2b6c60e91b16c63f69a1e33" /* ** CAPI3REF: Run-Time Library Version Numbers @@ -19361,7 +19361,7 @@ struct SrcList { #define WHERE_AGG_DISTINCT 0x0400 /* Query is "SELECT agg(DISTINCT ...)" */ #define WHERE_ORDERBY_LIMIT 0x0800 /* ORDERBY+LIMIT on the inner loop */ #define WHERE_RIGHT_JOIN 0x1000 /* Processing a RIGHT JOIN */ - /* 0x2000 not currently used */ +#define WHERE_KEEP_ALL_JOINS 0x2000 /* Do not do the omit-noop-join opt */ #define WHERE_USE_LIMIT 0x4000 /* Use the LIMIT in cost estimates */ /* 0x8000 not currently used */ @@ -90173,7 +90173,8 @@ SQLITE_PRIVATE sqlite3_value *sqlite3VdbeGetBoundValue(Vdbe *v, int iVar, u8 aff assert( iVar>0 ); if( v ){ Mem *pMem = &v->aVar[iVar-1]; - assert( (v->db->flags & SQLITE_EnableQPSG)==0 ); + assert( (v->db->flags & SQLITE_EnableQPSG)==0 + || (v->db->mDbFlags & DBFLAG_InternalFunc)!=0 ); if( 0==(pMem->flags & MEM_Null) ){ sqlite3_value *pRet = sqlite3ValueNew(v->db); if( pRet ){ @@ -90193,7 +90194,8 @@ SQLITE_PRIVATE sqlite3_value *sqlite3VdbeGetBoundValue(Vdbe *v, int iVar, u8 aff */ SQLITE_PRIVATE void sqlite3VdbeSetVarmask(Vdbe *v, int iVar){ assert( iVar>0 ); - assert( (v->db->flags & SQLITE_EnableQPSG)==0 ); + assert( (v->db->flags & SQLITE_EnableQPSG)==0 + || (v->db->mDbFlags & DBFLAG_InternalFunc)!=0 ); if( iVar>=32 ){ v->expmask |= 0x80000000; }else{ @@ -106950,7 +106952,7 @@ static void extendFJMatch( static SQLITE_NOINLINE int isValidSchemaTableName( const char *zTab, /* Name as it appears in the SQL */ Table *pTab, /* The schema table we are trying to match */ - Schema *pSchema /* non-NULL if a database qualifier is present */ + const char *zDb /* non-NULL if a database qualifier is present */ ){ const char *zLegacy; assert( pTab!=0 ); @@ -106961,7 +106963,7 @@ static SQLITE_NOINLINE int isValidSchemaTableName( if( sqlite3StrICmp(zTab+7, &PREFERRED_TEMP_SCHEMA_TABLE[7])==0 ){ return 1; } - if( pSchema==0 ) return 0; + if( zDb==0 ) return 0; if( sqlite3StrICmp(zTab+7, &LEGACY_SCHEMA_TABLE[7])==0 ) return 1; if( sqlite3StrICmp(zTab+7, &PREFERRED_SCHEMA_TABLE[7])==0 ) return 1; }else{ @@ -107144,7 +107146,7 @@ static int lookupName( } }else if( sqlite3StrICmp(zTab, pTab->zName)!=0 ){ if( pTab->tnum!=1 ) continue; - if( !isValidSchemaTableName(zTab, pTab, pSchema) ) continue; + if( !isValidSchemaTableName(zTab, pTab, zDb) ) continue; } assert( ExprUseYTab(pExpr) ); if( IN_RENAME_OBJECT && pItem->zAlias ){ @@ -108876,6 +108878,9 @@ SQLITE_PRIVATE int sqlite3ResolveExprNames( ** Resolve all names for all expression in an expression list. This is ** just like sqlite3ResolveExprNames() except that it works for an expression ** list rather than a single expression. +** +** The return value is SQLITE_OK (0) for success or SQLITE_ERROR (1) for a +** failure. */ SQLITE_PRIVATE int sqlite3ResolveExprListNames( NameContext *pNC, /* Namespace to resolve expressions in. */ @@ -108884,7 +108889,7 @@ SQLITE_PRIVATE int sqlite3ResolveExprListNames( int i; int savedHasAgg = 0; Walker w; - if( pList==0 ) return WRC_Continue; + if( pList==0 ) return SQLITE_OK; w.pParse = pNC->pParse; w.xExprCallback = resolveExprStep; w.xSelectCallback = resolveSelectStep; @@ -108898,7 +108903,7 @@ SQLITE_PRIVATE int sqlite3ResolveExprListNames( #if SQLITE_MAX_EXPR_DEPTH>0 w.pParse->nHeight += pExpr->nHeight; if( sqlite3ExprCheckHeight(w.pParse, w.pParse->nHeight) ){ - return WRC_Abort; + return SQLITE_ERROR; } #endif sqlite3WalkExprNN(&w, pExpr); @@ -108915,10 +108920,10 @@ SQLITE_PRIVATE int sqlite3ResolveExprListNames( (NC_HasAgg|NC_MinMaxAgg|NC_HasWin|NC_OrderAgg); pNC->ncFlags &= ~(NC_HasAgg|NC_MinMaxAgg|NC_HasWin|NC_OrderAgg); } - if( w.pParse->nErr>0 ) return WRC_Abort; + if( w.pParse->nErr>0 ) return SQLITE_ERROR; } pNC->ncFlags |= savedHasAgg; - return WRC_Continue; + return SQLITE_OK; } /* @@ -117457,7 +117462,7 @@ static int renameResolveTrigger(Parse *pParse){ /* ALWAYS() because if the table of the trigger does not exist, the ** error would have been hit before this point */ if( ALWAYS(pParse->pTriggerTab) ){ - rc = sqlite3ViewGetColumnNames(pParse, pParse->pTriggerTab); + rc = sqlite3ViewGetColumnNames(pParse, pParse->pTriggerTab)!=0; } /* Resolve symbols in WHEN clause */ @@ -124426,8 +124431,9 @@ create_view_fail: #if !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_VIRTUALTABLE) /* ** The Table structure pTable is really a VIEW. Fill in the names of -** the columns of the view in the pTable structure. Return the number -** of errors. If an error is seen leave an error message in pParse->zErrMsg. +** the columns of the view in the pTable structure. Return non-zero if +** there are errors. If an error is seen an error message is left +** in pParse->zErrMsg. */ static SQLITE_NOINLINE int viewGetColumnNames(Parse *pParse, Table *pTable){ Table *pSelTab; /* A fake table from which we get the result set */ @@ -124550,7 +124556,7 @@ static SQLITE_NOINLINE int viewGetColumnNames(Parse *pParse, Table *pTable){ sqlite3DeleteColumnNames(db, pTable); } #endif /* SQLITE_OMIT_VIEW */ - return nErr; + return nErr + pParse->nErr; } SQLITE_PRIVATE int sqlite3ViewGetColumnNames(Parse *pParse, Table *pTable){ assert( pTable!=0 ); @@ -130848,6 +130854,8 @@ static void groupConcatValue(sqlite3_context *context){ sqlite3_result_error_toobig(context); }else if( pAccum->accError==SQLITE_NOMEM ){ sqlite3_result_error_nomem(context); + }else if( pGCC->nAccum>0 && pAccum->nChar==0 ){ + sqlite3_result_text(context, "", 1, SQLITE_STATIC); }else{ const char *zText = sqlite3_str_value(pAccum); sqlite3_result_text(context, zText, pAccum->nChar, SQLITE_TRANSIENT); @@ -133602,6 +133610,7 @@ SQLITE_PRIVATE Select *sqlite3MultiValues(Parse *pParse, Select *pLeft, ExprList pRet->pSrc->nSrc = 1; pRet->pPrior = pLeft->pPrior; pRet->op = pLeft->op; + if( pRet->pPrior ) pRet->selFlags |= SF_Values; pLeft->pPrior = 0; pLeft->op = TK_SELECT; assert( pLeft->pNext==0 ); @@ -166067,7 +166076,9 @@ static int whereLoopAddBtree( " according to whereIsCoveringIndex()\n", pProbe->zName)); } } - }else if( m==0 ){ + }else if( m==0 + && (HasRowid(pTab) || pWInfo->pSelect!=0 || sqlite3FaultSim(700)) + ){ WHERETRACE(0x200, ("-> %s a covering index according to bitmasks\n", pProbe->zName, m==0 ? "is" : "is not")); @@ -167956,6 +167967,10 @@ static void showAllWhereLoops(WhereInfo *pWInfo, WhereClause *pWC){ ** the right-most table of a subquery that was flattened into the ** main query and that subquery was the right-hand operand of an ** inner join that held an ON or USING clause. +** 6) The ORDER BY clause has 63 or fewer terms +** 7) The omit-noop-join optimization is enabled. +** +** Items (1), (6), and (7) are checked by the caller. ** ** For example, given: ** @@ -168369,6 +168384,7 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( if( pOrderBy && pOrderBy->nExpr>=BMS ){ pOrderBy = 0; wctrlFlags &= ~WHERE_WANT_DISTINCT; + wctrlFlags |= WHERE_KEEP_ALL_JOINS; /* Disable omit-noop-join opt */ } /* The number of tables in the FROM clause is limited by the number of @@ -168669,10 +168685,10 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( ** in-line sqlite3WhereCodeOneLoopStart() for performance reasons. */ notReady = ~(Bitmask)0; - if( pWInfo->nLevel>=2 - && pResultSet!=0 /* these two combine to guarantee */ - && 0==(wctrlFlags & WHERE_AGG_DISTINCT) /* condition (1) above */ - && OptimizationEnabled(db, SQLITE_OmitNoopJoin) + if( pWInfo->nLevel>=2 /* Must be a join, or this opt8n is pointless */ + && pResultSet!=0 /* Condition (1) */ + && 0==(wctrlFlags & (WHERE_AGG_DISTINCT|WHERE_KEEP_ALL_JOINS)) /* (1),(6) */ + && OptimizationEnabled(db, SQLITE_OmitNoopJoin) /* (7) */ ){ notReady = whereOmitNoopJoin(pWInfo, notReady); nTabList = pWInfo->nLevel; @@ -168992,26 +169008,6 @@ whereBeginError: } #endif -#ifdef SQLITE_DEBUG -/* -** Return true if cursor iCur is opened by instruction k of the -** bytecode. Used inside of assert() only. -*/ -static int cursorIsOpen(Vdbe *v, int iCur, int k){ - while( k>=0 ){ - VdbeOp *pOp = sqlite3VdbeGetOp(v,k--); - if( pOp->p1!=iCur ) continue; - if( pOp->opcode==OP_Close ) return 0; - if( pOp->opcode==OP_OpenRead ) return 1; - if( pOp->opcode==OP_OpenWrite ) return 1; - if( pOp->opcode==OP_OpenDup ) return 1; - if( pOp->opcode==OP_OpenAutoindex ) return 1; - if( pOp->opcode==OP_OpenEphemeral ) return 1; - } - return 0; -} -#endif /* SQLITE_DEBUG */ - /* ** Generate the end of the WHERE loop. See comments on ** sqlite3WhereBegin() for additional information. @@ -169311,16 +169307,10 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ ** reference. Verify that this is harmless - that the ** table being referenced really is open. */ -#ifdef SQLITE_ENABLE_OFFSET_SQL_FUNC - assert( (pLoop->wsFlags & WHERE_IDX_ONLY)==0 - || cursorIsOpen(v,pOp->p1,k) - || pOp->opcode==OP_Offset - ); -#else - assert( (pLoop->wsFlags & WHERE_IDX_ONLY)==0 - || cursorIsOpen(v,pOp->p1,k) - ); -#endif + if( pLoop->wsFlags & WHERE_IDX_ONLY ){ + sqlite3ErrorMsg(pParse, "internal query planner error"); + pParse->rc = SQLITE_INTERNAL; + } } }else if( pOp->opcode==OP_Rowid ){ pOp->p1 = pLevel->iIdxCur; @@ -172591,9 +172581,9 @@ static void updateDeleteLimitError( break; } } - if( (p->selFlags & SF_MultiValue)==0 && - (mxSelect = pParse->db->aLimit[SQLITE_LIMIT_COMPOUND_SELECT])>0 && - cnt>mxSelect + if( (p->selFlags & (SF_MultiValue|SF_Values))==0 + && (mxSelect = pParse->db->aLimit[SQLITE_LIMIT_COMPOUND_SELECT])>0 + && cnt>mxSelect ){ sqlite3ErrorMsg(pParse, "too many terms in compound SELECT"); } @@ -237002,7 +236992,11 @@ static int sqlite3Fts5ExprNew( } sqlite3_free(sParse.apPhrase); - *pzErr = sParse.zErr; + if( 0==*pzErr ){ + *pzErr = sParse.zErr; + }else{ + sqlite3_free(sParse.zErr); + } return sParse.rc; } @@ -239130,6 +239124,7 @@ static Fts5ExprNode *sqlite3Fts5ParseImplicitAnd( assert( pRight->eType==FTS5_STRING || pRight->eType==FTS5_TERM || pRight->eType==FTS5_EOF + || (pRight->eType==FTS5_AND && pParse->bPhraseToAnd) ); if( pLeft->eType==FTS5_AND ){ @@ -251297,6 +251292,7 @@ static int fts5UpdateMethod( rc = SQLITE_ERROR; }else{ rc = fts5SpecialDelete(pTab, apVal); + bUpdateOrDelete = 1; } }else{ rc = fts5SpecialInsert(pTab, z, apVal[2 + pConfig->nCol + 1]); @@ -252471,14 +252467,16 @@ static int sqlite3Fts5GetTokenizer( if( pMod==0 ){ assert( nArg>0 ); rc = SQLITE_ERROR; - *pzErr = sqlite3_mprintf("no such tokenizer: %s", azArg[0]); + if( pzErr ) *pzErr = sqlite3_mprintf("no such tokenizer: %s", azArg[0]); }else{ rc = pMod->x.xCreate( pMod->pUserData, (azArg?&azArg[1]:0), (nArg?nArg-1:0), &pConfig->pTok ); pConfig->pTokApi = &pMod->x; if( rc!=SQLITE_OK ){ - if( pzErr ) *pzErr = sqlite3_mprintf("error in tokenizer constructor"); + if( pzErr && rc!=SQLITE_NOMEM ){ + *pzErr = sqlite3_mprintf("error in tokenizer constructor"); + } }else{ pConfig->ePattern = sqlite3Fts5TokenizerPattern( pMod->x.xCreate, pConfig->pTok @@ -252537,7 +252535,7 @@ static void fts5SourceIdFunc( ){ assert( nArg==0 ); UNUSED_PARAM2(nArg, apUnused); - sqlite3_result_text(pCtx, "fts5: 2024-05-23 13:25:27 96c92aba00c8375bc32fafcdf12429c58bd8aabfcadab6683e35bbb9cdebf19e", -1, SQLITE_TRANSIENT); + sqlite3_result_text(pCtx, "fts5: 2024-08-13 09:16:08 c9c2ab54ba1f5f46360f1b4f35d849cd3f080e6fc2b6c60e91b16c63f69a1e33", -1, SQLITE_TRANSIENT); } /* @@ -252572,17 +252570,23 @@ static int fts5IntegrityMethod( assert( pzErr!=0 && *pzErr==0 ); UNUSED_PARAM(isQuick); + assert( pTab->p.pConfig->pzErrmsg==0 ); + pTab->p.pConfig->pzErrmsg = pzErr; rc = sqlite3Fts5StorageIntegrity(pTab->pStorage, 0); - if( (rc&0xff)==SQLITE_CORRUPT ){ - *pzErr = sqlite3_mprintf("malformed inverted index for FTS5 table %s.%s", - zSchema, zTabname); - rc = (*pzErr) ? SQLITE_OK : SQLITE_NOMEM; - }else if( rc!=SQLITE_OK ){ - *pzErr = sqlite3_mprintf("unable to validate the inverted index for" - " FTS5 table %s.%s: %s", - zSchema, zTabname, sqlite3_errstr(rc)); + if( *pzErr==0 && rc!=SQLITE_OK ){ + if( (rc&0xff)==SQLITE_CORRUPT ){ + *pzErr = sqlite3_mprintf("malformed inverted index for FTS5 table %s.%s", + zSchema, zTabname); + rc = (*pzErr) ? SQLITE_OK : SQLITE_NOMEM; + }else{ + *pzErr = sqlite3_mprintf("unable to validate the inverted index for" + " FTS5 table %s.%s: %s", + zSchema, zTabname, sqlite3_errstr(rc)); + } } + sqlite3Fts5IndexCloseReader(pTab->p.pIndex); + pTab->p.pConfig->pzErrmsg = 0; return rc; } @@ -254016,7 +254020,7 @@ static int fts5AsciiCreate( int i; memset(p, 0, sizeof(AsciiTokenizer)); memcpy(p->aTokenChar, aAsciiTokenChar, sizeof(aAsciiTokenChar)); - for(i=0; rc==SQLITE_OK && ibFold = 1; pNew->iFoldParam = 0; - for(i=0; rc==SQLITE_OK && iiFoldParam!=0 && pNew->bFold==0 ){ rc = SQLITE_ERROR; diff --git a/src/database/sqlite3.h b/src/database/sqlite3.h index 57df8dcf..f64ca017 100644 --- a/src/database/sqlite3.h +++ b/src/database/sqlite3.h @@ -146,9 +146,9 @@ extern "C" { ** [sqlite3_libversion_number()], [sqlite3_sourceid()], ** [sqlite_version()] and [sqlite_source_id()]. */ -#define SQLITE_VERSION "3.46.0" -#define SQLITE_VERSION_NUMBER 3046000 -#define SQLITE_SOURCE_ID "2024-05-23 13:25:27 96c92aba00c8375bc32fafcdf12429c58bd8aabfcadab6683e35bbb9cdebf19e" +#define SQLITE_VERSION "3.46.1" +#define SQLITE_VERSION_NUMBER 3046001 +#define SQLITE_SOURCE_ID "2024-08-13 09:16:08 c9c2ab54ba1f5f46360f1b4f35d849cd3f080e6fc2b6c60e91b16c63f69a1e33" /* ** CAPI3REF: Run-Time Library Version Numbers From f2ae6db391bcb3e417e0813fbc2b32bee69f37bf Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 13 Aug 2024 16:52:11 +0200 Subject: [PATCH 257/339] Add further API debugging to top_client generation and fix an incorrect condition that skipped all clients without a name Signed-off-by: DL6ER --- src/api/stats.c | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/src/api/stats.c b/src/api/stats.c index 12024bfc..05075a3e 100644 --- a/src/api/stats.c +++ b/src/api/stats.c @@ -388,12 +388,26 @@ cJSON *get_top_clients(struct ftl_conn *api, const int count, // Skip invalid clients and also those managed by alias clients if(client == NULL || (!client->flags.aliasclient && client->aliasclient_id >= 0)) + { + log_debug(DEBUG_API, "Skipping client %i because %s", clientID, + client == NULL ? "it is invalid" : "it is an alias client"); continue; + } + + // Skip recycled clients + if(client->ippos == 0) + { + log_debug(DEBUG_API, "Skipping client %i because it is recycled", clientID); + continue; + } const char *client_ip = getstr(client->ippos); // Hidden client, probably due to privacy level. Skip this in the top lists if(strcmp(client_ip, HIDDEN_CLIENT) == 0) + { + log_debug(DEBUG_API, "Skipping client %i because it is hidden", clientID); continue; + } // Use either blocked or total count based on request string top_clients[added_clients].count = blocked ? client->blockedcount : client->count; @@ -405,6 +419,8 @@ cJSON *get_top_clients(struct ftl_conn *api, const int count, added_clients++; } + log_debug(DEBUG_API, "Found %u clients", added_clients); + // Unlock shared memory unlock_shm(); @@ -426,10 +442,6 @@ cJSON *get_top_clients(struct ftl_conn *api, const int count, for(unsigned int i = 0; i < added_clients; i++) { - // Skip e.g. recycled clients - if(top_clients[i].namepos == 0) - continue; - const char *client_ip = getstr(top_clients[i].ippos); const char *client_name = getstr(top_clients[i].namepos); @@ -457,7 +469,11 @@ cJSON *get_top_clients(struct ftl_conn *api, const int count, } if(skip_client || top_clients[i].count < 1) + { + log_debug(DEBUG_API, "Skipping client %s because it %s", client_ip, + skip_client ? "matches a filter" : "has no queries"); continue; + } if(clients_only) { From 07561835f017beb8a5e81d27fa5a8e3c4d2c21c1 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 15 Aug 2024 20:13:16 +0200 Subject: [PATCH 258/339] Update embedded CivetWeb to latest master of their repo Signed-off-by: DL6ER --- src/webserver/civetweb/civetweb.c | 948 ++++++++++++++++++------- src/webserver/civetweb/civetweb.h | 25 +- src/webserver/civetweb/handle_form.inl | 82 ++- src/webserver/civetweb/match.inl | 6 +- src/webserver/civetweb/mod_lua.inl | 37 +- src/webserver/civetweb/mod_mbedtls.inl | 48 +- src/webserver/civetweb/timer.inl | 6 +- 7 files changed, 797 insertions(+), 355 deletions(-) diff --git a/src/webserver/civetweb/civetweb.c b/src/webserver/civetweb/civetweb.c index 367e19ae..dced1362 100644 --- a/src/webserver/civetweb/civetweb.c +++ b/src/webserver/civetweb/civetweb.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2021 the Civetweb developers +/* Copyright (c) 2013-2024 the Civetweb developers * Copyright (c) 2004-2013 Sergey Lyubka * * Permission is hereby granted, free of charge, to any person obtaining a copy @@ -51,8 +51,8 @@ #if !defined(_CRT_SECURE_NO_WARNINGS) #define _CRT_SECURE_NO_WARNINGS /* Disable deprecation warning in VS2005 */ #endif -#if !defined(_WIN32_WINNT) /* defined for tdm-gcc so we can use getnameinfo */ -#define _WIN32_WINNT 0x0502 +#if !defined(_WIN32_WINNT) /* Minimum API version */ +#define _WIN32_WINNT 0x0601 #endif #else #if !defined(_GNU_SOURCE) @@ -239,10 +239,9 @@ static void DEBUG_TRACE_FUNC(const char *func, #endif #else -#include "log.h" #define DEBUG_TRACE(fmt, ...) \ - if(debug_flags[DEBUG_WEBSERVER]) {\ - log_web("DEBUG: " fmt " (%s:%d)", ##__VA_ARGS__, short_path(__FILE__), __LINE__); } + do { \ + } while (0) #endif /* DEBUG */ #endif /* DEBUG_TRACE */ @@ -1126,7 +1125,15 @@ mg_atomic_inc(volatile ptrdiff_t *addr) #if defined(_WIN64) && !defined(NO_ATOMICS) ret = InterlockedIncrement64(addr); #elif defined(_WIN32) && !defined(NO_ATOMICS) +#ifdef __cplusplus + /* For C++ the Microsoft Visual Studio compiler can not decide what + * overloaded function prototpye in the SDC corresponds to "ptrdiff_t". */ + static_assert(sizeof(ptrdiff_t) == sizeof(LONG), "Size mismatch"); + static_assert(sizeof(ptrdiff_t) == sizeof(int32_t), "Size mismatch"); + ret = InterlockedIncrement((LONG *)addr); +#else ret = InterlockedIncrement(addr); +#endif #elif defined(__GNUC__) \ && ((__GNUC__ > 4) || ((__GNUC__ == 4) && (__GNUC_MINOR__ > 0))) \ && !defined(NO_ATOMICS) @@ -1149,7 +1156,14 @@ mg_atomic_dec(volatile ptrdiff_t *addr) #if defined(_WIN64) && !defined(NO_ATOMICS) ret = InterlockedDecrement64(addr); #elif defined(_WIN32) && !defined(NO_ATOMICS) +#ifdef __cplusplus + /* see mg_atomic_inc */ + static_assert(sizeof(ptrdiff_t) == sizeof(LONG), "Size mismatch"); + static_assert(sizeof(ptrdiff_t) == sizeof(int32_t), "Size mismatch"); + ret = InterlockedDecrement((LONG *)addr); +#else ret = InterlockedDecrement(addr); +#endif #elif defined(__GNUC__) \ && ((__GNUC__ > 4) || ((__GNUC__ == 4) && (__GNUC_MINOR__ > 0))) \ && !defined(NO_ATOMICS) @@ -1308,13 +1322,13 @@ mg_malloc_ex(size_t size, #endif if (data) { + uintptr_t *tmp = (uintptr_t *)data; ptrdiff_t mmem = mg_atomic_add(&mstat->totalMemUsed, (ptrdiff_t)size); mg_atomic_max(&mstat->maxMemUsed, mmem); - mg_atomic_inc(&mstat->blockCount); - ((uintptr_t *)data)[0] = size; - ((uintptr_t *)data)[1] = (uintptr_t)mstat; - memory = (void *)(((char *)data) + 2 * sizeof(uintptr_t)); + tmp[0] = size; + tmp[1] = (uintptr_t)mstat; + memory = (void *)&tmp[2]; } #if defined(MEMORY_DEBUGGING) @@ -1537,11 +1551,13 @@ static void mg_snprintf(const struct mg_connection *conn, #if defined(vsnprintf) #undef vsnprintf #endif +#if !defined(NDEBUG) #define malloc DO_NOT_USE_THIS_FUNCTION__USE_mg_malloc #define calloc DO_NOT_USE_THIS_FUNCTION__USE_mg_calloc #define realloc DO_NOT_USE_THIS_FUNCTION__USE_mg_realloc #define free DO_NOT_USE_THIS_FUNCTION__USE_mg_free #define snprintf DO_NOT_USE_THIS_FUNCTION__USE_mg_snprintf +#endif #if defined(_WIN32) /* vsnprintf must not be used in any system, * but this define only works well for Windows. */ @@ -1907,7 +1923,9 @@ struct socket { unsigned char is_ssl; /* Is port SSL-ed */ unsigned char ssl_redir; /* Is port supposed to redirect everything to SSL * port */ - unsigned char in_use; /* 0: invalid, 1: valid, 2: free */ + unsigned char + is_optional; /* Shouldn't cause us to exit if we can't bind to it */ + unsigned char in_use; /* 0: invalid, 1: valid, 2: free */ }; @@ -1920,6 +1938,7 @@ enum { /* Once for each server */ LISTENING_PORTS, NUM_THREADS, + PRESPAWN_THREADS, RUN_AS_USER, CONFIG_TCP_NODELAY, /* Prepended CONFIG_ to avoid conflict with the * socket option typedef TCP_NODELAY. */ @@ -1953,6 +1972,7 @@ enum { /* Once for each domain */ DOCUMENT_ROOT, + FALLBACK_DOCUMENT_ROOT, ACCESS_LOG_FILE, ERROR_LOG_FILE, @@ -2034,6 +2054,7 @@ enum { #if defined(USE_WEBSOCKET) WEBSOCKET_ROOT, + FALLBACK_WEBSOCKET_ROOT, #endif #if defined(USE_LUA) && defined(USE_WEBSOCKET) LUA_WEBSOCKET_EXTENSIONS, @@ -2042,6 +2063,8 @@ enum { ACCESS_CONTROL_ALLOW_ORIGIN, ACCESS_CONTROL_ALLOW_METHODS, ACCESS_CONTROL_ALLOW_HEADERS, + ACCESS_CONTROL_EXPOSE_HEADERS, + ACCESS_CONTROL_ALLOW_CREDENTIALS, ERROR_PAGES, #if !defined(NO_CACHING) STATIC_FILE_MAX_AGE, @@ -2065,6 +2088,7 @@ static const struct mg_option config_options[] = { /* Once for each server */ {"listening_ports", MG_CONFIG_TYPE_STRING_LIST, "8080"}, {"num_threads", MG_CONFIG_TYPE_NUMBER, "50"}, + {"prespawn_threads", MG_CONFIG_TYPE_NUMBER, "0"}, {"run_as_user", MG_CONFIG_TYPE_STRING, NULL}, {"tcp_nodelay", MG_CONFIG_TYPE_NUMBER, "0"}, {"max_request_size", MG_CONFIG_TYPE_NUMBER, "16384"}, @@ -2097,6 +2121,7 @@ static const struct mg_option config_options[] = { /* Once for each domain */ {"document_root", MG_CONFIG_TYPE_DIRECTORY, NULL}, + {"fallback_document_root", MG_CONFIG_TYPE_DIRECTORY, NULL}, {"access_log_file", MG_CONFIG_TYPE_FILE, NULL}, {"error_log_file", MG_CONFIG_TYPE_FILE, NULL}, @@ -2195,6 +2220,7 @@ static const struct mg_option config_options[] = { #if defined(USE_WEBSOCKET) {"websocket_root", MG_CONFIG_TYPE_DIRECTORY, NULL}, + {"fallback_websocket_root", MG_CONFIG_TYPE_DIRECTORY, NULL}, #endif #if defined(USE_LUA) && defined(USE_WEBSOCKET) {"lua_websocket_pattern", MG_CONFIG_TYPE_EXT_PATTERN, "**.lua$"}, @@ -2202,6 +2228,8 @@ static const struct mg_option config_options[] = { {"access_control_allow_origin", MG_CONFIG_TYPE_STRING, "*"}, {"access_control_allow_methods", MG_CONFIG_TYPE_STRING, "*"}, {"access_control_allow_headers", MG_CONFIG_TYPE_STRING, "*"}, + {"access_control_expose_headers", MG_CONFIG_TYPE_STRING, ""}, + {"access_control_allow_credentials", MG_CONFIG_TYPE_STRING, ""}, {"error_pages", MG_CONFIG_TYPE_DIRECTORY, NULL}, #if !defined(NO_CACHING) {"static_file_max_age", MG_CONFIG_TYPE_NUMBER, "3600"}, @@ -2311,7 +2339,7 @@ STOP_FLAG_IS_TWO(stop_flag_t *f) static void STOP_FLAG_ASSIGN(stop_flag_t *f, stop_flag_t v) { - stop_flag_t sf; + stop_flag_t sf = 0; do { sf = mg_atomic_compare_and_swap(f, *f, v); } while (sf != v); @@ -2374,10 +2402,18 @@ struct mg_context { stop_flag_t stop_flag; /* Should we stop event loop */ pthread_mutex_t thread_mutex; /* Protects client_socks or queue */ - pthread_t masterthreadid; /* The master thread ID */ + pthread_t masterthreadid; /* The master thread ID */ + unsigned int cfg_max_worker_threads; /* How many worker-threads we are + allowed to create, total */ + + unsigned int spawned_worker_threads; /* How many worker-threads currently + exist (modified by master thread) */ unsigned int - cfg_worker_threads; /* The number of configured worker threads. */ - pthread_t *worker_threadids; /* The worker thread IDs */ + idle_worker_thread_count; /* How many worker-threads are currently + sitting around with nothing to do */ + /* Access to this value MUST be synchronized by thread_mutex */ + + pthread_t *worker_threadids; /* The worker thread IDs */ unsigned long starter_thread_idx; /* thread index which called mg_start */ /* Connection to thread dispatching */ @@ -2424,6 +2460,11 @@ struct mg_context { int lua_bg_log_available; /* Use Lua background state for access log */ #endif + int user_shutdown_notification_socket; /* mg_stop() will close this + socket... */ + int thread_shutdown_notification_socket; /* to cause poll() in all threads + to return immediately */ + /* Server nonce */ pthread_mutex_t nonce_mutex; /* Protects ssl_ctx, handlers, * ssl_cert_last_mtime, nonce_count, and @@ -4140,17 +4181,11 @@ send_additional_header(struct mg_connection *conn) } #endif + // Content-Security-Policy + if (header && header[0]) { mg_response_header_add_lines(conn, header); } - - /*************** Pi-hole modification ****************/ - if (pi_hole_extra_headers[0] != '\0') { - mg_response_header_add_lines(conn, pi_hole_extra_headers); - // Invalidate extra headers after having sent them to avoid repetitions - pi_hole_extra_headers[0] = '\0'; - } - /*****************************************************/ } @@ -4160,6 +4195,14 @@ send_cors_header(struct mg_connection *conn) const char *origin_hdr = mg_get_header(conn, "Origin"); const char *cors_orig_cfg = conn->dom_ctx->config[ACCESS_CONTROL_ALLOW_ORIGIN]; + const char *cors_cred_cfg = + conn->dom_ctx->config[ACCESS_CONTROL_ALLOW_CREDENTIALS]; + const char *cors_hdr_cfg = + conn->dom_ctx->config[ACCESS_CONTROL_ALLOW_HEADERS]; + const char *cors_exphdr_cfg = + conn->dom_ctx->config[ACCESS_CONTROL_EXPOSE_HEADERS]; + const char *cors_meth_cfg = + conn->dom_ctx->config[ACCESS_CONTROL_ALLOW_METHODS]; if (cors_orig_cfg && *cors_orig_cfg && origin_hdr && *origin_hdr) { /* Cross-origin resource sharing (CORS), see @@ -4171,6 +4214,37 @@ send_cors_header(struct mg_connection *conn) cors_orig_cfg, -1); } + + if (cors_cred_cfg && *cors_cred_cfg && origin_hdr && *origin_hdr) { + /* Cross-origin resource sharing (CORS), see + * https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Access-Control-Allow-Credentials + */ + mg_response_header_add(conn, + "Access-Control-Allow-Credentials", + cors_cred_cfg, + -1); + } + + if (cors_hdr_cfg && *cors_hdr_cfg) { + mg_response_header_add(conn, + "Access-Control-Allow-Headers", + cors_hdr_cfg, + -1); + } + + if (cors_exphdr_cfg && *cors_exphdr_cfg) { + mg_response_header_add(conn, + "Access-Control-Expose-Headers", + cors_exphdr_cfg, + -1); + } + + if (cors_meth_cfg && *cors_meth_cfg) { + mg_response_header_add(conn, + "Access-Control-Allow-Methods", + cors_meth_cfg, + -1); + } } @@ -4567,48 +4641,6 @@ mg_send_http_error_impl(struct mg_connection *conn, } -/************************************** Pi-hole method **************************************/ -CIVETWEB_API int -my_send_http_error_headers(struct mg_connection *conn, - int status, const char* mime_type, - long long content_length) -{ - if ((mime_type == NULL) || (*mime_type == 0)) { - /* No content type defined: default to text/html */ - mime_type = "text/html"; - } - - mg_response_header_start(conn, status); - send_no_cache_header(conn); - send_additional_header(conn); - mg_response_header_add(conn, "Content-Type", mime_type, -1); - if (content_length < 0) { - /* Size not known. Use chunked encoding (HTTP/1.x) */ - if (conn->protocol_type == PROTOCOL_TYPE_HTTP1) { - /* Only HTTP/1.x defines "chunked" encoding, HTTP/2 does not*/ - mg_response_header_add(conn, "Transfer-Encoding", "chunked", -1); - } - } else { - char len[32]; - int trunc = 0; - mg_snprintf(conn, - &trunc, - len, - sizeof(len), - "%" UINT64_FMT, - (uint64_t)content_length); - if (!trunc) { - /* Since 32 bytes is enough to hold any 64 bit decimal number, - * !trunc is always true */ - mg_response_header_add(conn, "Content-Length", len, -1); - } - } - mg_response_header_send(conn); - - return 0; -} -/********************************************************************************************/ - CIVETWEB_API int mg_send_http_error(struct mg_connection *conn, int status, const char *fmt, ...) { @@ -6179,12 +6211,20 @@ push_inner(struct mg_context *ctx, mg_sleep(5); } else { /* For sockets, wait for the socket using poll */ - struct mg_pollfd pfd[1]; + struct mg_pollfd pfd[2]; int pollres; + unsigned int num_sock = 1; pfd[0].fd = sock; pfd[0].events = POLLOUT; - pollres = mg_poll(pfd, 1, (int)(ms_wait), &(ctx->stop_flag)); + + if (ctx->context_type == CONTEXT_SERVER) { + pfd[num_sock].fd = ctx->thread_shutdown_notification_socket; + pfd[num_sock].events = POLLIN; + num_sock++; + } + + pollres = mg_poll(pfd, num_sock, (int)(ms_wait), &(ctx->stop_flag)); if (!STOP_FLAG_IS_ZERO(&ctx->stop_flag)) { return -2; } @@ -6292,9 +6332,10 @@ pull_inner(FILE *fp, #if defined(USE_MBEDTLS) } else if (conn->ssl != NULL) { - struct mg_pollfd pfd[1]; + struct mg_pollfd pfd[2]; int to_read; int pollres; + unsigned int num_sock = 1; to_read = mbedtls_ssl_get_bytes_avail(conn->ssl); @@ -6310,10 +6351,17 @@ pull_inner(FILE *fp, pfd[0].fd = conn->client.sock; pfd[0].events = POLLIN; + if (conn->phys_ctx->context_type == CONTEXT_SERVER) { + pfd[num_sock].fd = + conn->phys_ctx->thread_shutdown_notification_socket; + pfd[num_sock].events = POLLIN; + num_sock++; + } + to_read = len; pollres = mg_poll(pfd, - 1, + num_sock, (int)(timeout * 1000.0), &(conn->phys_ctx->stop_flag)); @@ -6348,8 +6396,9 @@ pull_inner(FILE *fp, #elif !defined(NO_SSL) } else if (conn->ssl != NULL) { int ssl_pending; - struct mg_pollfd pfd[1]; + struct mg_pollfd pfd[2]; int pollres; + unsigned int num_sock = 1; if ((ssl_pending = SSL_pending(conn->ssl)) > 0) { /* We already know there is no more data buffered in conn->buf @@ -6362,8 +6411,16 @@ pull_inner(FILE *fp, } else { pfd[0].fd = conn->client.sock; pfd[0].events = POLLIN; + + if (conn->phys_ctx->context_type == CONTEXT_SERVER) { + pfd[num_sock].fd = + conn->phys_ctx->thread_shutdown_notification_socket; + pfd[num_sock].events = POLLIN; + num_sock++; + } + pollres = mg_poll(pfd, - 1, + num_sock, (int)(timeout * 1000.0), &(conn->phys_ctx->stop_flag)); if (!STOP_FLAG_IS_ZERO(&conn->phys_ctx->stop_flag)) { @@ -6401,13 +6458,22 @@ pull_inner(FILE *fp, #endif } else { - struct mg_pollfd pfd[1]; + struct mg_pollfd pfd[2]; int pollres; + unsigned int num_sock = 1; pfd[0].fd = conn->client.sock; pfd[0].events = POLLIN; + + if (conn->phys_ctx->context_type == CONTEXT_SERVER) { + pfd[num_sock].fd = + conn->phys_ctx->thread_shutdown_notification_socket; + pfd[num_sock].events = POLLIN; + num_sock++; + } + pollres = mg_poll(pfd, - 1, + num_sock, (int)(timeout * 1000.0), &(conn->phys_ctx->stop_flag)); if (!STOP_FLAG_IS_ZERO(&conn->phys_ctx->stop_flag)) { @@ -6429,7 +6495,7 @@ pull_inner(FILE *fp, } } - if (!STOP_FLAG_IS_ZERO(&conn->phys_ctx->stop_flag)) { + if (conn != NULL && !STOP_FLAG_IS_ZERO(&conn->phys_ctx->stop_flag)) { return -2; } @@ -6453,7 +6519,7 @@ pull_inner(FILE *fp, /* See https://www.chilkatsoft.com/p/p_299.asp */ return -2; } else { - DEBUG_TRACE("recv() failed, error %d", err); + DEBUG_TRACE("read()/recv() failed, error %d", err); return -2; } #else @@ -6475,7 +6541,7 @@ pull_inner(FILE *fp, * (see signal(7)). * => stay in the while loop */ } else { - DEBUG_TRACE("recv() failed, error %d", err); + DEBUG_TRACE("read()/recv() failed, error %d", err); return -2; } #endif @@ -7622,10 +7688,10 @@ extention_matches_template_text( * Return 1 if index file has been found, 0 if not found. * If the file is found, it's stats is returned in stp. */ static int -substitute_index_file(struct mg_connection *conn, - char *path, - size_t path_len, - struct mg_file_stat *filestat) +substitute_index_file_aux(struct mg_connection *conn, + char *path, + size_t path_len, + struct mg_file_stat *filestat) { const char *list = conn->dom_ctx->config[INDEX_FILES]; struct vec filename_vec; @@ -7666,6 +7732,61 @@ substitute_index_file(struct mg_connection *conn, return found; } + +/* Same as above, except if the first try fails and a fallback-root is + * configured, we'll try there also */ +static int +substitute_index_file(struct mg_connection *conn, + char *path, + size_t path_len, + struct mg_file_stat *filestat) +{ + int ret = substitute_index_file_aux(conn, path, path_len, filestat); + if (ret == 0) { + const char *root_prefix = conn->dom_ctx->config[DOCUMENT_ROOT]; + const char *fallback_root_prefix = + conn->dom_ctx->config[FALLBACK_DOCUMENT_ROOT]; + if ((root_prefix) && (fallback_root_prefix)) { + const size_t root_prefix_len = strlen(root_prefix); + if ((strncmp(path, root_prefix, root_prefix_len) == 0)) { + char scratch_path[UTF8_PATH_MAX]; /* separate storage, to avoid + side effects if we fail */ + size_t sub_path_len; + + const size_t fallback_root_prefix_len = + strlen(fallback_root_prefix); + const char *sub_path = path + root_prefix_len; + while (*sub_path == '/') { + sub_path++; + } + sub_path_len = strlen(sub_path); + + if (((fallback_root_prefix_len + 1 + sub_path_len + 1) + < sizeof(scratch_path))) { + /* The concatenations below are all safe because we + * pre-verified string lengths above */ + char *nul; + strcpy(scratch_path, fallback_root_prefix); + nul = strchr(scratch_path, '\0'); + if ((nul > scratch_path) && (*(nul - 1) != '/')) { + *nul++ = '/'; + *nul = '\0'; + } + strcat(scratch_path, sub_path); + if (substitute_index_file_aux(conn, + scratch_path, + sizeof(scratch_path), + filestat)) { + mg_strlcpy(path, scratch_path, path_len); + return 1; + } + } + } + } + } + return ret; +} + #endif @@ -7686,12 +7807,16 @@ interpret_uri(struct mg_connection *conn, /* in/out: request (must be valid) */ #if !defined(NO_FILES) const char *uri = conn->request_info.local_uri; - const char *root = conn->dom_ctx->config[DOCUMENT_ROOT]; + const char *roots[] = {conn->dom_ctx->config[DOCUMENT_ROOT], + conn->dom_ctx->config[FALLBACK_DOCUMENT_ROOT], + NULL}; + int fileExists = 0; const char *rewrite; struct vec a, b; ptrdiff_t match_len; char gz_path[UTF8_PATH_MAX]; int truncated; + int i; #if !defined(NO_CGI) || defined(USE_LUA) || defined(USE_DUKTAPE) char *tmp_str; size_t tmp_str_len, sep_pos; @@ -7721,7 +7846,8 @@ interpret_uri(struct mg_connection *conn, /* in/out: request (must be valid) */ *is_websocket_request = (conn->protocol_type == PROTOCOL_TYPE_WEBSOCKET); #if !defined(NO_FILES) if ((*is_websocket_request) && conn->dom_ctx->config[WEBSOCKET_ROOT]) { - root = conn->dom_ctx->config[WEBSOCKET_ROOT]; + roots[0] = conn->dom_ctx->config[WEBSOCKET_ROOT]; + roots[1] = conn->dom_ctx->config[FALLBACK_WEBSOCKET_ROOT]; } #endif /* !NO_FILES */ #else /* USE_WEBSOCKET */ @@ -7738,53 +7864,63 @@ interpret_uri(struct mg_connection *conn, /* in/out: request (must be valid) */ #if !defined(NO_FILES) /* Step 5: If there is no root directory, don't look for files. */ - /* Note that root == NULL is a regular use case here. This occurs, + /* Note that roots[0] == NULL is a regular use case here. This occurs, * if all requests are handled by callbacks, so the WEBSOCKET_ROOT * config is not required. */ - if (root == NULL) { + if (roots[0] == NULL) { /* all file related outputs have already been set to 0, just return */ return; } - /* Step 6: Determine the local file path from the root path and the - * request uri. */ - /* Using filename_buf_len - 1 because memmove() for PATH_INFO may shift - * part of the path one byte on the right. */ - truncated = 0; - mg_snprintf( - conn, &truncated, filename, filename_buf_len - 1, "%s%s", root, uri); + for (i = 0; roots[i] != NULL; i++) { + /* Step 6: Determine the local file path from the root path and the + * request uri. */ + /* Using filename_buf_len - 1 because memmove() for PATH_INFO may shift + * part of the path one byte on the right. */ + truncated = 0; + mg_snprintf(conn, + &truncated, + filename, + filename_buf_len - 1, + "%s%s", + roots[i], + uri); - FTL_rewrite_pattern(filename, filename_buf_len - 1); + if (truncated) { + goto interpret_cleanup; + } - if (truncated) { - goto interpret_cleanup; - } + /* Step 7: URI rewriting */ + rewrite = conn->dom_ctx->config[URL_REWRITE_PATTERN]; + while ((rewrite = next_option(rewrite, &a, &b)) != NULL) { + if ((match_len = match_prefix(a.ptr, a.len, uri)) > 0) { + mg_snprintf(conn, + &truncated, + filename, + filename_buf_len - 1, + "%.*s%s", + (int)b.len, + b.ptr, + uri + match_len); + break; + } + } - /* Step 7: URI rewriting */ - rewrite = conn->dom_ctx->config[URL_REWRITE_PATTERN]; - while ((rewrite = next_option(rewrite, &a, &b)) != NULL) { - if ((match_len = match_prefix(a.ptr, a.len, uri)) > 0) { - mg_snprintf(conn, - &truncated, - filename, - filename_buf_len - 1, - "%.*s%s", - (int)b.len, - b.ptr, - uri + match_len); + if (truncated) { + goto interpret_cleanup; + } + + /* Step 8: Check if the file exists at the server */ + /* Local file path and name, corresponding to requested URI + * is now stored in "filename" variable. */ + if (mg_stat(conn, filename, filestat)) { + fileExists = 1; break; } } - if (truncated) { - goto interpret_cleanup; - } - - /* Step 8: Check if the file exists at the server */ - /* Local file path and name, corresponding to requested URI - * is now stored in "filename" variable. */ - if (mg_stat(conn, filename, filestat)) { + if (fileExists) { int uri_len = (int)strlen(uri); int is_uri_end_slash = (uri_len > 0) && (uri[uri_len - 1] == '/'); @@ -8289,10 +8425,12 @@ static const struct { {".iso", 4, "application/octet-stream"}, {".js", 3, "application/javascript"}, {".json", 5, "application/json"}, + {".mjs", 4, "application/javascript"}, {".msi", 4, "application/octet-stream"}, {".pdf", 4, "application/pdf"}, {".ps", 3, "application/postscript"}, {".rtf", 4, "application/rtf"}, + {".wasm", 5, "application/wasm"}, {".xhtml", 6, "application/xhtml+xml"}, {".xsl", 4, "application/xml"}, {".xslt", 5, "application/xml"}, @@ -8595,7 +8733,7 @@ open_auth_file(struct mg_connection *conn, /* Parsed Authorization header */ -struct ah { +struct auth_header { char *user; int type; /* 1 = basic, 2 = digest */ char *plain_password; /* Basic only */ @@ -8603,32 +8741,32 @@ struct ah { }; -/* Return 1 on success. Always initializes the ah structure. */ +/* Return 1 on success. Always initializes the auth_header structure. */ static int parse_auth_header(struct mg_connection *conn, char *buf, size_t buf_size, - struct ah *ah) + struct auth_header *auth_header) { char *name, *value, *s; - const char *auth_header; + const char *ah; uint64_t nonce; - if (!ah || !conn) { + if (!auth_header || !conn) { return 0; } - (void)memset(ah, 0, sizeof(*ah)); - auth_header = mg_get_header(conn, "Authorization"); + (void)memset(auth_header, 0, sizeof(*auth_header)); + ah = mg_get_header(conn, "Authorization"); - if (auth_header == NULL) { + if (ah == NULL) { /* No Authorization header at all */ return 0; } - if (0 == mg_strncasecmp(auth_header, "Basic ", 6)) { + if (0 == mg_strncasecmp(ah, "Basic ", 6)) { /* Basic Auth (we never asked for this, but some client may send it) */ char *split; - const char *userpw_b64 = auth_header + 6; + const char *userpw_b64 = ah + 6; size_t userpw_b64_len = strlen(userpw_b64); size_t buf_len_r = buf_size; if (mg_base64_decode( @@ -8645,15 +8783,15 @@ parse_auth_header(struct mg_connection *conn, *split = 0; /* User name is before ':', Password is after ':' */ - ah->user = buf; - ah->type = 1; - ah->plain_password = split + 1; + auth_header->user = buf; + auth_header->type = 1; + auth_header->plain_password = split + 1; return 1; - } else if (0 == mg_strncasecmp(auth_header, "Digest ", 7)) { + } else if (0 == mg_strncasecmp(ah, "Digest ", 7)) { /* Digest Auth ... implemented below */ - ah->type = 2; + auth_header->type = 2; } else { /* Unknown or invalid Auth method */ @@ -8661,7 +8799,7 @@ parse_auth_header(struct mg_connection *conn, } /* Make modifiable copy of the auth header */ - (void)mg_strlcpy(buf, auth_header + 7, buf_size); + (void)mg_strlcpy(buf, ah + 7, buf_size); s = buf; /* Parse authorization header */ @@ -8688,29 +8826,29 @@ parse_auth_header(struct mg_connection *conn, } if (!strcmp(name, "username")) { - ah->user = value; + auth_header->user = value; } else if (!strcmp(name, "cnonce")) { - ah->cnonce = value; + auth_header->cnonce = value; } else if (!strcmp(name, "response")) { - ah->response = value; + auth_header->response = value; } else if (!strcmp(name, "uri")) { - ah->uri = value; + auth_header->uri = value; } else if (!strcmp(name, "qop")) { - ah->qop = value; + auth_header->qop = value; } else if (!strcmp(name, "nc")) { - ah->nc = value; + auth_header->nc = value; } else if (!strcmp(name, "nonce")) { - ah->nonce = value; + auth_header->nonce = value; } } #if !defined(NO_NONCE_CHECK) /* Read the nonce from the response. */ - if (ah->nonce == NULL) { + if (auth_header->nonce == NULL) { return 0; } s = NULL; - nonce = strtoull(ah->nonce, &s, 10); + nonce = strtoull(auth_header->nonce, &s, 10); if ((s == NULL) || (*s != 0)) { return 0; } @@ -8741,7 +8879,7 @@ parse_auth_header(struct mg_connection *conn, (void)nonce; #endif - return (ah->user != NULL); + return (auth_header->user != NULL); } @@ -8773,7 +8911,7 @@ mg_fgets(char *buf, size_t size, struct mg_file *filep) #if !defined(NO_FILESYSTEMS) struct read_auth_file_struct { struct mg_connection *conn; - struct ah ah; + struct auth_header auth_header; const char *domain; char buf[256 + 256 + 40]; const char *f_user; @@ -8874,9 +9012,9 @@ read_auth_file(struct mg_file *filep, *(char *)(workdata->f_ha1) = 0; (workdata->f_ha1)++; - if (!strcmp(workdata->ah.user, workdata->f_user) + if (!strcmp(workdata->auth_header.user, workdata->f_user) && !strcmp(workdata->domain, workdata->f_domain)) { - switch (workdata->ah.type) { + switch (workdata->auth_header.type) { case 1: /* Basic */ { char md5[33]; @@ -8885,7 +9023,7 @@ read_auth_file(struct mg_file *filep, ":", workdata->domain, ":", - workdata->ah.plain_password, + workdata->auth_header.plain_password, NULL); return 0 == memcmp(workdata->f_ha1, md5, 33); } @@ -8893,12 +9031,12 @@ read_auth_file(struct mg_file *filep, return check_password_digest( workdata->conn->request_info.request_method, workdata->f_ha1, - workdata->ah.uri, - workdata->ah.nonce, - workdata->ah.nc, - workdata->ah.cnonce, - workdata->ah.qop, - workdata->ah.response); + workdata->auth_header.uri, + workdata->auth_header.nonce, + workdata->auth_header.nc, + workdata->auth_header.cnonce, + workdata->auth_header.qop, + workdata->auth_header.response); default: /* None/Other/Unknown */ return 0; } @@ -8923,13 +9061,13 @@ authorize(struct mg_connection *conn, struct mg_file *filep, const char *realm) memset(&workdata, 0, sizeof(workdata)); workdata.conn = conn; - if (!parse_auth_header(conn, buf, sizeof(buf), &workdata.ah)) { + if (!parse_auth_header(conn, buf, sizeof(buf), &workdata.auth_header)) { return 0; } /* CGI needs it as REMOTE_USER */ conn->request_info.remote_user = - mg_strdup_ctx(workdata.ah.user, conn->phys_ctx); + mg_strdup_ctx(workdata.auth_header.user, conn->phys_ctx); if (realm) { workdata.domain = realm; @@ -9564,11 +9702,11 @@ connect_socket( #endif /* Data for poll */ - struct mg_pollfd pfd[1]; + struct mg_pollfd pfd[2]; int pollres; - int ms_wait = 10000; /* 10 second timeout */ - stop_flag_t nonstop; - STOP_FLAG_ASSIGN(&nonstop, 0); + int ms_wait = 10000; /* 10 second timeout */ + stop_flag_t nonstop = 0; /* STOP_FLAG_ASSIGN(&nonstop, 0); */ + unsigned int num_sock = 1; /* use one or two sockets */ /* For a non-blocking socket, the connect sequence is: * 1) call connect (will not block) @@ -9577,7 +9715,15 @@ connect_socket( */ pfd[0].fd = *sock; pfd[0].events = POLLOUT; - pollres = mg_poll(pfd, 1, ms_wait, ctx ? &(ctx->stop_flag) : &nonstop); + + if (ctx && (ctx->context_type == CONTEXT_SERVER)) { + pfd[num_sock].fd = ctx->thread_shutdown_notification_socket; + pfd[num_sock].events = POLLIN; + num_sock++; + } + + pollres = + mg_poll(pfd, num_sock, ms_wait, ctx ? &(ctx->stop_flag) : &nonstop); if (pollres != 1) { /* Not connected */ @@ -10198,8 +10344,11 @@ send_file_data(struct mg_connection *conn, } /* Read from file, exit the loop on error */ - if ((num_read = - (int)fread(buf, 1, (size_t)to_read, filep->access.fp)) + if ((num_read = pull_inner(filep->access.fp, + NULL, + buf, + to_read, + /* unused */ 0.0)) <= 0) { break; } @@ -10733,6 +10882,7 @@ static int skip_to_end_of_word_and_terminate(char **ppw, int eol) { /* Forward until a space is found - use isgraph here */ + /* Extended ASCII characters are also treated as word characters. */ /* See http://www.cplusplus.com/reference/cctype/ */ while ((unsigned char)**ppw > 127 || isgraph((unsigned char)**ppw)) { (*ppw)++; @@ -10831,7 +10981,7 @@ parse_http_headers(char **buf, struct mg_header hdr[MG_MAX_HEADERS]) } /* here *dp is either 0 or '\n' */ - /* in any case, we have a new header */ + /* in any case, we have found a complete header */ num_headers = i + 1; if (*dp) { @@ -10840,9 +10990,11 @@ parse_http_headers(char **buf, struct mg_header hdr[MG_MAX_HEADERS]) *buf = dp; if ((dp[0] == '\r') || (dp[0] == '\n')) { - /* This is the end of the header */ + /* We've had CRLF twice in a row + * This is the end of the headers */ break; } + /* continue within the loop, find the next header */ } else { *buf = dp; break; @@ -11219,11 +11371,11 @@ read_message(FILE *fp, request_len = get_http_header_len(buf, *nread); } - if ((request_len == 0) && (request_timeout >= 0)) { + if ((n <= 0) && (request_timeout >= 0)) { if (mg_difftimespec(&last_action_time, &(conn->req_time)) > request_timeout) { /* Timeout */ - return -1; + return -3; } } } @@ -11443,6 +11595,11 @@ prepare_cgi_environment(struct mg_connection *conn, addenv(env, "SERVER_NAME=%s", conn->dom_ctx->config[AUTHENTICATION_DOMAIN]); addenv(env, "SERVER_ROOT=%s", conn->dom_ctx->config[DOCUMENT_ROOT]); addenv(env, "DOCUMENT_ROOT=%s", conn->dom_ctx->config[DOCUMENT_ROOT]); + if (conn->dom_ctx->config[FALLBACK_DOCUMENT_ROOT]) { + addenv(env, + "FALLBACK_DOCUMENT_ROOT=%s", + conn->dom_ctx->config[FALLBACK_DOCUMENT_ROOT]); + } addenv(env, "SERVER_SOFTWARE=CivetWeb/%s", mg_version()); /* Prepare the environment block */ @@ -12861,11 +13018,14 @@ dav_lock_file(struct mg_connection *conn, const char *path) int i; uint64_t LOCK_DURATION_NS = (uint64_t)(LOCK_DURATION_S) * (uint64_t)1000000000; - struct twebdav_lock *dav_lock = conn->phys_ctx->webdav_lock; + struct twebdav_lock *dav_lock = NULL; - if (!path || !conn->dom_ctx || !conn->request_info.remote_user) { + if (!path || !conn || !conn->dom_ctx || !conn->request_info.remote_user + || !conn->phys_ctx) { return; } + + dav_lock = conn->phys_ctx->webdav_lock; mg_get_request_link(conn, link_buf, sizeof(link_buf)); /* const char *refresh = mg_get_header(conn, "If"); */ @@ -14904,6 +15064,10 @@ handle_request(struct mg_connection *conn) get_header(ri->http_headers, ri->num_headers, "Access-Control-Request-Headers"); + const char *cors_cred_cfg = + conn->dom_ctx->config[ACCESS_CONTROL_ALLOW_CREDENTIALS]; + const char *cors_exphdr_cfg = + conn->dom_ctx->config[ACCESS_CONTROL_EXPOSE_HEADERS]; gmt_time_string(date, sizeof(date), &curtime); mg_printf(conn, @@ -14918,7 +15082,19 @@ handle_request(struct mg_connection *conn) ((cors_meth_cfg[0] == '*') ? cors_acrm : cors_meth_cfg), suggest_connection_header(conn)); - if (cors_acrh != NULL) { + if (cors_cred_cfg && *cors_cred_cfg) { + mg_printf(conn, + "Access-Control-Allow-Credentials: %s\r\n", + cors_cred_cfg); + } + + if (cors_exphdr_cfg && *cors_exphdr_cfg) { + mg_printf(conn, + "Access-Control-Expose-Headers: %s\r\n", + cors_exphdr_cfg); + } + + if (cors_acrh || (cors_cred_cfg && *cors_cred_cfg)) { /* CORS request is asking for additional headers */ const char *cors_hdr_cfg = conn->dom_ctx->config[ACCESS_CONTROL_ALLOW_HEADERS]; @@ -15556,7 +15732,7 @@ parse_port_string(const struct vec *vec, struct socket *so, int *ip_version) unsigned int a, b, c, d; unsigned port; unsigned long portUL; - int ch, len; + int len; const char *cb; char *endptr; #if defined(USE_IPV6) @@ -15709,14 +15885,38 @@ parse_port_string(const struct vec *vec, struct socket *so, int *ip_version) } /* sscanf and the option splitting code ensure the following condition - * Make sure the port is valid and vector ends with the port, 's' or 'r' */ - if ((len > 0) && is_valid_port(port) - && (((size_t)len == vec->len) || (((size_t)len + 1) == vec->len))) { - /* Next character after the port number */ - ch = ((size_t)len < vec->len) ? vec->ptr[len] : '\0'; - so->is_ssl = (ch == 's'); - so->ssl_redir = (ch == 'r'); - if ((ch == '\0') || (ch == 's') || (ch == 'r')) { + * Make sure the port is valid and vector ends with the port, 'o', 's', or + * 'r' */ + if ((len > 0) && (is_valid_port(port))) { + int bad_suffix = 0; + size_t i; + + /* Parse any suffix character(s) after the port number */ + for (i = len; i < vec->len; i++) { + unsigned char *opt = NULL; + switch (vec->ptr[i]) { + case 'o': + opt = &so->is_optional; + break; + case 'r': + opt = &so->ssl_redir; + break; + case 's': + opt = &so->is_ssl; + break; + default: /* empty */ + break; + } + + if ((opt) && (*opt == 0)) + *opt = 1; + else { + bad_suffix = 1; + break; + } + } + + if ((bad_suffix == 0) && ((so->is_ssl == 0) || (so->ssl_redir == 0))) { return 1; } } @@ -15771,8 +15971,14 @@ is_ssl_port_used(const char *ports) char prevIsNumber = 0; for (i = 0; i < portslen; i++) { - if (prevIsNumber && (ports[i] == 's' || ports[i] == 'r')) { - return 1; + if (prevIsNumber) { + int suffixCharIdx = (ports[i] == 'o') + ? (i + 1) + : i; /* allow "os" and "or" suffixes */ + if (ports[suffixCharIdx] == 's' + || ports[suffixCharIdx] == 'r') { + return 1; + } } if (ports[i] >= '0' && ports[i] <= '9') { prevIsNumber = 1; @@ -15955,6 +16161,10 @@ set_ports_option(struct mg_context *phys_ctx) strerror(errno)); closesocket(so.sock); so.sock = INVALID_SOCKET; + if (so.is_optional) { + portsOk++; /* it's okay if we couldn't bind, this port is + optional anyway */ + } continue; } } @@ -15971,6 +16181,10 @@ set_ports_option(struct mg_context *phys_ctx) strerror(errno)); closesocket(so.sock); so.sock = INVALID_SOCKET; + if (so.is_optional) { + portsOk++; /* it's okay if we couldn't bind, this port is + optional anyway */ + } continue; } } @@ -15987,6 +16201,10 @@ set_ports_option(struct mg_context *phys_ctx) strerror(errno)); closesocket(so.sock); so.sock = INVALID_SOCKET; + if (so.is_optional) { + portsOk++; /* it's okay if we couldn't bind, this port is + optional anyway */ + } continue; } } @@ -16064,9 +16282,14 @@ set_ports_option(struct mg_context *phys_ctx) continue; } + /* The +2 below includes the original +1 (for the socket we're about to + * add), plus another +1 for the thread_shutdown_notification_socket + * that we'll also want to poll() on so that mg_stop() can return + * quickly + */ if ((pfd = (struct mg_pollfd *) mg_realloc_ctx(phys_ctx->listening_socket_fds, - (phys_ctx->num_listening_sockets + 1) + (phys_ctx->num_listening_sockets + 2) * sizeof(phys_ctx->listening_socket_fds[0]), phys_ctx)) == NULL) { @@ -16588,15 +16811,26 @@ sslize(struct mg_connection *conn, /* Need to retry the function call "later". * See https://linux.die.net/man/3/ssl_get_error * This is typical for non-blocking sockets. */ - struct mg_pollfd pfd; + struct mg_pollfd pfd[2]; int pollres; - pfd.fd = conn->client.sock; - pfd.events = ((err == SSL_ERROR_WANT_CONNECT) - || (err == SSL_ERROR_WANT_WRITE)) - ? POLLOUT - : POLLIN; - pollres = - mg_poll(&pfd, 1, 50, &(conn->phys_ctx->stop_flag)); + unsigned int num_sock = 1; + pfd[0].fd = conn->client.sock; + pfd[0].events = ((err == SSL_ERROR_WANT_CONNECT) + || (err == SSL_ERROR_WANT_WRITE)) + ? POLLOUT + : POLLIN; + + if (conn->phys_ctx->context_type == CONTEXT_SERVER) { + pfd[num_sock].fd = + conn->phys_ctx->thread_shutdown_notification_socket; + pfd[num_sock].events = POLLIN; + num_sock++; + } + + pollres = mg_poll(pfd, + num_sock, + 50, + &(conn->phys_ctx->stop_flag)); if (pollres < 0) { /* Break if error occurred (-1) * or server shutdown (-2) */ @@ -17761,9 +17995,6 @@ reset_per_request_attributes(struct mg_connection *conn) } conn->request_info.local_uri = NULL; - /* Pi-hole addition */ - memset(conn->request_info.csrf_token, 0, sizeof(conn->request_info.csrf_token)); - #if defined(USE_SERVER_STATS) conn->processing_time = 0; #endif @@ -18019,7 +18250,7 @@ mg_close_connection(struct mg_connection *conn) * timeouts, we will just wait a few seconds in mg_join_thread. */ /* join worker thread */ - for (i = 0; i < conn->phys_ctx->cfg_worker_threads; i++) { + for (i = 0; i < conn->phys_ctx->spawned_worker_threads; i++) { mg_join_thread(conn->phys_ctx->worker_threadids[i]); } } @@ -18678,7 +18909,8 @@ get_message(struct mg_connection *conn, char *ebuf, size_t ebuf_len, int *err) ebuf, ebuf_len, "%s", - "Malformed message"); + conn->request_len == -3 ? "Request timeout" + : "Malformed message"); *err = 400; } else { /* Server did not recv anything -> just close the connection */ @@ -19049,6 +19281,24 @@ websocket_client_thread(void *data) #endif +#if defined(USE_WEBSOCKET) +static void +generate_websocket_magic(char *magic25) +{ + uint64_t rnd; + unsigned char buffer[2 * sizeof(rnd)]; + + rnd = get_random(); + memcpy(buffer, &rnd, sizeof(rnd)); + rnd = get_random(); + memcpy(buffer + sizeof(rnd), &rnd, sizeof(rnd)); + + size_t dst_len = 24 + 1; + mg_base64_encode(buffer, sizeof(buffer), magic25, &dst_len); +} +#endif + + static struct mg_connection * mg_connect_websocket_client_impl(const struct mg_client_options *client_options, int use_ssl, @@ -19065,7 +19315,8 @@ mg_connect_websocket_client_impl(const struct mg_client_options *client_options, #if defined(USE_WEBSOCKET) struct websocket_client_thread_data *thread_data; - static const char *magic = "x3JJHMbDL1EzLkh9GBhXDw=="; + char magic[32]; + generate_websocket_magic(magic); const char *host = client_options->host; int i; @@ -19229,7 +19480,8 @@ mg_connect_websocket_client_impl(const struct mg_client_options *client_options, /* Now upgrade to ws/wss client context */ conn->phys_ctx->user_data = user_data; conn->phys_ctx->context_type = CONTEXT_WS_CLIENT; - conn->phys_ctx->cfg_worker_threads = 1; /* one worker thread */ + conn->phys_ctx->cfg_max_worker_threads = 1; /* one worker thread */ + conn->phys_ctx->spawned_worker_threads = 1; /* one worker thread */ /* Start a thread to read the websocket client connection * This thread will automatically stop when mg_disconnect is @@ -19238,7 +19490,7 @@ mg_connect_websocket_client_impl(const struct mg_client_options *client_options, thread_data, conn->phys_ctx->worker_threadids) != 0) { - conn->phys_ctx->cfg_worker_threads = 0; + conn->phys_ctx->spawned_worker_threads = 0; mg_free(thread_data); mg_close_connection(conn); conn = NULL; @@ -19609,6 +19861,9 @@ process_new_connection(struct mg_connection *conn) #endif } +static int +mg_start_worker_thread(struct mg_context *ctx, + int only_if_no_idle_threads); /* forward declaration */ #if defined(ALTERNATIVE_QUEUE) @@ -19617,8 +19872,12 @@ produce_socket(struct mg_context *ctx, const struct socket *sp) { unsigned int i; + (void)mg_start_worker_thread( + ctx, 1); /* will start a worker-thread only if there aren't currently + any idle worker-threads */ + while (!ctx->stop_flag) { - for (i = 0; i < ctx->cfg_worker_threads; i++) { + for (i = 0; i < ctx->spawned_worker_threads; i++) { /* find a free worker slot and signal it */ if (ctx->client_socks[i].in_use == 2) { (void)pthread_mutex_lock(&ctx->thread_mutex); @@ -19643,10 +19902,18 @@ produce_socket(struct mg_context *ctx, const struct socket *sp) static int -consume_socket(struct mg_context *ctx, struct socket *sp, int thread_index) +consume_socket(struct mg_context *ctx, + struct socket *sp, + int thread_index, + int counter_was_preincremented) { DEBUG_TRACE("%s", "going idle"); (void)pthread_mutex_lock(&ctx->thread_mutex); + if (counter_was_preincremented + == 0) { /* first call only: the master-thread pre-incremented this + before he spawned us */ + ctx->idle_worker_thread_count++; + } ctx->client_socks[thread_index].in_use = 2; (void)pthread_mutex_unlock(&ctx->thread_mutex); @@ -19663,6 +19930,7 @@ consume_socket(struct mg_context *ctx, struct socket *sp, int thread_index) } return 0; } + ctx->idle_worker_thread_count--; (void)pthread_mutex_unlock(&ctx->thread_mutex); if (sp->in_use == 1) { DEBUG_TRACE("grabbed socket %d, going busy", sp->sock); @@ -19677,12 +19945,20 @@ consume_socket(struct mg_context *ctx, struct socket *sp, int thread_index) /* Worker threads take accepted socket from the queue */ static int -consume_socket(struct mg_context *ctx, struct socket *sp, int thread_index) +consume_socket(struct mg_context *ctx, + struct socket *sp, + int thread_index, + int counter_was_preincremented) { (void)thread_index; - (void)pthread_mutex_lock(&ctx->thread_mutex); DEBUG_TRACE("%s", "going idle"); + (void)pthread_mutex_lock(&ctx->thread_mutex); + if (counter_was_preincremented + == 0) { /* first call only: the master-thread pre-incremented this + before he spawned us */ + ctx->idle_worker_thread_count++; + } /* If the queue is empty, wait. We're idle at this point. */ while ((ctx->sq_head == ctx->sq_tail) @@ -19706,6 +19982,8 @@ consume_socket(struct mg_context *ctx, struct socket *sp, int thread_index) } (void)pthread_cond_signal(&ctx->sq_empty); + + ctx->idle_worker_thread_count--; (void)pthread_mutex_unlock(&ctx->thread_mutex); return STOP_FLAG_IS_ZERO(&ctx->stop_flag); @@ -19752,6 +20030,10 @@ produce_socket(struct mg_context *ctx, const struct socket *sp) (void)pthread_cond_signal(&ctx->sq_full); (void)pthread_mutex_unlock(&ctx->thread_mutex); + + (void)mg_start_worker_thread( + ctx, 1); /* will start a worker-thread only if there aren't currently + any idle worker-threads */ } #endif /* ALTERNATIVE_QUEUE */ @@ -19762,6 +20044,7 @@ worker_thread_run(struct mg_connection *conn) struct mg_context *ctx = conn->phys_ctx; int thread_index; struct mg_workerTLS tls; + int first_call_to_consume_socket = 1; mg_set_thread_name("worker"); @@ -19787,7 +20070,7 @@ worker_thread_run(struct mg_connection *conn) /* Connection structure has been pre-allocated */ thread_index = (int)(conn - ctx->worker_connections); if ((thread_index < 0) - || ((unsigned)thread_index >= (unsigned)ctx->cfg_worker_threads)) { + || ((unsigned)thread_index >= (unsigned)ctx->cfg_max_worker_threads)) { mg_cry_ctx_internal(ctx, "Internal error: Invalid worker index %i", thread_index); @@ -19828,7 +20111,9 @@ worker_thread_run(struct mg_connection *conn) /* Call consume_socket() even when ctx->stop_flag > 0, to let it * signal sq_empty condvar to wake up the master waiting in * produce_socket() */ - while (consume_socket(ctx, &conn->client, thread_index)) { + while (consume_socket( + ctx, &conn->client, thread_index, first_call_to_consume_socket)) { + first_call_to_consume_socket = 0; /* New connections must start with new protocol negotiation */ tls.alpn_proto = NULL; @@ -20035,6 +20320,7 @@ accept_new_connection(const struct socket *listener, struct mg_context *ctx) set_close_on_exec(so.sock, NULL, ctx); so.is_ssl = listener->is_ssl; so.ssl_redir = listener->ssl_redir; + so.is_optional = listener->is_optional; if (getsockname(so.sock, &so.lsa.sa, &len) != 0) { mg_cry_ctx_internal(ctx, "%s: getsockname() failed: %s", @@ -20183,8 +20469,17 @@ master_thread_run(struct mg_context *ctx) pfd[i].events = POLLIN; } + /* We listen on this socket just so that mg_stop() can cause mg_poll() + * to return ASAP. Don't worry, we did allocate an extra slot at the end + * of listening_socket_fds[] just to hold this + */ + pfd[ctx->num_listening_sockets].fd = + ctx->thread_shutdown_notification_socket; + pfd[ctx->num_listening_sockets].events = POLLIN; + if (mg_poll(pfd, - ctx->num_listening_sockets, + ctx->num_listening_sockets + + 1, // +1 for the thread_shutdown_notification_socket SOCKET_TIMEOUT_QUANTUM, &(ctx->stop_flag)) > 0) { @@ -20210,7 +20505,7 @@ master_thread_run(struct mg_context *ctx) /* Wakeup workers that are waiting for connections to handle. */ #if defined(ALTERNATIVE_QUEUE) - for (i = 0; i < ctx->cfg_worker_threads; i++) { + for (i = 0; i < ctx->spawned_worker_threads; i++) { event_signal(ctx->client_wait_events[i]); } #else @@ -20220,7 +20515,7 @@ master_thread_run(struct mg_context *ctx) #endif /* Join all worker threads to avoid leaking threads. */ - workerthreadcount = ctx->cfg_worker_threads; + workerthreadcount = ctx->spawned_worker_threads; for (i = 0; i < workerthreadcount; i++) { if (ctx->worker_threadids[i] != 0) { mg_join_thread(ctx->worker_threadids[i]); @@ -20324,7 +20619,7 @@ free_context(struct mg_context *ctx) #if defined(ALTERNATIVE_QUEUE) mg_free(ctx->client_socks); if (ctx->client_wait_events != NULL) { - for (i = 0; (unsigned)i < ctx->cfg_worker_threads; i++) { + for (i = 0; (unsigned)i < ctx->spawned_worker_threads; i++) { event_destroy(ctx->client_wait_events[i]); } mg_free(ctx->client_wait_events); @@ -20342,6 +20637,14 @@ free_context(struct mg_context *ctx) (void)pthread_mutex_destroy(&ctx->lua_bg_mutex); #endif + /* Deallocate shutdown-triggering socket-pair */ + if (ctx->user_shutdown_notification_socket >= 0) { + closesocket(ctx->user_shutdown_notification_socket); + } + if (ctx->thread_shutdown_notification_socket >= 0) { + closesocket(ctx->thread_shutdown_notification_socket); + } + /* Deallocate config parameters */ for (i = 0; i < NUM_OPTIONS; i++) { if (ctx->dd.config[i] != NULL) { @@ -20418,6 +20721,12 @@ mg_stop(struct mg_context *ctx) /* Set stop flag, so all threads know they have to exit. */ STOP_FLAG_ASSIGN(&ctx->stop_flag, 1); + /* Closing this socket will cause mg_poll() in all the I/O threads to return + * immediately */ + closesocket(ctx->user_shutdown_notification_socket); + ctx->user_shutdown_notification_socket = + -1; /* to avoid calling closesocket() again in free_context() */ + /* Join timer thread */ #if defined(USE_TIMERS) timers_exit(ctx); @@ -20515,13 +20824,122 @@ legacy_init(const char **options) } } +/* we'll assume it's only Windows that doesn't have socketpair() available */ +#if !defined(HAVE_SOCKETPAIR) && !defined(_WIN32) +#define HAVE_SOCKETPAIR 1 +#endif + +static int +mg_socketpair(int *sockA, int *sockB) +{ + int temp[2] = {-1, -1}; + int asock = -1; + + /** Default to unallocated */ + *sockA = -1; + *sockB = -1; + +#if defined(HAVE_SOCKETPAIR) + int ret = socketpair(AF_UNIX, SOCK_STREAM, 0, temp); + if (ret == 0) { + *sockA = temp[0]; + *sockB = temp[1]; + set_close_on_exec(*sockA, NULL, NULL); + set_close_on_exec(*sockB, NULL, NULL); + } + (void)asock; /* not used */ + return ret; +#else + /** No socketpair() call is available, so we'll have to roll our own + * implementation */ + asock = socket(PF_INET, SOCK_STREAM, 0); + if (asock >= 0) { + struct sockaddr_in addr; + struct sockaddr *pa = (struct sockaddr *)&addr; + socklen_t addrLen = sizeof(addr); + + memset(&addr, 0, sizeof(addr)); + addr.sin_family = AF_INET; + addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + addr.sin_port = 0; + + if ((bind(asock, pa, sizeof(addr)) == 0) + && (getsockname(asock, pa, &addrLen) == 0) + && (listen(asock, 1) == 0)) { + temp[0] = socket(PF_INET, SOCK_STREAM, 0); + if ((temp[0] >= 0) && (connect(temp[0], pa, sizeof(addr)) == 0)) { + temp[1] = accept(asock, pa, &addrLen); + if (temp[1] >= 0) { + closesocket(asock); + *sockA = temp[0]; + *sockB = temp[1]; + set_close_on_exec(*sockA, NULL, NULL); + set_close_on_exec(*sockB, NULL, NULL); + return 0; /* success! */ + } + } + } + } + + /* Cleanup */ + if (asock >= 0) + closesocket(asock); + if (temp[0] >= 0) + closesocket(temp[0]); + if (temp[1] >= 0) + closesocket(temp[1]); + return -1; /* fail! */ +#endif +} + +static int +mg_start_worker_thread(struct mg_context *ctx, int only_if_no_idle_threads) +{ + const unsigned int i = ctx->spawned_worker_threads; + if (i >= ctx->cfg_max_worker_threads) { + return -1; /* Oops, we hit our worker-thread limit! No more worker + threads, ever! */ + } + + (void)pthread_mutex_lock(&ctx->thread_mutex); +#if defined(ALTERNATIVE_QUEUE) + if ((only_if_no_idle_threads) && (ctx->idle_worker_thread_count > 0)) { +#else + if ((only_if_no_idle_threads) + && (ctx->idle_worker_thread_count + > (unsigned)(ctx->sq_head - ctx->sq_tail))) { +#endif + (void)pthread_mutex_unlock(&ctx->thread_mutex); + return -2; /* There are idle threads available, so no need to spawn a + new worker thread now */ + } + ctx->idle_worker_thread_count++; /* we do this here to avoid a race + condition while the thread is starting + up */ + (void)pthread_mutex_unlock(&ctx->thread_mutex); + + ctx->worker_connections[i].phys_ctx = ctx; + int ret = mg_start_thread_with_id(worker_thread, + &ctx->worker_connections[i], + &ctx->worker_threadids[i]); + if (ret == 0) { + ctx->spawned_worker_threads++; /* note that we've filled another slot in + the table */ + DEBUG_TRACE("Started worker_thread #%i", ctx->spawned_worker_threads); + } else { + (void)pthread_mutex_lock(&ctx->thread_mutex); + ctx->idle_worker_thread_count--; /* whoops, roll-back on error */ + (void)pthread_mutex_unlock(&ctx->thread_mutex); + } + return ret; +} CIVETWEB_API struct mg_context * mg_start2(struct mg_init_data *init, struct mg_error_data *error) { struct mg_context *ctx; const char *name, *value, *default_value; - int idx, ok, workerthreadcount; + int idx, ok, prespawnthreadcount, workerthreadcount; unsigned int i; int itmp; void (*exit_callback)(const struct mg_context *ctx) = 0; @@ -20598,6 +21016,15 @@ mg_start2(struct mg_init_data *init, struct mg_error_data *error) #if defined(USE_LUA) ok &= (0 == pthread_mutex_init(&ctx->lua_bg_mutex, &pthread_mutex_attr)); #endif + + /** mg_stop() will close the user_shutdown_notification_socket, and that + * will cause poll() to return immediately in the master-thread, so that + * mg_stop() can also return immediately. + */ + ok &= (0 + == mg_socketpair(&ctx->user_shutdown_notification_socket, + &ctx->thread_shutdown_notification_socket)); + if (!ok) { unsigned error_id = (unsigned)ERRNO; const char *err_msg = @@ -20765,6 +21192,13 @@ mg_start2(struct mg_init_data *init, struct mg_error_data *error) /* Worker thread count option */ workerthreadcount = atoi(ctx->dd.config[NUM_THREADS]); + prespawnthreadcount = atoi(ctx->dd.config[PRESPAWN_THREADS]); + + if ((prespawnthreadcount < 0) + || (prespawnthreadcount > workerthreadcount)) { + prespawnthreadcount = + workerthreadcount; /* can't prespawn more than all of them! */ + } if ((workerthreadcount > MAX_WORKER_THREADS) || (workerthreadcount <= 0)) { if (workerthreadcount <= 0) { @@ -21029,10 +21463,11 @@ mg_start2(struct mg_init_data *init, struct mg_error_data *error) return NULL; } - ctx->cfg_worker_threads = ((unsigned int)(workerthreadcount)); - ctx->worker_threadids = (pthread_t *)mg_calloc_ctx(ctx->cfg_worker_threads, - sizeof(pthread_t), - ctx); + ctx->cfg_max_worker_threads = ((unsigned int)(workerthreadcount)); + ctx->worker_threadids = + (pthread_t *)mg_calloc_ctx(ctx->cfg_max_worker_threads, + sizeof(pthread_t), + ctx); if (ctx->worker_threadids == NULL) { const char *err_msg = "Not enough memory for worker thread ID array"; @@ -21040,8 +21475,8 @@ mg_start2(struct mg_init_data *init, struct mg_error_data *error) if (error != NULL) { error->code = MG_ERROR_DATA_CODE_OUT_OF_MEMORY; - error->code_sub = - (unsigned)ctx->cfg_worker_threads * (unsigned)sizeof(pthread_t); + error->code_sub = (unsigned)ctx->cfg_max_worker_threads + * (unsigned)sizeof(pthread_t); mg_snprintf(NULL, NULL, /* No truncation check for error buffers */ error->text, @@ -21055,7 +21490,7 @@ mg_start2(struct mg_init_data *init, struct mg_error_data *error) return NULL; } ctx->worker_connections = - (struct mg_connection *)mg_calloc_ctx(ctx->cfg_worker_threads, + (struct mg_connection *)mg_calloc_ctx(ctx->cfg_max_worker_threads, sizeof(struct mg_connection), ctx); if (ctx->worker_connections == NULL) { @@ -21065,7 +21500,7 @@ mg_start2(struct mg_init_data *init, struct mg_error_data *error) if (error != NULL) { error->code = MG_ERROR_DATA_CODE_OUT_OF_MEMORY; - error->code_sub = (unsigned)ctx->cfg_worker_threads + error->code_sub = (unsigned)ctx->cfg_max_worker_threads * (unsigned)sizeof(struct mg_connection); mg_snprintf(NULL, NULL, /* No truncation check for error buffers */ @@ -21082,7 +21517,7 @@ mg_start2(struct mg_init_data *init, struct mg_error_data *error) #if defined(ALTERNATIVE_QUEUE) ctx->client_wait_events = - (void **)mg_calloc_ctx(ctx->cfg_worker_threads, + (void **)mg_calloc_ctx(ctx->cfg_max_worker_threads, sizeof(ctx->client_wait_events[0]), ctx); if (ctx->client_wait_events == NULL) { @@ -21092,7 +21527,7 @@ mg_start2(struct mg_init_data *init, struct mg_error_data *error) if (error != NULL) { error->code = MG_ERROR_DATA_CODE_OUT_OF_MEMORY; - error->code_sub = (unsigned)ctx->cfg_worker_threads + error->code_sub = (unsigned)ctx->cfg_max_worker_threads * (unsigned)sizeof(ctx->client_wait_events[0]); mg_snprintf(NULL, NULL, /* No truncation check for error buffers */ @@ -21108,7 +21543,7 @@ mg_start2(struct mg_init_data *init, struct mg_error_data *error) } ctx->client_socks = - (struct socket *)mg_calloc_ctx(ctx->cfg_worker_threads, + (struct socket *)mg_calloc_ctx(ctx->cfg_max_worker_threads, sizeof(ctx->client_socks[0]), ctx); if (ctx->client_socks == NULL) { @@ -21119,7 +21554,7 @@ mg_start2(struct mg_init_data *init, struct mg_error_data *error) if (error != NULL) { error->code = MG_ERROR_DATA_CODE_OUT_OF_MEMORY; - error->code_sub = (unsigned)ctx->cfg_worker_threads + error->code_sub = (unsigned)ctx->cfg_max_worker_threads * (unsigned)sizeof(ctx->client_socks[0]); mg_snprintf(NULL, NULL, /* No truncation check for error buffers */ @@ -21134,7 +21569,7 @@ mg_start2(struct mg_init_data *init, struct mg_error_data *error) return NULL; } - for (i = 0; (unsigned)i < ctx->cfg_worker_threads; i++) { + for (i = 0; (unsigned)i < ctx->cfg_max_worker_threads; i++) { ctx->client_wait_events[i] = event_create(); if (ctx->client_wait_events[i] == 0) { const char *err_msg = "Error creating worker event %i"; @@ -21198,23 +21633,18 @@ mg_start2(struct mg_init_data *init, struct mg_error_data *error) ctx->context_type = CONTEXT_SERVER; /* server context */ /* Start worker threads */ - for (i = 0; i < ctx->cfg_worker_threads; i++) { + for (i = 0; (int)i < prespawnthreadcount; i++) { /* worker_thread sets up the other fields */ - ctx->worker_connections[i].phys_ctx = ctx; - if (mg_start_thread_with_id(worker_thread, - &ctx->worker_connections[i], - &ctx->worker_threadids[i]) - != 0) { - + if (mg_start_worker_thread(ctx, 0) != 0) { long error_no = (long)ERRNO; /* thread was not created */ - if (i > 0) { + if (ctx->spawned_worker_threads > 0) { /* If the second, third, ... thread cannot be created, set a * warning, but keep running. */ mg_cry_ctx_internal(ctx, "Cannot start worker thread %i: error %ld", - i + 1, + ctx->spawned_worker_threads + 1, error_no); /* If the server initialization should stop here, all @@ -22144,7 +22574,7 @@ mg_get_connection_info(const struct mg_context *ctx, return 0; } - if ((unsigned)idx >= ctx->cfg_worker_threads) { + if ((unsigned)idx >= ctx->cfg_max_worker_threads) { /* Out of range */ return 0; } @@ -22354,44 +22784,44 @@ mg_get_connection_info(const struct mg_context *ctx, return (int)connection_info_length; } + #if 0 -/* Get handler information. It can be printed or stored by the caller. - * Return the size of available information. */ +/* Get handler information. Not fully implemented. Is it required? */ CIVETWEB_API int mg_get_handler_info(struct mg_context *ctx, - char *buffer, - int buflen) + char *buffer, + int buflen) { - int handler_info_len = 0; - struct mg_handler_info *tmp_rh; - mg_lock_context(ctx); + int handler_info_len = 0; + struct mg_handler_info *tmp_rh; + mg_lock_context(ctx); - for (tmp_rh = ctx->dd.handlers; tmp_rh != NULL; tmp_rh = tmp_rh->next) { + for (tmp_rh = ctx->dd.handlers; tmp_rh != NULL; tmp_rh = tmp_rh->next) { - if (buflen > handler_info_len+ tmp_rh->uri_len) { - memcpy(buffer+handler_info_len, tmp_rh->uri, tmp_rh->uri_len); - } - handler_info_len += tmp_rh->uri_len; + if (buflen > handler_info_len + tmp_rh->uri_len) { + memcpy(buffer + handler_info_len, tmp_rh->uri, tmp_rh->uri_len); + } + handler_info_len += tmp_rh->uri_len; - switch (tmp_rh->handler_type) { - case REQUEST_HANDLER: - (void)tmp_rh->handler; - break; - case WEBSOCKET_HANDLER: - (void)tmp_rh->connect_handler; - (void) tmp_rh->ready_handler; - (void) tmp_rh->data_handler; - (void) tmp_rh->close_handler; - break; - case AUTH_HANDLER: - (void) tmp_rh->auth_handler; - break; - } - (void)cbdata; - } + switch (tmp_rh->handler_type) { + case REQUEST_HANDLER: + (void)tmp_rh->handler; + break; + case WEBSOCKET_HANDLER: + (void)tmp_rh->connect_handler; + (void)tmp_rh->ready_handler; + (void)tmp_rh->data_handler; + (void)tmp_rh->close_handler; + break; + case AUTH_HANDLER: + (void)tmp_rh->auth_handler; + break; + } + (void)cbdata; + } - mg_unlock_context(ctx); - return handler_info_len; + mg_unlock_context(ctx); + return handler_info_len; } #endif #endif diff --git a/src/webserver/civetweb/civetweb.h b/src/webserver/civetweb/civetweb.h index a50be337..5ae6a0c7 100644 --- a/src/webserver/civetweb/civetweb.h +++ b/src/webserver/civetweb/civetweb.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2021 the Civetweb developers +/* Copyright (c) 2013-2024 the Civetweb developers * Copyright (c) 2004-2013 Sergey Lyubka * * Permission is hereby granted, free of charge, to any person obtaining a copy @@ -23,9 +23,9 @@ #ifndef CIVETWEB_HEADER_INCLUDED #define CIVETWEB_HEADER_INCLUDED -#define CIVETWEB_VERSION "1.16" +#define CIVETWEB_VERSION "1.17" #define CIVETWEB_VERSION_MAJOR (1) -#define CIVETWEB_VERSION_MINOR (16) +#define CIVETWEB_VERSION_MINOR (17) #define CIVETWEB_VERSION_PATCH (0) #ifndef CIVETWEB_API @@ -183,9 +183,6 @@ struct mg_request_info { const char *acceptedWebSocketSubprotocol; /* websocket subprotocol, * accepted during handshake */ - /* Pi-hole modification */ - char csrf_token[32]; - int is_authenticated; }; @@ -931,22 +928,6 @@ CIVETWEB_API int mg_send_http_error(struct mg_connection *conn, PRINTF_FORMAT_STRING(const char *fmt), ...) PRINTF_ARGS(3, 4); -/************************************** Pi-hole method **************************************/ -int my_send_http_error_headers(struct mg_connection *conn, - int status, const char* mime_type, - long long content_length); - -void FTL_rewrite_pattern(char *filename, size_t filename_buf_len); - -#define MG_CONFIG_MBEDTLS_DEBUG 3 -void FTL_mbed_debug(void *user_param, int level, const char *file, - int line, const char *message); - -// Buffer used for additional "Set-Cookie" headers -#define PIHOLE_HEADERS_MAXLEN 1024 -extern char pi_hole_extra_headers[PIHOLE_HEADERS_MAXLEN]; -/********************************************************************************************/ - /* Send "HTTP 200 OK" response header. * After calling this function, use mg_write or mg_send_chunk to send the diff --git a/src/webserver/civetweb/handle_form.inl b/src/webserver/civetweb/handle_form.inl index be477a05..4de8d4f5 100644 --- a/src/webserver/civetweb/handle_form.inl +++ b/src/webserver/civetweb/handle_form.inl @@ -162,14 +162,17 @@ search_boundary(const char *buf, const char *boundary, size_t boundary_len) { - /* We must do a binary search here, not a string search, since the buffer - * may contain '\x00' bytes, if binary data is transferred. */ - int clen = (int)buf_len - (int)boundary_len - 4; + char *boundary_start = "\r\n--"; + size_t boundary_start_len = strlen(boundary_start); + + /* We must do a binary search here, not a string search, since the + * buffer may contain '\x00' bytes, if binary data is transferred. */ + int clen = (int)buf_len - (int)boundary_len - boundary_start_len; int i; for (i = 0; i <= clen; i++) { - if (!memcmp(buf + i, "\r\n--", 4)) { - if (!memcmp(buf + i + 4, boundary, boundary_len)) { + if (!memcmp(buf + i, boundary_start, boundary_start_len)) { + if (!memcmp(buf + i + boundary_start_len, boundary, boundary_len)) { return buf + i; } } @@ -624,6 +627,7 @@ mg_handle_form_request(struct mg_connection *conn, } /* Copy boundary string to variable "boundary" */ + /* fbeg is pointer to start of value of boundary */ fbeg = content_type + bl + 9; bl = strlen(fbeg); boundary = (char *)mg_malloc(bl + 1); @@ -701,43 +705,75 @@ mg_handle_form_request(struct mg_connection *conn, return -1; } + /* @see https://www.rfc-editor.org/rfc/rfc2046.html#section-5.1.1 + * + * multipart-body := [preamble CRLF] + * dash-boundary transport-padding CRLF + * body-part *encapsulation + * close-delimiter transport-padding + * [CRLF epilogue] + */ + if (part_no == 0) { - int d = 0; - while ((d < buf_fill) && (buf[d] != '-')) { - d++; + size_t preamble_length = 0; + /* skip over the preamble until we find a complete boundary + * limit the preamble length to prevent abuse */ + /* +2 for the -- preceding the boundary */ + while (preamble_length < 1024 + && (preamble_length < buf_fill - bl) + && strncmp(buf + preamble_length + 2, boundary, bl)) { + preamble_length++; } - if ((d > 0) && (buf[d] == '-')) { - memmove(buf, buf + d, (unsigned)buf_fill - (unsigned)d); - buf_fill -= d; + /* reset the start of buf to remove the preamble */ + if (0 == strncmp(buf + preamble_length + 2, boundary, bl)) { + memmove(buf, + buf + preamble_length, + (unsigned)buf_fill - (unsigned)preamble_length); + buf_fill -= preamble_length; buf[buf_fill] = 0; } } - if (buf[0] != '-' || buf[1] != '-') { + /* either it starts with a boundary and it's fine, or it's malformed + * because: + * - the preamble was longer than accepted + * - couldn't find a boundary at all in the body + * - didn't have a terminating boundary */ + if (buf_fill < (bl + 2) || strncmp(buf, "--", 2) + || strncmp(buf + 2, boundary, bl)) { /* Malformed request */ mg_free(boundary); return -1; } - if (0 != strncmp(buf + 2, boundary, bl)) { - /* Malformed request */ - mg_free(boundary); - return -1; + + /* skip the -- */ + char *boundary_start = buf + 2; + size_t transport_padding = 0; + while (boundary_start[bl + transport_padding] == ' ' + || boundary_start[bl + transport_padding] == '\t') { + transport_padding++; } - if (buf[bl + 2] != '\r' || buf[bl + 3] != '\n') { - /* Every part must end with \r\n, if there is another part. - * The end of the request has an extra -- */ - if (((size_t)buf_fill != (size_t)(bl + 6)) - || (strncmp(buf + bl + 2, "--\r\n", 4))) { + char *boundary_end = boundary_start + bl + transport_padding; + + /* after the transport padding, if the boundary isn't + * immediately followed by a \r\n then it is either... */ + if (strncmp(boundary_end, "\r\n", 2)) + { + /* ...the final boundary, and it is followed by --, (in which + * case it's the end of the request) or it's a malformed + * request */ + if (strncmp(boundary_end, "--", 2)) { /* Malformed request */ mg_free(boundary); return -1; } - /* End of the request */ + /* Ingore any epilogue here */ break; } + /* skip the \r\n */ + hbuf = boundary_end + 2; /* Next, we need to get the part header: Read until \r\n\r\n */ - hbuf = buf + bl + 4; hend = strstr(hbuf, "\r\n\r\n"); if (!hend) { /* Malformed request */ diff --git a/src/webserver/civetweb/match.inl b/src/webserver/civetweb/match.inl index a5011f57..34ee00ef 100644 --- a/src/webserver/civetweb/match.inl +++ b/src/webserver/civetweb/match.inl @@ -47,8 +47,8 @@ mg_match_impl(const char *pat, /* Advance as long as there are ? */ i_pat++; i_str++; - } while ((pat[i_pat] == '?') && (str[i_str] != '\0') - && (str[i_str] != '/') && (i_pat < pat_len)); + } while ((i_pat < pat_len) && (pat[i_pat] == '?') + && (str[i_str] != '\0') && (str[i_str] != '/')); /* If we have a match context, add the substring we just found */ if (mcx) { @@ -72,7 +72,7 @@ mg_match_impl(const char *pat, ptrdiff_t ret; i_pat++; - if ((pat[i_pat] == '*') && (i_pat < pat_len)) { + if ((i_pat < pat_len) && (pat[i_pat] == '*')) { /* Pattern ** matches all */ i_pat++; len = strlen(str + i_str); diff --git a/src/webserver/civetweb/mod_lua.inl b/src/webserver/civetweb/mod_lua.inl index e9d90ca5..4ad3c38a 100644 --- a/src/webserver/civetweb/mod_lua.inl +++ b/src/webserver/civetweb/mod_lua.inl @@ -641,7 +641,10 @@ run_lsp_kepler(struct mg_connection *conn, /* Only send a HTML header, if this is the top level page. * If this page is included by some mg.include calls, do not add a * header. */ - mg_printf(conn, "HTTP/1.1 200 OK\r\n"); + if(conn->status_code < 0) + mg_printf(conn, "HTTP/1.1 200 OK\r\n"); + else + mg_printf(conn, "HTTP/1.1 %d %s\r\n", conn->status_code, mg_get_response_code_text(conn, conn->status_code)); send_no_cache_header(conn); send_additional_header(conn); mg_printf(conn, @@ -2603,10 +2606,6 @@ prepare_lua_request_info_inner(const struct mg_connection *conn, lua_State *L) reg_string(L, "finger", conn->request_info.client_cert->finger); lua_rawset(L, -3); } - - /* Pi-hole addition */ - reg_string(L, "csrf_token", conn->request_info.csrf_token); - reg_boolean(L, "is_authenticated", conn->request_info.is_authenticated != 0); } @@ -2793,11 +2792,7 @@ lua_error_handler(lua_State *L) static void prepare_lua_environment(struct mg_context *ctx, struct mg_connection *conn, -#if defined(USE_WEBSOCKET) struct lua_websock_data *ws_conn_list, -#else - void *ws_conn_list, -#endif lua_State *L, const char *script_name, int lua_env_type) @@ -2943,6 +2938,11 @@ prepare_lua_environment(struct mg_context *ctx, if ((conn != NULL) && (conn->dom_ctx != NULL)) { reg_string(L, "document_root", conn->dom_ctx->config[DOCUMENT_ROOT]); + if (conn->dom_ctx->config[FALLBACK_DOCUMENT_ROOT]) { + reg_string(L, + "fallback_document_root", + conn->dom_ctx->config[FALLBACK_DOCUMENT_ROOT]); + } reg_string(L, "auth_domain", conn->dom_ctx->config[AUTHENTICATION_DOMAIN]); @@ -2951,6 +2951,11 @@ prepare_lua_environment(struct mg_context *ctx, reg_string(L, "websocket_root", conn->dom_ctx->config[WEBSOCKET_ROOT]); + if (conn->dom_ctx->config[FALLBACK_WEBSOCKET_ROOT]) { + reg_string(L, + "fallback_websocket_root", + conn->dom_ctx->config[FALLBACK_WEBSOCKET_ROOT]); + } } else { reg_string(L, "websocket_root", @@ -3066,9 +3071,14 @@ mg_exec_lua_script(struct mg_connection *conn, } if (luaL_loadfile(L, path) != 0) { + mg_send_http_error(conn, 500, "Lua error:\r\n"); lua_error_handler(L); } else { - lua_pcall(L, 0, 0, -2); + int call_status = lua_pcall(L, 0, 0, 0); + if (call_status != 0) { + mg_send_http_error(conn, 500, "Lua error:\r\n"); + lua_error_handler(L); + } } DEBUG_TRACE("Close Lua environment %p", L); lua_close(L); @@ -3216,9 +3226,10 @@ handle_lsp_request(struct mg_connection *conn, * "pkey); mbedtls_ctr_drbg_init(&ctx->ctr); @@ -153,19 +141,6 @@ mbed_sslctx_init(SSL_CTX *ctx, const char *crt) DEBUG_TRACE("TLS cannot set certificate and private key (%i)", rc); return -1; } - -// /* Set ciphersuites */ -// static const int tls_cipher_suites[] = { -// MBEDTLS_CIPHER_CHACHA20_POLY1305, -// MBEDTLS_TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, -// MBEDTLS_TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, -// 0 -// }; -// mbedtls_ssl_conf_ciphersuites(conf, tls_cipher_suites); -// -// /* Set protocol version */ -// mbedtls_ssl_conf_min_version(conf, MBEDTLS_SSL_MAJOR_VERSION_3, MBEDTLS_SSL_MINOR_VERSION_3); - return 0; } @@ -213,7 +188,13 @@ mbed_ssl_accept(mbedtls_ssl_context **ssl, return -1; } - DEBUG_TRACE("TLS connection %p accepted, state: %d", ssl, (*ssl)->MBEDTLS_PRIVATE(state)); +#if MBEDTLS_VERSION_NUMBER >= 0x03000000 + DEBUG_TRACE("TLS connection %p accepted, state: %d", + ssl, + (*ssl)->MBEDTLS_PRIVATE(state)); +#else + DEBUG_TRACE("TLS connection %p accepted, state: %d", ssl, (*ssl)->state); +#endif return 0; } @@ -239,7 +220,13 @@ mbed_ssl_handshake(mbedtls_ssl_context *ssl) } } - DEBUG_TRACE("TLS handshake rc: %d, state: %d", rc, ssl->MBEDTLS_PRIVATE(state)); +#if MBEDTLS_VERSION_NUMBER >= 0x03000000 + DEBUG_TRACE("TLS handshake rc: %d, state: %d", + rc, + ssl->MBEDTLS_PRIVATE(state)); +#else + DEBUG_TRACE("TLS handshake rc: %d, state: %d", rc, ssl->state); +#endif return rc; } @@ -249,13 +236,6 @@ mbed_ssl_read(mbedtls_ssl_context *ssl, unsigned char *buf, int len) { int rc = mbedtls_ssl_read(ssl, buf, len); /* DEBUG_TRACE("mbedtls_ssl_read: %d", rc); */ - -#if defined(MBEDTLS_SSL_PROTO_TLS1_3) && defined(MBEDTLS_CLIENT_SSL_SESSION_TICKETS) - if (ret == MBEDTLS_ERR_SSL_RECEIVED_NEW_SESSION_TICKET) { - DEBUG_TRACE("got session ticket in TLS 1.3 connection, retrying read"); - rc = mbedtls_ssl_read(ssl, buf, len); - } -#endif return rc; } diff --git a/src/webserver/civetweb/timer.inl b/src/webserver/civetweb/timer.inl index 9b8d5539..39d68dfc 100644 --- a/src/webserver/civetweb/timer.inl +++ b/src/webserver/civetweb/timer.inl @@ -39,13 +39,16 @@ TIMER_API double timer_getcurrenttime(struct mg_context *ctx) { #if defined(_WIN32) + uint64_t now_tick64 = 0; +#if defined(_WIN64) + now_tick64 = GetTickCount64(); +#else /* GetTickCount returns milliseconds since system start as * unsigned 32 bit value. It will wrap around every 49.7 days. * We need to use a 64 bit counter (will wrap in 500 mio. years), * by adding the 32 bit difference since the last call to a * 64 bit counter. This algorithm will only work, if this * function is called at least once every 7 weeks. */ - uint64_t now_tick64 = 0; DWORD now_tick = GetTickCount(); if (ctx->timers) { @@ -55,6 +58,7 @@ timer_getcurrenttime(struct mg_context *ctx) ctx->timers->last_tick = now_tick; pthread_mutex_unlock(&ctx->timers->mutex); } +#endif return (double)now_tick64 * 1.0E-3; #else struct timespec now_ts; From ce62ddc9368719a74ea0a2e85d9d607ccf437ba8 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 15 Aug 2024 20:33:42 +0200 Subject: [PATCH 259/339] Apply and update Pi-hole patches Signed-off-by: DL6ER --- patch/civetweb.sh | 14 ++++- ...TL-URI-rewriting-changes-to-CivetWeb.patch | 16 ++--- ...EN-option-to-civetweb-s-LUA-routines.patch | 35 ----------- .../0001-Add-mbedTLS-debug-logging-hook.patch | 10 ++-- ...ow-extended-ASCII-characters-in-URIs.patch | 35 ----------- ...es-to-webserver.log-when-debug.webse.patch | 22 ------- ...ster-CSRF-token-in-conn-request_info.patch | 12 ++-- src/webserver/civetweb/civetweb.c | 60 ++++++++++++++++++- src/webserver/civetweb/civetweb.h | 20 +++++++ src/webserver/civetweb/handle_form.inl | 32 +++++----- src/webserver/civetweb/mod_lua.inl | 11 ++-- src/webserver/civetweb/mod_mbedtls.inl | 16 +++++ src/webserver/webserver.c | 3 +- 13 files changed, 150 insertions(+), 136 deletions(-) delete mode 100644 patch/civetweb/0001-Add-NO_DLOPEN-option-to-civetweb-s-LUA-routines.patch delete mode 100644 patch/civetweb/0001-Allow-extended-ASCII-characters-in-URIs.patch diff --git a/patch/civetweb.sh b/patch/civetweb.sh index f33dc8de..3fcc732a 100644 --- a/patch/civetweb.sh +++ b/patch/civetweb.sh @@ -1,13 +1,23 @@ #!/bin/sh set -e +echo "Applying patches for civetweb" +echo "Applying patch 0001-add-pihole-mods.patch" patch -p1 < patch/civetweb/0001-add-pihole-mods.patch -patch -p1 < patch/civetweb/0001-Add-NO_DLOPEN-option-to-civetweb-s-LUA-routines.patch + +echo "Applying patch 0001-Always-Kepler-syntax-for-Lua-server-pages.patch" patch -p1 < patch/civetweb/0001-Always-Kepler-syntax-for-Lua-server-pages.patch + +echo "Applying patch 0001-Add-FTL-URI-rewriting-changes-to-CivetWeb.patch" patch -p1 < patch/civetweb/0001-Add-FTL-URI-rewriting-changes-to-CivetWeb.patch + +echo "Applying patch 0001-Add-mbedTLS-debug-logging-hook.patch" patch -p1 < patch/civetweb/0001-Add-mbedTLS-debug-logging-hook.patch + +echo "Applying patch 0001-Add-Register-CSRF-token-in-conn-request_info.patch" patch -p1 < patch/civetweb/0001-Register-CSRF-token-in-conn-request_info.patch + +echo "Applying patch 0001-Log-debug-messages-to-webserver.log-when-debug.webse.patch" patch -p1 < patch/civetweb/0001-Log-debug-messages-to-webserver.log-when-debug.webse.patch -patch -p1 < patch/civetweb/0001-Allow-extended-ASCII-characters-in-URIs.patch echo "ALL PATCHES APPLIED OKAY" diff --git a/patch/civetweb/0001-Add-FTL-URI-rewriting-changes-to-CivetWeb.patch b/patch/civetweb/0001-Add-FTL-URI-rewriting-changes-to-CivetWeb.patch index f5439791..f0f78ea0 100644 --- a/patch/civetweb/0001-Add-FTL-URI-rewriting-changes-to-CivetWeb.patch +++ b/patch/civetweb/0001-Add-FTL-URI-rewriting-changes-to-CivetWeb.patch @@ -14,14 +14,14 @@ index 0d293f1f..44f6cf3d 100644 --- a/src/webserver/civetweb/civetweb.c +++ b/src/webserver/civetweb/civetweb.c @@ -7754,6 +7754,8 @@ interpret_uri(struct mg_connection *conn, /* in/out: request (must be valid) */ - mg_snprintf( - conn, &truncated, filename, filename_buf_len - 1, "%s%s", root, uri); + roots[i], + uri); -+ FTL_rewrite_pattern(filename, filename_buf_len - 1, root, uri); ++ FTL_rewrite_pattern(filename, filename_buf_len - 1); + - if (truncated) { - goto interpret_cleanup; - } + if (truncated) { + goto interpret_cleanup; + } diff --git a/src/webserver/civetweb/civetweb.h b/src/webserver/civetweb/civetweb.h index e71dfedc..2ad76693 100644 --- a/src/webserver/civetweb/civetweb.h @@ -30,8 +30,8 @@ index e71dfedc..2ad76693 100644 int status, const char* mime_type, long long content_length); -+void FTL_rewrite_pattern(char *filename, size_t filename_buf_len, -+ const char *root, const char *uri); ++void FTL_rewrite_pattern(char *filename, unsigned long filename_buf_len); ++ + // Buffer used for additional "Set-Cookie" headers #define PIHOLE_HEADERS_MAXLEN 1024 diff --git a/patch/civetweb/0001-Add-NO_DLOPEN-option-to-civetweb-s-LUA-routines.patch b/patch/civetweb/0001-Add-NO_DLOPEN-option-to-civetweb-s-LUA-routines.patch deleted file mode 100644 index ee48ec54..00000000 --- a/patch/civetweb/0001-Add-NO_DLOPEN-option-to-civetweb-s-LUA-routines.patch +++ /dev/null @@ -1,35 +0,0 @@ -From 1b81285fed48df6939d4b2569bba9e572f4c1137 Mon Sep 17 00:00:00 2001 -From: DL6ER -Date: Fri, 13 Jan 2023 21:37:31 +0100 -Subject: [PATCH] Add NO_DLOPEN option to civetweb's LUA routines - -Signed-off-by: DL6ER ---- - src/webserver/civetweb/mod_lua.inl | 4 ++-- - 1 file changed, 2 insertions(+), 2 deletions(-) - -diff --git a/src/webserver/civetweb/mod_lua.inl b/src/webserver/civetweb/mod_lua.inl -index 5cc94318..59c4f2b3 100644 ---- a/src/webserver/civetweb/mod_lua.inl -+++ b/src/webserver/civetweb/mod_lua.inl -@@ -3634,7 +3634,7 @@ lua_init_optional_libraries(void) - lua_shared_init(); - - /* UUID library */ --#if !defined(_WIN32) -+#if !defined(_WIN32) && !defined(NO_DLOPEN) - lib_handle_uuid = dlopen("libuuid.so", RTLD_LAZY); - pf_uuid_generate.p = - (lib_handle_uuid ? dlsym(lib_handle_uuid, "uuid_generate") : 0); -@@ -3648,7 +3648,7 @@ static void - lua_exit_optional_libraries(void) - { - /* UUID library */ --#if !defined(_WIN32) -+#if !defined(_WIN32) && !defined(NO_DLOPEN) - if (lib_handle_uuid) { - dlclose(lib_handle_uuid); - } --- -2.34.1 - diff --git a/patch/civetweb/0001-Add-mbedTLS-debug-logging-hook.patch b/patch/civetweb/0001-Add-mbedTLS-debug-logging-hook.patch index ba7e8b2a..71f5c0ea 100644 --- a/patch/civetweb/0001-Add-mbedTLS-debug-logging-hook.patch +++ b/patch/civetweb/0001-Add-mbedTLS-debug-logging-hook.patch @@ -14,8 +14,8 @@ index 2ad76693..52724199 100644 --- a/src/webserver/civetweb/civetweb.h +++ b/src/webserver/civetweb/civetweb.h @@ -938,6 +938,10 @@ int my_send_http_error_headers(struct mg_connection *conn, - void FTL_rewrite_pattern(char *filename, size_t filename_buf_len, - const char *root, const char *uri); + void FTL_rewrite_pattern(char *filename, size_t filename_buf_len); + +#define MG_CONFIG_MBEDTLS_DEBUG 3 +void FTL_mbed_debug(void *user_param, int level, const char *file, @@ -36,9 +36,9 @@ index e72685f4..00b9280a 100644 + mbedtls_ssl_conf_dbg(conf, FTL_mbed_debug, NULL); + /****************************************************/ + - #ifdef MBEDTLS_SSL_PROTO_TLS1_3 - psa_status_t status = psa_crypto_init(); - if (status != PSA_SUCCESS) { + /* Initialize TLS key and cert */ + mbedtls_pk_init(&ctx->pkey); + mbedtls_ctr_drbg_init(&ctx->ctr); -- 2.34.1 diff --git a/patch/civetweb/0001-Allow-extended-ASCII-characters-in-URIs.patch b/patch/civetweb/0001-Allow-extended-ASCII-characters-in-URIs.patch deleted file mode 100644 index d54ab29e..00000000 --- a/patch/civetweb/0001-Allow-extended-ASCII-characters-in-URIs.patch +++ /dev/null @@ -1,35 +0,0 @@ -From ebb27741b10ed2eac51ac356708800ae96cdd17a Mon Sep 17 00:00:00 2001 -From: DL6ER -Date: Tue, 31 Oct 2023 08:35:31 +0100 -Subject: [PATCH] Allow extended ASCII characters in URIs - -Signed-off-by: DL6ER ---- - src/webserver/civetweb/civetweb.c | 4 ++-- - 1 file changed, 2 insertions(+), 2 deletions(-) - -diff --git a/src/webserver/civetweb/civetweb.c b/src/webserver/civetweb/civetweb.c -index 9b0c6308..5320c4d4 100644 ---- a/src/webserver/civetweb/civetweb.c -+++ b/src/webserver/civetweb/civetweb.c -@@ -10734,7 +10734,7 @@ skip_to_end_of_word_and_terminate(char **ppw, int eol) - { - /* Forward until a space is found - use isgraph here */ - /* See http://www.cplusplus.com/reference/cctype/ */ -- while (isgraph((unsigned char)**ppw)) { -+ while ((unsigned char)**ppw > 127 || isgraph((unsigned char)**ppw)) { - (*ppw)++; - } - -@@ -18473,7 +18473,7 @@ get_uri_type(const char *uri) - * and % encoded symbols. - */ - for (i = 0; uri[i] != 0; i++) { -- if (uri[i] < 33) { -+ if ((unsigned char)uri[i] < 33) { - /* control characters and spaces are invalid */ - return 0; - } --- -2.34.1 - diff --git a/patch/civetweb/0001-Log-debug-messages-to-webserver.log-when-debug.webse.patch b/patch/civetweb/0001-Log-debug-messages-to-webserver.log-when-debug.webse.patch index c396cb0f..9dc655e8 100644 --- a/patch/civetweb/0001-Log-debug-messages-to-webserver.log-when-debug.webse.patch +++ b/patch/civetweb/0001-Log-debug-messages-to-webserver.log-when-debug.webse.patch @@ -27,28 +27,6 @@ index 3df8eab9..9b0c6308 100644 #endif /* DEBUG */ #endif /* DEBUG_TRACE */ -diff --git a/src/webserver/civetweb/mod_mbedtls.inl b/src/webserver/civetweb/mod_mbedtls.inl -index 00b9280a..6a450ba3 100644 ---- a/src/webserver/civetweb/mod_mbedtls.inl -+++ b/src/webserver/civetweb/mod_mbedtls.inl -@@ -213,7 +213,7 @@ mbed_ssl_accept(mbedtls_ssl_context **ssl, - return -1; - } - -- DEBUG_TRACE("TLS connection %p accepted, state: %d", ssl, (*ssl)->state); -+ DEBUG_TRACE("TLS connection %p accepted, state: %d", ssl, (*ssl)->MBEDTLS_PRIVATE(state)); - return 0; - } - -@@ -239,7 +239,7 @@ mbed_ssl_handshake(mbedtls_ssl_context *ssl) - } - } - -- DEBUG_TRACE("TLS handshake rc: %d, state: %d", rc, ssl->state); -+ DEBUG_TRACE("TLS handshake rc: %d, state: %d", rc, ssl->MBEDTLS_PRIVATE(state)); - return rc; - } - -- 2.34.1 diff --git a/patch/civetweb/0001-Register-CSRF-token-in-conn-request_info.patch b/patch/civetweb/0001-Register-CSRF-token-in-conn-request_info.patch index 575c5649..50506f0f 100644 --- a/patch/civetweb/0001-Register-CSRF-token-in-conn-request_info.patch +++ b/patch/civetweb/0001-Register-CSRF-token-in-conn-request_info.patch @@ -6,9 +6,9 @@ Subject: [PATCH] Register CSRF token and is_authenticated boolean in conn->reque Signed-off-by: DL6ER --- src/webserver/civetweb/civetweb.c | 3 +++ - src/webserver/civetweb/civetweb.h | 2 ++ - src/webserver/civetweb/mod_lua.inl | 3 +++ - 3 files changed, 8 insertions(+) + src/webserver/civetweb/civetweb.h | 3 +++ + src/webserver/civetweb/mod_lua.inl | 4 ++++ + 3 files changed, 10 insertions(+) diff --git a/src/webserver/civetweb/civetweb.c b/src/webserver/civetweb/civetweb.c index 233b342a..f44b17ba 100644 @@ -20,7 +20,6 @@ index 233b342a..f44b17ba 100644 + /* Pi-hole addition */ + memset(conn->request_info.csrf_token, 0, sizeof(conn->request_info.csrf_token)); -+ reg_boolean(L, "is_authenticated", conn->request_info.is_authenticated != 0); + #if defined(USE_SERVER_STATS) conn->processing_time = 0; @@ -29,7 +28,7 @@ diff --git a/src/webserver/civetweb/civetweb.h b/src/webserver/civetweb/civetweb index 5b3d596b..291ef683 100644 --- a/src/webserver/civetweb/civetweb.h +++ b/src/webserver/civetweb/civetweb.h -@@ -183,6 +183,8 @@ struct mg_request_info { +@@ -183,6 +183,9 @@ struct mg_request_info { const char *acceptedWebSocketSubprotocol; /* websocket subprotocol, * accepted during handshake */ @@ -43,13 +42,14 @@ diff --git a/src/webserver/civetweb/mod_lua.inl b/src/webserver/civetweb/mod_lua index e9a13835..92066b3f 100644 --- a/src/webserver/civetweb/mod_lua.inl +++ b/src/webserver/civetweb/mod_lua.inl -@@ -2603,6 +2603,9 @@ prepare_lua_request_info_inner(const struct mg_connection *conn, lua_State *L) +@@ -2603,6 +2603,10 @@ prepare_lua_request_info_inner(const struct mg_connection *conn, lua_State *L) reg_string(L, "finger", conn->request_info.client_cert->finger); lua_rawset(L, -3); } + + /* Pi-hole addition */ + reg_string(L, "csrf_token", conn->request_info.csrf_token); ++ reg_boolean(L, "is_authenticated", conn->request_info.is_authenticated != 0); } diff --git a/src/webserver/civetweb/civetweb.c b/src/webserver/civetweb/civetweb.c index dced1362..6cef4dc4 100644 --- a/src/webserver/civetweb/civetweb.c +++ b/src/webserver/civetweb/civetweb.c @@ -239,9 +239,10 @@ static void DEBUG_TRACE_FUNC(const char *func, #endif #else +#include "log.h" #define DEBUG_TRACE(fmt, ...) \ - do { \ - } while (0) + if(debug_flags[DEBUG_WEBSERVER]) {\ + log_web("DEBUG: " fmt " (%s:%d)", ##__VA_ARGS__, short_path(__FILE__), __LINE__); } #endif /* DEBUG */ #endif /* DEBUG_TRACE */ @@ -4186,6 +4187,14 @@ send_additional_header(struct mg_connection *conn) if (header && header[0]) { mg_response_header_add_lines(conn, header); } + + /*************** Pi-hole modification ****************/ + if (pi_hole_extra_headers[0] != '\0') { + mg_response_header_add_lines(conn, pi_hole_extra_headers); + // Invalidate extra headers after having sent them to avoid repetitions + pi_hole_extra_headers[0] = '\0'; + } + /*****************************************************/ } @@ -4641,6 +4650,48 @@ mg_send_http_error_impl(struct mg_connection *conn, } +/************************************** Pi-hole method **************************************/ +CIVETWEB_API int +my_send_http_error_headers(struct mg_connection *conn, + int status, const char* mime_type, + long long content_length) +{ + if ((mime_type == NULL) || (*mime_type == 0)) { + /* No content type defined: default to text/html */ + mime_type = "text/html"; + } + + mg_response_header_start(conn, status); + send_no_cache_header(conn); + send_additional_header(conn); + mg_response_header_add(conn, "Content-Type", mime_type, -1); + if (content_length < 0) { + /* Size not known. Use chunked encoding (HTTP/1.x) */ + if (conn->protocol_type == PROTOCOL_TYPE_HTTP1) { + /* Only HTTP/1.x defines "chunked" encoding, HTTP/2 does not*/ + mg_response_header_add(conn, "Transfer-Encoding", "chunked", -1); + } + } else { + char len[32]; + int trunc = 0; + mg_snprintf(conn, + &trunc, + len, + sizeof(len), + "%" UINT64_FMT, + (uint64_t)content_length); + if (!trunc) { + /* Since 32 bytes is enough to hold any 64 bit decimal number, + * !trunc is always true */ + mg_response_header_add(conn, "Content-Length", len, -1); + } + } + mg_response_header_send(conn); + + return 0; +} +/********************************************************************************************/ + CIVETWEB_API int mg_send_http_error(struct mg_connection *conn, int status, const char *fmt, ...) { @@ -7887,6 +7938,8 @@ interpret_uri(struct mg_connection *conn, /* in/out: request (must be valid) */ roots[i], uri); + FTL_rewrite_pattern(filename, filename_buf_len - 1); + if (truncated) { goto interpret_cleanup; } @@ -17995,6 +18048,9 @@ reset_per_request_attributes(struct mg_connection *conn) } conn->request_info.local_uri = NULL; + /* Pi-hole addition */ + memset(conn->request_info.csrf_token, 0, sizeof(conn->request_info.csrf_token)); + #if defined(USE_SERVER_STATS) conn->processing_time = 0; #endif diff --git a/src/webserver/civetweb/civetweb.h b/src/webserver/civetweb/civetweb.h index 5ae6a0c7..e0ef3443 100644 --- a/src/webserver/civetweb/civetweb.h +++ b/src/webserver/civetweb/civetweb.h @@ -183,6 +183,9 @@ struct mg_request_info { const char *acceptedWebSocketSubprotocol; /* websocket subprotocol, * accepted during handshake */ + /* Pi-hole modification */ + char csrf_token[32]; + int is_authenticated; }; @@ -928,6 +931,23 @@ CIVETWEB_API int mg_send_http_error(struct mg_connection *conn, PRINTF_FORMAT_STRING(const char *fmt), ...) PRINTF_ARGS(3, 4); +/************************************** Pi-hole method **************************************/ +int my_send_http_error_headers(struct mg_connection *conn, + int status, const char* mime_type, + long long content_length); + +void FTL_rewrite_pattern(char *filename, unsigned long filename_buf_len); + + +#define MG_CONFIG_MBEDTLS_DEBUG 3 +void FTL_mbed_debug(void *user_param, int level, const char *file, + int line, const char *message); + +// Buffer used for additional "Set-Cookie" headers +#define PIHOLE_HEADERS_MAXLEN 1024 +extern char pi_hole_extra_headers[PIHOLE_HEADERS_MAXLEN]; +/********************************************************************************************/ + /* Send "HTTP 200 OK" response header. * After calling this function, use mg_write or mg_send_chunk to send the diff --git a/src/webserver/civetweb/handle_form.inl b/src/webserver/civetweb/handle_form.inl index 4de8d4f5..a7b7fc10 100644 --- a/src/webserver/civetweb/handle_form.inl +++ b/src/webserver/civetweb/handle_form.inl @@ -188,7 +188,7 @@ mg_handle_form_request(struct mg_connection *conn, char path[512]; char buf[MG_BUF_LEN]; /* Must not be smaller than ~900 */ int field_storage; - int buf_fill = 0; + size_t buf_fill = 0; int r; int field_count = 0; struct mg_file fstore = STRUCT_FILE_INITIALIZER; @@ -397,10 +397,10 @@ mg_handle_form_request(struct mg_connection *conn, int end_of_key_value_pair_found = 0; int get_block; - if ((size_t)buf_fill < (sizeof(buf) - 1)) { + if (buf_fill < (sizeof(buf) - 1)) { - size_t to_read = sizeof(buf) - 1 - (size_t)buf_fill; - r = mg_read(conn, buf + (size_t)buf_fill, to_read); + size_t to_read = sizeof(buf) - 1 - buf_fill; + r = mg_read(conn, buf + buf_fill, to_read); if ((r < 0) || ((r == 0) && all_data_read)) { /* read error */ return -1; @@ -529,11 +529,11 @@ mg_handle_form_request(struct mg_connection *conn, buf + (size_t)used, sizeof(buf) - (size_t)used); next = buf; - buf_fill -= (int)used; - if ((size_t)buf_fill < (sizeof(buf) - 1)) { + buf_fill -= used; + if (buf_fill < (sizeof(buf) - 1)) { - size_t to_read = sizeof(buf) - 1 - (size_t)buf_fill; - r = mg_read(conn, buf + (size_t)buf_fill, to_read); + size_t to_read = sizeof(buf) - 1 - buf_fill; + r = mg_read(conn, buf + buf_fill, to_read); if ((r < 0) || ((r == 0) && all_data_read)) { #if !defined(NO_FILESYSTEMS) /* read error */ @@ -592,7 +592,7 @@ mg_handle_form_request(struct mg_connection *conn, /* Proceed to next entry */ used = next - buf; memmove(buf, buf + (size_t)used, sizeof(buf) - (size_t)used); - buf_fill -= (int)used; + buf_fill -= used; } return field_count; @@ -682,12 +682,12 @@ mg_handle_form_request(struct mg_connection *conn, for (part_no = 0;; part_no++) { size_t towrite, fnlen, n; int get_block; - size_t to_read = sizeof(buf) - 1 - (size_t)buf_fill; + size_t to_read = sizeof(buf) - 1 - buf_fill; /* Unused without filesystems */ (void)n; - r = mg_read(conn, buf + (size_t)buf_fill, to_read); + r = mg_read(conn, buf + buf_fill, to_read); if ((r < 0) || ((r == 0) && all_data_read)) { /* read error */ mg_free(boundary); @@ -1001,12 +1001,12 @@ mg_handle_form_request(struct mg_connection *conn, #endif /* NO_FILESYSTEMS */ memmove(buf, hend + towrite, bl + 4); - buf_fill = (int)(bl + 4); + buf_fill = bl + 4; hend = buf; /* Read new data */ - to_read = sizeof(buf) - 1 - (size_t)buf_fill; - r = mg_read(conn, buf + (size_t)buf_fill, to_read); + to_read = sizeof(buf) - 1 - buf_fill; + r = mg_read(conn, buf + buf_fill, to_read); if ((r < 0) || ((r == 0) && all_data_read)) { #if !defined(NO_FILESYSTEMS) /* read error */ @@ -1025,7 +1025,7 @@ mg_handle_form_request(struct mg_connection *conn, /* buf_fill is at least 8 here */ /* Find boundary */ - next = search_boundary(buf, (size_t)buf_fill, boundary, bl); + next = search_boundary(buf, buf_fill, boundary, bl); if (!next && (r == 0)) { /* incomplete request */ @@ -1100,7 +1100,7 @@ mg_handle_form_request(struct mg_connection *conn, if (next) { used = next - buf + 2; memmove(buf, buf + (size_t)used, sizeof(buf) - (size_t)used); - buf_fill -= (int)used; + buf_fill -= used; } else { buf_fill = 0; } diff --git a/src/webserver/civetweb/mod_lua.inl b/src/webserver/civetweb/mod_lua.inl index 4ad3c38a..b3ad4b14 100644 --- a/src/webserver/civetweb/mod_lua.inl +++ b/src/webserver/civetweb/mod_lua.inl @@ -2606,6 +2606,10 @@ prepare_lua_request_info_inner(const struct mg_connection *conn, lua_State *L) reg_string(L, "finger", conn->request_info.client_cert->finger); lua_rawset(L, -3); } + + /* Pi-hole addition */ + reg_string(L, "csrf_token", conn->request_info.csrf_token); + reg_boolean(L, "is_authenticated", conn->request_info.is_authenticated != 0); } @@ -3226,10 +3230,9 @@ handle_lsp_request(struct mg_connection *conn, * "pkey); mbedtls_ctr_drbg_init(&ctx->ctr); mbedtls_x509_crt_init(&ctx->cert); +#ifdef MBEDTLS_PSA_CRYPTO_C + /* Initialize PSA crypto (mandatory with TLS 1.3) + * This must be done before calling any other PSA Crypto + * functions or they will fail with PSA_ERROR_BAD_STATE + */ + const psa_status_t status = psa_crypto_init(); + if (status != PSA_SUCCESS) { + DEBUG_TRACE("Failed to initialize PSA crypto, returned %d\n", (int) status); + return -1; + } +#endif + rc = mbedtls_ctr_drbg_seed(&ctx->ctr, mbedtls_entropy_func, &ctx->entropy, diff --git a/src/webserver/webserver.c b/src/webserver/webserver.c index e16b6329..6e3a3598 100644 --- a/src/webserver/webserver.c +++ b/src/webserver/webserver.c @@ -565,8 +565,9 @@ static char *append_to_path(char *path, const char *append) return new_path; } -void FTL_rewrite_pattern(char *filename, size_t filename_buf_len) +void FTL_rewrite_pattern(char *filename, unsigned long filename_buf_len) { + log_debug(DEBUG_API, "Rewriting filename: %s", filename); const bool trailing_slash = filename[strlen(filename) - 1] == '/'; char *filename_lp = NULL; From 1c8579e668ffa09d6df143d28e294e8cb3fd6808 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 18 Aug 2024 19:06:15 +0200 Subject: [PATCH 260/339] Fix headers not being correctly handled for Kepler-style Lua server pages Signed-off-by: DL6ER --- src/webserver/civetweb/mod_lua.inl | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/src/webserver/civetweb/mod_lua.inl b/src/webserver/civetweb/mod_lua.inl index b3ad4b14..3ad3ba75 100644 --- a/src/webserver/civetweb/mod_lua.inl +++ b/src/webserver/civetweb/mod_lua.inl @@ -641,17 +641,22 @@ run_lsp_kepler(struct mg_connection *conn, /* Only send a HTML header, if this is the top level page. * If this page is included by some mg.include calls, do not add a * header. */ - if(conn->status_code < 0) - mg_printf(conn, "HTTP/1.1 200 OK\r\n"); - else - mg_printf(conn, "HTTP/1.1 %d %s\r\n", conn->status_code, mg_get_response_code_text(conn, conn->status_code)); + + /* Initialize a new HTTP response, either with some-predefined + * status code (e.g. 404 if this is called from an error + * handler) or with 200 OK */ + mg_response_header_start(conn, conn->status_code > 0 ? conn->status_code : 200); + + /* Add additional headers */ send_no_cache_header(conn); send_additional_header(conn); - mg_printf(conn, - "Date: %s\r\n" - "Connection: close\r\n" - "Content-Type: text/html; charset=utf-8\r\n\r\n", - date); + send_cors_header(conn); + + /* Add content type */ + mg_response_header_add(conn, "Content-Type", "text/html; charset=utf-8", -1); + + /* Send the HTTP response (status and all headers) */ + mg_response_header_send(conn); } data.begin = p; From 588ae42c11f182ae9c17e7a8f5d974915aa4ef8c Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 18 Aug 2024 20:18:49 +0200 Subject: [PATCH 261/339] Improve behavior on systems where mandatory config file locations aren't present (initial install interrupted/not completed) Signed-off-by: DL6ER --- src/config/config.c | 11 +++++++++-- src/config/toml_helper.c | 8 ++++---- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/src/config/config.c b/src/config/config.c index 42d1f7c7..e4800b03 100644 --- a/src/config/config.c +++ b/src/config/config.c @@ -1600,6 +1600,13 @@ bool readFTLconf(struct config *conf, const bool rewrite) log_info("No config file nor backup available, using defaults"); + // If we reach this point, we could not read the TOML config file When + // this functions is invoked to run without rewriting, we are likely + // running interactively and do not want to migrate settings (yet): + // using defaults is fine in this case + if(!rewrite) + return false; + // If no previous config file could be read, we are likely either running // for the first time or we are upgrading from a version prior to v6.0 // In this case, we try to read the legacy config files @@ -1661,8 +1668,8 @@ bool readFTLconf(struct config *conf, const bool rewrite) conf->webserver.port.v.s = ports; conf->webserver.port.t = CONF_STRING_ALLOCATED; - log_info("Initialised webserver ports at %d (HTTP) and %d (HTTPS), IPv6 support is %s", - http_port, https_port, have_ipv6 ? "enabled" : "disabled"); + log_info("Config initialized with webserver ports %d (HTTP) and %d (HTTPS), IPv6 support is %s", + http_port, https_port, have_ipv6 ? "enabled" : "disabled"); } // Initialize the TOML config file diff --git a/src/config/toml_helper.c b/src/config/toml_helper.c index 22b85dd0..dabc1f59 100644 --- a/src/config/toml_helper.c +++ b/src/config/toml_helper.c @@ -61,8 +61,8 @@ FILE * __attribute((malloc)) __attribute((nonnull(1))) openFTLtoml(const char *m // Return early if opening failed if(!fp) { - log_info("Config %sfile %s not available: %s", - version > 0 ? "backup " : "", filename, strerror(errno)); + log_info("Config %sfile %s not available (%s): %s", + version > 0 ? "backup " : "", filename, mode, strerror(errno)); return NULL; } @@ -70,8 +70,8 @@ FILE * __attribute((malloc)) __attribute((nonnull(1))) openFTLtoml(const char *m if(flock(fileno(fp), LOCK_EX) != 0) { const int _e = errno; - log_err("Cannot open config file %s in exclusive mode: %s", - filename, strerror(errno)); + log_err("Cannot open config file %s in exclusive mode (%s): %s", + filename, mode, strerror(errno)); fclose(fp); errno = _e; return NULL; From 72f9874366a3fb3567827c2b529c69cd7bd6ff08 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 18 Aug 2024 20:31:02 +0200 Subject: [PATCH 262/339] Clarify that our version of CivetWeb has been modified for our needs Signed-off-by: DL6ER --- src/args.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/args.c b/src/args.c index 2a6c2b8b..a162aeca 100644 --- a/src/args.c +++ b/src/args.c @@ -807,10 +807,10 @@ void parse_args(int argc, char* argv[]) printf("****************************** %s%sCivetWeb%s *****************************\n", yellow, bold, normal); #ifdef HAVE_MBEDTLS - printf("Version: %s%s%s%s with %smbed TLS %s%s"MBEDTLS_VERSION_STRING"%s\n", + printf("Version: %s%s%s%s (modified by Pi-hole) with %smbed TLS %s%s"MBEDTLS_VERSION_STRING"%s\n", green, bold, mg_version(), normal, yellow, green, bold, normal); #else - printf("Version: %s%s%s%s%s without %smbed TLS%s\n", + printf("Version: %s%s%s%s%s (modified by Pi-hole) without %smbed TLS%s\n", green, bold, mg_version(), normal, red, yellow, normal); #endif printf("Features: "); From 070544bde13ac7546efd08ed8e28badb3a35df2a Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 12 Aug 2024 19:42:11 +0200 Subject: [PATCH 263/339] Implement DNS caching for queries blocked upstream (NXDOMAIN + no RA, 0.0.0.0/::, and known Umbrella IP blocking pages) accompanied with suitable CI testing. This change improves handling of externally blocked queries in two ways: 1. Once we know the domain is externally blocked, we don't forward it again for the same client (it is currently forwarded each time) 2. We can recognize cache content with the specified addresses/flags as being upstream blocked and don't return them as "OK (cached)" through the API/in the database Signed-off-by: DL6ER --- src/api/padd.c | 2 +- src/api/stats.c | 2 +- src/datastructure.c | 10 +- src/datastructure.h | 10 +- src/dnsmasq/forward.c | 6 +- src/dnsmasq/rfc1035.c | 20 ++++ src/dnsmasq_interface.c | 246 ++++++++++++++++++++++++++-------------- src/dnsmasq_interface.h | 7 +- src/enums.h | 3 + src/gc.c | 4 +- src/shmem.c | 2 +- test/pdns/setup.sh | 11 ++ test/test_suite.bats | 155 +++++++++++++++++++++++++ 13 files changed, 375 insertions(+), 103 deletions(-) diff --git a/src/api/padd.c b/src/api/padd.c index a3db08b9..21a44cfa 100644 --- a/src/api/padd.c +++ b/src/api/padd.c @@ -52,7 +52,7 @@ int api_padd(struct ftl_conn *api) // Find most recently blocked query for(int queryID = counters->queries - 1; queryID > 0 ; queryID--) { - const queriesData* query = getQuery(queryID, true); + const queriesData *query = getQuery(queryID, true); if(query == NULL) continue; diff --git a/src/api/stats.c b/src/api/stats.c index 05075a3e..60525d9c 100644 --- a/src/api/stats.c +++ b/src/api/stats.c @@ -748,7 +748,7 @@ int api_stats_recentblocked(struct ftl_conn *api) cJSON *blocked = JSON_NEW_ARRAY(); for(int queryID = counters->queries - 1; queryID > 0 ; queryID--) { - const queriesData* query = getQuery(queryID, true); + const queriesData *query = getQuery(queryID, true); if(query == NULL) continue; diff --git a/src/datastructure.c b/src/datastructure.c index 7ac9a1e2..4adb5687 100644 --- a/src/datastructure.c +++ b/src/datastructure.c @@ -69,7 +69,7 @@ int findQueryID(const int id) // Check UUIDs of queries for(int i = start; i >= until; i--) { - const queriesData* query = getQuery(i, true); + const queriesData *query = getQuery(i, true); // Check if the returned pointer is valid before trying to access it if(query == NULL) @@ -462,7 +462,7 @@ bool isValidIPv6(const char *addr) // Privacy-level sensitive subroutine that returns the domain name // only when appropriate for the requested query -const char *getDomainString(const queriesData* query) +const char *getDomainString(const queriesData *query) { // Check if the returned pointer is valid before trying to access it if(query == NULL || query->domainID < 0) @@ -486,7 +486,7 @@ const char *getDomainString(const queriesData* query) // Privacy-level sensitive subroutine that returns the domain name // only when appropriate for the requested query -const char *getCNAMEDomainString(const queriesData* query) +const char *getCNAMEDomainString(const queriesData *query) { // Check if the returned pointer is valid before trying to access it if(query == NULL || query->CNAME_domainID < 0) @@ -510,7 +510,7 @@ const char *getCNAMEDomainString(const queriesData* query) // Privacy-level sensitive subroutine that returns the client IP // only when appropriate for the requested query -const char *getClientIPString(const queriesData* query) +const char *getClientIPString(const queriesData *query) { // Check if the returned pointer is valid before trying to access it if(query == NULL || query->clientID < 0) @@ -534,7 +534,7 @@ const char *getClientIPString(const queriesData* query) // Privacy-level sensitive subroutine that returns the client host name // only when appropriate for the requested query -const char *getClientNameString(const queriesData* query) +const char *getClientNameString(const queriesData *query) { // Check if the returned pointer is valid before trying to access it if(query == NULL || query->clientID < 0) diff --git a/src/datastructure.h b/src/datastructure.h index 48450a94..a1f7de2f 100644 --- a/src/datastructure.h +++ b/src/datastructure.h @@ -145,10 +145,10 @@ void _query_set_status(queriesData *query, const enum query_status new_status, c void FTL_reload_all_domainlists(void); void FTL_reset_per_client_domain_data(void); -const char *getDomainString(const queriesData* query); -const char *getCNAMEDomainString(const queriesData* query); -const char *getClientIPString(const queriesData* query); -const char *getClientNameString(const queriesData* query); +const char *getDomainString(const queriesData *query); +const char *getCNAMEDomainString(const queriesData *query); +const char *getClientIPString(const queriesData *query); +const char *getClientNameString(const queriesData *query); void change_clientcount(clientsData *client, int total, int blocked, int overTimeIdx, int overTimeMod); const char *get_query_type_str(const enum query_type type, const queriesData *query, char buffer[20]); @@ -171,7 +171,7 @@ int __attribute__ ((pure)) get_temp_unit_val(const char *temp_unit); // Pointer getter functions #define getQuery(queryID, checkMagic) _getQuery(queryID, checkMagic, __LINE__, __FUNCTION__, __FILE__) -queriesData* _getQuery(int queryID, bool checkMagic, int line, const char *func, const char *file); +queriesData *_getQuery(int queryID, bool checkMagic, int line, const char *func, const char *file); #define getClient(clientID, checkMagic) _getClient(clientID, checkMagic, __LINE__, __FUNCTION__, __FILE__) clientsData* _getClient(int clientID, bool checkMagic, int line, const char *func, const char *file); #define getDomain(domainID, checkMagic) _getDomain(domainID, checkMagic, __LINE__, __FUNCTION__, __FILE__) diff --git a/src/dnsmasq/forward.c b/src/dnsmasq/forward.c index b505d354..27c2c8e8 100644 --- a/src/dnsmasq/forward.c +++ b/src/dnsmasq/forward.c @@ -781,7 +781,7 @@ static size_t process_reply(struct dns_header *header, time_t now, struct server } } - FTL_header_analysis(header->hb4, rcode, server, daemon->log_display_id); + FTL_header_analysis(header->hb4, server, daemon->log_display_id); /* RFC 4035 sect 4.6 para 3 */ if (!is_sign && !option_bool(OPT_DNSSEC_PROXY)) @@ -1206,7 +1206,7 @@ void reply_query(int fd, time_t now) server = daemon->serverarray[c]; - FTL_header_analysis(header->hb4, RCODE(header), server, daemon->log_display_id); + FTL_header_analysis(header->hb4, server, daemon->log_display_id); if (RCODE(header) != REFUSED) daemon->serverarray[first]->last_server = c; @@ -2166,7 +2166,7 @@ static int tcp_key_recurse(time_t now, int status, struct dns_header *header, si unsigned char *packet = NULL; struct dns_header *new_header = NULL; - FTL_header_analysis(header->hb4, RCODE(header), server, daemon->log_display_id); + FTL_header_analysis(header->hb4, server, daemon->log_display_id); while (1) { diff --git a/src/dnsmasq/rfc1035.c b/src/dnsmasq/rfc1035.c index 1446908d..a35d9df9 100644 --- a/src/dnsmasq/rfc1035.c +++ b/src/dnsmasq/rfc1035.c @@ -792,6 +792,17 @@ int extract_addresses(struct dns_header *header, size_t qlen, char *name, time_t flags |= F_RR; else insert = 0; /* NOTE: do not cache data from CNAME queries. */ + + /*********** Pi-hole modification ***********/ + if(FTL_check_reply(RCODE(header), flags, NULL, daemon->log_display_id)) + { + // Found while processing a reply from upstream. We prevent cache insertion here + // This query is to be blocked as we found a blocked + // domain while walking the CNAME path. Log to pihole.log here + log_query(F_UPSTREAM, name, NULL, "blocked due to upstream response (header)", 0); + return 99; + } + /********************************************/ cname_loop1: if (!(p1 = skip_questions(header, qlen))) @@ -1017,6 +1028,15 @@ int extract_addresses(struct dns_header *header, size_t qlen, char *name, time_t log_query((flags & (F_IPV4 | F_IPV6)) | F_IPSET, nftsets->domain, &addr, *nftsets_cur, 0); #endif } + + /*********** Pi-hole modification ***********/ + if(FTL_check_reply(RCODE(header), flags, &addr, daemon->log_display_id)) + { + // Found while processing a reply from upstream + log_query(F_UPSTREAM, name, NULL, "blocked due to upstream response (IP)", 0); + return 99; + } + /********************************************/ if (insert) { diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c index fa9ca8d1..36d1e1c4 100644 --- a/src/dnsmasq_interface.c +++ b/src/dnsmasq_interface.c @@ -64,12 +64,11 @@ // Private prototypes static void print_flags(const unsigned int flags); #define query_set_reply(flags, reply, addr, query, response) _query_set_reply(flags, reply, addr, query, response, __FILE__, __LINE__) -static void _query_set_reply(const unsigned int flags, const enum reply_type reply, const union all_addr *addr, queriesData* query, +static void _query_set_reply(const unsigned int flags, const enum reply_type reply, const union all_addr *addr, queriesData *query, const struct timeval response, const char *file, const int line); #define FTL_check_blocking(queryID, domainID, clientID) _FTL_check_blocking(queryID, domainID, clientID, __FILE__, __LINE__) static bool _FTL_check_blocking(int queryID, int domainID, int clientID, const char* file, const int line); -static enum query_status detect_blocked_IP(const unsigned short flags, const union all_addr *addr, const queriesData *query, const domainsData *domain); -static void query_blocked(queriesData* query, domainsData* domain, clientsData* client, const enum query_status new_status); +static void query_blocked(queriesData *query, domainsData* domain, clientsData* client, const enum query_status new_status); static void FTL_forwarded(const unsigned int flags, const char *name, const union all_addr *addr, unsigned short port, const int id, const char* file, const int line); static void FTL_reply(const unsigned int flags, const char *name, const union all_addr *addr, const char* arg, unsigned short type, const int id, const char* file, const int line); static void FTL_upstream_error(const union all_addr *addr, const unsigned int flags, const int id, const char* file, const int line); @@ -83,7 +82,7 @@ static char *get_ptrname(struct in_addr *addr); static const char *check_dnsmasq_name(const char *name); // Static blocking metadata -static bool adbit = false; +static bool adbit = false, rabit = false; static const char *blockingreason = ""; static enum reply_type force_next_DNS_reply = REPLY_UNKNOWN; static int last_regex_idx = -1; @@ -462,6 +461,9 @@ size_t _FTL_make_answer(struct dns_header *header, char *limit, const size_t len if (trunc) header->hb3 |= HB3_TC; + // Unset the blocking reason + blockingreason = ""; + return p - (unsigned char *)header; } @@ -728,7 +730,7 @@ bool _FTL_new_query(const unsigned int flags, const char *name, const int domainID = findDomainID(domainString, true); // Save everything - queriesData* query = getQuery(queryID, false); + queriesData *query = getQuery(queryID, false); if(query == NULL) { // Encountered memory error, skip query @@ -1137,14 +1139,19 @@ static void check_pihole_PTR(char *domain) } } -inline static void set_dnscache_blockingstatus(DNSCacheData * dns_cache, clientsData *client, - enum domain_client_status new_status, const char *domain) +inline static void set_dnscache_blockingstatus(DNSCacheData * dns_cache, enum domain_client_status new_status, + const char *client, const char *domain) { // Memorize blocking status DNS cache for the domain/client combination dns_cache->blocking_status = new_status; - const char *clientip = client ? getstr(client->ippos) : "N/A"; - log_debug(DEBUG_QUERIES, "DNS cache: %s/%s is %s", clientip, domain, blockingreason); + if(!config.debug.queries.v.b) + return; + + // Debug logging + const char *qtype = get_query_type_str(dns_cache->query_type, NULL, NULL); + const char *clientstr = client ? client : ""; + log_debug(DEBUG_QUERIES, "DNS cache: %s/%s/%s is %s", qtype, clientstr, domain, blockingreason); } static bool check_domain_blocked(const char *domain, const int clientID, @@ -1164,7 +1171,7 @@ static bool check_domain_blocked(const char *domain, const int clientID, blockingreason = "exactly denied"; // Mark domain as exactly denied for this client - set_dnscache_blockingstatus(dns_cache, client, DENYLIST_BLOCKED, domain); + set_dnscache_blockingstatus(dns_cache, DENYLIST_BLOCKED, client ? getstr(client->ippos) : NULL, domain); // We block this domain return true; @@ -1202,7 +1209,7 @@ static bool check_domain_blocked(const char *domain, const int clientID, blockingreason = "gravity blocked"; // Mark domain as gravity blocked for this client - set_dnscache_blockingstatus(dns_cache, client, GRAVITY_BLOCKED, domain); + set_dnscache_blockingstatus(dns_cache, GRAVITY_BLOCKED, client ? getstr(client->ippos) : NULL, domain); log_debug(DEBUG_QUERIES, "Blocking query due to gravity match (list ID %i)", list_id); @@ -1262,7 +1269,7 @@ static bool check_domain_blocked(const char *domain, const int clientID, blockingreason = "regex denied"; // Mark domain as regex matched for this client - set_dnscache_blockingstatus(dns_cache, client, REGEX_BLOCKED, domain); + set_dnscache_blockingstatus(dns_cache, REGEX_BLOCKED, client ? getstr(client->ippos) : NULL, domain); // Regex may be overwriting reply type for this domain if(dns_cache->force_reply != REPLY_UNKNOWN) @@ -1362,9 +1369,8 @@ static bool _FTL_check_blocking(int queryID, int domainID, int clientID, const c // Skip the entire chain of tests if we already know the answer for this // particular client - unsigned char blockingStatus = dns_cache->blocking_status; char *domainstr = (char*)getstr(domain->domainpos); - switch(blockingStatus) + switch(dns_cache->blocking_status) { case UNKNOWN_BLOCKED: // New domain/client combination. @@ -1451,6 +1457,23 @@ static bool _FTL_check_blocking(int queryID, int domainID, int clientID, const c return false; break; + + case UPSTREAM_BLOCKED_IP: + case UPSTREAM_BLOCKED_NULL: + case UPSTREAM_BLOCKED_NXRA: + // Known as upstream blocked, we return this result + // early, skipping all the lengthy tests below + blockingreason = "upstream blocked"; + log_debug(DEBUG_QUERIES, "%s is known as %s", domainstr, blockingreason); + + force_next_DNS_reply = dns_cache->force_reply; + const enum query_status qstat = dns_cache->blocking_status == UPSTREAM_BLOCKED_IP ? + QUERY_EXTERNAL_BLOCKED_IP : + dns_cache->blocking_status == UPSTREAM_BLOCKED_NULL ? + QUERY_EXTERNAL_BLOCKED_NULL : QUERY_EXTERNAL_BLOCKED_NXRA; + query_blocked(query, domain, client, qstat); + return true; + break; } // Skip all checks and continue if we hit already at least one allowlist in the chain @@ -1501,7 +1524,7 @@ static bool _FTL_check_blocking(int queryID, int domainID, int clientID, const c // (defaulting to true) if(config.dns.blockESNI.v.b && !query->flags.allowed && blockDomain == NOT_FOUND && - strlen(domainstr) > 6 && strncasecmp(domainstr, "_esni.", 6u) == 0) + strlen(domainstr) > 6 && strncasecmp(domainstr, "_esni.", 6u) == 0) { blockDomain = check_domain_blocked(domainstr + 6u, clientID, client, query, dns_cache, &new_status, &db_okay); @@ -1544,7 +1567,8 @@ static bool _FTL_check_blocking(int queryID, int domainID, int clientID, const c // Debug output // client is guaranteed to be non-NULL above - log_debug(DEBUG_QUERIES, "DNS cache: %s/%s is %s (domainlist ID: %i)", getstr(client->ippos), + log_debug(DEBUG_QUERIES, "DNS cache: %s/%s/%s is %s (domainlist ID: %i)", + get_query_type_str(query->type, NULL, NULL), getstr(client->ippos), domainstr, query->flags.allowed ? "allowed" : "not blocked", dns_cache->list_id); } @@ -1580,7 +1604,7 @@ bool _FTL_CNAME(const char *dst, const char *src, const int id, const char* file // Get query pointer so we can later extract the client requesting this domain for // the per-client blocking evaluation - queriesData* query = getQuery(queryID, true); + queriesData *query = getQuery(queryID, true); if(query == NULL) { // Nothing to be done here @@ -1740,7 +1764,7 @@ static void FTL_forwarded(const unsigned int flags, const char *name, const unio } // Get query pointer - queriesData* query = getQuery(queryID, true); + queriesData *query = getQuery(queryID, true); if(query == NULL) { free(upstreamIP); @@ -2061,7 +2085,7 @@ static void FTL_reply(const unsigned int flags, const char *name, const union al } // Get and check query pointer - queriesData* query = getQuery(queryID, true); + queriesData *query = getQuery(queryID, true); if(query == NULL) { // Nothing to be done here @@ -2126,17 +2150,6 @@ static void FTL_reply(const unsigned int flags, const char *name, const union al if(!is_blocked(query->status)) query_set_status(query, qs); - // Detect if returned IP indicates that this query was blocked - const enum query_status new_status = detect_blocked_IP(flags, addr, query, domain); - - // Update status of this query if detected as external blocking - if(new_status != query->status) - { - clientsData *client = getClient(query->clientID, true); - if(client != NULL) - query_blocked(query, domain, client, new_status); - } - // Save reply type and update individual reply counters query_set_reply(flags, 0, addr, query, response); @@ -2227,21 +2240,6 @@ static void FTL_reply(const unsigned int flags, const char *name, const union al // Save reply type and update individual reply counters query_set_reply(reply_flags, 0, addr, query, response); - // Further checks if this is an IP address - if(addr) - { - // Detect if returned IP indicates that this query was blocked - const enum query_status new_status = detect_blocked_IP(flags, addr, query, domain); - - // Update status of this query if detected as external blocking - if(new_status != query->status) - { - clientsData *client = getClient(query->clientID, true); - if(client != NULL) - query_blocked(query, domain, client, new_status); - } - } - // Mark query for updating in the database query->flags.database.changed = true; } @@ -2304,15 +2302,10 @@ static void FTL_reply(const unsigned int flags, const char *name, const union al unlock_shm(); } -static enum query_status detect_blocked_IP(const unsigned short flags, const union all_addr *addr, const queriesData *query, const domainsData *domain) +static enum query_status detect_blocked_IP(const unsigned short flags, const union all_addr *addr) { // Compare returned IP against list of known blocking splash pages - if (!addr) - { - return query->status; - } - // First, we check if we want to skip this result even before comparing against the known IPs if(flags & F_HOSTS || flags & F_REVERSE) { @@ -2320,18 +2313,18 @@ static enum query_status detect_blocked_IP(const unsigned short flags, const uni // count gravity.list blocked queries as externally blocked. // Also: Do not mark responses of PTR requests as externally blocked. const char *cause = (flags & F_HOSTS) ? "origin is HOSTS" : "query is PTR"; - log_debug(DEBUG_QUERIES, "Skipping detection of external blocking IP for ID %i as %s", query->id, cause); + log_debug(DEBUG_QUERIES, "Skipping detection of external blocking IP as %s", cause); // Return early, do not compare against known blocking page IP addresses below - return query->status; + return QUERY_UNKNOWN; } // If received one of the following IPs as reply, OpenDNS // (Cisco Umbrella) blocked this query - // See https://support.opendns.com/hc/en-us/articles/227986927-What-are-the-Cisco-Umbrella-Block-Page-IP-Addresses- + // See https://support.opendns.com/hc/en-us/articles/227986927-What-are-the-Cisco-Umbrella-Block-Page-IP-Addresses // for a full list of these IP addresses - in_addr_t ipv4Addr = ntohl(addr->addr4.s_addr); - in_addr_t ipv6Addr = ntohl(addr->addr6.s6_addr32[3]); + const in_addr_t ipv4Addr = flags & F_IPV4 ? ntohl(addr->addr4.s_addr) : 0; + const in_addr_t ipv6Addr = flags & F_IPV6 ? ntohl(addr->addr6.s6_addr32[3]) : 0; // Check for IP block 146.112.61.104 - 146.112.61.110 if((flags & F_IPV4) && ipv4Addr >= 0x92703d68 && ipv4Addr <= 0x92703d6e) { @@ -2339,14 +2332,14 @@ static enum query_status detect_blocked_IP(const unsigned short flags, const uni { char answer[ADDRSTRLEN]; answer[0] = '\0'; inet_ntop(AF_INET, addr, answer, ADDRSTRLEN); - log_debug(DEBUG_QUERIES, "Upstream responded with known blocking page (IPv4), ID %i:\n\t\"%s\" -> \"%s\"", - query->id, getstr(domain->domainpos), answer); + blockingreason = "blocked upstream with known address (IPv4)"; + log_debug(DEBUG_QUERIES, "%s -> \"%s\"", blockingreason, answer); } // Update status return QUERY_EXTERNAL_BLOCKED_IP; } - // Check for IP block :ffff:146.112.61.104 - :ffff:146.112.61.110 + // Check for IP block ::ffff:146.112.61.104 - ::ffff:146.112.61.110 else if(flags & F_IPV6 && addr->addr6.s6_addr32[0] == 0 && addr->addr6.s6_addr32[1] == 0 && @@ -2357,8 +2350,8 @@ static enum query_status detect_blocked_IP(const unsigned short flags, const uni { char answer[ADDRSTRLEN]; answer[0] = '\0'; inet_ntop(AF_INET6, addr, answer, ADDRSTRLEN); - log_debug(DEBUG_QUERIES, "Upstream responded with known blocking page (IPv6), ID %i:\n\t\"%s\" -> \"%s\"", - query->id, getstr(domain->domainpos), answer); + blockingreason = "blocked upstream with known address (IPv6)"; + log_debug(DEBUG_QUERIES, "%s -> \"%s\"", blockingreason, answer); } // Update status @@ -2372,8 +2365,8 @@ static enum query_status detect_blocked_IP(const unsigned short flags, const uni { if(config.debug.queries.v.b) { - log_debug(DEBUG_QUERIES, "Upstream responded with 0.0.0.0, ID %i:\n\t\"%s\" -> \"0.0.0.0\"", - query->id, getstr(domain->domainpos)); + blockingreason = "blocked upstream with 0.0.0.0"; + log_debug(DEBUG_QUERIES, "%s", blockingreason); } // Update status @@ -2387,8 +2380,8 @@ static enum query_status detect_blocked_IP(const unsigned short flags, const uni { if(config.debug.queries.v.b) { - log_debug(DEBUG_QUERIES, "Upstream responded with ::, ID %i:\n\t\"%s\" -> \"::\"", - query->id, getstr(domain->domainpos)); + blockingreason = "blocked upstream with ::"; + log_debug(DEBUG_QUERIES, "%s", blockingreason); } // Update status @@ -2396,15 +2389,34 @@ static enum query_status detect_blocked_IP(const unsigned short flags, const uni } // Nothing happened here - return query->status; + return QUERY_UNKNOWN; } -static void query_blocked(queriesData* query, domainsData* domain, clientsData* client, const enum query_status new_status) +static void query_blocked(queriesData *query, domainsData *domain, clientsData *client, const enum query_status new_status) { // Get response time struct timeval response; gettimeofday(&response, 0); + // Memorize this in the DNS cache if blocked due to the response + if(new_status == QUERY_EXTERNAL_BLOCKED_IP || + new_status == QUERY_EXTERNAL_BLOCKED_NULL || + new_status == QUERY_EXTERNAL_BLOCKED_NXRA) + { + const int cacheID = findCacheID(query->domainID, query->clientID, query->type, true); + DNSCacheData *dns_cache = getDNSCache(cacheID, true); + if(dns_cache != NULL) + { + // Update status + enum domain_client_status cache_status = new_status == QUERY_EXTERNAL_BLOCKED_IP ? UPSTREAM_BLOCKED_IP : + new_status == QUERY_EXTERNAL_BLOCKED_NULL ? UPSTREAM_BLOCKED_NULL : + UPSTREAM_BLOCKED_NXRA; // can be nothing else due to if above + set_dnscache_blockingstatus(dns_cache, cache_status, + client ? getstr(client->ippos) : NULL, + domain ? getstr(domain->domainpos) : NULL); + } + } + // Adjust counters if we recorded a non-blocking status if(query->status == QUERY_FORWARDED) { @@ -2454,7 +2466,7 @@ static void FTL_dnssec(const char *arg, const union all_addr *addr, const int id } // Get query pointer - queriesData* query = getQuery(queryID, true); + queriesData *query = getQuery(queryID, true); if(query == NULL) { // Memory error, skip this DNSSEC details @@ -2536,7 +2548,7 @@ static void FTL_upstream_error(const union all_addr *addr, const unsigned int fl } // Get query pointer - queriesData* query = getQuery(queryID, true); + queriesData *query = getQuery(queryID, true); if(query == NULL) { // Memory error, skip this query @@ -2637,7 +2649,7 @@ static void FTL_upstream_error(const union all_addr *addr, const unsigned int fl unlock_shm(); } -static void FTL_mark_externally_blocked(const int id, const char* file, const int line) +static void FTL_NXRA(const int id, const char* file, const int line) { // Lock shared memory lock_shm(); @@ -2652,7 +2664,7 @@ static void FTL_mark_externally_blocked(const int id, const char* file, const in } // Get query pointer - queriesData* query = getQuery(queryID, true); + queriesData *query = getQuery(queryID, true); if(query == NULL) { // Memory error, skip this query @@ -2677,6 +2689,9 @@ static void FTL_mark_externally_blocked(const int id, const char* file, const in log_debug(DEBUG_QUERIES, "**** %s externally blocked (ID %i, FTL %i, %s:%i)", domainname, id, queryID, file, line); } + // Set blocking reason + blockingreason = "blocked upstream with NXDOMAIN and unset RA bit"; + // Get response time struct timeval response; gettimeofday(&response, 0); @@ -2696,10 +2711,10 @@ static void FTL_mark_externally_blocked(const int id, const char* file, const in unlock_shm(); } -void _FTL_header_analysis(const unsigned char header4, const unsigned int rcode, const struct server *server, - const int id, const char* file, const int line) +int _FTL_check_reply(const unsigned int rcode, const unsigned short flags, + const union all_addr *addr, + const int id, const char* file, const int line) { - // Analyze DNS header bits // Check if RA bit is unset in DNS header and rcode is NXDOMAIN // If the response code (rcode) is NXDOMAIN, we may be seeing a response from @@ -2708,14 +2723,77 @@ void _FTL_header_analysis(const unsigned char header4, const unsigned int rcode, // FTL_reply() is never getting called from within the cache routines. // Hence, we have to store the necessary information about the NXDOMAIN // reply already here. - if(!(header4 & 0x80) && rcode == NXDOMAIN) + if(addr == NULL && !rabit && rcode == NXDOMAIN) + { // RA bit is not set and rcode is NXDOMAIN - FTL_mark_externally_blocked(id, file, line); + FTL_NXRA(id, file, line); + + // Query is blocked + return 1; + } + // Further checks if this is an IP address + else if(addr != NULL) + { + // Detect if returned IP indicates that this query was blocked + const enum query_status new_status = detect_blocked_IP(flags, addr); + + // Update status of this query if detected as external blocking + if(new_status != QUERY_UNKNOWN) + { + // Lock shared memory + lock_shm(); + + // Save status in corresponding query identified by dnsmasq's ID + const int queryID = findQueryID(id); + if(queryID < 0) + { + // This may happen e.g. if the original query was "pi.hole" + log_debug(DEBUG_QUERIES, "FTL_check_reply(): Query %i has not been found", id); + unlock_shm(); + return 0; + } + + // Get query pointer + queriesData *query = getQuery(queryID, true); + if(query == NULL) + { + // Memory error, skip this query + log_debug(DEBUG_QUERIES, "FTL_check_reply(): Memory error (ID %i)", id); + unlock_shm(); + return 0; + } + clientsData *client = getClient(query->clientID, true); + domainsData *domain = getDomain(query->domainID, true); + if(client != NULL && domain != NULL) + query_blocked(query, domain, client, new_status); + + // Mark query for updating in the database + query->flags.database.changed = true; + + // Unlock shared memory + unlock_shm(); + + // Query is blocked + return 1; + } + } + + return 0; +} + +void _FTL_header_analysis(const unsigned char header4, const struct server *server, + const int id, const char* file, const int line) +{ + // Analyze DNS header bits // Check if AD bit is set in DNS header adbit = header4 & HB4_AD; - // Store server which sent this reply + // Check if RA bit is set in DNS header. We do it here as it is it is + // forced by dnsmasq shortly after calling FTL_header_analysis() + rabit = header4 & HB4_RA; + + // Store server which sent this reply (if applicable) if(server) { memcpy(&last_server, &server->addr, sizeof(last_server)); @@ -2724,13 +2802,15 @@ void _FTL_header_analysis(const unsigned char header4, const unsigned int rcode, char ip[ADDRSTRLEN+1] = { 0 }; in_port_t port = 0; mysockaddr_extract_ip_port(&last_server, ip, &port); - log_debug(DEBUG_EXTRA, "Got forward address: %s#%u (%s:%i)", ip, port, short_path(file), line); + log_debug(DEBUG_EXTRA, "Got forward address: %s#%u for ID %i (%s:%i)", + ip, port, id, short_path(file), line); } } else { memset(&last_server, 0, sizeof(last_server)); - log_debug(DEBUG_EXTRA, "Got forward address: NO"); + log_debug(DEBUG_EXTRA, "Got forward address: NO for ID %i (%s:%i)", + id, short_path(file), line); } } @@ -3148,7 +3228,7 @@ void FTL_forwarding_retried(const struct server *serv, const int oldID, const in if(queryID >= 0) { // Get query pointer - queriesData* query = getQuery(queryID, true); + queriesData *query = getQuery(queryID, true); // Set retried status if(query != NULL) @@ -3406,7 +3486,7 @@ void FTL_query_in_progress(const int id) } // Get query pointer - queriesData* query = getQuery(queryID, true); + queriesData *query = getQuery(queryID, true); if(query == NULL) { // Memory error, skip this DNSSEC details @@ -3475,9 +3555,9 @@ void FTL_multiple_replies(const int id, int *firstID) // Get (read-only) pointer of the query that contains all relevant // information (all others are mere duplicates and were only added to the // list of duplicates rather than havong been forwarded on their own) - const queriesData* source_query = getQuery(*firstID, true); + const queriesData *source_query = getQuery(*firstID, true); // Get query pointer of duplicated reply - queriesData* duplicated_query = getQuery(queryID, true); + queriesData *duplicated_query = getQuery(queryID, true); if(duplicated_query == NULL || source_query == NULL) { diff --git a/src/dnsmasq_interface.h b/src/dnsmasq_interface.h index 3335fc6a..6dc6c94d 100644 --- a/src/dnsmasq_interface.h +++ b/src/dnsmasq_interface.h @@ -27,8 +27,11 @@ void _FTL_iface(struct irec *recviface, const union all_addr *addr, const sa_fam #define FTL_new_query(flags, name, addr, arg, qtype, id, proto) _FTL_new_query(flags, name, addr, arg, qtype, id, proto, __FILE__, __LINE__) bool _FTL_new_query(const unsigned int flags, const char *name, union mysockaddr *addr, char *arg, const unsigned short qtype, const int id, enum protocol proto, const char* file, const int line); -#define FTL_header_analysis(header4, rcode, server, id) _FTL_header_analysis(header4, rcode, server, id, __FILE__, __LINE__) -void _FTL_header_analysis(const unsigned char header4, const unsigned int rcode, const struct server *server, const int id, const char* file, const int line); +#define FTL_header_analysis(header4, server, id) _FTL_header_analysis(header4, server, id, __FILE__, __LINE__) +void _FTL_header_analysis(const unsigned char header4, const struct server *server, const int id, const char* file, const int line); + +#define FTL_check_reply(rcode, flags, addr, id) _FTL_check_reply(rcode, flags, addr, id, __FILE__, __LINE__) +int _FTL_check_reply(const unsigned int rcode, const unsigned short flags, const union all_addr *addr, const int id, const char* file, const int line); void FTL_forwarding_retried(const struct server *server, const int oldID, const int newID, const bool dnssec); diff --git a/src/enums.h b/src/enums.h index 66ba915a..153098a5 100644 --- a/src/enums.h +++ b/src/enums.h @@ -130,6 +130,9 @@ enum domain_client_status { REGEX_BLOCKED, ALLOWED, SPECIAL_DOMAIN, + UPSTREAM_BLOCKED_NXRA, + UPSTREAM_BLOCKED_NULL, + UPSTREAM_BLOCKED_IP, NOT_BLOCKED } __attribute__ ((packed)); diff --git a/src/gc.c b/src/gc.c index 4b6a42f5..f514b1b0 100644 --- a/src/gc.c +++ b/src/gc.c @@ -62,7 +62,7 @@ static void recycle(void) // and recycle them for(int queryID = 0; queryID < counters->queries; queryID++) { - queriesData* query = getQuery(queryID, true); + queriesData *query = getQuery(queryID, true); if(query == NULL) continue; @@ -308,7 +308,7 @@ void runGC(const time_t now, time_t *lastGCrun, const bool flush) unsigned int removed = 0; for(long int i = 0; i < counters->queries; i++) { - queriesData* query = getQuery(i, true); + queriesData *query = getQuery(i, true); if(query == NULL) continue; diff --git a/src/shmem.c b/src/shmem.c index efa72428..3063e1de 100644 --- a/src/shmem.c +++ b/src/shmem.c @@ -1024,7 +1024,7 @@ static inline bool check_magic(int ID, bool checkMagic, unsigned char magic, con return true; } -queriesData* _getQuery(int queryID, bool checkMagic, int line, const char *func, const char *file) +queriesData *_getQuery(int queryID, bool checkMagic, int line, const char *func, const char *file) { // This does not exist, return a NULL pointer if(queryID == -1) diff --git a/test/pdns/setup.sh b/test/pdns/setup.sh index 7eb30126..e52d6015 100644 --- a/test/pdns/setup.sh +++ b/test/pdns/setup.sh @@ -117,6 +117,17 @@ pdnsutil add-record ftl. regex-notMultiple AAAA fe80::3f41 # TXT pdnsutil add-record ftl. any TXT "\"Some example text\"" +# NOERROR +pdnsutil add-record ftl. noerror A + +# Blocked Cisco Umbrella IP (https://support.opendns.com/hc/en-us/articles/227986927-What-are-the-Cisco-Umbrella-Block-Page-IP-Addresses) +pdnsutil add-record ftl. umbrella A 146.112.61.104 +pdnsutil add-record ftl. umbrella AAAA ::ffff:146.112.61.104 + +# Null address +pdnsutil add-record ftl. null A 0.0.0.0 +pdnsutil add-record ftl. null AAAA :: + # Create valid internal DNSSEC zone pdnsutil create-zone dnssec ns1.ftl pdnsutil add-record dnssec. a A 192.168.4.1 diff --git a/test/test_suite.bats b/test/test_suite.bats index edbb3149..9107857d 100644 --- a/test/test_suite.bats +++ b/test/test_suite.bats @@ -418,6 +418,161 @@ [[ ${lines[@]} == *"status: NOERROR"* ]] } +# NXRA + RA unset cannot be tested with PowerDNS as upstream provider + +@test "Externally blocked domain: NULL is recognized" { + # Get number of lines in the log before the test + before="$(grep -c ^ /var/log/pihole/FTL.log)" + + # Run test + run bash -c "dig A null.ftl @127.0.0.1" + printf "%s\n" "${lines[@]}" + [[ ${lines[@]} == *"status: NOERROR"* ]] + [[ ${lines[@]} == *"null.ftl."*"2"*"IN"*"A"*"0.0.0.0"* ]] + + # Get number of lines in the log after the test + after="$(grep -c ^ /var/log/pihole/FTL.log)" + + # Extract relevant log lines + log="$(sed -n "${before},${after}p" /var/log/pihole/FTL.log)" + # Split log into array by newline + lines=() + while IFS= read -r line; do + lines+=("$line") + done <<< "${log}" + printf "%s\n" "${lines[@]}" + [[ ${lines[@]} == *"DEBUG_QUERIES: DNS cache: A/127.0.0.1/null.ftl is not blocked (domainlist ID: -1)"* ]] + [[ ${lines[@]} == *"DEBUG_QUERIES: **** forwarded null.ftl to 127.0.0.1#5555"* ]] + [[ ${lines[@]} == *"DEBUG_QUERIES: DNS cache: A/127.0.0.1/null.ftl is blocked upstream with 0.0.0.0"* ]] + [[ ${lines[@]} == *"DEBUG_QUERIES: Adding RR: \"null.ftl A 0.0.0.0\""* ]] +} + +@test "Externally blocked domain: NULL is recognized (cached)" { + # Get number of lines in the log before the test + before="$(grep -c ^ /var/log/pihole/FTL.log)" + + # Run test + run bash -c "dig A null.ftl @127.0.0.1" + printf "%s\n" "${lines[@]}" + [[ ${lines[@]} == *"status: NOERROR"* ]] + [[ ${lines[@]} == *"null.ftl."*"2"*"IN"*"A"*"0.0.0.0"* ]] + + # Get number of lines in the log after the test + after="$(grep -c ^ /var/log/pihole/FTL.log)" + + # Extract relevant log lines + log="$(sed -n "${before},${after}p" /var/log/pihole/FTL.log)" + # Split log into array by newline + lines=() + while IFS= read -r line; do + lines+=("$line") + done <<< "${log}" + printf "%s\n" "${lines[@]}" + [[ ${lines[@]} == *"DEBUG_QUERIES: null.ftl is known as upstream blocked"* ]] + [[ ${lines[@]} == *"DEBUG_QUERIES: DNS cache: A/127.0.0.1/null.ftl is upstream blocked"* ]] + [[ ${lines[@]} != *"DEBUG_QUERIES: **** forwarded null.ftl to 127.0.0.1#5555"* ]] + [[ ${lines[@]} == *"DEBUG_QUERIES: Adding RR: \"null.ftl A 0.0.0.0\""* ]] +} + +@test "Externally blocked domain: NULL is recognized (IPv6)" { + # Get number of lines in the log before the test + before="$(grep -c ^ /var/log/pihole/FTL.log)" + + # Run test + run bash -c "dig AAAA null.ftl @127.0.0.1" + printf "%s\n" "${lines[@]}" + [[ ${lines[@]} == *"status: NOERROR"* ]] + [[ ${lines[@]} == *"null.ftl."*"2"*"IN"*"AAAA"*"::"* ]] + + # Get number of lines in the log after the test + after="$(grep -c ^ /var/log/pihole/FTL.log)" + + # Extract relevant log lines + log="$(sed -n "${before},${after}p" /var/log/pihole/FTL.log)" + # Split log into array by newline + lines=() + while IFS= read -r line; do + lines+=("$line") + done <<< "${log}" + printf "%s\n" "${lines[@]}" + [[ ${lines[@]} == *"DEBUG_QUERIES: DNS cache: AAAA/127.0.0.1/null.ftl is not blocked (domainlist ID: -1)"* ]] + [[ ${lines[@]} == *"DEBUG_QUERIES: **** forwarded null.ftl to 127.0.0.1#5555"* ]] + [[ ${lines[@]} == *"DEBUG_QUERIES: DNS cache: AAAA/127.0.0.1/null.ftl is blocked upstream with ::"* ]] + [[ ${lines[@]} == *"DEBUG_QUERIES: Adding RR: \"null.ftl AAAA ::\""* ]] +} + +@test "Externally blocked domain: IP is recognized" { + # Get number of lines in the log before the test + before="$(grep -c ^ /var/log/pihole/FTL.log)" + + # Run test + run bash -c "dig A umbrella.ftl @127.0.0.1" + + # Get number of lines in the log after the test + after="$(grep -c ^ /var/log/pihole/FTL.log)" + + # Extract relevant log lines + log="$(sed -n "${before},${after}p" /var/log/pihole/FTL.log)" + # Split log into array by newline + lines=() + while IFS= read -r line; do + lines+=("$line") + done <<< "${log}" + printf "%s\n" "${lines[@]}" + [[ ${lines[@]} == *"DEBUG_QUERIES: DNS cache: A/127.0.0.1/umbrella.ftl is not blocked (domainlist ID: -1)"* ]] + [[ ${lines[@]} == *"DEBUG_QUERIES: **** forwarded umbrella.ftl to 127.0.0.1#5555"* ]] + [[ ${lines[@]} == *"DEBUG_QUERIES: DNS cache: A/127.0.0.1/umbrella.ftl is blocked upstream with known address (IPv4)"* ]] + [[ ${lines[@]} == *"DEBUG_QUERIES: Adding RR: \"umbrella.ftl A 0.0.0.0\""* ]] +} + +@test "Externally blocked domain: IP is recognized (cached)" { + # Get number of lines in the log before the test + before="$(grep -c ^ /var/log/pihole/FTL.log)" + + # Run test + run bash -c "dig A umbrella.ftl @127.0.0.1" + + # Get number of lines in the log after the test + after="$(grep -c ^ /var/log/pihole/FTL.log)" + + # Extract relevant log lines + log="$(sed -n "${before},${after}p" /var/log/pihole/FTL.log)" + # Split log into array by newline + lines=() + while IFS= read -r line; do + lines+=("$line") + done <<< "${log}" + printf "%s\n" "${lines[@]}" + [[ ${lines[@]} == *"DEBUG_QUERIES: umbrella.ftl is known as upstream blocked"* ]] + [[ ${lines[@]} == *"DEBUG_QUERIES: DNS cache: A/127.0.0.1/umbrella.ftl is upstream blocked"* ]] + [[ ${lines[@]} != *"DEBUG_QUERIES: **** forwarded umbrella.ftl to 127.0.0.1#5555"* ]] + [[ ${lines[@]} == *"DEBUG_QUERIES: Adding RR: \"umbrella.ftl A 0.0.0.0\""* ]] +} + +@test "Externally blocked domain: IP is recognized (IPv6)" { + # Get number of lines in the log before the test + before="$(grep -c ^ /var/log/pihole/FTL.log)" + + # Run test + run bash -c "dig AAAA umbrella.ftl @127.0.0.1" + + # Get number of lines in the log after the test + after="$(grep -c ^ /var/log/pihole/FTL.log)" + + # Extract relevant log lines + log="$(sed -n "${before},${after}p" /var/log/pihole/FTL.log)" + # Split log into array by newline + lines=() + while IFS= read -r line; do + lines+=("$line") + done <<< "${log}" + printf "%s\n" "${lines[@]}" + [[ ${lines[@]} == *"DEBUG_QUERIES: DNS cache: AAAA/127.0.0.1/umbrella.ftl is not blocked (domainlist ID: -1)"* ]] + [[ ${lines[@]} == *"DEBUG_QUERIES: **** forwarded umbrella.ftl to 127.0.0.1#5555"* ]] + [[ ${lines[@]} == *"DEBUG_QUERIES: DNS cache: AAAA/127.0.0.1/umbrella.ftl is blocked upstream with known address (IPv6)"* ]] + [[ ${lines[@]} == *"DEBUG_QUERIES: Adding RR: \"umbrella.ftl AAAA ::\""* ]] +} + @test "ABP-style matching working as expected" { run bash -c "dig A special.gravity.ftl @127.0.0.1 +short" printf "%s\n" "${lines[@]}" From 17b9bfcc3466a687e40fc9065dc77bea79eea45e Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 12 Aug 2024 19:45:27 +0200 Subject: [PATCH 264/339] If multiple records are returned but at least one is a blocked IP address, the entire query should be blocked. This is already the case but this commit adds an explicit CI test for this desired behavior. Signed-off-by: DL6ER --- .github/.codespellignore | 1 + test/pdns/setup.sh | 5 +++++ test/test_suite.bats | 24 ++++++++++++++++++++++++ 3 files changed, 30 insertions(+) diff --git a/.github/.codespellignore b/.github/.codespellignore index 6ffa6265..c1b04b2a 100644 --- a/.github/.codespellignore +++ b/.github/.codespellignore @@ -12,3 +12,4 @@ dnsmasq iif prefered padd +rabit diff --git a/test/pdns/setup.sh b/test/pdns/setup.sh index e52d6015..e887fa03 100644 --- a/test/pdns/setup.sh +++ b/test/pdns/setup.sh @@ -124,6 +124,11 @@ pdnsutil add-record ftl. noerror A pdnsutil add-record ftl. umbrella A 146.112.61.104 pdnsutil add-record ftl. umbrella AAAA ::ffff:146.112.61.104 +# Special record which consists of both blocked and non-blocked IP +pdnsutil add-record ftl. umbrella-multi A 1.2.3.4 +pdnsutil add-record ftl. umbrella-multi A 146.112.61.104 +pdnsutil add-record ftl. umbrella-multi A 8.8.8.8 + # Null address pdnsutil add-record ftl. null A 0.0.0.0 pdnsutil add-record ftl. null AAAA :: diff --git a/test/test_suite.bats b/test/test_suite.bats index 9107857d..8a5019d2 100644 --- a/test/test_suite.bats +++ b/test/test_suite.bats @@ -573,6 +573,30 @@ [[ ${lines[@]} == *"DEBUG_QUERIES: Adding RR: \"umbrella.ftl AAAA ::\""* ]] } +@test "Externally blocked domain: IP is recognized (multi)" { + # Get number of lines in the log before the test + before="$(grep -c ^ /var/log/pihole/FTL.log)" + + # Run test + run bash -c "dig A umbrella-multi.ftl @127.0.0.1" + + # Get number of lines in the log after the test + after="$(grep -c ^ /var/log/pihole/FTL.log)" + + # Extract relevant log lines + log="$(sed -n "${before},${after}p" /var/log/pihole/FTL.log)" + # Split log into array by newline + lines=() + while IFS= read -r line; do + lines+=("$line") + done <<< "${log}" + printf "%s\n" "${lines[@]}" + [[ ${lines[@]} == *"DEBUG_QUERIES: DNS cache: A/127.0.0.1/umbrella-multi.ftl is not blocked (domainlist ID: -1)"* ]] + [[ ${lines[@]} == *"DEBUG_QUERIES: **** forwarded umbrella-multi.ftl to 127.0.0.1#5555"* ]] + [[ ${lines[@]} == *"DEBUG_QUERIES: DNS cache: A/127.0.0.1/umbrella-multi.ftl is blocked upstream with known address (IPv4)"* ]] + [[ ${lines[@]} == *"DEBUG_QUERIES: Adding RR: \"umbrella-multi.ftl A 0.0.0.0\""* ]] +} + @test "ABP-style matching working as expected" { run bash -c "dig A special.gravity.ftl @127.0.0.1 +short" printf "%s\n" "${lines[@]}" From af1c5b3039a1a5ac5755c5d058ffcf5a78b4b76e Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 21 Aug 2024 20:09:03 +0200 Subject: [PATCH 265/339] Add new config option dns.cache.upstreamTTL (defaulting to one day) for configuring how long relpies from upstream are cached Signed-off-by: DL6ER --- src/api/docs/content/specs/config.yaml | 3 ++ src/config/config.c | 6 ++++ src/config/config.h | 1 + src/datastructure.h | 1 + src/dnsmasq/rfc1035.c | 2 +- src/dnsmasq_interface.c | 40 +++++++++++++++++++++----- test/pihole.toml | 8 +++++- 7 files changed, 52 insertions(+), 9 deletions(-) diff --git a/src/api/docs/content/specs/config.yaml b/src/api/docs/content/specs/config.yaml index b8894f74..4e862601 100644 --- a/src/api/docs/content/specs/config.yaml +++ b/src/api/docs/content/specs/config.yaml @@ -239,6 +239,8 @@ components: type: integer optimizer: type: integer + upstreamTTL: + type: integer revServers: type: array items: @@ -663,6 +665,7 @@ components: cache: size: 10000 optimizer: 3600 + upstreamTTL: 86400 revServers: - "true,192.168.0.0/24,192.168.0.1,lan" blocking: diff --git a/src/config/config.c b/src/config/config.c index 42d1f7c7..0a4c0633 100644 --- a/src/config/config.c +++ b/src/config/config.c @@ -580,6 +580,12 @@ void initConfig(struct config *conf) conf->dns.cache.optimizer.d.i = 3600u; conf->dns.cache.optimizer.c = validate_stub; // Only type-based checking + conf->dns.cache.upstreamTTL.k = "dns.cache.upstreamTTL"; + conf->dns.cache.upstreamTTL.h = "This setting allows you to specify the TTL used for queries blocked upstream. Once the TTL expires, the query will be forwarded to the upstream server again to check if the block is still valid. Defaults to caching for one day (86400 seconds). Setting this value to zero disables caching of queries blocked upstream."; + conf->dns.cache.upstreamTTL.t = CONF_UINT; + conf->dns.cache.upstreamTTL.d.ui = 86400; + conf->dns.cache.upstreamTTL.c = validate_stub; // Only type-based checking + // sub-struct dns.blocking conf->dns.blocking.active.k = "dns.blocking.active"; conf->dns.blocking.active.h = "Should FTL block queries?"; diff --git a/src/config/config.h b/src/config/config.h index 0839c1b0..2f3538ae 100644 --- a/src/config/config.h +++ b/src/config/config.h @@ -147,6 +147,7 @@ struct config { struct { struct conf_item size; struct conf_item optimizer; + struct conf_item upstreamTTL; } cache; struct { struct conf_item active; diff --git a/src/datastructure.h b/src/datastructure.h index a1f7de2f..7d3ac932 100644 --- a/src/datastructure.h +++ b/src/datastructure.h @@ -114,6 +114,7 @@ typedef struct { int domainID; int clientID; int list_id; + time_t expires; char *cname_target; } DNSCacheData; diff --git a/src/dnsmasq/rfc1035.c b/src/dnsmasq/rfc1035.c index a35d9df9..f16ae343 100644 --- a/src/dnsmasq/rfc1035.c +++ b/src/dnsmasq/rfc1035.c @@ -1033,7 +1033,7 @@ int extract_addresses(struct dns_header *header, size_t qlen, char *name, time_t if(FTL_check_reply(RCODE(header), flags, &addr, daemon->log_display_id)) { // Found while processing a reply from upstream - log_query(F_UPSTREAM, name, NULL, "blocked due to upstream response (IP)", 0); + log_query(F_UPSTREAM, name, NULL, "blocked due to upstream response (answer)", 0); return 99; } /********************************************/ diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c index 36d1e1c4..d2628100 100644 --- a/src/dnsmasq_interface.c +++ b/src/dnsmasq_interface.c @@ -184,7 +184,8 @@ void FTL_hook(unsigned int flags, const char *name, union all_addr *addr, char * } // This is inspired by make_local_answer() -size_t _FTL_make_answer(struct dns_header *header, char *limit, const size_t len, int *ede, const char *file, const int line) +size_t _FTL_make_answer(struct dns_header *header, char *limit, const size_t len, int *ede, + const char *file, const int line) { log_debug(DEBUG_FLAGS, "FTL_make_answer() called from %s:%d", short_path(file), line); // Exit early if there are no questions in this query @@ -1139,19 +1140,34 @@ static void check_pihole_PTR(char *domain) } } -inline static void set_dnscache_blockingstatus(DNSCacheData * dns_cache, enum domain_client_status new_status, - const char *client, const char *domain) +static void set_dnscache_blockingstatus(DNSCacheData *dns_cache, enum domain_client_status new_status, + const char *client, const char *domain) { // Memorize blocking status DNS cache for the domain/client combination dns_cache->blocking_status = new_status; + // Set expiration time for this cache entry (if applicable) + // We set this only if not already set to avoid extending the TTL of an + // existing entry + if(config.dns.cache.upstreamTTL.v.ui > 0 && + dns_cache->expires == 0 && + (new_status == UPSTREAM_BLOCKED_NXRA || + new_status == UPSTREAM_BLOCKED_NULL || + new_status == UPSTREAM_BLOCKED_IP)) + { + // Set expiration time for this cache entry + dns_cache->expires = time(NULL) + config.dns.cache.upstreamTTL.v.ui; + } + if(!config.debug.queries.v.b) return; // Debug logging const char *qtype = get_query_type_str(dns_cache->query_type, NULL, NULL); const char *clientstr = client ? client : ""; - log_debug(DEBUG_QUERIES, "DNS cache: %s/%s/%s is %s", qtype, clientstr, domain, blockingreason); + log_debug(DEBUG_QUERIES, "DNS cache: %s/%s/%s is %s, expires in %lis", + qtype, clientstr, domain, blockingreason, + dns_cache->expires > 0 ? (long)(dns_cache->expires - time(NULL)) : -1); } static bool check_domain_blocked(const char *domain, const int clientID, @@ -1358,8 +1374,7 @@ static bool _FTL_check_blocking(int queryID, int domainID, int clientID, const c // already stored in the query, we have to re-lookup the cache ID. // This can happen when a CNAME chain is followed and analyzed const int cacheID = query->domainID == domainID && query->clientID == clientID ? - query->cacheID : - findCacheID(domainID, clientID, query->type, true); + query->cacheID : findCacheID(domainID, clientID, query->type, true); DNSCacheData *dns_cache = getDNSCache(cacheID, true); if(dns_cache == NULL) { @@ -1367,6 +1382,16 @@ static bool _FTL_check_blocking(int queryID, int domainID, int clientID, const c return false; } + // If this cache record can expire, check if it is still valid + if(dns_cache->expires > 0 && dns_cache->expires < time(NULL)) + { + // This cache record is expired, we have to re-check + log_debug(DEBUG_QUERIES, "DNS cache record expired"); + dns_cache->blocking_status = UNKNOWN_BLOCKED; + dns_cache->expires = 0; + dns_cache->list_id = -1; + } + // Skip the entire chain of tests if we already know the answer for this // particular client char *domainstr = (char*)getstr(domain->domainpos); @@ -1464,7 +1489,8 @@ static bool _FTL_check_blocking(int queryID, int domainID, int clientID, const c // Known as upstream blocked, we return this result // early, skipping all the lengthy tests below blockingreason = "upstream blocked"; - log_debug(DEBUG_QUERIES, "%s is known as %s", domainstr, blockingreason); + log_debug(DEBUG_QUERIES, "%s is known as %s (expires in %lus)", + domainstr, blockingreason, (unsigned long)(dns_cache->expires - time(NULL))); force_next_DNS_reply = dns_cache->force_reply; const enum query_status qstat = dns_cache->blocking_status == UPSTREAM_BLOCKED_IP ? diff --git a/test/pihole.toml b/test/pihole.toml index c6514e6f..5a712c92 100644 --- a/test/pihole.toml +++ b/test/pihole.toml @@ -251,6 +251,12 @@ # altogether. optimizer = 3600 + # This setting allows you to specify the TTL used for queries blocked upstream. Once + # the TTL expires, the query will be forwarded to the upstream server again to check + # if the block is still valid. Defaults to caching for one day (86400 seconds). + # Setting this value to zero disables caching of queries blocked upstream. + upstreamTTL = 86400 + [dns.blocking] # Should FTL block queries? active = true @@ -1097,7 +1103,7 @@ all = true ### CHANGED, default = false # Configuration statistics: -# 148 total entries out of which 93 entries are default +# 149 total entries out of which 94 entries are default # --> 55 entries are modified # 2 entries are forced through environment: # - misc.nice From 7fea06811d30824ef6461aa6776c0dc561697178 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 22 Aug 2024 16:49:27 +0200 Subject: [PATCH 266/339] Allow other FTL process inside docker Do not refuse to (re)start if there is another pihole-FTL process already running *inside* a docker container on the same host. Currently, (re)starting on the host is prevented when FTL detects during startup that another process is running on the machine (the one inside the container) even when they can obviously run alongside perfectly fine. This is caused by the host being able to see *all* processes, not only the one in its own cgroup. This actually hinders development/testing natively on the host. This commit fixes this by not considering pihole-FTL processes running in downstream (!) docker containers as duplicate instances. Signed-off-by: DL6ER --- src/procps.c | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/src/procps.c b/src/procps.c index 7362b98c..d23ce7d7 100644 --- a/src/procps.c +++ b/src/procps.c @@ -114,6 +114,30 @@ static bool get_process_creation_time(const pid_t pid, char timestr[TIMESTR_SIZE return true; } +// This function checks if a given PID is running inside a docker container +static bool is_in_docker(const pid_t pid) +{ + char filename[sizeof("/proc/%u/cgroup") + sizeof(int)*3]; + snprintf(filename, sizeof(filename), "/proc/%d/cgroup", pid); + + FILE *f = fopen(filename, "r"); + if(f == NULL) + return false; + + char buffer[128]; + while(fgets(buffer, sizeof(buffer), f) != NULL) + { + if(strstr(buffer, "/docker") != NULL) + { + fclose(f); + return true; + } + } + fclose(f); + + return false; +} + // This function prints an info message about if another FTL process is already // running. It returns true if another FTL process is already running, false // otherwise. @@ -219,7 +243,7 @@ bool another_FTL(void) if(pid == ourselves) continue; - // Only process this is this is our own process + // Only process this if this is our own process if(strcasecmp(name, PROCESS_NAME) != 0) continue; @@ -231,6 +255,10 @@ bool another_FTL(void) if(!get_process_name(ppid, ppid_name)) continue; + // Skip if this is an instance running inside a docker container + if(is_in_docker(pid)) + continue; + log_debug(DEBUG_SHMEM, " └ PPID: %d -> name: %s", ppid, ppid_name); char timestr[TIMESTR_SIZE] = { 0 }; From 267d150ac7512330d9ff4dadac5d0db8783c8780 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 25 Aug 2024 19:22:41 +0200 Subject: [PATCH 267/339] Make v5 -> v6 migration verbose by default (not only when debug.config = false). This will happen only once and may greatly ease debugging in case unintended side-effects (most often it will by typos in the old setupVars.conf) Signed-off-by: DL6ER --- src/api/history.c | 2 - src/config/setupVars.c | 110 ++++++++++++++++++++++++----------------- 2 files changed, 65 insertions(+), 47 deletions(-) diff --git a/src/api/history.c b/src/api/history.c index 75a67544..dda62eb2 100644 --- a/src/api/history.c +++ b/src/api/history.c @@ -18,8 +18,6 @@ #include "overTime.h" // config struct #include "config/config.h" -// read_setupVarsconf() -#include "config/setupVars.h" // get_aliasclient_list() #include "database/aliasclients.h" diff --git a/src/config/setupVars.c b/src/config/setupVars.c index e0c54009..c5ff27de 100644 --- a/src/config/setupVars.c +++ b/src/config/setupVars.c @@ -30,7 +30,7 @@ static void get_conf_string_from_setupVars(const char *key, struct conf_item *co if(setupVarsValue == NULL) { // Do not change default value, this value is not set in setupVars.conf - log_debug(DEBUG_CONFIG, "setupVars.conf:%s -> Not set", key); + log_info("setupVars.conf:%s -> Not set", key); // Free memory, harmless to call if read_setupVarsconf() didn't return a result clearSetupVarsArray(); @@ -48,7 +48,7 @@ static void get_conf_string_from_setupVars(const char *key, struct conf_item *co clearSetupVarsArray(); // Parameter present in setupVars.conf - log_debug(DEBUG_CONFIG, "setupVars.conf:%s -> Setting %s to %s", key, conf_item->k, conf_item->v.s); + log_info("setupVars.conf:%s -> Setting %s to %s", key, conf_item->k, conf_item->v.s); } static void get_conf_ipv4_from_setupVars(const char *key, struct conf_item *conf_item) @@ -64,7 +64,7 @@ static void get_conf_ipv4_from_setupVars(const char *key, struct conf_item *conf if(setupVarsValue == NULL) { // Do not change default value, this value is not set in setupVars.conf - log_debug(DEBUG_CONFIG, "setupVars.conf:%s -> Not set", key); + log_info("setupVars.conf:%s -> Not set", key); // Free memory, harmless to call if read_setupVarsconf() didn't return a result clearSetupVarsArray(); @@ -75,7 +75,7 @@ static void get_conf_ipv4_from_setupVars(const char *key, struct conf_item *conf memset(&conf_item->v.in_addr, 0, sizeof(struct in_addr)); else if(inet_pton(AF_INET, setupVarsValue, &conf_item->v.in_addr) != 1) { - log_debug(DEBUG_CONFIG, "setupVars.conf:%s -> Invalid IPv4 address: %s", key, setupVarsValue); + log_info("setupVars.conf:%s -> Invalid IPv4 address: %s", key, setupVarsValue); memset(&conf_item->v.in_addr, 0, sizeof(struct in_addr)); } @@ -83,7 +83,7 @@ static void get_conf_ipv4_from_setupVars(const char *key, struct conf_item *conf clearSetupVarsArray(); // Parameter present in setupVars.conf - log_debug(DEBUG_CONFIG, "setupVars.conf:%s -> Setting %s to %s", key, conf_item->k, inet_ntoa(conf_item->v.in_addr)); + log_info("setupVars.conf:%s -> Setting %s to %s", key, conf_item->k, inet_ntoa(conf_item->v.in_addr)); } static void get_conf_bool_from_setupVars(const char *key, struct conf_item *conf_item) @@ -100,7 +100,7 @@ static void get_conf_bool_from_setupVars(const char *key, struct conf_item *conf if(boolean == NULL) { // Do not change default value, this value is not set in setupVars.conf - log_debug(DEBUG_CONFIG, "setupVars.conf:%s -> Not set", key); + log_info("setupVars.conf:%s -> Not set", key); // Free memory, harmless to call if read_setupVarsconf() didn't return a result clearSetupVarsArray(); @@ -116,13 +116,13 @@ static void get_conf_bool_from_setupVars(const char *key, struct conf_item *conf clearSetupVarsArray(); // Parameter present in setupVars.conf - log_debug(DEBUG_CONFIG, "setupVars.conf:%s -> Setting %s to %s", - key, conf_item->k, conf_item->v.b ? "true" : "false"); + log_info("setupVars.conf:%s -> Setting %s to %s", + key, conf_item->k, conf_item->v.b ? "true" : "false"); } static void get_revServer_from_setupVars(void) { - bool active = false; + char *active = NULL; char *cidr = NULL; char *target = NULL; char *domain = NULL; @@ -130,17 +130,21 @@ static void get_revServer_from_setupVars(void) if(active_str == NULL) { // Do not change default value, this value is not set in setupVars.conf - log_debug(DEBUG_CONFIG, "setupVars.conf:REV_SERVER -> Not set"); + log_info("setupVars.conf:REV_SERVER -> Not set"); // Free memory, harmless to call if read_setupVarsconf() didn't return a result clearSetupVarsArray(); return; } - else + // Parameter present in setupVars.conf, check if either "true" or "false" + if(strcasecmp(active_str, "true") != 0 && strcasecmp(active_str, "false") != 0) { - // Parameter present in setupVars.conf - active = getSetupVarsBool(active_str); + log_info("setupVars.conf:REV_SERVER -> Invalid value: %s", active_str); + + clearSetupVarsArray(); + return; } + active = strdup(active_str); // Free memory, harmless to call if read_setupVarsconf() didn't return a result clearSetupVarsArray(); @@ -151,6 +155,8 @@ static void get_revServer_from_setupVars(void) cidr = strdup(cidr_str); trim_whitespace(cidr); } + else + log_info("setupVars.conf:REV_SERVER_CIDR -> Not set"); // Free memory, harmless to call if read_setupVarsconf() didn't return a result clearSetupVarsArray(); @@ -161,6 +167,8 @@ static void get_revServer_from_setupVars(void) target = strdup(target_str); trim_whitespace(target); } + else + log_info("setupVars.conf:REV_SERVER_TARGET -> Not set"); // Free memory, harmless to call if read_setupVarsconf() didn't return a result clearSetupVarsArray(); @@ -171,27 +179,40 @@ static void get_revServer_from_setupVars(void) domain = strdup(domain_str); trim_whitespace(domain); } + else + log_info("setupVars.conf:REV_SERVER_DOMAIN -> Not set"); // Free memory, harmless to call if read_setupVarsconf() didn't return a result clearSetupVarsArray(); // Only add the entry if all values are present and active - if(active && cidr != NULL && target != NULL && domain != NULL) + if(active != NULL && cidr != NULL && target != NULL && domain != NULL) { // Build comma-separated string of all values - // 8 = 3 commas, "true", and null terminator - char *old = calloc(strlen(cidr) + strlen(target) + strlen(domain) + 8, sizeof(char)); + // 9 = 3 commas, "true/false", and null terminator + char *old = calloc(strlen(cidr) + strlen(target) + strlen(domain) + 9, sizeof(char)); if(old) { // Add to new config // active is always true as we only add active entries - sprintf(old, "true,%s,%s,%s", cidr, target, domain); + sprintf(old, "%s,%s,%s,%s", active_str, cidr, target, domain); cJSON_AddItemToArray(config.dns.revServers.v.json, cJSON_CreateString(old)); + + // Parameter present in setupVars.conf + log_info("setupVars.conf:REV_SERVER -> Setting %s to %s", + config.dns.revServers.k, old); free(old); } } + else + { + // Parameter not present in setupVars.conf + log_info("setupVars.conf:REV_SERVER_* -> Not set (found invalid/incomplete parameters)"); + } // Free memory + if(active != NULL) + free(active); if(cidr != NULL) free(cidr); if(target != NULL) @@ -262,8 +283,8 @@ static void get_conf_string_array_from_setupVars_regex(const char *key, struct c cJSON *item = cJSON_CreateString(regex2); cJSON_AddItemToArray(conf_item->v.json, item); - log_debug(DEBUG_CONFIG, "setupVars.conf:%s -> Setting %s[%u] = %s\n", - key, conf_item->k, i, item->valuestring); + log_info("setupVars.conf:%s -> Setting %s[%u] = %s\n", + key, conf_item->k, i, item->valuestring); // Free memory free(regex2); @@ -295,13 +316,12 @@ static void get_conf_upstream_servers_from_setupVars(struct conf_item *conf_item if(value != NULL) { - log_debug(DEBUG_CONFIG, "%s = %s\n", server_key, value); // Add string to our JSON array cJSON *item = cJSON_CreateString(value); cJSON_AddItemToArray(conf_item->v.json, item); - log_debug(DEBUG_CONFIG, "setupVars.conf:PIHOLE_DNS_%u -> Setting %s[%u] = %s\n", - j, conf_item->k, j, item->valuestring); + log_info("setupVars.conf:PIHOLE_DNS_%u -> Setting %s[%u] = %s\n", + j, conf_item->k, j, item->valuestring); } // Free memory, harmless to call if read_setupVarsconf() didn't return a result @@ -317,7 +337,7 @@ static void get_conf_temp_limit_from_setupVars(void) if(temp_limit == NULL) { // Do not change default value, this value is not set in setupVars.conf - log_debug(DEBUG_CONFIG, "setupVars.conf:TEMPERATURE_LIMIT -> Not set"); + log_info("setupVars.conf:TEMPERATURE_LIMIT -> Not set"); // Free memory, harmless to call if read_setupVarsconf() didn't return a result clearSetupVarsArray(); @@ -339,13 +359,13 @@ static void get_conf_temp_limit_from_setupVars(void) if(set) { // Parameter present in setupVars.conf - log_debug(DEBUG_CONFIG, "setupVars.conf:TEMPERATURE_LIMIT -> Setting %s to %f", + log_info("setupVars.conf:TEMPERATURE_LIMIT -> Setting %s to %f", config.webserver.api.temp.limit.k, config.webserver.api.temp.limit.v.d); } else { // Parameter not present in setupVars.conf - log_debug(DEBUG_CONFIG, "setupVars.conf:TEMPERATURE_LIMIT -> Not set (found invalid value)"); + log_info("setupVars.conf:TEMPERATURE_LIMIT -> Not set (found invalid value)"); } } @@ -357,7 +377,7 @@ static void get_conf_weblayout_from_setupVars(void) if(web_layout == NULL) { // Do not change default value, this value is not set in setupVars.conf - log_debug(DEBUG_CONFIG, "setupVars.conf:WEBUIBOXEDLAYOUT -> Not set"); + log_info("setupVars.conf:WEBUIBOXEDLAYOUT -> Not set"); // Free memory, harmless to call if read_setupVarsconf() didn't return a result clearSetupVarsArray(); @@ -365,16 +385,14 @@ static void get_conf_weblayout_from_setupVars(void) } // If the property is set to false and different than "boxed", the property - // is disabled. This is consistent with the code in AdminLTE when writing - // this code - if(strcasecmp(web_layout, "boxed") != 0) - config.webserver.interface.boxed.v.b = false; + // is disabled + config.webserver.interface.boxed.v.b = strcasecmp(web_layout, "boxed") == 0; // Free memory, harmless to call if read_setupVarsconf() didn't return a result clearSetupVarsArray(); // Parameter present in setupVars.conf - log_debug(DEBUG_CONFIG, "setupVars.conf:WEBUIBOXEDLAYOUT -> Setting %s to %s", + log_info("setupVars.conf:WEBUIBOXEDLAYOUT -> Setting %s to %s", config.webserver.interface.boxed.k,config.webserver.interface.boxed.v.b ? "true" : "false"); } @@ -386,7 +404,7 @@ static void get_conf_webtheme_from_setupVars(void) if(webTheme == NULL) { // Do not change default value, this value is not set in setupVars.conf - log_debug(DEBUG_CONFIG, "setupVars.conf:WEBTHEME -> Not set"); + log_info("setupVars.conf:WEBTHEME -> Not set"); // Free memory, harmless to call if read_setupVarsconf() didn't return a result clearSetupVarsArray(); @@ -407,14 +425,14 @@ static void get_conf_webtheme_from_setupVars(void) if(set) { // Parameter present in setupVars.conf - log_debug(DEBUG_CONFIG, "setupVars.conf:WEBTHEME -> Setting %s to %s", - config.webserver.interface.theme.k, - get_web_theme_str(config.webserver.interface.theme.v.web_theme)); + log_info("setupVars.conf:WEBTHEME -> Setting %s to %s", + config.webserver.interface.theme.k, + get_web_theme_str(config.webserver.interface.theme.v.web_theme)); } else { // Parameter not present in setupVars.conf - log_debug(DEBUG_CONFIG, "setupVars.conf:WEBTHEME -> Not set (found invalid value)"); + log_info("setupVars.conf:WEBTHEME -> Not set (found invalid value)"); } } @@ -426,7 +444,7 @@ static void get_conf_temp_unit_from_setupVars(void) if(temp_unit == NULL) { // Do not change default value, this value is not set in setupVars.conf - log_debug(DEBUG_CONFIG, "setupVars.conf:TEMPERATURE_UNIT -> Not set"); + log_info("setupVars.conf:TEMPERATURE_UNIT -> Not set"); // Free memory, harmless to call if read_setupVarsconf() didn't return a result clearSetupVarsArray(); @@ -447,14 +465,14 @@ static void get_conf_temp_unit_from_setupVars(void) if(set) { // Parameter present in setupVars.conf - log_debug(DEBUG_CONFIG, "setupVars.conf:TEMPERATURE_UNIT -> Setting %s to %s", - config.webserver.interface.theme.k, - get_temp_unit_str(config.webserver.api.temp.unit.v.temp_unit)); + log_info("setupVars.conf:TEMPERATURE_UNIT -> Setting %s to %s", + config.webserver.interface.theme.k, + get_temp_unit_str(config.webserver.api.temp.unit.v.temp_unit)); } else { // Parameter not present in setupVars.conf - log_debug(DEBUG_CONFIG, "setupVars.conf:TEMPERATURE_UNIT -> Not set (found invalid value)"); + log_info("setupVars.conf:TEMPERATURE_UNIT -> Not set (found invalid value)"); } } @@ -466,7 +484,7 @@ static void get_conf_listeningMode_from_setupVars(void) if(listeningMode == NULL) { // Do not change default value, this value is not set in setupVars.conf - log_debug(DEBUG_CONFIG, "setupVars.conf:DNSMASQ_LISTENING -> Not set"); + log_info("setupVars.conf:DNSMASQ_LISTENING -> Not set"); // Free memory, harmless to call if read_setupVarsconf() didn't return a result clearSetupVarsArray(); @@ -487,13 +505,13 @@ static void get_conf_listeningMode_from_setupVars(void) if(set) { // Parameter present in setupVars.conf - log_debug(DEBUG_CONFIG, "setupVars.conf:DNSMASQ_LISTENING -> Setting %s to %s", - config.dns.listeningMode.k, get_listeningMode_str(config.dns.listeningMode.v.listeningMode)); + log_info("setupVars.conf:DNSMASQ_LISTENING -> Setting %s to %s", + config.dns.listeningMode.k, get_listeningMode_str(config.dns.listeningMode.v.listeningMode)); } else { // Parameter not present in setupVars.conf - log_debug(DEBUG_CONFIG, "setupVars.conf:DNSMASQ_LISTENING -> Not set (found invalid value)"); + log_info("setupVars.conf:DNSMASQ_LISTENING -> Not set (found invalid value)"); } } @@ -587,6 +605,8 @@ void importsetupVarsConf(void) else log_info("Moved %s to %s", config.files.setupVars.v.s, old_setupVars); free(old_setupVars); + + log_info("Migration complete"); } char* __attribute__((pure)) find_equals(char *s) From c98f1cf0bea19bc05ae7eef3f95634c5bd2c4925 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 20 Aug 2024 19:52:11 +0200 Subject: [PATCH 268/339] Avoid (unlikely) memory leaking and ensure we properly reset the config type to a non-allocated type after free'ing. Signed-off-by: DL6ER --- src/api/config.c | 1 + src/config/cli.c | 2 +- src/config/config.c | 13 ++++++++++--- src/config/config.h | 2 -- src/config/legacy_reader.c | 8 ++++++++ src/config/password.c | 21 +++++++++++++++++---- src/config/setupVars.c | 2 +- 7 files changed, 38 insertions(+), 11 deletions(-) diff --git a/src/api/config.c b/src/api/config.c index c5d005bb..5b12dd10 100644 --- a/src/api/config.c +++ b/src/api/config.c @@ -279,6 +279,7 @@ static const char *getJSONvalue(struct conf_item *conf_item, cJSON *elem, struct free(conf_item->v.s); // Set item conf_item->v.s = strdup(elem->valuestring); + conf_item->t = CONF_STRING_ALLOCATED; // allocated now log_debug(DEBUG_CONFIG, "%s = \"%s\"", conf_item->k, conf_item->v.s); break; } diff --git a/src/config/cli.c b/src/config/cli.c index 0bcc249d..7c2de6bd 100644 --- a/src/config/cli.c +++ b/src/config/cli.c @@ -182,7 +182,7 @@ static bool readStringValue(struct conf_item *conf_item, const char *value, stru // Free old password hash if it was allocated if(conf_item->t == CONF_STRING_ALLOCATED) - free(conf_item->v.s); + free(conf_item->v.s); // Store new password hash conf_item->v.s = pwhash; diff --git a/src/config/config.c b/src/config/config.c index e4800b03..03ca4231 100644 --- a/src/config/config.c +++ b/src/config/config.c @@ -42,6 +42,8 @@ uint8_t last_checksum[SHA256_DIGEST_SIZE] = { 0 }; // Private prototypes static bool port_in_use(const in_port_t port); +static void reset_config_default(struct conf_item *conf_item); +static void initConfig(struct config *conf); // Set debug flags from config struct to global debug_flags array // This is called whenever the config is reloaded and debug flags may have @@ -158,7 +160,10 @@ void free_config_path(char **paths) for(unsigned int i = 0; i < MAX_CONFIG_PATH_DEPTH; i++) if(paths[i] != NULL) + { free(paths[i]); + paths[i] = NULL; + } } bool __attribute__ ((pure)) check_paths_equal(char **paths1, char **paths2, unsigned int max_level) @@ -369,6 +374,8 @@ void free_config(struct config *conf) break; case CONF_STRING_ALLOCATED: free(copy_item->v.s); + copy_item->v.s = NULL; + copy_item->t = CONF_STRING; // not allocated anymore break; case CONF_JSON_STRING_ARRAY: cJSON_Delete(copy_item->v.json); @@ -377,7 +384,7 @@ void free_config(struct config *conf) } } -void initConfig(struct config *conf) +static void initConfig(struct config *conf) { if(config_initialized) return; @@ -1492,7 +1499,7 @@ void initConfig(struct config *conf) // Initialize config value with default one for all *except* the log file path if(conf_item != &conf->files.log.ftl) - reset_config(conf_item); + reset_config_default(conf_item); // Parse and split paths conf_item->p = gen_config_path(conf_item->k, '.'); @@ -1541,7 +1548,7 @@ void initConfig(struct config *conf) } } -void reset_config(struct conf_item *conf_item) +static void reset_config_default(struct conf_item *conf_item) { if(conf_item->t == CONF_JSON_STRING_ARRAY) { diff --git a/src/config/config.h b/src/config/config.h index 0839c1b0..811f48b3 100644 --- a/src/config/config.h +++ b/src/config/config.h @@ -348,8 +348,6 @@ extern struct config config; // Defined in config.c void set_debug_flags(struct config *conf); void set_all_debug(struct config *conf, const bool status); -void initConfig(struct config *conf); -void reset_config(struct conf_item *conf_item); bool readFTLconf(struct config *conf, const bool rewrite); bool getLogFilePath(void); struct conf_item *get_conf_item(struct config *conf, const unsigned int n); diff --git a/src/config/legacy_reader.c b/src/config/legacy_reader.c index 8e64cfed..4a5b1605 100644 --- a/src/config/legacy_reader.c +++ b/src/config/legacy_reader.c @@ -66,6 +66,10 @@ bool getLogFilePathLegacy(struct config *conf, FILE *fp) // No option set => use default log location if(buffer == NULL) { + // Free previously allocated memory (if any) + if(conf->files.log.ftl.t == CONF_STRING_ALLOCATED) + free(conf->files.log.ftl.v.s); + // Use standard path if no custom path was obtained from the config file conf->files.log.ftl.v.s = strdup("/var/log/pihole/FTL.log"); conf->files.log.ftl.t = CONF_STRING_ALLOCATED; @@ -92,6 +96,10 @@ bool getLogFilePathLegacy(struct config *conf, FILE *fp) conf->files.log.ftl.v.s = NULL; conf->files.log.ftl.t = CONF_STRING; log_info("Using syslog facility"); + + // Free buffer + if(val_buffer != NULL) + free(val_buffer); } // Set string if memory allocation was successful and a value was read diff --git a/src/config/password.c b/src/config/password.c index f90bc4ad..526c1229 100644 --- a/src/config/password.c +++ b/src/config/password.c @@ -199,6 +199,14 @@ static char * __attribute__((malloc)) balloon_password(const char *password, // Build PHC string-like output (output string is 101 bytes long (measured)) char *output = calloc(128, sizeof(char)); + + if(output == NULL || salt_base64 == NULL || scratch_base64 == NULL) + { + log_err("Error while allocating memory for PHC string: %s", strerror(errno)); + goto clean_and_exit; + } + + // Generate PHC string int size = snprintf(output, 128, "$BALLOON-SHA256$v=1$s=%zu,t=%zu$%s$%s", s_cost, t_cost, @@ -213,11 +221,15 @@ static char * __attribute__((malloc)) balloon_password(const char *password, } clean_and_exit: - free(scratch); - free(salt_base64); - free(scratch_base64); + // Clean up + if(scratch != NULL) + free(scratch); + if(salt_base64 != NULL) + free(salt_base64); + if(scratch_base64 != NULL) + free(scratch_base64); - return output; + return output; // may be NULL on failure (unlikely) } // Parse a PHC string and return the parameters and hash @@ -653,6 +665,7 @@ bool set_and_check_password(struct conf_item *conf_item, const char *password) // Set item conf_item->v.s = pwhash; + conf_item->t = CONF_STRING_ALLOCATED; log_debug(DEBUG_CONFIG, "Set %s to \"%s\"", conf_item->k, conf_item->v.s); return true; diff --git a/src/config/setupVars.c b/src/config/setupVars.c index e0c54009..18fedab1 100644 --- a/src/config/setupVars.c +++ b/src/config/setupVars.c @@ -553,7 +553,7 @@ void importsetupVarsConf(void) get_conf_string_from_setupVars("DHCP_LEASETIME", &config.dhcp.leaseTime); // If the DHCP lease time is set to "24", it is interpreted as "24h". - // This is some relic from the past that may still be present in some + // This is some relict from the past that may still be present in some // setups if(strcmp(config.dhcp.leaseTime.v.s, "24") == 0) { From 358478e333b9bccaff2fbc18458f5c193a85bf20 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 22 Aug 2024 19:48:55 +0200 Subject: [PATCH 269/339] Ensure we free all memory in ngethostbyname() Signed-off-by: DL6ER --- src/resolve.c | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/src/resolve.c b/src/resolve.c index 0b2a9864..0f717b47 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -403,19 +403,23 @@ static char *__attribute__((malloc)) ngethostbyname(const int sock, const bool t answers[i].resource = (struct R_DATA*)(reader); reader = reader + sizeof(struct R_DATA); + // Read the answer and convert from network to host representation + answers[i].rdata = name_fromDNS(reader, buf, &stop); + reader = reader + stop; + // We only care about PTR answers and ignore all others const uint16_t rtype = ntohs(answers[i].resource->type); if(rtype != T_PTR) { log_debug(DEBUG_RESOLVER, "Answer %u is not of type PTR but %u (skipping)", i, rtype); + + // Skip this answer + free(answers[i].name); + free(answers[i].rdata); continue; } - // Read the answer and convert from network to host representation - answers[i].rdata = name_fromDNS(reader, buf, &stop); - reader = reader + stop; - name = (char *)answers[i].rdata; log_debug(DEBUG_RESOLVER, "Answer %u is PTR \"%s\" => \"%s\"", i, answers[i].name, answers[i].rdata); @@ -423,25 +427,18 @@ static char *__attribute__((malloc)) ngethostbyname(const int sock, const bool t // We break out of the loop if this is a valid hostname if(strlen(name) > 0 && valid_hostname(name, ipaddr)) { + free(answers[i].name); break; } else { // Discard this answer: free memory and set name to NULL - free(name); + free(answers[i].name); + free(answers[i].rdata); name = NULL; } } - // Free memory - for(uint16_t i = 0; i < min(ntohs(dns->ans_count), ArraySize(answers)); i++) - { - if(answers[i].name != NULL) - free(answers[i].name); - if(answers[i].rdata != NULL && (char*)answers[i].rdata != name) - free(answers[i].rdata); - } - if(name != NULL) { // We have a valid hostname, return it From 8571c99aae9f62a75b4276200b99cd8fc34329bf Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 26 Aug 2024 19:52:00 +0200 Subject: [PATCH 270/339] Restart FTL on change of webserver.api.cli_pw Signed-off-by: DL6ER --- src/config/config.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/config/config.c b/src/config/config.c index e4800b03..3dd12e31 100644 --- a/src/config/config.c +++ b/src/config/config.c @@ -1086,6 +1086,7 @@ void initConfig(struct config *conf) conf->webserver.api.cli_pw.k = "webserver.api.cli_pw"; conf->webserver.api.cli_pw.h = "Should FTL create a temporary CLI password? This password is stored in clear in /etc/pihole and can be used by the CLI (pihole ... commands) to authenticate against the API. Note that the password is only valid for the current session and regenerated on each FTL restart. Sessions initiated with this password cannot modify the Pi-hole configuration (change passwords, etc.) for security reasons but can still use the API to query data and manage lists."; conf->webserver.api.cli_pw.t = CONF_BOOL; + conf->webserver.api.cli_pw.f = FLAG_RESTART_FTL; conf->webserver.api.cli_pw.d.b = true; conf->webserver.api.cli_pw.c = validate_stub; // Only type-based checking From 42b626f36c9d50f05d68bada5731e9ce3d8dcc53 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 26 Aug 2024 21:40:27 +0200 Subject: [PATCH 271/339] Rename dns.cache.upstreamTTL -> dns.cache.upstreamBlockedTTL Signed-off-by: DL6ER --- src/api/docs/content/specs/config.yaml | 4 ++-- src/config/config.c | 10 +++++----- src/config/config.h | 2 +- src/dnsmasq_interface.c | 4 ++-- test/pihole.toml | 2 +- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/api/docs/content/specs/config.yaml b/src/api/docs/content/specs/config.yaml index 4e862601..2fdd110b 100644 --- a/src/api/docs/content/specs/config.yaml +++ b/src/api/docs/content/specs/config.yaml @@ -239,7 +239,7 @@ components: type: integer optimizer: type: integer - upstreamTTL: + upstreamBlockedTTL: type: integer revServers: type: array @@ -665,7 +665,7 @@ components: cache: size: 10000 optimizer: 3600 - upstreamTTL: 86400 + upstreamBlockedTTL: 86400 revServers: - "true,192.168.0.0/24,192.168.0.1,lan" blocking: diff --git a/src/config/config.c b/src/config/config.c index 0a4c0633..30123705 100644 --- a/src/config/config.c +++ b/src/config/config.c @@ -580,11 +580,11 @@ void initConfig(struct config *conf) conf->dns.cache.optimizer.d.i = 3600u; conf->dns.cache.optimizer.c = validate_stub; // Only type-based checking - conf->dns.cache.upstreamTTL.k = "dns.cache.upstreamTTL"; - conf->dns.cache.upstreamTTL.h = "This setting allows you to specify the TTL used for queries blocked upstream. Once the TTL expires, the query will be forwarded to the upstream server again to check if the block is still valid. Defaults to caching for one day (86400 seconds). Setting this value to zero disables caching of queries blocked upstream."; - conf->dns.cache.upstreamTTL.t = CONF_UINT; - conf->dns.cache.upstreamTTL.d.ui = 86400; - conf->dns.cache.upstreamTTL.c = validate_stub; // Only type-based checking + conf->dns.cache.upstreamBlockedTTL.k = "dns.cache.upstreamBlockedTTL"; + conf->dns.cache.upstreamBlockedTTL.h = "This setting allows you to specify the TTL used for queries blocked upstream. Once the TTL expires, the query will be forwarded to the upstream server again to check if the block is still valid. Defaults to caching for one day (86400 seconds). Setting this value to zero disables caching of queries blocked upstream."; + conf->dns.cache.upstreamBlockedTTL.t = CONF_UINT; + conf->dns.cache.upstreamBlockedTTL.d.ui = 86400; + conf->dns.cache.upstreamBlockedTTL.c = validate_stub; // Only type-based checking // sub-struct dns.blocking conf->dns.blocking.active.k = "dns.blocking.active"; diff --git a/src/config/config.h b/src/config/config.h index 2f3538ae..ad453812 100644 --- a/src/config/config.h +++ b/src/config/config.h @@ -147,7 +147,7 @@ struct config { struct { struct conf_item size; struct conf_item optimizer; - struct conf_item upstreamTTL; + struct conf_item upstreamBlockedTTL; } cache; struct { struct conf_item active; diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c index d2628100..b4b99253 100644 --- a/src/dnsmasq_interface.c +++ b/src/dnsmasq_interface.c @@ -1149,14 +1149,14 @@ static void set_dnscache_blockingstatus(DNSCacheData *dns_cache, enum domain_cli // Set expiration time for this cache entry (if applicable) // We set this only if not already set to avoid extending the TTL of an // existing entry - if(config.dns.cache.upstreamTTL.v.ui > 0 && + if(config.dns.cache.upstreamBlockedTTL.v.ui > 0 && dns_cache->expires == 0 && (new_status == UPSTREAM_BLOCKED_NXRA || new_status == UPSTREAM_BLOCKED_NULL || new_status == UPSTREAM_BLOCKED_IP)) { // Set expiration time for this cache entry - dns_cache->expires = time(NULL) + config.dns.cache.upstreamTTL.v.ui; + dns_cache->expires = time(NULL) + config.dns.cache.upstreamBlockedTTL.v.ui; } if(!config.debug.queries.v.b) diff --git a/test/pihole.toml b/test/pihole.toml index 5a712c92..3946aa9e 100644 --- a/test/pihole.toml +++ b/test/pihole.toml @@ -255,7 +255,7 @@ # the TTL expires, the query will be forwarded to the upstream server again to check # if the block is still valid. Defaults to caching for one day (86400 seconds). # Setting this value to zero disables caching of queries blocked upstream. - upstreamTTL = 86400 + upstreamBlockedTTL = 86400 [dns.blocking] # Should FTL block queries? From 453cdefb0d2c319f2030e7aa51b8bdee62195135 Mon Sep 17 00:00:00 2001 From: Dominik Date: Mon, 26 Aug 2024 21:59:41 +0200 Subject: [PATCH 272/339] Update src/config/setupVars.c Co-authored-by: Adam Warner Signed-off-by: Dominik --- src/config/setupVars.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config/setupVars.c b/src/config/setupVars.c index c5ff27de..07333439 100644 --- a/src/config/setupVars.c +++ b/src/config/setupVars.c @@ -320,7 +320,7 @@ static void get_conf_upstream_servers_from_setupVars(struct conf_item *conf_item cJSON *item = cJSON_CreateString(value); cJSON_AddItemToArray(conf_item->v.json, item); - log_info("setupVars.conf:PIHOLE_DNS_%u -> Setting %s[%u] = %s\n", + log_info("setupVars.conf:PIHOLE_DNS_%u -> Setting %s[%u] = %s", j, conf_item->k, j, item->valuestring); } From c09ce7c8c311af2c62dd997c4f27b60b3637fe3c Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 29 Aug 2024 18:44:19 +0200 Subject: [PATCH 273/339] Remove CLI password file after emptying it Signed-off-by: DL6ER --- src/config/password.c | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/config/password.c b/src/config/password.c index f90bc4ad..9c9c4dda 100644 --- a/src/config/password.c +++ b/src/config/password.c @@ -758,6 +758,13 @@ bool create_cli_password(void) bool remove_cli_password(void) { + // Remove the CLI password from memory (if allocated) + if(cli_password != NULL) + { + free(cli_password); + cli_password = NULL; + } + // Empty the CLI password file FILE *file = fopen(CLI_PW_FILE, "w"); if(file == NULL) @@ -769,8 +776,13 @@ bool remove_cli_password(void) // Close file fclose(file); - // Remove the CLI password from memory - free(cli_password); + // Remove the CLI password file from disk + // If the file does not exist, we returned above already + if(unlink(CLI_PW_FILE) < 0) + { + log_err("Failed to remove CLI password file: %s", strerror(errno)); + return false; + } log_debug(DEBUG_API, "CLI password removed"); return true; From 1e0bec5e2e1c1e337dbb519cd02d8ad54770cb17 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 31 Aug 2024 16:26:49 +0200 Subject: [PATCH 274/339] Use ftl-build:v2.7 builder for Actions Signed-off-by: DL6ER --- .github/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/Dockerfile b/.github/Dockerfile index 40d7a614..e8b05edb 100644 --- a/.github/Dockerfile +++ b/.github/Dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/pi-hole/ftl-build:v2.6 AS builder +FROM ghcr.io/pi-hole/ftl-build:v2.7 AS builder WORKDIR /app From 0fa865a29e45db31a94e976a54905a4aed0dd3cc Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 2 Sep 2024 06:49:53 +0200 Subject: [PATCH 275/339] Add GET /api/stats/summary -> .gravity.last_update timestamp exposing the last gravity update via the API Signed-off-by: DL6ER --- src/api/docs/content/specs/stats.yaml | 4 ++++ src/api/stats.c | 1 + src/database/gravity-db.c | 5 +++++ src/database/gravity-db.h | 2 ++ 4 files changed, 12 insertions(+) diff --git a/src/api/docs/content/specs/stats.yaml b/src/api/docs/content/specs/stats.yaml index b2dfba01..1ee83228 100644 --- a/src/api/docs/content/specs/stats.yaml +++ b/src/api/docs/content/specs/stats.yaml @@ -560,6 +560,10 @@ components: type: integer description: Number of domain on your Pi-hole's gravity list example: 104756 + last_update: + type: integer + description: Unix timestamp of last gravity update (may be `0` if unknown) + example: 1725194639 upstreams: type: object properties: diff --git a/src/api/stats.c b/src/api/stats.c index 60525d9c..ae98f9df 100644 --- a/src/api/stats.c +++ b/src/api/stats.c @@ -165,6 +165,7 @@ int api_stats_summary(struct ftl_conn *api) cJSON *gravity = JSON_NEW_OBJECT(); JSON_ADD_NUMBER_TO_OBJECT(gravity, "domains_being_blocked", num_gravity); + JSON_ADD_NUMBER_TO_OBJECT(gravity, "last_update", gravity_last_updated()); cJSON *json = JSON_NEW_OBJECT(); JSON_ADD_ITEM_TO_OBJECT(json, "queries", queries); diff --git a/src/database/gravity-db.c b/src/database/gravity-db.c index f6744b0b..a90ac7ef 100644 --- a/src/database/gravity-db.c +++ b/src/database/gravity-db.c @@ -2796,3 +2796,8 @@ bool gravity_updated(void) return changed; } + +time_t __attribute__((pure)) gravity_last_updated(void) +{ + return last_updated > 0 ? (time_t)last_updated : 0; +} diff --git a/src/database/gravity-db.h b/src/database/gravity-db.h index 7bded00c..996569e8 100644 --- a/src/database/gravity-db.h +++ b/src/database/gravity-db.h @@ -70,4 +70,6 @@ bool gravityDB_delFromTable(const enum gravity_list_type listtype, const cJSON* bool gravityDB_edit_groups(const enum gravity_list_type listtype, cJSON *groups, const tablerow *row, const char **message); +time_t gravity_last_updated(void) __attribute__((pure)); + #endif //GRAVITY_H From 68a14bf12cbe788e89a33804c52a778ebc87a0da Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 31 Aug 2024 18:34:10 +0200 Subject: [PATCH 276/339] Use total number of *configured* (= existing) cores rather then the number of available (= online and assigned) cores when computing the CPU utilization of the system Signed-off-by: DL6ER --- src/api/info.c | 8 ++++++-- src/gc.c | 4 ++-- src/webserver/webserver.c | 7 +++++++ 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/src/api/info.c b/src/api/info.c index 6256d6a0..7daa83b8 100644 --- a/src/api/info.c +++ b/src/api/info.c @@ -12,7 +12,7 @@ #include "webserver/http-common.h" #include "webserver/json_macros.h" #include "api/api.h" -// sysinfo() +// sysinfo(), get_nprocs_conf() #include // get_blockingstatus() #include "config/setupVars.h" @@ -159,7 +159,11 @@ int api_info_database(struct ftl_conn *api) int get_system_obj(struct ftl_conn *api, cJSON *system) { - const int nprocs = get_nprocs(); + // Use total number of processors + // This difference is important for virtualized systems where the number + // of available (= online) processors can be lower than the total number + // (= configured) of processors + const int nprocs = get_nprocs_conf(); struct sysinfo info; if(sysinfo(&info) != 0) return send_json_error(api, 500, "error", strerror(errno), NULL); diff --git a/src/gc.c b/src/gc.c index f514b1b0..613ac6f6 100644 --- a/src/gc.c +++ b/src/gc.c @@ -262,8 +262,8 @@ static void check_load(void) if (getloadavg(load, 3) == -1) return; - // Get number of CPU cores - const int nprocs = get_nprocs(); + // Get total number of CPU cores + const int nprocs = get_nprocs_conf(); // Warn if 15 minute average of load exceeds number of available // processors diff --git a/src/webserver/webserver.c b/src/webserver/webserver.c index 6e3a3598..745a63b7 100644 --- a/src/webserver/webserver.c +++ b/src/webserver/webserver.c @@ -396,6 +396,13 @@ void http_init(void) // send no referrer information. // The latter four headers are set as expected by https://securityheaders.io char num_threads[3] = { 0 }; + // Use 16 threads if more than 8 cores are available, otherwise use + // 2*cores. This is to prevent overloading the system with too many + // threads. + // We use the number of available (= online) cores which may be less + // than the total number of cores in the system, e.g., if a + // virtualization environment is used and fewer cores are assigned to + // the VM than are available on the host. sprintf(num_threads, "%d", get_nprocs() > 8 ? 16 : 2*get_nprocs()); const char *options[] = { "document_root", config.webserver.paths.webroot.v.s, From ecc05dccf51459503390f5c6a1997325cd3640da Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 2 Sep 2024 19:14:46 +0200 Subject: [PATCH 277/339] Remove appending ?target=... when redirecting to the login page. web-PR #3124 removed the redirection feature from the login page Signed-off-by: DL6ER --- src/webserver/lua_web.c | 48 ++++------------------------------------- 1 file changed, 4 insertions(+), 44 deletions(-) diff --git a/src/webserver/lua_web.c b/src/webserver/lua_web.c index bef0cab1..1d53f9bb 100644 --- a/src/webserver/lua_web.c +++ b/src/webserver/lua_web.c @@ -74,7 +74,6 @@ int request_handler(struct mg_connection *conn, void *cbdata) /* Handler may access the request info using mg_get_request_info */ const struct mg_request_info *req_info = mg_get_request_info(conn); - const size_t uri_raw_len = strlen(req_info->local_uri_raw); // Build minimal api struct to check authentication struct ftl_conn api = { 0 }; @@ -122,50 +121,11 @@ int request_handler(struct mg_connection *conn, void *cbdata) // Check if the user is authenticated if(!authorized) { - // Append query string to target - char *target = NULL; - if(req_info->query_string != NULL) - { - target = calloc(uri_raw_len + strlen(req_info->query_string) + 2u, sizeof(char)); - strcpy(target, req_info->local_uri_raw); - strcat(target, "?"); - strcat(target, req_info->query_string); - } - else - { - target = strdup(req_info->local_uri_raw); - } - if(target == NULL) - { - log_err("Error allocating memory for redirection target"); - return send_json_error(&api, 500, - "internal_error", - "Internal server error", - "Cannot allocate memory for redirection target"); - } - - // Encode target string - const size_t encoded_target_len = strlen(target) * 3u + 1u; - char *encoded_target = calloc(encoded_target_len, sizeof(char)); - if(encoded_target == NULL) - { - log_err("Error allocating memory for encoded redirection target"); - return send_json_error(&api, 500, - "internal_error", - "Internal server error", - "Cannot allocate memory for encoded redirection target"); - } - - // Encode target string - mg_url_encode(target, encoded_target, encoded_target_len); - free(target); - // User is not authenticated, redirect to login page - log_web("Authentication required, redirecting to %slogin?target=%s", - config.webserver.paths.webhome.v.s, encoded_target); - mg_printf(conn, "HTTP/1.1 302 Found\r\nLocation: %slogin?target=%s\r\n\r\n", - config.webserver.paths.webhome.v.s, encoded_target); - free(encoded_target); + log_web("Authentication required, redirecting to %slogin", + config.webserver.paths.webhome.v.s); + mg_printf(conn, "HTTP/1.1 302 Found\r\nLocation: %slogin\r\n\r\n", + config.webserver.paths.webhome.v.s); return 302; } } From ae426776fd3c1ce0fc23bf456d7c3f008e7874ec Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Mon, 2 Sep 2024 22:24:49 +0100 Subject: [PATCH 278/339] Remove development-v6 references from dependabot Signed-off-by: Adam Warner --- .github/dependabot.yml | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 2f025b2e..ff4ebf1b 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -17,23 +17,3 @@ updates: github_action-dependencies: patterns: - "*" - -# As above, but for development-v6 -- package-ecosystem: github-actions - directory: "/" - schedule: - interval: weekly - day: saturday - time: "10:00" - open-pull-requests-limit: 10 - target-branch: development-v6 - reviewers: - - "pi-hole/ftl-maintainers" - pull-request-branch-name: - # Separate sections of the branch name with a hyphen - separator: "-" - groups: - github_action-dependencies: - patterns: - - "*" - From b7dfdb4ef82c1c44fb151a753f83927e220fec02 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 5 Sep 2024 18:04:42 +0200 Subject: [PATCH 279/339] Add new queries per second (QPS) metric exposed via GET /api/stats/summary as .queries.frequency and /api/info/ftl as .query_frequency. The QPS value is averaged over 30 seconds Signed-off-by: DL6ER --- src/FTL.h | 4 +++ src/api/docs/content/specs/info.yaml | 4 +++ src/api/docs/content/specs/stats.yaml | 4 +++ src/api/info.c | 2 ++ src/api/stats.c | 2 ++ src/dnsmasq_interface.c | 3 ++ src/shmem.c | 43 +++++++++++++++++++++++++++ src/shmem.h | 7 +++++ 8 files changed, 69 insertions(+) diff --git a/src/FTL.h b/src/FTL.h index 5a76e979..d6a7f1c3 100644 --- a/src/FTL.h +++ b/src/FTL.h @@ -143,6 +143,10 @@ // Default: 2592000 (once per month) #define DATABASE_MACVENDOR_INTERVAL 2592000 +// Over how many seconds should the query-per-second (QPS) value be averaged? +// Default: 30 (seconds) +#define QPS_AVGLEN 30 + // Use out own syscalls handling functions that will detect possible errors // and report accordingly in the log. This will make debugging FTL crash // caused by insufficient memory or by code bugs (not properly dealing diff --git a/src/api/docs/content/specs/info.yaml b/src/api/docs/content/specs/info.yaml index dadaee27..d76dbd6a 100644 --- a/src/api/docs/content/specs/info.yaml +++ b/src/api/docs/content/specs/info.yaml @@ -721,6 +721,10 @@ components: type: integer description: Currently used privacy level example: 0 + query_frequency: + type: number + description: Average number of queries per second + example: 1.1 clients: type: object properties: diff --git a/src/api/docs/content/specs/stats.yaml b/src/api/docs/content/specs/stats.yaml index 1ee83228..3af53b19 100644 --- a/src/api/docs/content/specs/stats.yaml +++ b/src/api/docs/content/specs/stats.yaml @@ -338,6 +338,10 @@ components: type: integer description: Number of queries replied to from cache or local configuration example: 9765 + frequency: + type: number + description: Average number of queries per second + example: 1.1 types: type: object description: Number of individual queries diff --git a/src/api/info.c b/src/api/info.c index 7daa83b8..ce7a7a8d 100644 --- a/src/api/info.c +++ b/src/api/info.c @@ -548,6 +548,7 @@ static int get_ftl_obj(struct ftl_conn *api, cJSON *ftl) const int db_denied = counters->database.domains.denied; const int clients_total = counters->clients; const int privacylevel = config.misc.privacylevel.v.privacy_level; + const double qps = get_qps(); // unique_clients: count only clients that have been active within the most recent 24 hours int activeclients = 0; @@ -575,6 +576,7 @@ static int get_ftl_obj(struct ftl_conn *api, cJSON *ftl) JSON_ADD_ITEM_TO_OBJECT(ftl, "database", database); JSON_ADD_NUMBER_TO_OBJECT(ftl, "privacy_level", privacylevel); + JSON_ADD_NUMBER_TO_OBJECT(ftl, "query_frequency", qps); cJSON *clients = JSON_NEW_OBJECT(); JSON_ADD_NUMBER_TO_OBJECT(clients, "total",clients_total); diff --git a/src/api/stats.c b/src/api/stats.c index ae98f9df..ccaffc61 100644 --- a/src/api/stats.c +++ b/src/api/stats.c @@ -143,6 +143,8 @@ int api_stats_summary(struct ftl_conn *api) JSON_ADD_NUMBER_TO_OBJECT(queries, "forwarded", forwarded); JSON_ADD_NUMBER_TO_OBJECT(queries, "cached", cached); + JSON_ADD_NUMBER_TO_OBJECT(queries, "frequency", get_qps()); + cJSON *types = JSON_NEW_OBJECT(); int ret = get_query_types_obj(api, types); if(ret != 0) diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c index b4b99253..d9be227d 100644 --- a/src/dnsmasq_interface.c +++ b/src/dnsmasq_interface.c @@ -664,6 +664,9 @@ bool _FTL_new_query(const unsigned int flags, const char *name, return false; } + // Update rolling window of queries per second + update_qps(querytimestamp); + // Interface name is only available for regular queries, not for // automatically generated DNSSEC queries const char *interface = internal_query ? "-" : next_iface.name; diff --git a/src/shmem.c b/src/shmem.c index 3063e1de..3f1c29c1 100644 --- a/src/shmem.c +++ b/src/shmem.c @@ -1206,3 +1206,46 @@ int __attribute__((pure)) is_shm_fd(const int fd) // Not found return 0; } + +// Update queries per second (qps) value +// This is done in shared memory to allow for both UDP and TCP workers to +// contribute. +void update_qps(const double timestamp) +{ + // Get the timeslot for the current timestamp + const unsigned int slot = (unsigned int)timestamp % QPS_AVGLEN; + + // Check if the timestamp is in the same slot as the last one + if(shmSettings->qps.last != slot) + { + // Reset all the slots in between + // This is relevant if less than one query per second is + // received and the intermediate slots are not updated + for(unsigned int i = (shmSettings->qps.last + 1) % QPS_AVGLEN; i != slot; i = (i + 1) % QPS_AVGLEN) + shmSettings->qps.buf[i] = 0; + + // Reset the current slot + shmSettings->qps.buf[slot] = 0; + + // Update the last slot index + shmSettings->qps.last = slot; + } + + // Add the query + shmSettings->qps.buf[slot]++; +} + +// Compute queries per second (qps) value +double __attribute__((pure)) get_qps(void) +{ + // Compute the arithmetic mean of all slots + // 1 N + // QPS = --- Σ buf[i] + // N i=0 + // + double qps = 0.0; + for(unsigned int i = 0; i < QPS_AVGLEN; i++) + qps += shmSettings->qps.buf[i]; + + return qps / QPS_AVGLEN; +} diff --git a/src/shmem.h b/src/shmem.h index ad35ebee..2492f986 100644 --- a/src/shmem.h +++ b/src/shmem.h @@ -31,6 +31,10 @@ typedef struct { pid_t pid; unsigned int global_shm_counter; unsigned int next_str_pos; + struct { + unsigned int last; + unsigned int buf[QPS_AVGLEN]; + } qps; } ShmSettings; typedef struct { @@ -145,4 +149,7 @@ void set_per_client_regex(const int clientID, const int regexID, const bool valu // Used in dnsmasq/utils.c int is_shm_fd(const int fd); +void update_qps(const double timestamp); +double get_qps(void) __attribute__((pure)); + #endif //SHARED_MEMORY_SERVER_H From 19175b10f73366ba6e4ee2bb567c900e4650886a Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 9 Sep 2024 17:44:19 +0200 Subject: [PATCH 280/339] Migrate legacy files (setupVars.conf, pihole-FTL.conf and custom.list) into /etc/pihole/migration_backup_v6 instead of leaving them inside /etc/pihole Signed-off-by: DL6ER --- src/config/config.c | 2 +- src/config/dnsmasq_config.c | 2 +- src/config/dnsmasq_config.h | 1 + src/config/setupVars.c | 18 +++++------------- 4 files changed, 8 insertions(+), 15 deletions(-) diff --git a/src/config/config.c b/src/config/config.c index 861ac4a9..5fe183d0 100644 --- a/src/config/config.c +++ b/src/config/config.c @@ -1627,7 +1627,7 @@ bool readFTLconf(struct config *conf, const bool rewrite) const char *path = ""; if((path = readFTLlegacy(conf)) != NULL) { - const char *target = "/etc/pihole/pihole-FTL.conf.bck"; + const char *target = "/etc/pihole/migration_backup_v6/pihole-FTL.conf"; log_info("Moving %s to %s", path, target); if(rename(path, target) != 0) log_warn("Unable to move %s to %s: %s", path, target, strerror(errno)); diff --git a/src/config/dnsmasq_config.c b/src/config/dnsmasq_config.c index abf076ca..e4c305ec 100644 --- a/src/config/dnsmasq_config.c +++ b/src/config/dnsmasq_config.c @@ -917,7 +917,7 @@ bool read_legacy_custom_hosts_config(void) { // Check if file exists, if not, there is nothing to do const char *path = DNSMASQ_CUSTOM_LIST_LEGACY; - const char *target = DNSMASQ_CUSTOM_LIST_LEGACY".bck"; + const char *target = DNSMASQ_CUSTOM_LIST_LEGACY_TARGET; if(!file_exists(path)) return true; diff --git a/src/config/dnsmasq_config.h b/src/config/dnsmasq_config.h index 4a9d9fad..2b40042a 100644 --- a/src/config/dnsmasq_config.h +++ b/src/config/dnsmasq_config.h @@ -29,6 +29,7 @@ bool write_custom_list(void); #define DNSMASQ_HOSTSDIR "/etc/pihole/hosts" #define DNSMASQ_CUSTOM_LIST DNSMASQ_HOSTSDIR"/custom.list" #define DNSMASQ_CUSTOM_LIST_LEGACY "/etc/pihole/custom.list" +#define DNSMASQ_CUSTOM_LIST_LEGACY_TARGET "/etc/pihole/migration_backup_v6/custom.list" #define DHCPLEASESFILE "/etc/pihole/dhcp.leases" #endif //DNSMASQ_CONFIG_H diff --git a/src/config/setupVars.c b/src/config/setupVars.c index 2d8bd48d..cd5df0ea 100644 --- a/src/config/setupVars.c +++ b/src/config/setupVars.c @@ -591,20 +591,12 @@ void importsetupVarsConf(void) // Ports may be temporarily stored when importing a legacy Teleporter v5 file get_conf_string_from_setupVars("WEB_PORTS", &config.webserver.port); - // Move the setupVars.conf file to setupVars.conf.old - char *old_setupVars = calloc(strlen(config.files.setupVars.v.s) + 5, sizeof(char)); - if(old_setupVars == NULL) - { - log_warn("Could not allocate memory for old_setupVars"); - return; - } - strcpy(old_setupVars, config.files.setupVars.v.s); - strcat(old_setupVars, ".old"); - if(rename(config.files.setupVars.v.s, old_setupVars) != 0) - log_warn("Could not move %s to %s", config.files.setupVars.v.s, old_setupVars); + // Move the setupVars.conf file to the migration directory + const char *setupVars_target = "/etc/pihole/migration_backup_v6/setupVars.conf"; + if(rename(config.files.setupVars.v.s, setupVars_target) != 0) + log_warn("Could not move %s to %s", config.files.setupVars.v.s, setupVars_target); else - log_info("Moved %s to %s", config.files.setupVars.v.s, old_setupVars); - free(old_setupVars); + log_info("Moved %s to %s", config.files.setupVars.v.s, setupVars_target); log_info("Migration complete"); } From f29749defbf5ed9a08a1e35d144832c856510d8a Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 9 Sep 2024 17:51:54 +0200 Subject: [PATCH 281/339] Ensure calls to free(ptr) set ptr to NULL afterwards. This guarantees there can be no double-free corruptions in the future. Instead, on trying to free a pointer which has already been free'd (or was never allocated!), we instead print a warning to the log file and continue normal operation Signed-off-by: DL6ER --- src/FTL.h | 2 +- src/config/toml_helper.c | 20 ++++++++++---------- src/syscalls/free.c | 14 ++++---------- src/syscalls/syscalls.h | 2 +- src/webserver/webserver.c | 4 +++- 5 files changed, 19 insertions(+), 23 deletions(-) diff --git a/src/FTL.h b/src/FTL.h index 5a76e979..292eb80b 100644 --- a/src/FTL.h +++ b/src/FTL.h @@ -148,7 +148,7 @@ // caused by insufficient memory or by code bugs (not properly dealing // with NULL pointers) much easier. #undef strdup // strdup() is a macro in itself, it needs special handling -#define free(ptr) FTLfree((void**)&ptr, __FILE__, __FUNCTION__, __LINE__) +#define free(ptr) { FTLfree(ptr, __FILE__, __FUNCTION__, __LINE__); ptr = NULL; } #define strdup(str_in) FTLstrdup(str_in, __FILE__, __FUNCTION__, __LINE__) #define calloc(numer_of_elements, element_size) FTLcalloc(numer_of_elements, element_size, __FILE__, __FUNCTION__, __LINE__) #define realloc(ptr, new_size) FTLrealloc(ptr, new_size, __FILE__, __FUNCTION__, __LINE__) diff --git a/src/config/toml_helper.c b/src/config/toml_helper.c index dabc1f59..37cf792a 100644 --- a/src/config/toml_helper.c +++ b/src/config/toml_helper.c @@ -543,7 +543,7 @@ void readTOMLvalue(struct conf_item *conf_item, const char* key, toml_table_t *t } case CONF_ENUM_PTR_TYPE: { - const toml_datum_t val = toml_string_in(toml, key); + toml_datum_t val = toml_string_in(toml, key); if(val.ok) { const int ptr_type = get_ptr_type_val(val.u.s); @@ -559,7 +559,7 @@ void readTOMLvalue(struct conf_item *conf_item, const char* key, toml_table_t *t } case CONF_ENUM_BUSY_TYPE: { - const toml_datum_t val = toml_string_in(toml, key); + toml_datum_t val = toml_string_in(toml, key); if(val.ok) { const int busy_reply = get_busy_reply_val(val.u.s); @@ -575,7 +575,7 @@ void readTOMLvalue(struct conf_item *conf_item, const char* key, toml_table_t *t } case CONF_ENUM_BLOCKING_MODE: { - const toml_datum_t val = toml_string_in(toml, key); + toml_datum_t val = toml_string_in(toml, key); if(val.ok) { const int blocking_mode = get_blocking_mode_val(val.u.s); @@ -591,7 +591,7 @@ void readTOMLvalue(struct conf_item *conf_item, const char* key, toml_table_t *t } case CONF_ENUM_REFRESH_HOSTNAMES: { - const toml_datum_t val = toml_string_in(toml, key); + toml_datum_t val = toml_string_in(toml, key); if(val.ok) { const int refresh_hostnames = get_refresh_hostnames_val(val.u.s); @@ -607,7 +607,7 @@ void readTOMLvalue(struct conf_item *conf_item, const char* key, toml_table_t *t } case CONF_ENUM_LISTENING_MODE: { - const toml_datum_t val = toml_string_in(toml, key); + toml_datum_t val = toml_string_in(toml, key); if(val.ok) { const int listeningMode = get_listeningMode_val(val.u.s); @@ -623,7 +623,7 @@ void readTOMLvalue(struct conf_item *conf_item, const char* key, toml_table_t *t } case CONF_ENUM_WEB_THEME: { - const toml_datum_t val = toml_string_in(toml, key); + toml_datum_t val = toml_string_in(toml, key); if(val.ok) { const int web_theme = get_web_theme_val(val.u.s); @@ -639,7 +639,7 @@ void readTOMLvalue(struct conf_item *conf_item, const char* key, toml_table_t *t } case CONF_ENUM_TEMP_UNIT: { - const toml_datum_t val = toml_string_in(toml, key); + toml_datum_t val = toml_string_in(toml, key); if(val.ok) { const int temp_unit = get_temp_unit_val(val.u.s); @@ -665,7 +665,7 @@ void readTOMLvalue(struct conf_item *conf_item, const char* key, toml_table_t *t case CONF_STRUCT_IN_ADDR: { struct in_addr addr4 = { 0 }; - const toml_datum_t val = toml_string_in(toml, key); + toml_datum_t val = toml_string_in(toml, key); if(val.ok) { if(strlen(val.u.s) == 0) @@ -686,7 +686,7 @@ void readTOMLvalue(struct conf_item *conf_item, const char* key, toml_table_t *t case CONF_STRUCT_IN6_ADDR: { struct in6_addr addr6 = { 0 }; - const toml_datum_t val = toml_string_in(toml, key); + toml_datum_t val = toml_string_in(toml, key); if(val.ok) { if(strlen(val.u.s) == 0) @@ -717,7 +717,7 @@ void readTOMLvalue(struct conf_item *conf_item, const char* key, toml_table_t *t for(unsigned int i = 0; i < nelem; i++) { // Get string from TOML - const toml_datum_t d = toml_string_at(array, i); + toml_datum_t d = toml_string_at(array, i); if(!d.ok) { log_warn("Config %s is an invalid array (found at index %u)", conf_item->k, i); diff --git a/src/syscalls/free.c b/src/syscalls/free.c index 360170ea..73c9c905 100644 --- a/src/syscalls/free.c +++ b/src/syscalls/free.c @@ -13,26 +13,20 @@ #include "log.h" #undef free -void FTLfree(void **ptr, const char *file, const char *func, const int line) +bool FTLfree(void *ptr, const char *file, const char *func, const int line) { // The free() function frees the memory space pointed to by ptr, which // must have been returned by a previous call to malloc(), calloc(), or // realloc(). Otherwise, or if free(ptr) has already been called before, // undefined behavior occurs. If ptr is NULL, no operation is performed. if(ptr == NULL) - { - log_warn("Trying to free NULL memory location in %s() (%s:%i)", func, file, line); - return; - } - if(*ptr == NULL) { log_warn("Trying to free NULL pointer in %s() (%s:%i)", func, file, line); - return; + return false; } // Actually free the memory - free(*ptr); + free(ptr); - // Set the pointer to NULL - *ptr = NULL; + return true; } diff --git a/src/syscalls/syscalls.h b/src/syscalls/syscalls.h index 77e7725c..02024a84 100644 --- a/src/syscalls/syscalls.h +++ b/src/syscalls/syscalls.h @@ -14,7 +14,7 @@ char *FTLstrdup(const char *src, const char *file, const char *func, const int line) __attribute__((malloc)); void *FTLcalloc(size_t n, size_t size, const char *file, const char *func, const int line) __attribute__((malloc)) __attribute__((alloc_size(1,2))); void *FTLrealloc(void *ptr_in, size_t size, const char *file, const char *func, const int line) __attribute__((alloc_size(2))); -void FTLfree(void **ptr, const char*file, const char *func, const int line); +bool FTLfree(void *ptr, const char*file, const char *func, const int line); int FTLfallocate(const int fd, const off_t offset, const off_t len, const char *file, const char *func, const int line); diff --git a/src/webserver/webserver.c b/src/webserver/webserver.c index 745a63b7..b3b05b94 100644 --- a/src/webserver/webserver.c +++ b/src/webserver/webserver.c @@ -303,6 +303,7 @@ unsigned short get_api_string(char **buf, const bool domain) if(this_len < 0) { log_err("Failed to append API URL to buffer: %s", strerror(errno)); + free(api_str); return 0; } @@ -311,6 +312,7 @@ unsigned short get_api_string(char **buf, const bool domain) if((size_t)this_len >= bufsz - len - 1) { log_err("API URL buffer too small!"); + free(api_str); return 0; } @@ -318,8 +320,8 @@ unsigned short get_api_string(char **buf, const bool domain) if(memmem(*buf, len, api_str, this_len) != NULL) { // This string is already present, so skip it - free(api_str); log_debug(DEBUG_API, "Skipping duplicate API URL: %s", api_str); + free(api_str); continue; } From c0aaad3a56ecf876f7882fb51c50ed0b1c4731bf Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 10 Sep 2024 20:29:54 +0200 Subject: [PATCH 282/339] Do not free allocated hash when we are still using it Signed-off-by: DL6ER --- src/config/password.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/config/password.c b/src/config/password.c index ee6e102f..ddae3856 100644 --- a/src/config/password.c +++ b/src/config/password.c @@ -440,7 +440,6 @@ enum password_result verify_password(const char *password, const char *pwhash, c config.webserver.api.pwhash.v.s = new_hash; config.webserver.api.pwhash.t = CONF_STRING_ALLOCATED; writeFTLtoml(true); - free(new_hash); } // Successful logins do not count against rate-limiting From 621d11697d50964831526d75ad2d8fbf307203f1 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 10 Sep 2024 22:15:37 +0200 Subject: [PATCH 283/339] Fix REV_SERVER parsing when importing setupVars.conf from v5 Pi-holes Signed-off-by: DL6ER --- src/config/setupVars.c | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/config/setupVars.c b/src/config/setupVars.c index 2d8bd48d..53481d86 100644 --- a/src/config/setupVars.c +++ b/src/config/setupVars.c @@ -122,7 +122,6 @@ static void get_conf_bool_from_setupVars(const char *key, struct conf_item *conf static void get_revServer_from_setupVars(void) { - char *active = NULL; char *cidr = NULL; char *target = NULL; char *domain = NULL; @@ -144,7 +143,7 @@ static void get_revServer_from_setupVars(void) clearSetupVarsArray(); return; } - active = strdup(active_str); + bool active = strcasecmp(active_str, "true") == 0; // Free memory, harmless to call if read_setupVarsconf() didn't return a result clearSetupVarsArray(); @@ -186,16 +185,16 @@ static void get_revServer_from_setupVars(void) clearSetupVarsArray(); // Only add the entry if all values are present and active - if(active != NULL && cidr != NULL && target != NULL && domain != NULL) + if(cidr != NULL && target != NULL && domain != NULL) { // Build comma-separated string of all values // 9 = 3 commas, "true/false", and null terminator char *old = calloc(strlen(cidr) + strlen(target) + strlen(domain) + 9, sizeof(char)); - if(old) + if(old != NULL) { // Add to new config // active is always true as we only add active entries - sprintf(old, "%s,%s,%s,%s", active_str, cidr, target, domain); + sprintf(old, "%s,%s,%s,%s", active ? "true" : "false", cidr, target, domain); cJSON_AddItemToArray(config.dns.revServers.v.json, cJSON_CreateString(old)); // Parameter present in setupVars.conf @@ -211,8 +210,6 @@ static void get_revServer_from_setupVars(void) } // Free memory - if(active != NULL) - free(active); if(cidr != NULL) free(cidr); if(target != NULL) From cc4a6ae44ed6cc7d0317de773950b7d72e4c9709 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 10 Sep 2024 22:23:26 +0200 Subject: [PATCH 284/339] Prevent endless resarting loop on crashes before the DNS resolver is fully seeded (moid was not set) Signed-off-by: DL6ER --- src/signals.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/signals.c b/src/signals.c index 392aff3a..35971f0d 100644 --- a/src/signals.c +++ b/src/signals.c @@ -29,7 +29,7 @@ #define BINARY_NAME "pihole-FTL" volatile sig_atomic_t killed = 0; -static volatile pid_t mpid = -1; +static volatile pid_t mpid = 0; static time_t FTLstarttime = 0; volatile int exit_code = EXIT_SUCCESS; @@ -253,7 +253,7 @@ static void __attribute__((noreturn)) signal_handler(int sig, siginfo_t *si, voi log_info("Thank you for helping us to improve our FTL engine!"); // Terminate main process if crash happened in a TCP worker - if(mpid != getpid()) + if(main_pid() != getpid()) { // This is a forked process log_info("Asking parent pihole-FTL (PID %i) to shut down", (int)mpid); @@ -474,7 +474,7 @@ void handle_realtime_signals(void) // Return PID of the main FTL process pid_t main_pid(void) { - if(mpid > -1) + if(mpid > 0) // Has already been set return mpid; else From dc3e1eb108435b94a717d67455012948688a8457 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 10 Sep 2024 23:27:13 +0200 Subject: [PATCH 285/339] Fix gzip help text Signed-off-by: DL6ER --- src/args.c | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/args.c b/src/args.c index a162aeca..03b0d3b1 100644 --- a/src/args.c +++ b/src/args.c @@ -1015,12 +1015,17 @@ void parse_args(int argc, char* argv[]) printf("%sEmbedded GZIP un-/compressor:%s\n", yellow, normal); printf(" A simple but fast in-memory gzip compressor\n\n"); - printf(" Usage: %spihole-FTL --compress %sinfile %s[outfile]%s\n", green, cyan, purple, normal); - printf(" Usage: %spihole-FTL --uncompress %sinfile %s[outfile]%s\n\n", green, cyan, purple, normal); - printf(" - %sinfile%s is the file to be compressed.\n", cyan, normal); + printf(" Usage: %spihole-FTL --gzip %sinfile %s[outfile]%s\n\n", green, cyan, purple, normal); + printf(" - %sinfile%s is the file to be processed. If the filename ends\n", cyan, normal); + printf(" in %s.gz%s, FTL will uncompress, otherwise it will compress\n\n", yellow, normal); printf(" - %s[outfile]%s is the optional target. If omitted, FTL will\n", purple, normal); - printf(" %s--compress%s: use the %sinfile%s and append %s.gz%s at the end\n", green, normal, cyan, normal, cyan, normal); - printf(" %s--uncompress%s: use the %sinfile%s and remove %s.gz%s at the end\n\n", green, normal, cyan, normal, cyan, normal); + printf(" - input is gz: use %sinfile%s.gz%s and remove %s.gz%s from the end\n", cyan, yellow, normal, purple, normal); + printf(" - otherwise: use %sinfile%s and append %s.gz%s at the end\n\n", cyan, normal, purple, normal); + printf(" Examples:\n"); + printf(" - %spihole-FTL --gzip %sfile.txt%s\n", green, cyan, normal); + printf(" compresses %sfile.txt%s to %sfile.txt.gz%s\n\n", cyan, normal, cyan, normal); + printf(" - %spihole-FTL --gzip %sfile.txt.gz%s\n", green, cyan, normal); + printf(" uncompresses %sfile.txt.gz%s to %sfile.txt%s\n\n", cyan, normal, cyan, normal); printf("%sTeleporter:%s\n", yellow, normal); printf("\t%s--teleporter%s Create a Teleporter archive in the\n", green, normal); From 7b60775fdc8c8c6a39cc756427d12f68810ec8cb Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 21 Feb 2024 19:48:40 +0100 Subject: [PATCH 286/339] Add missing import instruction for CNAME records file (if present in the archive) Signed-off-by: DL6ER --- src/api/teleporter.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/api/teleporter.c b/src/api/teleporter.c index b202cf5f..a8542362 100644 --- a/src/api/teleporter.c +++ b/src/api/teleporter.c @@ -736,6 +736,9 @@ static int process_received_tar_gz(struct ftl_conn *api, struct upload_data *dat // i = 3 .archive_name = "setupVars.conf", .destination = config.files.setupVars.v.s + },{ + .archive_name = "dnsmasq.d/05-pihole-custom-cname.conf", + .destination = DNSMASQ_CNAMES } }; for(size_t i = 0; i < sizeof(extract_files) / sizeof(*extract_files); i++) From 1f264eb6d526278364a7d8e5d6eaed12eba61c27 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 12 Sep 2024 17:11:47 +0200 Subject: [PATCH 287/339] Clarify exit messages Signed-off-by: DL6ER --- src/config/setupVars.c | 2 +- src/daemon.c | 5 ++++- src/main.c | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/config/setupVars.c b/src/config/setupVars.c index 57dfd818..e7e3a55c 100644 --- a/src/config/setupVars.c +++ b/src/config/setupVars.c @@ -595,7 +595,7 @@ void importsetupVarsConf(void) else log_info("Moved %s to %s", config.files.setupVars.v.s, setupVars_target); - log_info("Migration complete"); + log_info("setupVars.conf migration complete"); } char* __attribute__((pure)) find_equals(char *s) diff --git a/src/daemon.c b/src/daemon.c index b2204b6e..9c0e641a 100644 --- a/src/daemon.c +++ b/src/daemon.c @@ -401,7 +401,10 @@ void cleanup(const int ret) char buffer[42] = { 0 }; format_time(buffer, 0, timer_elapsed_msec(EXIT_TIMER)); - log_info("########## FTL terminated after%s (code %i)! ##########", buffer, ret); + if(ret == RESTART_FTL_CODE) + log_info("########## FTL terminated after%s (internal restart)! ##########", buffer); + else + log_info("########## FTL terminated after%s (code %i)! ##########", buffer, ret); } static float last_clock = 0.0f; diff --git a/src/main.c b/src/main.c index 518a64fa..c7f288c3 100644 --- a/src/main.c +++ b/src/main.c @@ -139,7 +139,7 @@ int main (int argc, char *argv[]) sleepms(100); } - log_info("Shutting down... // exit code %d // jmpret %d", exit_code, jmpret); + log_debug(DEBUG_ANY, "Shutting down... // exit code %d // jmpret %d", exit_code, jmpret); // Extra grace time is needed as dnsmasq script-helpers and the API may not // be terminating immediately sleepms(250); From db0054bdb6e0e1a51b2e526b4117d8f402250f0c Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 12 Sep 2024 17:13:12 +0200 Subject: [PATCH 288/339] Restart FTL after successful Teleporter import and provide more detailed log messages. Also reduce code duplication by defining a common restart_ftl() routine Signed-off-by: DL6ER --- src/api/action.c | 5 +---- src/api/api.c | 7 +------ src/api/config.c | 9 +++++++++ src/api/teleporter.c | 7 +++++++ src/config/config.c | 10 ++++------ src/ntp/client.c | 8 ++------ src/signals.c | 8 ++++++++ src/signals.h | 1 + src/webserver/http-common.h | 3 ++- src/zip/teleporter.c | 2 ++ 10 files changed, 37 insertions(+), 23 deletions(-) diff --git a/src/api/action.c b/src/api/action.c index 138c1c6c..4a55d6e5 100644 --- a/src/api/action.c +++ b/src/api/action.c @@ -128,10 +128,7 @@ int api_action_restartDNS(struct ftl_conn *api) "Restarting DNS is not allowed", "Check setting webserver.api.allow_destructive"); - log_info("Restarting FTL due to API action request"); - exit_code = RESTART_FTL_CODE; - // Send SIGTERM to FTL - kill(main_pid(), SIGTERM); + restart_ftl("API action request"); return send_json_success(api); } diff --git a/src/api/api.c b/src/api/api.c index 32acbb61..444dc45a 100644 --- a/src/api/api.c +++ b/src/api/api.c @@ -259,12 +259,7 @@ int api_handler(struct mg_connection *conn, void *ignored) // Restart FTL if requested if(api.ftl.restart) - { - log_info("Restarting FTL due to API config change"); - exit_code = RESTART_FTL_CODE; - // Send SIGTERM to FTL - kill(main_pid(), SIGTERM); - } + restart_ftl(api.ftl.restart_reason); return ret; } diff --git a/src/api/config.c b/src/api/config.c index 5b12dd10..5cf93d1f 100644 --- a/src/api/config.c +++ b/src/api/config.c @@ -807,7 +807,10 @@ static int api_config_patch(struct ftl_conn *api) // If the privacy level was decreased, we need to restart if(new_item == &newconf.misc.privacylevel && new_item->v.privacy_level < conf_item->v.privacy_level) + { + api->ftl.restart_reason = "Privacy level decreased"; api->ftl.restart = true; + } // Check if this item changed the password, if so, we need to // invalidate all currently active sessions @@ -823,7 +826,10 @@ static int api_config_patch(struct ftl_conn *api) { char errbuf[ERRBUF_SIZE] = { 0 }; if(write_dnsmasq_config(&newconf, true, errbuf)) + { + api->ftl.restart_reason = "dnsmasq config changed"; api->ftl.restart = true; + } else { free_config(&newconf); @@ -1031,7 +1037,10 @@ static int api_config_put_delete(struct ftl_conn *api) char errbuf[ERRBUF_SIZE] = { 0 }; // Request restart of FTL if(write_dnsmasq_config(&newconf, true, errbuf)) + { + api->ftl.restart_reason = "dnsmasq config changed"; api->ftl.restart = true; + } else { // The new config did not work diff --git a/src/api/teleporter.c b/src/api/teleporter.c index a8542362..bfead147 100644 --- a/src/api/teleporter.c +++ b/src/api/teleporter.c @@ -27,6 +27,8 @@ #include "files.h" //basename() #include +// restart_ftl() +#include "signals.h" #define MAXFILESIZE (50u*1024*1024) @@ -322,6 +324,10 @@ static int process_received_zip(struct ftl_conn *api, struct upload_data *data) // Free allocated memory free_upload_data(data); + // Signal FTL we want to restart for re-import + api->ftl.restart_reason = "Teleporter (ZIP) import"; + api->ftl.restart = true; + // Send response cJSON *json = JSON_NEW_OBJECT(); JSON_ADD_ITEM_TO_OBJECT(json, "files", json_files); @@ -818,6 +824,7 @@ static int process_received_tar_gz(struct ftl_conn *api, struct upload_data *dat free_upload_data(data); // Signal FTL we want to restart for re-import + api->ftl.restart_reason = "Teleporter (TAR.GZ) import"; api->ftl.restart = true; // Send response diff --git a/src/config/config.c b/src/config/config.c index 5fe183d0..71576ee8 100644 --- a/src/config/config.c +++ b/src/config/config.c @@ -35,7 +35,10 @@ #include "config/env.h" // sha256sum() #include "files.h" +// restart_ftl() +#include "signals.h" +// Global variables struct config config = { 0 }; static bool config_initialized = false; uint8_t last_checksum[SHA256_DIGEST_SIZE] = { 0 }; @@ -1867,12 +1870,7 @@ void reread_config(void) // If we need to restart FTL, we do so now if(restart) - { - log_info("Restarting FTL due to pihole.toml change"); - exit_code = RESTART_FTL_CODE; - // Send SIGTERM to FTL - kill(main_pid(), SIGTERM); - } + restart_ftl("pihole.toml change"); } // Very simple test of a port's availability by trying to bind a TCP socket to diff --git a/src/ntp/client.c b/src/ntp/client.c index 08f92b67..53a60f2a 100644 --- a/src/ntp/client.c +++ b/src/ntp/client.c @@ -614,12 +614,8 @@ static void *ntp_client_thread(void *arg) double time_delta = fabs(after - before); if(first_run && time_delta > GCinterval) { - log_info("System time was updated by %.1f seconds, restarting FTL to import recent data", - time_delta); - // Set the restart flag to true - exit_code = RESTART_FTL_CODE; - // Send SIGTERM to FTL - kill(main_pid(), SIGTERM); + log_info("System time was updated by %.1f seconds", time_delta); + restart_ftl("System time updated"); } // Set first run to false diff --git a/src/signals.c b/src/signals.c index 35971f0d..ba0107df 100644 --- a/src/signals.c +++ b/src/signals.c @@ -523,3 +523,11 @@ int sigtest(void) // Exit successfully return EXIT_SUCCESS; } + +void restart_ftl(const char *reason) +{ + log_info("Restarting FTL: %s", reason); + exit_code = RESTART_FTL_CODE; + // Send SIGTERM to FTL + kill(main_pid(), SIGTERM); +} diff --git a/src/signals.h b/src/signals.h index 1f397b26..3c2e8a75 100644 --- a/src/signals.h +++ b/src/signals.h @@ -21,6 +21,7 @@ pid_t main_pid(void); void thread_sleepms(const enum thread_types thread, const int milliseconds); void generate_backtrace(void); int sigtest(void); +void restart_ftl(const char *reason); extern volatile int exit_code; extern volatile sig_atomic_t killed; diff --git a/src/webserver/http-common.h b/src/webserver/http-common.h index bde500e5..ad85e11b 100644 --- a/src/webserver/http-common.h +++ b/src/webserver/http-common.h @@ -51,7 +51,8 @@ struct ftl_conn { long unsigned int size; } payload; struct { - bool restart; + bool restart :1; + const char *restart_reason; } ftl; struct session *session; diff --git a/src/zip/teleporter.c b/src/zip/teleporter.c index 47f2edc3..8e9af579 100644 --- a/src/zip/teleporter.c +++ b/src/zip/teleporter.c @@ -39,6 +39,8 @@ #include "events.h" // JSON_KEY_TRUE #include "webserver/json_macros.h" +// exit_code +#include "signals.h" // Tables to copy from the gravity database to the Teleporter database static const char *gravity_tables[] = { From a4eb3adf3ebf7270888131cad133d4be65b68df7 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 12 Sep 2024 22:10:36 +0200 Subject: [PATCH 289/339] Remove fixed number of unchanged config files from CI tests. Slow workers may detect more often that the config was not changed than faster ones. This is caused by inotify events arriving delayed and - in general - asynchroneous. Signed-off-by: DL6ER --- test/test_suite.bats | 9 --------- 1 file changed, 9 deletions(-) diff --git a/test/test_suite.bats b/test/test_suite.bats index 8a5019d2..1242c15a 100644 --- a/test/test_suite.bats +++ b/test/test_suite.bats @@ -1965,21 +1965,12 @@ run bash -c 'grep -c "INFO: Config file written to /etc/pihole/pihole.toml" /var/log/pihole/FTL.log' printf "%s\n" "${lines[@]}" [[ ${lines[0]} == "2" ]] - run bash -c 'grep -c "DEBUG_CONFIG: pihole.toml unchanged" /var/log/pihole/FTL.log' - printf "%s\n" "${lines[@]}" - [[ ${lines[0]} == "4" ]] run bash -c 'grep -c "DEBUG_CONFIG: Config file written to /etc/pihole/dnsmasq.conf" /var/log/pihole/FTL.log' printf "%s\n" "${lines[@]}" [[ ${lines[0]} == "1" ]] - run bash -c 'grep -c "DEBUG_CONFIG: dnsmasq.conf unchanged" /var/log/pihole/FTL.log' - printf "%s\n" "${lines[@]}" - [[ ${lines[0]} == "2" ]] run bash -c 'grep -c "DEBUG_CONFIG: HOSTS file written to /etc/pihole/hosts/custom.list" /var/log/pihole/FTL.log' printf "%s\n" "${lines[@]}" [[ ${lines[0]} == "1" ]] - run bash -c 'grep -c "DEBUG_CONFIG: custom.list unchanged" /var/log/pihole/FTL.log' - printf "%s\n" "${lines[@]}" - [[ ${lines[0]} == "3" ]] } @test "Check NTP server is broadcasting correct time" { From 850d263ec3bc6fb0faaf72c361e7b38ed534ebeb Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 13 Sep 2024 15:52:13 +0200 Subject: [PATCH 290/339] Check and create v6 migration directory before trying to move/write files there. This involves config migrations but also Teleporter importing Signed-off-by: DL6ER --- src/api/teleporter.c | 5 +++++ src/config/config.c | 37 ++++++++++++++++++++++++++++++++++++- src/config/config.h | 4 ++++ src/config/dnsmasq_config.h | 6 +++--- src/config/setupVars.c | 2 +- 5 files changed, 49 insertions(+), 5 deletions(-) diff --git a/src/api/teleporter.c b/src/api/teleporter.c index bfead147..d3e2b902 100644 --- a/src/api/teleporter.c +++ b/src/api/teleporter.c @@ -29,6 +29,8 @@ #include // restart_ftl() #include "signals.h" +// create_migration_target_v6() +#include "config/config.h" #define MAXFILESIZE (50u*1024*1024) @@ -264,6 +266,9 @@ static int api_teleporter_POST(struct ftl_conn *api) NULL); } + // Ensure v6 migration directory exists + create_migration_target_v6(); + // Check if we received something that claims to be a ZIP archive // - filename should end in ".zip" // - the data itself diff --git a/src/config/config.c b/src/config/config.c index 71576ee8..60b17acb 100644 --- a/src/config/config.c +++ b/src/config/config.c @@ -1624,13 +1624,19 @@ bool readFTLconf(struct config *conf, const bool rewrite) if(!rewrite) return false; + // Check if MIGRATION_TARGET_V6 exists and is a directory + // Ideally, this directory should be created by the installer but users + // may have deleted it manually and it is necessary for restoring + // Teleporter files + create_migration_target_v6(); + // If no previous config file could be read, we are likely either running // for the first time or we are upgrading from a version prior to v6.0 // In this case, we try to read the legacy config files const char *path = ""; if((path = readFTLlegacy(conf)) != NULL) { - const char *target = "/etc/pihole/migration_backup_v6/pihole-FTL.conf"; + const char *target = MIGRATION_TARGET_V6"/pihole-FTL.conf"; log_info("Moving %s to %s", path, target); if(rename(path, target) != 0) log_warn("Unable to move %s to %s: %s", path, target, strerror(errno)); @@ -1903,3 +1909,32 @@ static bool port_in_use(const in_port_t port) close(sock); return false; } + +/** + * @brief Create a migration target directory for version 6. + * + * This function creates a directory for migration target version 6. If the directory + * already exists, it does nothing. The function also changes the ownership of the + * directory to the user running the FTL program. + * + * @return true if the directory creation and ownership change were successful, false otherwise. + */ +bool create_migration_target_v6(void) +{ + if(mkdir(MIGRATION_TARGET_V6, 0755) != 0 && errno != EEXIST) + { + log_err("Unable to create directory %s: %s", MIGRATION_TARGET_V6, strerror(errno)); + return false; + } + else + { + // Change ownership of the directory to the user running FTL + if(chown(MIGRATION_TARGET_V6, getuid(), getgid()) != 0) + { + log_err("Unable to change ownership of %s: %s", MIGRATION_TARGET_V6, strerror(errno)); + return false; + } + } + + return true; +} diff --git a/src/config/config.h b/src/config/config.h index bdabc0f9..203347a3 100644 --- a/src/config/config.h +++ b/src/config/config.h @@ -39,6 +39,9 @@ // Location of the legacy (pre-v6.0) config file #define GLOBALCONFFILE_LEGACY "/etc/pihole/pihole-FTL.conf" +// Migration target for the legacy (pre-v6.0) config file +#define MIGRATION_TARGET_V6 "/etc/pihole/migration_backup_v6" + union conf_value { bool b; // boolean value int i; // integer value @@ -363,6 +366,7 @@ bool check_paths_equal(char **paths1, char **paths2, unsigned int max_level) __a const char *get_conf_type_str(const enum conf_type type) __attribute__ ((const)); void replace_config(struct config *newconf); void reread_config(void); +bool create_migration_target_v6(void); // Defined in toml_reader.c bool readDebugSettings(void); diff --git a/src/config/dnsmasq_config.h b/src/config/dnsmasq_config.h index 2b40042a..df660790 100644 --- a/src/config/dnsmasq_config.h +++ b/src/config/dnsmasq_config.h @@ -24,12 +24,12 @@ bool write_custom_list(void); #define DNSMASQ_PH_CONFIG "/etc/pihole/dnsmasq.conf" #define DNSMASQ_TEMP_CONF "/etc/pihole/dnsmasq.conf.temp" -#define DNSMASQ_STATIC_LEASES "/etc/pihole/migration_backup_v6/04-pihole-static-dhcp.conf" -#define DNSMASQ_CNAMES "/etc/pihole/migration_backup_v6/05-pihole-custom-cname.conf" +#define DNSMASQ_STATIC_LEASES MIGRATION_TARGET_V6"/04-pihole-static-dhcp.conf" +#define DNSMASQ_CNAMES MIGRATION_TARGET_V6"/05-pihole-custom-cname.conf" #define DNSMASQ_HOSTSDIR "/etc/pihole/hosts" #define DNSMASQ_CUSTOM_LIST DNSMASQ_HOSTSDIR"/custom.list" #define DNSMASQ_CUSTOM_LIST_LEGACY "/etc/pihole/custom.list" -#define DNSMASQ_CUSTOM_LIST_LEGACY_TARGET "/etc/pihole/migration_backup_v6/custom.list" +#define DNSMASQ_CUSTOM_LIST_LEGACY_TARGET MIGRATION_TARGET_V6"/custom.list" #define DHCPLEASESFILE "/etc/pihole/dhcp.leases" #endif //DNSMASQ_CONFIG_H diff --git a/src/config/setupVars.c b/src/config/setupVars.c index e7e3a55c..3a82beeb 100644 --- a/src/config/setupVars.c +++ b/src/config/setupVars.c @@ -589,7 +589,7 @@ void importsetupVarsConf(void) get_conf_string_from_setupVars("WEB_PORTS", &config.webserver.port); // Move the setupVars.conf file to the migration directory - const char *setupVars_target = "/etc/pihole/migration_backup_v6/setupVars.conf"; + const char *setupVars_target = MIGRATION_TARGET_V6"/setupVars.conf"; if(rename(config.files.setupVars.v.s, setupVars_target) != 0) log_warn("Could not move %s to %s", config.files.setupVars.v.s, setupVars_target); else From d67d13303d2dc839e6c0081e78f327422865366a Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 18 Aug 2024 21:12:37 +0200 Subject: [PATCH 291/339] Add downstream EDE info for synthesized replies Signed-off-by: DL6ER --- src/api/config.c | 15 ++++ src/api/docs/content/specs/config.yaml | 3 + src/config/cli.c | 15 ++++ src/config/config.c | 19 +++++ src/config/config.h | 3 + src/config/env.c | 19 +++++ src/config/toml_helper.c | 19 +++++ src/datastructure.c | 27 +++++++ src/datastructure.h | 2 + src/dnsmasq/forward.c | 37 +++++---- src/dnsmasq_interface.c | 101 +++++++++++++++++++++++-- src/dnsmasq_interface.h | 5 +- src/edns0.c | 9 ++- src/enums.h | 7 ++ test/pihole.toml | 14 +++- test/test_suite.bats | 40 +++++++++- 16 files changed, 306 insertions(+), 29 deletions(-) diff --git a/src/api/config.c b/src/api/config.c index 5cf93d1f..690f8103 100644 --- a/src/api/config.c +++ b/src/api/config.c @@ -129,6 +129,8 @@ cJSON *addJSONConfValue(const enum conf_type conf_type, union conf_value *val) return cJSON_CreateStringReference(get_web_theme_str(val->web_theme)); case CONF_ENUM_TEMP_UNIT: return cJSON_CreateStringReference(get_temp_unit_str(val->temp_unit)); + case CONF_ENUM_BLOCKING_EDNS_MODE: + return cJSON_CreateStringReference(get_edns_mode_str(val->edns_mode)); case CONF_STRUCT_IN_ADDR: { // Special case 0.0.0.0 -> return empty string @@ -391,6 +393,19 @@ static const char *getJSONvalue(struct conf_item *conf_item, cJSON *elem, struct log_debug(DEBUG_CONFIG, "%s = %d", conf_item->k, conf_item->v.temp_unit); break; } + case CONF_ENUM_BLOCKING_EDNS_MODE: + { + // Check type + if(!cJSON_IsString(elem)) + return "not of type string"; + const int edns_mode = get_edns_mode_val(elem->valuestring); + if(edns_mode == -1) + return "invalid option"; + // Set item + conf_item->v.edns_mode = edns_mode; + log_debug(DEBUG_CONFIG, "%s = %d", conf_item->k, conf_item->v.edns_mode); + break; + } case CONF_ENUM_PRIVACY_LEVEL: { // Check type diff --git a/src/api/docs/content/specs/config.yaml b/src/api/docs/content/specs/config.yaml index 2fdd110b..d9d7c612 100644 --- a/src/api/docs/content/specs/config.yaml +++ b/src/api/docs/content/specs/config.yaml @@ -252,6 +252,8 @@ components: type: boolean mode: type: string + edns: + type: string specialDomains: type: object properties: @@ -671,6 +673,7 @@ components: blocking: active: true mode: 'NULL' + edns: 'NONE' specialDomains: mozillaCanary: true iCloudPrivateRelay: true diff --git a/src/config/cli.c b/src/config/cli.c index 7c2de6bd..15b967a6 100644 --- a/src/config/cli.c +++ b/src/config/cli.c @@ -306,6 +306,21 @@ static bool readStringValue(struct conf_item *conf_item, const char *value, stru } break; } + case CONF_ENUM_BLOCKING_EDNS_MODE: + { + const int edns_mode = get_edns_mode_val(value); + if(edns_mode != -1) + conf_item->v.edns_mode = edns_mode; + else + { + char *allowed = NULL; + CONFIG_ITEM_ARRAY(conf_item->a, allowed); + log_err("Config setting %s is invalid, allowed options are: %s", conf_item->k, allowed); + free(allowed); + return false; + } + break; + } case CONF_STRUCT_IN_ADDR: { struct in_addr addr4 = { 0 }; diff --git a/src/config/config.c b/src/config/config.c index 71576ee8..c03e9379 100644 --- a/src/config/config.c +++ b/src/config/config.c @@ -278,6 +278,7 @@ void duplicate_config(struct config *dst, struct config *src) case CONF_ENUM_LISTENING_MODE: case CONF_ENUM_WEB_THEME: case CONF_ENUM_TEMP_UNIT: + case CONF_ENUM_BLOCKING_EDNS_MODE: case CONF_STRUCT_IN_ADDR: case CONF_STRUCT_IN6_ADDR: case CONF_ALL_DEBUG_BOOL: @@ -314,6 +315,7 @@ bool compare_config_item(const enum conf_type t, const union conf_value *val1, c case CONF_ENUM_LISTENING_MODE: case CONF_ENUM_WEB_THEME: case CONF_ENUM_TEMP_UNIT: + case CONF_ENUM_BLOCKING_EDNS_MODE: case CONF_STRUCT_IN_ADDR: case CONF_STRUCT_IN6_ADDR: case CONF_ALL_DEBUG_BOOL: @@ -370,6 +372,7 @@ void free_config(struct config *conf) case CONF_ENUM_LISTENING_MODE: case CONF_ENUM_WEB_THEME: case CONF_ENUM_TEMP_UNIT: + case CONF_ENUM_BLOCKING_EDNS_MODE: case CONF_STRUCT_IN_ADDR: case CONF_STRUCT_IN6_ADDR: case CONF_ALL_DEBUG_BOOL: @@ -620,6 +623,21 @@ static void initConfig(struct config *conf) conf->dns.blocking.mode.d.blocking_mode = MODE_NULL; conf->dns.blocking.mode.c = validate_stub; // Only type-based checking + conf->dns.blocking.edns.k = "dns.blocking.edns"; + conf->dns.blocking.edns.h = "Should FTL enrich blocked replies with EDNS0 information?"; + { + struct enum_options blocking_edns[] = + { + { get_edns_mode_str(EDNS_MODE_NONE), "In NONE mode, no additional EDNS information is added to blocked queries" }, + { get_edns_mode_str(EDNS_MODE_CODE), "In CODE mode, blocked queries will be enriched with EDNS info-code BLOCKED (15)" }, + { get_edns_mode_str(EDNS_MODE_TEXT), "In TEXT mode, blocked queries will be enriched with EDNS info-code BLOCKED (15) and a text message describing the reason for the block" } + }; + CONFIG_ADD_ENUM_OPTIONS(conf->dns.blocking.edns.a, blocking_edns); + } + conf->dns.blocking.edns.t = CONF_ENUM_BLOCKING_EDNS_MODE; + conf->dns.blocking.edns.d.edns_mode = EDNS_MODE_TEXT; + conf->dns.blocking.edns.c = validate_stub; // Only type-based checking + conf->dns.revServers.k = "dns.revServers"; conf->dns.revServers.h = "Reverse server (former also called \"conditional forwarding\") feature\n Array of reverse servers each one in one of the following forms: \",[/],[#],\"\n\n Individual components:\n\n : either \"true\" or \"false\"\n\n [/]: Address range for the reverse server feature in CIDR notation. If the prefix length is omitted, either 32 (IPv4) or 128 (IPv6) are substituted (exact address match). This is almost certainly not what you want here.\n Example: \"192.168.0.0/24\" for the range 192.168.0.1 - 192.168.0.255\n\n [#]: Target server to be used for the reverse server feature\n Example: \"192.168.0.1#53\"\n\n : Domain used for the reverse server feature (e.g., \"fritz.box\")\n Example: \"fritz.box\""; conf->dns.revServers.a = cJSON_CreateStringReference("array of reverse servers each one in one of the following forms: \",[/],[#],\", e.g., \"true,192.168.0.0/24,192.168.0.1,fritz.box\""); @@ -1766,6 +1784,7 @@ const char * __attribute__ ((const)) get_conf_type_str(const enum conf_type type case CONF_ENUM_LISTENING_MODE: case CONF_ENUM_WEB_THEME: case CONF_ENUM_TEMP_UNIT: + case CONF_ENUM_BLOCKING_EDNS_MODE: return "enum (string)"; case CONF_ENUM_PRIVACY_LEVEL: return "enum (unsigned integer)"; diff --git a/src/config/config.h b/src/config/config.h index bdabc0f9..d1c6c506 100644 --- a/src/config/config.h +++ b/src/config/config.h @@ -57,6 +57,7 @@ union conf_value { enum listening_mode listeningMode; // enum listening_mode value enum web_theme web_theme; // enum web_theme value enum temp_unit temp_unit; // enum temp_unit value + enum edns_mode edns_mode; // enum edns_mode value struct in_addr in_addr; // struct in_addr value struct in6_addr in6_addr; // struct in6_addr value cJSON *json; // cJSON * value @@ -80,6 +81,7 @@ enum conf_type { CONF_ENUM_PRIVACY_LEVEL, CONF_ENUM_LISTENING_MODE, CONF_ENUM_WEB_THEME, + CONF_ENUM_BLOCKING_EDNS_MODE, CONF_ENUM_TEMP_UNIT, CONF_STRUCT_IN_ADDR, CONF_STRUCT_IN6_ADDR, @@ -152,6 +154,7 @@ struct config { struct { struct conf_item active; struct conf_item mode; + struct conf_item edns; } blocking; struct { struct conf_item mozillaCanary; diff --git a/src/config/env.c b/src/config/env.c index f1e7bb5f..0981c619 100644 --- a/src/config/env.c +++ b/src/config/env.c @@ -499,6 +499,25 @@ bool __attribute__((nonnull(1,2,3))) readEnvValue(struct conf_item *conf_item, s } break; } + case CONF_ENUM_BLOCKING_EDNS_MODE: + { + const int edns_mode = get_edns_mode_val(envvar); + if(edns_mode != -1) + { + conf_item->v.edns_mode = edns_mode; + item->valid = true; + } + else + { + + item->error = "not an allowed option"; + item->allowed = conf_item->h; + log_warn("ENV %s is %s, allowed options are: %s", + conf_item->e, item->error, item->allowed); + item->valid = false; + } + break; + } case CONF_ENUM_PRIVACY_LEVEL: { int val = 0; diff --git a/src/config/toml_helper.c b/src/config/toml_helper.c index 37cf792a..58e8f8fc 100644 --- a/src/config/toml_helper.c +++ b/src/config/toml_helper.c @@ -362,6 +362,9 @@ void writeTOMLvalue(FILE * fp, const int indent, const enum conf_type t, union c case CONF_ENUM_TEMP_UNIT: printTOMLstring(fp, get_temp_unit_str(v->temp_unit), toml); break; + case CONF_ENUM_BLOCKING_EDNS_MODE: + printTOMLstring(fp, get_edns_mode_str(v->edns_mode), toml); + break; case CONF_STRUCT_IN_ADDR: { // Special case: 0.0.0.0 -> return empty string @@ -653,6 +656,22 @@ void readTOMLvalue(struct conf_item *conf_item, const char* key, toml_table_t *t log_debug(DEBUG_CONFIG, "%s DOES NOT EXIST or is not a valid string", conf_item->k); break; } + case CONF_ENUM_BLOCKING_EDNS_MODE: + { + const toml_datum_t val = toml_string_in(toml, key); + if(val.ok) + { + const int edns_mode = get_edns_mode_val(val.u.s); + free(val.u.s); + if(edns_mode != -1) + conf_item->v.edns_mode = edns_mode; + else + log_warn("Config setting %s is invalid, allowed options are: %s", conf_item->k, conf_item->h); + } + else + log_debug(DEBUG_CONFIG, "%s DOES NOT EXIST or is not a valid string", conf_item->k); + break; + } case CONF_ENUM_PRIVACY_LEVEL: { const toml_datum_t val = toml_int_in(toml, key); diff --git a/src/datastructure.c b/src/datastructure.c index 4adb5687..7d842845 100644 --- a/src/datastructure.c +++ b/src/datastructure.c @@ -1217,3 +1217,30 @@ int __attribute__ ((pure)) get_temp_unit_val(const char *temp_unit) // Invalid value return -1; } + +const char * __attribute__ ((const)) get_edns_mode_str(const enum edns_mode edns_mode) +{ + switch(edns_mode) + { + case EDNS_MODE_NONE: + return "NONE"; + case EDNS_MODE_CODE: + return "CODE"; + case EDNS_MODE_TEXT: + return "TEXT"; + } + return NULL; +} + +int __attribute__ ((pure)) get_edns_mode_val(const char *edns_mode) +{ + if(strcasecmp(edns_mode, "NONE") == 0) + return EDNS_MODE_NONE; + else if(strcasecmp(edns_mode, "CODE") == 0) + return EDNS_MODE_CODE; + else if(strcasecmp(edns_mode, "TEXT") == 0) + return EDNS_MODE_TEXT; + + // Invalid value + return -1; +} diff --git a/src/datastructure.h b/src/datastructure.h index 7d3ac932..5e7cedc4 100644 --- a/src/datastructure.h +++ b/src/datastructure.h @@ -169,6 +169,8 @@ const char * get_listeningMode_str(const enum listening_mode listeningMode) __at int get_listeningMode_val(const char *listeningMode) __attribute__ ((pure)); const char * __attribute__ ((const)) get_temp_unit_str(const enum temp_unit temp_unit); int __attribute__ ((pure)) get_temp_unit_val(const char *temp_unit); +const char * __attribute__ ((const)) get_edns_mode_str(const enum edns_mode edns_mode); +int __attribute__ ((pure)) get_edns_mode_val(const char *edns_mode); // Pointer getter functions #define getQuery(queryID, checkMagic) _getQuery(queryID, checkMagic, __LINE__, __FUNCTION__, __FILE__) diff --git a/src/dnsmasq/forward.c b/src/dnsmasq/forward.c index 27c2c8e8..c2824d85 100644 --- a/src/dnsmasq/forward.c +++ b/src/dnsmasq/forward.c @@ -714,6 +714,8 @@ static size_t process_reply(struct dns_header *header, time_t now, struct server size_t plen; /******** Pi-hole modification ********/ unsigned char *pheader_copy = NULL; + unsigned char ede_data[MAX_EDE_DATA] = { 0 }; + size_t ede_len = 0; /**************************************/ (void)ad_reqd; @@ -890,7 +892,7 @@ static size_t process_reply(struct dns_header *header, time_t now, struct server // Generate DNS packet for reply, a possibly existing pseudo header // will be restored later inside resize_packet() - n = FTL_make_answer(header, ((char *) header) + 65536, n, &ede); + n = FTL_make_answer(header, ((char *) header) + 65536, n, ede_data, &ede_len); } } @@ -930,13 +932,18 @@ static size_t process_reply(struct dns_header *header, time_t now, struct server // pheader_copy instead of pheader if(pheader_copy) free(pheader_copy); - /**************************************/ - if (pheader && ede != EDE_UNSET) + if (pheader && (ede != EDE_UNSET || ede_len > 0)) { - u16 swap = htons((u16)ede); - n = add_pseudoheader(header, n, limit, daemon->edns_pktsz, EDNS0_OPTION_EDE, (unsigned char *)&swap, 2, do_bit, 1); + if (ede_len > 0) + n = add_pseudoheader(header, n, limit, daemon->edns_pktsz, EDNS0_OPTION_EDE, ede_data, ede_len, do_bit, 1); + else + { + u16 swap = htons((u16)ede); + n = add_pseudoheader(header, n, limit, daemon->edns_pktsz, EDNS0_OPTION_EDE, (unsigned char *)&swap, 2, do_bit, 1); + } } + /**************************************/ if (RCODE(header) == NXDOMAIN) server->nxdomain_replies++; @@ -1930,8 +1937,9 @@ void receive_query(struct listener *listen, time_t now) if(piholeblocked) { // Generate DNS packet for reply - int ede = EDE_UNSET; - n = FTL_make_answer(header, ((char *) header) + udp_size, n, &ede); + unsigned char ede_data[MAX_EDE_DATA] = { 0 }; + size_t ede_len = 0; + n = FTL_make_answer(header, ((char *) header) + udp_size, n, ede_data, &ede_len); // The pseudoheader may contain important information such as EDNS0 version important for // some DNS resolvers (such as systemd-resolved) to work properly. We should not discard them. @@ -1941,10 +1949,9 @@ void receive_query(struct listener *listen, time_t now) if (have_pseudoheader) { - u16 swap = htons(ede); - if (ede != EDE_UNSET) // Add EDNS0 option EDE if applicable + if (ede_len > 0) // Add EDNS0 option EDE if applicable n = add_pseudoheader(header, n, ((unsigned char *) header) + udp_size, - daemon->edns_pktsz, EDNS0_OPTION_EDE, (unsigned char *)&swap, 2, do_bit, 0); + daemon->edns_pktsz, EDNS0_OPTION_EDE, ede_data, ede_len, do_bit, 0); else n = add_pseudoheader(header, n, ((unsigned char *) header) + udp_size, daemon->edns_pktsz, 0, NULL, 0, do_bit, 0); @@ -2470,18 +2477,18 @@ unsigned char *tcp_request(int confd, time_t now, // Interface name is known from before forking if(piholeblocked) { - int ede = EDE_UNSET; + unsigned char ede_data[MAX_EDE_DATA] = { 0 }; + size_t ede_len = 0; stale = 0; // Generate DNS packet for reply - m = FTL_make_answer(header, ((char *) header) + 65536, size, &ede); + m = FTL_make_answer(header, ((char *) header) + 65536, size, ede_data, &ede_len); // The pseudoheader may contain important information such as EDNS0 version important for // some DNS resolvers (such as systemd-resolved) to work properly. We should not discard them. if (have_pseudoheader && m > 0) { - u16 swap = htons(ede); - if (ede != -1) // Add EDNS0 option EDE if applicable + if (ede_len > 0) // Add EDNS0 option EDE if applicable m = add_pseudoheader(header, m, ((unsigned char *) header) + 65536, - daemon->edns_pktsz, EDNS0_OPTION_EDE, (unsigned char *)&swap, 2, do_bit, 0); + daemon->edns_pktsz, EDNS0_OPTION_EDE, ede_data, ede_len, do_bit, 0); else m = add_pseudoheader(header, m, ((unsigned char *) header) + 65536, daemon->edns_pktsz, 0, NULL, 0, do_bit, 0); diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c index d9be227d..63fc0d92 100644 --- a/src/dnsmasq_interface.c +++ b/src/dnsmasq_interface.c @@ -85,6 +85,7 @@ static const char *check_dnsmasq_name(const char *name); static bool adbit = false, rabit = false; static const char *blockingreason = ""; static enum reply_type force_next_DNS_reply = REPLY_UNKNOWN; +static enum domain_client_status cacheStatus = UNKNOWN_BLOCKED; static int last_regex_idx = -1; static char *pihole_suffix = NULL; static char *hostname_suffix = NULL; @@ -184,7 +185,8 @@ void FTL_hook(unsigned int flags, const char *name, union all_addr *addr, char * } // This is inspired by make_local_answer() -size_t _FTL_make_answer(struct dns_header *header, char *limit, const size_t len, int *ede, +size_t _FTL_make_answer(struct dns_header *header, char *limit, const size_t len, + unsigned char ede_data[MAX_EDE_DATA], size_t *ede_len, const char *file, const int line) { log_debug(DEBUG_FLAGS, "FTL_make_answer() called from %s:%d", short_path(file), line); @@ -199,7 +201,7 @@ size_t _FTL_make_answer(struct dns_header *header, char *limit, const size_t len return 0; // Debug logging - log_debug(DEBUG_QUERIES, "Preparing reply for \"%s\", EDE: %s (%d)", name, *ede != EDE_UNSET ? edestr(*ede) : "N/A", *ede); + log_debug(DEBUG_QUERIES, "Preparing reply for \"%s\"", name); // Get question type int qtype, flags = 0; @@ -247,9 +249,6 @@ size_t _FTL_make_answer(struct dns_header *header, char *limit, const size_t len // Debug logging log_debug(DEBUG_QUERIES, "Forced DNS reply to REFUSED"); - - // Set EDE code to blocked - *ede = EDE_BLOCKED; } else if(force_next_DNS_reply == REPLY_IP) { @@ -324,11 +323,79 @@ size_t _FTL_make_answer(struct dns_header *header, char *limit, const size_t len force_next_DNS_reply = REPLY_UNKNOWN; } + // Derive EDE code and text from cacheStatus + int ede_code = EDE_UNSET; + const char *ede_text = NULL; + switch(cacheStatus) + { + case UNKNOWN_BLOCKED: + case NOT_BLOCKED: + case ALLOWED: + // Not going through this function + break; + case GRAVITY_BLOCKED: + ede_code = EDE_BLOCKED; + ede_text = "gravity"; + break; + case DENYLIST_BLOCKED: + ede_code = EDE_BLOCKED; + ede_text = "denylist"; + break; + case REGEX_BLOCKED: + ede_code = EDE_BLOCKED; + ede_text = "regex"; + break; + case SPECIAL_DOMAIN: + ede_code = EDE_BLOCKED; + ede_text = "special"; + break; + case UPSTREAM_BLOCKED_NXRA: + ede_code = EDE_BLOCKED; + ede_text = "upstream NXRA"; + break; + case UPSTREAM_BLOCKED_NULL: + ede_code = EDE_BLOCKED; + ede_text = "upstream NULL"; + break; + case UPSTREAM_BLOCKED_IP: + ede_code = EDE_BLOCKED; + ede_text = "upstream IP"; + break; + case PIHOLE_SYNTH: + ede_code = EDE_SYNTHESIZED; + ede_text = "synthesized"; + break; + } + cacheStatus = UNKNOWN_BLOCKED; + + // Debug logging + log_debug(DEBUG_QUERIES, "Setting EDE: %s (%d) + \"%s\"", + ede_code != EDE_UNSET ? edestr(ede_code) : "---", ede_code, ede_text ? ede_text : "---"); + + if(ede_code != EDE_UNSET && config.dns.blocking.edns.v.edns_mode > EDNS_MODE_NONE) + { + // Set EDE INFO-CODE (network byte order) + uint16_t swap = htons(ede_code); + memcpy(ede_data, &swap, sizeof(swap)); + *ede_len = sizeof(swap); + + // Set EDE INFO-TEXT (if available) + if(ede_text && config.dns.blocking.edns.v.edns_mode > EDNS_MODE_CODE) + { + size_t extra_len = strlen(ede_text); + // Truncate if necessary + if(extra_len > MAX_EDE_DATA - *ede_len) + extra_len = MAX_EDE_DATA - *ede_len; + memcpy(ede_data + *ede_len, ede_text, extra_len); + *ede_len += extra_len; + } + } + // Debug logging print_flags(flags); // Setup reply header - setup_reply(header, flags, *ede); + setup_reply(header, flags, ede_code); // Add NEG flag when replying with NXDOMAIN or NODATA. This is necessary // to get proper logging in pihole.log At the same time, we cannot add @@ -590,6 +657,7 @@ bool _FTL_new_query(const unsigned int flags, const char *name, "interface-local IP address" : "NODATA due to missing iface address"); + cacheStatus = PIHOLE_SYNTH; return true; } else @@ -1395,6 +1463,10 @@ static bool _FTL_check_blocking(int queryID, int domainID, int clientID, const c dns_cache->list_id = -1; } + // Memorize blocking status DNS cache for the domain/client combination + if(dns_cache->blocking_status != UNKNOWN_BLOCKED) + cacheStatus = dns_cache->blocking_status; + // Skip the entire chain of tests if we already know the answer for this // particular client char *domainstr = (char*)getstr(domain->domainpos); @@ -1499,10 +1571,16 @@ static bool _FTL_check_blocking(int queryID, int domainID, int clientID, const c const enum query_status qstat = dns_cache->blocking_status == UPSTREAM_BLOCKED_IP ? QUERY_EXTERNAL_BLOCKED_IP : dns_cache->blocking_status == UPSTREAM_BLOCKED_NULL ? - QUERY_EXTERNAL_BLOCKED_NULL : QUERY_EXTERNAL_BLOCKED_NXRA; + QUERY_EXTERNAL_BLOCKED_NULL : QUERY_EXTERNAL_BLOCKED_NXRA; query_blocked(query, domain, client, qstat); return true; break; + + case PIHOLE_SYNTH: + // Known as a synthetic reply, we return this result early, skipping + // all the lengthy tests below + return false; + break; } // Skip all checks and continue if we hit already at least one allowlist in the chain @@ -1532,6 +1610,7 @@ static bool _FTL_check_blocking(int queryID, int domainID, int clientID, const c { // Set DNS cache properties dns_cache->blocking_status = SPECIAL_DOMAIN; + cacheStatus = SPECIAL_DOMAIN; dns_cache->force_reply = force_next_DNS_reply; // Adjust counters @@ -1601,6 +1680,9 @@ static bool _FTL_check_blocking(int queryID, int domainID, int clientID, const c domainstr, query->flags.allowed ? "allowed" : "not blocked", dns_cache->list_id); } + // Update DNS cache status + cacheStatus = dns_cache->blocking_status; + free(domainstr); return blockDomain; } @@ -2362,6 +2444,7 @@ static enum query_status detect_blocked_IP(const unsigned short flags, const uni char answer[ADDRSTRLEN]; answer[0] = '\0'; inet_ntop(AF_INET, addr, answer, ADDRSTRLEN); blockingreason = "blocked upstream with known address (IPv4)"; + cacheStatus = UPSTREAM_BLOCKED_IP; log_debug(DEBUG_QUERIES, "%s -> \"%s\"", blockingreason, answer); } @@ -2380,6 +2463,7 @@ static enum query_status detect_blocked_IP(const unsigned short flags, const uni char answer[ADDRSTRLEN]; answer[0] = '\0'; inet_ntop(AF_INET6, addr, answer, ADDRSTRLEN); blockingreason = "blocked upstream with known address (IPv6)"; + cacheStatus = UPSTREAM_BLOCKED_IP; log_debug(DEBUG_QUERIES, "%s -> \"%s\"", blockingreason, answer); } @@ -2395,6 +2479,7 @@ static enum query_status detect_blocked_IP(const unsigned short flags, const uni if(config.debug.queries.v.b) { blockingreason = "blocked upstream with 0.0.0.0"; + cacheStatus = UPSTREAM_BLOCKED_NULL; log_debug(DEBUG_QUERIES, "%s", blockingreason); } @@ -2410,6 +2495,7 @@ static enum query_status detect_blocked_IP(const unsigned short flags, const uni if(config.debug.queries.v.b) { blockingreason = "blocked upstream with ::"; + cacheStatus = UPSTREAM_BLOCKED_NULL; log_debug(DEBUG_QUERIES, "%s", blockingreason); } @@ -2720,6 +2806,7 @@ static void FTL_NXRA(const int id, const char* file, const int line) // Set blocking reason blockingreason = "blocked upstream with NXDOMAIN and unset RA bit"; + cacheStatus = UPSTREAM_BLOCKED_NXRA; // Get response time struct timeval response; diff --git a/src/dnsmasq_interface.h b/src/dnsmasq_interface.h index 6dc6c94d..4004183f 100644 --- a/src/dnsmasq_interface.h +++ b/src/dnsmasq_interface.h @@ -35,8 +35,9 @@ int _FTL_check_reply(const unsigned int rcode, const unsigned short flags, const void FTL_forwarding_retried(const struct server *server, const int oldID, const int newID, const bool dnssec); -#define FTL_make_answer(header, limit, len, ede) _FTL_make_answer(header, limit, len, ede, __FILE__, __LINE__) -size_t _FTL_make_answer(struct dns_header *header, char *limit, const size_t len, int *ede, const char* file, const int line); +#define MAX_EDE_DATA 128 +#define FTL_make_answer(header, limit, len, ede_data, ede_len) _FTL_make_answer(header, limit, len, ede_data, ede_len, __FILE__, __LINE__) +size_t _FTL_make_answer(struct dns_header *header, char *limit, const size_t len, unsigned char ede_data[MAX_EDE_DATA], size_t *ede_len, const char* file, const int line); #define FTL_CNAME(dst, src, id) _FTL_CNAME(dst, src, id, __FILE__, __LINE__) bool _FTL_CNAME(const char *dst, const char *src, const int id, const char* file, const int line); diff --git a/src/edns0.c b/src/edns0.c index 8f27a184..54f7f709 100644 --- a/src/edns0.c +++ b/src/edns0.c @@ -400,13 +400,16 @@ void FTL_parse_pseudoheaders(unsigned char *pheader, const size_t plen) // this document. The value of the INFO-CODE is encoded // as a two-octet unsigned integer in network byte // order. - // - // The EXTRA-TEXT from the EDE EDNS option is ignored by - // FTL // Debug output log_debug(DEBUG_EDNS0, "EDE: %s (code %d)", edestr(edns.ede), edns.ede); + if(optlen > 2) + { + // Debug output + log_debug(DEBUG_EDNS0, "EDE: EXTRA-TEXT: %.*s", optlen - 2, p + 2); + } + // Advance working pointer p += optlen; } diff --git a/src/enums.h b/src/enums.h index 153098a5..788c1987 100644 --- a/src/enums.h +++ b/src/enums.h @@ -133,6 +133,7 @@ enum domain_client_status { UPSTREAM_BLOCKED_NXRA, UPSTREAM_BLOCKED_NULL, UPSTREAM_BLOCKED_IP, + PIHOLE_SYNTH, NOT_BLOCKED } __attribute__ ((packed)); @@ -317,6 +318,12 @@ enum temp_unit { TEMP_UNIT_K } __attribute__ ((packed)); +enum edns_mode { + EDNS_MODE_NONE = 0, + EDNS_MODE_CODE, + EDNS_MODE_TEXT, +} __attribute__ ((packed)); + enum adlist_type { ADLIST_BLOCK = 0, ADLIST_ALLOW diff --git a/test/pihole.toml b/test/pihole.toml index 3946aa9e..e03fd65d 100644 --- a/test/pihole.toml +++ b/test/pihole.toml @@ -289,6 +289,18 @@ # exists, but there is no record for the requested query type. mode = "NULL" + # Should FTL enrich blocked replies with EDNS0 information? + # + # Possible values are: + # - "NONE" + # In NONE mode, no additional EDNS information is added to blocked queries + # - "CODE" + # In CODE mode, blocked queries will be enriched with EDNS info-code BLOCKED (15) + # - "TEXT" + # In TEXT mode, blocked queries will be enriched with EDNS info-code BLOCKED (15) + # and a text message describing the reason for the block + edns = "TEXT" + [dns.specialDomains] # Should Pi-hole always replies with NXDOMAIN to A and AAAA queries of # use-application-dns.net to disable Firefox automatic DNS-over-HTTP? This is @@ -1103,7 +1115,7 @@ all = true ### CHANGED, default = false # Configuration statistics: -# 149 total entries out of which 94 entries are default +# 150 total entries out of which 95 entries are default # --> 55 entries are modified # 2 entries are forced through environment: # - misc.nice diff --git a/test/test_suite.bats b/test/test_suite.bats index 1242c15a..e9a4b6de 100644 --- a/test/test_suite.bats +++ b/test/test_suite.bats @@ -38,6 +38,10 @@ printf "%s\n" "${lines[@]}" [[ ${lines[0]} == "0.0.0.0" ]] [[ ${lines[1]} == "" ]] + run bash -c "dig denied.ftl @127.0.0.1 | grep 'EDE: '" + printf "%s\n" "${lines[@]}" + [[ ${lines[0]} == *"EDE: 15 (Blocked): (denylist)" ]] + [[ ${lines[1]} == "" ]] } @test "Gravity domain is blocked" { @@ -45,6 +49,10 @@ printf "%s\n" "${lines[@]}" [[ ${lines[0]} == "0.0.0.0" ]] [[ ${lines[1]} == "" ]] + run bash -c "dig gravity.ftl @127.0.0.1 | grep 'EDE: '" + printf "%s\n" "${lines[@]}" + [[ ${lines[0]} == *"EDE: 15 (Blocked): (gravity)" ]] + [[ ${lines[1]} == "" ]] } @test "Gravity domain is blocked (TCP)" { @@ -52,6 +60,10 @@ printf "%s\n" "${lines[@]}" [[ ${lines[0]} == "0.0.0.0" ]] [[ ${lines[1]} == "" ]] + run bash -c "dig gravity.ftl @127.0.0.1 +tcp | grep 'EDE: '" + printf "%s\n" "${lines[@]}" + [[ ${lines[0]} == *"EDE: 15 (Blocked): (gravity)" ]] + [[ ${lines[1]} == "" ]] } @test "Gravity domain + allowed exact match is not blocked" { @@ -77,6 +89,10 @@ printf "%s\n" "${lines[@]}" [[ ${lines[0]} == "0.0.0.0" ]] [[ ${lines[1]} == "" ]] + run bash -c "dig regex5.ftl @127.0.0.1 | grep 'EDE: '" + printf "%s\n" "${lines[@]}" + [[ ${lines[0]} == *"EDE: 15 (Blocked): (regex)" ]] + [[ ${lines[1]} == "" ]] } @test "Regex denylist mismatch is not blocked" { @@ -429,6 +445,7 @@ printf "%s\n" "${lines[@]}" [[ ${lines[@]} == *"status: NOERROR"* ]] [[ ${lines[@]} == *"null.ftl."*"2"*"IN"*"A"*"0.0.0.0"* ]] + [[ ${lines[@]} == *"EDE: 15 (Blocked): (upstream NULL)"* ]] # Get number of lines in the log after the test after="$(grep -c ^ /var/log/pihole/FTL.log)" @@ -483,6 +500,7 @@ printf "%s\n" "${lines[@]}" [[ ${lines[@]} == *"status: NOERROR"* ]] [[ ${lines[@]} == *"null.ftl."*"2"*"IN"*"AAAA"*"::"* ]] + [[ ${lines[@]} == *"EDE: 15 (Blocked): (upstream NULL)"* ]] # Get number of lines in the log after the test after="$(grep -c ^ /var/log/pihole/FTL.log)" @@ -507,6 +525,7 @@ # Run test run bash -c "dig A umbrella.ftl @127.0.0.1" + [[ ${lines[@]} == *"EDE: 15 (Blocked): (upstream IP)"* ]] # Get number of lines in the log after the test after="$(grep -c ^ /var/log/pihole/FTL.log)" @@ -531,6 +550,7 @@ # Run test run bash -c "dig A umbrella.ftl @127.0.0.1" + [[ ${lines[@]} == *"EDE: 15 (Blocked): (upstream IP)"* ]] # Get number of lines in the log after the test after="$(grep -c ^ /var/log/pihole/FTL.log)" @@ -1151,7 +1171,7 @@ @test "Blocking status is correctly logged in pihole.log" { run bash -c 'grep -c "gravity blocked gravity.ftl is 0.0.0.0" /var/log/pihole/pihole.log' printf "%s\n" "${lines[@]}" - [[ ${lines[0]} == "2" ]] + [[ ${lines[0]} == "4" ]] } @test "HTTP server responds with JSON error 404 to unknown API path" { @@ -1369,6 +1389,15 @@ run bash -c "dig AAAA pi.hole +short @127.0.0.1" printf "AAAA: %s\n" "${lines[@]}" [[ "${lines[0]}" == "fe80::10" ]] + + run bash -c "dig A pi.hole @127.0.0.1 | grep 'EDE: '" + printf "%s\n" "${lines[@]}" + [[ ${lines[0]} == *"EDE: 29: (synthesized)" ]] + [[ ${lines[1]} == "" ]] + run bash -c "dig AAAA pi.hole @127.0.0.1 | grep 'EDE: '" + printf "%s\n" "${lines[@]}" + [[ ${lines[0]} == *"EDE: 29: (synthesized)" ]] + [[ ${lines[1]} == "" ]] } @test "Pi-hole uses dns.reply.host.IPv4/6 for hostname" { @@ -1378,6 +1407,15 @@ run bash -c "dig AAAA $(hostname) +short @127.0.0.1" printf "AAAA: %s\n" "${lines[@]}" [[ "${lines[0]}" == "fe80::10" ]] + + run bash -c "dig A $(hostname) @127.0.0.1 | grep 'EDE: '" + printf "%s\n" "${lines[@]}" + [[ ${lines[0]} == *"EDE: 29: (synthesized)" ]] + [[ ${lines[1]} == "" ]] + run bash -c "dig AAAA $(hostname) @127.0.0.1 | grep 'EDE: '" + printf "%s\n" "${lines[@]}" + [[ ${lines[0]} == *"EDE: 29: (synthesized)" ]] + [[ ${lines[1]} == "" ]] } @test "Pi-hole uses dns.reply.blocking.IPv4/6 for blocked domain" { From c12c47cf58210de437d47a4fed5f9a6c39c0d098 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 21 Aug 2024 21:32:12 +0200 Subject: [PATCH 292/339] Add EDE 15 from upstream => Blocked detection + new query type representing this Signed-off-by: DL6ER --- src/api/docs/content/specs/stats.yaml | 4 + src/api/queries.c | 1 + src/database/query-table.c | 7 +- src/datastructure.c | 6 + src/dnsmasq_interface.c | 218 +++++++++++++++++--------- src/enums.h | 2 + src/gc.c | 1 + test/pdns/luadns.lua | 41 +++++ test/pdns/recursor.conf | 3 + test/pdns/setup.sh | 1 + test/test_suite.bats | 81 ++++++++-- 11 files changed, 274 insertions(+), 91 deletions(-) create mode 100644 test/pdns/luadns.lua diff --git a/src/api/docs/content/specs/stats.yaml b/src/api/docs/content/specs/stats.yaml index 3af53b19..919ef7af 100644 --- a/src/api/docs/content/specs/stats.yaml +++ b/src/api/docs/content/specs/stats.yaml @@ -486,6 +486,10 @@ components: type: integer description: Type CACHE_STALE queries example: 0 + EXTERNAL_BLOCKED_EDE15: + type: integer + description: Type EXTERNAL_BLOCKED_EDE15 queries + example: 0 replies: type: object description: Number of individual replies diff --git a/src/api/queries.c b/src/api/queries.c index 5b94d940..077efc27 100644 --- a/src/api/queries.c +++ b/src/api/queries.c @@ -1015,6 +1015,7 @@ int api_queries(struct ftl_conn *api) case QUERY_EXTERNAL_BLOCKED_IP: case QUERY_EXTERNAL_BLOCKED_NULL: case QUERY_EXTERNAL_BLOCKED_NXRA: + case QUERY_EXTERNAL_BLOCKED_EDE15: case QUERY_RETRIED: case QUERY_RETRIED_DNSSEC: case QUERY_IN_PROGRESS: diff --git a/src/database/query-table.c b/src/database/query-table.c index 1b661e77..94620946 100644 --- a/src/database/query-table.c +++ b/src/database/query-table.c @@ -1256,9 +1256,10 @@ void DB_read_queries(void) case QUERY_GRAVITY: // Blocked by gravity case QUERY_REGEX: // Blocked by regex denylist case QUERY_DENYLIST: // Blocked by exact denylist - case QUERY_EXTERNAL_BLOCKED_IP: // Blocked by external provider - case QUERY_EXTERNAL_BLOCKED_NULL: // Blocked by external provider - case QUERY_EXTERNAL_BLOCKED_NXRA: // Blocked by external provider + case QUERY_EXTERNAL_BLOCKED_IP: // Blocked upstream + case QUERY_EXTERNAL_BLOCKED_NULL: // Blocked upstream + case QUERY_EXTERNAL_BLOCKED_NXRA: // Blocked upstream + case QUERY_EXTERNAL_BLOCKED_EDE15: // Blocked upstream case QUERY_GRAVITY_CNAME: // Blocked by gravity (inside CNAME path) case QUERY_REGEX_CNAME: // Blocked by regex denylist (inside CNAME path) case QUERY_DENYLIST_CNAME: // Blocked by exact denylist (inside CNAME path) diff --git a/src/datastructure.c b/src/datastructure.c index 7d842845..f0980c34 100644 --- a/src/datastructure.c +++ b/src/datastructure.c @@ -700,6 +700,8 @@ const char * __attribute__ ((const)) get_query_status_str(const enum query_statu return "SPECIAL_DOMAIN"; case QUERY_CACHE_STALE: return "CACHE_STALE"; + case QUERY_EXTERNAL_BLOCKED_EDE15: + return "EXTERNAL_BLOCKED_EDE15"; case QUERY_STATUS_MAX: default: return "INVALID"; @@ -872,6 +874,7 @@ bool __attribute__ ((const)) is_blocked(const enum query_status status) case QUERY_EXTERNAL_BLOCKED_IP: case QUERY_EXTERNAL_BLOCKED_NULL: case QUERY_EXTERNAL_BLOCKED_NXRA: + case QUERY_EXTERNAL_BLOCKED_EDE15: case QUERY_GRAVITY_CNAME: case QUERY_REGEX_CNAME: case QUERY_DENYLIST_CNAME: @@ -969,6 +972,7 @@ bool __attribute__ ((const)) is_cached(const enum query_status status) case QUERY_EXTERNAL_BLOCKED_IP: case QUERY_EXTERNAL_BLOCKED_NULL: case QUERY_EXTERNAL_BLOCKED_NXRA: + case QUERY_EXTERNAL_BLOCKED_EDE15: case QUERY_GRAVITY_CNAME: case QUERY_REGEX_CNAME: case QUERY_DENYLIST_CNAME: @@ -1019,6 +1023,8 @@ static const char* __attribute__ ((const)) query_status_str(const enum query_sta return "SPECIAL_DOMAIN"; case QUERY_CACHE_STALE: return "CACHE_STALE"; + case QUERY_EXTERNAL_BLOCKED_EDE15: + return "EXTERNAL_BLOCKED_EDE15"; case QUERY_STATUS_MAX: return NULL; } diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c index 63fc0d92..92b31d99 100644 --- a/src/dnsmasq_interface.c +++ b/src/dnsmasq_interface.c @@ -68,7 +68,8 @@ static void _query_set_reply(const unsigned int flags, const enum reply_type rep const struct timeval response, const char *file, const int line); #define FTL_check_blocking(queryID, domainID, clientID) _FTL_check_blocking(queryID, domainID, clientID, __FILE__, __LINE__) static bool _FTL_check_blocking(int queryID, int domainID, int clientID, const char* file, const int line); -static void query_blocked(queriesData *query, domainsData* domain, clientsData* client, const enum query_status new_status); +static void query_blocked(queriesData *query, domainsData *domain, clientsData *client, + const enum query_status new_status, const enum domain_client_status cache_status); static void FTL_forwarded(const unsigned int flags, const char *name, const union all_addr *addr, unsigned short port, const int id, const char* file, const int line); static void FTL_reply(const unsigned int flags, const char *name, const union all_addr *addr, const char* arg, unsigned short type, const int id, const char* file, const int line); static void FTL_upstream_error(const union all_addr *addr, const unsigned int flags, const int id, const char* file, const int line); @@ -361,6 +362,10 @@ size_t _FTL_make_answer(struct dns_header *header, char *limit, const size_t len ede_code = EDE_BLOCKED; ede_text = "upstream IP"; break; + case UPSTREAM_BLOCKED_EDE15: + ede_code = EDE_BLOCKED; + ede_text = "upstream EDE 15"; + break; case PIHOLE_SYNTH: ede_code = EDE_SYNTHESIZED; ede_text = "synthesized"; @@ -1224,7 +1229,8 @@ static void set_dnscache_blockingstatus(DNSCacheData *dns_cache, enum domain_cli dns_cache->expires == 0 && (new_status == UPSTREAM_BLOCKED_NXRA || new_status == UPSTREAM_BLOCKED_NULL || - new_status == UPSTREAM_BLOCKED_IP)) + new_status == UPSTREAM_BLOCKED_IP || + new_status == UPSTREAM_BLOCKED_EDE15)) { // Set expiration time for this cache entry dns_cache->expires = time(NULL) + config.dns.cache.upstreamBlockedTTL.v.ui; @@ -1464,8 +1470,8 @@ static bool _FTL_check_blocking(int queryID, int domainID, int clientID, const c } // Memorize blocking status DNS cache for the domain/client combination - if(dns_cache->blocking_status != UNKNOWN_BLOCKED) - cacheStatus = dns_cache->blocking_status; + cacheStatus = dns_cache->blocking_status; + log_info("Set global cache status to %d", cacheStatus); // Skip the entire chain of tests if we already know the answer for this // particular client @@ -1490,7 +1496,7 @@ static bool _FTL_check_blocking(int queryID, int domainID, int clientID, const c if(!query->flags.allowed) { force_next_DNS_reply = dns_cache->force_reply; - query_blocked(query, domain, client, QUERY_DENYLIST); + query_blocked(query, domain, client, QUERY_DENYLIST, DENYLIST_BLOCKED); return true; } break; @@ -1506,7 +1512,7 @@ static bool _FTL_check_blocking(int queryID, int domainID, int clientID, const c if(!query->flags.allowed) { force_next_DNS_reply = dns_cache->force_reply; - query_blocked(query, domain, client, QUERY_GRAVITY); + query_blocked(query, domain, client, QUERY_GRAVITY, GRAVITY_BLOCKED); return true; } break; @@ -1524,7 +1530,7 @@ static bool _FTL_check_blocking(int queryID, int domainID, int clientID, const c { force_next_DNS_reply = dns_cache->force_reply; last_regex_idx = dns_cache->list_id; - query_blocked(query, domain, client, QUERY_REGEX); + query_blocked(query, domain, client, QUERY_REGEX, REGEX_BLOCKED); return true; } break; @@ -1546,7 +1552,7 @@ static bool _FTL_check_blocking(int queryID, int domainID, int clientID, const c log_debug(DEBUG_QUERIES, "%s is known as special domain", domainstr); force_next_DNS_reply = dns_cache->force_reply; - query_blocked(query, domain, client, QUERY_SPECIAL_DOMAIN); + query_blocked(query, domain, client, QUERY_SPECIAL_DOMAIN, SPECIAL_DOMAIN); return true; break; @@ -1561,18 +1567,47 @@ static bool _FTL_check_blocking(int queryID, int domainID, int clientID, const c case UPSTREAM_BLOCKED_IP: case UPSTREAM_BLOCKED_NULL: case UPSTREAM_BLOCKED_NXRA: + case UPSTREAM_BLOCKED_EDE15: + + enum query_status qstat; + + switch(dns_cache->blocking_status) + { + case UNKNOWN_BLOCKED: + case GRAVITY_BLOCKED: + case DENYLIST_BLOCKED: + case REGEX_BLOCKED: + case ALLOWED: + case SPECIAL_DOMAIN: + case PIHOLE_SYNTH: + case NOT_BLOCKED: + // Cannot happen + break; + case UPSTREAM_BLOCKED_IP: + qstat = QUERY_EXTERNAL_BLOCKED_IP; + blockingreason = "blocked upstream with known address"; + break; + case UPSTREAM_BLOCKED_NULL: + qstat = QUERY_EXTERNAL_BLOCKED_NULL; + blockingreason = "blocked upstream with NULL address"; + break; + case UPSTREAM_BLOCKED_EDE15: + qstat = QUERY_EXTERNAL_BLOCKED_EDE15; + blockingreason = "blocked upstream with EDE15"; + break; + case UPSTREAM_BLOCKED_NXRA: + blockingreason = "blocked upstream with NXRA address"; + qstat = QUERY_EXTERNAL_BLOCKED_NXRA; + break; + } + // Known as upstream blocked, we return this result // early, skipping all the lengthy tests below - blockingreason = "upstream blocked"; log_debug(DEBUG_QUERIES, "%s is known as %s (expires in %lus)", domainstr, blockingreason, (unsigned long)(dns_cache->expires - time(NULL))); force_next_DNS_reply = dns_cache->force_reply; - const enum query_status qstat = dns_cache->blocking_status == UPSTREAM_BLOCKED_IP ? - QUERY_EXTERNAL_BLOCKED_IP : - dns_cache->blocking_status == UPSTREAM_BLOCKED_NULL ? - QUERY_EXTERNAL_BLOCKED_NULL : QUERY_EXTERNAL_BLOCKED_NXRA; - query_blocked(query, domain, client, qstat); + query_blocked(query, domain, client, qstat, dns_cache->blocking_status); return true; break; @@ -1614,7 +1649,7 @@ static bool _FTL_check_blocking(int queryID, int domainID, int clientID, const c dns_cache->force_reply = force_next_DNS_reply; // Adjust counters - query_blocked(query, domain, client, QUERY_SPECIAL_DOMAIN); + query_blocked(query, domain, client, QUERY_SPECIAL_DOMAIN, SPECIAL_DOMAIN); // Debug output log_debug(DEBUG_QUERIES, "Special domain: %s is %s", domainstr, blockingreason); @@ -1636,6 +1671,9 @@ static bool _FTL_check_blocking(int queryID, int domainID, int clientID, const c { blockDomain = check_domain_blocked(domainstr + 6u, clientID, client, query, dns_cache, &new_status, &db_okay); + // Update DNS cache status + cacheStatus = dns_cache->blocking_status; + if(blockDomain) { // Truncate "_esni." from queried domain if the parenting domain was @@ -1655,7 +1693,7 @@ static bool _FTL_check_blocking(int queryID, int domainID, int clientID, const c if(blockDomain) { // Adjust counters - query_blocked(query, domain, client, new_status); + query_blocked(query, domain, client, new_status, cacheStatus); // Debug output if(config.debug.queries.v.b) @@ -1680,9 +1718,6 @@ static bool _FTL_check_blocking(int queryID, int domainID, int clientID, const c domainstr, query->flags.allowed ? "allowed" : "not blocked", dns_cache->list_id); } - // Update DNS cache status - cacheStatus = dns_cache->blocking_status; - free(domainstr); return blockDomain; } @@ -2314,10 +2349,11 @@ static void FTL_reply(const unsigned int flags, const char *name, const union al upstream->rtuncertainty += (mean - query->response)*(mean - query->response); // Only proceed if query is not already known - // to have been blocked by Quad9 + // to have been blocked upstream if(query->status == QUERY_EXTERNAL_BLOCKED_IP || query->status == QUERY_EXTERNAL_BLOCKED_NULL || - query->status == QUERY_EXTERNAL_BLOCKED_NXRA) + query->status == QUERY_EXTERNAL_BLOCKED_NXRA || + query->status == QUERY_EXTERNAL_BLOCKED_EDE15) { unlock_shm(); return; @@ -2507,25 +2543,21 @@ static enum query_status detect_blocked_IP(const unsigned short flags, const uni return QUERY_UNKNOWN; } -static void query_blocked(queriesData *query, domainsData *domain, clientsData *client, const enum query_status new_status) +static void query_blocked(queriesData *query, domainsData *domain, clientsData *client, + const enum query_status new_status, const enum domain_client_status cache_status) { // Get response time struct timeval response; gettimeofday(&response, 0); // Memorize this in the DNS cache if blocked due to the response - if(new_status == QUERY_EXTERNAL_BLOCKED_IP || - new_status == QUERY_EXTERNAL_BLOCKED_NULL || - new_status == QUERY_EXTERNAL_BLOCKED_NXRA) + if(cache_status != UNKNOWN_BLOCKED) { const int cacheID = findCacheID(query->domainID, query->clientID, query->type, true); DNSCacheData *dns_cache = getDNSCache(cacheID, true); - if(dns_cache != NULL) + if(dns_cache != NULL && dns_cache->blocking_status != cache_status) { // Update status - enum domain_client_status cache_status = new_status == QUERY_EXTERNAL_BLOCKED_IP ? UPSTREAM_BLOCKED_IP : - new_status == QUERY_EXTERNAL_BLOCKED_NULL ? UPSTREAM_BLOCKED_NULL : - UPSTREAM_BLOCKED_NXRA; // can be nothing else due to if above set_dnscache_blockingstatus(dns_cache, cache_status, client ? getstr(client->ippos) : NULL, domain ? getstr(domain->domainpos) : NULL); @@ -2764,7 +2796,7 @@ static void FTL_upstream_error(const union all_addr *addr, const unsigned int fl unlock_shm(); } -static void FTL_NXRA(const int id, const char* file, const int line) +static void FTL_blocked_upstream_by_header(const enum domain_client_status new_status, const int id, const char* file, const int line) { // Lock shared memory lock_shm(); @@ -2800,13 +2832,15 @@ static void FTL_NXRA(const int id, const char* file, const int line) if(config.debug.queries.v.b) { // Get domain name (domain cannot be NULL here) - const char *domainname = getstr(domain->domainpos); - log_debug(DEBUG_QUERIES, "**** %s externally blocked (ID %i, FTL %i, %s:%i)", domainname, id, queryID, file, line); + const char *domainstr = getstr(domain->domainpos); + log_debug(DEBUG_QUERIES, "**** %s externally blocked by header (ID %i, FTL %i, %s:%i)", domainstr, id, queryID, file, line); } // Set blocking reason - blockingreason = "blocked upstream with NXDOMAIN and unset RA bit"; - cacheStatus = UPSTREAM_BLOCKED_NXRA; + blockingreason = new_status == UPSTREAM_BLOCKED_NXRA ? + "blocked upstream with NXDOMAIN + no RA" : + "blocked upstream with EDE15"; + cacheStatus = new_status; // Get response time struct timeval response; @@ -2815,7 +2849,12 @@ static void FTL_NXRA(const int id, const char* file, const int line) // Store query as externally blocked clientsData *client = getClient(query->clientID, true); if(client != NULL) - query_blocked(query, domain, client, QUERY_EXTERNAL_BLOCKED_NXRA); + { + const enum query_status new_qstatus = new_status == UPSTREAM_BLOCKED_NXRA ? + QUERY_EXTERNAL_BLOCKED_NXRA : + QUERY_EXTERNAL_BLOCKED_EDE15; + query_blocked(query, domain, client, new_qstatus, new_status); + } // Store reply type as replied with NXDOMAIN query_set_reply(F_NEG | F_NXDOMAIN, 0, NULL, query, response); @@ -2827,11 +2866,60 @@ static void FTL_NXRA(const int id, const char* file, const int line) unlock_shm(); } +static void FTL_blocked_upstream_by_addr(const enum query_status new_qstatus, const int id, const char* file, const int line) +{ + // Lock shared memory + lock_shm(); + + // Save status in corresponding query identified by dnsmasq's ID + const int queryID = findQueryID(id); + if(queryID < 0) + { + // This may happen e.g. if the original query was "pi.hole" + log_debug(DEBUG_QUERIES, "FTL_check_reply(): Query %i has not been found", id); + unlock_shm(); + return; + } + + // Get query pointer + queriesData *query = getQuery(queryID, true); + if(query == NULL) + { + // Memory error, skip this query + log_debug(DEBUG_QUERIES, "FTL_check_reply(): Memory error (ID %i)", id); + unlock_shm(); + return; + } + clientsData *client = getClient(query->clientID, true); + domainsData *domain = getDomain(query->domainID, true); + if(client != NULL && domain != NULL) + { + const enum domain_client_status new_status = new_qstatus == QUERY_EXTERNAL_BLOCKED_IP ? + UPSTREAM_BLOCKED_IP : + UPSTREAM_BLOCKED_NULL; + query_blocked(query, domain, client, new_qstatus, new_status); + } + + // Possible debugging information + if(config.debug.queries.v.b) + { + // Get domain name (domain cannot be NULL here) + const char *domainname = domain ? getstr(domain->domainpos) : ""; + log_debug(DEBUG_QUERIES, "**** %s externally blocked by address (ID %i, FTL %i, %s:%i)", domainname, id, queryID, file, line); + } + + // Mark query for updating in the database + query->flags.database.changed = true; + + // Unlock shared memory + unlock_shm(); +} + int _FTL_check_reply(const unsigned int rcode, const unsigned short flags, const union all_addr *addr, const int id, const char* file, const int line) { - + ednsData *edns = getEDNS(); // Check if RA bit is unset in DNS header and rcode is NXDOMAIN // If the response code (rcode) is NXDOMAIN, we may be seeing a response from // an externally blocked query. As they are not always accompany a necessary @@ -2839,55 +2927,37 @@ int _FTL_check_reply(const unsigned int rcode, const unsigned short flags, // FTL_reply() is never getting called from within the cache routines. // Hence, we have to store the necessary information about the NXDOMAIN // reply already here. - if(addr == NULL && !rabit && rcode == NXDOMAIN) + // Alternatively, we also consider EDE15 as a blocking reason. + if(addr == NULL) { // RA bit is not set and rcode is NXDOMAIN - FTL_NXRA(id, file, line); + if(!rabit && rcode == NXDOMAIN) + { + FTL_blocked_upstream_by_header(UPSTREAM_BLOCKED_NXRA, id, file, line); - // Query is blocked - return 1; + // Query is blocked + return 1; + } + + // EDE 15 + if(edns != NULL && edns->ede == EDE_BLOCKED) + { + FTL_blocked_upstream_by_header(UPSTREAM_BLOCKED_EDE15, id, file, line); + + // Query is blocked + return 1; + } } // Further checks if this is an IP address else if(addr != NULL) { // Detect if returned IP indicates that this query was blocked - const enum query_status new_status = detect_blocked_IP(flags, addr); + const enum query_status new_qstatus = detect_blocked_IP(flags, addr); // Update status of this query if detected as external blocking - if(new_status != QUERY_UNKNOWN) + if(new_qstatus != QUERY_UNKNOWN) { - // Lock shared memory - lock_shm(); - - // Save status in corresponding query identified by dnsmasq's ID - const int queryID = findQueryID(id); - if(queryID < 0) - { - // This may happen e.g. if the original query was "pi.hole" - log_debug(DEBUG_QUERIES, "FTL_check_reply(): Query %i has not been found", id); - unlock_shm(); - return 0; - } - - // Get query pointer - queriesData *query = getQuery(queryID, true); - if(query == NULL) - { - // Memory error, skip this query - log_debug(DEBUG_QUERIES, "FTL_check_reply(): Memory error (ID %i)", id); - unlock_shm(); - return 0; - } - clientsData *client = getClient(query->clientID, true); - domainsData *domain = getDomain(query->domainID, true); - if(client != NULL && domain != NULL) - query_blocked(query, domain, client, new_status); - - // Mark query for updating in the database - query->flags.database.changed = true; - - // Unlock shared memory - unlock_shm(); + FTL_blocked_upstream_by_addr(new_qstatus, id, file, line); // Query is blocked return 1; diff --git a/src/enums.h b/src/enums.h index 788c1987..4ea76b78 100644 --- a/src/enums.h +++ b/src/enums.h @@ -49,6 +49,7 @@ enum query_status { QUERY_DBBUSY, QUERY_SPECIAL_DOMAIN, QUERY_CACHE_STALE, + QUERY_EXTERNAL_BLOCKED_EDE15, QUERY_STATUS_MAX } __attribute__ ((packed)); @@ -133,6 +134,7 @@ enum domain_client_status { UPSTREAM_BLOCKED_NXRA, UPSTREAM_BLOCKED_NULL, UPSTREAM_BLOCKED_IP, + UPSTREAM_BLOCKED_EDE15, PIHOLE_SYNTH, NOT_BLOCKED } __attribute__ ((packed)); diff --git a/src/gc.c b/src/gc.c index 613ac6f6..1aaf036a 100644 --- a/src/gc.c +++ b/src/gc.c @@ -354,6 +354,7 @@ void runGC(const time_t now, time_t *lastGCrun, const bool flush) case QUERY_EXTERNAL_BLOCKED_IP: // Blocked by upstream provider (fall through) case QUERY_EXTERNAL_BLOCKED_NXRA: // Blocked by upstream provider (fall through) case QUERY_EXTERNAL_BLOCKED_NULL: // Blocked by upstream provider (fall through) + case QUERY_EXTERNAL_BLOCKED_EDE15: // Blocked by upstream provider (fall through) case QUERY_GRAVITY_CNAME: // Gravity domain in CNAME chain (fall through) case QUERY_REGEX_CNAME: // Regex denied domain in CNAME chain (fall through) case QUERY_DENYLIST_CNAME: // Exactly denied domain in CNAME chain (fall through) diff --git a/test/pdns/luadns.lua b/test/pdns/luadns.lua new file mode 100644 index 00000000..dd88dbe0 --- /dev/null +++ b/test/pdns/luadns.lua @@ -0,0 +1,41 @@ +refused_ede15 = newDN("refused.ede15.ftl") +nxdomain_ede15 = newDN("nxdomain.ede15.ftl") +null_ede15 = newDN("null.ede15.ftl") + + +-- this hook is called before doing any resolving +function preresolve(dq) + pdnslog("Got question for "..dq.qname:toString().." from "..dq.remoteaddr:toString().." to "..dq.localaddr:toString()) + + if dq.qname == refused_ede15 then + pdnslog("Blocking REFUSED + EDE 15 for "..dq.qname:toString()) + -- Set EDE 15 in response + dq.extendedErrorCode = 15 + -- Set REFUSED in response + dq.rcode = pdns.REFUSED + return true + end + + if dq.qname == nxdomain_ede15 then + pdnslog("Blocking NXDOMAIN + EDE 15 for "..dq.qname:toString()) + -- Set EDE 15 in response + dq.extendedErrorCode = 15 + -- Set NXDOMAIN in response + dq.rcode = pdns.NXDOMAIN + return true + end + + if dq.qname == null_ede15 then + pdnslog("Blocking NULL + EDE 15 for "..dq.qname:toString()) + -- Set EDE 15 in response + dq.extendedErrorCode = 15 + -- Add a NULL RR to the response + dq:addAnswer(pdns.A, "0.0.0.0") + dq:addAnswer(pdns.AAAA, "::") + return true + end + + -- as we do not set dq.variable, our decision here will be cached + + return false +end diff --git a/test/pdns/recursor.conf b/test/pdns/recursor.conf index 0694e52b..3021a700 100644 --- a/test/pdns/recursor.conf +++ b/test/pdns/recursor.conf @@ -21,3 +21,6 @@ forward-zones=ftl=127.0.0.1:5554,168.192.in-addr.arpa=127.0.0.1:5554,ip6.arpa=12 # requested by the client. # The default mode until PowerDNS Recursor 4.5.0. dnssec=process-no-validate + +# Enable LUA support +lua-dns-script=/etc/pdns/luadns.lua diff --git a/test/pdns/setup.sh b/test/pdns/setup.sh index e887fa03..9aaaf28d 100644 --- a/test/pdns/setup.sh +++ b/test/pdns/setup.sh @@ -25,6 +25,7 @@ else exit 1 fi +cp test/pdns/luadns.lua /etc/pdns/luadns.lua cp test/pdns/recursor.conf $RECURSOR_CONF # Create zone database diff --git a/test/test_suite.bats b/test/test_suite.bats index e9a4b6de..6f339cb9 100644 --- a/test/test_suite.bats +++ b/test/test_suite.bats @@ -436,7 +436,7 @@ # NXRA + RA unset cannot be tested with PowerDNS as upstream provider -@test "Externally blocked domain: NULL is recognized" { +@test "Upstream blocked domain: NULL is recognized" { # Get number of lines in the log before the test before="$(grep -c ^ /var/log/pihole/FTL.log)" @@ -460,11 +460,11 @@ printf "%s\n" "${lines[@]}" [[ ${lines[@]} == *"DEBUG_QUERIES: DNS cache: A/127.0.0.1/null.ftl is not blocked (domainlist ID: -1)"* ]] [[ ${lines[@]} == *"DEBUG_QUERIES: **** forwarded null.ftl to 127.0.0.1#5555"* ]] - [[ ${lines[@]} == *"DEBUG_QUERIES: DNS cache: A/127.0.0.1/null.ftl is blocked upstream with 0.0.0.0"* ]] + [[ ${lines[@]} == *"DEBUG_QUERIES: blocked upstream with 0.0.0.0"* ]] [[ ${lines[@]} == *"DEBUG_QUERIES: Adding RR: \"null.ftl A 0.0.0.0\""* ]] } -@test "Externally blocked domain: NULL is recognized (cached)" { +@test "Upstream blocked domain: NULL is recognized (cached)" { # Get number of lines in the log before the test before="$(grep -c ^ /var/log/pihole/FTL.log)" @@ -485,13 +485,12 @@ lines+=("$line") done <<< "${log}" printf "%s\n" "${lines[@]}" - [[ ${lines[@]} == *"DEBUG_QUERIES: null.ftl is known as upstream blocked"* ]] - [[ ${lines[@]} == *"DEBUG_QUERIES: DNS cache: A/127.0.0.1/null.ftl is upstream blocked"* ]] + [[ ${lines[@]} == *"DEBUG_QUERIES: null.ftl is known as blocked upstream with NULL address"* ]] [[ ${lines[@]} != *"DEBUG_QUERIES: **** forwarded null.ftl to 127.0.0.1#5555"* ]] [[ ${lines[@]} == *"DEBUG_QUERIES: Adding RR: \"null.ftl A 0.0.0.0\""* ]] } -@test "Externally blocked domain: NULL is recognized (IPv6)" { +@test "Upstream blocked domain: NULL is recognized (IPv6)" { # Get number of lines in the log before the test before="$(grep -c ^ /var/log/pihole/FTL.log)" @@ -515,16 +514,17 @@ printf "%s\n" "${lines[@]}" [[ ${lines[@]} == *"DEBUG_QUERIES: DNS cache: AAAA/127.0.0.1/null.ftl is not blocked (domainlist ID: -1)"* ]] [[ ${lines[@]} == *"DEBUG_QUERIES: **** forwarded null.ftl to 127.0.0.1#5555"* ]] - [[ ${lines[@]} == *"DEBUG_QUERIES: DNS cache: AAAA/127.0.0.1/null.ftl is blocked upstream with ::"* ]] + [[ ${lines[@]} == *"DEBUG_QUERIES: blocked upstream with ::"* ]] [[ ${lines[@]} == *"DEBUG_QUERIES: Adding RR: \"null.ftl AAAA ::\""* ]] } -@test "Externally blocked domain: IP is recognized" { +@test "Upstream blocked domain: IP is recognized" { # Get number of lines in the log before the test before="$(grep -c ^ /var/log/pihole/FTL.log)" # Run test run bash -c "dig A umbrella.ftl @127.0.0.1" + printf "%s\n" "${lines[@]}" [[ ${lines[@]} == *"EDE: 15 (Blocked): (upstream IP)"* ]] # Get number of lines in the log after the test @@ -540,16 +540,17 @@ printf "%s\n" "${lines[@]}" [[ ${lines[@]} == *"DEBUG_QUERIES: DNS cache: A/127.0.0.1/umbrella.ftl is not blocked (domainlist ID: -1)"* ]] [[ ${lines[@]} == *"DEBUG_QUERIES: **** forwarded umbrella.ftl to 127.0.0.1#5555"* ]] - [[ ${lines[@]} == *"DEBUG_QUERIES: DNS cache: A/127.0.0.1/umbrella.ftl is blocked upstream with known address (IPv4)"* ]] + [[ ${lines[@]} == *"DEBUG_QUERIES: blocked upstream with known address (IPv4)"* ]] [[ ${lines[@]} == *"DEBUG_QUERIES: Adding RR: \"umbrella.ftl A 0.0.0.0\""* ]] } -@test "Externally blocked domain: IP is recognized (cached)" { +@test "Upstream blocked domain: IP is recognized (cached)" { # Get number of lines in the log before the test before="$(grep -c ^ /var/log/pihole/FTL.log)" # Run test run bash -c "dig A umbrella.ftl @127.0.0.1" + printf "%s\n" "${lines[@]}" [[ ${lines[@]} == *"EDE: 15 (Blocked): (upstream IP)"* ]] # Get number of lines in the log after the test @@ -563,18 +564,18 @@ lines+=("$line") done <<< "${log}" printf "%s\n" "${lines[@]}" - [[ ${lines[@]} == *"DEBUG_QUERIES: umbrella.ftl is known as upstream blocked"* ]] - [[ ${lines[@]} == *"DEBUG_QUERIES: DNS cache: A/127.0.0.1/umbrella.ftl is upstream blocked"* ]] + [[ ${lines[@]} == *"DEBUG_QUERIES: umbrella.ftl is known as blocked upstream with known address"* ]] [[ ${lines[@]} != *"DEBUG_QUERIES: **** forwarded umbrella.ftl to 127.0.0.1#5555"* ]] [[ ${lines[@]} == *"DEBUG_QUERIES: Adding RR: \"umbrella.ftl A 0.0.0.0\""* ]] } -@test "Externally blocked domain: IP is recognized (IPv6)" { +@test "Upstream blocked domain: IP is recognized (IPv6)" { # Get number of lines in the log before the test before="$(grep -c ^ /var/log/pihole/FTL.log)" # Run test run bash -c "dig AAAA umbrella.ftl @127.0.0.1" + printf "%s\n" "${lines[@]}" # Get number of lines in the log after the test after="$(grep -c ^ /var/log/pihole/FTL.log)" @@ -593,12 +594,13 @@ [[ ${lines[@]} == *"DEBUG_QUERIES: Adding RR: \"umbrella.ftl AAAA ::\""* ]] } -@test "Externally blocked domain: IP is recognized (multi)" { +@test "Upstream blocked domain: IP is recognized (multi)" { # Get number of lines in the log before the test before="$(grep -c ^ /var/log/pihole/FTL.log)" # Run test run bash -c "dig A umbrella-multi.ftl @127.0.0.1" + printf "%s\n" "${lines[@]}" # Get number of lines in the log after the test after="$(grep -c ^ /var/log/pihole/FTL.log)" @@ -617,6 +619,57 @@ [[ ${lines[@]} == *"DEBUG_QUERIES: Adding RR: \"umbrella-multi.ftl A 0.0.0.0\""* ]] } +@test "Upstream blocked domain: EDE 15 is recognized" { + # Get number of lines in the log before the test + before="$(grep -c ^ /var/log/pihole/FTL.log)" + + # Run test + run bash -c "dig A nxdomain.ede15.ftl @127.0.0.1" + printf "%s\n" "${lines[@]}" + [[ ${lines[@]} == *"EDE: 15 (Blocked): (upstream EDE 15)"* ]] + + # Get number of lines in the log after the test + after="$(grep -c ^ /var/log/pihole/FTL.log)" + + # Extract relevant log lines + log="$(sed -n "${before},${after}p" /var/log/pihole/FTL.log)" + # Split log into array by newline + lines=() + while IFS= read -r line; do + lines+=("$line") + done <<< "${log}" + printf "%s\n" "${lines[@]}" + [[ ${lines[@]} == *"DEBUG_QUERIES: DNS cache: A/127.0.0.1/nxdomain.ede15.ftl is not blocked (domainlist ID: -1)"* ]] + [[ ${lines[@]} == *"DEBUG_QUERIES: **** forwarded nxdomain.ede15.ftl to 127.0.0.1#5555"* ]] + [[ ${lines[@]} == *"DEBUG_QUERIES: DNS cache: A/127.0.0.1/nxdomain.ede15.ftl is blocked upstream with EDE15"* ]] + [[ ${lines[@]} == *"DEBUG_QUERIES: Adding RR: \"nxdomain.ede15.ftl A 0.0.0.0\""* ]] +} + +@test "Upstream blocked domain: EDE 15 is recognized (cached)" { + # Get number of lines in the log before the test + before="$(grep -c ^ /var/log/pihole/FTL.log)" + + # Run test + run bash -c "dig A nxdomain.ede15.ftl @127.0.0.1" + printf "%s\n" "${lines[@]}" + [[ ${lines[@]} == *"EDE: 15 (Blocked): (upstream EDE 15)"* ]] + + # Get number of lines in the log after the test + after="$(grep -c ^ /var/log/pihole/FTL.log)" + + # Extract relevant log lines + log="$(sed -n "${before},${after}p" /var/log/pihole/FTL.log)" + # Split log into array by newline + lines=() + while IFS= read -r line; do + lines+=("$line") + done <<< "${log}" + printf "%s\n" "${lines[@]}" + [[ ${lines[@]} == *"DEBUG_QUERIES: nxdomain.ede15.ftl is known as blocked upstream with EDE15"* ]] + [[ ${lines[@]} != *"DEBUG_QUERIES: **** forwarded umbrella.ftl to 127.0.0.1#5555"* ]] + [[ ${lines[@]} == *"DEBUG_QUERIES: Adding RR: \"nxdomain.ede15.ftl A 0.0.0.0\""* ]] +} + @test "ABP-style matching working as expected" { run bash -c "dig A special.gravity.ftl @127.0.0.1 +short" printf "%s\n" "${lines[@]}" From 3e090f52c73d5999b726c6f517c38771f9c5b6d2 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 21 Aug 2024 22:17:01 +0200 Subject: [PATCH 293/339] Unify query and cache status enums Signed-off-by: DL6ER --- src/config/toml_helper.c | 2 +- src/datastructure.c | 60 ++++++++- src/datastructure.h | 5 +- src/dnsmasq_interface.c | 268 ++++++++++++++++----------------------- src/enums.h | 18 --- test/test_suite.bats | 14 +- 6 files changed, 183 insertions(+), 184 deletions(-) diff --git a/src/config/toml_helper.c b/src/config/toml_helper.c index 58e8f8fc..a4a402d7 100644 --- a/src/config/toml_helper.c +++ b/src/config/toml_helper.c @@ -658,7 +658,7 @@ void readTOMLvalue(struct conf_item *conf_item, const char* key, toml_table_t *t } case CONF_ENUM_BLOCKING_EDNS_MODE: { - const toml_datum_t val = toml_string_in(toml, key); + toml_datum_t val = toml_string_in(toml, key); if(val.ok) { const int edns_mode = get_edns_mode_val(val.u.s); diff --git a/src/datastructure.c b/src/datastructure.c index f0980c34..b9d0c1d9 100644 --- a/src/datastructure.c +++ b/src/datastructure.c @@ -435,7 +435,8 @@ int _findCacheID(const int domainID, const int clientID, const enum query_type q // Initialize cache entry dns_cache->magic = MAGICBYTE; - dns_cache->blocking_status = UNKNOWN_BLOCKED; + dns_cache->blocking_status = QUERY_UNKNOWN; + dns_cache->expires = 0; dns_cache->domainID = domainID; dns_cache->clientID = clientID; dns_cache->query_type = query_type; @@ -570,7 +571,9 @@ void FTL_reset_per_client_domain_data(void) continue; // Reset blocking status - dns_cache->blocking_status = UNKNOWN_BLOCKED; + dns_cache->blocking_status = QUERY_UNKNOWN; + // Reset expiry + dns_cache->expires = 0; // Reset domainlist ID dns_cache->list_id = -1; } @@ -1069,6 +1072,59 @@ void _query_set_status(queriesData *query, const enum query_status new_status, c return; } + // Memorize this in the DNS cache if blocked due to the response + // We do not cache intermittent statuses as they are subject to change + if(!init && + new_status != QUERY_UNKNOWN && + new_status != QUERY_DBBUSY && + new_status != QUERY_IN_PROGRESS && + new_status != QUERY_RETRIED && + new_status != QUERY_RETRIED_DNSSEC) + { + const int cacheID = findCacheID(query->domainID, query->clientID, query->type, true); + DNSCacheData *dns_cache = getDNSCache(cacheID, true); + if(dns_cache != NULL && dns_cache->blocking_status != new_status) + { + // Memorize blocking status DNS cache for the domain/client combination + dns_cache->blocking_status = new_status; + + // Set expiration time for this cache entry (if applicable) + // We set this only if not already set to avoid extending the TTL of an + // existing entry + if(config.dns.cache.upstreamBlockedTTL.v.ui > 0 && + dns_cache->expires == 0 && + (new_status == QUERY_EXTERNAL_BLOCKED_NXRA || + new_status == QUERY_EXTERNAL_BLOCKED_NULL || + new_status == QUERY_EXTERNAL_BLOCKED_IP || + new_status == QUERY_EXTERNAL_BLOCKED_EDE15)) + { + // Set expiration time for this cache entry + dns_cache->expires = time(NULL) + config.dns.cache.upstreamBlockedTTL.v.ui; + } + + if(config.debug.queries.v.b) + { + // Debug logging + const char *qtype = get_query_type_str(dns_cache->query_type, NULL, NULL); + const char *domain = getDomainString(query); + const char *clientstr = getClientIPString(query); + const char *statusstr = get_query_status_str(new_status); + + if(dns_cache->expires > 0) + { + log_debug(DEBUG_QUERIES, "DNS cache: %s/%s/%s -> %s, expires in %lis", + qtype, clientstr, domain, statusstr, + (long)(dns_cache->expires - time(NULL))); + } + else + { + log_debug(DEBUG_QUERIES, "DNS cache: %s/%s/%s -> %s, no expiry", + qtype, clientstr, domain, statusstr); + } + } + } + } + // else: update global counters, ... if(!init) { diff --git a/src/datastructure.h b/src/datastructure.h index 5e7cedc4..3e204939 100644 --- a/src/datastructure.h +++ b/src/datastructure.h @@ -108,7 +108,10 @@ typedef struct { typedef struct { unsigned char magic; - enum domain_client_status blocking_status; + struct { + bool allowed :1; + } flags; + enum query_status blocking_status; enum reply_type force_reply; enum query_type query_type; int domainID; diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c index 92b31d99..4d91ad12 100644 --- a/src/dnsmasq_interface.c +++ b/src/dnsmasq_interface.c @@ -68,8 +68,7 @@ static void _query_set_reply(const unsigned int flags, const enum reply_type rep const struct timeval response, const char *file, const int line); #define FTL_check_blocking(queryID, domainID, clientID) _FTL_check_blocking(queryID, domainID, clientID, __FILE__, __LINE__) static bool _FTL_check_blocking(int queryID, int domainID, int clientID, const char* file, const int line); -static void query_blocked(queriesData *query, domainsData *domain, clientsData *client, - const enum query_status new_status, const enum domain_client_status cache_status); +static void query_blocked(queriesData *query, domainsData *domain, clientsData *client, const enum query_status new_status); static void FTL_forwarded(const unsigned int flags, const char *name, const union all_addr *addr, unsigned short port, const int id, const char* file, const int line); static void FTL_reply(const unsigned int flags, const char *name, const union all_addr *addr, const char* arg, unsigned short type, const int id, const char* file, const int line); static void FTL_upstream_error(const union all_addr *addr, const unsigned int flags, const int id, const char* file, const int line); @@ -86,7 +85,7 @@ static const char *check_dnsmasq_name(const char *name); static bool adbit = false, rabit = false; static const char *blockingreason = ""; static enum reply_type force_next_DNS_reply = REPLY_UNKNOWN; -static enum domain_client_status cacheStatus = UNKNOWN_BLOCKED; +static enum query_status cacheStatus = QUERY_UNKNOWN; static int last_regex_idx = -1; static char *pihole_suffix = NULL; static char *hostname_suffix = NULL; @@ -329,49 +328,69 @@ size_t _FTL_make_answer(struct dns_header *header, char *limit, const size_t len const char *ede_text = NULL; switch(cacheStatus) { - case UNKNOWN_BLOCKED: - case NOT_BLOCKED: - case ALLOWED: + case QUERY_UNKNOWN: +// case QUERY_CACHE: + case QUERY_FORWARDED: + case QUERY_RETRIED: + case QUERY_RETRIED_DNSSEC: + case QUERY_IN_PROGRESS: + case QUERY_DBBUSY: + case QUERY_CACHE_STALE: + case QUERY_STATUS_MAX: // Not going through this function break; - case GRAVITY_BLOCKED: + case QUERY_GRAVITY: ede_code = EDE_BLOCKED; ede_text = "gravity"; break; - case DENYLIST_BLOCKED: + case QUERY_GRAVITY_CNAME: + ede_code = EDE_BLOCKED; + ede_text = "gravity (CNAME)"; + break; + case QUERY_DENYLIST: ede_code = EDE_BLOCKED; ede_text = "denylist"; break; - case REGEX_BLOCKED: + case QUERY_DENYLIST_CNAME: + ede_code = EDE_BLOCKED; + ede_text = "denylist (CNAME)"; + break; + case QUERY_REGEX: ede_code = EDE_BLOCKED; ede_text = "regex"; break; - case SPECIAL_DOMAIN: + case QUERY_REGEX_CNAME: + ede_code = EDE_BLOCKED; + ede_text = "regex (CNAME)"; + break; + case QUERY_SPECIAL_DOMAIN: ede_code = EDE_BLOCKED; ede_text = "special"; break; - case UPSTREAM_BLOCKED_NXRA: + case QUERY_EXTERNAL_BLOCKED_NXRA: ede_code = EDE_BLOCKED; ede_text = "upstream NXRA"; break; - case UPSTREAM_BLOCKED_NULL: + case QUERY_EXTERNAL_BLOCKED_NULL: ede_code = EDE_BLOCKED; ede_text = "upstream NULL"; break; - case UPSTREAM_BLOCKED_IP: + case QUERY_EXTERNAL_BLOCKED_IP: ede_code = EDE_BLOCKED; ede_text = "upstream IP"; break; - case UPSTREAM_BLOCKED_EDE15: + case QUERY_EXTERNAL_BLOCKED_EDE15: ede_code = EDE_BLOCKED; ede_text = "upstream EDE 15"; break; - case PIHOLE_SYNTH: + case QUERY_CACHE: ede_code = EDE_SYNTHESIZED; ede_text = "synthesized"; break; } - cacheStatus = UNKNOWN_BLOCKED; + + // Reset global DNS cache status + cacheStatus = QUERY_UNKNOWN; // Debug logging log_debug(DEBUG_QUERIES, "Setting EDE: %s (%d) + \"%s\"", @@ -662,7 +681,7 @@ bool _FTL_new_query(const unsigned int flags, const char *name, "interface-local IP address" : "NODATA due to missing iface address"); - cacheStatus = PIHOLE_SYNTH; + cacheStatus = QUERY_CACHE; return true; } else @@ -1216,37 +1235,6 @@ static void check_pihole_PTR(char *domain) } } -static void set_dnscache_blockingstatus(DNSCacheData *dns_cache, enum domain_client_status new_status, - const char *client, const char *domain) -{ - // Memorize blocking status DNS cache for the domain/client combination - dns_cache->blocking_status = new_status; - - // Set expiration time for this cache entry (if applicable) - // We set this only if not already set to avoid extending the TTL of an - // existing entry - if(config.dns.cache.upstreamBlockedTTL.v.ui > 0 && - dns_cache->expires == 0 && - (new_status == UPSTREAM_BLOCKED_NXRA || - new_status == UPSTREAM_BLOCKED_NULL || - new_status == UPSTREAM_BLOCKED_IP || - new_status == UPSTREAM_BLOCKED_EDE15)) - { - // Set expiration time for this cache entry - dns_cache->expires = time(NULL) + config.dns.cache.upstreamBlockedTTL.v.ui; - } - - if(!config.debug.queries.v.b) - return; - - // Debug logging - const char *qtype = get_query_type_str(dns_cache->query_type, NULL, NULL); - const char *clientstr = client ? client : ""; - log_debug(DEBUG_QUERIES, "DNS cache: %s/%s/%s is %s, expires in %lis", - qtype, clientstr, domain, blockingreason, - dns_cache->expires > 0 ? (long)(dns_cache->expires - time(NULL)) : -1); -} - static bool check_domain_blocked(const char *domain, const int clientID, clientsData *client, queriesData *query, DNSCacheData *dns_cache, enum query_status *new_status, bool *db_okay) @@ -1263,9 +1251,6 @@ static bool check_domain_blocked(const char *domain, const int clientID, *new_status = QUERY_DENYLIST; blockingreason = "exactly denied"; - // Mark domain as exactly denied for this client - set_dnscache_blockingstatus(dns_cache, DENYLIST_BLOCKED, client ? getstr(client->ippos) : NULL, domain); - // We block this domain return true; } @@ -1301,9 +1286,6 @@ static bool check_domain_blocked(const char *domain, const int clientID, *new_status = QUERY_GRAVITY; blockingreason = "gravity blocked"; - // Mark domain as gravity blocked for this client - set_dnscache_blockingstatus(dns_cache, GRAVITY_BLOCKED, client ? getstr(client->ippos) : NULL, domain); - log_debug(DEBUG_QUERIES, "Blocking query due to gravity match (list ID %i)", list_id); // Store ID of the matching gravity list @@ -1361,9 +1343,6 @@ static bool check_domain_blocked(const char *domain, const int clientID, *new_status = QUERY_REGEX; blockingreason = "regex denied"; - // Mark domain as regex matched for this client - set_dnscache_blockingstatus(dns_cache, REGEX_BLOCKED, client ? getstr(client->ippos) : NULL, domain); - // Regex may be overwriting reply type for this domain if(dns_cache->force_reply != REPLY_UNKNOWN) force_next_DNS_reply = dns_cache->force_reply; @@ -1464,7 +1443,8 @@ static bool _FTL_check_blocking(int queryID, int domainID, int clientID, const c { // This cache record is expired, we have to re-check log_debug(DEBUG_QUERIES, "DNS cache record expired"); - dns_cache->blocking_status = UNKNOWN_BLOCKED; + dns_cache->blocking_status = QUERY_UNKNOWN; + dns_cache->flags.allowed = false; dns_cache->expires = 0; dns_cache->list_id = -1; } @@ -1478,17 +1458,18 @@ static bool _FTL_check_blocking(int queryID, int domainID, int clientID, const c char *domainstr = (char*)getstr(domain->domainpos); switch(dns_cache->blocking_status) { - case UNKNOWN_BLOCKED: + case QUERY_UNKNOWN: // New domain/client combination. // We have to go through all the tests below log_debug(DEBUG_QUERIES, "%s is not known", domainstr); break; - case DENYLIST_BLOCKED: + case QUERY_DENYLIST: + case QUERY_DENYLIST_CNAME: // Known as exactly denied, we return this result early, skipping // all the lengthy tests below - blockingreason = "exactly denied"; + blockingreason = dns_cache->blocking_status == QUERY_DENYLIST ? "exactly denied" : "exactly denied (CNAME)"; log_debug(DEBUG_QUERIES, "%s is known as %s", domainstr, blockingreason); // Do not block if the entire query is to be permitted @@ -1496,15 +1477,16 @@ static bool _FTL_check_blocking(int queryID, int domainID, int clientID, const c if(!query->flags.allowed) { force_next_DNS_reply = dns_cache->force_reply; - query_blocked(query, domain, client, QUERY_DENYLIST, DENYLIST_BLOCKED); + query_blocked(query, domain, client, QUERY_DENYLIST); return true; } break; - case GRAVITY_BLOCKED: + case QUERY_GRAVITY: + case QUERY_GRAVITY_CNAME: // Known as gravity blocked, we return this result early, skipping // all the lengthy tests below - blockingreason = "gravity blocked"; + blockingreason = dns_cache->blocking_status == QUERY_GRAVITY ? "gravity blocked" : "gravity blocked (CNAME)"; log_debug(DEBUG_QUERIES, "%s is known as %s", domainstr, blockingreason); // Do not block if the entire query is to be permitted @@ -1512,15 +1494,16 @@ static bool _FTL_check_blocking(int queryID, int domainID, int clientID, const c if(!query->flags.allowed) { force_next_DNS_reply = dns_cache->force_reply; - query_blocked(query, domain, client, QUERY_GRAVITY, GRAVITY_BLOCKED); + query_blocked(query, domain, client, QUERY_GRAVITY); return true; } break; - case REGEX_BLOCKED: + case QUERY_REGEX: + case QUERY_REGEX_CNAME: // Known as regex denied, we return this result early, skipping all // the lengthy tests below - blockingreason = "regex denied"; + blockingreason = dns_cache->blocking_status == QUERY_REGEX ? "regex denied" : "regex denied (CNAME)"; log_debug(DEBUG_QUERIES, "%s is known as %s (cache regex ID: %i)", domainstr, blockingreason, dns_cache->list_id); @@ -1530,74 +1513,58 @@ static bool _FTL_check_blocking(int queryID, int domainID, int clientID, const c { force_next_DNS_reply = dns_cache->force_reply; last_regex_idx = dns_cache->list_id; - query_blocked(query, domain, client, QUERY_REGEX, REGEX_BLOCKED); + query_blocked(query, domain, client, QUERY_REGEX); return true; } break; - case ALLOWED: - // Known as allowed, we return this result early, skipping all the - // lengthy tests below - log_debug(DEBUG_QUERIES, "%s is known as not to be blocked (allowed)", domainstr); - - query->flags.allowed = true; - - return false; - break; - - case SPECIAL_DOMAIN: + case QUERY_SPECIAL_DOMAIN: // Known as a special domain, we return this result early, skipping // all the lengthy tests below blockingreason = "special domain"; log_debug(DEBUG_QUERIES, "%s is known as special domain", domainstr); force_next_DNS_reply = dns_cache->force_reply; - query_blocked(query, domain, client, QUERY_SPECIAL_DOMAIN, SPECIAL_DOMAIN); + query_blocked(query, domain, client, QUERY_SPECIAL_DOMAIN); return true; break; - case NOT_BLOCKED: - // Known as not blocked, we return this result early, skipping all - // the lengthy tests below - log_debug(DEBUG_QUERIES, "%s is known as not to be blocked", domainstr); - - return false; - break; - - case UPSTREAM_BLOCKED_IP: - case UPSTREAM_BLOCKED_NULL: - case UPSTREAM_BLOCKED_NXRA: - case UPSTREAM_BLOCKED_EDE15: - - enum query_status qstat; + case QUERY_EXTERNAL_BLOCKED_IP: + case QUERY_EXTERNAL_BLOCKED_NULL: + case QUERY_EXTERNAL_BLOCKED_NXRA: + case QUERY_EXTERNAL_BLOCKED_EDE15: switch(dns_cache->blocking_status) { - case UNKNOWN_BLOCKED: - case GRAVITY_BLOCKED: - case DENYLIST_BLOCKED: - case REGEX_BLOCKED: - case ALLOWED: - case SPECIAL_DOMAIN: - case PIHOLE_SYNTH: - case NOT_BLOCKED: + case QUERY_UNKNOWN: + case QUERY_GRAVITY: + case QUERY_DENYLIST: + case QUERY_REGEX: + case QUERY_FORWARDED: + case QUERY_CACHE: + case QUERY_GRAVITY_CNAME: + case QUERY_REGEX_CNAME: + case QUERY_DENYLIST_CNAME: + case QUERY_RETRIED: + case QUERY_RETRIED_DNSSEC: + case QUERY_IN_PROGRESS: + case QUERY_DBBUSY: + case QUERY_SPECIAL_DOMAIN: + case QUERY_CACHE_STALE: + case QUERY_STATUS_MAX: // Cannot happen break; - case UPSTREAM_BLOCKED_IP: - qstat = QUERY_EXTERNAL_BLOCKED_IP; + case QUERY_EXTERNAL_BLOCKED_IP: blockingreason = "blocked upstream with known address"; break; - case UPSTREAM_BLOCKED_NULL: - qstat = QUERY_EXTERNAL_BLOCKED_NULL; + case QUERY_EXTERNAL_BLOCKED_NULL: blockingreason = "blocked upstream with NULL address"; break; - case UPSTREAM_BLOCKED_EDE15: - qstat = QUERY_EXTERNAL_BLOCKED_EDE15; + case QUERY_EXTERNAL_BLOCKED_EDE15: blockingreason = "blocked upstream with EDE15"; break; - case UPSTREAM_BLOCKED_NXRA: + case QUERY_EXTERNAL_BLOCKED_NXRA: blockingreason = "blocked upstream with NXRA address"; - qstat = QUERY_EXTERNAL_BLOCKED_NXRA; break; } @@ -1607,13 +1574,27 @@ static bool _FTL_check_blocking(int queryID, int domainID, int clientID, const c domainstr, blockingreason, (unsigned long)(dns_cache->expires - time(NULL))); force_next_DNS_reply = dns_cache->force_reply; - query_blocked(query, domain, client, qstat, dns_cache->blocking_status); + query_blocked(query, domain, client, dns_cache->blocking_status); return true; break; - case PIHOLE_SYNTH: - // Known as a synthetic reply, we return this result early, skipping - // all the lengthy tests below + case QUERY_CACHE: + case QUERY_FORWARDED: + case QUERY_RETRIED: + case QUERY_RETRIED_DNSSEC: + case QUERY_IN_PROGRESS: + case QUERY_DBBUSY: + case QUERY_CACHE_STALE: + case QUERY_STATUS_MAX: + // Known as not to be blocked, possibly even explicitly + // allowed - we return this result early, skipping all + // the lengthy tests below + log_debug(DEBUG_QUERIES, "%s is known as not to be blocked%s", domainstr, + dns_cache->flags.allowed ? " (allowed)" : ""); + + if(dns_cache->flags.allowed) + query->flags.allowed = true; + return false; break; } @@ -1644,12 +1625,12 @@ static bool _FTL_check_blocking(int queryID, int domainID, int clientID, const c if(!query->flags.allowed && special_domain(query, domainstr)) { // Set DNS cache properties - dns_cache->blocking_status = SPECIAL_DOMAIN; - cacheStatus = SPECIAL_DOMAIN; + dns_cache->blocking_status = QUERY_SPECIAL_DOMAIN; + cacheStatus = dns_cache->blocking_status; dns_cache->force_reply = force_next_DNS_reply; // Adjust counters - query_blocked(query, domain, client, QUERY_SPECIAL_DOMAIN, SPECIAL_DOMAIN); + query_blocked(query, domain, client, QUERY_SPECIAL_DOMAIN); // Debug output log_debug(DEBUG_QUERIES, "Special domain: %s is %s", domainstr, blockingreason); @@ -1693,7 +1674,7 @@ static bool _FTL_check_blocking(int queryID, int domainID, int clientID, const c if(blockDomain) { // Adjust counters - query_blocked(query, domain, client, new_status, cacheStatus); + query_blocked(query, domain, client, new_status); // Debug output if(config.debug.queries.v.b) @@ -1709,7 +1690,7 @@ static bool _FTL_check_blocking(int queryID, int domainID, int clientID, const c // Explicitly mark as not blocked to skip the entire gravity/blacklist // chain when the same client asks for the same domain in the future. // Store domain as allowed if this is the case - dns_cache->blocking_status = query->flags.allowed ? ALLOWED : NOT_BLOCKED; + dns_cache->flags.allowed = query->flags.allowed; // Debug output // client is guaranteed to be non-NULL above @@ -2480,7 +2461,7 @@ static enum query_status detect_blocked_IP(const unsigned short flags, const uni char answer[ADDRSTRLEN]; answer[0] = '\0'; inet_ntop(AF_INET, addr, answer, ADDRSTRLEN); blockingreason = "blocked upstream with known address (IPv4)"; - cacheStatus = UPSTREAM_BLOCKED_IP; + cacheStatus = QUERY_EXTERNAL_BLOCKED_IP; log_debug(DEBUG_QUERIES, "%s -> \"%s\"", blockingreason, answer); } @@ -2499,7 +2480,7 @@ static enum query_status detect_blocked_IP(const unsigned short flags, const uni char answer[ADDRSTRLEN]; answer[0] = '\0'; inet_ntop(AF_INET6, addr, answer, ADDRSTRLEN); blockingreason = "blocked upstream with known address (IPv6)"; - cacheStatus = UPSTREAM_BLOCKED_IP; + cacheStatus = QUERY_EXTERNAL_BLOCKED_IP; log_debug(DEBUG_QUERIES, "%s -> \"%s\"", blockingreason, answer); } @@ -2515,7 +2496,7 @@ static enum query_status detect_blocked_IP(const unsigned short flags, const uni if(config.debug.queries.v.b) { blockingreason = "blocked upstream with 0.0.0.0"; - cacheStatus = UPSTREAM_BLOCKED_NULL; + cacheStatus = QUERY_EXTERNAL_BLOCKED_NULL; log_debug(DEBUG_QUERIES, "%s", blockingreason); } @@ -2531,7 +2512,7 @@ static enum query_status detect_blocked_IP(const unsigned short flags, const uni if(config.debug.queries.v.b) { blockingreason = "blocked upstream with ::"; - cacheStatus = UPSTREAM_BLOCKED_NULL; + cacheStatus = QUERY_EXTERNAL_BLOCKED_NULL; log_debug(DEBUG_QUERIES, "%s", blockingreason); } @@ -2543,27 +2524,12 @@ static enum query_status detect_blocked_IP(const unsigned short flags, const uni return QUERY_UNKNOWN; } -static void query_blocked(queriesData *query, domainsData *domain, clientsData *client, - const enum query_status new_status, const enum domain_client_status cache_status) +static void query_blocked(queriesData *query, domainsData *domain, clientsData *client, const enum query_status new_status) { // Get response time struct timeval response; gettimeofday(&response, 0); - // Memorize this in the DNS cache if blocked due to the response - if(cache_status != UNKNOWN_BLOCKED) - { - const int cacheID = findCacheID(query->domainID, query->clientID, query->type, true); - DNSCacheData *dns_cache = getDNSCache(cacheID, true); - if(dns_cache != NULL && dns_cache->blocking_status != cache_status) - { - // Update status - set_dnscache_blockingstatus(dns_cache, cache_status, - client ? getstr(client->ippos) : NULL, - domain ? getstr(domain->domainpos) : NULL); - } - } - // Adjust counters if we recorded a non-blocking status if(query->status == QUERY_FORWARDED) { @@ -2796,7 +2762,7 @@ static void FTL_upstream_error(const union all_addr *addr, const unsigned int fl unlock_shm(); } -static void FTL_blocked_upstream_by_header(const enum domain_client_status new_status, const int id, const char* file, const int line) +static void FTL_blocked_upstream_by_header(const enum query_status new_status, const int id, const char* file, const int line) { // Lock shared memory lock_shm(); @@ -2837,7 +2803,7 @@ static void FTL_blocked_upstream_by_header(const enum domain_client_status new_s } // Set blocking reason - blockingreason = new_status == UPSTREAM_BLOCKED_NXRA ? + blockingreason = new_status == QUERY_EXTERNAL_BLOCKED_NXRA ? "blocked upstream with NXDOMAIN + no RA" : "blocked upstream with EDE15"; cacheStatus = new_status; @@ -2849,12 +2815,7 @@ static void FTL_blocked_upstream_by_header(const enum domain_client_status new_s // Store query as externally blocked clientsData *client = getClient(query->clientID, true); if(client != NULL) - { - const enum query_status new_qstatus = new_status == UPSTREAM_BLOCKED_NXRA ? - QUERY_EXTERNAL_BLOCKED_NXRA : - QUERY_EXTERNAL_BLOCKED_EDE15; - query_blocked(query, domain, client, new_qstatus, new_status); - } + query_blocked(query, domain, client, new_status); // Store reply type as replied with NXDOMAIN query_set_reply(F_NEG | F_NXDOMAIN, 0, NULL, query, response); @@ -2866,7 +2827,7 @@ static void FTL_blocked_upstream_by_header(const enum domain_client_status new_s unlock_shm(); } -static void FTL_blocked_upstream_by_addr(const enum query_status new_qstatus, const int id, const char* file, const int line) +static void FTL_blocked_upstream_by_addr(const enum query_status new_status, const int id, const char* file, const int line) { // Lock shared memory lock_shm(); @@ -2893,12 +2854,7 @@ static void FTL_blocked_upstream_by_addr(const enum query_status new_qstatus, co clientsData *client = getClient(query->clientID, true); domainsData *domain = getDomain(query->domainID, true); if(client != NULL && domain != NULL) - { - const enum domain_client_status new_status = new_qstatus == QUERY_EXTERNAL_BLOCKED_IP ? - UPSTREAM_BLOCKED_IP : - UPSTREAM_BLOCKED_NULL; - query_blocked(query, domain, client, new_qstatus, new_status); - } + query_blocked(query, domain, client, new_status); // Possible debugging information if(config.debug.queries.v.b) @@ -2933,7 +2889,7 @@ int _FTL_check_reply(const unsigned int rcode, const unsigned short flags, // RA bit is not set and rcode is NXDOMAIN if(!rabit && rcode == NXDOMAIN) { - FTL_blocked_upstream_by_header(UPSTREAM_BLOCKED_NXRA, id, file, line); + FTL_blocked_upstream_by_header(QUERY_EXTERNAL_BLOCKED_NXRA, id, file, line); // Query is blocked return 1; @@ -2942,7 +2898,7 @@ int _FTL_check_reply(const unsigned int rcode, const unsigned short flags, // EDE 15 if(edns != NULL && edns->ede == EDE_BLOCKED) { - FTL_blocked_upstream_by_header(UPSTREAM_BLOCKED_EDE15, id, file, line); + FTL_blocked_upstream_by_header(QUERY_EXTERNAL_BLOCKED_EDE15, id, file, line); // Query is blocked return 1; diff --git a/src/enums.h b/src/enums.h index 4ea76b78..2fa3594a 100644 --- a/src/enums.h +++ b/src/enums.h @@ -121,24 +121,6 @@ enum blocking_status { BLOCKING_UNKNOWN } __attribute__ ((packed)); -// Blocking status constants used by the dns_cache->blocking_status vector -// We explicitly force UNKNOWN_BLOCKED to zero on all platforms as this is the -// default value set initially with calloc -enum domain_client_status { - UNKNOWN_BLOCKED = 0, - GRAVITY_BLOCKED, - DENYLIST_BLOCKED, - REGEX_BLOCKED, - ALLOWED, - SPECIAL_DOMAIN, - UPSTREAM_BLOCKED_NXRA, - UPSTREAM_BLOCKED_NULL, - UPSTREAM_BLOCKED_IP, - UPSTREAM_BLOCKED_EDE15, - PIHOLE_SYNTH, - NOT_BLOCKED -} __attribute__ ((packed)); - enum debug_flag { DEBUG_NONE = 0, DEBUG_DATABASE = 1, diff --git a/test/test_suite.bats b/test/test_suite.bats index 6f339cb9..2860776f 100644 --- a/test/test_suite.bats +++ b/test/test_suite.bats @@ -485,7 +485,7 @@ lines+=("$line") done <<< "${log}" printf "%s\n" "${lines[@]}" - [[ ${lines[@]} == *"DEBUG_QUERIES: null.ftl is known as blocked upstream with NULL address"* ]] + [[ ${lines[@]} == *"DEBUG_QUERIES: null.ftl is known as blocked upstream with NULL address (expires in"* ]] [[ ${lines[@]} != *"DEBUG_QUERIES: **** forwarded null.ftl to 127.0.0.1#5555"* ]] [[ ${lines[@]} == *"DEBUG_QUERIES: Adding RR: \"null.ftl A 0.0.0.0\""* ]] } @@ -541,6 +541,7 @@ [[ ${lines[@]} == *"DEBUG_QUERIES: DNS cache: A/127.0.0.1/umbrella.ftl is not blocked (domainlist ID: -1)"* ]] [[ ${lines[@]} == *"DEBUG_QUERIES: **** forwarded umbrella.ftl to 127.0.0.1#5555"* ]] [[ ${lines[@]} == *"DEBUG_QUERIES: blocked upstream with known address (IPv4)"* ]] + [[ ${lines[@]} == *"DEBUG_QUERIES: DNS cache: A/127.0.0.1/umbrella.ftl -> EXTERNAL_BLOCKED_IP"* ]] [[ ${lines[@]} == *"DEBUG_QUERIES: Adding RR: \"umbrella.ftl A 0.0.0.0\""* ]] } @@ -564,7 +565,7 @@ lines+=("$line") done <<< "${log}" printf "%s\n" "${lines[@]}" - [[ ${lines[@]} == *"DEBUG_QUERIES: umbrella.ftl is known as blocked upstream with known address"* ]] + [[ ${lines[@]} == *"DEBUG_QUERIES: umbrella.ftl is known as blocked upstream with known address (expires in"* ]] [[ ${lines[@]} != *"DEBUG_QUERIES: **** forwarded umbrella.ftl to 127.0.0.1#5555"* ]] [[ ${lines[@]} == *"DEBUG_QUERIES: Adding RR: \"umbrella.ftl A 0.0.0.0\""* ]] } @@ -590,7 +591,8 @@ printf "%s\n" "${lines[@]}" [[ ${lines[@]} == *"DEBUG_QUERIES: DNS cache: AAAA/127.0.0.1/umbrella.ftl is not blocked (domainlist ID: -1)"* ]] [[ ${lines[@]} == *"DEBUG_QUERIES: **** forwarded umbrella.ftl to 127.0.0.1#5555"* ]] - [[ ${lines[@]} == *"DEBUG_QUERIES: DNS cache: AAAA/127.0.0.1/umbrella.ftl is blocked upstream with known address (IPv6)"* ]] + [[ ${lines[@]} == *"DEBUG_QUERIES: blocked upstream with known address (IPv6)"* ]] + [[ ${lines[@]} == *"DEBUG_QUERIES: DNS cache: AAAA/127.0.0.1/umbrella.ftl -> EXTERNAL_BLOCKED_IP"* ]] [[ ${lines[@]} == *"DEBUG_QUERIES: Adding RR: \"umbrella.ftl AAAA ::\""* ]] } @@ -615,7 +617,7 @@ printf "%s\n" "${lines[@]}" [[ ${lines[@]} == *"DEBUG_QUERIES: DNS cache: A/127.0.0.1/umbrella-multi.ftl is not blocked (domainlist ID: -1)"* ]] [[ ${lines[@]} == *"DEBUG_QUERIES: **** forwarded umbrella-multi.ftl to 127.0.0.1#5555"* ]] - [[ ${lines[@]} == *"DEBUG_QUERIES: DNS cache: A/127.0.0.1/umbrella-multi.ftl is blocked upstream with known address (IPv4)"* ]] + [[ ${lines[@]} == *"DEBUG_QUERIES: DNS cache: A/127.0.0.1/umbrella-multi.ftl -> EXTERNAL_BLOCKED_IP"* ]] [[ ${lines[@]} == *"DEBUG_QUERIES: Adding RR: \"umbrella-multi.ftl A 0.0.0.0\""* ]] } @@ -641,7 +643,7 @@ printf "%s\n" "${lines[@]}" [[ ${lines[@]} == *"DEBUG_QUERIES: DNS cache: A/127.0.0.1/nxdomain.ede15.ftl is not blocked (domainlist ID: -1)"* ]] [[ ${lines[@]} == *"DEBUG_QUERIES: **** forwarded nxdomain.ede15.ftl to 127.0.0.1#5555"* ]] - [[ ${lines[@]} == *"DEBUG_QUERIES: DNS cache: A/127.0.0.1/nxdomain.ede15.ftl is blocked upstream with EDE15"* ]] + [[ ${lines[@]} == *"DEBUG_QUERIES: DNS cache: A/127.0.0.1/nxdomain.ede15.ftl -> EXTERNAL_BLOCKED_EDE15"* ]] [[ ${lines[@]} == *"DEBUG_QUERIES: Adding RR: \"nxdomain.ede15.ftl A 0.0.0.0\""* ]] } @@ -665,7 +667,7 @@ lines+=("$line") done <<< "${log}" printf "%s\n" "${lines[@]}" - [[ ${lines[@]} == *"DEBUG_QUERIES: nxdomain.ede15.ftl is known as blocked upstream with EDE15"* ]] + [[ ${lines[@]} == *"DEBUG_QUERIES: nxdomain.ede15.ftl is known as blocked upstream with EDE15 (expires in"* ]] [[ ${lines[@]} != *"DEBUG_QUERIES: **** forwarded umbrella.ftl to 127.0.0.1#5555"* ]] [[ ${lines[@]} == *"DEBUG_QUERIES: Adding RR: \"nxdomain.ede15.ftl A 0.0.0.0\""* ]] } From 21b77b7e8aa9260ef7a7f5fea90c6834cfc5f7ac Mon Sep 17 00:00:00 2001 From: Dominik Date: Sat, 14 Sep 2024 10:28:59 +0200 Subject: [PATCH 294/339] Apply suggestions from code review Co-authored-by: RD WebDesign Signed-off-by: Dominik --- src/args.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/args.c b/src/args.c index 03b0d3b1..af85cc69 100644 --- a/src/args.c +++ b/src/args.c @@ -1018,9 +1018,10 @@ void parse_args(int argc, char* argv[]) printf(" Usage: %spihole-FTL --gzip %sinfile %s[outfile]%s\n\n", green, cyan, purple, normal); printf(" - %sinfile%s is the file to be processed. If the filename ends\n", cyan, normal); printf(" in %s.gz%s, FTL will uncompress, otherwise it will compress\n\n", yellow, normal); - printf(" - %s[outfile]%s is the optional target. If omitted, FTL will\n", purple, normal); - printf(" - input is gz: use %sinfile%s.gz%s and remove %s.gz%s from the end\n", cyan, yellow, normal, purple, normal); - printf(" - otherwise: use %sinfile%s and append %s.gz%s at the end\n\n", cyan, normal, purple, normal); + printf(" - %s[outfile]%s is the optional target file.\n", purple, normal); + printf(" If omitted, FTL will modify the original filename:\n"); + printf(" - FTL will remove %s.gz%s from the end of the filename, if present.\n", yellow, normal); + printf(" - otherwise, FTL will append %s.gz%s to the filename\n\n", yellow, normal); printf(" Examples:\n"); printf(" - %spihole-FTL --gzip %sfile.txt%s\n", green, cyan, normal); printf(" compresses %sfile.txt%s to %sfile.txt.gz%s\n\n", cyan, normal, cyan, normal); From d8627a91e6f29325abfdcb6933d2f354f7094295 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 14 Sep 2024 21:40:26 +0200 Subject: [PATCH 295/339] Simplify gzip help text further Signed-off-by: DL6ER --- src/args.c | 25 ++++++++++++++++++------- src/args.h | 2 ++ 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/src/args.c b/src/args.c index af85cc69..bf2713d8 100644 --- a/src/args.c +++ b/src/args.c @@ -106,6 +106,7 @@ const char** argv_dnsmasq = NULL; #define COL_BLUE "\x1b[94m" // bright foreground color #define COL_PURPLE "\x1b[95m" // bright foreground color #define COL_CYAN "\x1b[96m" // bright foreground color +#define CLI_OVER "\r\x1b[K" // go back to beginning of line and erase to end of line static bool __attribute__ ((pure)) is_term(void) { @@ -149,6 +150,16 @@ const char __attribute__ ((pure)) *cli_bold(void) return is_term() ? COL_BOLD : ""; } +const char __attribute__ ((pure)) *cli_underline(void) +{ + return is_term() ? COL_ULINE : ""; +} + +const char __attribute__ ((pure)) *cli_italics(void) +{ + return is_term() ? COL_ITALIC : ""; +} + // Resets font to normal const char __attribute__ ((pure)) *cli_normal(void) { @@ -165,7 +176,7 @@ static const char __attribute__ ((pure)) *cli_color(const char *color) const char __attribute__ ((pure)) *cli_over(void) { // \x1b[K is the ANSI escape sequence for "erase to end of line" - return is_term() ? "\r\x1b[K" : "\r"; + return is_term() ? CLI_OVER : "\r"; } static inline bool strEndsWith(const char *input, const char *end) @@ -925,6 +936,7 @@ void parse_args(int argc, char* argv[]) if(strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "help") == 0 || strcmp(argv[i], "--help") == 0) { const char *bold = cli_bold(); + const char *uline = cli_underline(); const char *normal = cli_normal(); const char *blue = cli_color(COL_BLUE); const char *cyan = cli_color(COL_CYAN); @@ -1019,14 +1031,13 @@ void parse_args(int argc, char* argv[]) printf(" - %sinfile%s is the file to be processed. If the filename ends\n", cyan, normal); printf(" in %s.gz%s, FTL will uncompress, otherwise it will compress\n\n", yellow, normal); printf(" - %s[outfile]%s is the optional target file.\n", purple, normal); - printf(" If omitted, FTL will modify the original filename:\n"); - printf(" - FTL will remove %s.gz%s from the end of the filename, if present.\n", yellow, normal); - printf(" - otherwise, FTL will append %s.gz%s to the filename\n\n", yellow, normal); + printf(" If omitted, FTL will try to derive the target file from\n"); + printf(" the source file.\n\n"); printf(" Examples:\n"); printf(" - %spihole-FTL --gzip %sfile.txt%s\n", green, cyan, normal); - printf(" compresses %sfile.txt%s to %sfile.txt.gz%s\n\n", cyan, normal, cyan, normal); - printf(" - %spihole-FTL --gzip %sfile.txt.gz%s\n", green, cyan, normal); - printf(" uncompresses %sfile.txt.gz%s to %sfile.txt%s\n\n", cyan, normal, cyan, normal); + printf(" compresses %sfile.txt%s to %sfile.txt%s.gz%s\n\n", cyan, normal, cyan, yellow, normal); + printf(" - %spihole-FTL --gzip %sfile.txt%s.gz%s\n", green, cyan, yellow, normal); + printf(" %sun%scompresses %sfile.txt%s.gz%s to %sfile.txt%s\n\n", uline, normal, cyan, yellow, normal, cyan, normal); printf("%sTeleporter:%s\n", yellow, normal); printf("\t%s--teleporter%s Create a Teleporter archive in the\n", green, normal); diff --git a/src/args.h b/src/args.h index fa188f3f..f9688917 100644 --- a/src/args.h +++ b/src/args.h @@ -24,6 +24,8 @@ const char *cli_done(void) __attribute__ ((pure)); const char *cli_bold(void) __attribute__ ((pure)); const char *cli_normal(void) __attribute__ ((pure)); const char *cli_over(void) __attribute__ ((pure)); +const char *cli_underline(void) __attribute__ ((pure)); +const char *cli_italics(void) __attribute__ ((pure)); void test_dnsmasq_options(int argc, const char *argv[]); From e0d2271b3a5542a94aaa7361646c95be827d4e2f Mon Sep 17 00:00:00 2001 From: yubiuser Date: Sun, 15 Sep 2024 17:54:14 +0200 Subject: [PATCH 296/339] Add CodeQL Signed-off-by: yubiuser --- .github/workflows/codeql.yml | 100 +++++++++++++++++++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100644 .github/workflows/codeql.yml diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 00000000..e5495478 --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,100 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +# +# ******** NOTE ******** +# We have attempted to detect the languages in your repository. Please check +# the `language` matrix defined below to confirm you have the correct set of +# supported CodeQL languages. +# +name: "CodeQL Advanced" + +on: + push: + branches: [ "master", "development", "special/CI*", "update/dnsmasq" ] + pull_request: + branches: [ "master", "development", "special/CI*", "update/dnsmasq" ] + schedule: + - cron: '45 10 * * 6' + +jobs: + analyze: + name: Analyze (${{ matrix.language }}) + # Runner size impacts CodeQL analysis time. To learn more, please see: + # - https://gh.io/recommended-hardware-resources-for-running-codeql + # - https://gh.io/supported-runners-and-hardware-resources + # - https://gh.io/using-larger-runners (GitHub.com only) + # Consider using larger runners or machines with greater resources for possible analysis time improvements. + runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }} + permissions: + # required for all workflows + security-events: write + + # required to fetch internal or private CodeQL packs + packages: read + + # only required for workflows in private repositories + actions: read + contents: read + + strategy: + fail-fast: false + matrix: + include: + - language: c-cpp + build-mode: autobuild + # CodeQL supports the following values keywords for 'language': 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift' + # Use `c-cpp` to analyze code written in C, C++ or both + # Use 'java-kotlin' to analyze code written in Java, Kotlin or both + # Use 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both + # To learn more about changing the languages that are analyzed or customizing the build mode for your analysis, + # see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/customizing-your-advanced-setup-for-code-scanning. + # If you are analyzing a compiled language, you can modify the 'build-mode' for that language to customize how + # your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v3 + with: + languages: ${{ matrix.language }} + build-mode: ${{ matrix.build-mode }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + + # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs + # queries: security-extended,security-and-quality + + # If the analyze step fails for one of the languages you are analyzing with + # "We were unable to automatically build your code", modify the matrix above + # to set the build mode to "manual" for that language. Then modify this step + # to build your code. + # ℹ️ Command-line programs to run using the OS shell. + # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun + - if: matrix.build-mode == 'manual' + shell: bash + run: | + echo 'If you are using a "manual" build mode for one or more of the' \ + 'languages you are analyzing, replace this with the commands to build' \ + 'your code, for example:' + echo ' make bootstrap' + echo ' make release' + exit 1 + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v3 + with: + category: "/language:${{matrix.language}}" + + - name: Upload CodeQL results as an artifact + if: success() || failure() + uses: actions/upload-artifact@v4 + with: + name: codeql-results + path: ${{ steps.codeql_analysis.outputs.sarif-output }} + retention-days: 5 From bc5833d756a3c33d3180f84f8a6ba6163846cc2f Mon Sep 17 00:00:00 2001 From: yubiuser Date: Sun, 15 Sep 2024 19:05:11 +0200 Subject: [PATCH 297/339] Use build.sh Signed-off-by: yubiuser --- .github/workflows/codeql.yml | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index e5495478..ce224dd7 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -44,7 +44,7 @@ jobs: matrix: include: - language: c-cpp - build-mode: autobuild + build-mode: manual # CodeQL supports the following values keywords for 'language': 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift' # Use `c-cpp` to analyze code written in C, C++ or both # Use 'java-kotlin' to analyze code written in Java, Kotlin or both @@ -79,12 +79,7 @@ jobs: - if: matrix.build-mode == 'manual' shell: bash run: | - echo 'If you are using a "manual" build mode for one or more of the' \ - 'languages you are analyzing, replace this with the commands to build' \ - 'your code, for example:' - echo ' make bootstrap' - echo ' make release' - exit 1 + ./build.sh - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@v3 From 2b9932b5f5f82be82d4bd76941787498a65d13c1 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 15 Sep 2024 19:23:27 +0200 Subject: [PATCH 298/339] Move global cache status message into debug.queries mode instead of being a general INFO message Signed-off-by: DL6ER --- src/dnsmasq_interface.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c index 4d91ad12..c47e8ab0 100644 --- a/src/dnsmasq_interface.c +++ b/src/dnsmasq_interface.c @@ -1451,7 +1451,7 @@ static bool _FTL_check_blocking(int queryID, int domainID, int clientID, const c // Memorize blocking status DNS cache for the domain/client combination cacheStatus = dns_cache->blocking_status; - log_info("Set global cache status to %d", cacheStatus); + log_debug(DEBUG_QUERIES, "Set global cache status to %d", cacheStatus); // Skip the entire chain of tests if we already know the answer for this // particular client From 39626b25ee0b937730843366cf113cd124290485 Mon Sep 17 00:00:00 2001 From: yubiuser Date: Sun, 15 Sep 2024 19:22:02 +0200 Subject: [PATCH 299/339] Install nettle and mbedTLS and dependencies Signed-off-by: yubiuser --- .github/workflows/codeql.yml | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index ce224dd7..bea263fc 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -10,6 +10,9 @@ # supported CodeQL languages. # name: "CodeQL Advanced" +env: + nettleversion: 3.9.1 + mbedtlsversion: 3.6.1 on: push: @@ -57,6 +60,29 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y libidn2-0 libidn2-0-dev libunistring-dev + + - name: Install nettle + run: | + curl -sSL https://ftl.pi-hole.net/libraries/nettle-${nettleversion}.tar.gz | tar -xz + cd nettle-${nettleversion} + ./configure --enable-static --disable-shared --disable-openssl --disable-mini-gmp -disable-gcov --disable-documentation + sudo make -j $(nproc) install + + - name: Install mbedTLS + # Build static mbedTLS with pthread support + # Disable AESNI on linux/386 asit would possibly result in an incompatible + # binary in processors lacking the AESNI and SSE2 instruction sets + run: | + curl -sSL https://ftl.pi-hole.net/libraries/mbedtls-${mbedtlsversion}.tar.bz2 | tar -xj + cd mbedtls-${mbedtlsversion} + sed -i '/#define MBEDTLS_THREADING_C/s*^//**g' include/mbedtls/mbedtls_config.h + sed -i '/#define MBEDTLS_THREADING_PTHREAD/s*^//**g' include/mbedtls/mbedtls_config.h + sudo make -j $(nproc) install + # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL uses: github/codeql-action/init@v3 @@ -85,6 +111,7 @@ jobs: uses: github/codeql-action/analyze@v3 with: category: "/language:${{matrix.language}}" + id: codeql_analysis - name: Upload CodeQL results as an artifact if: success() || failure() From 7a398aaa199801b6c78950269c7166b4e8a2e1b7 Mon Sep 17 00:00:00 2001 From: yubiuser Date: Sun, 15 Sep 2024 20:41:39 +0200 Subject: [PATCH 300/339] Enable security-and-quality query pack Signed-off-by: yubiuser --- .github/workflows/codeql.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index bea263fc..0bdd785e 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -94,7 +94,7 @@ jobs: # Prefix the list here with "+" to use these queries and those in the config file. # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs - # queries: security-extended,security-and-quality + queries: security-and-quality # If the analyze step fails for one of the languages you are analyzing with # "We were unable to automatically build your code", modify the matrix above From d88eabed6b54d1c4178971f76c0d9e411196065e Mon Sep 17 00:00:00 2001 From: yubiuser Date: Sun, 15 Sep 2024 22:03:40 +0200 Subject: [PATCH 301/339] Filter alerts from dependencies Signed-off-by: yubiuser --- .github/workflows/codeql.yml | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 0bdd785e..a3878d25 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -111,12 +111,37 @@ jobs: uses: github/codeql-action/analyze@v3 with: category: "/language:${{matrix.language}}" - id: codeql_analysis + upload: failure-only # upload only in case of failure, otherwise upload later after filtering + output: codeql-results + + - name: Filter SARIF + uses: advanced-security/filter-sarif@v1 + with: + # filter out third-party dependencies + patterns: | + -src/dnsmasq/* + -src/webserver/civetweb/* + -src/webserver/cJSON/* + -src/tre-regex/* + -src/config/tomlc99/* + -src/database/shell.c + -src/database/sqlite3.c + -src/database/sqlite3.h + -src/zip/miniz/* + -src/lua/* + +src/lua/ftl_* + input: codeql-results/cpp.sarif + output: codeql-results/cpp.sarif + + - name: Upload SARIF + uses: github/codeql-action/upload-sarif@v3 + with: + sarif_file: codeql-results/cpp.sarif - name: Upload CodeQL results as an artifact if: success() || failure() uses: actions/upload-artifact@v4 with: name: codeql-results - path: ${{ steps.codeql_analysis.outputs.sarif-output }} + path: codeql-results retention-days: 5 From b0733a04a33c703dae39745169fffdf2528332eb Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 15 Sep 2024 22:07:51 +0200 Subject: [PATCH 302/339] Avoid using dangerous function localtime() Signed-off-by: DL6ER --- src/ntp/client.c | 9 +++++---- src/overTime.c | 11 ++++++++--- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/src/ntp/client.c b/src/ntp/client.c index 53a60f2a..340358bf 100644 --- a/src/ntp/client.c +++ b/src/ntp/client.c @@ -93,11 +93,12 @@ static void format_NTP_time(char time_str[TIMESTR_SIZE], const uint64_t ntp_time struct timeval client_time; client_time.tv_sec = NTPtoSEC(ntp_time); client_time.tv_usec = NTPtoUSEC(ntp_time); - struct tm *client_tm = localtime(&client_time.tv_sec); + struct tm client_tm = {0}; + localtime_r(&client_time.tv_sec, &client_tm); snprintf(time_str, TIMESTR_SIZE, "%04i-%02i-%02i %02i:%02i:%02i.%06li %s", - client_tm->tm_year + 1900, client_tm->tm_mon + 1, client_tm->tm_mday, - client_tm->tm_hour, client_tm->tm_min, client_tm->tm_sec, - (long int)client_time.tv_usec, client_tm->tm_zone); + client_tm.tm_year + 1900, client_tm.tm_mon + 1, client_tm.tm_mday, + client_tm.tm_hour, client_tm.tm_min, client_tm.tm_sec, + (long int)client_time.tv_usec, client_tm.tm_zone); time_str[TIMESTR_SIZE - 1] = '\0'; } diff --git a/src/overTime.c b/src/overTime.c index a7ad1a7e..34b51755 100644 --- a/src/overTime.c +++ b/src/overTime.c @@ -30,7 +30,9 @@ static void initSlot(const unsigned int index, const time_t timestamp) if(config.debug.overtime.v.b) { char timestr[20]; - strftime(timestr, 20, "%Y-%m-%d %H:%M:%S", localtime(×tamp)); + struct tm tm = { 0 }; + localtime_r(×tamp, &tm); + strftime(timestr, 20, "%Y-%m-%d %H:%M:%S", &tm); log_debug(DEBUG_OVERTIME, "initSlot(%u, %lu): Zeroing overTime slot at %s", index, (unsigned long)timestamp, timestr); } @@ -73,8 +75,11 @@ void initOverTime(void) if(config.debug.overtime.v.b) { char first[20], last[20]; - strftime(first, 20, "%Y-%m-%d %H:%M:%S", localtime(&oldest)); - strftime(last, 20, "%Y-%m-%d %H:%M:%S", localtime(&newest)); + struct tm tm_o = { 0 }, tm_n = { 0 }; + localtime_r(&oldest, &tm_o); + localtime_r(&newest, &tm_n); + strftime(first, 20, "%Y-%m-%d %H:%M:%S", &tm_o); + strftime(last, 20, "%Y-%m-%d %H:%M:%S", &tm_n); log_debug(DEBUG_OVERTIME, "initOverTime(): Initializing %i slots from %s (%lu) to %s (%lu)", OVERTIME_SLOTS, first, (unsigned long)oldest, last, (unsigned long)newest); } From e48cb188e5e685f34686303811ba53ce62ce3a35 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 15 Sep 2024 22:10:57 +0200 Subject: [PATCH 303/339] Fix multiplication result converted to larger type Signed-off-by: DL6ER --- src/shmem.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/shmem.c b/src/shmem.c index 3f1c29c1..e5569a5d 100644 --- a/src/shmem.c +++ b/src/shmem.c @@ -951,7 +951,7 @@ void reset_per_client_regex(const int clientID) void add_per_client_regex(unsigned int clientID) { const unsigned int num_regex_tot = get_num_regex(REGEX_MAX); // total number - const size_t size = get_optimal_object_size(1, counters->clients * num_regex_tot); + const size_t size = get_optimal_object_size(1, (size_t)counters->clients * num_regex_tot); if(size > shm_per_client_regex.size && realloc_shm(&shm_per_client_regex, 1, size, true)) { From cfdd7030dbaae2c669e5485f2c3c29dfb9de9621 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 15 Sep 2024 22:15:11 +0200 Subject: [PATCH 304/339] Fix comparison of narrow type with wide type in loop condition Signed-off-by: DL6ER --- src/log.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/log.c b/src/log.c index 0d089e02..8b885ca0 100644 --- a/src/log.c +++ b/src/log.c @@ -381,7 +381,7 @@ void FTL_log_helper(const unsigned int n, ...) va_list args; char **arg = calloc(n, sizeof(char*)); va_start(args, n); - for(unsigned char i = 0; i < n; i++) + for(unsigned int i = 0; i < n; i++) { const char *argin = va_arg(args, char*); if(argin == NULL) @@ -410,7 +410,7 @@ void FTL_log_helper(const unsigned int n, ...) } // Free allocated memory - for(unsigned char i = 0; i < n; i++) + for(unsigned int i = 0; i < n; i++) if(arg[i] != NULL) free(arg[i]); free(arg); From bf5ec1417cd2ba6eba13145f6382f649a6b39049 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 15 Sep 2024 22:26:13 +0200 Subject: [PATCH 305/339] Restrict permissions to owner read/write only when creating a new file Signed-off-by: DL6ER --- src/api/teleporter.c | 6 ++++++ src/webserver/x509.c | 4 ++++ src/zip/gzip.c | 9 +++++++++ 3 files changed, 19 insertions(+) diff --git a/src/api/teleporter.c b/src/api/teleporter.c index d3e2b902..16e15756 100644 --- a/src/api/teleporter.c +++ b/src/api/teleporter.c @@ -782,6 +782,12 @@ static int process_received_tar_gz(struct ftl_conn *api, struct upload_data *dat log_err("Unable to open file \"%s\" for writing: %s", extract_files[i].destination, strerror(errno)); continue; } + + // Restrict permissions to owner read/write only + if(fchmod(fileno(fp), S_IRUSR | S_IWUSR) != 0) + log_warn("Unable to set permissions on file \"%s\": %s", extract_files[i].destination, strerror(errno)); + + // Write file to disk if(fwrite(file, fileSize, 1, fp) != 1) { log_err("Unable to write file \"%s\": %s", extract_files[i].destination, strerror(errno)); diff --git a/src/webserver/x509.c b/src/webserver/x509.c index d8706022..31eb3ba5 100644 --- a/src/webserver/x509.c +++ b/src/webserver/x509.c @@ -110,6 +110,10 @@ static bool write_to_file(const char *filename, const char *type, const char *su return false; } + // Restrict permissions to owner read/write only + if(fchmod(fileno(f), S_IRUSR | S_IWUSR) != 0) + log_warn("Unable to set permissions on file \"%s\": %s", targetname, strerror(errno)); + // Write key (if provided) if(key != NULL) { diff --git a/src/zip/gzip.c b/src/zip/gzip.c index a192df5a..d930656a 100644 --- a/src/zip/gzip.c +++ b/src/zip/gzip.c @@ -8,6 +8,7 @@ * This file is copyright under the latest version of the EUPL. * Please see LICENSE file for your rights under this license. */ +#include "FTL.h" #include "gzip.h" #include "log.h" @@ -315,6 +316,10 @@ bool inflate_file(const char *infilename, const char *outfilename, bool verbose) return false; } + // Restrict permissions to owner read/write only + if(fchmod(fileno(outfile), S_IRUSR | S_IWUSR) != 0) + log_warn("Unable to set permissions on file \"%s\": %s", outfilename, strerror(errno)); + // Get file size fseek(infile, 0, SEEK_END); const long sc = ftell(infile); @@ -408,6 +413,10 @@ bool deflate_file(const char *infilename, const char *outfilename, bool verbose) return false; } + // Restrict permissions to owner read/write only + if(fchmod(fileno(outfile), S_IRUSR | S_IWUSR) != 0) + log_warn("Unable to set permissions on file \"%s\": %s", outfilename, strerror(errno)); + // Get file size fseek(infile, 0, SEEK_END); const long size_uncompressed = ftell(infile); From 31473f47b5a3095b49ca0c7d4c8020c097cbefc0 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 15 Sep 2024 22:32:13 +0200 Subject: [PATCH 306/339] Check sscanf result when reading Signed-off-by: DL6ER --- src/config/legacy_reader.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config/legacy_reader.c b/src/config/legacy_reader.c index 4a5b1605..fb698278 100644 --- a/src/config/legacy_reader.c +++ b/src/config/legacy_reader.c @@ -268,7 +268,7 @@ const char *readFTLlegacy(struct config *conf) buffer = parseFTLconf(fp, "DELAY_STARTUP"); unsigned int unum; - if(buffer != NULL && sscanf(buffer, "%u", &unum) && unum > 0 && unum <= 300) + if(buffer != NULL && sscanf(buffer, "%u", &unum) == 1 && unum > 0 && unum <= 300) conf->misc.delay_startup.v.ui = unum; // BLOCK_ESNI From 73b0d1383cae39cac66f7cfc437be6e63a02c91a Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 15 Sep 2024 22:32:30 +0200 Subject: [PATCH 307/339] Add NULL check for array allocation Signed-off-by: DL6ER --- src/zip/teleporter.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/zip/teleporter.c b/src/zip/teleporter.c index 8e9af579..0185edb8 100644 --- a/src/zip/teleporter.c +++ b/src/zip/teleporter.c @@ -816,6 +816,12 @@ bool read_teleporter_zip_from_disk(const char *filename) // Process ZIP archive char hint[ERRBUF_SIZE] = ""; cJSON *imported_files = cJSON_CreateArray(); + if(imported_files == NULL) + { + log_err("Failed to create JSON array for imported files"); + free(ptr); + return false; + } const char *error = read_teleporter_zip(ptr, size, hint, NULL, imported_files); if(error != NULL) From 45e9e4453a80d8845de2afdb87a962fa428dd084 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 15 Sep 2024 22:36:17 +0200 Subject: [PATCH 308/339] Fix using incorrect type Signed-off-by: DL6ER --- src/tools/netlink.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/tools/netlink.c b/src/tools/netlink.c index 1de4d67e..a0fe97b3 100644 --- a/src/tools/netlink.c +++ b/src/tools/netlink.c @@ -1118,17 +1118,16 @@ static int nlquery(const int type, cJSON *json, const bool detailed) memset(&sa, 0, sizeof(sa)); sa.nl_family = AF_NETLINK; - ssize_t len = nlrequest(fd, &sa, type); - if(len < 0) + if(!nlrequest(fd, &sa, type)) { log_info("nlrequest error: %s", strerror(errno)); return -1; } - char buf[BUFLEN]; uint32_t nl_msg_type; do { - len = nlgetmsg(fd, &sa, buf, BUFLEN); + char buf[BUFLEN]; + ssize_t len = nlgetmsg(fd, &sa, buf, BUFLEN); nl_msg_type = parse_nl_msg(buf, len, json, detailed); } while (nl_msg_type != NLMSG_DONE && nl_msg_type != NLMSG_ERROR); From 2153cbbee3abdb8d08cebbaafded58db0e9bf3b4 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 16 Sep 2024 06:22:20 +0200 Subject: [PATCH 309/339] Reduce global variable scope to file-local Signed-off-by: DL6ER --- src/timers.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/timers.c b/src/timers.c index d153ccdc..ddfddb0b 100644 --- a/src/timers.c +++ b/src/timers.c @@ -16,7 +16,7 @@ // set_blockingmode() #include "config/config.h" -struct timespec t0[NUMTIMERS]; +static struct timespec t0[NUMTIMERS]; void timer_start(const enum timers i) { From 8fed1361f15465cb7694665a93d5a6690756c2da Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 16 Sep 2024 06:26:50 +0200 Subject: [PATCH 310/339] Check for leap year, and adjust the date accordingly Signed-off-by: DL6ER --- src/webserver/x509.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/webserver/x509.c b/src/webserver/x509.c index 31eb3ba5..d53f65e7 100644 --- a/src/webserver/x509.c +++ b/src/webserver/x509.c @@ -238,6 +238,9 @@ bool generate_certificate(const char* certfile, bool rsa, const char *domain) char not_after[16] = { 0 }; strftime(not_before, sizeof(not_before), "%Y%m%d%H%M%S", tm); tm->tm_year += 30; // 30 years from now + // Check for leap year, and adjust the date accordingly + const bool isLeapYear = tm->tm_year % 4 == 0 && (tm->tm_year % 100 != 0 || tm->tm_year % 400 == 0); + tm->tm_mday = tm->tm_mon == 2 && tm->tm_mday == 29 && !isLeapYear ? 28 : tm->tm_mday; strftime(not_after, sizeof(not_after), "%Y%m%d%H%M%S", tm); // 1. Create CA certificate From 4df3c26ce75f1e18448e5480d4128dbe40164ff2 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 16 Sep 2024 06:35:59 +0200 Subject: [PATCH 311/339] Fix local variable address stored in non-local memory Signed-off-by: DL6ER --- src/regex.c | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/src/regex.c b/src/regex.c index ec80032f..2da9de08 100644 --- a/src/regex.c +++ b/src/regex.c @@ -34,7 +34,7 @@ const char *regextype[REGEX_MAX] = { "deny", "allow", "CLI" }; static regexData *allow_regex = NULL; static regexData *deny_regex = NULL; -static regexData *cli_regex = NULL; +static regexData cli_regex = { 0 }; static unsigned int num_regex[REGEX_MAX] = { 0 }; unsigned int regex_change = 0; static char regex_msg[REGEX_MSG_LEN] = { 0 }; @@ -48,7 +48,7 @@ static inline regexData *get_regex_ptr(const enum regex_type regexid) case REGEX_ALLOW: return allow_regex; case REGEX_CLI: - return cli_regex; + return &cli_regex; case REGEX_MAX: // Fall through default: // This is not possible return NULL; @@ -57,7 +57,7 @@ static inline regexData *get_regex_ptr(const enum regex_type regexid) static inline void free_regex_ptr(const enum regex_type regexid) { - regexData **regex; + regexData **regex = NULL; switch (regexid) { case REGEX_DENY: @@ -67,8 +67,8 @@ static inline void free_regex_ptr(const enum regex_type regexid) regex = &allow_regex; break; case REGEX_CLI: - regex = &cli_regex; - break; + // cannot be freed + return; case REGEX_MAX: // Fall through default: // This is not possible return; @@ -626,8 +626,7 @@ void free_regex(void) { // Return early if we don't use any regex filters if(allow_regex == NULL && - deny_regex == NULL && - cli_regex == NULL) + deny_regex == NULL) { log_debug(DEBUG_DATABASE, "Not using any regex filters, nothing to free or reset"); return; @@ -895,15 +894,13 @@ int regex_test(const bool debug_mode, const bool quiet, const char *domainin, co { // Compile CLI regex log_info("%s Compiling regex filter...", cli_info()); - regexData regex = { 0 }; - cli_regex = ®ex; num_regex[REGEX_CLI] = 1; // Compile CLI regex timer_start(REGEX_TIMER); log_ctrl(false, true); // Temporarily re-enable terminal output for error logging char *message = NULL; - if(!compile_regex(regexin, ®ex, &message) && message != NULL) + if(!compile_regex(regexin, &cli_regex, &message) && message != NULL) { logg_regex_warning("CLI", message, 0, regexin); free(message); From acbe08cbc5df6d601430be1e7b5b6ef6ff1fdcd7 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 16 Sep 2024 06:49:17 +0200 Subject: [PATCH 312/339] Add missing header guards Signed-off-by: DL6ER --- src/FTL.h | 1 - src/database/sqlite3-ext.h | 5 +++++ src/ntp/server.c | 1 - src/tools/gravity-parseList.h | 5 +++++ src/webserver/webserver.c | 2 +- 5 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/FTL.h b/src/FTL.h index 64986a20..5eb91005 100644 --- a/src/FTL.h +++ b/src/FTL.h @@ -33,7 +33,6 @@ #include #include #include -//#include #include // syslog #include diff --git a/src/database/sqlite3-ext.h b/src/database/sqlite3-ext.h index 2636eb5d..56426571 100644 --- a/src/database/sqlite3-ext.h +++ b/src/database/sqlite3-ext.h @@ -8,5 +8,10 @@ * This file is copyright under the latest version of the EUPL. * Please see LICENSE file for your rights under this license. */ +#ifndef SQLITE3_EXT_H +#define SQLITE3_EXT_H + // Initialization point for SQLite3 extensions extern int sqlite3_pihole_extensions_init(sqlite3 *db, const char **pzErrMsg, const struct sqlite3_api_routines *pApi); + +#endif // SQLITE3_EXT_H \ No newline at end of file diff --git a/src/ntp/server.c b/src/ntp/server.c index ae9eed9c..7c89fd07 100644 --- a/src/ntp/server.c +++ b/src/ntp/server.c @@ -19,7 +19,6 @@ #include // clock_gettime() #include -//#include #include // wait() #include diff --git a/src/tools/gravity-parseList.h b/src/tools/gravity-parseList.h index e9e54ac4..b3e816b0 100644 --- a/src/tools/gravity-parseList.h +++ b/src/tools/gravity-parseList.h @@ -8,7 +8,12 @@ * This file is copyright under the latest version of the EUPL. * Please see LICENSE file for your rights under this license. */ +#ifndef GRAVITY_PARSELIST_H +#define GRAVITY_PARSELIST_H + #include "FTL.h" int gravity_parseList(const char *infile, const char *outfile, const char *adlistID, const bool checkOnly, const bool antigravity); bool __attribute__((pure)) valid_domain(const char *domain, const size_t len, const bool fqdn_only); + +#endif // GRAVITY_PARSELIST_H \ No newline at end of file diff --git a/src/webserver/webserver.c b/src/webserver/webserver.c index b3b05b94..cc4fe929 100644 --- a/src/webserver/webserver.c +++ b/src/webserver/webserver.c @@ -606,7 +606,7 @@ void FTL_rewrite_pattern(char *filename, unsigned long filename_buf_len) filename_lp = append_to_path(filename, ".lp"); if(filename_lp == NULL) { - //Failed to allocate memory for filename!"); + // Failed to allocate memory for filename return; } From 9148224324ab24d5b92373ce02913da851e7527e Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 16 Sep 2024 07:02:35 +0200 Subject: [PATCH 313/339] Reduce superfluent enum initializer Signed-off-by: DL6ER --- src/enums.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/enums.h b/src/enums.h index 2fa3594a..ac7ed52c 100644 --- a/src/enums.h +++ b/src/enums.h @@ -123,7 +123,7 @@ enum blocking_status { enum debug_flag { DEBUG_NONE = 0, - DEBUG_DATABASE = 1, + DEBUG_DATABASE, DEBUG_NETWORKING, DEBUG_LOCKS, DEBUG_QUERIES, From 5552e33e559dc44313c6402fd62d94d7860a1bb4 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 16 Sep 2024 14:00:21 +0200 Subject: [PATCH 314/339] Fix whitespace issue Signed-off-by: DL6ER --- src/database/sqlite3-ext.h | 2 +- src/dnsmasq_interface.c | 2 +- src/tools/gravity-parseList.h | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/database/sqlite3-ext.h b/src/database/sqlite3-ext.h index 56426571..18eabd8c 100644 --- a/src/database/sqlite3-ext.h +++ b/src/database/sqlite3-ext.h @@ -14,4 +14,4 @@ // Initialization point for SQLite3 extensions extern int sqlite3_pihole_extensions_init(sqlite3 *db, const char **pzErrMsg, const struct sqlite3_api_routines *pApi); -#endif // SQLITE3_EXT_H \ No newline at end of file +#endif // SQLITE3_EXT_H diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c index c47e8ab0..362eb1bb 100644 --- a/src/dnsmasq_interface.c +++ b/src/dnsmasq_interface.c @@ -1567,7 +1567,7 @@ static bool _FTL_check_blocking(int queryID, int domainID, int clientID, const c blockingreason = "blocked upstream with NXRA address"; break; } - + // Known as upstream blocked, we return this result // early, skipping all the lengthy tests below log_debug(DEBUG_QUERIES, "%s is known as %s (expires in %lus)", diff --git a/src/tools/gravity-parseList.h b/src/tools/gravity-parseList.h index b3e816b0..34ca169c 100644 --- a/src/tools/gravity-parseList.h +++ b/src/tools/gravity-parseList.h @@ -16,4 +16,4 @@ int gravity_parseList(const char *infile, const char *outfile, const char *adlistID, const bool checkOnly, const bool antigravity); bool __attribute__((pure)) valid_domain(const char *domain, const size_t len, const bool fqdn_only); -#endif // GRAVITY_PARSELIST_H \ No newline at end of file +#endif // GRAVITY_PARSELIST_H From 66141d8b48612069f4d751886264d8c8ad9bf995 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 18 Sep 2024 15:35:36 +0200 Subject: [PATCH 315/339] Improve database updating procedure by avoiding opening a dedicated / separate database connection for updating the query counters when we already have the disk database attached to our in-memory database under the disk.* namespace. This also brings a bit of code duplication and removal of functions called only once. Signed-off-by: DL6ER --- src/database/common.c | 30 ------------------------------ src/database/common.h | 2 -- src/database/query-table.c | 22 ++++++++++++++-------- 3 files changed, 14 insertions(+), 40 deletions(-) diff --git a/src/database/common.c b/src/database/common.c index dee1019c..90895469 100644 --- a/src/database/common.c +++ b/src/database/common.c @@ -678,17 +678,6 @@ bool db_set_FTL_property(sqlite3 *db, const enum ftl_table_props ID, const int v return true; } -bool db_set_FTL_property_double(sqlite3 *db, const enum ftl_table_props ID, const double value) -{ - int ret = dbquery(db, "INSERT OR REPLACE INTO ftl (id, value) VALUES ( %u, %f );", ID, value); - if(ret != SQLITE_OK) - { - checkFTLDBrc(ret); - return false; - } - return true; -} - bool db_set_counter(sqlite3 *db, const enum counters_table_props ID, const int value) { int ret = dbquery(db, "INSERT OR REPLACE INTO counters (id, value) VALUES ( %u, %d );", ID, value); @@ -700,25 +689,6 @@ bool db_set_counter(sqlite3 *db, const enum counters_table_props ID, const int v return true; } -bool db_update_counters(sqlite3 *db, const int total, const int blocked) -{ - int ret = dbquery(db, "UPDATE counters SET value = value + %i WHERE id = %i;", total, DB_TOTALQUERIES); - if(ret != SQLITE_OK) - { - checkFTLDBrc(ret); - return false; - } - - ret = dbquery(db, "UPDATE counters SET value = value + %i WHERE id = %i;", total, DB_TOTALQUERIES); - if(ret != SQLITE_OK) - { - checkFTLDBrc(ret); - return false; - } - - return true; -} - int db_query_int(sqlite3 *db, const char* querystr) { log_debug(DEBUG_DATABASE, "dbquery: \"%s\"", querystr); diff --git a/src/database/common.h b/src/database/common.h index b5673f91..46d863eb 100644 --- a/src/database/common.h +++ b/src/database/common.h @@ -33,7 +33,6 @@ int db_get_int(sqlite3* db, const enum ftl_table_props ID); int db_get_FTL_property(sqlite3* db, const enum ftl_table_props ID); double db_get_FTL_property_double(sqlite3* db, const enum ftl_table_props ID); bool db_set_FTL_property(sqlite3* db, const enum ftl_table_props ID, const int value); -bool db_set_FTL_property_double(sqlite3* db, const enum ftl_table_props ID, const double value); /// Execute a formatted SQL query and get the return code int dbquery(sqlite3* db, const char *format, ...) __attribute__ ((format (printf, 2, 3)));; @@ -53,7 +52,6 @@ int db_query_int_from_until_type(sqlite3 *db, const char* querystr, const double void SQLite3LogCallback(void *pArg, int iErrCode, const char *zMsg); bool db_set_counter(sqlite3 *db, const enum counters_table_props ID, const int value); -bool db_update_counters(sqlite3 *db, const int total, const int blocked); const char *get_sqlite3_version(void); extern bool DBdeleteoldqueries; diff --git a/src/database/query-table.c b/src/database/query-table.c index 94620946..58469424 100644 --- a/src/database/query-table.c +++ b/src/database/query-table.c @@ -637,7 +637,6 @@ bool export_queries_to_disk(bool final) // Finalize statement sqlite3_finalize(stmt); - // Update last_disk_db_idx // Prepare SQLite3 statement log_debug(DEBUG_DATABASE, "Accessing in-memory database"); @@ -691,15 +690,22 @@ bool export_queries_to_disk(bool final) // All temp queries were stored to disk, update the IDs last_disk_db_idx += insertions; + /* + * If there are any insertions, we: + * 1. Insert (or replace) the last timestamp into the `disk.ftl` table. + * 2. Update the total queries counter in the `disk.counters` table. + * 3. Update the blocked queries counter in the `disk.counters` table. + */ if(insertions > 0) { - sqlite3 *db = dbopen(false, false); - if(db != NULL) - { - db_set_FTL_property_double(db, DB_LASTTIMESTAMP, new_last_timestamp); - db_update_counters(db, new_total, new_blocked); - dbclose(&db); - } + if((rc = dbquery(memdb, "INSERT OR REPLACE INTO disk.ftl (id, value) VALUES ( %i, %f );", DB_LASTTIMESTAMP, new_last_timestamp)) != SQLITE_OK) + log_err("export_queries_to_disk(): Cannot update timestamp: %s", sqlite3_errstr(rc)); + + if((rc = dbquery(memdb, "UPDATE disk.counters SET value = value + %u WHERE id = %i;", new_total, DB_TOTALQUERIES)) != SQLITE_OK) + log_err("export_queries_to_disk(): Cannot update total queries counter: %s", sqlite3_errstr(rc)); + + if((rc = dbquery(memdb, "UPDATE disk.counters SET value = value + %u WHERE id = %i;", new_blocked, DB_BLOCKEDQUERIES)) != SQLITE_OK) + log_err("export_queries_to_disk(): Cannot update blocked queries counter: %s", sqlite3_errstr(rc)); } log_debug(DEBUG_DATABASE, "Exported %u rows for disk.query_storage (took %.1f ms, last SQLite ID %lu)", From e3a7820aa893cc1bd78f94c337b031e876c2034f Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 18 Sep 2024 15:42:28 +0200 Subject: [PATCH 316/339] Include update inside running transaction Signed-off-by: DL6ER --- src/database/query-table.c | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/database/query-table.c b/src/database/query-table.c index 58469424..0cd9adfc 100644 --- a/src/database/query-table.c +++ b/src/database/query-table.c @@ -677,19 +677,6 @@ bool export_queries_to_disk(bool final) log_debug(DEBUG_DATABASE, "Exported %i rows to disk.%s", sqlite3_changes(memdb), subtable_names[i]); } - // End transaction - if((rc = sqlite3_exec(memdb, "END TRANSACTION", NULL, NULL, NULL)) != SQLITE_OK) - { - log_err("export_queries_to_disk(): Cannot end transaction: %s", sqlite3_errstr(rc)); - return false; - } - - // Update number of queries in the disk database - disk_db_num = get_number_of_queries_in_DB(memdb, "disk.query_storage"); - - // All temp queries were stored to disk, update the IDs - last_disk_db_idx += insertions; - /* * If there are any insertions, we: * 1. Insert (or replace) the last timestamp into the `disk.ftl` table. @@ -708,6 +695,19 @@ bool export_queries_to_disk(bool final) log_err("export_queries_to_disk(): Cannot update blocked queries counter: %s", sqlite3_errstr(rc)); } + // End transaction + if((rc = sqlite3_exec(memdb, "END TRANSACTION", NULL, NULL, NULL)) != SQLITE_OK) + { + log_err("export_queries_to_disk(): Cannot end transaction: %s", sqlite3_errstr(rc)); + return false; + } + + // Update number of queries in the disk database + disk_db_num = get_number_of_queries_in_DB(memdb, "disk.query_storage"); + + // All temp queries were stored to disk, update the IDs + last_disk_db_idx += insertions; + log_debug(DEBUG_DATABASE, "Exported %u rows for disk.query_storage (took %.1f ms, last SQLite ID %lu)", insertions, timer_elapsed_msec(DATABASE_WRITE_TIMER), last_disk_db_idx); From 40480814183db9af68b16d9f0b1ca7f6604d33b6 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 18 Sep 2024 15:47:15 +0200 Subject: [PATCH 317/339] Reset global counters after update Signed-off-by: DL6ER --- src/database/query-table.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/database/query-table.c b/src/database/query-table.c index 0cd9adfc..140805c8 100644 --- a/src/database/query-table.c +++ b/src/database/query-table.c @@ -682,6 +682,11 @@ bool export_queries_to_disk(bool final) * 1. Insert (or replace) the last timestamp into the `disk.ftl` table. * 2. Update the total queries counter in the `disk.counters` table. * 3. Update the blocked queries counter in the `disk.counters` table. + * + * Note that new_total does not need to match the total number of + * insertions here as storing queries to the database happens + * time-delayed. In the end, the total number of queries will be + * correct (after final synchronization during FTL shutdown). */ if(insertions > 0) { @@ -690,9 +695,15 @@ bool export_queries_to_disk(bool final) if((rc = dbquery(memdb, "UPDATE disk.counters SET value = value + %u WHERE id = %i;", new_total, DB_TOTALQUERIES)) != SQLITE_OK) log_err("export_queries_to_disk(): Cannot update total queries counter: %s", sqlite3_errstr(rc)); + else + // Success + new_total = 0; if((rc = dbquery(memdb, "UPDATE disk.counters SET value = value + %u WHERE id = %i;", new_blocked, DB_BLOCKEDQUERIES)) != SQLITE_OK) log_err("export_queries_to_disk(): Cannot update blocked queries counter: %s", sqlite3_errstr(rc)); + else + // Success + new_blocked = 0; } // End transaction From 75f11f3f4fe97c685f2ac6e5f573b4edde9bfbd9 Mon Sep 17 00:00:00 2001 From: yubiuser Date: Mon, 26 Aug 2024 20:30:49 +0200 Subject: [PATCH 318/339] Prettify BATS output Signed-off-by: yubiuser --- .github/Dockerfile | 9 +++++++++ test/run.sh | 2 +- test/test_suite.bats | 4 ++++ 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/.github/Dockerfile b/.github/Dockerfile index e8b05edb..0cf7f2df 100644 --- a/.github/Dockerfile +++ b/.github/Dockerfile @@ -13,6 +13,15 @@ ENV GIT_TAG=${GIT_TAG} ARG BUILD_OPTS="" ENV BUILD_OPTS=${BUILD_OPTS} +# Add ncurses used for pretty output in bats tests +RUN apk add --no-cache ncurses +ENV TERM=xterm + +# Monkeypatch BATS to remove duplicate output of starting and finished test +# BATS uses ANSI escape codes to overwrite the line after the test has finished +# This is not supported by Github Actions as it does not provide a TTY to the docker build container +RUN sed -i '/buffer_with_truncation /d' /bats-core/libexec/bats-core/bats-format-pretty + # Build FTL # Remove possible old build files RUN rm -rf cmake && \ diff --git a/test/run.sh b/test/run.sh index 5d490a82..626a5dc7 100755 --- a/test/run.sh +++ b/test/run.sh @@ -106,7 +106,7 @@ echo -n "Contained dnsmasq version (DNS): " dig TXT CHAOS version.bind @127.0.0.1 +short # Run tests -$BATS "test/test_suite.bats" +$BATS -p "test/test_suite.bats" RET=$? curl_to_tricorder() { diff --git a/test/test_suite.bats b/test/test_suite.bats index 2860776f..b610d1e1 100644 --- a/test/test_suite.bats +++ b/test/test_suite.bats @@ -1,5 +1,9 @@ #!./test/libs/bats/bin/bats +@test 'fail()' { + fail 'this test always fails' +} + @test "Compare template and test TOML config files" { # We skip the first 5 lines of the files as they contain the version and # timestamp of the file creation/modification From c83da0b68e5344711f06aea979dff16528ac686d Mon Sep 17 00:00:00 2001 From: yubiuser Date: Thu, 19 Sep 2024 21:11:00 +0200 Subject: [PATCH 319/339] Update base image and remove purposely added failing test Signed-off-by: yubiuser --- .github/Dockerfile | 5 ++--- test/test_suite.bats | 4 ---- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/.github/Dockerfile b/.github/Dockerfile index 0cf7f2df..094c9ef2 100644 --- a/.github/Dockerfile +++ b/.github/Dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/pi-hole/ftl-build:v2.7 AS builder +FROM ghcr.io/pi-hole/ftl-build:v2.8 AS builder WORKDIR /app @@ -13,8 +13,7 @@ ENV GIT_TAG=${GIT_TAG} ARG BUILD_OPTS="" ENV BUILD_OPTS=${BUILD_OPTS} -# Add ncurses used for pretty output in bats tests -RUN apk add --no-cache ncurses +# Setting TERM is needed for pretty output in BATS tests ENV TERM=xterm # Monkeypatch BATS to remove duplicate output of starting and finished test diff --git a/test/test_suite.bats b/test/test_suite.bats index b610d1e1..2860776f 100644 --- a/test/test_suite.bats +++ b/test/test_suite.bats @@ -1,9 +1,5 @@ #!./test/libs/bats/bin/bats -@test 'fail()' { - fail 'this test always fails' -} - @test "Compare template and test TOML config files" { # We skip the first 5 lines of the files as they contain the version and # timestamp of the file creation/modification From 8c92b20e5aa98a5a9043fd68b20bd3720f4e0b15 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 23 Sep 2024 12:32:45 +0200 Subject: [PATCH 320/339] Fix lua error handling issue present in current CivetWeb code Signed-off-by: DL6ER --- src/webserver/civetweb/mod_lua.inl | 8 ++++++-- src/webserver/lua_web.c | 7 +------ 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/webserver/civetweb/mod_lua.inl b/src/webserver/civetweb/mod_lua.inl index 3ad3ba75..739f1a2e 100644 --- a/src/webserver/civetweb/mod_lua.inl +++ b/src/webserver/civetweb/mod_lua.inl @@ -674,7 +674,9 @@ run_lsp_kepler(struct mg_connection *conn, } else { /* Success loading chunk. Call it. */ - lua_pcall(L, 0, 0, 1); + lua_ok = lua_pcall(L, 0, 0, 0); + if(lua_ok != LUA_OK) + lua_cry(conn, lua_ok, L, "LSP", "call"); } return 0; } @@ -790,7 +792,9 @@ run_lsp_civetweb(struct mg_connection *conn, lua_pcall(L, 1, 0, 0); } else { /* Success loading chunk. Call it. */ - lua_pcall(L, 0, 0, 1); + lua_ok = lua_pcall(L, 0, 0, 0); + if(lua_ok != LUA_OK) + lua_cry(conn, lua_ok, L, "LSP", "call"); } /* Progress until after the Lua closing tag. */ diff --git a/src/webserver/lua_web.c b/src/webserver/lua_web.c index 1d53f9bb..e1087035 100644 --- a/src/webserver/lua_web.c +++ b/src/webserver/lua_web.c @@ -57,12 +57,7 @@ void free_lua(void) void init_lua(const struct mg_connection *conn, void *L, unsigned context_flags) { - // Set onerror handler to print errors to the log - if(luaL_dostring(L, "mg.onerror = function(e) mg.cry('Error at ' .. e) end") != LUA_OK) - { - log_err("Error setting Lua onerror handler: %s", lua_tostring(L, -1)); - lua_pop(L, 1); - } + return; } int request_handler(struct mg_connection *conn, void *cbdata) From bdf0d91f8aeec07b6959a41fa2965731ca2dcdcd Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 23 Sep 2024 13:07:33 +0200 Subject: [PATCH 321/339] Abort early on Lua errors Signed-off-by: DL6ER --- src/webserver/civetweb/mod_lua.inl | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/src/webserver/civetweb/mod_lua.inl b/src/webserver/civetweb/mod_lua.inl index 739f1a2e..62cb54f1 100644 --- a/src/webserver/civetweb/mod_lua.inl +++ b/src/webserver/civetweb/mod_lua.inl @@ -670,13 +670,17 @@ run_lsp_kepler(struct mg_connection *conn, /* Syntax error or OOM. * Error message is pushed on stack. */ lua_pcall(L, 1, 0, 0); - lua_cry(conn, lua_ok, L, "LSP", "execute"); /* XXX TODO: everywhere ! */ + lua_cry(conn, lua_ok, L, "LSP Kepler", "execute"); + return 1; } else { /* Success loading chunk. Call it. */ lua_ok = lua_pcall(L, 0, 0, 0); - if(lua_ok != LUA_OK) - lua_cry(conn, lua_ok, L, "LSP", "call"); + if(lua_ok != LUA_OK) + { + lua_cry(conn, lua_ok, L, "LSP Kepler", "call"); + return 1; + } } return 0; } @@ -790,11 +794,16 @@ run_lsp_civetweb(struct mg_connection *conn, /* Syntax error or OOM. * Error message is pushed on stack. */ lua_pcall(L, 1, 0, 0); + lua_cry(conn, lua_ok, L, "LSP", "execute"); + return 1; } else { /* Success loading chunk. Call it. */ lua_ok = lua_pcall(L, 0, 0, 0); - if(lua_ok != LUA_OK) + if(lua_ok != LUA_OK) + { lua_cry(conn, lua_ok, L, "LSP", "call"); + return 1; + } } /* Progress until after the Lua closing tag. */ From 221389b06091da524c87ac06dd1ed9c1199358ac Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 23 Sep 2024 13:18:54 +0200 Subject: [PATCH 322/339] Print error to user Signed-off-by: DL6ER --- src/webserver/civetweb/mod_lua.inl | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/webserver/civetweb/mod_lua.inl b/src/webserver/civetweb/mod_lua.inl index 62cb54f1..1d297f29 100644 --- a/src/webserver/civetweb/mod_lua.inl +++ b/src/webserver/civetweb/mod_lua.inl @@ -10,6 +10,9 @@ #include "civetweb_lua.h" #include "civetweb_private_lua.h" +static int +lua_error_handler(lua_State *L); + #if defined(_WIN32) static void * @@ -670,7 +673,7 @@ run_lsp_kepler(struct mg_connection *conn, /* Syntax error or OOM. * Error message is pushed on stack. */ lua_pcall(L, 1, 0, 0); - lua_cry(conn, lua_ok, L, "LSP Kepler", "execute"); + lua_error_handler(L); return 1; } else { @@ -678,7 +681,7 @@ run_lsp_kepler(struct mg_connection *conn, lua_ok = lua_pcall(L, 0, 0, 0); if(lua_ok != LUA_OK) { - lua_cry(conn, lua_ok, L, "LSP Kepler", "call"); + lua_error_handler(L); return 1; } } @@ -794,14 +797,14 @@ run_lsp_civetweb(struct mg_connection *conn, /* Syntax error or OOM. * Error message is pushed on stack. */ lua_pcall(L, 1, 0, 0); - lua_cry(conn, lua_ok, L, "LSP", "execute"); + lua_error_handler(L); return 1; } else { /* Success loading chunk. Call it. */ lua_ok = lua_pcall(L, 0, 0, 0); if(lua_ok != LUA_OK) { - lua_cry(conn, lua_ok, L, "LSP", "call"); + lua_error_handler(L); return 1; } } @@ -2792,6 +2795,9 @@ lua_error_handler(lua_State *L) { const char *error_msg = lua_isstring(L, -1) ? lua_tostring(L, -1) : "?\n"; + /* Log error message */ + lua_cry(NULL, 0, L, error_msg, "error"); + lua_getglobal(L, "mg"); if (!lua_isnil(L, -1)) { lua_getfield(L, -1, "write"); /* call mg.write() */ From 3a5f5450fd085f3d9e48c4f201f5745d65a6c1e5 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 23 Sep 2024 13:24:27 +0200 Subject: [PATCH 323/339] Print to user *and* log to log file Signed-off-by: DL6ER --- src/webserver/civetweb/mod_lua.inl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/webserver/civetweb/mod_lua.inl b/src/webserver/civetweb/mod_lua.inl index 1d297f29..f8b4d3fd 100644 --- a/src/webserver/civetweb/mod_lua.inl +++ b/src/webserver/civetweb/mod_lua.inl @@ -13,7 +13,6 @@ static int lua_error_handler(lua_State *L); - #if defined(_WIN32) static void * mmap(void *addr, int64_t len, int prot, int flags, int fd, int offset) @@ -673,6 +672,7 @@ run_lsp_kepler(struct mg_connection *conn, /* Syntax error or OOM. * Error message is pushed on stack. */ lua_pcall(L, 1, 0, 0); + lua_cry(conn, lua_ok, L, "LSP Kepler", "execute"); lua_error_handler(L); return 1; @@ -681,6 +681,7 @@ run_lsp_kepler(struct mg_connection *conn, lua_ok = lua_pcall(L, 0, 0, 0); if(lua_ok != LUA_OK) { + lua_cry(conn, lua_ok, L, "LSP Kepler", "call"); lua_error_handler(L); return 1; } @@ -797,6 +798,7 @@ run_lsp_civetweb(struct mg_connection *conn, /* Syntax error or OOM. * Error message is pushed on stack. */ lua_pcall(L, 1, 0, 0); + lua_cry(conn, lua_ok, L, "LSP", "call"); lua_error_handler(L); return 1; } else { @@ -804,6 +806,7 @@ run_lsp_civetweb(struct mg_connection *conn, lua_ok = lua_pcall(L, 0, 0, 0); if(lua_ok != LUA_OK) { + lua_cry(conn, lua_ok, L, "LSP", "execute"); lua_error_handler(L); return 1; } From 835933f8501e517a781b380b8cfafa656adc6fa7 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 23 Sep 2024 13:25:34 +0200 Subject: [PATCH 324/339] Increase LUA_IDSIZE so that long script filenames as well as long script lines fit into the error logging buffer Signed-off-by: DL6ER --- src/lua/luaconf.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lua/luaconf.h b/src/lua/luaconf.h index 33bb580d..dacc5221 100644 --- a/src/lua/luaconf.h +++ b/src/lua/luaconf.h @@ -765,7 +765,7 @@ ** of a function in debug information. ** CHANGE it if you want a different size. */ -#define LUA_IDSIZE 60 +#define LUA_IDSIZE 256 /* From 59cfeb4ed51560ec9ef12fc10784d192b6aa90e0 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 23 Sep 2024 13:26:24 +0200 Subject: [PATCH 325/339] Add new Lua patch Signed-off-by: DL6ER --- patch/lua.sh | 1 + ...IZE-so-that-long-script-filenames-as.patch | 27 +++++++++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 patch/lua/0001-Increase-LUA_IDSIZE-so-that-long-script-filenames-as.patch diff --git a/patch/lua.sh b/patch/lua.sh index 9987b222..e1dcfb2b 100644 --- a/patch/lua.sh +++ b/patch/lua.sh @@ -2,5 +2,6 @@ set -e patch -p1 < patch/lua/0001-add-pihole-library.patch +patch -p1 < patch/lua/0001-Increase-LUA_IDSIZE-so-that-long-script-filenames-as.patch echo "ALL PATCHES APPLIED OKAY" diff --git a/patch/lua/0001-Increase-LUA_IDSIZE-so-that-long-script-filenames-as.patch b/patch/lua/0001-Increase-LUA_IDSIZE-so-that-long-script-filenames-as.patch new file mode 100644 index 00000000..64e2658f --- /dev/null +++ b/patch/lua/0001-Increase-LUA_IDSIZE-so-that-long-script-filenames-as.patch @@ -0,0 +1,27 @@ +From 835933f8501e517a781b380b8cfafa656adc6fa7 Mon Sep 17 00:00:00 2001 +From: DL6ER +Date: Mon, 23 Sep 2024 13:25:34 +0200 +Subject: [PATCH] Increase LUA_IDSIZE so that long script filenames as well as + long script lines fit into the error logging buffer + +Signed-off-by: DL6ER +--- + src/lua/luaconf.h | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/src/lua/luaconf.h b/src/lua/luaconf.h +index 33bb580d..dacc5221 100644 +--- a/src/lua/luaconf.h ++++ b/src/lua/luaconf.h +@@ -765,7 +765,7 @@ + ** of a function in debug information. + ** CHANGE it if you want a different size. + */ +-#define LUA_IDSIZE 60 ++#define LUA_IDSIZE 256 + + + /* +-- +2.34.1 + From dce844b599da39a1b7995707645ae2b19036e7d9 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 23 Sep 2024 20:25:25 +0200 Subject: [PATCH 326/339] Properly generate traceback in lua_error_handler() avoiding the first line showing the manual call to debug.traceback() itself Signed-off-by: DL6ER --- src/webserver/civetweb/mod_lua.inl | 34 +++++++++++++++++++++++------- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/src/webserver/civetweb/mod_lua.inl b/src/webserver/civetweb/mod_lua.inl index f8b4d3fd..f75746ca 100644 --- a/src/webserver/civetweb/mod_lua.inl +++ b/src/webserver/civetweb/mod_lua.inl @@ -2798,23 +2798,41 @@ lua_error_handler(lua_State *L) { const char *error_msg = lua_isstring(L, -1) ? lua_tostring(L, -1) : "?\n"; - /* Log error message */ - lua_cry(NULL, 0, L, error_msg, "error"); - lua_getglobal(L, "mg"); if (!lua_isnil(L, -1)) { - lua_getfield(L, -1, "write"); /* call mg.write() */ + /* Write the error message to the error log */ + lua_getfield(L, -1, "write"); lua_pushstring(L, error_msg); lua_pushliteral(L, "\n"); - lua_call(L, 2, 0); - IGNORE_UNUSED_RESULT( - luaL_dostring(L, "mg.write(debug.traceback(), '\\n')")); + lua_call(L, 2, 0); /* call mg.write(error_msg + \n) */ + lua_pop(L, 1); /* pop mg */ + + /* Get Lua traceback */ + lua_getglobal(L, "debug"); + lua_getfield(L, -1, "traceback"); + lua_call(L, 0, 1); /* call debug.traceback() */ + lua_remove(L, -2); /* remove debug */ + + /* Write the Lua traceback to the error log */ + lua_getglobal(L, "mg"); + lua_getfield(L, -1, "write"); + lua_pushvalue(L, -3); /* push the traceback */ + + /* Only print the traceback if it is not empty */ + if (strcmp(lua_tostring(L, -1), "stack traceback:") != 0) { + lua_pushliteral(L, "\n"); /* append a newline */ + lua_call(L, 2, 0); /* call mg.write(traceback + \n) */ + lua_pop(L, 2); /* pop mg and traceback */ + } else { + lua_pop(L, 3); /* pop mg, traceback and error message */ + } + } else { printf("Lua error: [%s]\n", error_msg); IGNORE_UNUSED_RESULT( luaL_dostring(L, "print(debug.traceback(), '\\n')")); } - /* TODO(lsm, low): leave the stack balanced */ + lua_pop(L, 1); /* pop error message */ return 0; } From 0ff00e1c838ec91a31970c2b51a7651954cba3d6 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 23 Sep 2024 21:42:21 +0200 Subject: [PATCH 327/339] Add bundled script loading into luaL_openlibs to make them available globally (also in the webserver) Signed-off-by: DL6ER --- src/lua/ftl_lua.h | 2 -- src/lua/linit.c | 6 ++++++ src/lua/lua.c | 13 +------------ src/webserver/civetweb/mod_lua.inl | 2 +- 4 files changed, 8 insertions(+), 15 deletions(-) diff --git a/src/lua/ftl_lua.h b/src/lua/ftl_lua.h index d986498a..30bad1f9 100644 --- a/src/lua/ftl_lua.h +++ b/src/lua/ftl_lua.h @@ -21,8 +21,6 @@ int run_luac(const int argc, char **argv); int lua_main (int argc, char **argv); int luac_main (int argc, char **argv); -extern int dolibrary (lua_State *L, char *name); - void print_embedded_scripts(void); void ftl_lua_init(lua_State *L); diff --git a/src/lua/linit.c b/src/lua/linit.c index 9a5bcfdc..787865c0 100644 --- a/src/lua/linit.c +++ b/src/lua/linit.c @@ -8,6 +8,10 @@ #define linit_c #define LUA_LIB +/** Pi-hole modification **/ +#include "ftl_lua.h" +/**************************/ + /* ** If you embed Lua in your program and need to open the standard ** libraries, call luaL_openlibs in your program. If you need a @@ -64,5 +68,7 @@ LUALIB_API void luaL_openlibs (lua_State *L) { luaL_requiref(L, lib->name, lib->func, 1); lua_pop(L, 1); /* remove lib */ } + // Load and enable libraries bundled with Pi-hole + ftl_lua_init(L); } diff --git a/src/lua/lua.c b/src/lua/lua.c index 35fb281d..111a1b2b 100644 --- a/src/lua/lua.c +++ b/src/lua/lua.c @@ -20,10 +20,6 @@ #include "lauxlib.h" #include "lualib.h" -/** Pi-hole modification **/ -#include "ftl_lua.h" -/**************************/ - #if !defined(LUA_PROGNAME) #define LUA_PROGNAME "lua" @@ -218,9 +214,7 @@ static int dostring (lua_State *L, const char *s, const char *name) { ** If there is no explicit modname and globname contains a '-', cut ** the suffix after '-' (the "version") to make the global name. */ -/************** Pi-hole modification ***************/ -int dolibrary (lua_State *L, char *globname) { -/***************************************************/ +static int dolibrary (lua_State *L, char *globname) { int status; char *suffix = NULL; char *modname = strchr(globname, '='); @@ -655,11 +649,6 @@ static int pmain (lua_State *L) { return 0; /* error running LUA_INIT */ } - /************** Pi-hole modification ***************/ - // Load and enable libraries bundled with Pi-hole - ftl_lua_init(L); - /***************************************************/ - if (!runargs(L, argv, optlim)) /* execute arguments -e and -l */ return 0; /* something failed */ if (script > 0) { /* execute main script (if there is one) */ diff --git a/src/webserver/civetweb/mod_lua.inl b/src/webserver/civetweb/mod_lua.inl index f75746ca..49129e09 100644 --- a/src/webserver/civetweb/mod_lua.inl +++ b/src/webserver/civetweb/mod_lua.inl @@ -2824,7 +2824,7 @@ lua_error_handler(lua_State *L) lua_call(L, 2, 0); /* call mg.write(traceback + \n) */ lua_pop(L, 2); /* pop mg and traceback */ } else { - lua_pop(L, 3); /* pop mg, traceback and error message */ + lua_pop(L, 3); /* pop mg, traceback and write */ } } else { From ddbce6392ee7642d6fa6f1a50e780f6ebecf76ac Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 23 Sep 2024 21:43:28 +0200 Subject: [PATCH 328/339] Add new Lua patch Signed-off-by: DL6ER --- patch/lua.sh | 1 + ...pt-loading-into-luaL_openlibs-to-mak.patch | 90 +++++++++++++++++++ 2 files changed, 91 insertions(+) create mode 100644 patch/lua/0001-Add-bundled-script-loading-into-luaL_openlibs-to-mak.patch diff --git a/patch/lua.sh b/patch/lua.sh index e1dcfb2b..80bc6f70 100644 --- a/patch/lua.sh +++ b/patch/lua.sh @@ -3,5 +3,6 @@ set -e patch -p1 < patch/lua/0001-add-pihole-library.patch patch -p1 < patch/lua/0001-Increase-LUA_IDSIZE-so-that-long-script-filenames-as.patch +patch -p1 < patch/lua/0001-Add-bundled-script-loading-into-luaL_openlibs-to-mak.patch echo "ALL PATCHES APPLIED OKAY" diff --git a/patch/lua/0001-Add-bundled-script-loading-into-luaL_openlibs-to-mak.patch b/patch/lua/0001-Add-bundled-script-loading-into-luaL_openlibs-to-mak.patch new file mode 100644 index 00000000..2270a56f --- /dev/null +++ b/patch/lua/0001-Add-bundled-script-loading-into-luaL_openlibs-to-mak.patch @@ -0,0 +1,90 @@ +From 0ff00e1c838ec91a31970c2b51a7651954cba3d6 Mon Sep 17 00:00:00 2001 +From: DL6ER +Date: Mon, 23 Sep 2024 21:42:21 +0200 +Subject: [PATCH] Add bundled script loading into luaL_openlibs to make them + available globally (also in the webserver) + +Signed-off-by: DL6ER +--- + src/lua/ftl_lua.h | 2 -- + src/lua/linit.c | 6 ++++++ + src/lua/lua.c | 13 +------------ + 3 files changed, 7 insertions(+), 14 deletions(-) + +diff --git a/src/lua/ftl_lua.h b/src/lua/ftl_lua.h +index d986498a..30bad1f9 100644 +--- a/src/lua/ftl_lua.h ++++ b/src/lua/ftl_lua.h +@@ -21,8 +21,6 @@ int run_luac(const int argc, char **argv); + int lua_main (int argc, char **argv); + int luac_main (int argc, char **argv); + +-extern int dolibrary (lua_State *L, char *name); +- + void print_embedded_scripts(void); + void ftl_lua_init(lua_State *L); + +diff --git a/src/lua/linit.c b/src/lua/linit.c +index 9a5bcfdc..787865c0 100644 +--- a/src/lua/linit.c ++++ b/src/lua/linit.c +@@ -8,6 +8,10 @@ + #define linit_c + #define LUA_LIB + ++/** Pi-hole modification **/ ++#include "ftl_lua.h" ++/**************************/ ++ + /* + ** If you embed Lua in your program and need to open the standard + ** libraries, call luaL_openlibs in your program. If you need a +@@ -64,5 +68,7 @@ LUALIB_API void luaL_openlibs (lua_State *L) { + luaL_requiref(L, lib->name, lib->func, 1); + lua_pop(L, 1); /* remove lib */ + } ++ // Load and enable libraries bundled with Pi-hole ++ ftl_lua_init(L); + } + +diff --git a/src/lua/lua.c b/src/lua/lua.c +index 35fb281d..111a1b2b 100644 +--- a/src/lua/lua.c ++++ b/src/lua/lua.c +@@ -20,10 +20,6 @@ + #include "lauxlib.h" + #include "lualib.h" + +-/** Pi-hole modification **/ +-#include "ftl_lua.h" +-/**************************/ +- + + #if !defined(LUA_PROGNAME) + #define LUA_PROGNAME "lua" +@@ -218,9 +214,7 @@ static int dostring (lua_State *L, const char *s, const char *name) { + ** If there is no explicit modname and globname contains a '-', cut + ** the suffix after '-' (the "version") to make the global name. + */ +-/************** Pi-hole modification ***************/ +-int dolibrary (lua_State *L, char *globname) { +-/***************************************************/ ++static int dolibrary (lua_State *L, char *globname) { + int status; + char *suffix = NULL; + char *modname = strchr(globname, '='); +@@ -655,11 +649,6 @@ static int pmain (lua_State *L) { + return 0; /* error running LUA_INIT */ + } + +- /************** Pi-hole modification ***************/ +- // Load and enable libraries bundled with Pi-hole +- ftl_lua_init(L); +- /***************************************************/ +- + if (!runargs(L, argv, optlim)) /* execute arguments -e and -l */ + return 0; /* something failed */ + if (script > 0) { /* execute main script (if there is one) */ +-- +2.34.1 + From 31d06849124e60d847e277b1571f3acc9721ec5d Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 23 Sep 2024 22:37:01 +0200 Subject: [PATCH 329/339] Add CI test for proper Lua backtrace generation Signed-off-by: DL6ER --- .devcontainer/devcontainer.json | 2 +- test/broken_lua.lp | 4 ++++ test/run.sh | 5 ++++- test/test_suite.bats | 20 +++++++++++++++++--- 4 files changed, 26 insertions(+), 5 deletions(-) create mode 100644 test/broken_lua.lp diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 646a1a08..dbb9de7a 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -16,7 +16,7 @@ }, "mounts": [ "type=bind,source=/home/${localEnv:USER}/.ssh,target=/root/.ssh,readonly", - "type=bind,source=/var/www/html,target=/var/www/html,readonly" + "type=bind,source=/var/www/html/admin,target=/var/www/html/admin,readonly" ] } diff --git a/test/broken_lua.lp b/test/broken_lua.lp new file mode 100644 index 00000000..32f57aff --- /dev/null +++ b/test/broken_lua.lp @@ -0,0 +1,4 @@ + \ No newline at end of file diff --git a/test/run.sh b/test/run.sh index 5d490a82..3a2ee4cf 100755 --- a/test/run.sh +++ b/test/run.sh @@ -23,7 +23,7 @@ done rm -rf /etc/pihole /var/log/pihole /dev/shm/FTL-* # Create necessary directories and files -mkdir -p /home/pihole /etc/pihole /run/pihole /var/log/pihole /etc/pihole/config_backups +mkdir -p /home/pihole /etc/pihole /run/pihole /var/log/pihole /etc/pihole/config_backups /var/www/html echo "" > /var/log/pihole/FTL.log echo "" > /var/log/pihole/pihole.log echo "" > /var/log/pihole/webserver.log @@ -62,6 +62,9 @@ cp test/01-pihole-tests.conf /etc/dnsmasq.d/01-pihole-tests.conf # Prepare versions file (read by /api/version) cp test/versions /etc/pihole/versions +# Prepare Lua test script +cp test/broken_lua.lp /var/www/html/broken_lua.lp + # Prepare local powerDNS resolver bash test/pdns/setup.sh diff --git a/test/test_suite.bats b/test/test_suite.bats index 2860776f..59b98902 100644 --- a/test/test_suite.bats +++ b/test/test_suite.bats @@ -1235,10 +1235,10 @@ [[ ${lines[0]} == '{"error":{"key":"not_found","message":"Not found","hint":"/api/undefined"},"took":'*'}' ]] } -@test "HTTP server responds with normal error 404 to path outside /admin" { - run bash -c 'curl -s 127.0.0.1/undefined' +@test "HTTP server responds with error 404 to path outside /admin" { + run bash -c 'curl -sI 127.0.0.1/undefined' printf "%s\n" "${lines[@]}" - [[ ${lines[0]} == "Error 404: Not Found" ]] + [[ ${lines[@]} == *"HTTP/1.1 404 Not Found"* ]] } @test "LUA: Interpreter returns FTL version" { @@ -1684,6 +1684,20 @@ [[ ${lines[0]} == "0" ]] } +# This test should run before a password it set +@test "Lua server page is generating proper backtrace" { + # Run a page with a syntax error + run bash -c 'curl -s 127.0.0.1/broken_lua' + printf "%s\n" "${lines[@]}" + [[ ${lines[0]} == 'Hello, world!' ]] + [[ ${lines[1]} == '[string "/var/www/html/broken_lua.lp"]:4: Cannot include [/var/www/html/does_not_exist.lp]: not found' ]] + [[ ${lines[2]} == '' ]] + + # Check if the error is logged (-F = fixed string (no regex), -q = quiet) + run grep -qF 'LSP Kepler: call failed: runtime error: [string "/var/www/html/broken_lua.lp"]:4: Cannot include [/var/www/html/does_not_exist.lp]: not found' /var/log/pihole/webserver.log + [[ $status == 0 ]] +} + @test "API authorization (without password): No login required" { run bash -c 'curl -s 127.0.0.1/api/auth' printf "%s\n" "${lines[@]}" From 9de2bbdad4b98ee47237cff4ee4b722eb572015d Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 23 Sep 2024 22:55:33 +0200 Subject: [PATCH 330/339] Make the CI test a bit more complex so we actually get some stack traceback Signed-off-by: DL6ER --- test/broken_lua.lp | 6 +++--- test/broken_lua_2.lp | 4 ++++ test/run.sh | 1 + test/test_suite.bats | 13 +++++++++---- 4 files changed, 17 insertions(+), 7 deletions(-) create mode 100644 test/broken_lua_2.lp diff --git a/test/broken_lua.lp b/test/broken_lua.lp index 32f57aff..bf8a5fef 100644 --- a/test/broken_lua.lp +++ b/test/broken_lua.lp @@ -1,4 +1,4 @@ \ No newline at end of file +mg.write("Hello, world 1!\n") +mg.include("broken_lua_2.lp", "r") +?> diff --git a/test/broken_lua_2.lp b/test/broken_lua_2.lp new file mode 100644 index 00000000..e0ada85c --- /dev/null +++ b/test/broken_lua_2.lp @@ -0,0 +1,4 @@ + diff --git a/test/run.sh b/test/run.sh index 3a2ee4cf..e5d9331e 100755 --- a/test/run.sh +++ b/test/run.sh @@ -64,6 +64,7 @@ cp test/versions /etc/pihole/versions # Prepare Lua test script cp test/broken_lua.lp /var/www/html/broken_lua.lp +cp test/broken_lua_2.lp /var/www/html/broken_lua_2.lp # Prepare local powerDNS resolver bash test/pdns/setup.sh diff --git a/test/test_suite.bats b/test/test_suite.bats index 59b98902..b61733b9 100644 --- a/test/test_suite.bats +++ b/test/test_suite.bats @@ -1689,12 +1689,17 @@ # Run a page with a syntax error run bash -c 'curl -s 127.0.0.1/broken_lua' printf "%s\n" "${lines[@]}" - [[ ${lines[0]} == 'Hello, world!' ]] - [[ ${lines[1]} == '[string "/var/www/html/broken_lua.lp"]:4: Cannot include [/var/www/html/does_not_exist.lp]: not found' ]] - [[ ${lines[2]} == '' ]] + [[ ${lines[0]} == 'Hello, world 1!' ]] + [[ ${lines[1]} == 'Hello, world 2!' ]] + [[ ${lines[2]} == '[string "/var/www/html/broken_lua_2.lp"]:4: Cannot include [/var/www/html/does_not_exist.lp]: not found' ]] + [[ ${lines[3]} == 'stack traceback:' ]] + [[ ${lines[4]} == " [C]: in field 'include'" ]] + [[ ${lines[5]} == ' [string "/var/www/html/broken_lua.lp"]:4: in main chunk' ]] + [[ ${lines[6]} == 'aborting' ]] + [[ ${lines[7]} == '' ]] # Check if the error is logged (-F = fixed string (no regex), -q = quiet) - run grep -qF 'LSP Kepler: call failed: runtime error: [string "/var/www/html/broken_lua.lp"]:4: Cannot include [/var/www/html/does_not_exist.lp]: not found' /var/log/pihole/webserver.log + run grep -qF 'LSP Kepler: call failed: runtime error: [string "/var/www/html/broken_lua_2.lp"]:4: Cannot include [/var/www/html/does_not_exist.lp]: not found' /var/log/pihole/webserver.log [[ $status == 0 ]] } From 92e259021e1457dafac6a50188ae4020b8c8a07d Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 25 Sep 2024 10:05:50 +0200 Subject: [PATCH 331/339] Disentangle info/ftl : ftl.database.domains from newly added ftl.database.regex Signed-off-by: DL6ER --- src/api/docs/content/specs/info.yaml | 11 +++++++++++ src/api/info.c | 15 +++++++++++---- src/database/gravity-db.c | 6 ------ src/datastructure.c | 6 ++++-- src/enums.h | 2 -- src/shmem.h | 10 ++++++++-- 6 files changed, 34 insertions(+), 16 deletions(-) diff --git a/src/api/docs/content/specs/info.yaml b/src/api/docs/content/specs/info.yaml index d76dbd6a..6be2b1ac 100644 --- a/src/api/docs/content/specs/info.yaml +++ b/src/api/docs/content/specs/info.yaml @@ -717,6 +717,17 @@ components: type: integer description: Number of denied domains example: 3 + regex: + type: object + properties: + allowed: + type: integer + description: Number of allowed regex filters + example: 4 + denied: + type: integer + description: Number of denied regex filters + example: 2 privacy_level: type: integer description: Currently used privacy level diff --git a/src/api/info.c b/src/api/info.c index ce7a7a8d..6d5acdf7 100644 --- a/src/api/info.c +++ b/src/api/info.c @@ -544,8 +544,10 @@ static int get_ftl_obj(struct ftl_conn *api, cJSON *ftl) const int db_groups = counters->database.groups; const int db_lists = counters->database.lists; const int db_clients = counters->database.clients; - const int db_allowed = counters->database.domains.allowed; - const int db_denied = counters->database.domains.denied; + const int db_allowed_exact = counters->database.domains.allowed.exact; + const int db_denied_exact = counters->database.domains.denied.exact; + const int db_allowed_regex = counters->database.domains.allowed.regex; + const int db_denied_regex = counters->database.domains.denied.regex; const int clients_total = counters->clients; const int privacylevel = config.misc.privacylevel.v.privacy_level; const double qps = get_qps(); @@ -570,9 +572,14 @@ static int get_ftl_obj(struct ftl_conn *api, cJSON *ftl) JSON_ADD_NUMBER_TO_OBJECT(database, "clients", db_clients); cJSON *domains = JSON_NEW_OBJECT(); - JSON_ADD_NUMBER_TO_OBJECT(domains, "allowed", db_allowed); - JSON_ADD_NUMBER_TO_OBJECT(domains, "denied", db_denied); + JSON_ADD_NUMBER_TO_OBJECT(domains, "allowed", db_allowed_exact); + JSON_ADD_NUMBER_TO_OBJECT(domains, "denied", db_denied_exact); JSON_ADD_ITEM_TO_OBJECT(database, "domains", domains); + + cJSON *regex = JSON_NEW_OBJECT(); + JSON_ADD_NUMBER_TO_OBJECT(regex, "allowed", db_allowed_regex); + JSON_ADD_NUMBER_TO_OBJECT(regex, "denied", db_denied_regex); + JSON_ADD_ITEM_TO_OBJECT(database, "regex", regex); JSON_ADD_ITEM_TO_OBJECT(ftl, "database", database); JSON_ADD_NUMBER_TO_OBJECT(ftl, "privacy_level", privacylevel); diff --git a/src/database/gravity-db.c b/src/database/gravity-db.c index a90ac7ef..282c80dc 100644 --- a/src/database/gravity-db.c +++ b/src/database/gravity-db.c @@ -1112,12 +1112,6 @@ int gravityDB_count(const enum gravity_tables list) case ADLISTS_TABLE: querystr = "SELECT COUNT(1) FROM adlist WHERE enabled != 0"; break; - case DENIED_DOMAINS_TABLE: - querystr = "SELECT COUNT(1) FROM domainlist WHERE (type = 0 OR type = 2) AND enabled != 0"; - break; - case ALLOWED_DOMAINS_TABLE: - querystr = "SELECT COUNT(1) FROM domainlist WHERE (type = 1 OR type = 3) AND enabled != 0"; - break; case UNKNOWN_TABLE: log_err("List type %u unknown!", list); gravityDB_close(); diff --git a/src/datastructure.c b/src/datastructure.c index b9d0c1d9..7d2504fa 100644 --- a/src/datastructure.c +++ b/src/datastructure.c @@ -594,8 +594,10 @@ void FTL_reload_all_domainlists(void) counters->database.groups = gravityDB_count(GROUPS_TABLE); counters->database.clients = gravityDB_count(CLIENTS_TABLE); counters->database.lists = gravityDB_count(ADLISTS_TABLE); - counters->database.domains.allowed = gravityDB_count(DENIED_DOMAINS_TABLE); - counters->database.domains.denied = gravityDB_count(ALLOWED_DOMAINS_TABLE); + counters->database.domains.allowed.exact = gravityDB_count(EXACT_WHITELIST_TABLE); + counters->database.domains.denied.exact = gravityDB_count(EXACT_BLACKLIST_TABLE); + counters->database.domains.allowed.regex = gravityDB_count(REGEX_ALLOW_TABLE); + counters->database.domains.denied.regex = gravityDB_count(REGEX_DENY_TABLE); // Read and compile possible regex filters // only after having called gravityDB_reopen() diff --git a/src/enums.h b/src/enums.h index ac7ed52c..e748562c 100644 --- a/src/enums.h +++ b/src/enums.h @@ -193,8 +193,6 @@ enum gravity_tables { CLIENTS_TABLE, GROUPS_TABLE, ADLISTS_TABLE, - DENIED_DOMAINS_TABLE, - ALLOWED_DOMAINS_TABLE, UNKNOWN_TABLE } __attribute__ ((packed)); diff --git a/src/shmem.h b/src/shmem.h index 2492f986..f717445e 100644 --- a/src/shmem.h +++ b/src/shmem.h @@ -62,8 +62,14 @@ typedef struct { int groups; int lists; struct { - int allowed; - int denied; + struct { + int exact; + int regex; + } allowed; + struct { + int exact; + int regex; + } denied; } domains; } database; int querytype[TYPE_MAX]; From bb9346744228a83e1b40782516e1c59cd9a756bb Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 26 Sep 2024 11:02:39 +0200 Subject: [PATCH 332/339] Add new binary integrity verification function Signed-off-by: DL6ER --- src/CMakeLists.txt | 7 +++ src/args.c | 30 ++++++++++-- src/config/config.c | 2 +- src/config/toml_writer.c | 4 +- src/files.c | 103 ++++++++++++++++++++++++++++++++++++--- src/files.h | 3 +- 6 files changed, 134 insertions(+), 15 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 008e6d7d..778764a6 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -371,6 +371,13 @@ else() target_compile_definitions(civetweb PRIVATE NO_SSL) endif() +# After finishing building the FTL binary, we append the sha256sum of the binary in raw form to itself +add_custom_command(TARGET pihole-FTL POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy $ $/pihole-FTL.tmp + COMMAND sha256sum $.tmp | cut -d ' ' -f 1 | xxd -r -p >> $.tmp + COMMAND mv $.tmp $ + ) + find_program(SETCAP setcap) install(TARGETS pihole-FTL RUNTIME DESTINATION bin diff --git a/src/args.c b/src/args.c index bf2713d8..085b633a 100644 --- a/src/args.c +++ b/src/args.c @@ -532,12 +532,13 @@ void parse_args(int argc, char* argv[]) } // sha256sum mode - if(argc == 3 && strcmp(argv[1], "sha256sum") == 0) + if((argc == 3 || (argc == 4 && strcmp(argv[2], "--skip-end"))) && strcmp(argv[1], "sha256sum") == 0) { + const bool skip_end = argc == 4; // Enable stdout printing cli_mode = true; uint8_t checksum[SHA256_DIGEST_SIZE]; - if(!sha256sum(argv[2], checksum)) + if(!sha256sum(argv[skip_end ? 3 : 2], checksum, skip_end)) exit(EXIT_FAILURE); // Convert checksum to hex string @@ -545,10 +546,23 @@ void parse_args(int argc, char* argv[]) sha256_raw_to_hex(checksum, hex); // Print result - printf("%s %s\n", hex, argv[2]); + printf("%s %s\n", hex, argv[skip_end ? 3 : 2]); exit(EXIT_SUCCESS); } + // Checksum verification mode + if(argc == 2 && strcmp(argv[1], "verify") == 0) + { + // Enable stdout printing + cli_mode = true; + const bool match = verify_self_hash(true); + if(match) + printf("%s SHA256 checksum matches\n", cli_tick()); + else + printf("%s SHA256 checksum does not match\n", cli_cross()); + exit(match ? EXIT_SUCCESS : EXIT_FAILURE); + } + // Local reverse name resolver if((argc == 3 || argc == 4) && strcasecmp(argv[1], "ptr") == 0) { @@ -1082,10 +1096,18 @@ void parse_args(int argc, char* argv[]) printf(" %s--update%s flag is given.\n\n", purple, normal); printf(" Usage: %spihole-FTL ntp %s[server]%s %s[--update]%s\n\n", green, cyan, normal, purple, normal); + printf("%sSHA256 checksum tools:%s\n", yellow, normal); + printf(" Calculates the SHA256 checksum of a file.\n\n"); + printf(" Usage: %spihole-FTL sha256sum %sfile%s\n\n", green, cyan, normal); + printf(" The special flag %s--skip-end%s can be used to skip the last 32\n", purple, normal); + printf(" bytes of the file. This is useful for files which have their\n"); + printf(" checksum appended at the end of the file, e.g., pihole-FTL:\n\n"); + printf(" %spihole-FTL sha256sum %s--skip_end %sfile%s\n\n", green, purple, cyan, normal); + printf("%sOther:%s\n", yellow, normal); + printf("\t%sverify%s Verify the integrity of the FTL binary\n", green, normal); printf("\t%sptr %sIP%s %s[tcp]%s Resolve IP address to hostname\n", green, cyan, normal, purple, normal); printf("\t Append %stcp%s to use TCP instead of UDP\n", purple, normal); - printf("\t%ssha256sum %sfile%s Calculate SHA256 checksum of a file\n", green, cyan, normal); printf("\t%sdhcp-discover%s Discover DHCP servers in the local\n", green, normal); printf("\t network\n"); printf("\t%sarp-scan %s[-a/-x]%s Use ARP to scan local network for\n", green, cyan, normal); diff --git a/src/config/config.c b/src/config/config.c index b984af70..2331adbd 100644 --- a/src/config/config.c +++ b/src/config/config.c @@ -1831,7 +1831,7 @@ void reread_config(void) // Create checksum of config file uint8_t checksum[SHA256_DIGEST_SIZE]; - if(!sha256sum(GLOBALTOMLPATH, checksum)) + if(!sha256sum(GLOBALTOMLPATH, checksum, false)) { log_err("Unable to create checksum of %s, not re-reading config file", GLOBALTOMLPATH); return; diff --git a/src/config/toml_writer.c b/src/config/toml_writer.c index 8361c324..57a2090a 100644 --- a/src/config/toml_writer.c +++ b/src/config/toml_writer.c @@ -35,7 +35,7 @@ bool writeFTLtoml(const bool verbose) // We need to (re-)calculate the checksum here as it'd otherwise // be outdated (in non-read-only mode, it's calculated at the // end of this function) - if(!sha256sum(GLOBALTOMLPATH, last_checksum)) + if(!sha256sum(GLOBALTOMLPATH, last_checksum, false)) log_err("Unable to create checksum of %s", GLOBALTOMLPATH); return true; } @@ -209,7 +209,7 @@ bool writeFTLtoml(const bool verbose) log_debug(DEBUG_CONFIG, "pihole.toml unchanged"); } - if(!sha256sum(GLOBALTOMLPATH, last_checksum)) + if(!sha256sum(GLOBALTOMLPATH, last_checksum, false)) log_err("Unable to create checksum of %s", GLOBALTOMLPATH); return true; diff --git a/src/files.c b/src/files.c index 8158f60a..9e934b13 100644 --- a/src/files.c +++ b/src/files.c @@ -13,6 +13,8 @@ #include "config/config.h" #include "config/setupVars.h" #include "log.h" +// sha256_raw_to_hex() +#include "config/password.h" // opendir(), readdir() #include @@ -27,10 +29,8 @@ // sendfile() #include #include - // PRIu64 #include - //basename() #include @@ -714,7 +714,7 @@ bool files_different(const char *pathA, const char* pathB, unsigned int from) } // Create SHA256 checksum of a file -bool sha256sum(const char *path, uint8_t checksum[SHA256_DIGEST_SIZE]) +bool sha256sum(const char *path, uint8_t checksum[SHA256_DIGEST_SIZE], const bool skip_end) { // Open file FILE *fp = fopen(path, "rb"); @@ -728,14 +728,30 @@ bool sha256sum(const char *path, uint8_t checksum[SHA256_DIGEST_SIZE]) struct sha256_ctx ctx; sha256_init(&ctx); - // Read file in chunks of bytes - const size_t pagesize = getpagesize(); - unsigned char *buf = calloc(pagesize, sizeof(char)); + // Get size of file + fseek(fp, 0, SEEK_END); + size_t filesize = ftell(fp); + fseek(fp, 0, SEEK_SET); + + // Determine chunk size + size_t chunksize = getpagesize(); + + // Read file in chunks + unsigned char *buf = calloc(chunksize, sizeof(char)); size_t len; - while((len = fread(buf, sizeof(char), pagesize, fp)) > 0) + while((len = fread(buf, sizeof(char), chunksize, fp)) > 0) { // Update SHA256 context sha256_update(&ctx, len, buf); + + // Reduce filesize by the number of bytes read + filesize -= len; + + // If we want to skip the end of the file, we have to adjust the + // chunk size to the remaining bytes minus the size of the SHA256 + // checksum itself + if(skip_end && filesize <= chunksize + SHA256_DIGEST_SIZE) + chunksize = filesize - SHA256_DIGEST_SIZE; } // Finalize SHA256 context @@ -749,3 +765,76 @@ bool sha256sum(const char *path, uint8_t checksum[SHA256_DIGEST_SIZE]) return true; } + +/** + * @brief Verifies the integrity of the current executable file by comparing its + * SHA256 checksum with a pre-computed hash stored in the last 8 bytes of the + * binary. + * + * @param verbose A boolean value indicating whether verbose output should be + * enabled. + * @return Returns true if the checksum matches the expected value, false + * otherwise. + */ +bool verify_self_hash(bool verbose) +{ + // Get the filename of the current executable + char filename[PATH_MAX] = { 0 }; + if(readlink("/proc/self/exe", filename, sizeof(filename)) == -1) + { + log_err("Failed to read self filename: %s", strerror(errno)); + return -1; + } + + // Read the pre-computed hash - it is stored in the last 8 bytes of the + // binary itself + uint8_t self_hash[SHA256_DIGEST_SIZE]; + FILE *f = fopen(filename, "r"); + if(f == NULL) + { + log_err("Failed to open self file \"%s\": %s", filename, strerror(errno)); + return -1; + } + if(fseek(f, -SHA256_DIGEST_SIZE, SEEK_END) != 0) + { + log_err("Failed to seek to hash: %s", strerror(errno)); + fclose(f); + return -1; + } + if(fread(self_hash, SHA256_DIGEST_SIZE, 1, f) != 1) + { + log_err("Failed to read hash: %s", strerror(errno)); + fclose(f); + return -1; + } + fclose(f); + + // Calculate the hash of the binary + uint8_t checksum[SHA256_DIGEST_SIZE]; + if(!sha256sum(filename, checksum, true)) + { + log_err("Failed to calculate SHA256 checksum of %s", filename); + return false; + } + + // Compare the checksums + bool success = memcmp(checksum, self_hash, SHA256_DIGEST_SIZE) == 0; + if(!success) + log_err("SHA256 checksum of %s does not match the expected value", filename); + + // Log the checksums if the verification failed or if verbose output is + // requested + if(!success || verbose) + { + // Convert checksums to human-readable hex strings + char expected_hex[SHA256_DIGEST_SIZE*2+1]; + sha256_raw_to_hex(self_hash, expected_hex); + char actual_hex[SHA256_DIGEST_SIZE*2+1]; + sha256_raw_to_hex(checksum, actual_hex); + + log_info("Expected: %s", expected_hex); + log_info("Actual: %s", actual_hex); + } + + return success; +} diff --git a/src/files.h b/src/files.h index fc10ad23..6af94bd3 100644 --- a/src/files.h +++ b/src/files.h @@ -36,7 +36,8 @@ bool directory_exists(const char *path); bool chown_pihole(const char *path, struct passwd *pwd); void rotate_files(const char *path, char **first_file); bool files_different(const char *pathA, const char* pathB, unsigned int from); -bool sha256sum(const char *path, uint8_t checksum[SHA256_DIGEST_SIZE]); +bool sha256sum(const char *path, uint8_t checksum[SHA256_DIGEST_SIZE], const bool skip_end); +bool verify_self_hash(bool verbose); int parse_line(char *line, char **key, char **value); From 3ca04acd75cfd0e80ce13849ae353cb8e436cfd6 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 26 Sep 2024 11:43:22 +0200 Subject: [PATCH 333/339] Add message table entry on binary verification failure Signed-off-by: DL6ER --- src/args.c | 2 +- src/database/message-table.c | 75 ++++++++++++++++++++++++++++++++++++ src/database/message-table.h | 1 + src/dnsmasq_interface.c | 6 +++ src/enums.h | 1 + src/files.c | 19 +++++---- src/files.h | 2 +- src/main.c | 2 + 8 files changed, 98 insertions(+), 10 deletions(-) diff --git a/src/args.c b/src/args.c index 085b633a..c9bcbe15 100644 --- a/src/args.c +++ b/src/args.c @@ -555,7 +555,7 @@ void parse_args(int argc, char* argv[]) { // Enable stdout printing cli_mode = true; - const bool match = verify_self_hash(true); + const bool match = verify_FTL(true); if(match) printf("%s SHA256 checksum matches\n", cli_tick()); else diff --git a/src/database/message-table.c b/src/database/message-table.c index 409d151b..a563a939 100644 --- a/src/database/message-table.c +++ b/src/database/message-table.c @@ -31,6 +31,8 @@ #include "database/query-table.h" // escape_html() #include "webserver/http-common.h" +// GIT_HASH, FTL_ARCH +#include "version.h" // Number of arguments in a variadic macro // Credit: https://stackoverflow.com/a/35693080/2087442 @@ -99,6 +101,8 @@ static const char *get_message_type_str(const enum message_type type) return "CONNECTION_ERROR"; case NTP_MESSAGE: return "NTP"; + case VERIFY_MESSAGE: + return "VERIFY"; case MAX_MESSAGE: default: return "UNKNOWN"; @@ -135,6 +139,8 @@ static enum message_type get_message_type_from_string(const char *typestr) return CONNECTION_ERROR_MESSAGE; else if (strcmp(typestr, "NTP") == 0) return NTP_MESSAGE; + else if (strcmp(typestr, "VERIFY") == 0) + return VERIFY_MESSAGE; else return MAX_MESSAGE; } @@ -242,6 +248,14 @@ static unsigned char message_blob_types[MAX_MESSAGE][5] = SQLITE_NULL, // not used SQLITE_NULL, // not used SQLITE_NULL // not used + }, + { + // VERIFY_MESSAGE: The message column contains the error + SQLITE_TEXT, // expected checksum + SQLITE_TEXT, // actual checksum + SQLITE_TEXT, // FTL commit hash + SQLITE_TEXT, // FTL architecture + SQLITE_NULL // not used } }; // Create message table in the database @@ -928,6 +942,39 @@ static void format_ntp_message(char *plain, const int sizeof_plain, char *html, log_warn("format_ntp_message(): Buffer too small to hold HTML message, warning truncated"); } +static void format_verify_message(char *plain, const int sizeof_plain, char *html, const int sizeof_html, + const char *message, const char *expected, const char *actual, + const char *commit, const char *arch) +{ + if(snprintf(plain, sizeof_plain, "%s - expected \"%s\", but got \"%s\" - FTL commit is %s on %s", + message, expected, actual, commit, arch) > sizeof_plain) + log_warn("format_verify_message(): Buffer too small to hold plain message, warning truncated"); + + // Return early if HTML text is not required + if(sizeof_html < 1 || html == NULL) + return; + + char *escaped_message = escape_html(message); + char *escaped_expected = escape_html(expected); + char *escaped_actual = escape_html(actual); + char *escaped_commit = escape_html(commit); + char *escaped_arch = escape_html(arch); + + // Return early if memory allocation failed + if(escaped_message == NULL || escaped_expected == NULL || escaped_actual == NULL || escaped_commit == NULL || escaped_arch == NULL) + return; + + if(snprintf(html, sizeof_html, "%s
    Expected: 
    %s

    Actual: 
    %s

    FTL commit is %s on %s", + escaped_message, escaped_expected, escaped_actual, escaped_commit, escaped_arch) > sizeof_html) + log_warn("format_verify_message(): Buffer too small to hold HTML message, warning truncated"); + + free(escaped_message); + free(escaped_expected); + free(escaped_actual); + free(escaped_commit); + free(escaped_arch); +} + int count_messages(const bool filter_dnsmasq_warnings) { int count = 0; @@ -1187,6 +1234,20 @@ bool format_messages(cJSON *array) break; } + case VERIFY_MESSAGE: + { + const char *message = (const char*)sqlite3_column_text(stmt, 3); + const char *expected = (const char*)sqlite3_column_text(stmt, 4); + const char *actual = (const char*)sqlite3_column_text(stmt, 5); + const char *hash = (const char*)sqlite3_column_text(stmt, 6); + const char *arch = (const char*)sqlite3_column_text(stmt, 7); + + format_verify_message(plain, sizeof(plain), html, sizeof(html), + message, expected, actual, hash, arch); + + break; + } + case MAX_MESSAGE: // Fall through default: log_warn("format_messages() - Unknown message type: %s", mtypestr); @@ -1458,3 +1519,17 @@ void log_ntp_message(const bool error, const bool server, const char *message) add_message(NTP_MESSAGE, message, level, who); } + +void log_verify_message(const char *expected, const char *actual) +{ + // Create message + char buf[2048]; + snprintf(buf, sizeof(buf), "Corrupt binary detected - this may lead to unexpected behaviour!"); + + // Log to FTL.log + log_crit("%s", buf); + + // Log to database + add_message(VERIFY_MESSAGE, buf, expected, actual, GIT_HASH, FTL_ARCH); + +} diff --git a/src/database/message-table.h b/src/database/message-table.h index 5354af6f..d230f066 100644 --- a/src/database/message-table.h +++ b/src/database/message-table.h @@ -31,5 +31,6 @@ void logg_inaccessible_adlist(const int dbindex, const char *address); void log_certificate_domain_mismatch(const char *certfile, const char *domain); void log_connection_error(const char *server, const char *reason, const char *error); void log_ntp_message(const bool error, const bool server, const char *message); +void log_verify_message(const char *expected, const char *actual); #endif //MESSAGETABLE_H diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c index 362eb1bb..9eec8cda 100644 --- a/src/dnsmasq_interface.c +++ b/src/dnsmasq_interface.c @@ -3130,6 +3130,12 @@ void FTL_fork_and_bind_sockets(struct passwd *ent_pw, bool dnsmasq_start) // Flush messages stored in the long-term database flush_message_table(); + // Verify checksum of this binary early on to ensure that the binary is + // not corrupted and that the binary is not tampered with. We can only + // do this here as we need the database to be properly initialized + // in case we need to store the verification result + verify_FTL(false); + // Initialize in-memory database starting index update_disk_db_idx(); diff --git a/src/enums.h b/src/enums.h index e748562c..1dd8c929 100644 --- a/src/enums.h +++ b/src/enums.h @@ -264,6 +264,7 @@ enum message_type { CERTIFICATE_DOMAIN_MISMATCH_MESSAGE, CONNECTION_ERROR_MESSAGE, NTP_MESSAGE, + VERIFY_MESSAGE, MAX_MESSAGE, } __attribute__ ((packed)); diff --git a/src/files.c b/src/files.c index 9e934b13..98834f19 100644 --- a/src/files.c +++ b/src/files.c @@ -15,6 +15,8 @@ #include "log.h" // sha256_raw_to_hex() #include "config/password.h" +// log_verify_message() +#include "database/message-table.h" // opendir(), readdir() #include @@ -776,7 +778,7 @@ bool sha256sum(const char *path, uint8_t checksum[SHA256_DIGEST_SIZE], const boo * @return Returns true if the checksum matches the expected value, false * otherwise. */ -bool verify_self_hash(bool verbose) +bool verify_FTL(bool verbose) { // Get the filename of the current executable char filename[PATH_MAX] = { 0 }; @@ -820,11 +822,6 @@ bool verify_self_hash(bool verbose) // Compare the checksums bool success = memcmp(checksum, self_hash, SHA256_DIGEST_SIZE) == 0; if(!success) - log_err("SHA256 checksum of %s does not match the expected value", filename); - - // Log the checksums if the verification failed or if verbose output is - // requested - if(!success || verbose) { // Convert checksums to human-readable hex strings char expected_hex[SHA256_DIGEST_SIZE*2+1]; @@ -832,8 +829,14 @@ bool verify_self_hash(bool verbose) char actual_hex[SHA256_DIGEST_SIZE*2+1]; sha256_raw_to_hex(checksum, actual_hex); - log_info("Expected: %s", expected_hex); - log_info("Actual: %s", actual_hex); + if(!verbose) // during startup + log_verify_message(expected_hex, actual_hex); + else // CLI verification + { + log_err("Checksum verification failed!"); + log_err("Expected: %s", expected_hex); + log_err("Actual: %s", actual_hex); + } } return success; diff --git a/src/files.h b/src/files.h index 6af94bd3..b1096e14 100644 --- a/src/files.h +++ b/src/files.h @@ -37,7 +37,7 @@ bool chown_pihole(const char *path, struct passwd *pwd); void rotate_files(const char *path, char **first_file); bool files_different(const char *pathA, const char* pathB, unsigned int from); bool sha256sum(const char *path, uint8_t checksum[SHA256_DIGEST_SIZE], const bool skip_end); -bool verify_self_hash(bool verbose); +bool verify_FTL(bool verbose); int parse_line(char *line, char **key, char **value); diff --git a/src/main.c b/src/main.c index c7f288c3..d6909495 100644 --- a/src/main.c +++ b/src/main.c @@ -27,6 +27,8 @@ #include "overTime.h" // export_queries_to_disk() #include "database/query-table.h" +// verify_FTL() +#include "files.h" char *username; bool needGC = false; From cdba28e0c5285e088ac8868e5e660f24e5ef80ff Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 26 Sep 2024 11:57:53 +0200 Subject: [PATCH 334/339] Add verification CI test Signed-off-by: DL6ER --- test/test_suite.bats | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/test_suite.bats b/test/test_suite.bats index b61733b9..f39c23a8 100644 --- a/test/test_suite.bats +++ b/test/test_suite.bats @@ -8,6 +8,12 @@ [[ "${lines[@]}" == "" ]] } +@test "Check FTL binary integrity" { + run bash -c './pihole-FTL verify' + printf "%s\n" "${lines[@]}" + [[ "${lines[0]}" == *"SHA256 checksum matches" ]] +} + @test "Running a second instance is detected and prevented" { run bash -c 'su pihole -s /bin/sh -c "./pihole-FTL -f"' printf "%s\n" "${lines[@]}" From c2a2df8a77fbca30738a47879a5dff6ce0c01b2a Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 26 Sep 2024 18:19:27 +0200 Subject: [PATCH 335/339] Restart FTL to enforce flushing the cache when dns.piholePTR is changed Signed-off-by: DL6ER --- src/config/config.c | 3 ++- test/pihole.toml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/config/config.c b/src/config/config.c index b984af70..5cf9a837 100644 --- a/src/config/config.c +++ b/src/config/config.c @@ -442,7 +442,7 @@ static void initConfig(struct config *conf) conf->dns.analyzeOnlyAandAAAA.c = validate_stub; // Only type-based checking conf->dns.piholePTR.k = "dns.piholePTR"; - conf->dns.piholePTR.h = "Controls whether and how FTL will reply with for address for which a local interface exists."; + conf->dns.piholePTR.h = "Controls whether and how FTL will reply with for address for which a local interface exists. Changing this setting causes FTL to restart."; { struct enum_options piholePTR[] = { @@ -455,6 +455,7 @@ static void initConfig(struct config *conf) } conf->dns.piholePTR.t = CONF_ENUM_PTR_TYPE; conf->dns.piholePTR.d.ptr_type = PTR_PIHOLE; + conf->dns.piholePTR.f = FLAG_RESTART_FTL; conf->dns.piholePTR.c = validate_stub; // Only type-based checking conf->dns.replyWhenBusy.k = "dns.replyWhenBusy"; diff --git a/test/pihole.toml b/test/pihole.toml index e03fd65d..e30dedb6 100644 --- a/test/pihole.toml +++ b/test/pihole.toml @@ -47,7 +47,7 @@ analyzeOnlyAandAAAA = false # Controls whether and how FTL will reply with for address for which a local interface - # exists. + # exists. Changing this setting causes FTL to restart. # # Possible values are: # - "NONE" From e62e58619e096307e75b8b44c0e28f988823e82e Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 26 Sep 2024 19:25:04 +0200 Subject: [PATCH 336/339] Do not take the not-filtered shortcut if one of the magic upstream={blocklist,cache} is selected. This is currently broken as detection of ongoing filtering is through variable bindung to the SQL string. However, these two special upstreams work by using the IN operator which does not support binding to a prepared statement. Signed-off-by: DL6ER --- src/api/queries.c | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/src/api/queries.c b/src/api/queries.c index 077efc27..20768774 100644 --- a/src/api/queries.c +++ b/src/api/queries.c @@ -307,6 +307,11 @@ int api_queries(struct ftl_conn *api) bool cursor_set = false, where = false; double timestamp_from = 0.0, timestamp_until = 0.0; + // We use this boolean to memorize if we are filtering at all. It is used + // later to decide if we can short-circuit the query counting for + // performance reasons. + bool filtering = false; + // Filter-/sorting based on GET parameters? if(api->request->query_string != NULL) { @@ -331,11 +336,17 @@ int api_queries(struct ftl_conn *api) if(GET_STR("upstream", upstreamname, api->request->query_string) > 0) { if(strcmp(upstreamname, "blocklist") == 0) + { // Pseudo-upstream for blocked queries add_querystr_string(api, querystr, "q.status IN ", get_blocked_statuslist(), &where); + filtering = true; + } else if(strcmp(upstreamname, "cache") == 0) + { // Pseudo-upstream for cached queries add_querystr_string(api, querystr, "q.status IN ", get_cached_statuslist(), &where); + filtering = true; + } else { if(is_wildcard(upstreamname)) @@ -510,11 +521,6 @@ int api_queries(struct ftl_conn *api) } } - // We use this boolean to memorize if we are filtering at all. It is used - // later to decide if we can short-circuit the query counting for - // performance reasons. - bool filtering = false; - // Regex filtering? regex_t *regex_domains = NULL; unsigned int N_regex_domains = 0; From 450964edefd444768517b55c342111bd6ff5000e Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 29 Sep 2024 10:40:29 +0200 Subject: [PATCH 337/339] Remove sha256 --skip-end feature as discussed during PR review process Signed-off-by: DL6ER --- src/args.c | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/src/args.c b/src/args.c index c9bcbe15..1a04a8ad 100644 --- a/src/args.c +++ b/src/args.c @@ -532,13 +532,12 @@ void parse_args(int argc, char* argv[]) } // sha256sum mode - if((argc == 3 || (argc == 4 && strcmp(argv[2], "--skip-end"))) && strcmp(argv[1], "sha256sum") == 0) + if(argc == 3 && strcmp(argv[1], "sha256sum") == 0) { - const bool skip_end = argc == 4; // Enable stdout printing cli_mode = true; uint8_t checksum[SHA256_DIGEST_SIZE]; - if(!sha256sum(argv[skip_end ? 3 : 2], checksum, skip_end)) + if(!sha256sum(argv[2], checksum, false)) exit(EXIT_FAILURE); // Convert checksum to hex string @@ -546,7 +545,7 @@ void parse_args(int argc, char* argv[]) sha256_raw_to_hex(checksum, hex); // Print result - printf("%s %s\n", hex, argv[skip_end ? 3 : 2]); + printf("%s %s\n", hex, argv[2]); exit(EXIT_SUCCESS); } @@ -1097,12 +1096,11 @@ void parse_args(int argc, char* argv[]) printf(" Usage: %spihole-FTL ntp %s[server]%s %s[--update]%s\n\n", green, cyan, normal, purple, normal); printf("%sSHA256 checksum tools:%s\n", yellow, normal); - printf(" Calculates the SHA256 checksum of a file.\n\n"); + printf(" Calculates the SHA256 checksum of a file. The checksum is\n"); + printf(" computed as described in FIPS-180-2 and uses streaming\n"); + printf(" to allow processing arbitrary large files with a small\n"); + printf(" memory footprint.\n\n"); printf(" Usage: %spihole-FTL sha256sum %sfile%s\n\n", green, cyan, normal); - printf(" The special flag %s--skip-end%s can be used to skip the last 32\n", purple, normal); - printf(" bytes of the file. This is useful for files which have their\n"); - printf(" checksum appended at the end of the file, e.g., pihole-FTL:\n\n"); - printf(" %spihole-FTL sha256sum %s--skip_end %sfile%s\n\n", green, purple, cyan, normal); printf("%sOther:%s\n", yellow, normal); printf("\t%sverify%s Verify the integrity of the FTL binary\n", green, normal); From 8d1394d00eb78ba294cadd88889166df7043258f Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 29 Sep 2024 10:43:15 +0200 Subject: [PATCH 338/339] Simplify text Signed-off-by: DL6ER --- src/args.c | 7 +++---- test/test_suite.bats | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/args.c b/src/args.c index 1a04a8ad..529bb488 100644 --- a/src/args.c +++ b/src/args.c @@ -555,10 +555,9 @@ void parse_args(int argc, char* argv[]) // Enable stdout printing cli_mode = true; const bool match = verify_FTL(true); - if(match) - printf("%s SHA256 checksum matches\n", cli_tick()); - else - printf("%s SHA256 checksum does not match\n", cli_cross()); + printf("%s Binary integrity check: %s\n", + match ? cli_tick() : cli_cross() , + match ? "OK" : "FAILED"); exit(match ? EXIT_SUCCESS : EXIT_FAILURE); } diff --git a/test/test_suite.bats b/test/test_suite.bats index f39c23a8..133e3905 100644 --- a/test/test_suite.bats +++ b/test/test_suite.bats @@ -11,7 +11,7 @@ @test "Check FTL binary integrity" { run bash -c './pihole-FTL verify' printf "%s\n" "${lines[@]}" - [[ "${lines[0]}" == *"SHA256 checksum matches" ]] + [[ "${lines[0]}" == *"Binary integrity check: OK" ]] } @test "Running a second instance is detected and prevented" { From dec2c8a70059589549875251f2c0932e2fbd98f1 Mon Sep 17 00:00:00 2001 From: Dominik Date: Mon, 30 Sep 2024 21:34:16 +0200 Subject: [PATCH 339/339] Review comment extending documentation Co-authored-by: yubiuser Signed-off-by: Dominik --- src/files.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/files.c b/src/files.c index 98834f19..7a1a02d5 100644 --- a/src/files.c +++ b/src/files.c @@ -812,6 +812,7 @@ bool verify_FTL(bool verbose) fclose(f); // Calculate the hash of the binary + // Skip the last 256 bit as it contains the hast itself uint8_t checksum[SHA256_DIGEST_SIZE]; if(!sha256sum(filename, checksum, true)) {