From 56a1c15b8938de1c73e94466ef4ac6fa68fef48d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sun, 3 Jul 2022 14:27:42 +0200 Subject: [PATCH 001/218] Run stale workflow at 8:00 once per day MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- .github/workflows/stale.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 4bafe052..7300cff5 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -2,7 +2,7 @@ name: Mark stale issues on: schedule: - - cron: '0 * * * *' + - cron: '0 8 * * *' workflow_dispatch: jobs: From dcc2e5d5a35d769de15da3581964eee7f45b8b4a Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 3 Jul 2022 18:36:39 +0200 Subject: [PATCH 002/218] Change default log file path to /var/log/pihole/FTL.log Signed-off-by: DL6ER --- src/config.c | 4 +- src/database/common.c | 2 +- src/database/gravity-db.c | 7 +- src/database/message-table.c | 12 ++-- test/run.sh | 12 ++-- test/test_suite.bats | 136 +++++++++++++++++------------------ 6 files changed, 86 insertions(+), 87 deletions(-) diff --git a/src/config.c b/src/config.c index 70677d22..56c4d782 100644 --- a/src/config.c +++ b/src/config.c @@ -78,7 +78,7 @@ void getLogFilePath(void) } // Read LOGFILE value if available - // defaults to: "/var/log/pihole/pihole-FTL.log" + // defaults to: "/var/log/pihole/FTL.log" buffer = parse_FTLconf(fp, "LOGFILE"); errno = 0; @@ -86,7 +86,7 @@ void getLogFilePath(void) if(buffer == NULL || sscanf(buffer, "%127ms", &FTLfiles.log) != 1) { // Use standard path if no custom path was obtained from the config file - FTLfiles.log = strdup("/var/log/pihole/pihole-FTL.log"); + FTLfiles.log = strdup("/var/log/pihole/FTL.log"); } // Test if memory allocation was successful diff --git a/src/database/common.c b/src/database/common.c index 6fbc0da9..e8bfb5a1 100644 --- a/src/database/common.c +++ b/src/database/common.c @@ -241,7 +241,7 @@ void SQLite3LogCallback(void *pArg, int iErrCode, const char *zMsg) void db_init(void) { // Initialize SQLite3 logging callback - // This ensures SQLite3 errors and warnings are logged to pihole-FTL.log + // This ensures SQLite3 errors and warnings are logged to FTL.log // We use this to possibly catch even more errors in places we do not // explicitly check for failures to have happened sqlite3_config(SQLITE_CONFIG_LOG, SQLite3LogCallback, NULL); diff --git a/src/database/gravity-db.c b/src/database/gravity-db.c index ee328e65..06551dfe 100644 --- a/src/database/gravity-db.c +++ b/src/database/gravity-db.c @@ -1021,7 +1021,7 @@ bool gravityDB_getTable(const unsigned char list) // as there are domains available. Once we reached the // end of the table, it returns NULL. It also returns // NULL when it encounters an error (e.g., on reading -// errors). Errors are logged to pihole-FTL.log +// errors). Errors are logged to FTL.log // This function is performance critical as it might // be called millions of times for large blocking lists inline const char* gravityDB_getDomain(int *rowid) @@ -1066,9 +1066,8 @@ void gravityDB_finalizeTable(void) table_stmt = NULL; } -// Get number of domains in a specified table of the gravity database -// We return the constant DB_FAILED and log to pihole-FTL.log if we -// encounter any error +// Get number of domains in a specified table of the gravity database We return +// the constant DB_FAILED and log to FTL.log if we encounter any error int gravityDB_count(const enum gravity_tables list) { if(!gravityDB_opened && !gravityDB_open()) diff --git a/src/database/message-table.c b/src/database/message-table.c index 8ec02b32..317b849a 100644 --- a/src/database/message-table.c +++ b/src/database/message-table.c @@ -304,7 +304,7 @@ void logg_regex_warning(const char *type, const char *warning, const int dbindex if(getpid() != main_pid()) return; - // Log to pihole-FTL.log + // Log to FTL.log logg("REGEX WARNING: Invalid regex %s filter \"%s\": %s", type, regex, warning); @@ -317,7 +317,7 @@ void logg_subnet_warning(const char *ip, const int matching_count, const char *m const int matching_bits, const char *chosen_match_text, const int chosen_match_id) { - // Log to pihole-FTL.log + // Log to FTL.log logg("SUBNET WARNING: Client %s is managed by %i groups (IDs %s), all describing /%i subnets. " "FTL chose the most recent entry %s (ID %i) for this client.", ip, matching_count, matching_ids, matching_bits, @@ -331,7 +331,7 @@ void logg_subnet_warning(const char *ip, const int matching_count, const char *m void logg_hostname_warning(const char *ip, const char *name, const unsigned int pos) { - // Log to pihole-FTL.log + // Log to FTL.log logg("HOSTNAME WARNING: Host name of client \"%s\" => \"%s\" contains (at least) one invalid character at position %d", ip, name, pos); @@ -341,7 +341,7 @@ void logg_hostname_warning(const char *ip, const char *name, const unsigned int void logg_fatal_dnsmasq_message(const char *message) { - // Log to pihole-FTL.log + // Log to FTL.log logg("FATAL ERROR in dnsmasq core: %s", message); // Log to database @@ -356,7 +356,7 @@ void logg_rate_limit_message(const char *clientIP, const unsigned int rate_limit { const time_t turnaround = get_rate_limit_turnaround(rate_limit_count); - // Log to pihole-FTL.log + // Log to FTL.log logg("Rate-limiting %s for at least %ld second%s", clientIP, turnaround, turnaround == 1 ? "" : "s"); @@ -366,7 +366,7 @@ void logg_rate_limit_message(const char *clientIP, const unsigned int rate_limit void logg_warn_dnsmasq_message(char *message) { - // Log to pihole-FTL.log + // Log to FTL.log logg("WARNING in dnsmasq core: %s", message); // Log to database diff --git a/test/run.sh b/test/run.sh index fbfffbb1..63d75d7f 100755 --- a/test/run.sh +++ b/test/run.sh @@ -20,14 +20,14 @@ while pidof -s pihole-FTL > /dev/null; do done # Clean up possible old files from earlier test runs -rm -f /etc/pihole/gravity.db /etc/pihole/pihole-FTL.db /var/log/pihole/pihole.log /var/log/pihole/pihole-FTL.log /dev/shm/FTL-* +rm -f /etc/pihole/gravity.db /etc/pihole/pihole-FTL.db /var/log/pihole/pihole.log /var/log/pihole/FTL.log /dev/shm/FTL-* # Create necessary directories and files mkdir -p /home/pihole /etc/pihole /run/pihole /var/log/pihole -echo "" > /var/log/pihole/pihole-FTL.log +echo "" > /var/log/pihole/FTL.log echo "" > /var/log/pihole/pihole.log touch /run/pihole-FTL.pid /run/pihole-FTL.port dig.log ptr.log -chown pihole:pihole /etc/pihole /run/pihole /var/log/pihole/pihole.log /var/log/pihole/pihole-FTL.log /run/pihole-FTL.pid /run/pihole-FTL.port +chown pihole:pihole /etc/pihole /run/pihole /var/log/pihole/pihole.log /var/log/pihole/FTL.log /run/pihole-FTL.pid /run/pihole-FTL.port # Copy binary into a location the new user pihole can access cp ./pihole-FTL /home/pihole/pihole-FTL @@ -97,11 +97,11 @@ curl_to_tricorder() { } if [[ $RET != 0 ]]; then - echo -n "pihole.log: " + echo -n "pihole/pihole.log: " curl_to_tricorder /var/log/pihole/pihole.log echo "" - echo -n "pihole-FTL.log: " - curl_to_tricorder /var/log/pihole/pihole-FTL.log + echo -n "pihole/FTL.log: " + curl_to_tricorder /var/log/pihole/FTL.log echo "" echo -n "dig.log: " curl_to_tricorder ./dig.log diff --git a/test/test_suite.bats b/test/test_suite.bats index f7680346..6a5a9d04 100644 --- a/test/test_suite.bats +++ b/test/test_suite.bats @@ -33,19 +33,19 @@ } @test "Starting tests without prior history" { - run bash -c 'grep -c "Total DNS queries: 0" /var/log/pihole/pihole-FTL.log' + run bash -c 'grep -c "Total DNS queries: 0" /var/log/pihole/FTL.log' printf "%s\n" "${lines[@]}" [[ ${lines[0]} == "1" ]] } @test "Initial blocking status is enabled" { - run bash -c 'grep -c "Blocking status is enabled" /var/log/pihole/pihole-FTL.log' + run bash -c 'grep -c "Blocking status is enabled" /var/log/pihole/FTL.log' printf "%s\n" "${lines[@]}" [[ ${lines[0]} == "1" ]] } @test "Number of compiled regex filters as expected" { - run bash -c 'grep "Compiled [0-9]* whitelist" /var/log/pihole/pihole-FTL.log' + run bash -c 'grep "Compiled [0-9]* whitelist" /var/log/pihole/FTL.log' printf "%s\n" "${lines[@]}" [[ ${lines[0]} == *"Compiled 2 whitelist and 9 blacklist regex filters"* ]] } @@ -159,31 +159,31 @@ @test "Client 4: Client is recognized by MAC address" { run bash -c "dig TXT CHAOS version.bind -b 127.0.0.4 @127.0.0.1 +short" run sleep 0.1 - run bash -c "grep -c \"Found database hardware address 127.0.0.4 -> aa:bb:cc:dd:ee:ff\" /var/log/pihole/pihole-FTL.log" + run bash -c "grep -c \"Found database hardware address 127.0.0.4 -> aa:bb:cc:dd:ee:ff\" /var/log/pihole/FTL.log" printf "%s\n" "${lines[@]}" [[ ${lines[0]} == "1" ]] - run bash -c "grep -c \"Gravity database: Client aa:bb:cc:dd:ee:ff found. Using groups (4)\" /var/log/pihole/pihole-FTL.log" + run bash -c "grep -c \"Gravity database: Client aa:bb:cc:dd:ee:ff found. Using groups (4)\" /var/log/pihole/FTL.log" printf "%s\n" "${lines[@]}" [[ ${lines[0]} != "0" ]] - run bash -c "grep -c 'Regex blacklist: Querying groups for client 127.0.0.4: \"SELECT id from vw_regex_blacklist WHERE group_id IN (4);\"' /var/log/pihole/pihole-FTL.log" + run bash -c "grep -c 'Regex blacklist: Querying groups for client 127.0.0.4: \"SELECT id from vw_regex_blacklist WHERE group_id IN (4);\"' /var/log/pihole/FTL.log" printf "%s\n" "${lines[@]}" [[ ${lines[0]} == "1" ]] - run bash -c "grep -c 'Regex whitelist: Querying groups for client 127.0.0.4: \"SELECT id from vw_regex_whitelist WHERE group_id IN (4);\"' /var/log/pihole/pihole-FTL.log" + run bash -c "grep -c 'Regex whitelist: Querying groups for client 127.0.0.4: \"SELECT id from vw_regex_whitelist WHERE group_id IN (4);\"' /var/log/pihole/FTL.log" printf "%s\n" "${lines[@]}" [[ ${lines[0]} == "1" ]] - run bash -c "grep -c 'get_client_querystr: SELECT EXISTS(SELECT domain from vw_whitelist WHERE domain = ? AND group_id IN (4));' /var/log/pihole/pihole-FTL.log" + run bash -c "grep -c 'get_client_querystr: SELECT EXISTS(SELECT domain from vw_whitelist WHERE domain = ? AND group_id IN (4));' /var/log/pihole/FTL.log" printf "%s\n" "${lines[@]}" [[ ${lines[0]} != "0" ]] - run bash -c "grep -c 'get_client_querystr: SELECT EXISTS(SELECT domain from vw_blacklist WHERE domain = ? AND group_id IN (4));' /var/log/pihole/pihole-FTL.log" + run bash -c "grep -c 'get_client_querystr: SELECT EXISTS(SELECT domain from vw_blacklist WHERE domain = ? AND group_id IN (4));' /var/log/pihole/FTL.log" printf "%s\n" "${lines[@]}" [[ ${lines[0]} != "0" ]] - run bash -c "grep -c 'get_client_querystr: SELECT EXISTS(SELECT domain from vw_gravity WHERE domain = ? AND group_id IN (4));' /var/log/pihole/pihole-FTL.log" + run bash -c "grep -c 'get_client_querystr: SELECT EXISTS(SELECT domain from vw_gravity WHERE domain = ? AND group_id IN (4));' /var/log/pihole/FTL.log" printf "%s\n" "${lines[@]}" [[ ${lines[0]} != "0" ]] - run bash -c "grep -c 'Regex whitelist ([[:digit:]]*, DB ID [[:digit:]]*) .* NOT ENABLED for client 127.0.0.4' /var/log/pihole/pihole-FTL.log" + run bash -c "grep -c 'Regex whitelist ([[:digit:]]*, DB ID [[:digit:]]*) .* NOT ENABLED for client 127.0.0.4' /var/log/pihole/FTL.log" printf "%s\n" "${lines[@]}" [[ ${lines[0]} == "2" ]] - run bash -c "grep -c 'Regex blacklist ([[:digit:]]*, DB ID [[:digit:]]*) .* NOT ENABLED for client 127.0.0.4' /var/log/pihole/pihole-FTL.log" + run bash -c "grep -c 'Regex blacklist ([[:digit:]]*, DB ID [[:digit:]]*) .* NOT ENABLED for client 127.0.0.4' /var/log/pihole/FTL.log" printf "%s\n" "${lines[@]}" [[ ${lines[0]} == "9" ]] } @@ -191,31 +191,31 @@ @test "Client 5: Client is recognized by MAC address" { run bash -c "dig TXT CHAOS version.bind -b 127.0.0.5 @127.0.0.1 +short" run sleep 0.1 - run bash -c "grep -c \"Found database hardware address 127.0.0.5 -> aa:bb:cc:dd:ee:ff\" /var/log/pihole/pihole-FTL.log" + run bash -c "grep -c \"Found database hardware address 127.0.0.5 -> aa:bb:cc:dd:ee:ff\" /var/log/pihole/FTL.log" printf "%s\n" "${lines[@]}" [[ ${lines[0]} == "1" ]] - run bash -c "grep -c \"Gravity database: Client aa:bb:cc:dd:ee:ff found. Using groups (4)\" /var/log/pihole/pihole-FTL.log" + run bash -c "grep -c \"Gravity database: Client aa:bb:cc:dd:ee:ff found. Using groups (4)\" /var/log/pihole/FTL.log" printf "%s\n" "${lines[@]}" [[ ${lines[0]} != "0" ]] - run bash -c "grep -c 'Regex blacklist: Querying groups for client 127.0.0.5: \"SELECT id from vw_regex_blacklist WHERE group_id IN (4);\"' /var/log/pihole/pihole-FTL.log" + run bash -c "grep -c 'Regex blacklist: Querying groups for client 127.0.0.5: \"SELECT id from vw_regex_blacklist WHERE group_id IN (4);\"' /var/log/pihole/FTL.log" printf "%s\n" "${lines[@]}" [[ ${lines[0]} == "1" ]] - run bash -c "grep -c 'Regex whitelist: Querying groups for client 127.0.0.5: \"SELECT id from vw_regex_whitelist WHERE group_id IN (4);\"' /var/log/pihole/pihole-FTL.log" + run bash -c "grep -c 'Regex whitelist: Querying groups for client 127.0.0.5: \"SELECT id from vw_regex_whitelist WHERE group_id IN (4);\"' /var/log/pihole/FTL.log" printf "%s\n" "${lines[@]}" [[ ${lines[0]} == "1" ]] - run bash -c "grep -c 'get_client_querystr: SELECT EXISTS(SELECT domain from vw_whitelist WHERE domain = ? AND group_id IN (4));' /var/log/pihole/pihole-FTL.log" + run bash -c "grep -c 'get_client_querystr: SELECT EXISTS(SELECT domain from vw_whitelist WHERE domain = ? AND group_id IN (4));' /var/log/pihole/FTL.log" printf "%s\n" "${lines[@]}" [[ ${lines[0]} != "0" ]] - run bash -c "grep -c 'get_client_querystr: SELECT EXISTS(SELECT domain from vw_blacklist WHERE domain = ? AND group_id IN (4));' /var/log/pihole/pihole-FTL.log" + run bash -c "grep -c 'get_client_querystr: SELECT EXISTS(SELECT domain from vw_blacklist WHERE domain = ? AND group_id IN (4));' /var/log/pihole/FTL.log" printf "%s\n" "${lines[@]}" [[ ${lines[0]} != "0" ]] - run bash -c "grep -c 'get_client_querystr: SELECT EXISTS(SELECT domain from vw_gravity WHERE domain = ? AND group_id IN (4));' /var/log/pihole/pihole-FTL.log" + run bash -c "grep -c 'get_client_querystr: SELECT EXISTS(SELECT domain from vw_gravity WHERE domain = ? AND group_id IN (4));' /var/log/pihole/FTL.log" printf "%s\n" "${lines[@]}" [[ ${lines[0]} != "0" ]] - run bash -c "grep -c 'Regex whitelist ([[:digit:]]*, DB ID [[:digit:]]*) .* NOT ENABLED for client 127.0.0.5' /var/log/pihole/pihole-FTL.log" + run bash -c "grep -c 'Regex whitelist ([[:digit:]]*, DB ID [[:digit:]]*) .* NOT ENABLED for client 127.0.0.5' /var/log/pihole/FTL.log" printf "%s\n" "${lines[@]}" [[ ${lines[0]} == "2" ]] - run bash -c "grep -c 'Regex blacklist ([[:digit:]]*, DB ID [[:digit:]]*) .* NOT ENABLED for client 127.0.0.5' /var/log/pihole/pihole-FTL.log" + run bash -c "grep -c 'Regex blacklist ([[:digit:]]*, DB ID [[:digit:]]*) .* NOT ENABLED for client 127.0.0.5' /var/log/pihole/FTL.log" printf "%s\n" "${lines[@]}" [[ ${lines[0]} == "9" ]] } @@ -223,37 +223,37 @@ @test "Client 6: Client is recognized by interface name" { run bash -c "dig TXT CHAOS version.bind -b 127.0.0.6 @127.0.0.1 +short" run sleep 0.1 - run bash -c "grep -c \"Found database hardware address 127.0.0.6 -> 00:11:22:33:44:55\" /var/log/pihole/pihole-FTL.log" + run bash -c "grep -c \"Found database hardware address 127.0.0.6 -> 00:11:22:33:44:55\" /var/log/pihole/FTL.log" printf "%s\n" "${lines[@]}" [[ ${lines[0]} == "1" ]] - run bash -c "grep -c \"There is no record for 00:11:22:33:44:55 in the client table\" /var/log/pihole/pihole-FTL.log" + run bash -c "grep -c \"There is no record for 00:11:22:33:44:55 in the client table\" /var/log/pihole/FTL.log" printf "%s\n" "${lines[@]}" [[ ${lines[0]} == "1" ]] - run bash -c "grep -c \"Found database interface 127.0.0.6 -> enp0s123\" /var/log/pihole/pihole-FTL.log" + run bash -c "grep -c \"Found database interface 127.0.0.6 -> enp0s123\" /var/log/pihole/FTL.log" printf "%s\n" "${lines[@]}" [[ ${lines[0]} == "1" ]] - run bash -c "grep -c \"Gravity database: Client 00:11:22:33:44:55 found (identified by interface enp0s123). Using groups (5)\" /var/log/pihole/pihole-FTL.log" + run bash -c "grep -c \"Gravity database: Client 00:11:22:33:44:55 found (identified by interface enp0s123). Using groups (5)\" /var/log/pihole/FTL.log" printf "%s\n" "${lines[@]}" [[ ${lines[0]} == "1" ]] - run bash -c "grep -c 'Regex blacklist: Querying groups for client 127.0.0.6: \"SELECT id from vw_regex_blacklist WHERE group_id IN (5);\"' /var/log/pihole/pihole-FTL.log" + run bash -c "grep -c 'Regex blacklist: Querying groups for client 127.0.0.6: \"SELECT id from vw_regex_blacklist WHERE group_id IN (5);\"' /var/log/pihole/FTL.log" printf "%s\n" "${lines[@]}" [[ ${lines[0]} == "1" ]] - run bash -c "grep -c 'Regex whitelist: Querying groups for client 127.0.0.6: \"SELECT id from vw_regex_whitelist WHERE group_id IN (5);\"' /var/log/pihole/pihole-FTL.log" + run bash -c "grep -c 'Regex whitelist: Querying groups for client 127.0.0.6: \"SELECT id from vw_regex_whitelist WHERE group_id IN (5);\"' /var/log/pihole/FTL.log" printf "%s\n" "${lines[@]}" [[ ${lines[0]} == "1" ]] - run bash -c "grep -c 'get_client_querystr: SELECT EXISTS(SELECT domain from vw_whitelist WHERE domain = ? AND group_id IN (5));' /var/log/pihole/pihole-FTL.log" + run bash -c "grep -c 'get_client_querystr: SELECT EXISTS(SELECT domain from vw_whitelist WHERE domain = ? AND group_id IN (5));' /var/log/pihole/FTL.log" printf "%s\n" "${lines[@]}" [[ ${lines[0]} == "1" ]] - run bash -c "grep -c 'get_client_querystr: SELECT EXISTS(SELECT domain from vw_blacklist WHERE domain = ? AND group_id IN (5));' /var/log/pihole/pihole-FTL.log" + run bash -c "grep -c 'get_client_querystr: SELECT EXISTS(SELECT domain from vw_blacklist WHERE domain = ? AND group_id IN (5));' /var/log/pihole/FTL.log" printf "%s\n" "${lines[@]}" [[ ${lines[0]} == "1" ]] - run bash -c "grep -c 'get_client_querystr: SELECT EXISTS(SELECT domain from vw_gravity WHERE domain = ? AND group_id IN (5));' /var/log/pihole/pihole-FTL.log" + run bash -c "grep -c 'get_client_querystr: SELECT EXISTS(SELECT domain from vw_gravity WHERE domain = ? AND group_id IN (5));' /var/log/pihole/FTL.log" printf "%s\n" "${lines[@]}" [[ ${lines[0]} == "1" ]] - run bash -c "grep -c 'Regex whitelist ([[:digit:]]*, DB ID [[:digit:]]*) .* NOT ENABLED for client 127.0.0.6' /var/log/pihole/pihole-FTL.log" + run bash -c "grep -c 'Regex whitelist ([[:digit:]]*, DB ID [[:digit:]]*) .* NOT ENABLED for client 127.0.0.6' /var/log/pihole/FTL.log" printf "%s\n" "${lines[@]}" [[ ${lines[0]} == "2" ]] - run bash -c "grep -c 'Regex blacklist ([[:digit:]]*, DB ID [[:digit:]]*) .* NOT ENABLED for client 127.0.0.6' /var/log/pihole/pihole-FTL.log" + run bash -c "grep -c 'Regex blacklist ([[:digit:]]*, DB ID [[:digit:]]*) .* NOT ENABLED for client 127.0.0.6' /var/log/pihole/FTL.log" printf "%s\n" "${lines[@]}" [[ ${lines[0]} == "9" ]] } @@ -652,48 +652,48 @@ [[ ${lines[3]} == "Available arguments:" ]] } -@test "No WARNING messages in pihole-FTL.log (besides known capability issues)" { - run bash -c 'grep "WARNING" /var/log/pihole/pihole-FTL.log' +@test "No WARNING messages in FTL.log (besides known capability issues)" { + run bash -c 'grep "WARNING" /var/log/pihole/FTL.log' printf "%s\n" "${lines[@]}" - run bash -c 'grep "WARNING" /var/log/pihole/pihole-FTL.log | grep -c -v -E "CAP_NET_ADMIN|CAP_NET_RAW|CAP_SYS_NICE|CAP_IPC_LOCK|CAP_CHOWN"' + run bash -c 'grep "WARNING" /var/log/pihole/FTL.log | grep -c -v -E "CAP_NET_ADMIN|CAP_NET_RAW|CAP_SYS_NICE|CAP_IPC_LOCK|CAP_CHOWN"' printf "%s\n" "${lines[@]}" [[ ${lines[0]} == "0" ]] } -@test "No \"database not available\" messages in pihole-FTL.log" { - run bash -c 'grep -c "database not available" /var/log/pihole/pihole-FTL.log' +@test "No \"database not available\" messages in FTL.log" { + run bash -c 'grep -c "database not available" /var/log/pihole/FTL.log' printf "%s\n" "${lines[@]}" [[ ${lines[0]} == "0" ]] } -@test "No ERROR messages in pihole-FTL.log" { - run bash -c 'grep "ERROR" /var/log/pihole/pihole-FTL.log' +@test "No ERROR messages in FTL.log" { + run bash -c 'grep "ERROR" /var/log/pihole/FTL.log' printf "%s\n" "${lines[@]}" - run bash -c 'grep -c "ERROR" /var/log/pihole/pihole-FTL.log' + run bash -c 'grep -c "ERROR" /var/log/pihole/FTL.log' printf "%s\n" "${lines[@]}" [[ ${lines[0]} == "0" ]] } -@test "No FATAL messages in pihole-FTL.log (besides error due to starting FTL more than once)" { - run bash -c 'grep "FATAL" /var/log/pihole/pihole-FTL.log' +@test "No FATAL messages in FTL.log (besides error due to starting FTL more than once)" { + run bash -c 'grep "FATAL" /var/log/pihole/FTL.log' printf "%s\n" "${lines[@]}" - run bash -c 'grep "FATAL:" /var/log/pihole/pihole-FTL.log | grep -c -v "FATAL: create_shm(): Failed to create shared memory object \"FTL-lock\": File exists"' + run bash -c 'grep "FATAL:" /var/log/pihole/FTL.log | grep -c -v "FATAL: create_shm(): Failed to create shared memory object \"FTL-lock\": File exists"' printf "%s\n" "${lines[@]}" [[ ${lines[0]} == "0" ]] } # Regex tests @test "Compiled blacklist regex as expected" { - run bash -c 'grep -c "Compiling blacklist regex 0 (DB ID 6): regex\[0-9\].ftl" /var/log/pihole/pihole-FTL.log' + run bash -c 'grep -c "Compiling blacklist regex 0 (DB ID 6): regex\[0-9\].ftl" /var/log/pihole/FTL.log' printf "%s\n" "${lines[@]}" [[ ${lines[0]} == "1" ]] } @test "Compiled whitelist regex as expected" { - run bash -c 'grep -c "Compiling whitelist regex 0 (DB ID 3): regex2" /var/log/pihole/pihole-FTL.log' + run bash -c 'grep -c "Compiling whitelist regex 0 (DB ID 3): regex2" /var/log/pihole/FTL.log' printf "%s\n" "${lines[@]}" [[ ${lines[0]} == "1" ]] - run bash -c 'grep -c "Compiling whitelist regex 1 (DB ID 4): ^gravity-whitelisted" /var/log/pihole/pihole-FTL.log' + run bash -c 'grep -c "Compiling whitelist regex 1 (DB ID 4): ^gravity-whitelisted" /var/log/pihole/FTL.log' printf "%s\n" "${lines[@]}" [[ ${lines[0]} == "1" ]] } @@ -1078,21 +1078,21 @@ } @test "Architecture is correctly reported on startup" { - run bash -c 'grep "Compiled for" /var/log/pihole/pihole-FTL.log' + run bash -c 'grep "Compiled for" /var/log/pihole/FTL.log' printf "Output: %s\n\$CI_ARCH: %s\nuname -m: %s\n" "${lines[@]:-not set}" "${CI_ARCH:-not set}" "$(uname -m)" [[ ${lines[0]} == *"Compiled for ${CI_ARCH:-$(uname -m)}"* ]] } @test "Building machine (CI) is reported on startup" { [[ ${CI_ARCH} != "" ]] && compiled_str="on CI" || compiled_str="locally" && export compiled_str - run bash -c 'grep "Compiled for" /var/log/pihole/pihole-FTL.log' + run bash -c 'grep "Compiled for" /var/log/pihole/FTL.log' printf "Output: %s\n\$CI_ARCH: %s\n" "${lines[@]:-not set}" "${CI_ARCH:-not set}" [[ ${lines[0]} == *"(compiled ${compiled_str})"* ]] } @test "Compiler version is correctly reported on startup" { compiler_version="$(${CC} --version | head -n1)" && export compiler_version - run bash -c 'grep "Compiled for" /var/log/pihole/pihole-FTL.log' + run bash -c 'grep "Compiled for" /var/log/pihole/FTL.log' printf "Output: %s\n\$CC: %s\nVersion: %s\n" "${lines[@]:-not set}" "${CC:-not set}" "${compiler_version:-not set}" [[ ${lines[0]} == *"using ${compiler_version}"* ]] } @@ -1104,7 +1104,7 @@ } @test "No errors on setting busy handlers for the databases" { - run bash -c 'grep -c "Cannot set busy handler" /var/log/pihole/pihole-FTL.log' + run bash -c 'grep -c "Cannot set busy handler" /var/log/pihole/FTL.log' printf "%s\n" "${lines[@]}" [[ ${lines[0]} == "0" ]] } @@ -1136,7 +1136,7 @@ @test "EDNS(0) analysis working as expected" { # Get number of lines in the log before the test - before="$(grep -c ^ /var/log/pihole/pihole-FTL.log)" + before="$(grep -c ^ /var/log/pihole/FTL.log)" # Run test command # CLIENT SUBNET COOKIE MAC HEX MAC TEXT CPE-ID @@ -1146,10 +1146,10 @@ [[ $status == 0 ]] # Get number of lines in the log after the test - after="$(grep -c ^ /var/log/pihole/pihole-FTL.log)" + after="$(grep -c ^ /var/log/pihole/FTL.log)" # Extract relevant log lines - log="$(sed -n "${before},${after}p" /var/log/pihole/pihole-FTL.log)" + log="$(sed -n "${before},${after}p" /var/log/pihole/FTL.log)" printf "%s\n" "${log}" # Start actual test @@ -1172,7 +1172,7 @@ @test "EDNS(0) ECS can overwrite client address (IPv4)" { # Get number of lines in the log before the test - before="$(grep -c ^ /var/log/pihole/pihole-FTL.log)" + before="$(grep -c ^ /var/log/pihole/FTL.log)" # Run test command run bash -c 'dig localhost +short +subnet=192.168.47.97/32 @127.0.0.1' @@ -1181,17 +1181,17 @@ [[ $status == 0 ]] # Get number of lines in the log after the test - after="$(grep -c ^ /var/log/pihole/pihole-FTL.log)" + after="$(grep -c ^ /var/log/pihole/FTL.log)" # Extract relevant log lines - run bash -c "sed -n \"${before},${after}p\" /var/log/pihole/pihole-FTL.log" + run bash -c "sed -n \"${before},${after}p\" /var/log/pihole/FTL.log" printf "%s\n" "${lines[@]}" [[ "${lines[@]}" == *"**** new UDP IPv4 query[A] query \"localhost\" from lo/192.168.47.97#53 "* ]] } @test "EDNS(0) ECS can overwrite client address (IPv6)" { # Get number of lines in the log before the test - before="$(grep -c ^ /var/log/pihole/pihole-FTL.log)" + before="$(grep -c ^ /var/log/pihole/FTL.log)" # Run test command run bash -c 'dig localhost +short +subnet=fe80::b167:af1e:968b:dead/128 @127.0.0.1' @@ -1200,29 +1200,29 @@ [[ $status == 0 ]] # Get number of lines in the log after the test - after="$(grep -c ^ /var/log/pihole/pihole-FTL.log)" + after="$(grep -c ^ /var/log/pihole/FTL.log)" # Extract relevant log lines - run bash -c "sed -n \"${before},${after}p\" /var/log/pihole/pihole-FTL.log" + run bash -c "sed -n \"${before},${after}p\" /var/log/pihole/FTL.log" printf "%s\n" "${lines[@]}" [[ "${lines[@]}" == *"**** new UDP IPv4 query[A] query \"localhost\" from lo/fe80::b167:af1e:968b:dead#53 "* ]] } @test "alias-client is imported and used for configured client" { - run bash -c 'grep -c "Added alias-client \"some-aliasclient\" (aliasclient-0) with FTL ID 0" /var/log/pihole/pihole-FTL.log' + run bash -c 'grep -c "Added alias-client \"some-aliasclient\" (aliasclient-0) with FTL ID 0" /var/log/pihole/FTL.log' printf "Added: %s\n" "${lines[@]}" [[ ${lines[0]} == "1" ]] - run bash -c 'grep -c "Aliasclient ID 127.0.0.6 -> 0" /var/log/pihole/pihole-FTL.log' + run bash -c 'grep -c "Aliasclient ID 127.0.0.6 -> 0" /var/log/pihole/FTL.log' printf "Found ID: %s\n" "${lines[@]}" [[ ${lines[0]} == "1" ]] - run bash -c 'grep -c "Client .* (127.0.0.6) IS managed by this alias-client, adding counts" /var/log/pihole/pihole-FTL.log' + run bash -c 'grep -c "Client .* (127.0.0.6) IS managed by this alias-client, adding counts" /var/log/pihole/FTL.log' printf "Adding counts: %s\n" "${lines[@]}" [[ ${lines[0]} == "1" ]] } @test "EDNS(0) ECS skipped for loopback address (IPv4)" { # Get number of lines in the log before the test - before="$(grep -c ^ /var/log/pihole/pihole-FTL.log)" + before="$(grep -c ^ /var/log/pihole/FTL.log)" # Run test command run bash -c 'dig localhost +short +subnet=127.0.0.1/32 @127.0.0.1' @@ -1231,17 +1231,17 @@ [[ $status == 0 ]] # Get number of lines in the log after the test - after="$(grep -c ^ /var/log/pihole/pihole-FTL.log)" + after="$(grep -c ^ /var/log/pihole/FTL.log)" # Extract relevant log lines - run bash -c "sed -n \"${before},${after}p\" /var/log/pihole/pihole-FTL.log" + run bash -c "sed -n \"${before},${after}p\" /var/log/pihole/FTL.log" printf "%s\n" "${lines[@]}" [[ "${lines[@]}" == *"EDNS(0) CLIENT SUBNET: Skipped 127.0.0.1/32 (IPv4 loopback address)"* ]] } @test "EDNS(0) ECS skipped for loopback address (IPv6)" { # Get number of lines in the log before the test - before="$(grep -c ^ /var/log/pihole/pihole-FTL.log)" + before="$(grep -c ^ /var/log/pihole/FTL.log)" # Run test command run bash -c 'dig localhost +short +subnet=::1/128 @127.0.0.1' @@ -1250,10 +1250,10 @@ [[ $status == 0 ]] # Get number of lines in the log after the test - after="$(grep -c ^ /var/log/pihole/pihole-FTL.log)" + after="$(grep -c ^ /var/log/pihole/FTL.log)" # Extract relevant log lines - run bash -c "sed -n \"${before},${after}p\" /var/log/pihole/pihole-FTL.log" + run bash -c "sed -n \"${before},${after}p\" /var/log/pihole/FTL.log" printf "%s\n" "${lines[@]}" [[ "${lines[@]}" == *"EDNS(0) CLIENT SUBNET: Skipped ::1/128 (IPv6 loopback address)"* ]] } From 390d456b71152b1abce339f8396312708f2c2869 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Tue, 5 Jul 2022 10:21:55 +0200 Subject: [PATCH 003/218] Fix close-issue-reason MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- .github/workflows/stale.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 7300cff5..81f3a97b 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -23,4 +23,4 @@ jobs: exempt-issue-labels: 'Fixed in next release, Bug, Bug:Confirmed, Bugfix in progress, documentation needed, internal' exempt-all-issue-assignees: true operations-per-run: 300 - stale-issue-reason: 'not_planned' + close-issue-reason: 'not_planned' From 37e605201bf76c9143171770c8d7dfd0ae367697 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Fri, 8 Jul 2022 00:23:45 +0200 Subject: [PATCH 004/218] Log new telnet connections only when debug flag is set MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- src/api/socket.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/api/socket.c b/src/api/socket.c index 6dd9907e..f46948a0 100644 --- a/src/api/socket.c +++ b/src/api/socket.c @@ -502,7 +502,8 @@ void *telnet_listening_thread_IPv4(void *args) continue; } - logg("Accepting new telnet connection at socket %d", csck); + if(config.debug & DEBUG_API) + logg("Accepting new telnet connection at socket %d", csck); // Allocate memory used to transport client socket ID to client listening thread int *newsock; From 54fc5b6b4237ff8e72e2ad46d8bf03865d6a43a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Fri, 8 Jul 2022 07:56:09 +0200 Subject: [PATCH 005/218] Apply to terminating as well MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- src/api/socket.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/api/socket.c b/src/api/socket.c index f46948a0..e04dfd58 100644 --- a/src/api/socket.c +++ b/src/api/socket.c @@ -519,8 +519,8 @@ void *telnet_listening_thread_IPv4(void *args) logg("WARNING: Unable to open telnet processing thread: %s", strerror(errno)); } } - - logg("Terminating IPv4 telnet thread"); + if(config.debug & DEBUG_API) + logg("Terminating IPv4 telnet thread"); return NULL; } From 00dafbfae6110c83301cad19f8dfc9d298ddcb10 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 8 Jul 2022 22:24:48 +0200 Subject: [PATCH 006/218] Fix status of blocked special domains when they are in FTL's DNS cache Signed-off-by: DL6ER --- src/dnsmasq_interface.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c index 0730c40b..41c122d1 100644 --- a/src/dnsmasq_interface.c +++ b/src/dnsmasq_interface.c @@ -1361,7 +1361,7 @@ static bool _FTL_check_blocking(int queryID, int domainID, int clientID, const c } force_next_DNS_reply = dns_cache->force_reply; - query_blocked(query, domain, client, QUERY_CACHE); + query_blocked(query, domain, client, QUERY_SPECIAL_DOMAIN); return true; break; From 43917f017d0106c0dc2900bc61b9ecb5046d3371 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 10 Jul 2022 15:01:24 +0200 Subject: [PATCH 007/218] Revert "Remove PID and PORT file during shutdown" --- src/api/socket.c | 12 +----------- src/daemon.c | 11 ----------- 2 files changed, 1 insertion(+), 22 deletions(-) diff --git a/src/api/socket.c b/src/api/socket.c index e04dfd58..6495aa14 100644 --- a/src/api/socket.c +++ b/src/api/socket.c @@ -54,18 +54,8 @@ void saveport(int port) } else { - // FTL is terminating: Truncate file and remove it (if possible) + // FTL is terminating: Leave file truncated fclose(f); - // We also try to remove the file. We still empty the file above - // to ensure it is at least empty when it cannot be removed. - // because removing files on Linux is actually unlinking them. - // If any processes still have the file open, it will remain - // in existence until the last file descriptor referring to - // it is closed. - if(remove(FTLfiles.port) != 0) - { - logg("WARNING: Unable to remove PORT file: %s", strerror(errno)); - } } } diff --git a/src/daemon.c b/src/daemon.c index 14c8f34b..a1b26012 100644 --- a/src/daemon.c +++ b/src/daemon.c @@ -124,17 +124,6 @@ static void removepid(void) return; } fclose(f); - - // We also try to remove the file. We still empty the file above - // to ensure it is at least empty when it cannot be removed. - // because removing files on Linux is actually unlinking them. - // If any processes still have the file open, it will remain - // in existence until the last file descriptor referring to - // it is closed. - if(remove(FTLfiles.pid) != 0) - { - logg("WARNING: Unable to remove PID file: %s", strerror(errno)); - } } char *getUserName(void) From fa02dcc9247fd894254786d39a43fe37ecc1c118 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 10 Jul 2022 22:01:57 +0200 Subject: [PATCH 008/218] Change DELAY_UPTIME to 180 seconds Signed-off-by: DL6ER --- src/FTL.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/FTL.h b/src/FTL.h index 593aef00..4c819e19 100644 --- a/src/FTL.h +++ b/src/FTL.h @@ -115,8 +115,8 @@ // DELAY_STARTUP should only delay the startup of the resolver during a starting up system // This setting control how long after boot we consider a system to be in starting-up mode -// Default: 60 [seconds] -#define DELAY_UPTIME 60 +// Default: 180 [seconds] +#define DELAY_UPTIME 180 // Use out own syscalls handling functions that will detect possible errors // and report accordingly in the log. This will make debugging FTL crash From 902fc2154aed1edc5eaf00ea6d2736ee2b01a2cf Mon Sep 17 00:00:00 2001 From: Marc Date: Fri, 15 Jul 2022 10:44:52 +0200 Subject: [PATCH 009/218] Change configuration option for REPLY_WHEN_BUSY ACCEPT > ALLOW Signed-off-by: Marc --- src/config.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config.c b/src/config.c index 56c4d782..798b7ddb 100644 --- a/src/config.c +++ b/src/config.c @@ -691,7 +691,7 @@ void read_FTLconf(void) // defaults to: DROP buffer = parse_FTLconf(fp, "REPLY_WHEN_BUSY"); - if(buffer != NULL && strcasecmp(buffer, "ACCEPT") == 0) + if(buffer != NULL && strcasecmp(buffer, "ALLOW") == 0) { config.reply_when_busy = BUSY_ALLOW; logg(" REPLY_WHEN_BUSY: Permit queries when the database is busy"); From a65e3961ffb4e4c8fed8702e98512efc4f5a5059 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sun, 24 Jul 2022 22:14:24 +0200 Subject: [PATCH 010/218] Explicit set minor and patch version on used actions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- .github/workflows/build.yml | 10 +++++----- .github/workflows/codespell.yml | 2 +- .github/workflows/stale.yml | 2 +- .github/workflows/sync-back-to-dev.yml | 4 ++-- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d3c909df..ee132c76 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -57,7 +57,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v3.0.0 - name: Calculate required build variables id: variables @@ -92,7 +92,7 @@ jobs: - name: Upload artifacts to job for later processing if: github.event_name != 'pull_request' - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v3.0.0 with: name: tmp-binary-storage path: '${{ matrix.bin_name }}*' @@ -104,7 +104,7 @@ jobs: steps: - name: Get Binaries built in previous jobs - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v3.0.0 with: name: tmp-binary-storage - @@ -112,7 +112,7 @@ jobs: run: ls -R - name: Install SSH Key - uses: shimataro/ssh-key-action@v2 + uses: shimataro/ssh-key-action@v2.0.0 with: key: ${{ secrets.SSH_KEY }} known_hosts: ${{ secrets.KNOWN_HOSTS }} @@ -124,7 +124,7 @@ jobs: - name: Attach binaries to release if: github.event_name == 'release' - uses: softprops/action-gh-release@v1 + uses: softprops/action-gh-release@v1.0.0 with: files: | * diff --git a/.github/workflows/codespell.yml b/.github/workflows/codespell.yml index 8ee2a92d..31e4109c 100644 --- a/.github/workflows/codespell.yml +++ b/.github/workflows/codespell.yml @@ -10,7 +10,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v3.0.0 - name: Spell-Checking uses: codespell-project/actions-codespell@master diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 81f3a97b..a024688a 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -13,7 +13,7 @@ jobs: issues: write steps: - - uses: actions/stale@v5 + - uses: actions/stale@v5.0.0 with: repo-token: ${{ secrets.GITHUB_TOKEN }} days-before-stale: 30 diff --git a/.github/workflows/sync-back-to-dev.yml b/.github/workflows/sync-back-to-dev.yml index f79b69f2..30b1a970 100644 --- a/.github/workflows/sync-back-to-dev.yml +++ b/.github/workflows/sync-back-to-dev.yml @@ -11,7 +11,7 @@ jobs: name: Syncing branches steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v3.0.0 - name: Opening pull request id: pull uses: tretuna/sync-branches@1.4.0 @@ -20,7 +20,7 @@ jobs: FROM_BRANCH: 'master' TO_BRANCH: 'development' - name: Label the pull request to ignore for release note generation - uses: actions-ecosystem/action-add-labels@v1 + uses: actions-ecosystem/action-add-labels@v1.0.0 with: labels: internal repo: ${{ github.repository }} From bb44d6e20bc33f2523fbd12af51654e1b254cdc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sun, 24 Jul 2022 22:23:40 +0200 Subject: [PATCH 011/218] action-gh-release uses chaotic version tagging MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ee132c76..f406a4fb 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -124,7 +124,7 @@ jobs: - name: Attach binaries to release if: github.event_name == 'release' - uses: softprops/action-gh-release@v1.0.0 + uses: softprops/action-gh-release@v1 with: files: | * From 8b6814895a31776f5a0b8497b47c6b61914a8e87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sun, 24 Jul 2022 22:39:06 +0200 Subject: [PATCH 012/218] Update ssh-key-action to current version MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f406a4fb..787f3e5a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -112,7 +112,7 @@ jobs: run: ls -R - name: Install SSH Key - uses: shimataro/ssh-key-action@v2.0.0 + uses: shimataro/ssh-key-action@v2.3.1 with: key: ${{ secrets.SSH_KEY }} known_hosts: ${{ secrets.KNOWN_HOSTS }} From e9d4de40f6d66e744509c0bcca33772a96bba2fa Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 25 Jul 2022 15:21:59 +0200 Subject: [PATCH 013/218] Update SQLite3 from 3.39.0 to 3.39.2 Signed-off-by: DL6ER --- src/database/shell.c | 4 +- src/database/sqlite3.c | 246 +++++++++++++++++++++++++---------------- src/database/sqlite3.h | 8 +- 3 files changed, 157 insertions(+), 101 deletions(-) diff --git a/src/database/shell.c b/src/database/shell.c index 00266127..6b40debc 100644 --- a/src/database/shell.c +++ b/src/database/shell.c @@ -4411,7 +4411,7 @@ static const char *re_compile(ReCompiled **ppRe, const char *zIn, int noCase){ pRe->zInit[j++] = (unsigned char)(0xc0 | (x>>6)); pRe->zInit[j++] = 0x80 | (x&0x3f); }else if( x<=0xffff ){ - pRe->zInit[j++] = (unsigned char)(0xd0 | (x>>12)); + pRe->zInit[j++] = (unsigned char)(0xe0 | (x>>12)); pRe->zInit[j++] = 0x80 | ((x>>6)&0x3f); pRe->zInit[j++] = 0x80 | (x&0x3f); }else{ @@ -23080,7 +23080,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ char **argv; #endif #ifdef SQLITE_DEBUG - sqlite3_uint64 mem_main_enter = sqlite3_memory_used(); + sqlite3_int64 mem_main_enter = sqlite3_memory_used(); #endif char *zErrMsg = 0; #ifdef SQLITE_SHELL_WASM_MODE diff --git a/src/database/sqlite3.c b/src/database/sqlite3.c index 6a7b52d9..b8f98c7c 100644 --- a/src/database/sqlite3.c +++ b/src/database/sqlite3.c @@ -1,6 +1,6 @@ /****************************************************************************** ** This file is an amalgamation of many separate C source files from SQLite -** version 3.39.0. By combining all the individual C code files into this +** version 3.39.2. By combining all the individual C code files into this ** single large file, the entire code can be compiled as a single translation ** unit. This allows many compilers to do optimizations that would not be ** possible if the files were compiled separately. Performance improvements @@ -452,9 +452,9 @@ extern "C" { ** [sqlite3_libversion_number()], [sqlite3_sourceid()], ** [sqlite_version()] and [sqlite_source_id()]. */ -#define SQLITE_VERSION "3.39.0" -#define SQLITE_VERSION_NUMBER 3039000 -#define SQLITE_SOURCE_ID "2022-06-25 14:57:57 14e166f40dbfa6e055543f8301525f2ca2e96a02a57269818b9e69e162e98918" +#define SQLITE_VERSION "3.39.2" +#define SQLITE_VERSION_NUMBER 3039002 +#define SQLITE_SOURCE_ID "2022-07-21 15:24:47 698edb77537b67c41adc68f9b892db56bcf9a55e00371a61420f3ddd668e6603" /* ** CAPI3REF: Run-Time Library Version Numbers @@ -6588,7 +6588,7 @@ SQLITE_API sqlite3 *sqlite3_db_handle(sqlite3_stmt*); ** ** ^The sqlite3_db_name(D,N) interface returns a pointer to the schema name ** for the N-th database on database connection D, or a NULL pointer of N is -** out of range. An N alue of 0 means the main database file. An N of 1 is +** out of range. An N value of 0 means the main database file. An N of 1 is ** the "temp" schema. Larger values of N correspond to various ATTACH-ed ** databases. ** @@ -19779,6 +19779,7 @@ SQLITE_PRIVATE void sqlite3TreeViewSrcList(TreeView*, const SrcList*); SQLITE_PRIVATE void sqlite3TreeViewSelect(TreeView*, const Select*, u8); SQLITE_PRIVATE void sqlite3TreeViewWith(TreeView*, const With*, u8); SQLITE_PRIVATE void sqlite3TreeViewUpsert(TreeView*, const Upsert*, u8); +#if TREETRACE_ENABLED SQLITE_PRIVATE void sqlite3TreeViewDelete(const With*, const SrcList*, const Expr*, const ExprList*,const Expr*, const Trigger*); SQLITE_PRIVATE void sqlite3TreeViewInsert(const With*, const SrcList*, @@ -19787,6 +19788,7 @@ SQLITE_PRIVATE void sqlite3TreeViewInsert(const With*, const SrcList*, SQLITE_PRIVATE void sqlite3TreeViewUpdate(const With*, const SrcList*, const ExprList*, const Expr*, int, const ExprList*, const Expr*, const Upsert*, const Trigger*); +#endif #ifndef SQLITE_OMIT_TRIGGER SQLITE_PRIVATE void sqlite3TreeViewTriggerStep(TreeView*, const TriggerStep*, u8, u8); SQLITE_PRIVATE void sqlite3TreeViewTrigger(TreeView*, const Trigger*, u8, u8); @@ -30428,8 +30430,8 @@ SQLITE_API void sqlite3_str_vappendf( case etSQLESCAPE: /* %q: Escape ' characters */ case etSQLESCAPE2: /* %Q: Escape ' and enclose in '...' */ case etSQLESCAPE3: { /* %w: Escape " characters */ - int i, j, k, n, isnull; - int needQuote; + i64 i, j, k, n; + int needQuote, isnull; char ch; char q = ((xtype==etSQLESCAPE3)?'"':'\''); /* Quote character */ char *escarg; @@ -31117,8 +31119,8 @@ SQLITE_PRIVATE void sqlite3TreeViewColumnList( sqlite3TreeViewLine(pView, "COLUMNS"); for(i=0; ipTab; sqlite3TreeViewColumnList(pView, pTab->aCol, pTab->nCol, 1); } - assert( pItem->fg.isNestedFrom == IsNestedFrom(pItem->pSelect) ); + assert( (int)pItem->fg.isNestedFrom == IsNestedFrom(pItem->pSelect) ); sqlite3TreeViewSelect(pView, pItem->pSelect, (--n)>0); } if( pItem->fg.isTabFunc ){ @@ -32016,6 +32018,7 @@ SQLITE_PRIVATE void sqlite3TreeViewUpsert( sqlite3TreeViewPop(&pView); } +#if TREETRACE_ENABLED /* ** Generate a human-readable diagram of the data structure that go ** into generating an DELETE statement. @@ -32069,7 +32072,9 @@ SQLITE_PRIVATE void sqlite3TreeViewDelete( } sqlite3TreeViewPop(&pView); } +#endif /* TREETRACE_ENABLED */ +#if TREETRACE_ENABLED /* ** Generate a human-readable diagram of the data structure that go ** into generating an INSERT statement. @@ -32137,7 +32142,9 @@ SQLITE_PRIVATE void sqlite3TreeViewInsert( } sqlite3TreeViewPop(&pView); } +#endif /* TREETRACE_ENABLED */ +#if TREETRACE_ENABLED /* ** Generate a human-readable diagram of the data structure that go ** into generating an UPDATE statement. @@ -32213,6 +32220,7 @@ SQLITE_PRIVATE void sqlite3TreeViewUpdate( } sqlite3TreeViewPop(&pView); } +#endif /* TREETRACE_ENABLED */ #ifndef SQLITE_OMIT_TRIGGER /* @@ -101532,33 +101540,23 @@ static void resolveAlias( sqlite3ExprDelete(db, pDup); pDup = 0; }else{ + Expr temp; incrAggFunctionDepth(pDup, nSubquery); if( pExpr->op==TK_COLLATE ){ assert( !ExprHasProperty(pExpr, EP_IntValue) ); pDup = sqlite3ExprAddCollateString(pParse, pDup, pExpr->u.zToken); } - - /* Before calling sqlite3ExprDelete(), set the EP_Static flag. This - ** prevents ExprDelete() from deleting the Expr structure itself, - ** allowing it to be repopulated by the memcpy() on the following line. - ** The pExpr->u.zToken might point into memory that will be freed by the - ** sqlite3DbFree(db, pDup) on the last line of this block, so be sure to - ** make a copy of the token before doing the sqlite3DbFree(). - */ - ExprSetProperty(pExpr, EP_Static); - sqlite3ExprDelete(db, pExpr); - memcpy(pExpr, pDup, sizeof(*pExpr)); - if( !ExprHasProperty(pExpr, EP_IntValue) && pExpr->u.zToken!=0 ){ - assert( (pExpr->flags & (EP_Reduced|EP_TokenOnly))==0 ); - pExpr->u.zToken = sqlite3DbStrDup(db, pExpr->u.zToken); - pExpr->flags |= EP_MemToken; - } + memcpy(&temp, pDup, sizeof(Expr)); + memcpy(pDup, pExpr, sizeof(Expr)); + memcpy(pExpr, &temp, sizeof(Expr)); if( ExprHasProperty(pExpr, EP_WinFunc) ){ if( ALWAYS(pExpr->y.pWin!=0) ){ pExpr->y.pWin->pOwner = pExpr; } } - sqlite3DbFree(db, pDup); + sqlite3ParserAddCleanup(pParse, + (void(*)(sqlite3*,void*))sqlite3ExprDelete, + pDup); } } @@ -101761,7 +101759,7 @@ static int lookupName( pTab = pItem->pTab; assert( pTab!=0 && pTab->zName!=0 ); assert( pTab->nCol>0 || pParse->nErr ); - assert( pItem->fg.isNestedFrom == IsNestedFrom(pItem->pSelect) ); + assert( (int)pItem->fg.isNestedFrom == IsNestedFrom(pItem->pSelect) ); if( pItem->fg.isNestedFrom ){ /* In this case, pItem is a subquery that has been formed from a ** parenthesized subset of the FROM clause terms. Example: @@ -115451,8 +115449,6 @@ SQLITE_PRIVATE void sqlite3NestedParse(Parse *pParse, const char *zFormat, ...){ memset(PARSE_TAIL(pParse), 0, PARSE_TAIL_SZ); db->mDbFlags |= DBFLAG_PreferBuiltin; sqlite3RunParser(pParse, zSql); - sqlite3DbFree(db, pParse->zErrMsg); - pParse->zErrMsg = 0; db->mDbFlags = savedDbFlags; sqlite3DbFree(db, zSql); memcpy(PARSE_TAIL(pParse), saveBuf, PARSE_TAIL_SZ); @@ -135853,7 +135849,7 @@ SQLITE_PRIVATE int sqlite3ColumnIndex(Table *pTab, const char *zCol){ */ SQLITE_PRIVATE void sqlite3SrcItemColumnUsed(SrcItem *pItem, int iCol){ assert( pItem!=0 ); - assert( pItem->fg.isNestedFrom == IsNestedFrom(pItem->pSelect) ); + assert( (int)pItem->fg.isNestedFrom == IsNestedFrom(pItem->pSelect) ); if( pItem->fg.isNestedFrom ){ ExprList *pResults; assert( pItem->pSelect!=0 ); @@ -139651,6 +139647,9 @@ static void renumberCursors( ** (17d2) DISTINCT ** (17e) the subquery may not contain window functions, and ** (17f) the subquery must not be the RHS of a LEFT JOIN. +** (17g) either the subquery is the first element of the outer +** query or there are no RIGHT or FULL JOINs in any arm +** of the subquery. (This is a duplicate of condition (27b).) ** ** The parent and sub-query may contain WHERE clauses. Subject to ** rules (11), (13) and (14), they may also contain ORDER BY, @@ -139702,7 +139701,11 @@ static void renumberCursors( ** See also (3) for restrictions on LEFT JOIN. ** ** (27) The subquery may not contain a FULL or RIGHT JOIN unless it -** is the first element of the parent query. +** is the first element of the parent query. This must be the +** the case if: +** (27a) the subquery is not compound query, and +** (27b) the subquery is a compound query and the RIGHT JOIN occurs +** in any arm of the compound query. (See also (17g).) ** ** (28) The subquery is not a MATERIALIZED CTE. ** @@ -139832,7 +139835,7 @@ static int flattenSubquery( assert( pSubSrc->nSrc>0 ); /* True by restriction (7) */ if( iFrom>0 && (pSubSrc->a[0].fg.jointype & JT_LTORJ)!=0 ){ - return 0; /* Restriction (27) */ + return 0; /* Restriction (27a) */ } if( pSubitem->fg.isCte && pSubitem->u2.pCteUse->eM10d==M10d_Yes ){ return 0; /* (28) */ @@ -139852,7 +139855,7 @@ static int flattenSubquery( ** NATURAL join or a join that as an ON or USING clause. ** ** These conditions are sufficient to keep an EP_OuterON from being - ** flattened into an EP_InnerON. Restrictions (3a) and (27) prevent + ** flattened into an EP_InnerON. Restrictions (3a) and (27a) prevent ** an EP_InnerON from being flattened into an EP_OuterON. */ if( pSubSrc->nSrc>=2 @@ -139894,6 +139897,12 @@ static int flattenSubquery( ){ return 0; } + if( iFrom>0 && (pSub1->pSrc->a[0].fg.jointype & JT_LTORJ)!=0 ){ + /* Without this restriction, the JT_LTORJ flag would end up being + ** omitted on left-hand tables of the right join that is being + ** flattened. */ + return 0; /* Restrictions (17g), (27b) */ + } testcase( pSub1->pSrc->nSrc>1 ); } @@ -141427,7 +141436,7 @@ static int selectExpander(Walker *pWalker, Select *p){ zTabName = pTab->zName; } if( db->mallocFailed ) break; - assert( pFrom->fg.isNestedFrom == IsNestedFrom(pFrom->pSelect) ); + assert( (int)pFrom->fg.isNestedFrom == IsNestedFrom(pFrom->pSelect) ); if( pFrom->fg.isNestedFrom ){ assert( pFrom->pSelect!=0 ); pNestedFrom = pFrom->pSelect->pEList; @@ -142356,7 +142365,9 @@ SQLITE_PRIVATE int sqlite3Select( ){ SELECTTRACE(0x100,pParse,p, ("omit superfluous ORDER BY on %r FROM-clause subquery\n",i+1)); - sqlite3ExprListDelete(db, pSub->pOrderBy); + sqlite3ParserAddCleanup(pParse, + (void(*)(sqlite3*,void*))sqlite3ExprListDelete, + pSub->pOrderBy); pSub->pOrderBy = 0; } @@ -155356,7 +155367,7 @@ static int whereKeyStats( #endif assert( pRec!=0 ); assert( pIdx->nSample>0 ); - assert( pRec->nField>0 && pRec->nField<=pIdx->nSampleCol ); + assert( pRec->nField>0 ); /* Do a binary search to find the first sample greater than or equal ** to pRec. If pRec contains a single field, the set of samples to search @@ -155402,7 +155413,7 @@ static int whereKeyStats( ** it is extended to two fields. The duplicates that this creates do not ** cause any problems. */ - nField = pRec->nField; + nField = MIN(pRec->nField, pIdx->nSample); iCol = 0; iSample = pIdx->nSample * nField; do{ @@ -158091,6 +158102,7 @@ static int whereLoopAddAll(WhereLoopBuilder *pBuilder){ sqlite3 *db = pWInfo->pParse->db; int rc = SQLITE_OK; int bFirstPastRJ = 0; + int hasRightJoin = 0; WhereLoop *pNew; @@ -158111,15 +158123,16 @@ static int whereLoopAddAll(WhereLoopBuilder *pBuilder){ ** prevents the right operand of a RIGHT JOIN from being swapped with ** other elements even further to the right. ** - ** The JT_LTORJ term prevents any FROM-clause term reordering for terms - ** to the left of a RIGHT JOIN. This is conservative. Relaxing this - ** constraint somewhat to prevent terms from crossing from the right - ** side of a LEFT JOIN over to the left side when they are on the - ** left side of a RIGHT JOIN would be sufficient for all known failure - ** cases. FIX ME: Implement this optimization. + ** The JT_LTORJ case and the hasRightJoin flag work together to + ** prevent FROM-clause terms from moving from the right side of + ** a LEFT JOIN over to the left side of that join if the LEFT JOIN + ** is itself on the left side of a RIGHT JOIN. */ + if( pItem->fg.jointype & JT_LTORJ ) hasRightJoin = 1; mPrereq |= mPrior; bFirstPastRJ = (pItem->fg.jointype & JT_RIGHT)!=0; + }else if( !hasRightJoin ){ + mPrereq = 0; } #ifndef SQLITE_OMIT_VIRTUALTABLE if( IsVirtual(pItem->pTab) ){ @@ -169960,6 +169973,7 @@ SQLITE_PRIVATE int sqlite3RunParser(Parse *pParse, const char *zSql){ mxSqlLen -= n; if( mxSqlLen<0 ){ pParse->rc = SQLITE_TOOBIG; + pParse->nErr++; break; } #ifndef SQLITE_OMIT_WINDOWFUNC @@ -174700,8 +174714,11 @@ SQLITE_API int sqlite3_test_control(int op, ...){ sqlite3ShowTriggerStepList(0); sqlite3ShowTrigger(0); sqlite3ShowTriggerList(0); +#ifndef SQLITE_OMIT_WINDOWFUNC sqlite3ShowWindow(0); sqlite3ShowWinFunc(0); +#endif + sqlite3ShowSelect(0); } #endif break; @@ -181035,8 +181052,7 @@ static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){ char *aPoslist = 0; /* Position list for deferred tokens */ int nPoslist = 0; /* Number of bytes in aPoslist */ int iPrev = -1; /* Token number of previous deferred token */ - - assert( pPhrase->doclist.bFreeList==0 ); + char *aFree = (pPhrase->doclist.bFreeList ? pPhrase->doclist.pList : 0); for(iToken=0; iTokennToken; iToken++){ Fts3PhraseToken *pToken = &pPhrase->aToken[iToken]; @@ -181050,6 +181066,7 @@ static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){ if( pList==0 ){ sqlite3_free(aPoslist); + sqlite3_free(aFree); pPhrase->doclist.pList = 0; pPhrase->doclist.nList = 0; return SQLITE_OK; @@ -181070,6 +181087,7 @@ static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){ nPoslist = (int)(aOut - aPoslist); if( nPoslist==0 ){ sqlite3_free(aPoslist); + sqlite3_free(aFree); pPhrase->doclist.pList = 0; pPhrase->doclist.nList = 0; return SQLITE_OK; @@ -181109,6 +181127,7 @@ static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){ } pPhrase->doclist.pList = aOut; + assert( p1 && p2 ); if( fts3PoslistPhraseMerge(&aOut, nDistance, 0, 1, &p1, &p2) ){ pPhrase->doclist.bFreeList = 1; pPhrase->doclist.nList = (int)(aOut - pPhrase->doclist.pList); @@ -181121,6 +181140,7 @@ static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){ } } + if( pPhrase->doclist.pList!=aFree ) sqlite3_free(aFree); return SQLITE_OK; } #endif /* SQLITE_DISABLE_FTS4_DEFERRED */ @@ -182295,11 +182315,10 @@ static int fts3EvalTestExpr( default: { #ifndef SQLITE_DISABLE_FTS4_DEFERRED - if( pCsr->pDeferred - && (pExpr->iDocid==pCsr->iPrevId || pExpr->bDeferred) - ){ + if( pCsr->pDeferred && (pExpr->bDeferred || ( + pExpr->iDocid==pCsr->iPrevId && pExpr->pPhrase->doclist.pList + ))){ Fts3Phrase *pPhrase = pExpr->pPhrase; - assert( pExpr->bDeferred || pPhrase->doclist.bFreeList==0 ); if( pExpr->bDeferred ){ fts3EvalInvalidatePoslist(pPhrase); } @@ -236617,7 +236636,7 @@ static void fts5SourceIdFunc( ){ assert( nArg==0 ); UNUSED_PARAM2(nArg, apUnused); - sqlite3_result_text(pCtx, "fts5: 2022-06-25 14:57:57 14e166f40dbfa6e055543f8301525f2ca2e96a02a57269818b9e69e162e98918", -1, SQLITE_TRANSIENT); + sqlite3_result_text(pCtx, "fts5: 2022-07-21 15:24:47 698edb77537b67c41adc68f9b892db56bcf9a55e00371a61420f3ddd668e6603", -1, SQLITE_TRANSIENT); } /* @@ -241288,6 +241307,16 @@ SQLITE_EXTENSION_INIT1 #ifndef SQLITE_OMIT_VIRTUALTABLE + +#define STMT_NUM_INTEGER_COLUMN 10 +typedef struct StmtRow StmtRow; +struct StmtRow { + sqlite3_int64 iRowid; /* Rowid value */ + char *zSql; /* column "sql" */ + int aCol[STMT_NUM_INTEGER_COLUMN+1]; /* all other column values */ + StmtRow *pNext; /* Next row to return */ +}; + /* stmt_vtab is a subclass of sqlite3_vtab which will ** serve as the underlying representation of a stmt virtual table */ @@ -241305,8 +241334,7 @@ typedef struct stmt_cursor stmt_cursor; struct stmt_cursor { sqlite3_vtab_cursor base; /* Base class - must be first */ sqlite3 *db; /* Database connection for this cursor */ - sqlite3_stmt *pStmt; /* Statement cursor is currently pointing at */ - sqlite3_int64 iRowid; /* The rowid */ + StmtRow *pRow; /* Current row */ }; /* @@ -241350,7 +241378,7 @@ static int stmtConnect( "CREATE TABLE x(sql,ncol,ro,busy,nscan,nsort,naidx,nstep," "reprep,run,mem)"); if( rc==SQLITE_OK ){ - pNew = sqlite3_malloc( sizeof(*pNew) ); + pNew = sqlite3_malloc64( sizeof(*pNew) ); *ppVtab = (sqlite3_vtab*)pNew; if( pNew==0 ) return SQLITE_NOMEM; memset(pNew, 0, sizeof(*pNew)); @@ -241372,7 +241400,7 @@ static int stmtDisconnect(sqlite3_vtab *pVtab){ */ static int stmtOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ stmt_cursor *pCur; - pCur = sqlite3_malloc( sizeof(*pCur) ); + pCur = sqlite3_malloc64( sizeof(*pCur) ); if( pCur==0 ) return SQLITE_NOMEM; memset(pCur, 0, sizeof(*pCur)); pCur->db = ((stmt_vtab*)p)->db; @@ -241380,10 +241408,21 @@ static int stmtOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ return SQLITE_OK; } +static void stmtCsrReset(stmt_cursor *pCur){ + StmtRow *pRow = 0; + StmtRow *pNext = 0; + for(pRow=pCur->pRow; pRow; pRow=pNext){ + pNext = pRow->pNext; + sqlite3_free(pRow); + } + pCur->pRow = 0; +} + /* ** Destructor for a stmt_cursor. */ static int stmtClose(sqlite3_vtab_cursor *cur){ + stmtCsrReset((stmt_cursor*)cur); sqlite3_free(cur); return SQLITE_OK; } @@ -241394,8 +241433,9 @@ static int stmtClose(sqlite3_vtab_cursor *cur){ */ static int stmtNext(sqlite3_vtab_cursor *cur){ stmt_cursor *pCur = (stmt_cursor*)cur; - pCur->iRowid++; - pCur->pStmt = sqlite3_next_stmt(pCur->db, pCur->pStmt); + StmtRow *pNext = pCur->pRow->pNext; + sqlite3_free(pCur->pRow); + pCur->pRow = pNext; return SQLITE_OK; } @@ -241409,39 +241449,11 @@ static int stmtColumn( int i /* Which column to return */ ){ stmt_cursor *pCur = (stmt_cursor*)cur; - switch( i ){ - case STMT_COLUMN_SQL: { - sqlite3_result_text(ctx, sqlite3_sql(pCur->pStmt), -1, SQLITE_TRANSIENT); - break; - } - case STMT_COLUMN_NCOL: { - sqlite3_result_int(ctx, sqlite3_column_count(pCur->pStmt)); - break; - } - case STMT_COLUMN_RO: { - sqlite3_result_int(ctx, sqlite3_stmt_readonly(pCur->pStmt)); - break; - } - case STMT_COLUMN_BUSY: { - sqlite3_result_int(ctx, sqlite3_stmt_busy(pCur->pStmt)); - break; - } - default: { - assert( i==STMT_COLUMN_MEM ); - i = SQLITE_STMTSTATUS_MEMUSED + - STMT_COLUMN_NSCAN - SQLITE_STMTSTATUS_FULLSCAN_STEP; - /* Fall thru */ - } - case STMT_COLUMN_NSCAN: - case STMT_COLUMN_NSORT: - case STMT_COLUMN_NAIDX: - case STMT_COLUMN_NSTEP: - case STMT_COLUMN_REPREP: - case STMT_COLUMN_RUN: { - sqlite3_result_int(ctx, sqlite3_stmt_status(pCur->pStmt, - i-STMT_COLUMN_NSCAN+SQLITE_STMTSTATUS_FULLSCAN_STEP, 0)); - break; - } + StmtRow *pRow = pCur->pRow; + if( i==STMT_COLUMN_SQL ){ + sqlite3_result_text(ctx, pRow->zSql, -1, SQLITE_TRANSIENT); + }else{ + sqlite3_result_int(ctx, pRow->aCol[i]); } return SQLITE_OK; } @@ -241452,7 +241464,7 @@ static int stmtColumn( */ static int stmtRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ stmt_cursor *pCur = (stmt_cursor*)cur; - *pRowid = pCur->iRowid; + *pRowid = pCur->pRow->iRowid; return SQLITE_OK; } @@ -241462,7 +241474,7 @@ static int stmtRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ */ static int stmtEof(sqlite3_vtab_cursor *cur){ stmt_cursor *pCur = (stmt_cursor*)cur; - return pCur->pStmt==0; + return pCur->pRow==0; } /* @@ -241477,9 +241489,53 @@ static int stmtFilter( int argc, sqlite3_value **argv ){ stmt_cursor *pCur = (stmt_cursor *)pVtabCursor; - pCur->pStmt = 0; - pCur->iRowid = 0; - return stmtNext(pVtabCursor); + sqlite3_stmt *p = 0; + sqlite3_int64 iRowid = 1; + StmtRow **ppRow = 0; + + stmtCsrReset(pCur); + ppRow = &pCur->pRow; + for(p=sqlite3_next_stmt(pCur->db, 0); p; p=sqlite3_next_stmt(pCur->db, p)){ + const char *zSql = sqlite3_sql(p); + sqlite3_int64 nSql = zSql ? strlen(zSql)+1 : 0; + StmtRow *pNew = (StmtRow*)sqlite3_malloc64(sizeof(StmtRow) + nSql); + + if( pNew==0 ) return SQLITE_NOMEM; + memset(pNew, 0, sizeof(StmtRow)); + if( zSql ){ + pNew->zSql = (char*)&pNew[1]; + memcpy(pNew->zSql, zSql, nSql); + } + pNew->aCol[STMT_COLUMN_NCOL] = sqlite3_column_count(p); + pNew->aCol[STMT_COLUMN_RO] = sqlite3_stmt_readonly(p); + pNew->aCol[STMT_COLUMN_BUSY] = sqlite3_stmt_busy(p); + pNew->aCol[STMT_COLUMN_NSCAN] = sqlite3_stmt_status( + p, SQLITE_STMTSTATUS_FULLSCAN_STEP, 0 + ); + pNew->aCol[STMT_COLUMN_NSORT] = sqlite3_stmt_status( + p, SQLITE_STMTSTATUS_SORT, 0 + ); + pNew->aCol[STMT_COLUMN_NAIDX] = sqlite3_stmt_status( + p, SQLITE_STMTSTATUS_AUTOINDEX, 0 + ); + pNew->aCol[STMT_COLUMN_NSTEP] = sqlite3_stmt_status( + p, SQLITE_STMTSTATUS_VM_STEP, 0 + ); + pNew->aCol[STMT_COLUMN_REPREP] = sqlite3_stmt_status( + p, SQLITE_STMTSTATUS_REPREPARE, 0 + ); + pNew->aCol[STMT_COLUMN_RUN] = sqlite3_stmt_status( + p, SQLITE_STMTSTATUS_RUN, 0 + ); + pNew->aCol[STMT_COLUMN_MEM] = sqlite3_stmt_status( + p, SQLITE_STMTSTATUS_MEMUSED, 0 + ); + pNew->iRowid = iRowid++; + *ppRow = pNew; + ppRow = &pNew->pNext; + } + + return SQLITE_OK; } /* diff --git a/src/database/sqlite3.h b/src/database/sqlite3.h index 26b4a31f..f0df724d 100644 --- a/src/database/sqlite3.h +++ b/src/database/sqlite3.h @@ -146,9 +146,9 @@ extern "C" { ** [sqlite3_libversion_number()], [sqlite3_sourceid()], ** [sqlite_version()] and [sqlite_source_id()]. */ -#define SQLITE_VERSION "3.39.0" -#define SQLITE_VERSION_NUMBER 3039000 -#define SQLITE_SOURCE_ID "2022-06-25 14:57:57 14e166f40dbfa6e055543f8301525f2ca2e96a02a57269818b9e69e162e98918" +#define SQLITE_VERSION "3.39.2" +#define SQLITE_VERSION_NUMBER 3039002 +#define SQLITE_SOURCE_ID "2022-07-21 15:24:47 698edb77537b67c41adc68f9b892db56bcf9a55e00371a61420f3ddd668e6603" /* ** CAPI3REF: Run-Time Library Version Numbers @@ -6282,7 +6282,7 @@ SQLITE_API sqlite3 *sqlite3_db_handle(sqlite3_stmt*); ** ** ^The sqlite3_db_name(D,N) interface returns a pointer to the schema name ** for the N-th database on database connection D, or a NULL pointer of N is -** out of range. An N alue of 0 means the main database file. An N of 1 is +** out of range. An N value of 0 means the main database file. An N of 1 is ** the "temp" schema. Larger values of N correspond to various ATTACH-ed ** databases. ** From 7a61a7cd0e00ec63028539024beee23fb091f615 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Mon, 25 Jul 2022 12:53:13 +0100 Subject: [PATCH 014/218] Update build.yml to use latest FTL build containers Signed-off-by: Adam Warner --- .github/workflows/build.yml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d3c909df..79e73f46 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -25,7 +25,7 @@ jobs: runs-on: ubuntu-latest - container: ghcr.io/pi-hole/ftl-build:v1.16-${{ matrix.arch }} + container: ghcr.io/pi-hole/ftl-build:v1.18-${{ matrix.arch }} strategy: fail-fast: false @@ -68,6 +68,17 @@ jobs: echo ::set-output name=GIT_BRANCH::${GIT_BRANCH} echo ::set-output name=GIT_TAG::${GIT_TAG} echo ::set-output name=OUTPUT_DIR::${GIT_TAG:-${GIT_BRANCH}} + - + name: "Print environment details" + env: + GIT_BRANCH: ${{ steps.variables.outputs.GIT_BRANCH }} + GIT_TAG: ${{ steps.variables.outputs.GIT_TAG }} + run: | + printenv + find . ! -user root -exec ls -lh {} \; + chown -R root . + git --no-pager describe --always --dirty + git log -1 --format='%h' - name: "Build" env: From 4788c09bce9518ed4bcaec349619d8d9c8392fe5 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 25 Jul 2022 18:01:42 +0200 Subject: [PATCH 015/218] Fix ownership of repository to avoid failure because of "fatal: detected dubious ownership in repository at '/__w/FTL/FTL'" even though "set-safe-directory: true" is set in the Github checkout action Signed-off-by: DL6ER --- .github/workflows/build.yml | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 79e73f46..45e01296 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -69,16 +69,8 @@ jobs: echo ::set-output name=GIT_TAG::${GIT_TAG} echo ::set-output name=OUTPUT_DIR::${GIT_TAG:-${GIT_BRANCH}} - - name: "Print environment details" - env: - GIT_BRANCH: ${{ steps.variables.outputs.GIT_BRANCH }} - GIT_TAG: ${{ steps.variables.outputs.GIT_TAG }} - run: | - printenv - find . ! -user root -exec ls -lh {} \; - chown -R root . - git --no-pager describe --always --dirty - git log -1 --format='%h' + name: "Fix ownership of repository" + run: chown -R root . - name: "Build" env: From 086adf90f185e3b868062fd2965023e74c5b561e Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sun, 23 Jan 2022 15:52:17 +0000 Subject: [PATCH 016/218] Add basic devcontainer.json based on ftl-build for x86_64 arch, with a few helpful extensions. Signed-off-by: Adam Warner --- .devcontainer/devcontainer.json | 10 ++++++++++ .gitignore | 4 ++++ 2 files changed, 14 insertions(+) create mode 100644 .devcontainer/devcontainer.json diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 00000000..d4420369 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,10 @@ +{ + "name": "FTL x86_64 Build Env", + "image": "ghcr.io/pi-hole/ftl-build:x86_64", + "extensions": [ + "jetmartin.bats", + "ms-vscode.cpptools", + "ms-vscode.cmake-tools", + "eamodio.gitlens" + ], +} \ No newline at end of file diff --git a/.gitignore b/.gitignore index 7e971607..9cf9308d 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,7 @@ tools/macvendor.db # Test dependencies /node_modules/ + +# Test Leftovers +dig.log +ptr.log From 8d84c311e0012bbde9bae8624fdcb1e86eb1ae9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Mon, 25 Jul 2022 22:29:15 +0200 Subject: [PATCH 017/218] Update actions to latest versions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- .github/workflows/build.yml | 4 ++-- .github/workflows/codespell.yml | 2 +- .github/workflows/stale.yml | 2 +- .github/workflows/sync-back-to-dev.yml | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 787f3e5a..288e89aa 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -57,7 +57,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v3.0.0 + uses: actions/checkout@v3.0.2 - name: Calculate required build variables id: variables @@ -92,7 +92,7 @@ jobs: - name: Upload artifacts to job for later processing if: github.event_name != 'pull_request' - uses: actions/upload-artifact@v3.0.0 + uses: actions/upload-artifact@v3.1.0 with: name: tmp-binary-storage path: '${{ matrix.bin_name }}*' diff --git a/.github/workflows/codespell.yml b/.github/workflows/codespell.yml index 31e4109c..69f6cc38 100644 --- a/.github/workflows/codespell.yml +++ b/.github/workflows/codespell.yml @@ -10,7 +10,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3.0.0 + uses: actions/checkout@v3.0.2 - name: Spell-Checking uses: codespell-project/actions-codespell@master diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index a024688a..cd73e1cd 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -13,7 +13,7 @@ jobs: issues: write steps: - - uses: actions/stale@v5.0.0 + - uses: actions/stale@v5.1.0 with: repo-token: ${{ secrets.GITHUB_TOKEN }} days-before-stale: 30 diff --git a/.github/workflows/sync-back-to-dev.yml b/.github/workflows/sync-back-to-dev.yml index 30b1a970..27d966d2 100644 --- a/.github/workflows/sync-back-to-dev.yml +++ b/.github/workflows/sync-back-to-dev.yml @@ -11,7 +11,7 @@ jobs: name: Syncing branches steps: - name: Checkout - uses: actions/checkout@v3.0.0 + uses: actions/checkout@v3.0.2 - name: Opening pull request id: pull uses: tretuna/sync-branches@1.4.0 @@ -20,7 +20,7 @@ jobs: FROM_BRANCH: 'master' TO_BRANCH: 'development' - name: Label the pull request to ignore for release note generation - uses: actions-ecosystem/action-add-labels@v1.0.0 + uses: actions-ecosystem/action-add-labels@v1.1.0 with: labels: internal repo: ${{ github.repository }} From 66acc6cf5ce562ad91a43c6b8255b11bfb03b6f7 Mon Sep 17 00:00:00 2001 From: RD WebDesign Date: Wed, 27 Jul 2022 01:17:59 -0300 Subject: [PATCH 018/218] Add darkmode logos Signed-off-by: RD WebDesign --- README.md | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 3f823f3e..34876e72 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,21 @@

-Pi-hole -
-Network-wide ad blocking via your own Linux hardware

-FTLDNS
+ + + + + Pi-hole website + + +
+ Network-wide ad blocking via your own Linux hardware

+ + + + + FTLDNS + + +

FTLDNS (`pihole-FTL`) provides an interactive API and also generates statistics for Pi-hole[®](https://pi-hole.net/trademark-rules-and-brand-guidelines/)'s Web interface. From 5a095b1cb15b729aebb6ab280f44b217f1e11810 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 27 Jul 2022 19:20:03 +0200 Subject: [PATCH 019/218] Ensure errno value is restored on return of syscall functions. Signed-off-by: DL6ER --- src/api/api.c | 360 ++++++++++++++--------------- src/api/api.h | 40 ++-- src/api/request.c | 22 +- src/api/request.h | 2 +- src/api/socket.c | 371 ++++++++++++++---------------- src/api/socket.h | 5 +- src/daemon.c | 6 +- src/dnsmasq_interface.c | 13 +- src/dnsmasq_interface.h | 1 - src/syscalls/accept.c | 12 +- src/syscalls/calloc.c | 7 + src/syscalls/fopen.c | 7 + src/syscalls/ftlallocate.c | 1 + src/syscalls/pthread_mutex_lock.c | 12 +- src/syscalls/realloc.c | 6 + src/syscalls/recv.c | 12 +- src/syscalls/recvfrom.c | 12 +- src/syscalls/select.c | 12 +- src/syscalls/sendto.c | 12 +- src/syscalls/vasprintf.c | 3 + src/syscalls/vfprintf.c | 9 + src/syscalls/vsnprintf.c | 4 + src/syscalls/vsprintf.c | 3 + src/syscalls/write.c | 23 +- 24 files changed, 495 insertions(+), 460 deletions(-) diff --git a/src/api/api.c b/src/api/api.c index 0857b5f6..32047a2f 100644 --- a/src/api/api.c +++ b/src/api/api.c @@ -76,7 +76,7 @@ static int __attribute__((pure)) cmpdesc(const void *a, const void *b) return 0; } -void getStats(const int *sock) +void getStats(const int sock) { const int blocked = blocked_queries(); const int total = counters->queries; @@ -87,11 +87,11 @@ void getStats(const int *sock) percentage = 1e2f*blocked/total; // Send domains being blocked - if(istelnet[*sock]) { - ssend(*sock, "domains_being_blocked %i\n", counters->gravity); + if(istelnet[sock]) { + ssend(sock, "domains_being_blocked %i\n", counters->gravity); } else - pack_int32(*sock, counters->gravity); + pack_int32(sock, counters->gravity); // unique_clients: count only clients that have been active within the most recent 24 hours int activeclients = 0; @@ -106,13 +106,13 @@ void getStats(const int *sock) activeclients++; } - if(istelnet[*sock]) { - ssend(*sock, "dns_queries_today %i\nads_blocked_today %i\nads_percentage_today %f\n", + if(istelnet[sock]) { + ssend(sock, "dns_queries_today %i\nads_blocked_today %i\nads_percentage_today %f\n", total, blocked, percentage); - ssend(*sock, "unique_domains %i\nqueries_forwarded %i\nqueries_cached %i\n", + ssend(sock, "unique_domains %i\nqueries_forwarded %i\nqueries_cached %i\n", counters->domains, forwarded_queries(), cached_queries()); - ssend(*sock, "clients_ever_seen %i\n", counters->clients); - ssend(*sock, "unique_clients %i\n", activeclients); + ssend(sock, "clients_ever_seen %i\n", counters->clients); + ssend(sock, "unique_clients %i\n", activeclients); // Sum up all query types (A, AAAA, ANY, SRV, SOA, ...) int sumalltypes = 0; @@ -120,45 +120,45 @@ void getStats(const int *sock) { sumalltypes += counters->querytype[queryType]; } - ssend(*sock, "dns_queries_all_types %i\n", sumalltypes); + ssend(sock, "dns_queries_all_types %i\n", sumalltypes); // Send individual reply type counters int sumallreplies = 0; for(enum reply_type reply = REPLY_UNKNOWN; reply < QUERY_REPLY_MAX; reply++) { - ssend(*sock, "reply_%s %i\n", get_query_reply_str(reply), counters->reply[reply]); + ssend(sock, "reply_%s %i\n", get_query_reply_str(reply), counters->reply[reply]); sumallreplies += counters->reply[reply]; } - ssend(*sock, "dns_queries_all_replies %i\n", sumallreplies); - ssend(*sock, "privacy_level %i\n", config.privacylevel); + ssend(sock, "dns_queries_all_replies %i\n", sumallreplies); + ssend(sock, "privacy_level %i\n", config.privacylevel); } else { - pack_int32(*sock, total); - pack_int32(*sock, blocked); - pack_float(*sock, percentage); - pack_int32(*sock, counters->domains); - pack_int32(*sock, forwarded_queries()); - pack_int32(*sock, cached_queries()); - pack_int32(*sock, counters->clients); - pack_int32(*sock, activeclients); + pack_int32(sock, total); + pack_int32(sock, blocked); + pack_float(sock, percentage); + pack_int32(sock, counters->domains); + pack_int32(sock, forwarded_queries()); + pack_int32(sock, cached_queries()); + pack_int32(sock, counters->clients); + pack_int32(sock, activeclients); } // Send status - if(istelnet[*sock]) { - ssend(*sock, "status %s\n", blockingstatus ? "enabled" : "disabled"); + if(istelnet[sock]) { + ssend(sock, "status %s\n", blockingstatus ? "enabled" : "disabled"); } else - pack_uint8(*sock, blockingstatus); + pack_uint8(sock, blockingstatus); } -void getOverTime(const int *sock) +void getOverTime(const int sock) { - if(istelnet[*sock]) + if(istelnet[sock]) { for(int slot = 0; slot < OVERTIME_SLOTS; slot++) { - ssend(*sock,"%lli %i %i\n", + ssend(sock,"%lli %i %i\n", (long long)overTime[slot].timestamp, overTime[slot].total, overTime[slot].blocked); @@ -170,22 +170,22 @@ void getOverTime(const int *sock) // and map16 can hold up to (2^16)-1 = 65535 pairs // Send domains over time - pack_map16_start(*sock, (uint16_t) OVERTIME_SLOTS); + pack_map16_start(sock, (uint16_t) OVERTIME_SLOTS); for(int slot = 0; slot < OVERTIME_SLOTS; slot++) { - pack_int32(*sock, (int32_t)overTime[slot].timestamp); - pack_int32(*sock, overTime[slot].total); + pack_int32(sock, (int32_t)overTime[slot].timestamp); + pack_int32(sock, overTime[slot].total); } // Send ads over time - pack_map16_start(*sock, (uint16_t) OVERTIME_SLOTS); + pack_map16_start(sock, (uint16_t) OVERTIME_SLOTS); for(int slot = 0; slot < OVERTIME_SLOTS; slot++) { - pack_int32(*sock, (int32_t)overTime[slot].timestamp); - pack_int32(*sock, overTime[slot].blocked); + pack_int32(sock, (int32_t)overTime[slot].timestamp); + pack_int32(sock, overTime[slot].blocked); } } } -void getTopDomains(const char *client_message, const int *sock) +void getTopDomains(const char *client_message, const int sock) { int temparray[counters->domains][2], count=10, num; bool audit = false, asc = false; @@ -196,8 +196,8 @@ void getTopDomains(const char *client_message, const int *sock) get_privacy_level(NULL); if(config.privacylevel >= PRIVACY_HIDE_DOMAINS) { // Always send the total number of domains, but pretend it's 0 - if(!istelnet[*sock]) - pack_int32(*sock, 0); + if(!istelnet[sock]) + pack_int32(sock, 0); return; } @@ -269,13 +269,13 @@ void getTopDomains(const char *client_message, const int *sock) } } - if(!istelnet[*sock]) + if(!istelnet[sock]) { // Send the data required to get the percentage each domain has been blocked / queried if(blocked) - pack_int32(*sock, blocked_queries()); + pack_int32(sock, blocked_queries()); else - pack_int32(*sock, counters->queries); + pack_int32(sock, counters->queries); } int n = 0; @@ -306,26 +306,26 @@ void getTopDomains(const char *client_message, const int *sock) if(blocked && showblocked && domain->blockedcount > 0) { - if(istelnet[*sock]) - ssend(*sock, "%i %i %s\n", n, domain->blockedcount, getstr(domain->domainpos)); + if(istelnet[sock]) + ssend(sock, "%i %i %s\n", n, domain->blockedcount, getstr(domain->domainpos)); else { - if(!pack_str32(*sock, getstr(domain->domainpos))) + if(!pack_str32(sock, getstr(domain->domainpos))) return; - pack_int32(*sock, domain->blockedcount); + pack_int32(sock, domain->blockedcount); } n++; } else if(!blocked && showpermitted && (domain->count - domain->blockedcount) > 0) { - if(istelnet[*sock]) - ssend(*sock,"%i %i %s\n",n,(domain->count - domain->blockedcount),getstr(domain->domainpos)); + if(istelnet[sock]) + ssend(sock,"%i %i %s\n",n,(domain->count - domain->blockedcount),getstr(domain->domainpos)); else { - if(!pack_str32(*sock, getstr(domain->domainpos))) + if(!pack_str32(sock, getstr(domain->domainpos))) return; - pack_int32(*sock, domain->count - domain->blockedcount); + pack_int32(sock, domain->count - domain->blockedcount); } n++; } @@ -339,7 +339,7 @@ void getTopDomains(const char *client_message, const int *sock) clearSetupVarsArray(); } -void getTopClients(const char *client_message, const int *sock) +void getTopClients(const char *client_message, const int sock) { int temparray[counters->clients][2], count=10, num; @@ -347,8 +347,8 @@ void getTopClients(const char *client_message, const int *sock) get_privacy_level(NULL); if(config.privacylevel >= PRIVACY_HIDE_DOMAINS_CLIENTS) { // Always send the total number of clients, but pretend it's 0 - if(!istelnet[*sock]) - pack_int32(*sock, 0); + if(!istelnet[sock]) + pack_int32(sock, 0); return; } @@ -408,10 +408,10 @@ void getTopClients(const char *client_message, const int *sock) getSetupVarsArray(excludeclients); } - if(!istelnet[*sock]) + if(!istelnet[sock]) { // Send the total queries so they can make percentages from this data - pack_int32(*sock, counters->queries); + pack_int32(sock, counters->queries); } int n = 0; @@ -450,14 +450,14 @@ void getTopClients(const char *client_message, const int *sock) // - the client made at least one query within the most recent 24 hours if(includezeroclients || ccount > 0) { - if(istelnet[*sock]) - ssend(*sock,"%i %i %s %s\n", n, ccount, client_ip, client_name); + if(istelnet[sock]) + ssend(sock,"%i %i %s %s\n", n, ccount, client_ip, client_name); else { - if(!pack_str32(*sock, "") || !pack_str32(*sock, client_ip)) + if(!pack_str32(sock, "") || !pack_str32(sock, client_ip)) return; - pack_int32(*sock, ccount); + pack_int32(sock, ccount); } n++; } @@ -471,7 +471,7 @@ void getTopClients(const char *client_message, const int *sock) } -void getUpstreamDestinations(const char *client_message, const int *sock) +void getUpstreamDestinations(const char *client_message, const int sock) { bool sort = true; int temparray[counters->upstreams][2], sumforwarded = 0; @@ -575,24 +575,24 @@ void getUpstreamDestinations(const char *client_message, const int *sock) // - only if percentage > 0.0 for all others (i > 0) if(percentage > 0.0f || i < 0) { - if(istelnet[*sock]) + if(istelnet[sock]) if(upstream_port != 0) - ssend(*sock, "%i %.2f %s#%u %s#%u\n", i, percentage, + ssend(sock, "%i %.2f %s#%u %s#%u\n", i, percentage, ip, upstream_port, name, upstream_port); else - ssend(*sock, "%i %.2f %s %s\n", i, percentage, ip, name); + ssend(sock, "%i %.2f %s %s\n", i, percentage, ip, name); else { - if(!pack_str32(*sock, name) || !pack_str32(*sock, ip)) + if(!pack_str32(sock, name) || !pack_str32(sock, ip)) return; - pack_float(*sock, percentage); + pack_float(sock, percentage); } } } } -void getQueryTypes(const int *sock) +void getQueryTypes(const int sock) { int total = 0; for(enum query_types type = TYPE_A; type < TYPE_MAX; type++) @@ -611,8 +611,8 @@ void getQueryTypes(const int *sock) } } - if(istelnet[*sock]) { - ssend(*sock, "A (IPv4): %.2f\nAAAA (IPv6): %.2f\nANY: %.2f\nSRV: %.2f\n" + if(istelnet[sock]) { + ssend(sock, "A (IPv4): %.2f\nAAAA (IPv6): %.2f\nANY: %.2f\nSRV: %.2f\n" "SOA: %.2f\nPTR: %.2f\nTXT: %.2f\nNAPTR: %.2f\n" "MX: %.2f\nDS: %.2f\nRRSIG: %.2f\nDNSKEY: %.2f\n" "NS: %.2f\n" "OTHER: %.2f\n\nSVCB: %.2f\nHTTPS: %.2f\n", @@ -622,42 +622,42 @@ void getQueryTypes(const int *sock) percentage[TYPE_NS], percentage[TYPE_OTHER], percentage[TYPE_SVCB], percentage[TYPE_HTTPS]); } else { - pack_str32(*sock, "A (IPv4)"); - pack_float(*sock, percentage[TYPE_A]); - pack_str32(*sock, "AAAA (IPv6)"); - pack_float(*sock, percentage[TYPE_AAAA]); - pack_str32(*sock, "ANY"); - pack_float(*sock, percentage[TYPE_ANY]); - pack_str32(*sock, "SRV"); - pack_float(*sock, percentage[TYPE_SRV]); - pack_str32(*sock, "SOA"); - pack_float(*sock, percentage[TYPE_SOA]); - pack_str32(*sock, "PTR"); - pack_float(*sock, percentage[TYPE_PTR]); - pack_str32(*sock, "TXT"); - pack_float(*sock, percentage[TYPE_TXT]); - pack_str32(*sock, "NAPTR"); - pack_float(*sock, percentage[TYPE_NAPTR]); - pack_str32(*sock, "MX"); - pack_float(*sock, percentage[TYPE_MX]); - pack_str32(*sock, "DS"); - pack_float(*sock, percentage[TYPE_DS]); - pack_str32(*sock, "RRSIG"); - pack_float(*sock, percentage[TYPE_RRSIG]); - pack_str32(*sock, "DNSKEY"); - pack_float(*sock, percentage[TYPE_DNSKEY]); - pack_str32(*sock, "NS"); - pack_float(*sock, percentage[TYPE_NS]); - pack_str32(*sock, "OTHER"); - pack_float(*sock, percentage[TYPE_OTHER]); - pack_str32(*sock, "SVCB"); - pack_float(*sock, percentage[TYPE_SVCB]); - pack_str32(*sock, "HTTPS"); - pack_float(*sock, percentage[TYPE_HTTPS]); + pack_str32(sock, "A (IPv4)"); + pack_float(sock, percentage[TYPE_A]); + pack_str32(sock, "AAAA (IPv6)"); + pack_float(sock, percentage[TYPE_AAAA]); + pack_str32(sock, "ANY"); + pack_float(sock, percentage[TYPE_ANY]); + pack_str32(sock, "SRV"); + pack_float(sock, percentage[TYPE_SRV]); + pack_str32(sock, "SOA"); + pack_float(sock, percentage[TYPE_SOA]); + pack_str32(sock, "PTR"); + pack_float(sock, percentage[TYPE_PTR]); + pack_str32(sock, "TXT"); + pack_float(sock, percentage[TYPE_TXT]); + pack_str32(sock, "NAPTR"); + pack_float(sock, percentage[TYPE_NAPTR]); + pack_str32(sock, "MX"); + pack_float(sock, percentage[TYPE_MX]); + pack_str32(sock, "DS"); + pack_float(sock, percentage[TYPE_DS]); + pack_str32(sock, "RRSIG"); + pack_float(sock, percentage[TYPE_RRSIG]); + pack_str32(sock, "DNSKEY"); + pack_float(sock, percentage[TYPE_DNSKEY]); + pack_str32(sock, "NS"); + pack_float(sock, percentage[TYPE_NS]); + pack_str32(sock, "OTHER"); + pack_float(sock, percentage[TYPE_OTHER]); + pack_str32(sock, "SVCB"); + pack_float(sock, percentage[TYPE_SVCB]); + pack_str32(sock, "HTTPS"); + pack_float(sock, percentage[TYPE_HTTPS]); } } -void getAllQueries(const char *client_message, const int *sock) +void getAllQueries(const char *client_message, const int sock) { // Exit before processing any data if requested via config setting get_privacy_level(NULL); @@ -1053,9 +1053,9 @@ void getAllQueries(const char *client_message, const int *sock) delay = 0UL; } - if(istelnet[*sock]) + if(istelnet[sock]) { - ssend(*sock,"%lli %s %s %s %i %i %i %lu %s %i %s#%u \"%s\"", + ssend(sock,"%lli %s %s %s %i %i %i %lu %s %i %s#%u \"%s\"", (long long)query->timestamp, qtype, domain, @@ -1071,23 +1071,23 @@ void getAllQueries(const char *client_message, const int *sock) query->ede == -1 ? "" : get_edestr(query->ede)); if(config.debug & DEBUG_API) - ssend(*sock, " \"%i\"", queryID); - ssend(*sock, "\n"); + ssend(sock, " \"%i\"", queryID); + ssend(sock, "\n"); } else { - pack_int32(*sock, (int32_t)query->timestamp); + pack_int32(sock, (int32_t)query->timestamp); // Use a fixstr because the length of qtype is always 4 (max is 31 for fixstr) - if(!pack_fixstr(*sock, qtype)) + if(!pack_fixstr(sock, qtype)) break; // Use str32 for domain and client because we have no idea how long they will be (max is 4294967295 for str32) - if(!pack_str32(*sock, domain) || !pack_str32(*sock, clientIPName)) + if(!pack_str32(sock, domain) || !pack_str32(sock, clientIPName)) break; - pack_uint8(*sock, query->status); - pack_uint8(*sock, query->dnssec); + pack_uint8(sock, query->status); + pack_uint8(sock, query->dnssec); } } @@ -1105,7 +1105,7 @@ void getAllQueries(const char *client_message, const int *sock) free(clientid_list); } -void getRecentBlocked(const char *client_message, const int *sock) +void getRecentBlocked(const char *client_message, const int sock) { int num=1; @@ -1132,9 +1132,9 @@ void getRecentBlocked(const char *client_message, const int *sock) if(domain == NULL) continue; - if(istelnet[*sock]) - ssend(*sock,"%s\n", domain); - else if(!pack_str32(*sock, domain)) + if(istelnet[sock]) + ssend(sock,"%s\n", domain); + else if(!pack_str32(sock, domain)) return; // Only count when sent successfully @@ -1146,15 +1146,15 @@ void getRecentBlocked(const char *client_message, const int *sock) } } -void getClientID(const int *sock) +void getClientID(const int sock) { - if(istelnet[*sock]) - ssend(*sock,"%i\n", *sock); + if(istelnet[sock]) + ssend(sock,"%i\n", sock); else - pack_int32(*sock, *sock); + pack_int32(sock, sock); } -void getVersion(const int *sock) +void getVersion(const int sock) { const char *commit = GIT_HASH; const char *tag = GIT_TAG; @@ -1165,25 +1165,25 @@ void getVersion(const int *sock) memcpy(hash, commit, min((size_t)7, strlen(commit))); if(strlen(tag) > 1) { - if(istelnet[*sock]) + if(istelnet[sock]) ssend( - *sock, + sock, "version %s\ntag %s\nbranch %s\nhash %s\ndate %s\n", version, tag, GIT_BRANCH, hash, GIT_DATE ); else { - if(!pack_str32(*sock, version) || - !pack_str32(*sock, (char *) tag) || - !pack_str32(*sock, GIT_BRANCH) || - !pack_str32(*sock, hash) || - !pack_str32(*sock, GIT_DATE)) + if(!pack_str32(sock, version) || + !pack_str32(sock, (char *) tag) || + !pack_str32(sock, GIT_BRANCH) || + !pack_str32(sock, hash) || + !pack_str32(sock, GIT_DATE)) return; } } else { - if(istelnet[*sock]) + if(istelnet[sock]) ssend( - *sock, + sock, "version vDev-%s\ntag %s\nbranch %s\nhash %s\ndate %s\n", hash, tag, GIT_BRANCH, hash, GIT_DATE ); @@ -1192,11 +1192,11 @@ void getVersion(const int *sock) if(hashVersion == NULL) return; sprintf(hashVersion, "vDev-%s", hash); - if(!pack_str32(*sock, hashVersion) || - !pack_str32(*sock, (char *) tag) || - !pack_str32(*sock, GIT_BRANCH) || - !pack_str32(*sock, hash) || - !pack_str32(*sock, GIT_DATE)) + if(!pack_str32(sock, hashVersion) || + !pack_str32(sock, (char *) tag) || + !pack_str32(sock, GIT_BRANCH) || + !pack_str32(sock, hash) || + !pack_str32(sock, GIT_DATE)) return; free(hashVersion); @@ -1204,7 +1204,7 @@ void getVersion(const int *sock) } } -void getDBstats(const int *sock) +void getDBstats(const int sock) { // Get file details unsigned long long int filesize = get_FTL_db_filesize(); @@ -1213,19 +1213,19 @@ void getDBstats(const int *sock) double formatted = 0.0; format_memory_size(prefix, filesize, &formatted); - if(istelnet[*sock]) - ssend(*sock, "queries in database: %i\ndatabase filesize: %.2f %sB\nSQLite version: %s\n", + if(istelnet[sock]) + ssend(sock, "queries in database: %i\ndatabase filesize: %.2f %sB\nSQLite version: %s\n", get_number_of_queries_in_DB(NULL), formatted, prefix, get_sqlite3_version()); else { - pack_int32(*sock, get_number_of_queries_in_DB(NULL)); - pack_int64(*sock, filesize); + pack_int32(sock, get_number_of_queries_in_DB(NULL)); + pack_int64(sock, filesize); - if(!pack_str32(*sock, (char *) get_sqlite3_version())) + if(!pack_str32(sock, (char *) get_sqlite3_version())) return; } } -void getClientsOverTime(const int *sock) +void getClientsOverTime(const int sock) { // Exit before processing any data if requested via config setting get_privacy_level(NULL); @@ -1263,10 +1263,10 @@ void getClientsOverTime(const int *sock) // Main return loop for(int slot = 0; slot < OVERTIME_SLOTS; slot++) { - if(istelnet[*sock]) - ssend(*sock, "%lli", (long long)overTime[slot].timestamp); + if(istelnet[sock]) + ssend(sock, "%lli", (long long)overTime[slot].timestamp); else - pack_int32(*sock, (int32_t)overTime[slot].timestamp); + pack_int32(sock, (int32_t)overTime[slot].timestamp); // Loop over forward destinations to generate output to be sent to the client for(int clientID = 0; clientID < counters->clients; clientID++) @@ -1284,23 +1284,23 @@ void getClientsOverTime(const int *sock) continue; const int thisclient = client->overTime[slot]; - if(istelnet[*sock]) - ssend(*sock, " %i", thisclient); + if(istelnet[sock]) + ssend(sock, " %i", thisclient); else - pack_int32(*sock, thisclient); + pack_int32(sock, thisclient); } - if(istelnet[*sock]) - ssend(*sock, "\n"); + if(istelnet[sock]) + ssend(sock, "\n"); else - pack_int32(*sock, -1); + pack_int32(sock, -1); } if(excludeclients != NULL) clearSetupVarsArray(); } -void getClientNames(const int *sock) +void getClientNames(const int sock) { // Exit before processing any data if requested via config setting get_privacy_level(NULL); @@ -1353,11 +1353,11 @@ void getClientNames(const int *sock) const char *client_ip = getstr(client->ippos); const char *client_name = getstr(client->namepos); - if(istelnet[*sock]) - ssend(*sock, "%s %s\n", client_name, client_ip); + if(istelnet[sock]) + ssend(sock, "%s %s\n", client_name, client_ip); else { - pack_str32(*sock, client_name); - pack_str32(*sock, client_ip); + pack_str32(sock, client_name); + pack_str32(sock, client_ip); } } @@ -1365,7 +1365,7 @@ void getClientNames(const int *sock) clearSetupVarsArray(); } -void getUnknownQueries(const int *sock) +void getUnknownQueries(const int sock) { // Exit before processing any data if requested via config setting get_privacy_level(NULL); @@ -1401,22 +1401,22 @@ void getUnknownQueries(const int *sock) // Get client IP string const char *clientIP = getstr(client->ippos); - if(istelnet[*sock]) - ssend(*sock, "%lli %i %i %s %s %s %i %s\n", (long long)query->timestamp, queryID, query->id, type, getstr(domain->domainpos), clientIP, query->status, query->flags.complete ? "true" : "false"); + if(istelnet[sock]) + ssend(sock, "%lli %i %i %s %s %s %i %s\n", (long long)query->timestamp, queryID, query->id, type, getstr(domain->domainpos), clientIP, query->status, query->flags.complete ? "true" : "false"); else { - pack_int32(*sock, (int32_t)query->timestamp); - pack_int32(*sock, query->id); + pack_int32(sock, (int32_t)query->timestamp); + pack_int32(sock, query->id); // Use a fixstr because the length of qtype is always 4 (max is 31 for fixstr) - if(!pack_fixstr(*sock, type)) + if(!pack_fixstr(sock, type)) return; // Use str32 for domain and client because we have no idea how long they will be (max is 4294967295 for str32) - if(!pack_str32(*sock, getstr(domain->domainpos)) || !pack_str32(*sock, clientIP)) + if(!pack_str32(sock, getstr(domain->domainpos)) || !pack_str32(sock, clientIP)) return; - pack_uint8(*sock, query->status); - pack_bool(*sock, query->flags.complete); + pack_uint8(sock, query->status); + pack_bool(sock, query->flags.complete); } } } @@ -1424,12 +1424,12 @@ void getUnknownQueries(const int *sock) // FTL_unlink_DHCP_lease() extern bool FTL_unlink_DHCP_lease(const char *ipaddr); -void delete_lease(const char *client_message, const int *sock) +void delete_lease(const char *client_message, const int sock) { // Extract IP address from request char ipaddr[INET6_ADDRSTRLEN] = { 0 }; if(sscanf(client_message, ">delete-lease %"xstr(INET6_ADDRSTRLEN)"s", ipaddr) < 1) { - ssend(*sock, "ERROR: No IP address specified!\n"); + ssend(sock, "ERROR: No IP address specified!\n"); return; } ipaddr[sizeof(ipaddr) - 1] = '\0'; @@ -1438,24 +1438,24 @@ void delete_lease(const char *client_message, const int *sock) logg("Received request to delete lease for %s", ipaddr); if(FTL_unlink_DHCP_lease(ipaddr)) - ssend(*sock, "OK: Removed specified lease\n"); + ssend(sock, "OK: Removed specified lease\n"); else - ssend(*sock, "ERROR: Specified IP address invalid!\n"); + ssend(sock, "ERROR: Specified IP address invalid!\n"); if(config.debug & DEBUG_API) logg("...done"); } -void getDNSport(const int *sock) +void getDNSport(const int sock) { // Return DNS port used by FTL - ssend(*sock, "%d\n", config.dns_port); + ssend(sock, "%d\n", config.dns_port); } -void getMAXLOGAGE(const int *sock) +void getMAXLOGAGE(const int sock) { // Return maxlogage used by FTL - ssend(*sock, "%d\n", config.maxlogage); + ssend(sock, "%d\n", config.maxlogage); } static bool getDefaultInterface(char iface[IF_NAMESIZE], in_addr_t *gw) @@ -1506,13 +1506,13 @@ static bool getDefaultInterface(char iface[IF_NAMESIZE], in_addr_t *gw) return gw != 0; } -void getGateway(const int *sock) +void getGateway(const int sock) { in_addr_t gw = 0; char iface[IF_NAMESIZE] = { 0 }; getDefaultInterface(iface, &gw); - ssend(*sock, "%s %s\n", inet_ntoa(*(struct in_addr *) &gw), iface); + ssend(sock, "%s %s\n", inet_ntoa(*(struct in_addr *) &gw), iface); } struct if_info { @@ -1710,22 +1710,22 @@ static bool listInterfaces(struct if_info **head, char default_iface[IF_NAMESIZE return true; } -static void send_iface(const int *sock, struct if_info *iface) +static bool send_iface(const int sock, struct if_info *iface) { double tx = 0.0, rx = 0.0; char txp[2] = { 0 }, rxp[2] = { 0 }; format_memory_size(txp, iface->tx_bytes, &tx); format_memory_size(rxp, iface->rx_bytes, &rx); - ssend(*sock, "%s %s %i %.1f%sB %.1f%sB %s %s\n", - iface->name, - iface->carrier ? "UP" : "DOWN", - iface->speed, - tx, txp, rx, rxp, - iface->carrier && iface->ip.v4 ? iface->ip.v4 : "-", - iface->carrier && iface->ip.v6 ? iface->ip.v6 : "-"); + return ssend(sock, "%s %s %i %.1f%sB %.1f%sB %s %s\n", + iface->name, + iface->carrier ? "UP" : "DOWN", + iface->speed, + tx, txp, rx, rxp, + iface->carrier && iface->ip.v4 ? iface->ip.v4 : "-", + iface->carrier && iface->ip.v6 ? iface->ip.v6 : "-"); } -void getInterfaces(const int *sock) +void getInterfaces(const int sock) { // Get interface with default route in_addr_t gw = 0; @@ -1736,7 +1736,7 @@ void getInterfaces(const int *sock) struct if_info *ifinfo = NULL; if(!listInterfaces(&ifinfo, default_iface)) { - ssend(*sock, "ERROR"); + ssend(sock, "ERROR"); return; } diff --git a/src/api/api.h b/src/api/api.h index afa1a935..5867859d 100644 --- a/src/api/api.h +++ b/src/api/api.h @@ -11,29 +11,29 @@ #define API_H // Statistic methods -void getStats(const int *sock); -void getOverTime(const int *sock); -void getTopDomains(const char *client_message, const int *sock); -void getTopClients(const char *client_message, const int *sock); -void getUpstreamDestinations(const char *client_message, const int *sock); -void getQueryTypes(const int *sock); -void getAllQueries(const char *client_message, const int *sock); -void getRecentBlocked(const char *client_message, const int *sock); -void getClientsOverTime(const int *sock); -void getClientNames(const int *sock); +void getStats(const int sock); +void getOverTime(const int sock); +void getTopDomains(const char *client_message, const int sock); +void getTopClients(const char *client_message, const int sock); +void getUpstreamDestinations(const char *client_message, const int sock); +void getQueryTypes(const int sock); +void getAllQueries(const char *client_message, const int sock); +void getRecentBlocked(const char *client_message, const int sock); +void getClientsOverTime(const int sock); +void getClientNames(const int sock); // FTL methods -void getClientID(const int *sock); -void getVersion(const int *sock); -void getDBstats(const int *sock); -void getUnknownQueries(const int *sock); -void getMAXLOGAGE(const int *sock); -void getGateway(const int *sock); -void getInterfaces(const int *sock); +void getClientID(const int sock); +void getVersion(const int sock); +void getDBstats(const int sock); +void getUnknownQueries(const int sock); +void getMAXLOGAGE(const int sock); +void getGateway(const int sock); +void getInterfaces(const int sock); // DNS resolver methods (dnsmasq_interface.c) -void getCacheInformation(const int *sock); -void getDNSport(const int *sock); +void getCacheInformation(const int sock); +void getDNSport(const int sock); // MessagePack serialization helpers void pack_eom(const int sock); @@ -48,6 +48,6 @@ bool pack_str32(const int sock, const char *string); void pack_map16_start(const int sock, const uint16_t length); // DHCP lease management -void delete_lease(const char *client_message, const int *sock); +void delete_lease(const char *client_message, const int sock); #endif // API_H diff --git a/src/api/request.c b/src/api/request.c index 5ebc28cc..88bb5a01 100644 --- a/src/api/request.c +++ b/src/api/request.c @@ -26,7 +26,7 @@ bool __attribute__((pure)) command(const char *client_message, const char* cmd) return strstr(client_message, cmd) != NULL; } -void process_request(const char *client_message, int *sock) +bool process_request(const char *client_message, const int sock) { char EOT[2]; EOT[0] = 0x04; @@ -191,21 +191,15 @@ void process_request(const char *client_message, int *sock) if(command(client_message, ">quit") || command(client_message, EOT)) { if(config.debug & DEBUG_API) - logg("Received >quit or EOT on socket %d", *sock); - processed = true; - close(*sock); - *sock = 0; + logg("Received >quit or EOT on socket %d", sock); + return true; } if(!processed) - { - ssend(*sock, "unknown command: %s\n", client_message); - } + ssend(sock, "unknown command: %s\n", client_message); - // End of queryable commands - if(*sock != 0) - { - // Send EOM - seom(*sock); - } + // End of queryable commands: Send EOM + seom(sock); + + return false; } diff --git a/src/api/request.h b/src/api/request.h index ab2baa0a..5e57f582 100644 --- a/src/api/request.h +++ b/src/api/request.h @@ -10,7 +10,7 @@ #ifndef REQUEST_H #define REQUEST_H -void process_request(const char *client_message, int *sock); +bool process_request(const char *client_message, const int sock); bool command(const char *client_message, const char* cmd) __attribute__((pure)); #endif //REQUEST_H diff --git a/src/api/socket.c b/src/api/socket.c index 6495aa14..c3d62f86 100644 --- a/src/api/socket.c +++ b/src/api/socket.c @@ -31,9 +31,7 @@ #define BACKLOG 5 // File descriptors -int socketfd = 0, telnetfd4 = 0, telnetfd6 = 0; bool dualstack = false; -bool ipv4telnet = false, ipv6telnet = false, sock_avail = false; bool istelnet[MAXCONNS]; void saveport(int port) @@ -59,12 +57,11 @@ void saveport(int port) } } -static bool bind_to_telnet_port_IPv4(int *socketdescriptor) +static int bind_to_telnet_port_IPv4(void) { // IPv4 socket - *socketdescriptor = socket(AF_INET, SOCK_STREAM, 0); - - if(*socketdescriptor < 0) + const int socketdescriptor = socket(AF_INET, SOCK_STREAM, 0); + if(socketdescriptor < 0) { logg("Error opening IPv4 telnet socket: %s (%i)", strerror(errno), errno); exit(EXIT_FAILURE); @@ -77,7 +74,7 @@ static bool bind_to_telnet_port_IPv4(int *socketdescriptor) // new instance will fail if there were connections open to the previous // instance when you killed it. Those connections will hold the TCP port in // the TIME_WAIT state for 30-120 seconds, so you fall into case 1 above. - if(setsockopt(*socketdescriptor, SOL_SOCKET, SO_REUSEADDR, &(int){ 1 }, sizeof(int)) != 0) + if(setsockopt(socketdescriptor, SOL_SOCKET, SO_REUSEADDR, &(int){ 1 }, sizeof(int)) != 0) logg("WARN: allowing re-binding (IPv6) failed: %s", strerror(errno)); struct sockaddr_in serv_addr4; @@ -92,29 +89,28 @@ static bool bind_to_telnet_port_IPv4(int *socketdescriptor) // Bind to IPv4 port serv_addr4.sin_port = htons(config.port); - if(bind(*socketdescriptor, (struct sockaddr *) &serv_addr4, sizeof(serv_addr4)) < 0) + if(bind(socketdescriptor, (struct sockaddr *) &serv_addr4, sizeof(serv_addr4)) < 0) { logg("Error listening on IPv4 port %i: %s (%i)", config.port, strerror(errno), errno); - return false; + return -1; } // The listen system call allows the process to listen on the socket for connections - if(listen(*socketdescriptor, BACKLOG) == -1) + if(listen(socketdescriptor, BACKLOG) == -1) { logg("Error listening on IPv4 socket: %s (%i)", strerror(errno), errno); - return false; + return -1; } logg("Listening on port %i for incoming IPv4 telnet connections", config.port); - return true; + return socketdescriptor; } -static bool bind_to_telnet_port_IPv6(int *socketdescriptor) +static int bind_to_telnet_port_IPv6(void) { // IPv6 socket - *socketdescriptor = socket(AF_INET6, SOCK_STREAM, 0); - - if(*socketdescriptor < 0) + const int socketdescriptor = socket(AF_INET6, SOCK_STREAM, 0); + if(socketdescriptor < 0) { logg("Error opening IPv6 telnet socket: %s (%i)", strerror(errno), errno); exit(EXIT_FAILURE); @@ -124,7 +120,7 @@ static bool bind_to_telnet_port_IPv6(int *socketdescriptor) // stricted to sending and receiving IPv6 packets only. In this // case, an IPv4 and an IPv6 application can bind to a single port // at the same time. - if(setsockopt(*socketdescriptor, IPPROTO_IPV6, IPV6_V6ONLY, &(int){ 1 }, sizeof(int)) != 0) + if(setsockopt(socketdescriptor, IPPROTO_IPV6, IPV6_V6ONLY, &(int){ 1 }, sizeof(int)) != 0) logg("WARN: setting socket to IPv6-only failed: %s", strerror(errno)); // Set SO_REUSEADDR to allow re-binding to the port that has been used @@ -134,7 +130,7 @@ static bool bind_to_telnet_port_IPv6(int *socketdescriptor) // new instance will fail if there were connections open to the previous // instance when you killed it. Those connections will hold the TCP port in // the TIME_WAIT state for 30-120 seconds, so you fall into case 1 above. - if(setsockopt(*socketdescriptor, SOL_SOCKET, SO_REUSEADDR, &(int){ 1 }, sizeof(int)) != 0) + if(setsockopt(socketdescriptor, SOL_SOCKET, SO_REUSEADDR, &(int){ 1 }, sizeof(int)) != 0) logg("WARN: allowing re-binding (IPv6) failed: %s", strerror(errno)); struct sockaddr_in6 serv_addr; @@ -156,31 +152,30 @@ static bool bind_to_telnet_port_IPv6(int *socketdescriptor) // Bind to IPv6 socket serv_addr.sin6_port = htons(config.port); - if(bind(*socketdescriptor, (struct sockaddr *) &serv_addr, sizeof(serv_addr)) < 0) + if(bind(socketdescriptor, (struct sockaddr *) &serv_addr, sizeof(serv_addr)) < 0) { logg("Error listening on IPv6 port %i: %s (%i)", config.port, strerror(errno), errno); - return false; + return -1; } // The listen system call allows the process to listen on the socket for connections - if(listen(*socketdescriptor, BACKLOG) == -1) + if(listen(socketdescriptor, BACKLOG) == -1) { logg("Error listening on IPv6 socket: %s (%i)", strerror(errno), errno); - return false; + return -1; } logg("Listening on port %i for incoming IPv6 telnet connections", config.port); - return true; + return socketdescriptor; } -static bool bind_to_unix_socket(int *socketdescriptor) +static int bind_to_unix_socket(void) { - *socketdescriptor = socket(AF_LOCAL, SOCK_STREAM, 0); - - if(*socketdescriptor < 0) + const int socketdescriptor = socket(AF_LOCAL, SOCK_STREAM, 0); + if(socketdescriptor < 0) { logg("WARNING: Error opening Unix socket."); - return false; + return -1; } // Make sure unix socket file handle does not exist, if it exists, remove it @@ -198,21 +193,21 @@ static bool bind_to_unix_socket(int *socketdescriptor) // Bind to Unix socket handle errno = 0; - if(bind(*socketdescriptor, (struct sockaddr *) &address, sizeof (address)) != 0) + if(bind(socketdescriptor, (struct sockaddr *) &address, sizeof (address)) != 0) { logg("WARNING: Cannot bind on Unix socket %s: %s (%i)", FTLfiles.socketfile, strerror(errno), errno); - return false; + return -1; } // The listen system call allows the process to listen on the Unix socket for connections - if(listen(*socketdescriptor, BACKLOG) == -1) + if(listen(socketdescriptor, BACKLOG) == -1) { logg("WARNING: Cannot listen on Unix socket: %s (%i)", strerror(errno), errno); - return false; + return -1; } logg("Listening on Unix socket"); - return true; + return socketdescriptor; } void seom(const int sock) @@ -223,7 +218,7 @@ void seom(const int sock) pack_eom(sock); } -void __attribute__ ((format (gnu_printf, 2, 3))) ssend(const int sock, const char *format, ...) +bool __attribute__ ((format (gnu_printf, 5, 6))) _ssend(const int sock, const char *file, const char *func, const int line, const char *format, ...) { char *buffer; va_list args; @@ -232,68 +227,16 @@ void __attribute__ ((format (gnu_printf, 2, 3))) ssend(const int sock, const cha va_end(args); if(bytes > 0 && buffer != NULL) { - write(sock, buffer, bytes); + FTLwrite(sock, buffer, bytes, short_path(file), func, line); free(buffer); } -} - -static int listener(const int sockfd, const char type) -{ - socklen_t socklen = 0; - struct sockaddr_un un_addr; - struct sockaddr_in in4_addr; - struct sockaddr_in6 in6_addr; - - switch(type) - { - case 0: // Unix socket - memset(&un_addr, 0, sizeof(un_addr)); - socklen = sizeof(un_addr); - return accept(sockfd, (struct sockaddr *) &un_addr, &socklen); - - case 4: // Internet socket (IPv4) - memset(&in4_addr, 0, sizeof(in4_addr)); - socklen = sizeof(un_addr); - return accept(sockfd, (struct sockaddr *) &in4_addr, &socklen); - - case 6: // Internet socket (IPv6) - memset(&in6_addr, 0, sizeof(in6_addr)); - socklen = sizeof(un_addr); - return accept(sockfd, (struct sockaddr *) &in6_addr, &socklen); - - default: // Should not happen - logg("Cannot listen on type %i connection, code error!", type); - exit(EXIT_FAILURE); - } -} - -void close_telnet_socket(void) -{ - // Using global variable here - if(telnetfd4 != 0) - close(telnetfd4); - if(telnetfd6 != 0) - close(telnetfd6); -} - -void close_unix_socket(bool unlink_file) -{ - // Using global variable here - if(sock_avail != 0) - close(socketfd); - - // The process has to take care of unlinking the socket file description - // on exit - if(unlink_file) - { - unlink(FTLfiles.socketfile); - } + return errno == 0; } static void *telnet_connection_handler_thread(void *socket_desc) { //Get the socket descriptor - int sock = *(int*)socket_desc; + const int sock = *(int*)socket_desc; // Set connection type to telnet istelnet[sock] = true; @@ -324,7 +267,7 @@ static void *telnet_connection_handler_thread(void *socket_desc) } if(config.debug & DEBUG_API) - logg("New telnet thread for socket %d", *(int*)socket_desc); + logg("New telnet thread for socket %d", sock); // Receive from client ssize_t n; @@ -338,7 +281,7 @@ static void *telnet_connection_handler_thread(void *socket_desc) if(message == NULL) { if(config.debug & DEBUG_API) - logg("Break in telnet thread for socket %d: Memory error", *(int*)socket_desc); + logg("Break in telnet thread for socket %d: Memory error", sock); break; } @@ -346,31 +289,23 @@ static void *telnet_connection_handler_thread(void *socket_desc) memset(client_message, 0, sizeof client_message); // Process received message - process_request(message, &sock); + const bool eom = process_request(message, sock); free(message); - - if(sock == 0) - { - // Client disconnected by sending EOT or ">quit" - if(config.debug & DEBUG_API) - logg("Break in telnet thread for socket %d: Client disconnected", *(int*)socket_desc); - break; - } + if(eom) break; } else if(n == -1) { if(config.debug & DEBUG_API) - logg("Break in telnet thread for socket %d: No data received", *(int*)socket_desc); + logg("Break in telnet thread for socket %d: No data received", sock); break; } } if(config.debug & DEBUG_API) - logg("Terminating telnet thread for socket %d", *(int*)socket_desc); + logg("Terminating telnet thread for socket %d", sock); - // Free the socket pointer - if(sock != 0) - close(sock); + // Close the socket pointer + close(sock); free(socket_desc); // Release thread from list @@ -383,7 +318,7 @@ static void *telnet_connection_handler_thread(void *socket_desc) static void *socket_connection_handler_thread(void *socket_desc) { //Get the socket descriptor - int sock = *(int*)socket_desc; + const int sock = *(int*)socket_desc; // Set connection type to not telnet istelnet[sock] = false; @@ -420,7 +355,7 @@ static void *socket_connection_handler_thread(void *socket_desc) // Receive from client ssize_t n; - while((n = recv(sock,client_message,SOCKETBUFFERLEN-1, 0))) + while((n = recv(sock, client_message, SOCKETBUFFERLEN-1, 0))) { if (n > 0 && n < SOCKETBUFFERLEN) { @@ -433,14 +368,9 @@ static void *socket_connection_handler_thread(void *socket_desc) memset(client_message, 0, sizeof client_message); // Process received message - process_request(message, &sock); + const bool eom = process_request(message, sock); free(message); - - if(sock == 0) - { - // Socket connection interrupted by sending EOT or ">quit" - break; - } + if(eom) break; } else if(n == -1) { @@ -448,9 +378,8 @@ static void *socket_connection_handler_thread(void *socket_desc) } } - // Free the socket pointer - if(sock != 0) - close(sock); + // Close the socket pointer + close(sock); free(socket_desc); // Release thread from list @@ -474,43 +403,58 @@ void *telnet_listening_thread_IPv4(void *args) thread_names[TELNETv4] = "telnet-IPv4"; prctl(PR_SET_NAME, thread_names[TELNETv4], 0, 0, 0); - // Initialize IPv4 telnet socket - ipv4telnet = bind_to_telnet_port_IPv4(&telnetfd4); - if(!ipv4telnet) - return NULL; - - thread_cancellable[TELNETv4] = true; - - // Listen as long as this thread is not canceled while(!killed) { - // Look for new clients that want to connect - const int csck = listener(telnetfd4, 4); - if(csck == -1) + // Initialize IPv4 telnet socket´ + const int telnetfd4 = bind_to_telnet_port_IPv4(); + if(telnetfd4 < 0) + return NULL; + + if(config.debug & DEBUG_API) + logg("Telnet-IPv4 listener accepting on fd %d", telnetfd4); + + thread_cancellable[TELNETv4] = true; + + // Listen as long as this thread is not canceled + int errors = 0; + while(!killed) { - logg("IPv4 telnet error: %s (%i)", strerror(errno), errno); - continue; + // Look for new clients that want to connect + const int csck = accept(telnetfd4, NULL, NULL); + if(csck == -1) + { + logg("IPv4 telnet error: %s (%i, sockfd: %d)", strerror(errno), errno, telnetfd4); + if(errors++ > 20) + break; + continue; + } + + if(config.debug & DEBUG_API) + logg("Accepting new telnet connection at socket %d", csck); + + // Allocate memory used to transport client socket ID to client listening thread + int *newsock; + newsock = calloc(1,sizeof(int)); + if(newsock == NULL) break; + *newsock = csck; + + pthread_t telnet_connection_thread; + // Create a new thread + if(pthread_create( &telnet_connection_thread, &attr, telnet_connection_handler_thread, (void*) newsock ) != 0) + { + // Log the error code description + logg("WARNING: Unable to open telnet processing thread: %s", strerror(errno)); + } } if(config.debug & DEBUG_API) - logg("Accepting new telnet connection at socket %d", csck); + logg("Telnet-IPv4 listener closed (fd %d)", telnetfd4); - // Allocate memory used to transport client socket ID to client listening thread - int *newsock; - newsock = calloc(1,sizeof(int)); - if(newsock == NULL) break; - *newsock = csck; - - pthread_t telnet_connection_thread; - // Create a new thread - if(pthread_create( &telnet_connection_thread, &attr, telnet_connection_handler_thread, (void*) newsock ) != 0) - { - // Log the error code description - logg("WARNING: Unable to open telnet processing thread: %s", strerror(errno)); - } + // Close telnet socket + close(telnetfd4); + thread_sleepms(TELNETv4, 1000); } - if(config.debug & DEBUG_API) - logg("Terminating IPv4 telnet thread"); + logg("Terminating IPv4 telnet thread"); return NULL; } @@ -532,38 +476,53 @@ void *telnet_listening_thread_IPv6(void *args) if(!ipv6_available()) return NULL; - ipv6telnet = bind_to_telnet_port_IPv6(&telnetfd6); - if(!ipv6telnet) - return NULL; - - thread_cancellable[TELNETv6] = true; - - // Listen as long as this thread is not canceled while(!killed) { - // Look for new clients that want to connect - const int csck = listener(telnetfd6, 6); - if(csck == -1) + const int telnetfd6 = bind_to_telnet_port_IPv6(); + if(telnetfd6 < 0) + return NULL; + + if(config.debug & DEBUG_API) + logg("Telnet-IPv6 listener accepting on fd %d", telnetfd6); + + thread_cancellable[TELNETv6] = true; + + // Listen as long as this thread is not canceled + int errors = 0; + while(!killed) { - logg("IPv6 telnet error: %s (%i)", strerror(errno), errno); - continue; + // Look for new clients that want to connect + const int csck = accept(telnetfd6, NULL, NULL); + if(csck == -1) + { + logg("IPv6 telnet error: %s (%i, sockfd: %d)", strerror(errno), errno, telnetfd6); + if(errors++ > 20) + break; + continue; + } + + // Allocate memory used to transport client socket ID to client listening thread + int *newsock; + newsock = calloc(1,sizeof(int)); + if(newsock == NULL) break; + *newsock = csck; + + pthread_t telnet_connection_thread; + // Create a new thread + if(pthread_create( &telnet_connection_thread, &attr, telnet_connection_handler_thread, (void*) newsock ) != 0) + { + // Log the error code description + logg("WARNING: Unable to open telnet processing thread: %s", strerror(errno)); + } } - // Allocate memory used to transport client socket ID to client listening thread - int *newsock; - newsock = calloc(1,sizeof(int)); - if(newsock == NULL) break; - *newsock = csck; + if(config.debug & DEBUG_API) + logg("Telnet-IPv6 listener closed (fd %d)", telnetfd6); - pthread_t telnet_connection_thread; - // Create a new thread - if(pthread_create( &telnet_connection_thread, &attr, telnet_connection_handler_thread, (void*) newsock ) != 0) - { - // Log the error code description - logg("WARNING: Unable to open telnet processing thread: %s", strerror(errno)); - } + // Close telnet socket + close(telnetfd6); + thread_sleepms(TELNETv6, 1000); } - logg("Terminating IPv6 telnet thread"); return NULL; } @@ -582,39 +541,53 @@ void *socket_listening_thread(void *args) thread_names[SOCKET] = "telnet-socket"; prctl(PR_SET_NAME, thread_names[SOCKET], 0, 0, 0); - // Return early to avoid CPU spinning if Unix socket is not available - sock_avail = bind_to_unix_socket(&socketfd); - if(!sock_avail) - { - logg("INFO: Unix socket will not be available"); - return NULL; - } - - thread_cancellable[SOCKET] = true; - - // Listen as long as this thread is not canceled while(!killed) { - // Look for new clients that want to connect - const int csck = listener(socketfd, 0); - if(csck < 0) - continue; - - // Allocate memory used to transport client socket ID to client listening thread - int *newsock; - newsock = calloc(1,sizeof(int)); - if(newsock == NULL) break; - *newsock = csck; - - pthread_t socket_connection_thread; - // Create a new thread - if(pthread_create( &socket_connection_thread, &attr, socket_connection_handler_thread, (void*) newsock ) != 0) + // Return early to avoid CPU spinning if Unix socket is not available + const int socketfd = bind_to_unix_socket(); + if(socketfd < 0) { - // Log the error code description - logg("WARNING: Unable to open socket processing thread: %s", strerror(errno)); + logg("INFO: Unix socket will not be available"); + return NULL; } - } + if(config.debug & DEBUG_API) + logg("Telnet-socket listener accepting on fd %d", socketfd); + + thread_cancellable[SOCKET] = true; + + // Listen as long as this thread is not canceled + while(!killed) + { + // Look for new clients that want to connect + const int csck = accept(socketfd, NULL, NULL); + if(csck < 0) + continue; + + // Allocate memory used to transport client socket ID to client listening thread + int *newsock; + newsock = calloc(1,sizeof(int)); + if(newsock == NULL) break; + *newsock = csck; + + pthread_t socket_connection_thread; + // Create a new thread + if(pthread_create( &socket_connection_thread, &attr, socket_connection_handler_thread, (void*) newsock ) != 0) + { + // Log the error code description + logg("WARNING: Unable to open socket processing thread: %s", strerror(errno)); + } + } + + if(config.debug & DEBUG_API) + logg("Telnet-socket listener closed (fd %d)", socketfd); + + // Close Unix socket + close(socketfd); + } + // The process has to take care of unlinking the socket file description + // on exit + unlink(FTLfiles.socketfile); logg("Terminating socket thread"); return NULL; } diff --git a/src/api/socket.h b/src/api/socket.h index bfb7150f..8ee6a173 100644 --- a/src/api/socket.h +++ b/src/api/socket.h @@ -11,17 +11,16 @@ #define SOCKET_H void saveport(int port); -void close_telnet_socket(void); void close_unix_socket(bool unlink_file); void seom(const int sock); -void ssend(const int sock, const char *format, ...) __attribute__ ((format (gnu_printf, 2, 3))); +#define ssend(sock, format, ...) _ssend(sock, __FILE__, __FUNCTION__, __LINE__, format, ##__VA_ARGS__) +bool _ssend(const int sock, const char *file, const char *func, const int line, const char *format, ...) __attribute__ ((format (gnu_printf, 5, 6))); void *telnet_listening_thread_IPv4(void *args); void *telnet_listening_thread_IPv6(void *args); void *socket_listening_thread(void *args); bool ipv6_available(void); void bind_sockets(void); -extern bool ipv4telnet, ipv6telnet; extern bool istelnet[MAXCONNS]; #endif //SOCKET_H diff --git a/src/daemon.c b/src/daemon.c index a1b26012..c316f282 100644 --- a/src/daemon.c +++ b/src/daemon.c @@ -14,7 +14,7 @@ #include "log.h" // sleepms() #include "timers.h" -// close_telnet_socket() +// saveport() #include "api/socket.h" // gravityDB_close() #include "database/gravity-db.h" @@ -277,10 +277,6 @@ void cleanup(const int ret) lock_shm(); gravityDB_close(); unlock_shm(); - - // Close sockets and delete Unix socket file handle - close_telnet_socket(); - close_unix_socket(true); } // Empty API port file, port 0 = truncate file diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c index 41c122d1..3c989f5f 100644 --- a/src/dnsmasq_interface.c +++ b/src/dnsmasq_interface.c @@ -2851,11 +2851,11 @@ static char *get_ptrname(struct in_addr *addr) } // int cache_inserted, cache_live_freed are defined in dnsmasq/cache.c -void getCacheInformation(const int *sock) +void getCacheInformation(const int sock) { struct cache_info ci; get_dnsmasq_cache_info(&ci); - ssend(*sock,"cache-size: %i\ncache-live-freed: %i\ncache-inserted: %i\nipv4: %i\nipv6: %i\nsrv: %i\ncname: %i\nds: %i\ndnskey: %i\nother: %i\nexpired: %i\nimmortal: %i\n", + ssend(sock, "cache-size: %i\ncache-live-freed: %i\ncache-inserted: %i\nipv4: %i\nipv6: %i\nsrv: %i\ncname: %i\nds: %i\ndnskey: %i\nother: %i\nexpired: %i\nimmortal: %i\n", daemon->cachesize, daemon->metrics[METRIC_DNS_CACHE_LIVE_FREED], daemon->metrics[METRIC_DNS_CACHE_INSERTED], @@ -3133,15 +3133,6 @@ void FTL_TCP_worker_created(const int confd) if(config.debug != 0) logg("Reopening Gravity database for this fork"); gravityDB_forked(); - - // Children inherit file descriptors from their parents - // We don't need them in the forks, so we clean them up - if(config.debug != 0) - logg("Closing Telnet socket for this fork"); - close_telnet_socket(); - if(config.debug != 0) - logg("Closing Unix socket for this fork"); - close_unix_socket(false); } bool FTL_unlink_DHCP_lease(const char *ipaddr) diff --git a/src/dnsmasq_interface.h b/src/dnsmasq_interface.h index 7abca12e..a35b7381 100644 --- a/src/dnsmasq_interface.h +++ b/src/dnsmasq_interface.h @@ -15,7 +15,6 @@ #include "edns0.h" -extern int socketfd, telnetfd4, telnetfd6; extern unsigned char* pihole_privacylevel; enum protocol { TCP, UDP, INTERNAL }; diff --git a/src/syscalls/accept.c b/src/syscalls/accept.c index 535f18aa..5c90d2bf 100644 --- a/src/syscalls/accept.c +++ b/src/syscalls/accept.c @@ -15,7 +15,7 @@ #undef accept int FTLaccept(int sockfd, struct sockaddr *addr, socklen_t *addrlen, const char *file, const char *func, const int line) { - int ret = 0; + int ret = 0; do { // Reset errno before trying to write @@ -26,11 +26,17 @@ int FTLaccept(int sockfd, struct sockaddr *addr, socklen_t *addrlen, const char // incoming signal while(ret < 0 && errno == EINTR); + // Backup errno value + const int _errno = errno; + // Final error checking (may have failed for some other reason then an // EINTR = interrupted system call) if(ret < 0) logg("WARN: Could not accept() in %s() (%s:%i): %s", - func, file, line, strerror(errno)); + func, file, line, strerror(errno)); - return ret; + // Restore errno value + errno = _errno; + + return ret; } \ No newline at end of file diff --git a/src/syscalls/calloc.c b/src/syscalls/calloc.c index b79ad5fd..54621347 100644 --- a/src/syscalls/calloc.c +++ b/src/syscalls/calloc.c @@ -30,10 +30,17 @@ void* __attribute__((malloc)) __attribute__((alloc_size(1,2))) FTLcalloc(const s // an incoming signal while(ptr == NULL && errno == EINTR); + // Backup errno value + const int _errno = errno; + // Handle other errors than EINTR if(ptr == NULL) logg("FATAL: Memory allocation (%zu x %zu) failed in %s() (%s:%i)", nmemb, size, func, file, line); + // Restore errno value + errno = _errno; + + // Return memory pointer return ptr; } diff --git a/src/syscalls/fopen.c b/src/syscalls/fopen.c index f11c4375..d6416533 100644 --- a/src/syscalls/fopen.c +++ b/src/syscalls/fopen.c @@ -28,6 +28,9 @@ FILE *FTLfopen(const char *pathname, const char *mode, const char *file, const c // incoming signal while(file_ptr == NULL && errno == EINTR); + // Backup errno value + const int _errno = errno; + // Final error checking (may have failed for some other reason then an // EINTR = interrupted system call) // The already_writing counter prevents a possible infinite loop @@ -38,5 +41,9 @@ FILE *FTLfopen(const char *pathname, const char *mode, const char *file, const c // Decrement warning counter already_writing--; + // Restore errno value + errno = _errno; + + // Return file pointer return file_ptr; } diff --git a/src/syscalls/ftlallocate.c b/src/syscalls/ftlallocate.c index 91597cb7..56bc3944 100644 --- a/src/syscalls/ftlallocate.c +++ b/src/syscalls/ftlallocate.c @@ -35,5 +35,6 @@ int FTLfallocate(const int fd, const off_t offset, const off_t len, const char * // Set errno ourselves as posix_fallocate doesn't do it errno = ret; + return ret; } diff --git a/src/syscalls/pthread_mutex_lock.c b/src/syscalls/pthread_mutex_lock.c index 2b6c942a..5f66549e 100644 --- a/src/syscalls/pthread_mutex_lock.c +++ b/src/syscalls/pthread_mutex_lock.c @@ -17,7 +17,7 @@ #undef pthread_mutex_lock int FTLpthread_mutex_lock(pthread_mutex_t *__mutex, const char *file, const char *func, const int line) { - ssize_t ret = 0; + ssize_t ret = 0; do { // Reset errno before trying to write @@ -28,11 +28,17 @@ int FTLpthread_mutex_lock(pthread_mutex_t *__mutex, const char *file, const char // incoming signal while(ret < 0 && errno == EINTR); + // Backup errno value + const int _errno = errno; + // Final error checking (may have failed for some other reason then an // EINTR = interrupted system call) if(ret < 0) logg("WARN: Could not pthread_mutex_lock() in %s() (%s:%i): %s", - func, file, line, strerror(errno)); + func, file, line, strerror(errno)); - return ret; + // Restore errno value + errno = _errno; + + return ret; } \ No newline at end of file diff --git a/src/syscalls/realloc.c b/src/syscalls/realloc.c index 53df93b1..9bdb3fa5 100644 --- a/src/syscalls/realloc.c +++ b/src/syscalls/realloc.c @@ -34,10 +34,16 @@ void __attribute__((alloc_size(2))) *FTLrealloc(void *ptr_in, const size_t size, // an incoming signal while(ptr_out == NULL && errno == EINTR); + // Backup errno value + const int _errno = errno; + // Handle other errors than EINTR if(ptr_out == NULL) logg("FATAL: Memory reallocation (%p -> %zu) failed in %s() (%s:%i)", ptr_in, size, func, file, line); + // Restore errno value + errno = _errno; + return ptr_out; } \ No newline at end of file diff --git a/src/syscalls/recv.c b/src/syscalls/recv.c index 999edc4c..aff6ea56 100644 --- a/src/syscalls/recv.c +++ b/src/syscalls/recv.c @@ -17,7 +17,7 @@ #undef recv ssize_t FTLrecv(int sockfd, void *buf, size_t len, int flags, const char *file, const char *func, const int line) { - ssize_t ret = 0; + ssize_t ret = 0; do { // Reset errno before trying to write @@ -28,11 +28,17 @@ ssize_t FTLrecv(int sockfd, void *buf, size_t len, int flags, const char *file, // incoming signal while(ret < 0 && errno == EINTR); + // Backup errno value + const int _errno = errno; + // Final error checking (may have failed for some other reason then an // EINTR = interrupted system call) if(ret < 0) logg("WARN: Could not recv() in %s() (%s:%i): %s", - func, file, line, strerror(errno)); + func, file, line, strerror(errno)); - return ret; + // Restore errno value + errno = _errno; + + return ret; } \ No newline at end of file diff --git a/src/syscalls/recvfrom.c b/src/syscalls/recvfrom.c index 322a9727..8db0ac46 100644 --- a/src/syscalls/recvfrom.c +++ b/src/syscalls/recvfrom.c @@ -18,7 +18,7 @@ #undef recvfrom ssize_t FTLrecvfrom(int sockfd, void *buf, size_t len, int flags, struct sockaddr *src_addr, socklen_t *addrlen, const char *file, const char *func, const int line) { - ssize_t ret = 0; + ssize_t ret = 0; do { // Reset errno before trying to write @@ -29,11 +29,17 @@ ssize_t FTLrecvfrom(int sockfd, void *buf, size_t len, int flags, struct sockadd // incoming signal while(ret < 0 && errno == EINTR); + // Backup errno value + const int _errno = errno; + // Final error checking (may have failed for some other reason then an // EINTR = interrupted system call) if(ret < 0) logg("WARN: Could not recvfrom() in %s() (%s:%i): %s", - func, file, line, strerror(errno)); + func, file, line, strerror(errno)); - return ret; + // Restore errno value + errno = _errno; + + return ret; } \ No newline at end of file diff --git a/src/syscalls/select.c b/src/syscalls/select.c index 41c384ca..4e73b6c3 100644 --- a/src/syscalls/select.c +++ b/src/syscalls/select.c @@ -17,7 +17,7 @@ #undef select int FTLselect(int nfds, fd_set *readfds, fd_set *writefds, fd_set *exceptfds, struct timeval *timeout, const char *file, const char *func, const int line) { - int ret = 0; + int ret = 0; do { // Reset errno before trying to write @@ -28,11 +28,17 @@ int FTLselect(int nfds, fd_set *readfds, fd_set *writefds, fd_set *exceptfds, st // incoming signal while(ret < 0 && errno == EINTR); + // Backup errno value + const int _errno = errno; + // Final error checking (may have failed for some other reason then an // EINTR = interrupted system call) if(ret < 0) logg("WARN: Could not select() in %s() (%s:%i): %s", - func, file, line, strerror(errno)); + func, file, line, strerror(errno)); - return ret; + // Restore errno value + errno = _errno; + + return ret; } \ No newline at end of file diff --git a/src/syscalls/sendto.c b/src/syscalls/sendto.c index db5e4ff4..cc7234c9 100644 --- a/src/syscalls/sendto.c +++ b/src/syscalls/sendto.c @@ -18,7 +18,7 @@ #undef sendto ssize_t FTLsendto(int sockfd, void *buf, size_t len, int flags, const struct sockaddr *dest_addr, socklen_t addrlen, const char *file, const char *func, const int line) { - ssize_t ret = 0; + ssize_t ret = 0; do { // Reset errno before trying to write @@ -29,11 +29,17 @@ ssize_t FTLsendto(int sockfd, void *buf, size_t len, int flags, const struct soc // incoming signal while(ret < 0 && errno == EINTR); + // Backup errno value + const int _errno = errno; + // Final error checking (may have failed for some other reason then an // EINTR = interrupted system call) if(ret < 0) logg("WARN: Could not sendto() in %s() (%s:%i): %s", - func, file, line, strerror(errno)); + func, file, line, strerror(errno)); - return ret; + // Restore errno value + errno = _errno; + + return ret; } \ No newline at end of file diff --git a/src/syscalls/vasprintf.c b/src/syscalls/vasprintf.c index 8300a8c5..3ac340e6 100644 --- a/src/syscalls/vasprintf.c +++ b/src/syscalls/vasprintf.c @@ -54,6 +54,9 @@ int FTLvasprintf(const char *file, const char *func, const int line, char **buff stdout, _errno, format, func, file, line); } + // Restore errno value + errno = _errno; + // Return number of written bytes return length; } diff --git a/src/syscalls/vfprintf.c b/src/syscalls/vfprintf.c index b945ae4f..7a5ffae9 100644 --- a/src/syscalls/vfprintf.c +++ b/src/syscalls/vfprintf.c @@ -134,6 +134,9 @@ int FTLvfprintf(FILE *stream, const char *file, const char *func, const int line if(buffer != NULL) free(buffer); + // Restore errno value + errno = _errno; + // Return early, there isn't anything we can do here return length; } @@ -152,6 +155,9 @@ int FTLvfprintf(FILE *stream, const char *file, const char *func, const int line // to an interruption by an incoming signal while(_buffer < buffer && errno == EINTR); + // Backup errno value + _errno = errno; + // Final error checking (may have failed for some other reason then an // EINTR = interrupted system call) if(_buffer < buffer) @@ -163,6 +169,9 @@ int FTLvfprintf(FILE *stream, const char *file, const char *func, const int line // Free allocated memory free(buffer); + // Restore errno value + errno = _errno; + // Return number of written bytes return length; } diff --git a/src/syscalls/vsnprintf.c b/src/syscalls/vsnprintf.c index 2e1b5f08..4f4badfc 100644 --- a/src/syscalls/vsnprintf.c +++ b/src/syscalls/vsnprintf.c @@ -47,6 +47,10 @@ int FTLvsnprintf(const char *file, const char *func, const int line, char *__res stdout, _errno, format, func, file, line); } + + // Restore errno value + errno = _errno; + // Return number of written bytes return length; } diff --git a/src/syscalls/vsprintf.c b/src/syscalls/vsprintf.c index b31ba414..cce0b35f 100644 --- a/src/syscalls/vsprintf.c +++ b/src/syscalls/vsprintf.c @@ -54,6 +54,9 @@ int FTLvsprintf(const char *file, const char *func, const int line, char *__rest stdout, _errno, format, func, file, line); } + // Restore errno value + errno = _errno; + // Return number of written bytes return length; } diff --git a/src/syscalls/write.c b/src/syscalls/write.c index fc7ee2f1..0aba271f 100644 --- a/src/syscalls/write.c +++ b/src/syscalls/write.c @@ -21,26 +21,33 @@ ssize_t FTLwrite(int fd, const void *buf, size_t total, const char *file, const return 0; } - ssize_t ret = 0; - size_t written = 0; + ssize_t ret = 0; + size_t written = 0; do { // Reset errno before trying to write errno = 0; ret = write(fd, buf, total); - if(ret > 0) - written += ret; + if(ret > 0) + written += ret; } // Try to write the remaining content into the stream if - // (a) we haven't written all the data, however, there was no other error - // (b) the last write() call failed due to an interruption by an incoming signal + // (a) we haven't written all the data, however, there was no other error + // (b) the last write() call failed due to an interruption by an incoming signal while((written < total && errno == 0) || (ret < 0 && errno == EINTR)); + // Backup errno value + const int _errno = errno; + // Final error checking (may have failed for some other reason then an // EINTR = interrupted system call) if(written < total) logg("WARN: Could not write() everything in %s() [%s:%i]: %s", - func, file, line, strerror(errno)); + func, file, line, strerror(errno)); - return written; + // Restore errno value + errno = _errno; + + // Return number of written bytes + return written; } \ No newline at end of file From 3ecb14640c07a2a39c323292ba626de785d50c72 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 30 Jul 2022 13:53:02 +0200 Subject: [PATCH 020/218] Major rewrite of the socket processing routines. Before, we launched a new thread for each incoming connection (more than 40 threads were never allowed). This could potentially be used to DoS FTL by opening and closing telnet sessions in very quick succession. The new behavior is to, instead, launch five (compile-time setting) threads per type (5 for telnet IPv4, 5 for telnet IPv6, and 5 for Unix socket communication) and let them handle incoming connections in a FIFO manner. If too many requests are sent to FTL at once, they will simply have to wait until they are accepted. Signed-off-by: DL6ER --- src/api/api.c | 91 +++--- src/api/api.h | 28 +- src/api/request.c | 34 +- src/api/request.h | 2 +- src/api/socket.c | 700 +++++++++++----------------------------- src/api/socket.h | 20 +- src/daemon.c | 20 +- src/daemon.h | 5 +- src/dnsmasq_interface.c | 24 +- src/enums.h | 10 +- 10 files changed, 291 insertions(+), 643 deletions(-) diff --git a/src/api/api.c b/src/api/api.c index 32047a2f..32a270a0 100644 --- a/src/api/api.c +++ b/src/api/api.c @@ -15,7 +15,7 @@ #include "../shmem.h" // read_setupVarsconf() #include "../setupVars.h" -// istelnet() +// ssend() #include "socket.h" // get_FTL_db_filesize() #include "../files.h" @@ -76,7 +76,7 @@ static int __attribute__((pure)) cmpdesc(const void *a, const void *b) return 0; } -void getStats(const int sock) +void getStats(const int sock, const bool istelnet) { const int blocked = blocked_queries(); const int total = counters->queries; @@ -87,7 +87,7 @@ void getStats(const int sock) percentage = 1e2f*blocked/total; // Send domains being blocked - if(istelnet[sock]) { + if(istelnet) { ssend(sock, "domains_being_blocked %i\n", counters->gravity); } else @@ -106,7 +106,7 @@ void getStats(const int sock) activeclients++; } - if(istelnet[sock]) { + if(istelnet) { ssend(sock, "dns_queries_today %i\nads_blocked_today %i\nads_percentage_today %f\n", total, blocked, percentage); ssend(sock, "unique_domains %i\nqueries_forwarded %i\nqueries_cached %i\n", @@ -145,16 +145,16 @@ void getStats(const int sock) } // Send status - if(istelnet[sock]) { + if(istelnet) { ssend(sock, "status %s\n", blockingstatus ? "enabled" : "disabled"); } else pack_uint8(sock, blockingstatus); } -void getOverTime(const int sock) +void getOverTime(const int sock, const bool istelnet) { - if(istelnet[sock]) + if(istelnet) { for(int slot = 0; slot < OVERTIME_SLOTS; slot++) { @@ -185,7 +185,7 @@ void getOverTime(const int sock) } } -void getTopDomains(const char *client_message, const int sock) +void getTopDomains(const char *client_message, const int sock, const bool istelnet) { int temparray[counters->domains][2], count=10, num; bool audit = false, asc = false; @@ -196,7 +196,7 @@ void getTopDomains(const char *client_message, const int sock) get_privacy_level(NULL); if(config.privacylevel >= PRIVACY_HIDE_DOMAINS) { // Always send the total number of domains, but pretend it's 0 - if(!istelnet[sock]) + if(!istelnet) pack_int32(sock, 0); return; @@ -269,7 +269,7 @@ void getTopDomains(const char *client_message, const int sock) } } - if(!istelnet[sock]) + if(!istelnet) { // Send the data required to get the percentage each domain has been blocked / queried if(blocked) @@ -306,7 +306,7 @@ void getTopDomains(const char *client_message, const int sock) if(blocked && showblocked && domain->blockedcount > 0) { - if(istelnet[sock]) + if(istelnet) ssend(sock, "%i %i %s\n", n, domain->blockedcount, getstr(domain->domainpos)); else { if(!pack_str32(sock, getstr(domain->domainpos))) @@ -318,7 +318,7 @@ void getTopDomains(const char *client_message, const int sock) } else if(!blocked && showpermitted && (domain->count - domain->blockedcount) > 0) { - if(istelnet[sock]) + if(istelnet) ssend(sock,"%i %i %s\n",n,(domain->count - domain->blockedcount),getstr(domain->domainpos)); else { @@ -339,7 +339,7 @@ void getTopDomains(const char *client_message, const int sock) clearSetupVarsArray(); } -void getTopClients(const char *client_message, const int sock) +void getTopClients(const char *client_message, const int sock, const bool istelnet) { int temparray[counters->clients][2], count=10, num; @@ -347,7 +347,7 @@ void getTopClients(const char *client_message, const int sock) get_privacy_level(NULL); if(config.privacylevel >= PRIVACY_HIDE_DOMAINS_CLIENTS) { // Always send the total number of clients, but pretend it's 0 - if(!istelnet[sock]) + if(!istelnet) pack_int32(sock, 0); return; @@ -408,7 +408,7 @@ void getTopClients(const char *client_message, const int sock) getSetupVarsArray(excludeclients); } - if(!istelnet[sock]) + if(!istelnet) { // Send the total queries so they can make percentages from this data pack_int32(sock, counters->queries); @@ -450,7 +450,7 @@ void getTopClients(const char *client_message, const int sock) // - the client made at least one query within the most recent 24 hours if(includezeroclients || ccount > 0) { - if(istelnet[sock]) + if(istelnet) ssend(sock,"%i %i %s %s\n", n, ccount, client_ip, client_name); else { @@ -471,7 +471,7 @@ void getTopClients(const char *client_message, const int sock) } -void getUpstreamDestinations(const char *client_message, const int sock) +void getUpstreamDestinations(const char *client_message, const int sock, const bool istelnet) { bool sort = true; int temparray[counters->upstreams][2], sumforwarded = 0; @@ -575,7 +575,7 @@ void getUpstreamDestinations(const char *client_message, const int sock) // - only if percentage > 0.0 for all others (i > 0) if(percentage > 0.0f || i < 0) { - if(istelnet[sock]) + if(istelnet) if(upstream_port != 0) ssend(sock, "%i %.2f %s#%u %s#%u\n", i, percentage, ip, upstream_port, name, upstream_port); @@ -592,7 +592,7 @@ void getUpstreamDestinations(const char *client_message, const int sock) } } -void getQueryTypes(const int sock) +void getQueryTypes(const int sock, const bool istelnet) { int total = 0; for(enum query_types type = TYPE_A; type < TYPE_MAX; type++) @@ -611,7 +611,7 @@ void getQueryTypes(const int sock) } } - if(istelnet[sock]) { + if(istelnet) { ssend(sock, "A (IPv4): %.2f\nAAAA (IPv6): %.2f\nANY: %.2f\nSRV: %.2f\n" "SOA: %.2f\nPTR: %.2f\nTXT: %.2f\nNAPTR: %.2f\n" "MX: %.2f\nDS: %.2f\nRRSIG: %.2f\nDNSKEY: %.2f\n" @@ -657,7 +657,7 @@ void getQueryTypes(const int sock) } } -void getAllQueries(const char *client_message, const int sock) +void getAllQueries(const char *client_message, const int sock, const bool istelnet) { // Exit before processing any data if requested via config setting get_privacy_level(NULL); @@ -1053,7 +1053,7 @@ void getAllQueries(const char *client_message, const int sock) delay = 0UL; } - if(istelnet[sock]) + if(istelnet) { ssend(sock,"%lli %s %s %s %i %i %i %lu %s %i %s#%u \"%s\"", (long long)query->timestamp, @@ -1105,7 +1105,7 @@ void getAllQueries(const char *client_message, const int sock) free(clientid_list); } -void getRecentBlocked(const char *client_message, const int sock) +void getRecentBlocked(const char *client_message, const int sock, const bool istelnet) { int num=1; @@ -1132,7 +1132,7 @@ void getRecentBlocked(const char *client_message, const int sock) if(domain == NULL) continue; - if(istelnet[sock]) + if(istelnet) ssend(sock,"%s\n", domain); else if(!pack_str32(sock, domain)) return; @@ -1146,15 +1146,15 @@ void getRecentBlocked(const char *client_message, const int sock) } } -void getClientID(const int sock) +void getClientID(const int sock, const bool istelnet) { - if(istelnet[sock]) + if(istelnet) ssend(sock,"%i\n", sock); else pack_int32(sock, sock); } -void getVersion(const int sock) +void getVersion(const int sock, const bool istelnet) { const char *commit = GIT_HASH; const char *tag = GIT_TAG; @@ -1165,12 +1165,8 @@ void getVersion(const int sock) memcpy(hash, commit, min((size_t)7, strlen(commit))); if(strlen(tag) > 1) { - if(istelnet[sock]) - ssend( - sock, - "version %s\ntag %s\nbranch %s\nhash %s\ndate %s\n", - version, tag, GIT_BRANCH, hash, GIT_DATE - ); + if(istelnet) + ssend(sock, "version %s\ntag %s\nbranch %s\nhash %s\ndate %s\n", version, tag, GIT_BRANCH, hash, GIT_DATE); else { if(!pack_str32(sock, version) || !pack_str32(sock, (char *) tag) || @@ -1181,12 +1177,8 @@ void getVersion(const int sock) } } else { - if(istelnet[sock]) - ssend( - sock, - "version vDev-%s\ntag %s\nbranch %s\nhash %s\ndate %s\n", - hash, tag, GIT_BRANCH, hash, GIT_DATE - ); + if(istelnet) + ssend(sock, "version vDev-%s\ntag %s\nbranch %s\nhash %s\ndate %s\n", hash, tag, GIT_BRANCH, hash, GIT_DATE); else { char *hashVersion = calloc(6 + strlen(hash), sizeof(char)); if(hashVersion == NULL) return; @@ -1204,7 +1196,7 @@ void getVersion(const int sock) } } -void getDBstats(const int sock) +void getDBstats(const int sock, const bool istelnet) { // Get file details unsigned long long int filesize = get_FTL_db_filesize(); @@ -1213,7 +1205,7 @@ void getDBstats(const int sock) double formatted = 0.0; format_memory_size(prefix, filesize, &formatted); - if(istelnet[sock]) + if(istelnet) ssend(sock, "queries in database: %i\ndatabase filesize: %.2f %sB\nSQLite version: %s\n", get_number_of_queries_in_DB(NULL), formatted, prefix, get_sqlite3_version()); else { @@ -1225,7 +1217,7 @@ void getDBstats(const int sock) } } -void getClientsOverTime(const int sock) +void getClientsOverTime(const int sock, const bool istelnet) { // Exit before processing any data if requested via config setting get_privacy_level(NULL); @@ -1263,7 +1255,7 @@ void getClientsOverTime(const int sock) // Main return loop for(int slot = 0; slot < OVERTIME_SLOTS; slot++) { - if(istelnet[sock]) + if(istelnet) ssend(sock, "%lli", (long long)overTime[slot].timestamp); else pack_int32(sock, (int32_t)overTime[slot].timestamp); @@ -1284,13 +1276,13 @@ void getClientsOverTime(const int sock) continue; const int thisclient = client->overTime[slot]; - if(istelnet[sock]) + if(istelnet) ssend(sock, " %i", thisclient); else pack_int32(sock, thisclient); } - if(istelnet[sock]) + if(istelnet) ssend(sock, "\n"); else pack_int32(sock, -1); @@ -1300,7 +1292,7 @@ void getClientsOverTime(const int sock) clearSetupVarsArray(); } -void getClientNames(const int sock) +void getClientNames(const int sock, const bool istelnet) { // Exit before processing any data if requested via config setting get_privacy_level(NULL); @@ -1353,7 +1345,7 @@ void getClientNames(const int sock) const char *client_ip = getstr(client->ippos); const char *client_name = getstr(client->namepos); - if(istelnet[sock]) + if(istelnet) ssend(sock, "%s %s\n", client_name, client_ip); else { pack_str32(sock, client_name); @@ -1365,7 +1357,7 @@ void getClientNames(const int sock) clearSetupVarsArray(); } -void getUnknownQueries(const int sock) +void getUnknownQueries(const int sock, const bool istelnet) { // Exit before processing any data if requested via config setting get_privacy_level(NULL); @@ -1401,7 +1393,7 @@ void getUnknownQueries(const int sock) // Get client IP string const char *clientIP = getstr(client->ippos); - if(istelnet[sock]) + if(istelnet) ssend(sock, "%lli %i %i %s %s %s %i %s\n", (long long)query->timestamp, queryID, query->id, type, getstr(domain->domainpos), clientIP, query->status, query->flags.complete ? "true" : "false"); else { pack_int32(sock, (int32_t)query->timestamp); @@ -1691,6 +1683,7 @@ static bool listInterfaces(struct if_info **head, char default_iface[IF_NAMESIZE rx_sum += new->rx_bytes; } + closedir(dfd); freeifaddrs(ifap); // Create sum entry only if there is more than one interface diff --git a/src/api/api.h b/src/api/api.h index 5867859d..9e1bbbd8 100644 --- a/src/api/api.h +++ b/src/api/api.h @@ -11,22 +11,22 @@ #define API_H // Statistic methods -void getStats(const int sock); -void getOverTime(const int sock); -void getTopDomains(const char *client_message, const int sock); -void getTopClients(const char *client_message, const int sock); -void getUpstreamDestinations(const char *client_message, const int sock); -void getQueryTypes(const int sock); -void getAllQueries(const char *client_message, const int sock); -void getRecentBlocked(const char *client_message, const int sock); -void getClientsOverTime(const int sock); -void getClientNames(const int sock); +void getStats(const int sock, const bool istelnet); +void getOverTime(const int sock, const bool istelnet); +void getTopDomains(const char *client_message, const int sock, const bool istelnet); +void getTopClients(const char *client_message, const int sock, const bool istelnet); +void getUpstreamDestinations(const char *client_message, const int sock, const bool istelnet); +void getQueryTypes(const int sock, const bool istelnet); +void getAllQueries(const char *client_message, const int sock, const bool istelnet); +void getRecentBlocked(const char *client_message, const int sock, const bool istelnet); +void getClientsOverTime(const int sock, const bool istelnet); +void getClientNames(const int sock, const bool istelnet); // FTL methods -void getClientID(const int sock); -void getVersion(const int sock); -void getDBstats(const int sock); -void getUnknownQueries(const int sock); +void getClientID(const int sock, const bool istelnet); +void getVersion(const int sock, const bool istelnet); +void getDBstats(const int sock, const bool istelnet); +void getUnknownQueries(const int sock, const bool istelnet); void getMAXLOGAGE(const int sock); void getGateway(const int sock); void getInterfaces(const int sock); diff --git a/src/api/request.c b/src/api/request.c index 88bb5a01..45cdb00a 100644 --- a/src/api/request.c +++ b/src/api/request.c @@ -26,7 +26,7 @@ bool __attribute__((pure)) command(const char *client_message, const char* cmd) return strstr(client_message, cmd) != NULL; } -bool process_request(const char *client_message, const int sock) +bool process_request(const char *client_message, const int sock, const bool istelnet) { char EOT[2]; EOT[0] = 0x04; @@ -37,104 +37,104 @@ bool process_request(const char *client_message, const int sock) { processed = true; lock_shm(); - getStats(sock); + getStats(sock, istelnet); unlock_shm(); } else if(command(client_message, ">overTime")) { processed = true; lock_shm(); - getOverTime(sock); + getOverTime(sock, istelnet); unlock_shm(); } else if(command(client_message, ">top-domains") || command(client_message, ">top-ads")) { processed = true; lock_shm(); - getTopDomains(client_message, sock); + getTopDomains(client_message, sock, istelnet); unlock_shm(); } else if(command(client_message, ">top-clients")) { processed = true; lock_shm(); - getTopClients(client_message, sock); + getTopClients(client_message, sock, istelnet); unlock_shm(); } else if(command(client_message, ">forward-dest")) { processed = true; lock_shm(); - getUpstreamDestinations(client_message, sock); + getUpstreamDestinations(client_message, sock, istelnet); unlock_shm(); } else if(command(client_message, ">forward-names")) { processed = true; lock_shm(); - getUpstreamDestinations(">forward-dest unsorted", sock); + getUpstreamDestinations(">forward-dest unsorted", sock, istelnet); unlock_shm(); } else if(command(client_message, ">querytypes")) { processed = true; lock_shm(); - getQueryTypes(sock); + getQueryTypes(sock, istelnet); unlock_shm(); } else if(command(client_message, ">getallqueries")) { processed = true; lock_shm(); - getAllQueries(client_message, sock); + getAllQueries(client_message, sock, istelnet); unlock_shm(); } else if(command(client_message, ">recentBlocked")) { processed = true; lock_shm(); - getRecentBlocked(client_message, sock); + getRecentBlocked(client_message, sock, istelnet); unlock_shm(); } else if(command(client_message, ">clientID")) { processed = true; lock_shm(); - getClientID(sock); + getClientID(sock, istelnet); unlock_shm(); } else if(command(client_message, ">version")) { processed = true; // No lock required - getVersion(sock); + getVersion(sock, istelnet); } else if(command(client_message, ">dbstats")) { processed = true; // No lock required. Access to the database // is guaranteed to be atomic - getDBstats(sock); + getDBstats(sock, istelnet); } else if(command(client_message, ">ClientsoverTime")) { processed = true; lock_shm(); - getClientsOverTime(sock); + getClientsOverTime(sock, istelnet); unlock_shm(); } else if(command(client_message, ">client-names")) { processed = true; lock_shm(); - getClientNames(sock); + getClientNames(sock, istelnet); unlock_shm(); } else if(command(client_message, ">unknown")) { processed = true; lock_shm(); - getUnknownQueries(sock); + getUnknownQueries(sock, istelnet); unlock_shm(); } else if(command(client_message, ">cacheinfo")) @@ -199,7 +199,7 @@ bool process_request(const char *client_message, const int sock) ssend(sock, "unknown command: %s\n", client_message); // End of queryable commands: Send EOM - seom(sock); + seom(sock, istelnet); return false; } diff --git a/src/api/request.h b/src/api/request.h index 5e57f582..ad3ade50 100644 --- a/src/api/request.h +++ b/src/api/request.h @@ -10,7 +10,7 @@ #ifndef REQUEST_H #define REQUEST_H -bool process_request(const char *client_message, const int sock); +bool process_request(const char *client_message, const int sock, const bool istelnet); bool command(const char *client_message, const char* cmd) __attribute__((pure)); #endif //REQUEST_H diff --git a/src/api/socket.c b/src/api/socket.c index c3d62f86..d03c08f0 100644 --- a/src/api/socket.c +++ b/src/api/socket.c @@ -14,6 +14,8 @@ #include "socket.h" #include "request.h" #include "../config.h" +// sleepms() +#include "../timers.h" // global variable killed #include "../signals.h" // API thread storage @@ -30,10 +32,6 @@ // reattempt at connection succeeds. #define BACKLOG 5 -// File descriptors -bool dualstack = false; -bool istelnet[MAXCONNS]; - void saveport(int port) { FILE *f; @@ -57,162 +55,225 @@ void saveport(int port) } } -static int bind_to_telnet_port_IPv4(void) +static int bind_to_telnet_socket(const enum telnet_type type, const char *stype) { - // IPv4 socket - const int socketdescriptor = socket(AF_INET, SOCK_STREAM, 0); + const int socketdescriptor = socket(type == TELNET_SOCK ? AF_LOCAL : (type == TELNETv4 ? AF_INET : AF_INET6), SOCK_STREAM, 0); if(socketdescriptor < 0) { - logg("Error opening IPv4 telnet socket: %s (%i)", strerror(errno), errno); + logg("Error opening %s telnet socket: %s (%i)", stype, strerror(errno), errno); exit(EXIT_FAILURE); } - // Set SO_REUSEADDR to allow re-binding to the port that has been used - // previously by FTL. A common pattern is that you change FTL's - // configuration file and need to restart that server to make it reload - // its configuration. Without SO_REUSEADDR, the bind() call in the restarted - // new instance will fail if there were connections open to the previous - // instance when you killed it. Those connections will hold the TCP port in - // the TIME_WAIT state for 30-120 seconds, so you fall into case 1 above. - if(setsockopt(socketdescriptor, SOL_SOCKET, SO_REUSEADDR, &(int){ 1 }, sizeof(int)) != 0) - logg("WARN: allowing re-binding (IPv6) failed: %s", strerror(errno)); + const size_t addrlen = MAX(sizeof(struct sockaddr_un), MAX(sizeof(struct sockaddr_in), sizeof(struct sockaddr_in6))); + void *address = calloc(1, addrlen); - struct sockaddr_in serv_addr4; - // set all values in the buffer to zero - memset(&serv_addr4, 0, sizeof(serv_addr4)); - serv_addr4.sin_family = AF_INET; - - if(config.socket_listenlocal) - serv_addr4.sin_addr.s_addr = htonl(INADDR_LOOPBACK); - else - serv_addr4.sin_addr.s_addr = INADDR_ANY; - - // Bind to IPv4 port - serv_addr4.sin_port = htons(config.port); - if(bind(socketdescriptor, (struct sockaddr *) &serv_addr4, sizeof(serv_addr4)) < 0) + if(type == TELNETv4 || type == TELNETv6) { - logg("Error listening on IPv4 port %i: %s (%i)", config.port, strerror(errno), errno); + // Set SO_REUSEADDR to allow re-binding to the port that has been used + // previously by FTL. A common pattern is that you change FTL's + // configuration file and need to restart that server to make it reload + // its configuration. Without SO_REUSEADDR, the bind() call in the restarted + // new instance will fail if there were connections open to the previous + // instance when you killed it. Those connections will hold the TCP port in + // the TIME_WAIT state for 30-120 seconds, so you fall into case 1 above. + if(setsockopt(socketdescriptor, SOL_SOCKET, SO_REUSEADDR, &(int){ 1 }, sizeof(int)) != 0) + logg("WARN: allowing re-binding (%s) failed: %s", stype, strerror(errno)); + + if(type == TELNETv6) + { + // If this flag is set to true (nonzero), then the socket is re‐ + // stricted to sending and receiving IPv6 packets only. In this + // case, an IPv4 and an IPv6 application can bind to a single port + // at the same time. + if(setsockopt(socketdescriptor, IPPROTO_IPV6, IPV6_V6ONLY, &(int){ 1 }, sizeof(int)) != 0) + logg("WARN: setting socket to IPv6-only failed: %s", strerror(errno)); + + if(config.socket_listenlocal) + ((struct sockaddr_in6*) address)->sin6_addr = in6addr_loopback; + else + ((struct sockaddr_in6*) address)->sin6_addr = in6addr_any; + + // The bind() system call binds a socket to an address, + // in this case the address of the current host and + // port number on which the server will run. + // convert this to network byte order using the function htons() + // which converts a port number in host byte order to a port number + // in network byte order + + // Bind to IPv6 socket + ((struct sockaddr_in6*) address)->sin6_family = AF_INET6; + ((struct sockaddr_in6*) address)->sin6_port = htons(config.port); + } + else // IPv4 + { + + if(config.socket_listenlocal) + ((struct sockaddr_in*) address)->sin_addr.s_addr = htonl(INADDR_LOOPBACK); + else + ((struct sockaddr_in*) address)->sin_addr.s_addr = INADDR_ANY; + + // Bind to IPv4 port + ((struct sockaddr_in*) address)->sin_family = AF_INET; + ((struct sockaddr_in*) address)->sin_port = htons(config.port); + } + } + else // socket + { + // Make sure unix socket file handle does not exist, if it exists, remove it + unlink(FTLfiles.socketfile); + + ((struct sockaddr_un*) address)->sun_family = AF_LOCAL; + // The sockaddr_un.sum_path may be shorter than the size of the FTLfiles.socketfile + // buffer. Ensure that the string is null-terminated even when the string is too large. + // In case strlen(FTLfiles.socketfile) < sizeof(address.sun_path) [this will virtually + // always be the case], the explicit setting of the last byte to zero is a no-op as + // strncpy() writes additional null bytes to ensure that a total of n bytes are written. + strncpy(((struct sockaddr_un*) address)->sun_path, FTLfiles.socketfile, sizeof(((struct sockaddr_un*) address)->sun_path)); + ((struct sockaddr_un*) address)->sun_path[sizeof(((struct sockaddr_un*) address)->sun_path)-1] = '\0'; + + } + + // Bind to socket + if(bind(socketdescriptor, (struct sockaddr *) address, addrlen) < 0) + { + logg("Error listening on Unix socket %s: %s (%i)", FTLfiles.socketfile, strerror(errno), errno); return -1; } // The listen system call allows the process to listen on the socket for connections if(listen(socketdescriptor, BACKLOG) == -1) { - logg("Error listening on IPv4 socket: %s (%i)", strerror(errno), errno); + logg("Error listening on %s socket: %s (%i)", stype, strerror(errno), errno); return -1; } - logg("Listening on port %i for incoming IPv4 telnet connections", config.port); + logg("Listening on port %i for incoming %s telnet connections", config.port, stype); return socketdescriptor; } -static int bind_to_telnet_port_IPv6(void) +static void *telnet_connection_handler_thread(void *args) { - // IPv6 socket - const int socketdescriptor = socket(AF_INET6, SOCK_STREAM, 0); - if(socketdescriptor < 0) + struct thread_info *tinfo = args; + // Set thread name + char threadname[16] = { 0 }; + snprintf(threadname, sizeof(threadname), "telnet-%s-%i", tinfo->stype, tinfo->tid); + prctl(PR_SET_NAME, threadname, 0, 0, 0); + + // Ensure this thread can be canceled at any time (not only at + // cancellation points) + pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, NULL); + + if(config.debug & DEBUG_API) + logg("Started telnet thread %s", threadname); + + // Listen as long as this thread is not canceled + int errors = 0; + while(!killed) { - logg("Error opening IPv6 telnet socket: %s (%i)", strerror(errno), errno); - exit(EXIT_FAILURE); + // Look for new clients that want to connect + const int csck = accept(tinfo->fd, NULL, NULL); + if(csck == -1) + { + logg("Telnet error in %s: %s (%i, fd: %d)", threadname, strerror(errno), errno, tinfo->fd); + if(errors++ > 20) + break; + sleepms(100); + continue; + } + + // Define buffer for client's message + char client_message[SOCKETBUFFERLEN] ={ 0 }; + + // Receive from client + ssize_t n; + while((n = recv(csck, client_message, SOCKETBUFFERLEN-1, 0))) + { + if (n > 0 && n < SOCKETBUFFERLEN) + { + // Null-terminate client string + client_message[n] = '\0'; + char *message = strdup(client_message); + if(message == NULL) + { + if(config.debug & DEBUG_API) + logg("Break in telnet thread for socket %d/%d: Memory error", tinfo->fd, csck); + break; + } + + // Clear client message receive buffer + memset(client_message, 0, sizeof client_message); + + // Process received message + const bool eom = process_request(message, csck, tinfo->istelnet); + free(message); + if(eom) break; + } + else if(n == -1) + { + if(config.debug & DEBUG_API) + logg("Break in telnet thread for socket %d/%d: No data received", tinfo->fd, csck); + break; + } + } + + // Close client socket + close(csck); } - // If this flag is set to true (nonzero), then the socket is re‐ - // stricted to sending and receiving IPv6 packets only. In this - // case, an IPv4 and an IPv6 application can bind to a single port - // at the same time. - if(setsockopt(socketdescriptor, IPPROTO_IPV6, IPV6_V6ONLY, &(int){ 1 }, sizeof(int)) != 0) - logg("WARN: setting socket to IPv6-only failed: %s", strerror(errno)); + if(config.debug & DEBUG_API) + logg("Terminating telnet thread %s (%d errors)", threadname, errors); - // Set SO_REUSEADDR to allow re-binding to the port that has been used - // previously by FTL. A common pattern is that you change FTL's - // configuration file and need to restart that server to make it reload - // its configuration. Without SO_REUSEADDR, the bind() call in the restarted - // new instance will fail if there were connections open to the previous - // instance when you killed it. Those connections will hold the TCP port in - // the TIME_WAIT state for 30-120 seconds, so you fall into case 1 above. - if(setsockopt(socketdescriptor, SOL_SOCKET, SO_REUSEADDR, &(int){ 1 }, sizeof(int)) != 0) - logg("WARN: allowing re-binding (IPv6) failed: %s", strerror(errno)); - - struct sockaddr_in6 serv_addr; - // set all values in the buffer to zero - memset(&serv_addr, 0, sizeof(serv_addr)); - serv_addr.sin6_family = AF_INET6; - - if(config.socket_listenlocal) - serv_addr.sin6_addr = in6addr_loopback; - else - serv_addr.sin6_addr = in6addr_any; - - // The bind() system call binds a socket to an address, - // in this case the address of the current host and - // port number on which the server will run. - // convert this to network byte order using the function htons() - // which converts a port number in host byte order to a port number - // in network byte order - - // Bind to IPv6 socket - serv_addr.sin6_port = htons(config.port); - if(bind(socketdescriptor, (struct sockaddr *) &serv_addr, sizeof(serv_addr)) < 0) - { - logg("Error listening on IPv6 port %i: %s (%i)", config.port, strerror(errno), errno); - return -1; - } - - // The listen system call allows the process to listen on the socket for connections - if(listen(socketdescriptor, BACKLOG) == -1) - { - logg("Error listening on IPv6 socket: %s (%i)", strerror(errno), errno); - return -1; - } - - logg("Listening on port %i for incoming IPv6 telnet connections", config.port); - return socketdescriptor; + // Free thread-private memory + free(tinfo); + return NULL; } -static int bind_to_unix_socket(void) +void listen_telnet(const enum telnet_type type) { - const int socketdescriptor = socket(AF_LOCAL, SOCK_STREAM, 0); - if(socketdescriptor < 0) + // We will use the attributes object later to start all threads in detached mode + pthread_attr_t attr; + // Initialize thread attributes object with default attribute values + pthread_attr_init(&attr); + // When a detached thread terminates, its resources are automatically released back to + // the system without the need for another thread to join with the terminated thread + pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); + + // Initialize telnet socket + const char *stype = type == TELNET_SOCK ? "socket" : (type == TELNETv4 ? "IPv4" : "IPv6"); + const int fd = bind_to_telnet_socket(type, stype); + if(fd < 0) { - logg("WARNING: Error opening Unix socket."); - return -1; + logg("WARN: Cannot bind to %s telnet socket", stype); + return; } - // Make sure unix socket file handle does not exist, if it exists, remove it - unlink(FTLfiles.socketfile); + if(config.debug & DEBUG_API) + logg("Telnet-%s listener accepting on fd %d", stype, fd); - struct sockaddr_un address; - address.sun_family = AF_LOCAL; - // The sockaddr_un.sum_path may be shorter than the size of the FTLfiles.socketfile - // buffer. Ensure that the string is null-terminated even when the string is too large. - // In case strlen(FTLfiles.socketfile) < sizeof(address.sun_path) [this will virtually - // always be the case], the explicit setting of the last byte to zero is a no-op as - // strncpy() writes additional null bytes to ensure that a total of n bytes are written. - strncpy(address.sun_path, FTLfiles.socketfile, sizeof(address.sun_path)); - address.sun_path[sizeof(address.sun_path)-1] = '\0'; - - // Bind to Unix socket handle - errno = 0; - if(bind(socketdescriptor, (struct sockaddr *) &address, sizeof (address)) != 0) + for(unsigned int i = 0; i < MAX_API_THREADS; i++) { - logg("WARNING: Cannot bind on Unix socket %s: %s (%i)", FTLfiles.socketfile, strerror(errno), errno); - return -1; - } + // Spawn telnet thread + pthread_t telnet_connection_thread; + // Create a private copy of the socket fd for the child thread + struct thread_info *tinfo = calloc(1, sizeof(struct thread_info)); + if(!tinfo) + continue; - // The listen system call allows the process to listen on the Unix socket for connections - if(listen(socketdescriptor, BACKLOG) == -1) - { - logg("WARNING: Cannot listen on Unix socket: %s (%i)", strerror(errno), errno); - return -1; + tinfo->fd = fd; + tinfo->tid = i; + tinfo->istelnet = (type == TELNETv4 || type == TELNETv6); + tinfo->stype = stype; + if(pthread_create(&telnet_connection_thread, &attr, telnet_connection_handler_thread, (void*) tinfo) != 0) + { + // Log the error code description + logg("WARNING: Unable to open telnet processing thread: %s", strerror(errno)); + } } - - logg("Listening on Unix socket"); - return socketdescriptor; } -void seom(const int sock) +void seom(const int sock, const bool istelnet) { - if(istelnet[sock]) + if(istelnet) ssend(sock, "---EOM---\n\n"); else pack_eom(sock); @@ -232,398 +293,3 @@ bool __attribute__ ((format (gnu_printf, 5, 6))) _ssend(const int sock, const ch } return errno == 0; } - -static void *telnet_connection_handler_thread(void *socket_desc) -{ - //Get the socket descriptor - const int sock = *(int*)socket_desc; - // Set connection type to telnet - istelnet[sock] = true; - - // Define buffer for client's message - char client_message[SOCKETBUFFERLEN] = ""; - - // Set thread name - char threadname[16] = { 0 }; - sprintf(threadname, "telnet-%i", sock); - prctl(PR_SET_NAME, threadname, 0, 0, 0); - - // Store TID of this thread - lock_shm(); - unsigned int tid; - for(tid = 0; tid < MAX_API_THREADS; tid++) - { - if(api_threads[tid] == 0) - { - api_threads[tid] = pthread_self(); - break; - } - } - unlock_shm(); - if(tid == MAX_API_THREADS) - { - logg("Not able to spawn new API thread, limit of " str(MAX_API_THREADS) " threads reached."); - return NULL; - } - - if(config.debug & DEBUG_API) - logg("New telnet thread for socket %d", sock); - - // Receive from client - ssize_t n; - while((n = recv(sock, client_message, SOCKETBUFFERLEN-1, 0))) - { - if (n > 0 && n < SOCKETBUFFERLEN) - { - // Null-terminate client string - client_message[n] = '\0'; - char *message = strdup(client_message); - if(message == NULL) - { - if(config.debug & DEBUG_API) - logg("Break in telnet thread for socket %d: Memory error", sock); - break; - } - - // Clear client message receive buffer - memset(client_message, 0, sizeof client_message); - - // Process received message - const bool eom = process_request(message, sock); - free(message); - if(eom) break; - } - else if(n == -1) - { - if(config.debug & DEBUG_API) - logg("Break in telnet thread for socket %d: No data received", sock); - break; - } - } - - if(config.debug & DEBUG_API) - logg("Terminating telnet thread for socket %d", sock); - - // Close the socket pointer - close(sock); - free(socket_desc); - - // Release thread from list - api_threads[tid] = 0; - - return NULL; -} - - -static void *socket_connection_handler_thread(void *socket_desc) -{ - //Get the socket descriptor - const int sock = *(int*)socket_desc; - // Set connection type to not telnet - istelnet[sock] = false; - - // Define buffer for client's message - char client_message[SOCKETBUFFERLEN] = ""; - - // Set thread name - char threadname[16]; - sprintf(threadname, "socket-%i", sock); - prctl(PR_SET_NAME, threadname, 0, 0, 0); - - // Ensure this thread can be canceled at any time (not only at - // cancellation points) - pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, NULL); - - // Store TID of this thread - lock_shm(); - unsigned int tid; - for(tid = 0; tid < MAX_API_THREADS; tid++) - { - if(api_threads[tid] == 0) - { - api_threads[tid] = pthread_self(); - api_tids[tid] = gettid(); - break; - } - } - unlock_shm(); - if(tid == MAX_API_THREADS) - { - logg("Not able to spawn new API thread, limit of " str(MAX_API_THREADS) " threads reached."); - return NULL; - } - - // Receive from client - ssize_t n; - while((n = recv(sock, client_message, SOCKETBUFFERLEN-1, 0))) - { - if (n > 0 && n < SOCKETBUFFERLEN) - { - // Null-terminate client string - client_message[n] = '\0'; - char *message = strdup(client_message); - if(message == NULL) break; - - // Clear client message receive buffer - memset(client_message, 0, sizeof client_message); - - // Process received message - const bool eom = process_request(message, sock); - free(message); - if(eom) break; - } - else if(n == -1) - { - break; - } - } - - // Close the socket pointer - close(sock); - free(socket_desc); - - // Release thread from list - api_threads[tid] = 0; - api_tids[tid] = 0; - - return NULL; -} - -void *telnet_listening_thread_IPv4(void *args) -{ - // We will use the attributes object later to start all threads in detached mode - pthread_attr_t attr; - // Initialize thread attributes object with default attribute values - pthread_attr_init(&attr); - // When a detached thread terminates, its resources are automatically released back to - // the system without the need for another thread to join with the terminated thread - pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); - - // Set thread name - thread_names[TELNETv4] = "telnet-IPv4"; - prctl(PR_SET_NAME, thread_names[TELNETv4], 0, 0, 0); - - while(!killed) - { - // Initialize IPv4 telnet socket´ - const int telnetfd4 = bind_to_telnet_port_IPv4(); - if(telnetfd4 < 0) - return NULL; - - if(config.debug & DEBUG_API) - logg("Telnet-IPv4 listener accepting on fd %d", telnetfd4); - - thread_cancellable[TELNETv4] = true; - - // Listen as long as this thread is not canceled - int errors = 0; - while(!killed) - { - // Look for new clients that want to connect - const int csck = accept(telnetfd4, NULL, NULL); - if(csck == -1) - { - logg("IPv4 telnet error: %s (%i, sockfd: %d)", strerror(errno), errno, telnetfd4); - if(errors++ > 20) - break; - continue; - } - - if(config.debug & DEBUG_API) - logg("Accepting new telnet connection at socket %d", csck); - - // Allocate memory used to transport client socket ID to client listening thread - int *newsock; - newsock = calloc(1,sizeof(int)); - if(newsock == NULL) break; - *newsock = csck; - - pthread_t telnet_connection_thread; - // Create a new thread - if(pthread_create( &telnet_connection_thread, &attr, telnet_connection_handler_thread, (void*) newsock ) != 0) - { - // Log the error code description - logg("WARNING: Unable to open telnet processing thread: %s", strerror(errno)); - } - } - - if(config.debug & DEBUG_API) - logg("Telnet-IPv4 listener closed (fd %d)", telnetfd4); - - // Close telnet socket - close(telnetfd4); - thread_sleepms(TELNETv4, 1000); - } - logg("Terminating IPv4 telnet thread"); - return NULL; -} - -void *telnet_listening_thread_IPv6(void *args) -{ - // We will use the attributes object later to start all threads in detached mode - pthread_attr_t attr; - // Initialize thread attributes object with default attribute values - pthread_attr_init(&attr); - // When a detached thread terminates, its resources are automatically released back to - // the system without the need for another thread to join with the terminated thread - pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); - - // Set thread name - thread_names[TELNETv6] = "telnet-IPv6"; - prctl(PR_SET_NAME, thread_names[TELNETv6], 0, 0, 0); - - // Initialize IPv6 telnet socket but only if IPv6 interfaces are available - if(!ipv6_available()) - return NULL; - - while(!killed) - { - const int telnetfd6 = bind_to_telnet_port_IPv6(); - if(telnetfd6 < 0) - return NULL; - - if(config.debug & DEBUG_API) - logg("Telnet-IPv6 listener accepting on fd %d", telnetfd6); - - thread_cancellable[TELNETv6] = true; - - // Listen as long as this thread is not canceled - int errors = 0; - while(!killed) - { - // Look for new clients that want to connect - const int csck = accept(telnetfd6, NULL, NULL); - if(csck == -1) - { - logg("IPv6 telnet error: %s (%i, sockfd: %d)", strerror(errno), errno, telnetfd6); - if(errors++ > 20) - break; - continue; - } - - // Allocate memory used to transport client socket ID to client listening thread - int *newsock; - newsock = calloc(1,sizeof(int)); - if(newsock == NULL) break; - *newsock = csck; - - pthread_t telnet_connection_thread; - // Create a new thread - if(pthread_create( &telnet_connection_thread, &attr, telnet_connection_handler_thread, (void*) newsock ) != 0) - { - // Log the error code description - logg("WARNING: Unable to open telnet processing thread: %s", strerror(errno)); - } - } - - if(config.debug & DEBUG_API) - logg("Telnet-IPv6 listener closed (fd %d)", telnetfd6); - - // Close telnet socket - close(telnetfd6); - thread_sleepms(TELNETv6, 1000); - } - logg("Terminating IPv6 telnet thread"); - return NULL; -} - -void *socket_listening_thread(void *args) -{ - // We will use the attributes object later to start all threads in detached mode - pthread_attr_t attr; - // Initialize thread attributes object with default attribute values - pthread_attr_init(&attr); - // When a detached thread terminates, its resources are automatically released back to - // the system without the need for another thread to join with the terminated thread - pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); - - // Set thread name - thread_names[SOCKET] = "telnet-socket"; - prctl(PR_SET_NAME, thread_names[SOCKET], 0, 0, 0); - - while(!killed) - { - // Return early to avoid CPU spinning if Unix socket is not available - const int socketfd = bind_to_unix_socket(); - if(socketfd < 0) - { - logg("INFO: Unix socket will not be available"); - return NULL; - } - - if(config.debug & DEBUG_API) - logg("Telnet-socket listener accepting on fd %d", socketfd); - - thread_cancellable[SOCKET] = true; - - // Listen as long as this thread is not canceled - while(!killed) - { - // Look for new clients that want to connect - const int csck = accept(socketfd, NULL, NULL); - if(csck < 0) - continue; - - // Allocate memory used to transport client socket ID to client listening thread - int *newsock; - newsock = calloc(1,sizeof(int)); - if(newsock == NULL) break; - *newsock = csck; - - pthread_t socket_connection_thread; - // Create a new thread - if(pthread_create( &socket_connection_thread, &attr, socket_connection_handler_thread, (void*) newsock ) != 0) - { - // Log the error code description - logg("WARNING: Unable to open socket processing thread: %s", strerror(errno)); - } - } - - if(config.debug & DEBUG_API) - logg("Telnet-socket listener closed (fd %d)", socketfd); - - // Close Unix socket - close(socketfd); - } - // The process has to take care of unlinking the socket file description - // on exit - unlink(FTLfiles.socketfile); - logg("Terminating socket thread"); - return NULL; -} - -bool ipv6_available(void) -{ - struct ifaddrs *allInterfaces; - enum { IPv4, IPv6 }; - int iface[2] = { 0 }; - - // Get all interfaces - if (getifaddrs(&allInterfaces) == 0) - { - struct ifaddrs *interface; - // Loop over interfaces - for (interface = allInterfaces; interface != NULL; interface = interface->ifa_next) - { - const unsigned int flags = interface->ifa_flags; - const struct sockaddr *addr = interface->ifa_addr; - - // Check only for up and running IPv4, IPv6 interfaces - if ((flags & (IFF_UP|IFF_RUNNING)) && addr != NULL) - { - iface[addr->sa_family == AF_INET6 ? IPv6 : IPv4]++; - - if(config.debug & DEBUG_NETWORKING) - logg("Interface %s is %s", interface->ifa_name, addr->sa_family == AF_INET6 ? "IPv6" : "IPv4"); - } - } - freeifaddrs(allInterfaces); - } - - if(config.debug & DEBUG_NETWORKING) - { - logg("Found %i IPv4 and %i IPv6 capable interfaces", iface[IPv4], iface[IPv6]); - } - - return (iface[IPv6] > 0); -} diff --git a/src/api/socket.h b/src/api/socket.h index 8ee6a173..4ee4404b 100644 --- a/src/api/socket.h +++ b/src/api/socket.h @@ -10,17 +10,21 @@ #ifndef SOCKET_H #define SOCKET_H +// enum telnet_type +#include "../enums.h" + +struct thread_info { + int fd; + int tid; + bool istelnet; + const char *stype; +}; + void saveport(int port); void close_unix_socket(bool unlink_file); -void seom(const int sock); +void seom(const int sock, const bool istelnet); #define ssend(sock, format, ...) _ssend(sock, __FILE__, __FUNCTION__, __LINE__, format, ##__VA_ARGS__) bool _ssend(const int sock, const char *file, const char *func, const int line, const char *format, ...) __attribute__ ((format (gnu_printf, 5, 6))); -void *telnet_listening_thread_IPv4(void *args); -void *telnet_listening_thread_IPv6(void *args); -void *socket_listening_thread(void *args); -bool ipv6_available(void); -void bind_sockets(void); - -extern bool istelnet[MAXCONNS]; +void listen_telnet(const enum telnet_type type); #endif //SOCKET_H diff --git a/src/daemon.c b/src/daemon.c index c316f282..1c62a7ed 100644 --- a/src/daemon.c +++ b/src/daemon.c @@ -29,8 +29,7 @@ #include pthread_t threads[THREADS_MAX] = { 0 }; -pthread_t api_threads[MAX_API_THREADS] = { 0 }; -pid_t api_tids[MAX_API_THREADS] = { 0 }; +pthread_t api_threads[TELNET_MAX][MAX_API_THREADS] = {{ 0 }}; bool resolver_ready = false; void go_daemon(void) @@ -261,16 +260,15 @@ void cleanup(const int ret) terminate_threads(); // Cancel and join possibly still running API worker threads - for(unsigned int tid = 0; tid < MAX_API_THREADS; tid++) + for(enum telnet_type tt = 0; tt < TELNET_MAX; tt++) { - // Skip if this is an unused slot - if(api_threads[tid] == 0) - continue; - - // Otherwise, cancel and join the thread - logg("Joining API worker thread %d", tid); - pthread_cancel(api_threads[tid]); - pthread_join(api_threads[tid], NULL); + for(unsigned int tid = 0; tid < MAX_API_THREADS; tid++) + { + // Otherwise, cancel and join the thread + logg("Joining API worker thread %d/%d", tt,tid); + pthread_cancel(api_threads[tt][tid]); + pthread_join(api_threads[tt][tid], NULL); + } } // Close database connection diff --git a/src/daemon.h b/src/daemon.h index 12b49942..edcd9227 100644 --- a/src/daemon.h +++ b/src/daemon.h @@ -12,9 +12,8 @@ #include "enums.h" extern pthread_t threads[THREADS_MAX]; -#define MAX_API_THREADS 40 -extern pthread_t api_threads[MAX_API_THREADS]; -extern pid_t api_tids[MAX_API_THREADS]; +#define MAX_API_THREADS 5 +extern pthread_t api_threads[TELNET_MAX][MAX_API_THREADS]; void go_daemon(void); void savepid(void); diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c index 3c989f5f..dddd4e89 100644 --- a/src/dnsmasq_interface.c +++ b/src/dnsmasq_interface.c @@ -2683,26 +2683,10 @@ void FTL_fork_and_bind_sockets(struct passwd *ent_pw) // Initialize thread attributes object with default attribute values pthread_attr_init(&attr); - // Start TELNET IPv4 thread - if(pthread_create( &threads[TELNETv4], &attr, telnet_listening_thread_IPv4, NULL ) != 0) - { - logg("Unable to open IPv4 telnet listening thread. Exiting..."); - exit(EXIT_FAILURE); - } - - // Start TELNET IPv6 thread - if(pthread_create( &threads[TELNETv6], &attr, telnet_listening_thread_IPv6, NULL ) != 0) - { - logg("Unable to open IPv6 telnet listening thread. Exiting..."); - exit(EXIT_FAILURE); - } - - // Start SOCKET thread - if(pthread_create( &threads[SOCKET], &attr, socket_listening_thread, NULL ) != 0) - { - logg("Unable to open Unix socket listening thread. Exiting..."); - exit(EXIT_FAILURE); - } + // Start listening on telnet-like interface + listen_telnet(TELNETv4); + listen_telnet(TELNETv6); + listen_telnet(TELNET_SOCK); // Start database thread if database is used if(pthread_create( &threads[DB], &attr, DB_thread, NULL ) != 0) diff --git a/src/enums.h b/src/enums.h index 361cf3c2..08982f56 100644 --- a/src/enums.h +++ b/src/enums.h @@ -186,15 +186,19 @@ enum busy_reply { } __attribute__ ((packed)); enum thread_types { - TELNETv4, - TELNETv6, - SOCKET, DB, GC, DNSclient, THREADS_MAX } __attribute__ ((packed)); +enum telnet_type { + TELNETv4, + TELNETv6, + TELNET_SOCK, + TELNET_MAX +} __attribute__ ((packed)); + enum message_type { REGEX_MESSAGE, SUBNET_MESSAGE, From 56b0124d64d7d0570195ec3b8b2b1dd83a539063 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 25 Jul 2022 21:31:36 +0200 Subject: [PATCH 021/218] Update binary classification after ftl-build container upgrade to Debian Buster Signed-off-by: DL6ER --- test/arch_test.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/test/arch_test.sh b/test/arch_test.sh index e97bb7d5..f4bb5433 100644 --- a/test/arch_test.sh +++ b/test/arch_test.sh @@ -77,7 +77,7 @@ if [[ "${CI_ARCH}" == "x86_64" ]]; then check_machine "ELF64" "Advanced Micro Devices X86-64" check_libs "[libm.so.6] [librt.so.1] [libpthread.so.0] [libc.so.6]" - check_file "ELF 64-bit LSB shared object, x86-64, version 1 (SYSV), dynamically linked, interpreter /lib64/ld-linux-x86-64.so.2, for GNU/Linux 2.6.32, not stripped" + check_file "ELF 64-bit LSB shared object, x86-64, version 1 (SYSV), dynamically linked, interpreter /lib64/ld-linux-x86-64.so.2, for GNU/Linux 3.2.0, with debug_info, not stripped" elif [[ "${CI_ARCH}" == "x86_64-musl" ]]; then @@ -90,19 +90,19 @@ elif [[ "${CI_ARCH}" == "x86_32" ]]; then check_machine "ELF32" "Intel 80386" check_libs "[libm.so.6] [librt.so.1] [libpthread.so.0] [libc.so.6]" - check_file "ELF 32-bit LSB shared object, Intel 80386, version 1 (SYSV), dynamically linked, interpreter /lib/ld-linux.so.2, for GNU/Linux 2.6.32, not stripped" + check_file "ELF 32-bit LSB shared object, Intel 80386, version 1 (SYSV), dynamically linked, interpreter /lib/ld-linux.so.2, for GNU/Linux 3.2.0, with debug_info, not stripped" elif [[ "${CI_ARCH}" == "aarch64" ]]; then check_machine "ELF64" "AArch64" check_libs "[libm.so.6] [librt.so.1] [libpthread.so.0] [libc.so.6] [ld-linux-aarch64.so.1]" - check_file "ELF 64-bit LSB shared object, ARM aarch64, version 1 (SYSV), dynamically linked, interpreter /lib/ld-linux-aarch64.so.1, for GNU/Linux 3.7.0, not stripped" + check_file "ELF 64-bit LSB shared object, ARM aarch64, version 1 (SYSV), dynamically linked, interpreter /lib/ld-linux-aarch64.so.1, for GNU/Linux 3.7.0, with debug_info, not stripped" elif [[ "${CI_ARCH}" == "armv4t" ]]; then check_machine "ELF32" "ARM" check_libs "[libm.so.6] [librt.so.1] [libgcc_s.so.1] [libpthread.so.0] [libc.so.6] [ld-linux.so.3]" - check_file "ELF 32-bit LSB shared object, ARM, EABI5 version 1 (SYSV), dynamically linked, interpreter /lib/ld-linux.so.3, for GNU/Linux 3.2.0, not stripped" + check_file "ELF 32-bit LSB shared object, ARM, EABI5 version 1 (SYSV), dynamically linked, interpreter /lib/ld-linux.so.3, for GNU/Linux 3.2.0, with debug_info, not stripped" check_CPU_arch "v4T" check_FP_arch "" # No specified FP arch @@ -111,7 +111,7 @@ elif [[ "${CI_ARCH}" == "armv5te" ]]; then check_machine "ELF32" "ARM" check_libs "[libm.so.6] [librt.so.1] [libgcc_s.so.1] [libpthread.so.0] [libc.so.6] [ld-linux.so.3]" - check_file "ELF 32-bit LSB shared object, ARM, EABI5 version 1 (SYSV), dynamically linked, interpreter /lib/ld-linux.so.3, for GNU/Linux 3.2.0, not stripped" + check_file "ELF 32-bit LSB shared object, ARM, EABI5 version 1 (SYSV), dynamically linked, interpreter /lib/ld-linux.so.3, for GNU/Linux 3.2.0, with debug_info, not stripped" check_CPU_arch "v4T" check_FP_arch "" # No specified FP arch @@ -120,7 +120,7 @@ elif [[ "${CI_ARCH}" == "armv6hf" ]]; then check_machine "ELF32" "ARM" check_libs "[libm.so.6] [librt.so.1] [libgcc_s.so.1] [libpthread.so.0] [libc.so.6] [ld-linux-armhf.so.3]" - check_file "ELF 32-bit LSB shared object, ARM, EABI5 version 1 (SYSV), dynamically linked, interpreter /lib/ld-linux-armhf.so.3, for GNU/Linux 2.6.32, not stripped" + check_file "ELF 32-bit LSB shared object, ARM, EABI5 version 1 (SYSV), dynamically linked, interpreter /lib/ld-linux-armhf.so.3, for GNU/Linux 3.2.0, with debug_info, not stripped" check_CPU_arch "v6" check_FP_arch "VFPv2" @@ -129,7 +129,7 @@ elif [[ "${CI_ARCH}" == "armv7hf" ]]; then check_machine "ELF32" "ARM" check_libs "[libm.so.6] [librt.so.1] [libgcc_s.so.1] [libpthread.so.0] [libc.so.6] [ld-linux-armhf.so.3]" - check_file "ELF 32-bit LSB shared object, ARM, EABI5 version 1 (SYSV), dynamically linked, interpreter /lib/ld-linux-armhf.so.3, for GNU/Linux 3.2.0, not stripped" + check_file "ELF 32-bit LSB shared object, ARM, EABI5 version 1 (SYSV), dynamically linked, interpreter /lib/ld-linux-armhf.so.3, for GNU/Linux 3.2.0, with debug_info, not stripped" check_CPU_arch "v7" check_FP_arch "VFPv3-D16" @@ -138,7 +138,7 @@ elif [[ "${CI_ARCH}" == "armv8a" ]]; then check_machine "ELF32" "ARM" check_libs "[libm.so.6] [librt.so.1] [libgcc_s.so.1] [libpthread.so.0] [libc.so.6] [ld-linux-armhf.so.3]" - check_file "ELF 32-bit LSB shared object, ARM, EABI5 version 1 (SYSV), dynamically linked, interpreter /lib/ld-linux-armhf.so.3, for GNU/Linux 3.2.0, not stripped" + check_file "ELF 32-bit LSB shared object, ARM, EABI5 version 1 (SYSV), dynamically linked, interpreter /lib/ld-linux-armhf.so.3, for GNU/Linux 3.2.0, with debug_info, not stripped" check_CPU_arch "v8" check_FP_arch "VFPv3-D16" From 9545dda5b22289a970bbf4473afafebab22433c6 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 26 Jul 2022 20:39:17 +0200 Subject: [PATCH 022/218] The armv4t builder has to stay on stretch as Buster discontinued armv4t support Signed-off-by: DL6ER --- test/arch_test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/arch_test.sh b/test/arch_test.sh index f4bb5433..79ab1575 100644 --- a/test/arch_test.sh +++ b/test/arch_test.sh @@ -102,7 +102,7 @@ elif [[ "${CI_ARCH}" == "armv4t" ]]; then check_machine "ELF32" "ARM" check_libs "[libm.so.6] [librt.so.1] [libgcc_s.so.1] [libpthread.so.0] [libc.so.6] [ld-linux.so.3]" - check_file "ELF 32-bit LSB shared object, ARM, EABI5 version 1 (SYSV), dynamically linked, interpreter /lib/ld-linux.so.3, for GNU/Linux 3.2.0, with debug_info, not stripped" + check_file "ELF 32-bit LSB shared object, ARM, EABI5 version 1 (SYSV), dynamically linked, interpreter /lib/ld-linux.so.3, for GNU/Linux 3.2.0, not stripped" check_CPU_arch "v4T" check_FP_arch "" # No specified FP arch From 5d8f44d0941756e2390927cc22a902f4338184ad Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 27 Jul 2022 05:39:02 +0200 Subject: [PATCH 023/218] The official Raspberry Pi toolchain does not support Buster Signed-off-by: DL6ER --- test/arch_test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/arch_test.sh b/test/arch_test.sh index 79ab1575..b8a5dd18 100644 --- a/test/arch_test.sh +++ b/test/arch_test.sh @@ -120,7 +120,7 @@ elif [[ "${CI_ARCH}" == "armv6hf" ]]; then check_machine "ELF32" "ARM" check_libs "[libm.so.6] [librt.so.1] [libgcc_s.so.1] [libpthread.so.0] [libc.so.6] [ld-linux-armhf.so.3]" - check_file "ELF 32-bit LSB shared object, ARM, EABI5 version 1 (SYSV), dynamically linked, interpreter /lib/ld-linux-armhf.so.3, for GNU/Linux 3.2.0, with debug_info, not stripped" + check_file "ELF 32-bit LSB shared object, ARM, EABI5 version 1 (SYSV), dynamically linked, interpreter /lib/ld-linux-armhf.so.3, for GNU/Linux 2.6.32, not stripped" check_CPU_arch "v6" check_FP_arch "VFPv2" From bfd9c02e89d056ba467f5ee3e99a57883620d294 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 27 Jul 2022 05:13:09 +0200 Subject: [PATCH 024/218] Buster correctly builds ARMv5T binaries (Stretch still built for ARMv4 instead) Signed-off-by: DL6ER --- test/arch_test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/arch_test.sh b/test/arch_test.sh index b8a5dd18..929f7db1 100644 --- a/test/arch_test.sh +++ b/test/arch_test.sh @@ -113,7 +113,7 @@ elif [[ "${CI_ARCH}" == "armv5te" ]]; then check_libs "[libm.so.6] [librt.so.1] [libgcc_s.so.1] [libpthread.so.0] [libc.so.6] [ld-linux.so.3]" check_file "ELF 32-bit LSB shared object, ARM, EABI5 version 1 (SYSV), dynamically linked, interpreter /lib/ld-linux.so.3, for GNU/Linux 3.2.0, with debug_info, not stripped" - check_CPU_arch "v4T" + check_CPU_arch "v5TE" check_FP_arch "" # No specified FP arch elif [[ "${CI_ARCH}" == "armv6hf" ]]; then From d79abd3000652dde446a287b0ad8ad57c2ef12ed Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 26 Jul 2022 20:36:01 +0200 Subject: [PATCH 025/218] Adjustments for upgrade to PowerDNS v4.6 Signed-off-by: DL6ER --- test/pdns/pdns.conf | 4 +++- test/pdns/setup.sh | 2 +- test/test_suite.bats | 4 ++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/test/pdns/pdns.conf b/test/pdns/pdns.conf index ebe41ead..2449edf5 100644 --- a/test/pdns/pdns.conf +++ b/test/pdns/pdns.conf @@ -9,7 +9,6 @@ # Local DNS address and port local-address=127.0.0.1:5554 -local-ipv6= # Do not enforce TCP for ANY queries any-to-tcp=false @@ -19,3 +18,6 @@ launch=gsqlite3 # Database location gsqlite3-database=/var/lib/powerdns/pdns.sqlite3 + +# Used when creating a new zone +default-soa-content=ns1.@ hostmaster.@ 0 10800 3600 604800 3600 diff --git a/test/pdns/setup.sh b/test/pdns/setup.sh index 5a0bcf75..a5832674 100644 --- a/test/pdns/setup.sh +++ b/test/pdns/setup.sh @@ -101,7 +101,7 @@ fi # HTTPS if ! pdnsutil add-record ftl. https HTTPS '1 . alpn="h3,h2"'; then # comment above applies - pdnsutil add-record ftl. https TYPE65 "\# 15 000100000100080322683303683222" + pdnsutil add-record ftl. https TYPE65 "\# 13 00010000010006026833026832" fi # Create reverse lookup zone diff --git a/test/test_suite.bats b/test/test_suite.bats index 6a5a9d04..dff4f6ec 100644 --- a/test/test_suite.bats +++ b/test/test_suite.bats @@ -317,7 +317,7 @@ @test "Local DNS test: SOA ftl" { run bash -c "dig SOA ftl @127.0.0.1 +short" printf "%s\n" "${lines[@]}" - [[ ${lines[0]} == "ns1.ftl. hostmaster.ftl. 1 10800 3600 604800 3600" ]] + [[ ${lines[0]} == "ns1.ftl. hostmaster.ftl. 0 10800 3600 604800 3600" ]] [[ ${lines[1]} == "" ]] } @@ -366,7 +366,7 @@ @test "Local DNS test: HTTPS https.ftl" { run bash -c "dig +unknown TYPE65 https.ftl @127.0.0.1 +short" printf "%s\n" "${lines[@]}" - [[ ${lines[0]} == '\# 15 000100000100080322683303683222' ]] + [[ ${lines[0]} == '\# 13 00010000010006026833026832' ]] [[ ${lines[1]} == "" ]] } From dbd2cb722c2cb8d5a0b60a8a5c5c26c9e6cb2b8f Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 27 Jul 2022 08:46:23 +0200 Subject: [PATCH 026/218] Tests: Explicityl disable DNSSEC for zone ftl. Signed-off-by: DL6ER --- test/pdns/recursor.conf | 9 +++++++++ test/pdns/setup.sh | 15 +++++---------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/test/pdns/recursor.conf b/test/pdns/recursor.conf index 6919a556..06f40946 100644 --- a/test/pdns/recursor.conf +++ b/test/pdns/recursor.conf @@ -12,3 +12,12 @@ local-address=127.0.0.1:5555 # Use authoritative server for ftl. and arpa. zones forward-zones=ftl=127.0.0.1:5554,168.192.in-addr.arpa=127.0.0.1:5554,ip6.arpa=127.0.0.1:5554 + +# In this mode the Recursor acts as a “security aware, non-validating” +# nameserver, meaning it will set the DO-bit on outgoing queries and will +# provide DNSSEC related RRsets (NSEC, RRSIG) to clients that ask for them (by +# means of a DO-bit in the query), except for zones provided through the +# auth-zones setting. It will not do any validation in this mode, not even when +# requested by the client. +# The default mode until PowerDNS Recursor 4.5.0. +dnssec=process-no-validate diff --git a/test/pdns/setup.sh b/test/pdns/setup.sh index a5832674..a751f85e 100644 --- a/test/pdns/setup.sh +++ b/test/pdns/setup.sh @@ -40,7 +40,7 @@ else fi # Create zone ftl pdnsutil create-zone ftl ns1.ftl -pdnsutil add-record ftl. . SOA "ns1.ftl. hostmaster.ftl. 1 10800 3600 604800 3600" +pdnsutil disable-dnssec ftl # Create A records pdnsutil add-record ftl. a A 192.168.1.1 @@ -92,17 +92,10 @@ pdnsutil add-record ftl. naptr NAPTR '20 10 "s" "http+N2L+N2C+N2R" "" ftl.' pdnsutil add-record ftl. mx MX "50 ns1.ftl." # SVCB + HTTPS -if ! pdnsutil add-record ftl. svcb SVCB '1 port="80"'; then - # see RFC3597: Handling of Unknown DNS Resource Record (RR) Types - # and https://ypcs.fi/howto/2020/09/30/announce-https-via-dns/ - pdnsutil add-record ftl. svcb TYPE64 "\# 13 000109706F72743D2238302200" -fi +pdnsutil add-record ftl. svcb SVCB '1 port="80"' # HTTPS -if ! pdnsutil add-record ftl. https HTTPS '1 . alpn="h3,h2"'; then - # comment above applies - pdnsutil add-record ftl. https TYPE65 "\# 13 00010000010006026833026832" -fi +pdnsutil add-record ftl. https HTTPS '1 . alpn="h3,h2"' # Create reverse lookup zone pdnsutil create-zone arpa ns1.ftl @@ -120,6 +113,8 @@ pdnsutil rectify-all-zones pdnsutil check-zone ftl pdnsutil check-zone arpa +pdnsutil list-all-zones + echo "********* Done installing PowerDNS configuration **********" # Start services From cf97933ca2d1ad1afd94d77e96c398d5a891aa13 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 27 Jul 2022 09:33:06 +0200 Subject: [PATCH 027/218] Tests: dig should now support SVCB and HTTPS directly Signed-off-by: DL6ER --- test/test_suite.bats | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/test_suite.bats b/test/test_suite.bats index dff4f6ec..1931e6b4 100644 --- a/test/test_suite.bats +++ b/test/test_suite.bats @@ -357,16 +357,16 @@ } @test "Local DNS test: SVCB svcb.ftl" { - run bash -c "dig +unknown TYPE64 svcb.ftl @127.0.0.1 +short" + run bash -c "dig SVCB svcb.ftl @127.0.0.1 +short" printf "%s\n" "${lines[@]}" - [[ ${lines[0]} == '\# 13 000109706F72743D2238302200' ]] + [[ ${lines[0]} == '1 port=\"80\".' ]] [[ ${lines[1]} == "" ]] } @test "Local DNS test: HTTPS https.ftl" { - run bash -c "dig +unknown TYPE65 https.ftl @127.0.0.1 +short" + run bash -c "dig HTTPS https.ftl @127.0.0.1 +short" printf "%s\n" "${lines[@]}" - [[ ${lines[0]} == '\# 13 00010000010006026833026832' ]] + [[ ${lines[0]} == '1 . alpn="h3,h2"' ]] [[ ${lines[1]} == "" ]] } From ea537ddac3c0bf4411c87ec47d2cd6f0a32e2402 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 27 Jul 2022 13:15:33 +0200 Subject: [PATCH 028/218] Update expected binary information for cross-compiled armv6hf binary Signed-off-by: DL6ER --- test/arch_test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/arch_test.sh b/test/arch_test.sh index 929f7db1..11b47b37 100644 --- a/test/arch_test.sh +++ b/test/arch_test.sh @@ -120,7 +120,7 @@ elif [[ "${CI_ARCH}" == "armv6hf" ]]; then check_machine "ELF32" "ARM" check_libs "[libm.so.6] [librt.so.1] [libgcc_s.so.1] [libpthread.so.0] [libc.so.6] [ld-linux-armhf.so.3]" - check_file "ELF 32-bit LSB shared object, ARM, EABI5 version 1 (SYSV), dynamically linked, interpreter /lib/ld-linux-armhf.so.3, for GNU/Linux 2.6.32, not stripped" + check_file "ELF 32-bit LSB shared object, ARM, EABI5 version 1 (SYSV), dynamically linked, interpreter /lib/ld-linux-armhf.so.3, for GNU/Linux 3.2.0, with debug_info, not stripped" check_CPU_arch "v6" check_FP_arch "VFPv2" From 9e8b7818072e6d9c0fe19a45c86848f09f6c0ef5 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sat, 30 Jul 2022 18:10:54 +0100 Subject: [PATCH 029/218] Update build.yml to use latest FTL build containers Signed-off-by: Adam Warner --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 45e01296..45c6f87f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -25,7 +25,7 @@ jobs: runs-on: ubuntu-latest - container: ghcr.io/pi-hole/ftl-build:v1.18-${{ matrix.arch }} + container: ghcr.io/pi-hole/ftl-build:v1.19-${{ matrix.arch }} strategy: fail-fast: false From fb40a811465b36c2285bdb4dc26234482ed37e9a Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Mon, 1 Aug 2022 22:24:25 +0100 Subject: [PATCH 030/218] Allow nested dirs when deploying builds Add pre-build smoke-test to check git branch depth Co-authored-by: yubiuser Signed-off-by: Adam Warner --- .github/workflows/build.yml | 64 ++++++++++++++++++++++++++++--------- deploy.sh | 46 ++++++++++++++++++++++++++ 2 files changed, 95 insertions(+), 15 deletions(-) create mode 100755 deploy.sh diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 92188e0e..ad88bdee 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -12,7 +12,7 @@ on: types: [published] jobs: - build: + smoke-tests: if: | github.event_name == 'push' || github.event_name == 'release' @@ -24,6 +24,37 @@ jobs: OUTPUT_DIR: ${{ steps.variables.outputs.OUTPUT_DIR }} runs-on: ubuntu-latest + steps: + - + name: Checkout code + uses: actions/checkout@v3 + - + name: "Calculate required variables" + id: variables + run: | + GIT_TAG=${{ github.event.release.tag_name }} + # If GIT_TAG is set then GIT BRANCH should be "master", else set it from GITHUB_REF + GIT_BRANCH=$([ -n "${GIT_TAG}" ] && echo "master" || echo "${GITHUB_REF#refs/*/}") + echo ::set-output name=GIT_BRANCH::${GIT_BRANCH} + echo ::set-output name=GIT_TAG::${GIT_TAG} + echo ::set-output name=OUTPUT_DIR::${GIT_TAG:-${GIT_BRANCH}} + - + name: "Check git branch name depth" + env: + GIT_BRANCH: ${{ steps.variables.outputs.GIT_BRANCH }} + run: | + IFS='/'; + read -r -a branch <<<"${GIT_BRANCH}"; + if [[ "${#branch[@]}" -gt 2 ]]; then echo "Error: Your branch name contains more than one subdir, which will cause issues with the build process." && FAIL=1; fi; + unset IFS; + # If FAIL is 1 then we fail. + [[ $FAIL == 1 ]] && exit 1 || echo "Branch name depth check passed." + shell: bash + + build: + runs-on: ubuntu-latest + + needs: smoke-tests container: ghcr.io/pi-hole/ftl-build:v1.18-${{ matrix.arch }} @@ -59,23 +90,16 @@ jobs: name: Checkout code uses: actions/checkout@v3.0.2 - - name: Calculate required build variables - id: variables - run: | - GIT_TAG=${{ github.event.release.tag_name }} - # If GIT_TAG is set then GIT BRANCH should be "master", else set it from GITHUB_REF - GIT_BRANCH=$([ -n "${GIT_TAG}" ] && echo "master" || echo "${GITHUB_REF#refs/*/}") - echo ::set-output name=GIT_BRANCH::${GIT_BRANCH} - echo ::set-output name=GIT_TAG::${GIT_TAG} - echo ::set-output name=OUTPUT_DIR::${GIT_TAG:-${GIT_BRANCH}} + name: "Fix ownership of repository" + run: chown -R root . - name: "Fix ownership of repository" run: chown -R root . - name: "Build" env: - GIT_BRANCH: ${{ steps.variables.outputs.GIT_BRANCH }} - GIT_TAG: ${{ steps.variables.outputs.GIT_TAG }} + GIT_BRANCH: ${{ needs.smoke-tests.outputs.GIT_BRANCH }} + GIT_TAG: ${{ needs.smoke-tests.outputs.GIT_TAG }} run: | bash build.sh "-DSTATIC=${STATIC}" - @@ -102,17 +126,23 @@ jobs: deploy: if: github.event_name != 'pull_request' - needs: build + needs: [smoke-tests, build] runs-on: ubuntu-latest steps: + - + name: Checkout code + uses: actions/checkout@v3 - name: Get Binaries built in previous jobs uses: actions/download-artifact@v3.0.0 + id: download with: name: tmp-binary-storage + path: ftl-builds/ - name: Display structure of downloaded files run: ls -R + working-directory: ${{steps.download.outputs.download-path}} - name: Install SSH Key uses: shimataro/ssh-key-action@v2.3.1 @@ -121,9 +151,13 @@ jobs: known_hosts: ${{ secrets.KNOWN_HOSTS }} - name: Transfer Builds to Pi-hole server for pihole checkout + env: + USER: ${{ secrets.SSH_USER }} + HOST: ${{ secrets.SSH_HOST }} + TARGET_DIR: ${{ needs.smoke-tests.outputs.OUTPUT_DIR }} + SOURCE_DIR: ${{ steps.download.outputs.download-path }} run: | - sftp -b - ${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST }} <<< "-mkdir ${{ needs.build.outputs.OUTPUT_DIR }} - put * ${{ needs.build.outputs.OUTPUT_DIR }}" + bash ./deploy.sh - name: Attach binaries to release if: github.event_name == 'release' diff --git a/deploy.sh b/deploy.sh new file mode 100755 index 00000000..59c54669 --- /dev/null +++ b/deploy.sh @@ -0,0 +1,46 @@ +#!/bin/bash +# Pi-hole: A black hole for Internet advertisements +# (c) 2022 Pi-hole, LLC (https://pi-hole.net) +# Network-wide ad blocking via your own hardware. +# +# FTL Engine +# Deploy script for FTL +# +# This file is copyright under the latest version of the EUPL. +# Please see LICENSE file for your rights under this license. + + +# Transfer Builds to Pi-hole server for pihole checkout +# We use sftp for secure transfer and use the branch name as dir on the server. +# The branch name could contain slashes, creating hierarchical dirs. However, +# this is not supported by sftp's `mkdir` (option -p) is not available. Therefore, +# we need to loop over each dir level and create them one by one. + + +# Safeguard: do not deploy if TARGET_DIR is empty +if [[ -z ${TARGET_DIR} ]]; then + echo "Error: Empty target dir." + exit 1 +fi + +IFS='/' +read -r -a path <<<"${TARGET_DIR}" + +# Safeguard: do not deploy if more than one subdir (eg. /tweak/feature/subfeature) needs to be created +if [[ "${#path[@]}" -gt 2 ]]; then + echo "Error: Your branch name contains more then one subdir. We won't deploy that." + exit 1 +fi + +unset IFS + +old_path="." + +for dir in "${path[@]}"; do + sftp -b - "${USER}"@"${HOST}" <<< "cd ${old_path} + -mkdir ${dir}" + old_path="${old_path}/${dir}" +done + +sftp -r -b - "${USER}"@"${HOST}" <<< "cd ${old_path} +put ${SOURCE_DIR}/* ./" From 5f83ba9ff860bc4da76fba2fefc939a26bb8a781 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Tue, 2 Aug 2022 00:02:13 +0200 Subject: [PATCH 031/218] Only try to create the subdir if does not already exist MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- deploy.sh | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/deploy.sh b/deploy.sh index 59c54669..fb7c3a4b 100755 --- a/deploy.sh +++ b/deploy.sh @@ -37,8 +37,20 @@ unset IFS old_path="." for dir in "${path[@]}"; do - sftp -b - "${USER}"@"${HOST}" <<< "cd ${old_path} - -mkdir ${dir}" + mapfile -t dir_content <<< "$( + sftp -b - "${USER}"@"${HOST}" <<< "cd ${old_path} + ls -1" + )" + + # Only try to create the subdir if does not already exist + if [[ "${dir_content[*]}" =~ "${dir}" ]]; then + echo "Dir: ${old_path}/${dir} already exists" + else + echo "Creating dir: ${old_path}/${dir}" + sftp -b - "${USER}"@"${HOST}" <<< "cd ${old_path} + -mkdir ${dir}" + fi + old_path="${old_path}/${dir}" done From 5620a1949ba4a14c1129a0b82b58ab21c2958dd9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 6 Aug 2022 10:15:59 +0000 Subject: [PATCH 032/218] Bump actions/stale from 5.1.0 to 5.1.1 Bumps [actions/stale](https://github.com/actions/stale) from 5.1.0 to 5.1.1. - [Release notes](https://github.com/actions/stale/releases) - [Changelog](https://github.com/actions/stale/blob/v5.1.1/CHANGELOG.md) - [Commits](https://github.com/actions/stale/compare/v5.1.0...v5.1.1) --- updated-dependencies: - dependency-name: actions/stale dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/stale.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index cd73e1cd..bce8bbea 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -13,7 +13,7 @@ jobs: issues: write steps: - - uses: actions/stale@v5.1.0 + - uses: actions/stale@v5.1.1 with: repo-token: ${{ secrets.GITHUB_TOKEN }} days-before-stale: 30 From 91126cb936857551c47b06563f72c6ddef81e8db Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 6 Aug 2022 10:16:02 +0000 Subject: [PATCH 033/218] Bump actions-ecosystem/action-add-labels from 1.1.0 to 1.1.3 Bumps [actions-ecosystem/action-add-labels](https://github.com/actions-ecosystem/action-add-labels) from 1.1.0 to 1.1.3. - [Release notes](https://github.com/actions-ecosystem/action-add-labels/releases) - [Commits](https://github.com/actions-ecosystem/action-add-labels/compare/v1.1.0...v1.1.3) --- updated-dependencies: - dependency-name: actions-ecosystem/action-add-labels dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/sync-back-to-dev.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sync-back-to-dev.yml b/.github/workflows/sync-back-to-dev.yml index 27d966d2..dd66ebdc 100644 --- a/.github/workflows/sync-back-to-dev.yml +++ b/.github/workflows/sync-back-to-dev.yml @@ -20,7 +20,7 @@ jobs: FROM_BRANCH: 'master' TO_BRANCH: 'development' - name: Label the pull request to ignore for release note generation - uses: actions-ecosystem/action-add-labels@v1.1.0 + uses: actions-ecosystem/action-add-labels@v1.1.3 with: labels: internal repo: ${{ github.repository }} From d8ce2e826bfaa583c1066442276fcdab4b8ccc5d Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 27 Jul 2022 19:20:03 +0200 Subject: [PATCH 034/218] Ensure errno value is restored on return of syscall functions. Signed-off-by: DL6ER --- src/api/api.c | 360 ++++++++++++++--------------- src/api/api.h | 40 ++-- src/api/request.c | 22 +- src/api/request.h | 2 +- src/api/socket.c | 371 ++++++++++++++---------------- src/api/socket.h | 5 +- src/daemon.c | 6 +- src/dnsmasq_interface.c | 13 +- src/dnsmasq_interface.h | 1 - src/syscalls/accept.c | 12 +- src/syscalls/calloc.c | 7 + src/syscalls/fopen.c | 7 + src/syscalls/ftlallocate.c | 1 + src/syscalls/pthread_mutex_lock.c | 12 +- src/syscalls/realloc.c | 6 + src/syscalls/recv.c | 12 +- src/syscalls/recvfrom.c | 12 +- src/syscalls/select.c | 12 +- src/syscalls/sendto.c | 12 +- src/syscalls/vasprintf.c | 3 + src/syscalls/vfprintf.c | 9 + src/syscalls/vsnprintf.c | 4 + src/syscalls/vsprintf.c | 3 + src/syscalls/write.c | 23 +- 24 files changed, 495 insertions(+), 460 deletions(-) diff --git a/src/api/api.c b/src/api/api.c index 0857b5f6..32047a2f 100644 --- a/src/api/api.c +++ b/src/api/api.c @@ -76,7 +76,7 @@ static int __attribute__((pure)) cmpdesc(const void *a, const void *b) return 0; } -void getStats(const int *sock) +void getStats(const int sock) { const int blocked = blocked_queries(); const int total = counters->queries; @@ -87,11 +87,11 @@ void getStats(const int *sock) percentage = 1e2f*blocked/total; // Send domains being blocked - if(istelnet[*sock]) { - ssend(*sock, "domains_being_blocked %i\n", counters->gravity); + if(istelnet[sock]) { + ssend(sock, "domains_being_blocked %i\n", counters->gravity); } else - pack_int32(*sock, counters->gravity); + pack_int32(sock, counters->gravity); // unique_clients: count only clients that have been active within the most recent 24 hours int activeclients = 0; @@ -106,13 +106,13 @@ void getStats(const int *sock) activeclients++; } - if(istelnet[*sock]) { - ssend(*sock, "dns_queries_today %i\nads_blocked_today %i\nads_percentage_today %f\n", + if(istelnet[sock]) { + ssend(sock, "dns_queries_today %i\nads_blocked_today %i\nads_percentage_today %f\n", total, blocked, percentage); - ssend(*sock, "unique_domains %i\nqueries_forwarded %i\nqueries_cached %i\n", + ssend(sock, "unique_domains %i\nqueries_forwarded %i\nqueries_cached %i\n", counters->domains, forwarded_queries(), cached_queries()); - ssend(*sock, "clients_ever_seen %i\n", counters->clients); - ssend(*sock, "unique_clients %i\n", activeclients); + ssend(sock, "clients_ever_seen %i\n", counters->clients); + ssend(sock, "unique_clients %i\n", activeclients); // Sum up all query types (A, AAAA, ANY, SRV, SOA, ...) int sumalltypes = 0; @@ -120,45 +120,45 @@ void getStats(const int *sock) { sumalltypes += counters->querytype[queryType]; } - ssend(*sock, "dns_queries_all_types %i\n", sumalltypes); + ssend(sock, "dns_queries_all_types %i\n", sumalltypes); // Send individual reply type counters int sumallreplies = 0; for(enum reply_type reply = REPLY_UNKNOWN; reply < QUERY_REPLY_MAX; reply++) { - ssend(*sock, "reply_%s %i\n", get_query_reply_str(reply), counters->reply[reply]); + ssend(sock, "reply_%s %i\n", get_query_reply_str(reply), counters->reply[reply]); sumallreplies += counters->reply[reply]; } - ssend(*sock, "dns_queries_all_replies %i\n", sumallreplies); - ssend(*sock, "privacy_level %i\n", config.privacylevel); + ssend(sock, "dns_queries_all_replies %i\n", sumallreplies); + ssend(sock, "privacy_level %i\n", config.privacylevel); } else { - pack_int32(*sock, total); - pack_int32(*sock, blocked); - pack_float(*sock, percentage); - pack_int32(*sock, counters->domains); - pack_int32(*sock, forwarded_queries()); - pack_int32(*sock, cached_queries()); - pack_int32(*sock, counters->clients); - pack_int32(*sock, activeclients); + pack_int32(sock, total); + pack_int32(sock, blocked); + pack_float(sock, percentage); + pack_int32(sock, counters->domains); + pack_int32(sock, forwarded_queries()); + pack_int32(sock, cached_queries()); + pack_int32(sock, counters->clients); + pack_int32(sock, activeclients); } // Send status - if(istelnet[*sock]) { - ssend(*sock, "status %s\n", blockingstatus ? "enabled" : "disabled"); + if(istelnet[sock]) { + ssend(sock, "status %s\n", blockingstatus ? "enabled" : "disabled"); } else - pack_uint8(*sock, blockingstatus); + pack_uint8(sock, blockingstatus); } -void getOverTime(const int *sock) +void getOverTime(const int sock) { - if(istelnet[*sock]) + if(istelnet[sock]) { for(int slot = 0; slot < OVERTIME_SLOTS; slot++) { - ssend(*sock,"%lli %i %i\n", + ssend(sock,"%lli %i %i\n", (long long)overTime[slot].timestamp, overTime[slot].total, overTime[slot].blocked); @@ -170,22 +170,22 @@ void getOverTime(const int *sock) // and map16 can hold up to (2^16)-1 = 65535 pairs // Send domains over time - pack_map16_start(*sock, (uint16_t) OVERTIME_SLOTS); + pack_map16_start(sock, (uint16_t) OVERTIME_SLOTS); for(int slot = 0; slot < OVERTIME_SLOTS; slot++) { - pack_int32(*sock, (int32_t)overTime[slot].timestamp); - pack_int32(*sock, overTime[slot].total); + pack_int32(sock, (int32_t)overTime[slot].timestamp); + pack_int32(sock, overTime[slot].total); } // Send ads over time - pack_map16_start(*sock, (uint16_t) OVERTIME_SLOTS); + pack_map16_start(sock, (uint16_t) OVERTIME_SLOTS); for(int slot = 0; slot < OVERTIME_SLOTS; slot++) { - pack_int32(*sock, (int32_t)overTime[slot].timestamp); - pack_int32(*sock, overTime[slot].blocked); + pack_int32(sock, (int32_t)overTime[slot].timestamp); + pack_int32(sock, overTime[slot].blocked); } } } -void getTopDomains(const char *client_message, const int *sock) +void getTopDomains(const char *client_message, const int sock) { int temparray[counters->domains][2], count=10, num; bool audit = false, asc = false; @@ -196,8 +196,8 @@ void getTopDomains(const char *client_message, const int *sock) get_privacy_level(NULL); if(config.privacylevel >= PRIVACY_HIDE_DOMAINS) { // Always send the total number of domains, but pretend it's 0 - if(!istelnet[*sock]) - pack_int32(*sock, 0); + if(!istelnet[sock]) + pack_int32(sock, 0); return; } @@ -269,13 +269,13 @@ void getTopDomains(const char *client_message, const int *sock) } } - if(!istelnet[*sock]) + if(!istelnet[sock]) { // Send the data required to get the percentage each domain has been blocked / queried if(blocked) - pack_int32(*sock, blocked_queries()); + pack_int32(sock, blocked_queries()); else - pack_int32(*sock, counters->queries); + pack_int32(sock, counters->queries); } int n = 0; @@ -306,26 +306,26 @@ void getTopDomains(const char *client_message, const int *sock) if(blocked && showblocked && domain->blockedcount > 0) { - if(istelnet[*sock]) - ssend(*sock, "%i %i %s\n", n, domain->blockedcount, getstr(domain->domainpos)); + if(istelnet[sock]) + ssend(sock, "%i %i %s\n", n, domain->blockedcount, getstr(domain->domainpos)); else { - if(!pack_str32(*sock, getstr(domain->domainpos))) + if(!pack_str32(sock, getstr(domain->domainpos))) return; - pack_int32(*sock, domain->blockedcount); + pack_int32(sock, domain->blockedcount); } n++; } else if(!blocked && showpermitted && (domain->count - domain->blockedcount) > 0) { - if(istelnet[*sock]) - ssend(*sock,"%i %i %s\n",n,(domain->count - domain->blockedcount),getstr(domain->domainpos)); + if(istelnet[sock]) + ssend(sock,"%i %i %s\n",n,(domain->count - domain->blockedcount),getstr(domain->domainpos)); else { - if(!pack_str32(*sock, getstr(domain->domainpos))) + if(!pack_str32(sock, getstr(domain->domainpos))) return; - pack_int32(*sock, domain->count - domain->blockedcount); + pack_int32(sock, domain->count - domain->blockedcount); } n++; } @@ -339,7 +339,7 @@ void getTopDomains(const char *client_message, const int *sock) clearSetupVarsArray(); } -void getTopClients(const char *client_message, const int *sock) +void getTopClients(const char *client_message, const int sock) { int temparray[counters->clients][2], count=10, num; @@ -347,8 +347,8 @@ void getTopClients(const char *client_message, const int *sock) get_privacy_level(NULL); if(config.privacylevel >= PRIVACY_HIDE_DOMAINS_CLIENTS) { // Always send the total number of clients, but pretend it's 0 - if(!istelnet[*sock]) - pack_int32(*sock, 0); + if(!istelnet[sock]) + pack_int32(sock, 0); return; } @@ -408,10 +408,10 @@ void getTopClients(const char *client_message, const int *sock) getSetupVarsArray(excludeclients); } - if(!istelnet[*sock]) + if(!istelnet[sock]) { // Send the total queries so they can make percentages from this data - pack_int32(*sock, counters->queries); + pack_int32(sock, counters->queries); } int n = 0; @@ -450,14 +450,14 @@ void getTopClients(const char *client_message, const int *sock) // - the client made at least one query within the most recent 24 hours if(includezeroclients || ccount > 0) { - if(istelnet[*sock]) - ssend(*sock,"%i %i %s %s\n", n, ccount, client_ip, client_name); + if(istelnet[sock]) + ssend(sock,"%i %i %s %s\n", n, ccount, client_ip, client_name); else { - if(!pack_str32(*sock, "") || !pack_str32(*sock, client_ip)) + if(!pack_str32(sock, "") || !pack_str32(sock, client_ip)) return; - pack_int32(*sock, ccount); + pack_int32(sock, ccount); } n++; } @@ -471,7 +471,7 @@ void getTopClients(const char *client_message, const int *sock) } -void getUpstreamDestinations(const char *client_message, const int *sock) +void getUpstreamDestinations(const char *client_message, const int sock) { bool sort = true; int temparray[counters->upstreams][2], sumforwarded = 0; @@ -575,24 +575,24 @@ void getUpstreamDestinations(const char *client_message, const int *sock) // - only if percentage > 0.0 for all others (i > 0) if(percentage > 0.0f || i < 0) { - if(istelnet[*sock]) + if(istelnet[sock]) if(upstream_port != 0) - ssend(*sock, "%i %.2f %s#%u %s#%u\n", i, percentage, + ssend(sock, "%i %.2f %s#%u %s#%u\n", i, percentage, ip, upstream_port, name, upstream_port); else - ssend(*sock, "%i %.2f %s %s\n", i, percentage, ip, name); + ssend(sock, "%i %.2f %s %s\n", i, percentage, ip, name); else { - if(!pack_str32(*sock, name) || !pack_str32(*sock, ip)) + if(!pack_str32(sock, name) || !pack_str32(sock, ip)) return; - pack_float(*sock, percentage); + pack_float(sock, percentage); } } } } -void getQueryTypes(const int *sock) +void getQueryTypes(const int sock) { int total = 0; for(enum query_types type = TYPE_A; type < TYPE_MAX; type++) @@ -611,8 +611,8 @@ void getQueryTypes(const int *sock) } } - if(istelnet[*sock]) { - ssend(*sock, "A (IPv4): %.2f\nAAAA (IPv6): %.2f\nANY: %.2f\nSRV: %.2f\n" + if(istelnet[sock]) { + ssend(sock, "A (IPv4): %.2f\nAAAA (IPv6): %.2f\nANY: %.2f\nSRV: %.2f\n" "SOA: %.2f\nPTR: %.2f\nTXT: %.2f\nNAPTR: %.2f\n" "MX: %.2f\nDS: %.2f\nRRSIG: %.2f\nDNSKEY: %.2f\n" "NS: %.2f\n" "OTHER: %.2f\n\nSVCB: %.2f\nHTTPS: %.2f\n", @@ -622,42 +622,42 @@ void getQueryTypes(const int *sock) percentage[TYPE_NS], percentage[TYPE_OTHER], percentage[TYPE_SVCB], percentage[TYPE_HTTPS]); } else { - pack_str32(*sock, "A (IPv4)"); - pack_float(*sock, percentage[TYPE_A]); - pack_str32(*sock, "AAAA (IPv6)"); - pack_float(*sock, percentage[TYPE_AAAA]); - pack_str32(*sock, "ANY"); - pack_float(*sock, percentage[TYPE_ANY]); - pack_str32(*sock, "SRV"); - pack_float(*sock, percentage[TYPE_SRV]); - pack_str32(*sock, "SOA"); - pack_float(*sock, percentage[TYPE_SOA]); - pack_str32(*sock, "PTR"); - pack_float(*sock, percentage[TYPE_PTR]); - pack_str32(*sock, "TXT"); - pack_float(*sock, percentage[TYPE_TXT]); - pack_str32(*sock, "NAPTR"); - pack_float(*sock, percentage[TYPE_NAPTR]); - pack_str32(*sock, "MX"); - pack_float(*sock, percentage[TYPE_MX]); - pack_str32(*sock, "DS"); - pack_float(*sock, percentage[TYPE_DS]); - pack_str32(*sock, "RRSIG"); - pack_float(*sock, percentage[TYPE_RRSIG]); - pack_str32(*sock, "DNSKEY"); - pack_float(*sock, percentage[TYPE_DNSKEY]); - pack_str32(*sock, "NS"); - pack_float(*sock, percentage[TYPE_NS]); - pack_str32(*sock, "OTHER"); - pack_float(*sock, percentage[TYPE_OTHER]); - pack_str32(*sock, "SVCB"); - pack_float(*sock, percentage[TYPE_SVCB]); - pack_str32(*sock, "HTTPS"); - pack_float(*sock, percentage[TYPE_HTTPS]); + pack_str32(sock, "A (IPv4)"); + pack_float(sock, percentage[TYPE_A]); + pack_str32(sock, "AAAA (IPv6)"); + pack_float(sock, percentage[TYPE_AAAA]); + pack_str32(sock, "ANY"); + pack_float(sock, percentage[TYPE_ANY]); + pack_str32(sock, "SRV"); + pack_float(sock, percentage[TYPE_SRV]); + pack_str32(sock, "SOA"); + pack_float(sock, percentage[TYPE_SOA]); + pack_str32(sock, "PTR"); + pack_float(sock, percentage[TYPE_PTR]); + pack_str32(sock, "TXT"); + pack_float(sock, percentage[TYPE_TXT]); + pack_str32(sock, "NAPTR"); + pack_float(sock, percentage[TYPE_NAPTR]); + pack_str32(sock, "MX"); + pack_float(sock, percentage[TYPE_MX]); + pack_str32(sock, "DS"); + pack_float(sock, percentage[TYPE_DS]); + pack_str32(sock, "RRSIG"); + pack_float(sock, percentage[TYPE_RRSIG]); + pack_str32(sock, "DNSKEY"); + pack_float(sock, percentage[TYPE_DNSKEY]); + pack_str32(sock, "NS"); + pack_float(sock, percentage[TYPE_NS]); + pack_str32(sock, "OTHER"); + pack_float(sock, percentage[TYPE_OTHER]); + pack_str32(sock, "SVCB"); + pack_float(sock, percentage[TYPE_SVCB]); + pack_str32(sock, "HTTPS"); + pack_float(sock, percentage[TYPE_HTTPS]); } } -void getAllQueries(const char *client_message, const int *sock) +void getAllQueries(const char *client_message, const int sock) { // Exit before processing any data if requested via config setting get_privacy_level(NULL); @@ -1053,9 +1053,9 @@ void getAllQueries(const char *client_message, const int *sock) delay = 0UL; } - if(istelnet[*sock]) + if(istelnet[sock]) { - ssend(*sock,"%lli %s %s %s %i %i %i %lu %s %i %s#%u \"%s\"", + ssend(sock,"%lli %s %s %s %i %i %i %lu %s %i %s#%u \"%s\"", (long long)query->timestamp, qtype, domain, @@ -1071,23 +1071,23 @@ void getAllQueries(const char *client_message, const int *sock) query->ede == -1 ? "" : get_edestr(query->ede)); if(config.debug & DEBUG_API) - ssend(*sock, " \"%i\"", queryID); - ssend(*sock, "\n"); + ssend(sock, " \"%i\"", queryID); + ssend(sock, "\n"); } else { - pack_int32(*sock, (int32_t)query->timestamp); + pack_int32(sock, (int32_t)query->timestamp); // Use a fixstr because the length of qtype is always 4 (max is 31 for fixstr) - if(!pack_fixstr(*sock, qtype)) + if(!pack_fixstr(sock, qtype)) break; // Use str32 for domain and client because we have no idea how long they will be (max is 4294967295 for str32) - if(!pack_str32(*sock, domain) || !pack_str32(*sock, clientIPName)) + if(!pack_str32(sock, domain) || !pack_str32(sock, clientIPName)) break; - pack_uint8(*sock, query->status); - pack_uint8(*sock, query->dnssec); + pack_uint8(sock, query->status); + pack_uint8(sock, query->dnssec); } } @@ -1105,7 +1105,7 @@ void getAllQueries(const char *client_message, const int *sock) free(clientid_list); } -void getRecentBlocked(const char *client_message, const int *sock) +void getRecentBlocked(const char *client_message, const int sock) { int num=1; @@ -1132,9 +1132,9 @@ void getRecentBlocked(const char *client_message, const int *sock) if(domain == NULL) continue; - if(istelnet[*sock]) - ssend(*sock,"%s\n", domain); - else if(!pack_str32(*sock, domain)) + if(istelnet[sock]) + ssend(sock,"%s\n", domain); + else if(!pack_str32(sock, domain)) return; // Only count when sent successfully @@ -1146,15 +1146,15 @@ void getRecentBlocked(const char *client_message, const int *sock) } } -void getClientID(const int *sock) +void getClientID(const int sock) { - if(istelnet[*sock]) - ssend(*sock,"%i\n", *sock); + if(istelnet[sock]) + ssend(sock,"%i\n", sock); else - pack_int32(*sock, *sock); + pack_int32(sock, sock); } -void getVersion(const int *sock) +void getVersion(const int sock) { const char *commit = GIT_HASH; const char *tag = GIT_TAG; @@ -1165,25 +1165,25 @@ void getVersion(const int *sock) memcpy(hash, commit, min((size_t)7, strlen(commit))); if(strlen(tag) > 1) { - if(istelnet[*sock]) + if(istelnet[sock]) ssend( - *sock, + sock, "version %s\ntag %s\nbranch %s\nhash %s\ndate %s\n", version, tag, GIT_BRANCH, hash, GIT_DATE ); else { - if(!pack_str32(*sock, version) || - !pack_str32(*sock, (char *) tag) || - !pack_str32(*sock, GIT_BRANCH) || - !pack_str32(*sock, hash) || - !pack_str32(*sock, GIT_DATE)) + if(!pack_str32(sock, version) || + !pack_str32(sock, (char *) tag) || + !pack_str32(sock, GIT_BRANCH) || + !pack_str32(sock, hash) || + !pack_str32(sock, GIT_DATE)) return; } } else { - if(istelnet[*sock]) + if(istelnet[sock]) ssend( - *sock, + sock, "version vDev-%s\ntag %s\nbranch %s\nhash %s\ndate %s\n", hash, tag, GIT_BRANCH, hash, GIT_DATE ); @@ -1192,11 +1192,11 @@ void getVersion(const int *sock) if(hashVersion == NULL) return; sprintf(hashVersion, "vDev-%s", hash); - if(!pack_str32(*sock, hashVersion) || - !pack_str32(*sock, (char *) tag) || - !pack_str32(*sock, GIT_BRANCH) || - !pack_str32(*sock, hash) || - !pack_str32(*sock, GIT_DATE)) + if(!pack_str32(sock, hashVersion) || + !pack_str32(sock, (char *) tag) || + !pack_str32(sock, GIT_BRANCH) || + !pack_str32(sock, hash) || + !pack_str32(sock, GIT_DATE)) return; free(hashVersion); @@ -1204,7 +1204,7 @@ void getVersion(const int *sock) } } -void getDBstats(const int *sock) +void getDBstats(const int sock) { // Get file details unsigned long long int filesize = get_FTL_db_filesize(); @@ -1213,19 +1213,19 @@ void getDBstats(const int *sock) double formatted = 0.0; format_memory_size(prefix, filesize, &formatted); - if(istelnet[*sock]) - ssend(*sock, "queries in database: %i\ndatabase filesize: %.2f %sB\nSQLite version: %s\n", + if(istelnet[sock]) + ssend(sock, "queries in database: %i\ndatabase filesize: %.2f %sB\nSQLite version: %s\n", get_number_of_queries_in_DB(NULL), formatted, prefix, get_sqlite3_version()); else { - pack_int32(*sock, get_number_of_queries_in_DB(NULL)); - pack_int64(*sock, filesize); + pack_int32(sock, get_number_of_queries_in_DB(NULL)); + pack_int64(sock, filesize); - if(!pack_str32(*sock, (char *) get_sqlite3_version())) + if(!pack_str32(sock, (char *) get_sqlite3_version())) return; } } -void getClientsOverTime(const int *sock) +void getClientsOverTime(const int sock) { // Exit before processing any data if requested via config setting get_privacy_level(NULL); @@ -1263,10 +1263,10 @@ void getClientsOverTime(const int *sock) // Main return loop for(int slot = 0; slot < OVERTIME_SLOTS; slot++) { - if(istelnet[*sock]) - ssend(*sock, "%lli", (long long)overTime[slot].timestamp); + if(istelnet[sock]) + ssend(sock, "%lli", (long long)overTime[slot].timestamp); else - pack_int32(*sock, (int32_t)overTime[slot].timestamp); + pack_int32(sock, (int32_t)overTime[slot].timestamp); // Loop over forward destinations to generate output to be sent to the client for(int clientID = 0; clientID < counters->clients; clientID++) @@ -1284,23 +1284,23 @@ void getClientsOverTime(const int *sock) continue; const int thisclient = client->overTime[slot]; - if(istelnet[*sock]) - ssend(*sock, " %i", thisclient); + if(istelnet[sock]) + ssend(sock, " %i", thisclient); else - pack_int32(*sock, thisclient); + pack_int32(sock, thisclient); } - if(istelnet[*sock]) - ssend(*sock, "\n"); + if(istelnet[sock]) + ssend(sock, "\n"); else - pack_int32(*sock, -1); + pack_int32(sock, -1); } if(excludeclients != NULL) clearSetupVarsArray(); } -void getClientNames(const int *sock) +void getClientNames(const int sock) { // Exit before processing any data if requested via config setting get_privacy_level(NULL); @@ -1353,11 +1353,11 @@ void getClientNames(const int *sock) const char *client_ip = getstr(client->ippos); const char *client_name = getstr(client->namepos); - if(istelnet[*sock]) - ssend(*sock, "%s %s\n", client_name, client_ip); + if(istelnet[sock]) + ssend(sock, "%s %s\n", client_name, client_ip); else { - pack_str32(*sock, client_name); - pack_str32(*sock, client_ip); + pack_str32(sock, client_name); + pack_str32(sock, client_ip); } } @@ -1365,7 +1365,7 @@ void getClientNames(const int *sock) clearSetupVarsArray(); } -void getUnknownQueries(const int *sock) +void getUnknownQueries(const int sock) { // Exit before processing any data if requested via config setting get_privacy_level(NULL); @@ -1401,22 +1401,22 @@ void getUnknownQueries(const int *sock) // Get client IP string const char *clientIP = getstr(client->ippos); - if(istelnet[*sock]) - ssend(*sock, "%lli %i %i %s %s %s %i %s\n", (long long)query->timestamp, queryID, query->id, type, getstr(domain->domainpos), clientIP, query->status, query->flags.complete ? "true" : "false"); + if(istelnet[sock]) + ssend(sock, "%lli %i %i %s %s %s %i %s\n", (long long)query->timestamp, queryID, query->id, type, getstr(domain->domainpos), clientIP, query->status, query->flags.complete ? "true" : "false"); else { - pack_int32(*sock, (int32_t)query->timestamp); - pack_int32(*sock, query->id); + pack_int32(sock, (int32_t)query->timestamp); + pack_int32(sock, query->id); // Use a fixstr because the length of qtype is always 4 (max is 31 for fixstr) - if(!pack_fixstr(*sock, type)) + if(!pack_fixstr(sock, type)) return; // Use str32 for domain and client because we have no idea how long they will be (max is 4294967295 for str32) - if(!pack_str32(*sock, getstr(domain->domainpos)) || !pack_str32(*sock, clientIP)) + if(!pack_str32(sock, getstr(domain->domainpos)) || !pack_str32(sock, clientIP)) return; - pack_uint8(*sock, query->status); - pack_bool(*sock, query->flags.complete); + pack_uint8(sock, query->status); + pack_bool(sock, query->flags.complete); } } } @@ -1424,12 +1424,12 @@ void getUnknownQueries(const int *sock) // FTL_unlink_DHCP_lease() extern bool FTL_unlink_DHCP_lease(const char *ipaddr); -void delete_lease(const char *client_message, const int *sock) +void delete_lease(const char *client_message, const int sock) { // Extract IP address from request char ipaddr[INET6_ADDRSTRLEN] = { 0 }; if(sscanf(client_message, ">delete-lease %"xstr(INET6_ADDRSTRLEN)"s", ipaddr) < 1) { - ssend(*sock, "ERROR: No IP address specified!\n"); + ssend(sock, "ERROR: No IP address specified!\n"); return; } ipaddr[sizeof(ipaddr) - 1] = '\0'; @@ -1438,24 +1438,24 @@ void delete_lease(const char *client_message, const int *sock) logg("Received request to delete lease for %s", ipaddr); if(FTL_unlink_DHCP_lease(ipaddr)) - ssend(*sock, "OK: Removed specified lease\n"); + ssend(sock, "OK: Removed specified lease\n"); else - ssend(*sock, "ERROR: Specified IP address invalid!\n"); + ssend(sock, "ERROR: Specified IP address invalid!\n"); if(config.debug & DEBUG_API) logg("...done"); } -void getDNSport(const int *sock) +void getDNSport(const int sock) { // Return DNS port used by FTL - ssend(*sock, "%d\n", config.dns_port); + ssend(sock, "%d\n", config.dns_port); } -void getMAXLOGAGE(const int *sock) +void getMAXLOGAGE(const int sock) { // Return maxlogage used by FTL - ssend(*sock, "%d\n", config.maxlogage); + ssend(sock, "%d\n", config.maxlogage); } static bool getDefaultInterface(char iface[IF_NAMESIZE], in_addr_t *gw) @@ -1506,13 +1506,13 @@ static bool getDefaultInterface(char iface[IF_NAMESIZE], in_addr_t *gw) return gw != 0; } -void getGateway(const int *sock) +void getGateway(const int sock) { in_addr_t gw = 0; char iface[IF_NAMESIZE] = { 0 }; getDefaultInterface(iface, &gw); - ssend(*sock, "%s %s\n", inet_ntoa(*(struct in_addr *) &gw), iface); + ssend(sock, "%s %s\n", inet_ntoa(*(struct in_addr *) &gw), iface); } struct if_info { @@ -1710,22 +1710,22 @@ static bool listInterfaces(struct if_info **head, char default_iface[IF_NAMESIZE return true; } -static void send_iface(const int *sock, struct if_info *iface) +static bool send_iface(const int sock, struct if_info *iface) { double tx = 0.0, rx = 0.0; char txp[2] = { 0 }, rxp[2] = { 0 }; format_memory_size(txp, iface->tx_bytes, &tx); format_memory_size(rxp, iface->rx_bytes, &rx); - ssend(*sock, "%s %s %i %.1f%sB %.1f%sB %s %s\n", - iface->name, - iface->carrier ? "UP" : "DOWN", - iface->speed, - tx, txp, rx, rxp, - iface->carrier && iface->ip.v4 ? iface->ip.v4 : "-", - iface->carrier && iface->ip.v6 ? iface->ip.v6 : "-"); + return ssend(sock, "%s %s %i %.1f%sB %.1f%sB %s %s\n", + iface->name, + iface->carrier ? "UP" : "DOWN", + iface->speed, + tx, txp, rx, rxp, + iface->carrier && iface->ip.v4 ? iface->ip.v4 : "-", + iface->carrier && iface->ip.v6 ? iface->ip.v6 : "-"); } -void getInterfaces(const int *sock) +void getInterfaces(const int sock) { // Get interface with default route in_addr_t gw = 0; @@ -1736,7 +1736,7 @@ void getInterfaces(const int *sock) struct if_info *ifinfo = NULL; if(!listInterfaces(&ifinfo, default_iface)) { - ssend(*sock, "ERROR"); + ssend(sock, "ERROR"); return; } diff --git a/src/api/api.h b/src/api/api.h index afa1a935..5867859d 100644 --- a/src/api/api.h +++ b/src/api/api.h @@ -11,29 +11,29 @@ #define API_H // Statistic methods -void getStats(const int *sock); -void getOverTime(const int *sock); -void getTopDomains(const char *client_message, const int *sock); -void getTopClients(const char *client_message, const int *sock); -void getUpstreamDestinations(const char *client_message, const int *sock); -void getQueryTypes(const int *sock); -void getAllQueries(const char *client_message, const int *sock); -void getRecentBlocked(const char *client_message, const int *sock); -void getClientsOverTime(const int *sock); -void getClientNames(const int *sock); +void getStats(const int sock); +void getOverTime(const int sock); +void getTopDomains(const char *client_message, const int sock); +void getTopClients(const char *client_message, const int sock); +void getUpstreamDestinations(const char *client_message, const int sock); +void getQueryTypes(const int sock); +void getAllQueries(const char *client_message, const int sock); +void getRecentBlocked(const char *client_message, const int sock); +void getClientsOverTime(const int sock); +void getClientNames(const int sock); // FTL methods -void getClientID(const int *sock); -void getVersion(const int *sock); -void getDBstats(const int *sock); -void getUnknownQueries(const int *sock); -void getMAXLOGAGE(const int *sock); -void getGateway(const int *sock); -void getInterfaces(const int *sock); +void getClientID(const int sock); +void getVersion(const int sock); +void getDBstats(const int sock); +void getUnknownQueries(const int sock); +void getMAXLOGAGE(const int sock); +void getGateway(const int sock); +void getInterfaces(const int sock); // DNS resolver methods (dnsmasq_interface.c) -void getCacheInformation(const int *sock); -void getDNSport(const int *sock); +void getCacheInformation(const int sock); +void getDNSport(const int sock); // MessagePack serialization helpers void pack_eom(const int sock); @@ -48,6 +48,6 @@ bool pack_str32(const int sock, const char *string); void pack_map16_start(const int sock, const uint16_t length); // DHCP lease management -void delete_lease(const char *client_message, const int *sock); +void delete_lease(const char *client_message, const int sock); #endif // API_H diff --git a/src/api/request.c b/src/api/request.c index 5ebc28cc..88bb5a01 100644 --- a/src/api/request.c +++ b/src/api/request.c @@ -26,7 +26,7 @@ bool __attribute__((pure)) command(const char *client_message, const char* cmd) return strstr(client_message, cmd) != NULL; } -void process_request(const char *client_message, int *sock) +bool process_request(const char *client_message, const int sock) { char EOT[2]; EOT[0] = 0x04; @@ -191,21 +191,15 @@ void process_request(const char *client_message, int *sock) if(command(client_message, ">quit") || command(client_message, EOT)) { if(config.debug & DEBUG_API) - logg("Received >quit or EOT on socket %d", *sock); - processed = true; - close(*sock); - *sock = 0; + logg("Received >quit or EOT on socket %d", sock); + return true; } if(!processed) - { - ssend(*sock, "unknown command: %s\n", client_message); - } + ssend(sock, "unknown command: %s\n", client_message); - // End of queryable commands - if(*sock != 0) - { - // Send EOM - seom(*sock); - } + // End of queryable commands: Send EOM + seom(sock); + + return false; } diff --git a/src/api/request.h b/src/api/request.h index ab2baa0a..5e57f582 100644 --- a/src/api/request.h +++ b/src/api/request.h @@ -10,7 +10,7 @@ #ifndef REQUEST_H #define REQUEST_H -void process_request(const char *client_message, int *sock); +bool process_request(const char *client_message, const int sock); bool command(const char *client_message, const char* cmd) __attribute__((pure)); #endif //REQUEST_H diff --git a/src/api/socket.c b/src/api/socket.c index e04dfd58..55a4016c 100644 --- a/src/api/socket.c +++ b/src/api/socket.c @@ -31,9 +31,7 @@ #define BACKLOG 5 // File descriptors -int socketfd = 0, telnetfd4 = 0, telnetfd6 = 0; bool dualstack = false; -bool ipv4telnet = false, ipv6telnet = false, sock_avail = false; bool istelnet[MAXCONNS]; void saveport(int port) @@ -69,12 +67,11 @@ void saveport(int port) } } -static bool bind_to_telnet_port_IPv4(int *socketdescriptor) +static int bind_to_telnet_port_IPv4(void) { // IPv4 socket - *socketdescriptor = socket(AF_INET, SOCK_STREAM, 0); - - if(*socketdescriptor < 0) + const int socketdescriptor = socket(AF_INET, SOCK_STREAM, 0); + if(socketdescriptor < 0) { logg("Error opening IPv4 telnet socket: %s (%i)", strerror(errno), errno); exit(EXIT_FAILURE); @@ -87,7 +84,7 @@ static bool bind_to_telnet_port_IPv4(int *socketdescriptor) // new instance will fail if there were connections open to the previous // instance when you killed it. Those connections will hold the TCP port in // the TIME_WAIT state for 30-120 seconds, so you fall into case 1 above. - if(setsockopt(*socketdescriptor, SOL_SOCKET, SO_REUSEADDR, &(int){ 1 }, sizeof(int)) != 0) + if(setsockopt(socketdescriptor, SOL_SOCKET, SO_REUSEADDR, &(int){ 1 }, sizeof(int)) != 0) logg("WARN: allowing re-binding (IPv6) failed: %s", strerror(errno)); struct sockaddr_in serv_addr4; @@ -102,29 +99,28 @@ static bool bind_to_telnet_port_IPv4(int *socketdescriptor) // Bind to IPv4 port serv_addr4.sin_port = htons(config.port); - if(bind(*socketdescriptor, (struct sockaddr *) &serv_addr4, sizeof(serv_addr4)) < 0) + if(bind(socketdescriptor, (struct sockaddr *) &serv_addr4, sizeof(serv_addr4)) < 0) { logg("Error listening on IPv4 port %i: %s (%i)", config.port, strerror(errno), errno); - return false; + return -1; } // The listen system call allows the process to listen on the socket for connections - if(listen(*socketdescriptor, BACKLOG) == -1) + if(listen(socketdescriptor, BACKLOG) == -1) { logg("Error listening on IPv4 socket: %s (%i)", strerror(errno), errno); - return false; + return -1; } logg("Listening on port %i for incoming IPv4 telnet connections", config.port); - return true; + return socketdescriptor; } -static bool bind_to_telnet_port_IPv6(int *socketdescriptor) +static int bind_to_telnet_port_IPv6(void) { // IPv6 socket - *socketdescriptor = socket(AF_INET6, SOCK_STREAM, 0); - - if(*socketdescriptor < 0) + const int socketdescriptor = socket(AF_INET6, SOCK_STREAM, 0); + if(socketdescriptor < 0) { logg("Error opening IPv6 telnet socket: %s (%i)", strerror(errno), errno); exit(EXIT_FAILURE); @@ -134,7 +130,7 @@ static bool bind_to_telnet_port_IPv6(int *socketdescriptor) // stricted to sending and receiving IPv6 packets only. In this // case, an IPv4 and an IPv6 application can bind to a single port // at the same time. - if(setsockopt(*socketdescriptor, IPPROTO_IPV6, IPV6_V6ONLY, &(int){ 1 }, sizeof(int)) != 0) + if(setsockopt(socketdescriptor, IPPROTO_IPV6, IPV6_V6ONLY, &(int){ 1 }, sizeof(int)) != 0) logg("WARN: setting socket to IPv6-only failed: %s", strerror(errno)); // Set SO_REUSEADDR to allow re-binding to the port that has been used @@ -144,7 +140,7 @@ static bool bind_to_telnet_port_IPv6(int *socketdescriptor) // new instance will fail if there were connections open to the previous // instance when you killed it. Those connections will hold the TCP port in // the TIME_WAIT state for 30-120 seconds, so you fall into case 1 above. - if(setsockopt(*socketdescriptor, SOL_SOCKET, SO_REUSEADDR, &(int){ 1 }, sizeof(int)) != 0) + if(setsockopt(socketdescriptor, SOL_SOCKET, SO_REUSEADDR, &(int){ 1 }, sizeof(int)) != 0) logg("WARN: allowing re-binding (IPv6) failed: %s", strerror(errno)); struct sockaddr_in6 serv_addr; @@ -166,31 +162,30 @@ static bool bind_to_telnet_port_IPv6(int *socketdescriptor) // Bind to IPv6 socket serv_addr.sin6_port = htons(config.port); - if(bind(*socketdescriptor, (struct sockaddr *) &serv_addr, sizeof(serv_addr)) < 0) + if(bind(socketdescriptor, (struct sockaddr *) &serv_addr, sizeof(serv_addr)) < 0) { logg("Error listening on IPv6 port %i: %s (%i)", config.port, strerror(errno), errno); - return false; + return -1; } // The listen system call allows the process to listen on the socket for connections - if(listen(*socketdescriptor, BACKLOG) == -1) + if(listen(socketdescriptor, BACKLOG) == -1) { logg("Error listening on IPv6 socket: %s (%i)", strerror(errno), errno); - return false; + return -1; } logg("Listening on port %i for incoming IPv6 telnet connections", config.port); - return true; + return socketdescriptor; } -static bool bind_to_unix_socket(int *socketdescriptor) +static int bind_to_unix_socket(void) { - *socketdescriptor = socket(AF_LOCAL, SOCK_STREAM, 0); - - if(*socketdescriptor < 0) + const int socketdescriptor = socket(AF_LOCAL, SOCK_STREAM, 0); + if(socketdescriptor < 0) { logg("WARNING: Error opening Unix socket."); - return false; + return -1; } // Make sure unix socket file handle does not exist, if it exists, remove it @@ -208,21 +203,21 @@ static bool bind_to_unix_socket(int *socketdescriptor) // Bind to Unix socket handle errno = 0; - if(bind(*socketdescriptor, (struct sockaddr *) &address, sizeof (address)) != 0) + if(bind(socketdescriptor, (struct sockaddr *) &address, sizeof (address)) != 0) { logg("WARNING: Cannot bind on Unix socket %s: %s (%i)", FTLfiles.socketfile, strerror(errno), errno); - return false; + return -1; } // The listen system call allows the process to listen on the Unix socket for connections - if(listen(*socketdescriptor, BACKLOG) == -1) + if(listen(socketdescriptor, BACKLOG) == -1) { logg("WARNING: Cannot listen on Unix socket: %s (%i)", strerror(errno), errno); - return false; + return -1; } logg("Listening on Unix socket"); - return true; + return socketdescriptor; } void seom(const int sock) @@ -233,7 +228,7 @@ void seom(const int sock) pack_eom(sock); } -void __attribute__ ((format (gnu_printf, 2, 3))) ssend(const int sock, const char *format, ...) +bool __attribute__ ((format (gnu_printf, 5, 6))) _ssend(const int sock, const char *file, const char *func, const int line, const char *format, ...) { char *buffer; va_list args; @@ -242,68 +237,16 @@ void __attribute__ ((format (gnu_printf, 2, 3))) ssend(const int sock, const cha va_end(args); if(bytes > 0 && buffer != NULL) { - write(sock, buffer, bytes); + FTLwrite(sock, buffer, bytes, short_path(file), func, line); free(buffer); } -} - -static int listener(const int sockfd, const char type) -{ - socklen_t socklen = 0; - struct sockaddr_un un_addr; - struct sockaddr_in in4_addr; - struct sockaddr_in6 in6_addr; - - switch(type) - { - case 0: // Unix socket - memset(&un_addr, 0, sizeof(un_addr)); - socklen = sizeof(un_addr); - return accept(sockfd, (struct sockaddr *) &un_addr, &socklen); - - case 4: // Internet socket (IPv4) - memset(&in4_addr, 0, sizeof(in4_addr)); - socklen = sizeof(un_addr); - return accept(sockfd, (struct sockaddr *) &in4_addr, &socklen); - - case 6: // Internet socket (IPv6) - memset(&in6_addr, 0, sizeof(in6_addr)); - socklen = sizeof(un_addr); - return accept(sockfd, (struct sockaddr *) &in6_addr, &socklen); - - default: // Should not happen - logg("Cannot listen on type %i connection, code error!", type); - exit(EXIT_FAILURE); - } -} - -void close_telnet_socket(void) -{ - // Using global variable here - if(telnetfd4 != 0) - close(telnetfd4); - if(telnetfd6 != 0) - close(telnetfd6); -} - -void close_unix_socket(bool unlink_file) -{ - // Using global variable here - if(sock_avail != 0) - close(socketfd); - - // The process has to take care of unlinking the socket file description - // on exit - if(unlink_file) - { - unlink(FTLfiles.socketfile); - } + return errno == 0; } static void *telnet_connection_handler_thread(void *socket_desc) { //Get the socket descriptor - int sock = *(int*)socket_desc; + const int sock = *(int*)socket_desc; // Set connection type to telnet istelnet[sock] = true; @@ -334,7 +277,7 @@ static void *telnet_connection_handler_thread(void *socket_desc) } if(config.debug & DEBUG_API) - logg("New telnet thread for socket %d", *(int*)socket_desc); + logg("New telnet thread for socket %d", sock); // Receive from client ssize_t n; @@ -348,7 +291,7 @@ static void *telnet_connection_handler_thread(void *socket_desc) if(message == NULL) { if(config.debug & DEBUG_API) - logg("Break in telnet thread for socket %d: Memory error", *(int*)socket_desc); + logg("Break in telnet thread for socket %d: Memory error", sock); break; } @@ -356,31 +299,23 @@ static void *telnet_connection_handler_thread(void *socket_desc) memset(client_message, 0, sizeof client_message); // Process received message - process_request(message, &sock); + const bool eom = process_request(message, sock); free(message); - - if(sock == 0) - { - // Client disconnected by sending EOT or ">quit" - if(config.debug & DEBUG_API) - logg("Break in telnet thread for socket %d: Client disconnected", *(int*)socket_desc); - break; - } + if(eom) break; } else if(n == -1) { if(config.debug & DEBUG_API) - logg("Break in telnet thread for socket %d: No data received", *(int*)socket_desc); + logg("Break in telnet thread for socket %d: No data received", sock); break; } } if(config.debug & DEBUG_API) - logg("Terminating telnet thread for socket %d", *(int*)socket_desc); + logg("Terminating telnet thread for socket %d", sock); - // Free the socket pointer - if(sock != 0) - close(sock); + // Close the socket pointer + close(sock); free(socket_desc); // Release thread from list @@ -393,7 +328,7 @@ static void *telnet_connection_handler_thread(void *socket_desc) static void *socket_connection_handler_thread(void *socket_desc) { //Get the socket descriptor - int sock = *(int*)socket_desc; + const int sock = *(int*)socket_desc; // Set connection type to not telnet istelnet[sock] = false; @@ -430,7 +365,7 @@ static void *socket_connection_handler_thread(void *socket_desc) // Receive from client ssize_t n; - while((n = recv(sock,client_message,SOCKETBUFFERLEN-1, 0))) + while((n = recv(sock, client_message, SOCKETBUFFERLEN-1, 0))) { if (n > 0 && n < SOCKETBUFFERLEN) { @@ -443,14 +378,9 @@ static void *socket_connection_handler_thread(void *socket_desc) memset(client_message, 0, sizeof client_message); // Process received message - process_request(message, &sock); + const bool eom = process_request(message, sock); free(message); - - if(sock == 0) - { - // Socket connection interrupted by sending EOT or ">quit" - break; - } + if(eom) break; } else if(n == -1) { @@ -458,9 +388,8 @@ static void *socket_connection_handler_thread(void *socket_desc) } } - // Free the socket pointer - if(sock != 0) - close(sock); + // Close the socket pointer + close(sock); free(socket_desc); // Release thread from list @@ -484,43 +413,58 @@ void *telnet_listening_thread_IPv4(void *args) thread_names[TELNETv4] = "telnet-IPv4"; prctl(PR_SET_NAME, thread_names[TELNETv4], 0, 0, 0); - // Initialize IPv4 telnet socket - ipv4telnet = bind_to_telnet_port_IPv4(&telnetfd4); - if(!ipv4telnet) - return NULL; - - thread_cancellable[TELNETv4] = true; - - // Listen as long as this thread is not canceled while(!killed) { - // Look for new clients that want to connect - const int csck = listener(telnetfd4, 4); - if(csck == -1) + // Initialize IPv4 telnet socket´ + const int telnetfd4 = bind_to_telnet_port_IPv4(); + if(telnetfd4 < 0) + return NULL; + + if(config.debug & DEBUG_API) + logg("Telnet-IPv4 listener accepting on fd %d", telnetfd4); + + thread_cancellable[TELNETv4] = true; + + // Listen as long as this thread is not canceled + int errors = 0; + while(!killed) { - logg("IPv4 telnet error: %s (%i)", strerror(errno), errno); - continue; + // Look for new clients that want to connect + const int csck = accept(telnetfd4, NULL, NULL); + if(csck == -1) + { + logg("IPv4 telnet error: %s (%i, sockfd: %d)", strerror(errno), errno, telnetfd4); + if(errors++ > 20) + break; + continue; + } + + if(config.debug & DEBUG_API) + logg("Accepting new telnet connection at socket %d", csck); + + // Allocate memory used to transport client socket ID to client listening thread + int *newsock; + newsock = calloc(1,sizeof(int)); + if(newsock == NULL) break; + *newsock = csck; + + pthread_t telnet_connection_thread; + // Create a new thread + if(pthread_create( &telnet_connection_thread, &attr, telnet_connection_handler_thread, (void*) newsock ) != 0) + { + // Log the error code description + logg("WARNING: Unable to open telnet processing thread: %s", strerror(errno)); + } } if(config.debug & DEBUG_API) - logg("Accepting new telnet connection at socket %d", csck); + logg("Telnet-IPv4 listener closed (fd %d)", telnetfd4); - // Allocate memory used to transport client socket ID to client listening thread - int *newsock; - newsock = calloc(1,sizeof(int)); - if(newsock == NULL) break; - *newsock = csck; - - pthread_t telnet_connection_thread; - // Create a new thread - if(pthread_create( &telnet_connection_thread, &attr, telnet_connection_handler_thread, (void*) newsock ) != 0) - { - // Log the error code description - logg("WARNING: Unable to open telnet processing thread: %s", strerror(errno)); - } + // Close telnet socket + close(telnetfd4); + thread_sleepms(TELNETv4, 1000); } - if(config.debug & DEBUG_API) - logg("Terminating IPv4 telnet thread"); + logg("Terminating IPv4 telnet thread"); return NULL; } @@ -542,38 +486,53 @@ void *telnet_listening_thread_IPv6(void *args) if(!ipv6_available()) return NULL; - ipv6telnet = bind_to_telnet_port_IPv6(&telnetfd6); - if(!ipv6telnet) - return NULL; - - thread_cancellable[TELNETv6] = true; - - // Listen as long as this thread is not canceled while(!killed) { - // Look for new clients that want to connect - const int csck = listener(telnetfd6, 6); - if(csck == -1) + const int telnetfd6 = bind_to_telnet_port_IPv6(); + if(telnetfd6 < 0) + return NULL; + + if(config.debug & DEBUG_API) + logg("Telnet-IPv6 listener accepting on fd %d", telnetfd6); + + thread_cancellable[TELNETv6] = true; + + // Listen as long as this thread is not canceled + int errors = 0; + while(!killed) { - logg("IPv6 telnet error: %s (%i)", strerror(errno), errno); - continue; + // Look for new clients that want to connect + const int csck = accept(telnetfd6, NULL, NULL); + if(csck == -1) + { + logg("IPv6 telnet error: %s (%i, sockfd: %d)", strerror(errno), errno, telnetfd6); + if(errors++ > 20) + break; + continue; + } + + // Allocate memory used to transport client socket ID to client listening thread + int *newsock; + newsock = calloc(1,sizeof(int)); + if(newsock == NULL) break; + *newsock = csck; + + pthread_t telnet_connection_thread; + // Create a new thread + if(pthread_create( &telnet_connection_thread, &attr, telnet_connection_handler_thread, (void*) newsock ) != 0) + { + // Log the error code description + logg("WARNING: Unable to open telnet processing thread: %s", strerror(errno)); + } } - // Allocate memory used to transport client socket ID to client listening thread - int *newsock; - newsock = calloc(1,sizeof(int)); - if(newsock == NULL) break; - *newsock = csck; + if(config.debug & DEBUG_API) + logg("Telnet-IPv6 listener closed (fd %d)", telnetfd6); - pthread_t telnet_connection_thread; - // Create a new thread - if(pthread_create( &telnet_connection_thread, &attr, telnet_connection_handler_thread, (void*) newsock ) != 0) - { - // Log the error code description - logg("WARNING: Unable to open telnet processing thread: %s", strerror(errno)); - } + // Close telnet socket + close(telnetfd6); + thread_sleepms(TELNETv6, 1000); } - logg("Terminating IPv6 telnet thread"); return NULL; } @@ -592,39 +551,53 @@ void *socket_listening_thread(void *args) thread_names[SOCKET] = "telnet-socket"; prctl(PR_SET_NAME, thread_names[SOCKET], 0, 0, 0); - // Return early to avoid CPU spinning if Unix socket is not available - sock_avail = bind_to_unix_socket(&socketfd); - if(!sock_avail) - { - logg("INFO: Unix socket will not be available"); - return NULL; - } - - thread_cancellable[SOCKET] = true; - - // Listen as long as this thread is not canceled while(!killed) { - // Look for new clients that want to connect - const int csck = listener(socketfd, 0); - if(csck < 0) - continue; - - // Allocate memory used to transport client socket ID to client listening thread - int *newsock; - newsock = calloc(1,sizeof(int)); - if(newsock == NULL) break; - *newsock = csck; - - pthread_t socket_connection_thread; - // Create a new thread - if(pthread_create( &socket_connection_thread, &attr, socket_connection_handler_thread, (void*) newsock ) != 0) + // Return early to avoid CPU spinning if Unix socket is not available + const int socketfd = bind_to_unix_socket(); + if(socketfd < 0) { - // Log the error code description - logg("WARNING: Unable to open socket processing thread: %s", strerror(errno)); + logg("INFO: Unix socket will not be available"); + return NULL; } - } + if(config.debug & DEBUG_API) + logg("Telnet-socket listener accepting on fd %d", socketfd); + + thread_cancellable[SOCKET] = true; + + // Listen as long as this thread is not canceled + while(!killed) + { + // Look for new clients that want to connect + const int csck = accept(socketfd, NULL, NULL); + if(csck < 0) + continue; + + // Allocate memory used to transport client socket ID to client listening thread + int *newsock; + newsock = calloc(1,sizeof(int)); + if(newsock == NULL) break; + *newsock = csck; + + pthread_t socket_connection_thread; + // Create a new thread + if(pthread_create( &socket_connection_thread, &attr, socket_connection_handler_thread, (void*) newsock ) != 0) + { + // Log the error code description + logg("WARNING: Unable to open socket processing thread: %s", strerror(errno)); + } + } + + if(config.debug & DEBUG_API) + logg("Telnet-socket listener closed (fd %d)", socketfd); + + // Close Unix socket + close(socketfd); + } + // The process has to take care of unlinking the socket file description + // on exit + unlink(FTLfiles.socketfile); logg("Terminating socket thread"); return NULL; } diff --git a/src/api/socket.h b/src/api/socket.h index bfb7150f..8ee6a173 100644 --- a/src/api/socket.h +++ b/src/api/socket.h @@ -11,17 +11,16 @@ #define SOCKET_H void saveport(int port); -void close_telnet_socket(void); void close_unix_socket(bool unlink_file); void seom(const int sock); -void ssend(const int sock, const char *format, ...) __attribute__ ((format (gnu_printf, 2, 3))); +#define ssend(sock, format, ...) _ssend(sock, __FILE__, __FUNCTION__, __LINE__, format, ##__VA_ARGS__) +bool _ssend(const int sock, const char *file, const char *func, const int line, const char *format, ...) __attribute__ ((format (gnu_printf, 5, 6))); void *telnet_listening_thread_IPv4(void *args); void *telnet_listening_thread_IPv6(void *args); void *socket_listening_thread(void *args); bool ipv6_available(void); void bind_sockets(void); -extern bool ipv4telnet, ipv6telnet; extern bool istelnet[MAXCONNS]; #endif //SOCKET_H diff --git a/src/daemon.c b/src/daemon.c index 14c8f34b..b9dfe1f3 100644 --- a/src/daemon.c +++ b/src/daemon.c @@ -14,7 +14,7 @@ #include "log.h" // sleepms() #include "timers.h" -// close_telnet_socket() +// saveport() #include "api/socket.h" // gravityDB_close() #include "database/gravity-db.h" @@ -288,10 +288,6 @@ void cleanup(const int ret) lock_shm(); gravityDB_close(); unlock_shm(); - - // Close sockets and delete Unix socket file handle - close_telnet_socket(); - close_unix_socket(true); } // Empty API port file, port 0 = truncate file diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c index 0730c40b..286a2bb2 100644 --- a/src/dnsmasq_interface.c +++ b/src/dnsmasq_interface.c @@ -2851,11 +2851,11 @@ static char *get_ptrname(struct in_addr *addr) } // int cache_inserted, cache_live_freed are defined in dnsmasq/cache.c -void getCacheInformation(const int *sock) +void getCacheInformation(const int sock) { struct cache_info ci; get_dnsmasq_cache_info(&ci); - ssend(*sock,"cache-size: %i\ncache-live-freed: %i\ncache-inserted: %i\nipv4: %i\nipv6: %i\nsrv: %i\ncname: %i\nds: %i\ndnskey: %i\nother: %i\nexpired: %i\nimmortal: %i\n", + ssend(sock, "cache-size: %i\ncache-live-freed: %i\ncache-inserted: %i\nipv4: %i\nipv6: %i\nsrv: %i\ncname: %i\nds: %i\ndnskey: %i\nother: %i\nexpired: %i\nimmortal: %i\n", daemon->cachesize, daemon->metrics[METRIC_DNS_CACHE_LIVE_FREED], daemon->metrics[METRIC_DNS_CACHE_INSERTED], @@ -3133,15 +3133,6 @@ void FTL_TCP_worker_created(const int confd) if(config.debug != 0) logg("Reopening Gravity database for this fork"); gravityDB_forked(); - - // Children inherit file descriptors from their parents - // We don't need them in the forks, so we clean them up - if(config.debug != 0) - logg("Closing Telnet socket for this fork"); - close_telnet_socket(); - if(config.debug != 0) - logg("Closing Unix socket for this fork"); - close_unix_socket(false); } bool FTL_unlink_DHCP_lease(const char *ipaddr) diff --git a/src/dnsmasq_interface.h b/src/dnsmasq_interface.h index 7abca12e..a35b7381 100644 --- a/src/dnsmasq_interface.h +++ b/src/dnsmasq_interface.h @@ -15,7 +15,6 @@ #include "edns0.h" -extern int socketfd, telnetfd4, telnetfd6; extern unsigned char* pihole_privacylevel; enum protocol { TCP, UDP, INTERNAL }; diff --git a/src/syscalls/accept.c b/src/syscalls/accept.c index 535f18aa..5c90d2bf 100644 --- a/src/syscalls/accept.c +++ b/src/syscalls/accept.c @@ -15,7 +15,7 @@ #undef accept int FTLaccept(int sockfd, struct sockaddr *addr, socklen_t *addrlen, const char *file, const char *func, const int line) { - int ret = 0; + int ret = 0; do { // Reset errno before trying to write @@ -26,11 +26,17 @@ int FTLaccept(int sockfd, struct sockaddr *addr, socklen_t *addrlen, const char // incoming signal while(ret < 0 && errno == EINTR); + // Backup errno value + const int _errno = errno; + // Final error checking (may have failed for some other reason then an // EINTR = interrupted system call) if(ret < 0) logg("WARN: Could not accept() in %s() (%s:%i): %s", - func, file, line, strerror(errno)); + func, file, line, strerror(errno)); - return ret; + // Restore errno value + errno = _errno; + + return ret; } \ No newline at end of file diff --git a/src/syscalls/calloc.c b/src/syscalls/calloc.c index b79ad5fd..54621347 100644 --- a/src/syscalls/calloc.c +++ b/src/syscalls/calloc.c @@ -30,10 +30,17 @@ void* __attribute__((malloc)) __attribute__((alloc_size(1,2))) FTLcalloc(const s // an incoming signal while(ptr == NULL && errno == EINTR); + // Backup errno value + const int _errno = errno; + // Handle other errors than EINTR if(ptr == NULL) logg("FATAL: Memory allocation (%zu x %zu) failed in %s() (%s:%i)", nmemb, size, func, file, line); + // Restore errno value + errno = _errno; + + // Return memory pointer return ptr; } diff --git a/src/syscalls/fopen.c b/src/syscalls/fopen.c index f11c4375..d6416533 100644 --- a/src/syscalls/fopen.c +++ b/src/syscalls/fopen.c @@ -28,6 +28,9 @@ FILE *FTLfopen(const char *pathname, const char *mode, const char *file, const c // incoming signal while(file_ptr == NULL && errno == EINTR); + // Backup errno value + const int _errno = errno; + // Final error checking (may have failed for some other reason then an // EINTR = interrupted system call) // The already_writing counter prevents a possible infinite loop @@ -38,5 +41,9 @@ FILE *FTLfopen(const char *pathname, const char *mode, const char *file, const c // Decrement warning counter already_writing--; + // Restore errno value + errno = _errno; + + // Return file pointer return file_ptr; } diff --git a/src/syscalls/ftlallocate.c b/src/syscalls/ftlallocate.c index 91597cb7..56bc3944 100644 --- a/src/syscalls/ftlallocate.c +++ b/src/syscalls/ftlallocate.c @@ -35,5 +35,6 @@ int FTLfallocate(const int fd, const off_t offset, const off_t len, const char * // Set errno ourselves as posix_fallocate doesn't do it errno = ret; + return ret; } diff --git a/src/syscalls/pthread_mutex_lock.c b/src/syscalls/pthread_mutex_lock.c index 2b6c942a..5f66549e 100644 --- a/src/syscalls/pthread_mutex_lock.c +++ b/src/syscalls/pthread_mutex_lock.c @@ -17,7 +17,7 @@ #undef pthread_mutex_lock int FTLpthread_mutex_lock(pthread_mutex_t *__mutex, const char *file, const char *func, const int line) { - ssize_t ret = 0; + ssize_t ret = 0; do { // Reset errno before trying to write @@ -28,11 +28,17 @@ int FTLpthread_mutex_lock(pthread_mutex_t *__mutex, const char *file, const char // incoming signal while(ret < 0 && errno == EINTR); + // Backup errno value + const int _errno = errno; + // Final error checking (may have failed for some other reason then an // EINTR = interrupted system call) if(ret < 0) logg("WARN: Could not pthread_mutex_lock() in %s() (%s:%i): %s", - func, file, line, strerror(errno)); + func, file, line, strerror(errno)); - return ret; + // Restore errno value + errno = _errno; + + return ret; } \ No newline at end of file diff --git a/src/syscalls/realloc.c b/src/syscalls/realloc.c index 53df93b1..9bdb3fa5 100644 --- a/src/syscalls/realloc.c +++ b/src/syscalls/realloc.c @@ -34,10 +34,16 @@ void __attribute__((alloc_size(2))) *FTLrealloc(void *ptr_in, const size_t size, // an incoming signal while(ptr_out == NULL && errno == EINTR); + // Backup errno value + const int _errno = errno; + // Handle other errors than EINTR if(ptr_out == NULL) logg("FATAL: Memory reallocation (%p -> %zu) failed in %s() (%s:%i)", ptr_in, size, func, file, line); + // Restore errno value + errno = _errno; + return ptr_out; } \ No newline at end of file diff --git a/src/syscalls/recv.c b/src/syscalls/recv.c index 999edc4c..aff6ea56 100644 --- a/src/syscalls/recv.c +++ b/src/syscalls/recv.c @@ -17,7 +17,7 @@ #undef recv ssize_t FTLrecv(int sockfd, void *buf, size_t len, int flags, const char *file, const char *func, const int line) { - ssize_t ret = 0; + ssize_t ret = 0; do { // Reset errno before trying to write @@ -28,11 +28,17 @@ ssize_t FTLrecv(int sockfd, void *buf, size_t len, int flags, const char *file, // incoming signal while(ret < 0 && errno == EINTR); + // Backup errno value + const int _errno = errno; + // Final error checking (may have failed for some other reason then an // EINTR = interrupted system call) if(ret < 0) logg("WARN: Could not recv() in %s() (%s:%i): %s", - func, file, line, strerror(errno)); + func, file, line, strerror(errno)); - return ret; + // Restore errno value + errno = _errno; + + return ret; } \ No newline at end of file diff --git a/src/syscalls/recvfrom.c b/src/syscalls/recvfrom.c index 322a9727..8db0ac46 100644 --- a/src/syscalls/recvfrom.c +++ b/src/syscalls/recvfrom.c @@ -18,7 +18,7 @@ #undef recvfrom ssize_t FTLrecvfrom(int sockfd, void *buf, size_t len, int flags, struct sockaddr *src_addr, socklen_t *addrlen, const char *file, const char *func, const int line) { - ssize_t ret = 0; + ssize_t ret = 0; do { // Reset errno before trying to write @@ -29,11 +29,17 @@ ssize_t FTLrecvfrom(int sockfd, void *buf, size_t len, int flags, struct sockadd // incoming signal while(ret < 0 && errno == EINTR); + // Backup errno value + const int _errno = errno; + // Final error checking (may have failed for some other reason then an // EINTR = interrupted system call) if(ret < 0) logg("WARN: Could not recvfrom() in %s() (%s:%i): %s", - func, file, line, strerror(errno)); + func, file, line, strerror(errno)); - return ret; + // Restore errno value + errno = _errno; + + return ret; } \ No newline at end of file diff --git a/src/syscalls/select.c b/src/syscalls/select.c index 41c384ca..4e73b6c3 100644 --- a/src/syscalls/select.c +++ b/src/syscalls/select.c @@ -17,7 +17,7 @@ #undef select int FTLselect(int nfds, fd_set *readfds, fd_set *writefds, fd_set *exceptfds, struct timeval *timeout, const char *file, const char *func, const int line) { - int ret = 0; + int ret = 0; do { // Reset errno before trying to write @@ -28,11 +28,17 @@ int FTLselect(int nfds, fd_set *readfds, fd_set *writefds, fd_set *exceptfds, st // incoming signal while(ret < 0 && errno == EINTR); + // Backup errno value + const int _errno = errno; + // Final error checking (may have failed for some other reason then an // EINTR = interrupted system call) if(ret < 0) logg("WARN: Could not select() in %s() (%s:%i): %s", - func, file, line, strerror(errno)); + func, file, line, strerror(errno)); - return ret; + // Restore errno value + errno = _errno; + + return ret; } \ No newline at end of file diff --git a/src/syscalls/sendto.c b/src/syscalls/sendto.c index db5e4ff4..cc7234c9 100644 --- a/src/syscalls/sendto.c +++ b/src/syscalls/sendto.c @@ -18,7 +18,7 @@ #undef sendto ssize_t FTLsendto(int sockfd, void *buf, size_t len, int flags, const struct sockaddr *dest_addr, socklen_t addrlen, const char *file, const char *func, const int line) { - ssize_t ret = 0; + ssize_t ret = 0; do { // Reset errno before trying to write @@ -29,11 +29,17 @@ ssize_t FTLsendto(int sockfd, void *buf, size_t len, int flags, const struct soc // incoming signal while(ret < 0 && errno == EINTR); + // Backup errno value + const int _errno = errno; + // Final error checking (may have failed for some other reason then an // EINTR = interrupted system call) if(ret < 0) logg("WARN: Could not sendto() in %s() (%s:%i): %s", - func, file, line, strerror(errno)); + func, file, line, strerror(errno)); - return ret; + // Restore errno value + errno = _errno; + + return ret; } \ No newline at end of file diff --git a/src/syscalls/vasprintf.c b/src/syscalls/vasprintf.c index 8300a8c5..3ac340e6 100644 --- a/src/syscalls/vasprintf.c +++ b/src/syscalls/vasprintf.c @@ -54,6 +54,9 @@ int FTLvasprintf(const char *file, const char *func, const int line, char **buff stdout, _errno, format, func, file, line); } + // Restore errno value + errno = _errno; + // Return number of written bytes return length; } diff --git a/src/syscalls/vfprintf.c b/src/syscalls/vfprintf.c index b945ae4f..7a5ffae9 100644 --- a/src/syscalls/vfprintf.c +++ b/src/syscalls/vfprintf.c @@ -134,6 +134,9 @@ int FTLvfprintf(FILE *stream, const char *file, const char *func, const int line if(buffer != NULL) free(buffer); + // Restore errno value + errno = _errno; + // Return early, there isn't anything we can do here return length; } @@ -152,6 +155,9 @@ int FTLvfprintf(FILE *stream, const char *file, const char *func, const int line // to an interruption by an incoming signal while(_buffer < buffer && errno == EINTR); + // Backup errno value + _errno = errno; + // Final error checking (may have failed for some other reason then an // EINTR = interrupted system call) if(_buffer < buffer) @@ -163,6 +169,9 @@ int FTLvfprintf(FILE *stream, const char *file, const char *func, const int line // Free allocated memory free(buffer); + // Restore errno value + errno = _errno; + // Return number of written bytes return length; } diff --git a/src/syscalls/vsnprintf.c b/src/syscalls/vsnprintf.c index 2e1b5f08..4f4badfc 100644 --- a/src/syscalls/vsnprintf.c +++ b/src/syscalls/vsnprintf.c @@ -47,6 +47,10 @@ int FTLvsnprintf(const char *file, const char *func, const int line, char *__res stdout, _errno, format, func, file, line); } + + // Restore errno value + errno = _errno; + // Return number of written bytes return length; } diff --git a/src/syscalls/vsprintf.c b/src/syscalls/vsprintf.c index b31ba414..cce0b35f 100644 --- a/src/syscalls/vsprintf.c +++ b/src/syscalls/vsprintf.c @@ -54,6 +54,9 @@ int FTLvsprintf(const char *file, const char *func, const int line, char *__rest stdout, _errno, format, func, file, line); } + // Restore errno value + errno = _errno; + // Return number of written bytes return length; } diff --git a/src/syscalls/write.c b/src/syscalls/write.c index fc7ee2f1..0aba271f 100644 --- a/src/syscalls/write.c +++ b/src/syscalls/write.c @@ -21,26 +21,33 @@ ssize_t FTLwrite(int fd, const void *buf, size_t total, const char *file, const return 0; } - ssize_t ret = 0; - size_t written = 0; + ssize_t ret = 0; + size_t written = 0; do { // Reset errno before trying to write errno = 0; ret = write(fd, buf, total); - if(ret > 0) - written += ret; + if(ret > 0) + written += ret; } // Try to write the remaining content into the stream if - // (a) we haven't written all the data, however, there was no other error - // (b) the last write() call failed due to an interruption by an incoming signal + // (a) we haven't written all the data, however, there was no other error + // (b) the last write() call failed due to an interruption by an incoming signal while((written < total && errno == 0) || (ret < 0 && errno == EINTR)); + // Backup errno value + const int _errno = errno; + // Final error checking (may have failed for some other reason then an // EINTR = interrupted system call) if(written < total) logg("WARN: Could not write() everything in %s() [%s:%i]: %s", - func, file, line, strerror(errno)); + func, file, line, strerror(errno)); - return written; + // Restore errno value + errno = _errno; + + // Return number of written bytes + return written; } \ No newline at end of file From b92126fd6f1b2fb7493d6ae765942a4727ebe9ce Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 30 Jul 2022 13:53:02 +0200 Subject: [PATCH 035/218] Major rewrite of the socket processing routines. Before, we launched a new thread for each incoming connection (more than 40 threads were never allowed). This could potentially be used to DoS FTL by opening and closing telnet sessions in very quick succession. The new behavior is to, instead, launch five (compile-time setting) threads per type (5 for telnet IPv4, 5 for telnet IPv6, and 5 for Unix socket communication) and let them handle incoming connections in a FIFO manner. If too many requests are sent to FTL at once, they will simply have to wait until they are accepted. Signed-off-by: DL6ER --- src/api/api.c | 91 +++--- src/api/api.h | 28 +- src/api/request.c | 34 +- src/api/request.h | 2 +- src/api/socket.c | 700 +++++++++++----------------------------- src/api/socket.h | 20 +- src/daemon.c | 20 +- src/daemon.h | 5 +- src/dnsmasq_interface.c | 24 +- src/enums.h | 10 +- 10 files changed, 291 insertions(+), 643 deletions(-) diff --git a/src/api/api.c b/src/api/api.c index 32047a2f..32a270a0 100644 --- a/src/api/api.c +++ b/src/api/api.c @@ -15,7 +15,7 @@ #include "../shmem.h" // read_setupVarsconf() #include "../setupVars.h" -// istelnet() +// ssend() #include "socket.h" // get_FTL_db_filesize() #include "../files.h" @@ -76,7 +76,7 @@ static int __attribute__((pure)) cmpdesc(const void *a, const void *b) return 0; } -void getStats(const int sock) +void getStats(const int sock, const bool istelnet) { const int blocked = blocked_queries(); const int total = counters->queries; @@ -87,7 +87,7 @@ void getStats(const int sock) percentage = 1e2f*blocked/total; // Send domains being blocked - if(istelnet[sock]) { + if(istelnet) { ssend(sock, "domains_being_blocked %i\n", counters->gravity); } else @@ -106,7 +106,7 @@ void getStats(const int sock) activeclients++; } - if(istelnet[sock]) { + if(istelnet) { ssend(sock, "dns_queries_today %i\nads_blocked_today %i\nads_percentage_today %f\n", total, blocked, percentage); ssend(sock, "unique_domains %i\nqueries_forwarded %i\nqueries_cached %i\n", @@ -145,16 +145,16 @@ void getStats(const int sock) } // Send status - if(istelnet[sock]) { + if(istelnet) { ssend(sock, "status %s\n", blockingstatus ? "enabled" : "disabled"); } else pack_uint8(sock, blockingstatus); } -void getOverTime(const int sock) +void getOverTime(const int sock, const bool istelnet) { - if(istelnet[sock]) + if(istelnet) { for(int slot = 0; slot < OVERTIME_SLOTS; slot++) { @@ -185,7 +185,7 @@ void getOverTime(const int sock) } } -void getTopDomains(const char *client_message, const int sock) +void getTopDomains(const char *client_message, const int sock, const bool istelnet) { int temparray[counters->domains][2], count=10, num; bool audit = false, asc = false; @@ -196,7 +196,7 @@ void getTopDomains(const char *client_message, const int sock) get_privacy_level(NULL); if(config.privacylevel >= PRIVACY_HIDE_DOMAINS) { // Always send the total number of domains, but pretend it's 0 - if(!istelnet[sock]) + if(!istelnet) pack_int32(sock, 0); return; @@ -269,7 +269,7 @@ void getTopDomains(const char *client_message, const int sock) } } - if(!istelnet[sock]) + if(!istelnet) { // Send the data required to get the percentage each domain has been blocked / queried if(blocked) @@ -306,7 +306,7 @@ void getTopDomains(const char *client_message, const int sock) if(blocked && showblocked && domain->blockedcount > 0) { - if(istelnet[sock]) + if(istelnet) ssend(sock, "%i %i %s\n", n, domain->blockedcount, getstr(domain->domainpos)); else { if(!pack_str32(sock, getstr(domain->domainpos))) @@ -318,7 +318,7 @@ void getTopDomains(const char *client_message, const int sock) } else if(!blocked && showpermitted && (domain->count - domain->blockedcount) > 0) { - if(istelnet[sock]) + if(istelnet) ssend(sock,"%i %i %s\n",n,(domain->count - domain->blockedcount),getstr(domain->domainpos)); else { @@ -339,7 +339,7 @@ void getTopDomains(const char *client_message, const int sock) clearSetupVarsArray(); } -void getTopClients(const char *client_message, const int sock) +void getTopClients(const char *client_message, const int sock, const bool istelnet) { int temparray[counters->clients][2], count=10, num; @@ -347,7 +347,7 @@ void getTopClients(const char *client_message, const int sock) get_privacy_level(NULL); if(config.privacylevel >= PRIVACY_HIDE_DOMAINS_CLIENTS) { // Always send the total number of clients, but pretend it's 0 - if(!istelnet[sock]) + if(!istelnet) pack_int32(sock, 0); return; @@ -408,7 +408,7 @@ void getTopClients(const char *client_message, const int sock) getSetupVarsArray(excludeclients); } - if(!istelnet[sock]) + if(!istelnet) { // Send the total queries so they can make percentages from this data pack_int32(sock, counters->queries); @@ -450,7 +450,7 @@ void getTopClients(const char *client_message, const int sock) // - the client made at least one query within the most recent 24 hours if(includezeroclients || ccount > 0) { - if(istelnet[sock]) + if(istelnet) ssend(sock,"%i %i %s %s\n", n, ccount, client_ip, client_name); else { @@ -471,7 +471,7 @@ void getTopClients(const char *client_message, const int sock) } -void getUpstreamDestinations(const char *client_message, const int sock) +void getUpstreamDestinations(const char *client_message, const int sock, const bool istelnet) { bool sort = true; int temparray[counters->upstreams][2], sumforwarded = 0; @@ -575,7 +575,7 @@ void getUpstreamDestinations(const char *client_message, const int sock) // - only if percentage > 0.0 for all others (i > 0) if(percentage > 0.0f || i < 0) { - if(istelnet[sock]) + if(istelnet) if(upstream_port != 0) ssend(sock, "%i %.2f %s#%u %s#%u\n", i, percentage, ip, upstream_port, name, upstream_port); @@ -592,7 +592,7 @@ void getUpstreamDestinations(const char *client_message, const int sock) } } -void getQueryTypes(const int sock) +void getQueryTypes(const int sock, const bool istelnet) { int total = 0; for(enum query_types type = TYPE_A; type < TYPE_MAX; type++) @@ -611,7 +611,7 @@ void getQueryTypes(const int sock) } } - if(istelnet[sock]) { + if(istelnet) { ssend(sock, "A (IPv4): %.2f\nAAAA (IPv6): %.2f\nANY: %.2f\nSRV: %.2f\n" "SOA: %.2f\nPTR: %.2f\nTXT: %.2f\nNAPTR: %.2f\n" "MX: %.2f\nDS: %.2f\nRRSIG: %.2f\nDNSKEY: %.2f\n" @@ -657,7 +657,7 @@ void getQueryTypes(const int sock) } } -void getAllQueries(const char *client_message, const int sock) +void getAllQueries(const char *client_message, const int sock, const bool istelnet) { // Exit before processing any data if requested via config setting get_privacy_level(NULL); @@ -1053,7 +1053,7 @@ void getAllQueries(const char *client_message, const int sock) delay = 0UL; } - if(istelnet[sock]) + if(istelnet) { ssend(sock,"%lli %s %s %s %i %i %i %lu %s %i %s#%u \"%s\"", (long long)query->timestamp, @@ -1105,7 +1105,7 @@ void getAllQueries(const char *client_message, const int sock) free(clientid_list); } -void getRecentBlocked(const char *client_message, const int sock) +void getRecentBlocked(const char *client_message, const int sock, const bool istelnet) { int num=1; @@ -1132,7 +1132,7 @@ void getRecentBlocked(const char *client_message, const int sock) if(domain == NULL) continue; - if(istelnet[sock]) + if(istelnet) ssend(sock,"%s\n", domain); else if(!pack_str32(sock, domain)) return; @@ -1146,15 +1146,15 @@ void getRecentBlocked(const char *client_message, const int sock) } } -void getClientID(const int sock) +void getClientID(const int sock, const bool istelnet) { - if(istelnet[sock]) + if(istelnet) ssend(sock,"%i\n", sock); else pack_int32(sock, sock); } -void getVersion(const int sock) +void getVersion(const int sock, const bool istelnet) { const char *commit = GIT_HASH; const char *tag = GIT_TAG; @@ -1165,12 +1165,8 @@ void getVersion(const int sock) memcpy(hash, commit, min((size_t)7, strlen(commit))); if(strlen(tag) > 1) { - if(istelnet[sock]) - ssend( - sock, - "version %s\ntag %s\nbranch %s\nhash %s\ndate %s\n", - version, tag, GIT_BRANCH, hash, GIT_DATE - ); + if(istelnet) + ssend(sock, "version %s\ntag %s\nbranch %s\nhash %s\ndate %s\n", version, tag, GIT_BRANCH, hash, GIT_DATE); else { if(!pack_str32(sock, version) || !pack_str32(sock, (char *) tag) || @@ -1181,12 +1177,8 @@ void getVersion(const int sock) } } else { - if(istelnet[sock]) - ssend( - sock, - "version vDev-%s\ntag %s\nbranch %s\nhash %s\ndate %s\n", - hash, tag, GIT_BRANCH, hash, GIT_DATE - ); + if(istelnet) + ssend(sock, "version vDev-%s\ntag %s\nbranch %s\nhash %s\ndate %s\n", hash, tag, GIT_BRANCH, hash, GIT_DATE); else { char *hashVersion = calloc(6 + strlen(hash), sizeof(char)); if(hashVersion == NULL) return; @@ -1204,7 +1196,7 @@ void getVersion(const int sock) } } -void getDBstats(const int sock) +void getDBstats(const int sock, const bool istelnet) { // Get file details unsigned long long int filesize = get_FTL_db_filesize(); @@ -1213,7 +1205,7 @@ void getDBstats(const int sock) double formatted = 0.0; format_memory_size(prefix, filesize, &formatted); - if(istelnet[sock]) + if(istelnet) ssend(sock, "queries in database: %i\ndatabase filesize: %.2f %sB\nSQLite version: %s\n", get_number_of_queries_in_DB(NULL), formatted, prefix, get_sqlite3_version()); else { @@ -1225,7 +1217,7 @@ void getDBstats(const int sock) } } -void getClientsOverTime(const int sock) +void getClientsOverTime(const int sock, const bool istelnet) { // Exit before processing any data if requested via config setting get_privacy_level(NULL); @@ -1263,7 +1255,7 @@ void getClientsOverTime(const int sock) // Main return loop for(int slot = 0; slot < OVERTIME_SLOTS; slot++) { - if(istelnet[sock]) + if(istelnet) ssend(sock, "%lli", (long long)overTime[slot].timestamp); else pack_int32(sock, (int32_t)overTime[slot].timestamp); @@ -1284,13 +1276,13 @@ void getClientsOverTime(const int sock) continue; const int thisclient = client->overTime[slot]; - if(istelnet[sock]) + if(istelnet) ssend(sock, " %i", thisclient); else pack_int32(sock, thisclient); } - if(istelnet[sock]) + if(istelnet) ssend(sock, "\n"); else pack_int32(sock, -1); @@ -1300,7 +1292,7 @@ void getClientsOverTime(const int sock) clearSetupVarsArray(); } -void getClientNames(const int sock) +void getClientNames(const int sock, const bool istelnet) { // Exit before processing any data if requested via config setting get_privacy_level(NULL); @@ -1353,7 +1345,7 @@ void getClientNames(const int sock) const char *client_ip = getstr(client->ippos); const char *client_name = getstr(client->namepos); - if(istelnet[sock]) + if(istelnet) ssend(sock, "%s %s\n", client_name, client_ip); else { pack_str32(sock, client_name); @@ -1365,7 +1357,7 @@ void getClientNames(const int sock) clearSetupVarsArray(); } -void getUnknownQueries(const int sock) +void getUnknownQueries(const int sock, const bool istelnet) { // Exit before processing any data if requested via config setting get_privacy_level(NULL); @@ -1401,7 +1393,7 @@ void getUnknownQueries(const int sock) // Get client IP string const char *clientIP = getstr(client->ippos); - if(istelnet[sock]) + if(istelnet) ssend(sock, "%lli %i %i %s %s %s %i %s\n", (long long)query->timestamp, queryID, query->id, type, getstr(domain->domainpos), clientIP, query->status, query->flags.complete ? "true" : "false"); else { pack_int32(sock, (int32_t)query->timestamp); @@ -1691,6 +1683,7 @@ static bool listInterfaces(struct if_info **head, char default_iface[IF_NAMESIZE rx_sum += new->rx_bytes; } + closedir(dfd); freeifaddrs(ifap); // Create sum entry only if there is more than one interface diff --git a/src/api/api.h b/src/api/api.h index 5867859d..9e1bbbd8 100644 --- a/src/api/api.h +++ b/src/api/api.h @@ -11,22 +11,22 @@ #define API_H // Statistic methods -void getStats(const int sock); -void getOverTime(const int sock); -void getTopDomains(const char *client_message, const int sock); -void getTopClients(const char *client_message, const int sock); -void getUpstreamDestinations(const char *client_message, const int sock); -void getQueryTypes(const int sock); -void getAllQueries(const char *client_message, const int sock); -void getRecentBlocked(const char *client_message, const int sock); -void getClientsOverTime(const int sock); -void getClientNames(const int sock); +void getStats(const int sock, const bool istelnet); +void getOverTime(const int sock, const bool istelnet); +void getTopDomains(const char *client_message, const int sock, const bool istelnet); +void getTopClients(const char *client_message, const int sock, const bool istelnet); +void getUpstreamDestinations(const char *client_message, const int sock, const bool istelnet); +void getQueryTypes(const int sock, const bool istelnet); +void getAllQueries(const char *client_message, const int sock, const bool istelnet); +void getRecentBlocked(const char *client_message, const int sock, const bool istelnet); +void getClientsOverTime(const int sock, const bool istelnet); +void getClientNames(const int sock, const bool istelnet); // FTL methods -void getClientID(const int sock); -void getVersion(const int sock); -void getDBstats(const int sock); -void getUnknownQueries(const int sock); +void getClientID(const int sock, const bool istelnet); +void getVersion(const int sock, const bool istelnet); +void getDBstats(const int sock, const bool istelnet); +void getUnknownQueries(const int sock, const bool istelnet); void getMAXLOGAGE(const int sock); void getGateway(const int sock); void getInterfaces(const int sock); diff --git a/src/api/request.c b/src/api/request.c index 88bb5a01..45cdb00a 100644 --- a/src/api/request.c +++ b/src/api/request.c @@ -26,7 +26,7 @@ bool __attribute__((pure)) command(const char *client_message, const char* cmd) return strstr(client_message, cmd) != NULL; } -bool process_request(const char *client_message, const int sock) +bool process_request(const char *client_message, const int sock, const bool istelnet) { char EOT[2]; EOT[0] = 0x04; @@ -37,104 +37,104 @@ bool process_request(const char *client_message, const int sock) { processed = true; lock_shm(); - getStats(sock); + getStats(sock, istelnet); unlock_shm(); } else if(command(client_message, ">overTime")) { processed = true; lock_shm(); - getOverTime(sock); + getOverTime(sock, istelnet); unlock_shm(); } else if(command(client_message, ">top-domains") || command(client_message, ">top-ads")) { processed = true; lock_shm(); - getTopDomains(client_message, sock); + getTopDomains(client_message, sock, istelnet); unlock_shm(); } else if(command(client_message, ">top-clients")) { processed = true; lock_shm(); - getTopClients(client_message, sock); + getTopClients(client_message, sock, istelnet); unlock_shm(); } else if(command(client_message, ">forward-dest")) { processed = true; lock_shm(); - getUpstreamDestinations(client_message, sock); + getUpstreamDestinations(client_message, sock, istelnet); unlock_shm(); } else if(command(client_message, ">forward-names")) { processed = true; lock_shm(); - getUpstreamDestinations(">forward-dest unsorted", sock); + getUpstreamDestinations(">forward-dest unsorted", sock, istelnet); unlock_shm(); } else if(command(client_message, ">querytypes")) { processed = true; lock_shm(); - getQueryTypes(sock); + getQueryTypes(sock, istelnet); unlock_shm(); } else if(command(client_message, ">getallqueries")) { processed = true; lock_shm(); - getAllQueries(client_message, sock); + getAllQueries(client_message, sock, istelnet); unlock_shm(); } else if(command(client_message, ">recentBlocked")) { processed = true; lock_shm(); - getRecentBlocked(client_message, sock); + getRecentBlocked(client_message, sock, istelnet); unlock_shm(); } else if(command(client_message, ">clientID")) { processed = true; lock_shm(); - getClientID(sock); + getClientID(sock, istelnet); unlock_shm(); } else if(command(client_message, ">version")) { processed = true; // No lock required - getVersion(sock); + getVersion(sock, istelnet); } else if(command(client_message, ">dbstats")) { processed = true; // No lock required. Access to the database // is guaranteed to be atomic - getDBstats(sock); + getDBstats(sock, istelnet); } else if(command(client_message, ">ClientsoverTime")) { processed = true; lock_shm(); - getClientsOverTime(sock); + getClientsOverTime(sock, istelnet); unlock_shm(); } else if(command(client_message, ">client-names")) { processed = true; lock_shm(); - getClientNames(sock); + getClientNames(sock, istelnet); unlock_shm(); } else if(command(client_message, ">unknown")) { processed = true; lock_shm(); - getUnknownQueries(sock); + getUnknownQueries(sock, istelnet); unlock_shm(); } else if(command(client_message, ">cacheinfo")) @@ -199,7 +199,7 @@ bool process_request(const char *client_message, const int sock) ssend(sock, "unknown command: %s\n", client_message); // End of queryable commands: Send EOM - seom(sock); + seom(sock, istelnet); return false; } diff --git a/src/api/request.h b/src/api/request.h index 5e57f582..ad3ade50 100644 --- a/src/api/request.h +++ b/src/api/request.h @@ -10,7 +10,7 @@ #ifndef REQUEST_H #define REQUEST_H -bool process_request(const char *client_message, const int sock); +bool process_request(const char *client_message, const int sock, const bool istelnet); bool command(const char *client_message, const char* cmd) __attribute__((pure)); #endif //REQUEST_H diff --git a/src/api/socket.c b/src/api/socket.c index 55a4016c..1314b939 100644 --- a/src/api/socket.c +++ b/src/api/socket.c @@ -14,6 +14,8 @@ #include "socket.h" #include "request.h" #include "../config.h" +// sleepms() +#include "../timers.h" // global variable killed #include "../signals.h" // API thread storage @@ -30,10 +32,6 @@ // reattempt at connection succeeds. #define BACKLOG 5 -// File descriptors -bool dualstack = false; -bool istelnet[MAXCONNS]; - void saveport(int port) { FILE *f; @@ -67,162 +65,225 @@ void saveport(int port) } } -static int bind_to_telnet_port_IPv4(void) +static int bind_to_telnet_socket(const enum telnet_type type, const char *stype) { - // IPv4 socket - const int socketdescriptor = socket(AF_INET, SOCK_STREAM, 0); + const int socketdescriptor = socket(type == TELNET_SOCK ? AF_LOCAL : (type == TELNETv4 ? AF_INET : AF_INET6), SOCK_STREAM, 0); if(socketdescriptor < 0) { - logg("Error opening IPv4 telnet socket: %s (%i)", strerror(errno), errno); + logg("Error opening %s telnet socket: %s (%i)", stype, strerror(errno), errno); exit(EXIT_FAILURE); } - // Set SO_REUSEADDR to allow re-binding to the port that has been used - // previously by FTL. A common pattern is that you change FTL's - // configuration file and need to restart that server to make it reload - // its configuration. Without SO_REUSEADDR, the bind() call in the restarted - // new instance will fail if there were connections open to the previous - // instance when you killed it. Those connections will hold the TCP port in - // the TIME_WAIT state for 30-120 seconds, so you fall into case 1 above. - if(setsockopt(socketdescriptor, SOL_SOCKET, SO_REUSEADDR, &(int){ 1 }, sizeof(int)) != 0) - logg("WARN: allowing re-binding (IPv6) failed: %s", strerror(errno)); + const size_t addrlen = MAX(sizeof(struct sockaddr_un), MAX(sizeof(struct sockaddr_in), sizeof(struct sockaddr_in6))); + void *address = calloc(1, addrlen); - struct sockaddr_in serv_addr4; - // set all values in the buffer to zero - memset(&serv_addr4, 0, sizeof(serv_addr4)); - serv_addr4.sin_family = AF_INET; - - if(config.socket_listenlocal) - serv_addr4.sin_addr.s_addr = htonl(INADDR_LOOPBACK); - else - serv_addr4.sin_addr.s_addr = INADDR_ANY; - - // Bind to IPv4 port - serv_addr4.sin_port = htons(config.port); - if(bind(socketdescriptor, (struct sockaddr *) &serv_addr4, sizeof(serv_addr4)) < 0) + if(type == TELNETv4 || type == TELNETv6) { - logg("Error listening on IPv4 port %i: %s (%i)", config.port, strerror(errno), errno); + // Set SO_REUSEADDR to allow re-binding to the port that has been used + // previously by FTL. A common pattern is that you change FTL's + // configuration file and need to restart that server to make it reload + // its configuration. Without SO_REUSEADDR, the bind() call in the restarted + // new instance will fail if there were connections open to the previous + // instance when you killed it. Those connections will hold the TCP port in + // the TIME_WAIT state for 30-120 seconds, so you fall into case 1 above. + if(setsockopt(socketdescriptor, SOL_SOCKET, SO_REUSEADDR, &(int){ 1 }, sizeof(int)) != 0) + logg("WARN: allowing re-binding (%s) failed: %s", stype, strerror(errno)); + + if(type == TELNETv6) + { + // If this flag is set to true (nonzero), then the socket is re‐ + // stricted to sending and receiving IPv6 packets only. In this + // case, an IPv4 and an IPv6 application can bind to a single port + // at the same time. + if(setsockopt(socketdescriptor, IPPROTO_IPV6, IPV6_V6ONLY, &(int){ 1 }, sizeof(int)) != 0) + logg("WARN: setting socket to IPv6-only failed: %s", strerror(errno)); + + if(config.socket_listenlocal) + ((struct sockaddr_in6*) address)->sin6_addr = in6addr_loopback; + else + ((struct sockaddr_in6*) address)->sin6_addr = in6addr_any; + + // The bind() system call binds a socket to an address, + // in this case the address of the current host and + // port number on which the server will run. + // convert this to network byte order using the function htons() + // which converts a port number in host byte order to a port number + // in network byte order + + // Bind to IPv6 socket + ((struct sockaddr_in6*) address)->sin6_family = AF_INET6; + ((struct sockaddr_in6*) address)->sin6_port = htons(config.port); + } + else // IPv4 + { + + if(config.socket_listenlocal) + ((struct sockaddr_in*) address)->sin_addr.s_addr = htonl(INADDR_LOOPBACK); + else + ((struct sockaddr_in*) address)->sin_addr.s_addr = INADDR_ANY; + + // Bind to IPv4 port + ((struct sockaddr_in*) address)->sin_family = AF_INET; + ((struct sockaddr_in*) address)->sin_port = htons(config.port); + } + } + else // socket + { + // Make sure unix socket file handle does not exist, if it exists, remove it + unlink(FTLfiles.socketfile); + + ((struct sockaddr_un*) address)->sun_family = AF_LOCAL; + // The sockaddr_un.sum_path may be shorter than the size of the FTLfiles.socketfile + // buffer. Ensure that the string is null-terminated even when the string is too large. + // In case strlen(FTLfiles.socketfile) < sizeof(address.sun_path) [this will virtually + // always be the case], the explicit setting of the last byte to zero is a no-op as + // strncpy() writes additional null bytes to ensure that a total of n bytes are written. + strncpy(((struct sockaddr_un*) address)->sun_path, FTLfiles.socketfile, sizeof(((struct sockaddr_un*) address)->sun_path)); + ((struct sockaddr_un*) address)->sun_path[sizeof(((struct sockaddr_un*) address)->sun_path)-1] = '\0'; + + } + + // Bind to socket + if(bind(socketdescriptor, (struct sockaddr *) address, addrlen) < 0) + { + logg("Error listening on Unix socket %s: %s (%i)", FTLfiles.socketfile, strerror(errno), errno); return -1; } // The listen system call allows the process to listen on the socket for connections if(listen(socketdescriptor, BACKLOG) == -1) { - logg("Error listening on IPv4 socket: %s (%i)", strerror(errno), errno); + logg("Error listening on %s socket: %s (%i)", stype, strerror(errno), errno); return -1; } - logg("Listening on port %i for incoming IPv4 telnet connections", config.port); + logg("Listening on port %i for incoming %s telnet connections", config.port, stype); return socketdescriptor; } -static int bind_to_telnet_port_IPv6(void) +static void *telnet_connection_handler_thread(void *args) { - // IPv6 socket - const int socketdescriptor = socket(AF_INET6, SOCK_STREAM, 0); - if(socketdescriptor < 0) + struct thread_info *tinfo = args; + // Set thread name + char threadname[16] = { 0 }; + snprintf(threadname, sizeof(threadname), "telnet-%s-%i", tinfo->stype, tinfo->tid); + prctl(PR_SET_NAME, threadname, 0, 0, 0); + + // Ensure this thread can be canceled at any time (not only at + // cancellation points) + pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, NULL); + + if(config.debug & DEBUG_API) + logg("Started telnet thread %s", threadname); + + // Listen as long as this thread is not canceled + int errors = 0; + while(!killed) { - logg("Error opening IPv6 telnet socket: %s (%i)", strerror(errno), errno); - exit(EXIT_FAILURE); + // Look for new clients that want to connect + const int csck = accept(tinfo->fd, NULL, NULL); + if(csck == -1) + { + logg("Telnet error in %s: %s (%i, fd: %d)", threadname, strerror(errno), errno, tinfo->fd); + if(errors++ > 20) + break; + sleepms(100); + continue; + } + + // Define buffer for client's message + char client_message[SOCKETBUFFERLEN] ={ 0 }; + + // Receive from client + ssize_t n; + while((n = recv(csck, client_message, SOCKETBUFFERLEN-1, 0))) + { + if (n > 0 && n < SOCKETBUFFERLEN) + { + // Null-terminate client string + client_message[n] = '\0'; + char *message = strdup(client_message); + if(message == NULL) + { + if(config.debug & DEBUG_API) + logg("Break in telnet thread for socket %d/%d: Memory error", tinfo->fd, csck); + break; + } + + // Clear client message receive buffer + memset(client_message, 0, sizeof client_message); + + // Process received message + const bool eom = process_request(message, csck, tinfo->istelnet); + free(message); + if(eom) break; + } + else if(n == -1) + { + if(config.debug & DEBUG_API) + logg("Break in telnet thread for socket %d/%d: No data received", tinfo->fd, csck); + break; + } + } + + // Close client socket + close(csck); } - // If this flag is set to true (nonzero), then the socket is re‐ - // stricted to sending and receiving IPv6 packets only. In this - // case, an IPv4 and an IPv6 application can bind to a single port - // at the same time. - if(setsockopt(socketdescriptor, IPPROTO_IPV6, IPV6_V6ONLY, &(int){ 1 }, sizeof(int)) != 0) - logg("WARN: setting socket to IPv6-only failed: %s", strerror(errno)); + if(config.debug & DEBUG_API) + logg("Terminating telnet thread %s (%d errors)", threadname, errors); - // Set SO_REUSEADDR to allow re-binding to the port that has been used - // previously by FTL. A common pattern is that you change FTL's - // configuration file and need to restart that server to make it reload - // its configuration. Without SO_REUSEADDR, the bind() call in the restarted - // new instance will fail if there were connections open to the previous - // instance when you killed it. Those connections will hold the TCP port in - // the TIME_WAIT state for 30-120 seconds, so you fall into case 1 above. - if(setsockopt(socketdescriptor, SOL_SOCKET, SO_REUSEADDR, &(int){ 1 }, sizeof(int)) != 0) - logg("WARN: allowing re-binding (IPv6) failed: %s", strerror(errno)); - - struct sockaddr_in6 serv_addr; - // set all values in the buffer to zero - memset(&serv_addr, 0, sizeof(serv_addr)); - serv_addr.sin6_family = AF_INET6; - - if(config.socket_listenlocal) - serv_addr.sin6_addr = in6addr_loopback; - else - serv_addr.sin6_addr = in6addr_any; - - // The bind() system call binds a socket to an address, - // in this case the address of the current host and - // port number on which the server will run. - // convert this to network byte order using the function htons() - // which converts a port number in host byte order to a port number - // in network byte order - - // Bind to IPv6 socket - serv_addr.sin6_port = htons(config.port); - if(bind(socketdescriptor, (struct sockaddr *) &serv_addr, sizeof(serv_addr)) < 0) - { - logg("Error listening on IPv6 port %i: %s (%i)", config.port, strerror(errno), errno); - return -1; - } - - // The listen system call allows the process to listen on the socket for connections - if(listen(socketdescriptor, BACKLOG) == -1) - { - logg("Error listening on IPv6 socket: %s (%i)", strerror(errno), errno); - return -1; - } - - logg("Listening on port %i for incoming IPv6 telnet connections", config.port); - return socketdescriptor; + // Free thread-private memory + free(tinfo); + return NULL; } -static int bind_to_unix_socket(void) +void listen_telnet(const enum telnet_type type) { - const int socketdescriptor = socket(AF_LOCAL, SOCK_STREAM, 0); - if(socketdescriptor < 0) + // We will use the attributes object later to start all threads in detached mode + pthread_attr_t attr; + // Initialize thread attributes object with default attribute values + pthread_attr_init(&attr); + // When a detached thread terminates, its resources are automatically released back to + // the system without the need for another thread to join with the terminated thread + pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); + + // Initialize telnet socket + const char *stype = type == TELNET_SOCK ? "socket" : (type == TELNETv4 ? "IPv4" : "IPv6"); + const int fd = bind_to_telnet_socket(type, stype); + if(fd < 0) { - logg("WARNING: Error opening Unix socket."); - return -1; + logg("WARN: Cannot bind to %s telnet socket", stype); + return; } - // Make sure unix socket file handle does not exist, if it exists, remove it - unlink(FTLfiles.socketfile); + if(config.debug & DEBUG_API) + logg("Telnet-%s listener accepting on fd %d", stype, fd); - struct sockaddr_un address; - address.sun_family = AF_LOCAL; - // The sockaddr_un.sum_path may be shorter than the size of the FTLfiles.socketfile - // buffer. Ensure that the string is null-terminated even when the string is too large. - // In case strlen(FTLfiles.socketfile) < sizeof(address.sun_path) [this will virtually - // always be the case], the explicit setting of the last byte to zero is a no-op as - // strncpy() writes additional null bytes to ensure that a total of n bytes are written. - strncpy(address.sun_path, FTLfiles.socketfile, sizeof(address.sun_path)); - address.sun_path[sizeof(address.sun_path)-1] = '\0'; - - // Bind to Unix socket handle - errno = 0; - if(bind(socketdescriptor, (struct sockaddr *) &address, sizeof (address)) != 0) + for(unsigned int i = 0; i < MAX_API_THREADS; i++) { - logg("WARNING: Cannot bind on Unix socket %s: %s (%i)", FTLfiles.socketfile, strerror(errno), errno); - return -1; - } + // Spawn telnet thread + pthread_t telnet_connection_thread; + // Create a private copy of the socket fd for the child thread + struct thread_info *tinfo = calloc(1, sizeof(struct thread_info)); + if(!tinfo) + continue; - // The listen system call allows the process to listen on the Unix socket for connections - if(listen(socketdescriptor, BACKLOG) == -1) - { - logg("WARNING: Cannot listen on Unix socket: %s (%i)", strerror(errno), errno); - return -1; + tinfo->fd = fd; + tinfo->tid = i; + tinfo->istelnet = (type == TELNETv4 || type == TELNETv6); + tinfo->stype = stype; + if(pthread_create(&telnet_connection_thread, &attr, telnet_connection_handler_thread, (void*) tinfo) != 0) + { + // Log the error code description + logg("WARNING: Unable to open telnet processing thread: %s", strerror(errno)); + } } - - logg("Listening on Unix socket"); - return socketdescriptor; } -void seom(const int sock) +void seom(const int sock, const bool istelnet) { - if(istelnet[sock]) + if(istelnet) ssend(sock, "---EOM---\n\n"); else pack_eom(sock); @@ -242,398 +303,3 @@ bool __attribute__ ((format (gnu_printf, 5, 6))) _ssend(const int sock, const ch } return errno == 0; } - -static void *telnet_connection_handler_thread(void *socket_desc) -{ - //Get the socket descriptor - const int sock = *(int*)socket_desc; - // Set connection type to telnet - istelnet[sock] = true; - - // Define buffer for client's message - char client_message[SOCKETBUFFERLEN] = ""; - - // Set thread name - char threadname[16] = { 0 }; - sprintf(threadname, "telnet-%i", sock); - prctl(PR_SET_NAME, threadname, 0, 0, 0); - - // Store TID of this thread - lock_shm(); - unsigned int tid; - for(tid = 0; tid < MAX_API_THREADS; tid++) - { - if(api_threads[tid] == 0) - { - api_threads[tid] = pthread_self(); - break; - } - } - unlock_shm(); - if(tid == MAX_API_THREADS) - { - logg("Not able to spawn new API thread, limit of " str(MAX_API_THREADS) " threads reached."); - return NULL; - } - - if(config.debug & DEBUG_API) - logg("New telnet thread for socket %d", sock); - - // Receive from client - ssize_t n; - while((n = recv(sock, client_message, SOCKETBUFFERLEN-1, 0))) - { - if (n > 0 && n < SOCKETBUFFERLEN) - { - // Null-terminate client string - client_message[n] = '\0'; - char *message = strdup(client_message); - if(message == NULL) - { - if(config.debug & DEBUG_API) - logg("Break in telnet thread for socket %d: Memory error", sock); - break; - } - - // Clear client message receive buffer - memset(client_message, 0, sizeof client_message); - - // Process received message - const bool eom = process_request(message, sock); - free(message); - if(eom) break; - } - else if(n == -1) - { - if(config.debug & DEBUG_API) - logg("Break in telnet thread for socket %d: No data received", sock); - break; - } - } - - if(config.debug & DEBUG_API) - logg("Terminating telnet thread for socket %d", sock); - - // Close the socket pointer - close(sock); - free(socket_desc); - - // Release thread from list - api_threads[tid] = 0; - - return NULL; -} - - -static void *socket_connection_handler_thread(void *socket_desc) -{ - //Get the socket descriptor - const int sock = *(int*)socket_desc; - // Set connection type to not telnet - istelnet[sock] = false; - - // Define buffer for client's message - char client_message[SOCKETBUFFERLEN] = ""; - - // Set thread name - char threadname[16]; - sprintf(threadname, "socket-%i", sock); - prctl(PR_SET_NAME, threadname, 0, 0, 0); - - // Ensure this thread can be canceled at any time (not only at - // cancellation points) - pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, NULL); - - // Store TID of this thread - lock_shm(); - unsigned int tid; - for(tid = 0; tid < MAX_API_THREADS; tid++) - { - if(api_threads[tid] == 0) - { - api_threads[tid] = pthread_self(); - api_tids[tid] = gettid(); - break; - } - } - unlock_shm(); - if(tid == MAX_API_THREADS) - { - logg("Not able to spawn new API thread, limit of " str(MAX_API_THREADS) " threads reached."); - return NULL; - } - - // Receive from client - ssize_t n; - while((n = recv(sock, client_message, SOCKETBUFFERLEN-1, 0))) - { - if (n > 0 && n < SOCKETBUFFERLEN) - { - // Null-terminate client string - client_message[n] = '\0'; - char *message = strdup(client_message); - if(message == NULL) break; - - // Clear client message receive buffer - memset(client_message, 0, sizeof client_message); - - // Process received message - const bool eom = process_request(message, sock); - free(message); - if(eom) break; - } - else if(n == -1) - { - break; - } - } - - // Close the socket pointer - close(sock); - free(socket_desc); - - // Release thread from list - api_threads[tid] = 0; - api_tids[tid] = 0; - - return NULL; -} - -void *telnet_listening_thread_IPv4(void *args) -{ - // We will use the attributes object later to start all threads in detached mode - pthread_attr_t attr; - // Initialize thread attributes object with default attribute values - pthread_attr_init(&attr); - // When a detached thread terminates, its resources are automatically released back to - // the system without the need for another thread to join with the terminated thread - pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); - - // Set thread name - thread_names[TELNETv4] = "telnet-IPv4"; - prctl(PR_SET_NAME, thread_names[TELNETv4], 0, 0, 0); - - while(!killed) - { - // Initialize IPv4 telnet socket´ - const int telnetfd4 = bind_to_telnet_port_IPv4(); - if(telnetfd4 < 0) - return NULL; - - if(config.debug & DEBUG_API) - logg("Telnet-IPv4 listener accepting on fd %d", telnetfd4); - - thread_cancellable[TELNETv4] = true; - - // Listen as long as this thread is not canceled - int errors = 0; - while(!killed) - { - // Look for new clients that want to connect - const int csck = accept(telnetfd4, NULL, NULL); - if(csck == -1) - { - logg("IPv4 telnet error: %s (%i, sockfd: %d)", strerror(errno), errno, telnetfd4); - if(errors++ > 20) - break; - continue; - } - - if(config.debug & DEBUG_API) - logg("Accepting new telnet connection at socket %d", csck); - - // Allocate memory used to transport client socket ID to client listening thread - int *newsock; - newsock = calloc(1,sizeof(int)); - if(newsock == NULL) break; - *newsock = csck; - - pthread_t telnet_connection_thread; - // Create a new thread - if(pthread_create( &telnet_connection_thread, &attr, telnet_connection_handler_thread, (void*) newsock ) != 0) - { - // Log the error code description - logg("WARNING: Unable to open telnet processing thread: %s", strerror(errno)); - } - } - - if(config.debug & DEBUG_API) - logg("Telnet-IPv4 listener closed (fd %d)", telnetfd4); - - // Close telnet socket - close(telnetfd4); - thread_sleepms(TELNETv4, 1000); - } - logg("Terminating IPv4 telnet thread"); - return NULL; -} - -void *telnet_listening_thread_IPv6(void *args) -{ - // We will use the attributes object later to start all threads in detached mode - pthread_attr_t attr; - // Initialize thread attributes object with default attribute values - pthread_attr_init(&attr); - // When a detached thread terminates, its resources are automatically released back to - // the system without the need for another thread to join with the terminated thread - pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); - - // Set thread name - thread_names[TELNETv6] = "telnet-IPv6"; - prctl(PR_SET_NAME, thread_names[TELNETv6], 0, 0, 0); - - // Initialize IPv6 telnet socket but only if IPv6 interfaces are available - if(!ipv6_available()) - return NULL; - - while(!killed) - { - const int telnetfd6 = bind_to_telnet_port_IPv6(); - if(telnetfd6 < 0) - return NULL; - - if(config.debug & DEBUG_API) - logg("Telnet-IPv6 listener accepting on fd %d", telnetfd6); - - thread_cancellable[TELNETv6] = true; - - // Listen as long as this thread is not canceled - int errors = 0; - while(!killed) - { - // Look for new clients that want to connect - const int csck = accept(telnetfd6, NULL, NULL); - if(csck == -1) - { - logg("IPv6 telnet error: %s (%i, sockfd: %d)", strerror(errno), errno, telnetfd6); - if(errors++ > 20) - break; - continue; - } - - // Allocate memory used to transport client socket ID to client listening thread - int *newsock; - newsock = calloc(1,sizeof(int)); - if(newsock == NULL) break; - *newsock = csck; - - pthread_t telnet_connection_thread; - // Create a new thread - if(pthread_create( &telnet_connection_thread, &attr, telnet_connection_handler_thread, (void*) newsock ) != 0) - { - // Log the error code description - logg("WARNING: Unable to open telnet processing thread: %s", strerror(errno)); - } - } - - if(config.debug & DEBUG_API) - logg("Telnet-IPv6 listener closed (fd %d)", telnetfd6); - - // Close telnet socket - close(telnetfd6); - thread_sleepms(TELNETv6, 1000); - } - logg("Terminating IPv6 telnet thread"); - return NULL; -} - -void *socket_listening_thread(void *args) -{ - // We will use the attributes object later to start all threads in detached mode - pthread_attr_t attr; - // Initialize thread attributes object with default attribute values - pthread_attr_init(&attr); - // When a detached thread terminates, its resources are automatically released back to - // the system without the need for another thread to join with the terminated thread - pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); - - // Set thread name - thread_names[SOCKET] = "telnet-socket"; - prctl(PR_SET_NAME, thread_names[SOCKET], 0, 0, 0); - - while(!killed) - { - // Return early to avoid CPU spinning if Unix socket is not available - const int socketfd = bind_to_unix_socket(); - if(socketfd < 0) - { - logg("INFO: Unix socket will not be available"); - return NULL; - } - - if(config.debug & DEBUG_API) - logg("Telnet-socket listener accepting on fd %d", socketfd); - - thread_cancellable[SOCKET] = true; - - // Listen as long as this thread is not canceled - while(!killed) - { - // Look for new clients that want to connect - const int csck = accept(socketfd, NULL, NULL); - if(csck < 0) - continue; - - // Allocate memory used to transport client socket ID to client listening thread - int *newsock; - newsock = calloc(1,sizeof(int)); - if(newsock == NULL) break; - *newsock = csck; - - pthread_t socket_connection_thread; - // Create a new thread - if(pthread_create( &socket_connection_thread, &attr, socket_connection_handler_thread, (void*) newsock ) != 0) - { - // Log the error code description - logg("WARNING: Unable to open socket processing thread: %s", strerror(errno)); - } - } - - if(config.debug & DEBUG_API) - logg("Telnet-socket listener closed (fd %d)", socketfd); - - // Close Unix socket - close(socketfd); - } - // The process has to take care of unlinking the socket file description - // on exit - unlink(FTLfiles.socketfile); - logg("Terminating socket thread"); - return NULL; -} - -bool ipv6_available(void) -{ - struct ifaddrs *allInterfaces; - enum { IPv4, IPv6 }; - int iface[2] = { 0 }; - - // Get all interfaces - if (getifaddrs(&allInterfaces) == 0) - { - struct ifaddrs *interface; - // Loop over interfaces - for (interface = allInterfaces; interface != NULL; interface = interface->ifa_next) - { - const unsigned int flags = interface->ifa_flags; - const struct sockaddr *addr = interface->ifa_addr; - - // Check only for up and running IPv4, IPv6 interfaces - if ((flags & (IFF_UP|IFF_RUNNING)) && addr != NULL) - { - iface[addr->sa_family == AF_INET6 ? IPv6 : IPv4]++; - - if(config.debug & DEBUG_NETWORKING) - logg("Interface %s is %s", interface->ifa_name, addr->sa_family == AF_INET6 ? "IPv6" : "IPv4"); - } - } - freeifaddrs(allInterfaces); - } - - if(config.debug & DEBUG_NETWORKING) - { - logg("Found %i IPv4 and %i IPv6 capable interfaces", iface[IPv4], iface[IPv6]); - } - - return (iface[IPv6] > 0); -} diff --git a/src/api/socket.h b/src/api/socket.h index 8ee6a173..4ee4404b 100644 --- a/src/api/socket.h +++ b/src/api/socket.h @@ -10,17 +10,21 @@ #ifndef SOCKET_H #define SOCKET_H +// enum telnet_type +#include "../enums.h" + +struct thread_info { + int fd; + int tid; + bool istelnet; + const char *stype; +}; + void saveport(int port); void close_unix_socket(bool unlink_file); -void seom(const int sock); +void seom(const int sock, const bool istelnet); #define ssend(sock, format, ...) _ssend(sock, __FILE__, __FUNCTION__, __LINE__, format, ##__VA_ARGS__) bool _ssend(const int sock, const char *file, const char *func, const int line, const char *format, ...) __attribute__ ((format (gnu_printf, 5, 6))); -void *telnet_listening_thread_IPv4(void *args); -void *telnet_listening_thread_IPv6(void *args); -void *socket_listening_thread(void *args); -bool ipv6_available(void); -void bind_sockets(void); - -extern bool istelnet[MAXCONNS]; +void listen_telnet(const enum telnet_type type); #endif //SOCKET_H diff --git a/src/daemon.c b/src/daemon.c index b9dfe1f3..9ba11f06 100644 --- a/src/daemon.c +++ b/src/daemon.c @@ -29,8 +29,7 @@ #include pthread_t threads[THREADS_MAX] = { 0 }; -pthread_t api_threads[MAX_API_THREADS] = { 0 }; -pid_t api_tids[MAX_API_THREADS] = { 0 }; +pthread_t api_threads[TELNET_MAX][MAX_API_THREADS] = {{ 0 }}; bool resolver_ready = false; void go_daemon(void) @@ -272,16 +271,15 @@ void cleanup(const int ret) terminate_threads(); // Cancel and join possibly still running API worker threads - for(unsigned int tid = 0; tid < MAX_API_THREADS; tid++) + for(enum telnet_type tt = 0; tt < TELNET_MAX; tt++) { - // Skip if this is an unused slot - if(api_threads[tid] == 0) - continue; - - // Otherwise, cancel and join the thread - logg("Joining API worker thread %d", tid); - pthread_cancel(api_threads[tid]); - pthread_join(api_threads[tid], NULL); + for(unsigned int tid = 0; tid < MAX_API_THREADS; tid++) + { + // Otherwise, cancel and join the thread + logg("Joining API worker thread %d/%d", tt,tid); + pthread_cancel(api_threads[tt][tid]); + pthread_join(api_threads[tt][tid], NULL); + } } // Close database connection diff --git a/src/daemon.h b/src/daemon.h index 12b49942..edcd9227 100644 --- a/src/daemon.h +++ b/src/daemon.h @@ -12,9 +12,8 @@ #include "enums.h" extern pthread_t threads[THREADS_MAX]; -#define MAX_API_THREADS 40 -extern pthread_t api_threads[MAX_API_THREADS]; -extern pid_t api_tids[MAX_API_THREADS]; +#define MAX_API_THREADS 5 +extern pthread_t api_threads[TELNET_MAX][MAX_API_THREADS]; void go_daemon(void); void savepid(void); diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c index 286a2bb2..a013cdda 100644 --- a/src/dnsmasq_interface.c +++ b/src/dnsmasq_interface.c @@ -2683,26 +2683,10 @@ void FTL_fork_and_bind_sockets(struct passwd *ent_pw) // Initialize thread attributes object with default attribute values pthread_attr_init(&attr); - // Start TELNET IPv4 thread - if(pthread_create( &threads[TELNETv4], &attr, telnet_listening_thread_IPv4, NULL ) != 0) - { - logg("Unable to open IPv4 telnet listening thread. Exiting..."); - exit(EXIT_FAILURE); - } - - // Start TELNET IPv6 thread - if(pthread_create( &threads[TELNETv6], &attr, telnet_listening_thread_IPv6, NULL ) != 0) - { - logg("Unable to open IPv6 telnet listening thread. Exiting..."); - exit(EXIT_FAILURE); - } - - // Start SOCKET thread - if(pthread_create( &threads[SOCKET], &attr, socket_listening_thread, NULL ) != 0) - { - logg("Unable to open Unix socket listening thread. Exiting..."); - exit(EXIT_FAILURE); - } + // Start listening on telnet-like interface + listen_telnet(TELNETv4); + listen_telnet(TELNETv6); + listen_telnet(TELNET_SOCK); // Start database thread if database is used if(pthread_create( &threads[DB], &attr, DB_thread, NULL ) != 0) diff --git a/src/enums.h b/src/enums.h index 361cf3c2..08982f56 100644 --- a/src/enums.h +++ b/src/enums.h @@ -186,15 +186,19 @@ enum busy_reply { } __attribute__ ((packed)); enum thread_types { - TELNETv4, - TELNETv6, - SOCKET, DB, GC, DNSclient, THREADS_MAX } __attribute__ ((packed)); +enum telnet_type { + TELNETv4, + TELNETv6, + TELNET_SOCK, + TELNET_MAX +} __attribute__ ((packed)); + enum message_type { REGEX_MESSAGE, SUBNET_MESSAGE, From 35bd9a63d1ea3763f3d7e1ec0ab9a8683bf28ce5 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Wed, 17 Aug 2022 21:51:37 +0100 Subject: [PATCH 036/218] [skip ci] a couple of spelling changes Signed-off-by: Adam Warner --- src/api/api.c | 2 +- src/database/common.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/api/api.c b/src/api/api.c index 32a270a0..1c2a7265 100644 --- a/src/api/api.c +++ b/src/api/api.c @@ -1467,7 +1467,7 @@ static bool getDefaultInterface(char iface[IF_NAMESIZE], in_addr_t *gw) if(sscanf(buf, "%s %lx %lx %x %*i %*i %i", iface_r, &dest_r, &gw_r, &flags, &metric) != 5) continue; - // Only anaylze routes which are UP and whose + // Only analyze routes which are UP and whose // destinations are a gateway if(!(flags & RTF_UP) || !(flags & RTF_GATEWAY)) continue; diff --git a/src/database/common.h b/src/database/common.h index eda311bf..bacc5016 100644 --- a/src/database/common.h +++ b/src/database/common.h @@ -48,7 +48,7 @@ extern long int lastdbindex; extern bool DBdeleteoldqueries; // Return if FTL's database is known to be broken -// We abort execution of all database-related activitites in this case +// We abort execution of all database-related activities in this case bool FTLDBerror(void) __attribute__ ((pure)); // Check SQLite3 non-success return codes for possible database corruption From be3bad3818671479007a73bac376548e2f03e653 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Wed, 17 Aug 2022 21:51:37 +0100 Subject: [PATCH 037/218] a couple of spelling changes Signed-off-by: Adam Warner --- src/api/api.c | 2 +- src/database/common.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/api/api.c b/src/api/api.c index 32a270a0..1c2a7265 100644 --- a/src/api/api.c +++ b/src/api/api.c @@ -1467,7 +1467,7 @@ static bool getDefaultInterface(char iface[IF_NAMESIZE], in_addr_t *gw) if(sscanf(buf, "%s %lx %lx %x %*i %*i %i", iface_r, &dest_r, &gw_r, &flags, &metric) != 5) continue; - // Only anaylze routes which are UP and whose + // Only analyze routes which are UP and whose // destinations are a gateway if(!(flags & RTF_UP) || !(flags & RTF_GATEWAY)) continue; diff --git a/src/database/common.h b/src/database/common.h index eda311bf..bacc5016 100644 --- a/src/database/common.h +++ b/src/database/common.h @@ -48,7 +48,7 @@ extern long int lastdbindex; extern bool DBdeleteoldqueries; // Return if FTL's database is known to be broken -// We abort execution of all database-related activitites in this case +// We abort execution of all database-related activities in this case bool FTLDBerror(void) __attribute__ ((pure)); // Check SQLite3 non-success return codes for possible database corruption From 9e81025dab42f057ed18603488fa5b5a2818d88a Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 19 Aug 2022 19:34:05 +0200 Subject: [PATCH 038/218] Don't fail hard when we cannot open all the telnet sockets (e.g., IPv6 is disabled on the kernel level). Signed-off-by: DL6ER --- src/api/socket.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/api/socket.c b/src/api/socket.c index 1314b939..9072781e 100644 --- a/src/api/socket.c +++ b/src/api/socket.c @@ -71,7 +71,7 @@ static int bind_to_telnet_socket(const enum telnet_type type, const char *stype) if(socketdescriptor < 0) { logg("Error opening %s telnet socket: %s (%i)", stype, strerror(errno), errno); - exit(EXIT_FAILURE); + return -1; } const size_t addrlen = MAX(sizeof(struct sockaddr_un), MAX(sizeof(struct sockaddr_in), sizeof(struct sockaddr_in6))); From 15942b17cebe00825d535ea8e77f5487889d8eae Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 18 Aug 2022 22:42:37 +0200 Subject: [PATCH 039/218] Store domainlist IDs for blocked/permitted queries Signed-off-by: DL6ER --- src/api/api.c | 9 +++--- src/database/gravity-db.c | 59 ++++++++++++++------------------------ src/database/gravity-db.h | 2 +- src/database/query-table.c | 52 ++++++++++++++++----------------- src/datastructure.c | 1 + src/datastructure.h | 2 +- src/dnsmasq_interface.c | 15 +++++----- src/regex.c | 22 ++++++++++++-- src/regex_r.h | 3 +- test/run.sh | 3 ++ test/test_suite.bats | 32 ++++++++++----------- 11 files changed, 101 insertions(+), 99 deletions(-) diff --git a/src/api/api.c b/src/api/api.c index 1c2a7265..67da3334 100644 --- a/src/api/api.c +++ b/src/api/api.c @@ -1009,14 +1009,13 @@ void getAllQueries(const char *client_message, const int sock, const bool isteln } // Get ID of blocking regex, if applicable and permitted by privacy settings - int regex_idx = -1; - if ((query->status == QUERY_REGEX || query->status == QUERY_REGEX_CNAME) && - config.privacylevel < PRIVACY_HIDE_DOMAINS) + int domainlist_id = -1; + if (config.privacylevel < PRIVACY_HIDE_DOMAINS) { unsigned int cacheID = findCacheID(query->domainID, query->clientID, query->type); DNSCacheData *dns_cache = getDNSCache(cacheID, true); if(dns_cache != NULL) - regex_idx = dns_cache->black_regex_idx; + domainlist_id = dns_cache->domainlist_id; } // Get IP of upstream destination, if applicable @@ -1065,7 +1064,7 @@ void getAllQueries(const char *client_message, const int sock, const bool isteln reply, delay, CNAME_domain, - regex_idx, + domainlist_id, upstream_name, upstream_port, query->ede == -1 ? "" : get_edestr(query->ede)); diff --git a/src/database/gravity-db.c b/src/database/gravity-db.c index 06551dfe..d4c5d6d0 100644 --- a/src/database/gravity-db.c +++ b/src/database/gravity-db.c @@ -15,8 +15,6 @@ #include "../config.h" // logg() #include "../log.h" -// match_regex() -#include "../regex_r.h" // getstr() #include "../shmem.h" // SQLite3 prepared statement vectors @@ -210,11 +208,11 @@ bool gravityDB_reopen(void) return gravityDB_open(); } -static char* get_client_querystr(const char* table, const char* groups) +static char* get_client_querystr(const char *table, const char *column, const char *groups) { // Build query string with group filtering char *querystr = NULL; - if(asprintf(&querystr, "SELECT EXISTS(SELECT domain from %s WHERE domain = ? AND group_id IN (%s));", table, groups) < 1) + if(asprintf(&querystr, "SELECT %s from %s WHERE domain = ? AND group_id IN (%s);", column, table, groups) < 1) { logg("get_client_querystr(%s, %s) - asprintf() error", table, groups); return NULL; @@ -858,19 +856,14 @@ bool gravityDB_prepare_client_statements(clientsData *client) return false; // Prepare whitelist statement - // We use SELECT EXISTS() as this is known to efficiently use the index - // We are only interested in whether the domain exists or not in the - // list but don't case about duplicates or similar. SELECT EXISTS(...) - // returns true as soon as it sees the first row from the query inside - // of EXISTS(). if(config.debug & DEBUG_DATABASE) logg("gravityDB_open(): Preparing vw_whitelist statement for client %s", clientip); - querystr = get_client_querystr("vw_whitelist", getstr(client->groupspos)); + querystr = get_client_querystr("vw_whitelist", "id", getstr(client->groupspos)); sqlite3_stmt* stmt = NULL; int rc = sqlite3_prepare_v3(gravity_db, querystr, -1, SQLITE_PREPARE_PERSISTENT, &stmt, NULL); if( rc != SQLITE_OK ) { - logg("gravityDB_open(\"SELECT EXISTS(... vw_whitelist ...)\") - SQL error prepare: %s", sqlite3_errstr(rc)); + logg("gravityDB_open(\"SELECT(... vw_whitelist ...)\") - SQL error prepare: %s", sqlite3_errstr(rc)); gravityDB_close(); return false; } @@ -880,11 +873,11 @@ bool gravityDB_prepare_client_statements(clientsData *client) // Prepare gravity statement if(config.debug & DEBUG_DATABASE) logg("gravityDB_open(): Preparing vw_gravity statement for client %s", clientip); - querystr = get_client_querystr("vw_gravity", getstr(client->groupspos)); + querystr = get_client_querystr("vw_gravity", "domain", getstr(client->groupspos)); rc = sqlite3_prepare_v3(gravity_db, querystr, -1, SQLITE_PREPARE_PERSISTENT, &stmt, NULL); if( rc != SQLITE_OK ) { - logg("gravityDB_open(\"SELECT EXISTS(... vw_gravity ...)\") - SQL error prepare: %s", sqlite3_errstr(rc)); + logg("gravityDB_open(\"SELECT(... vw_gravity ...)\") - SQL error prepare: %s", sqlite3_errstr(rc)); gravityDB_close(); return false; } @@ -894,11 +887,11 @@ bool gravityDB_prepare_client_statements(clientsData *client) // Prepare blacklist statement if(config.debug & DEBUG_DATABASE) logg("gravityDB_open(): Preparing vw_blacklist statement for client %s", clientip); - querystr = get_client_querystr("vw_blacklist", getstr(client->groupspos)); + querystr = get_client_querystr("vw_blacklist", "id", getstr(client->groupspos)); rc = sqlite3_prepare_v3(gravity_db, querystr, -1, SQLITE_PREPARE_PERSISTENT, &stmt, NULL); if( rc != SQLITE_OK ) { - logg("gravityDB_open(\"SELECT EXISTS(... vw_blacklist ...)\") - SQL error prepare: %s", sqlite3_errstr(rc)); + logg("gravityDB_open(\"SELECT(... vw_blacklist ...)\") - SQL error prepare: %s", sqlite3_errstr(rc)); gravityDB_close(); return false; } @@ -1146,7 +1139,7 @@ int gravityDB_count(const enum gravity_tables list) return result; } -static enum db_result domain_in_list(const char *domain, sqlite3_stmt *stmt, const char *listname) +static enum db_result domain_in_list(const char *domain, sqlite3_stmt *stmt, const char *listname, int *domain_id) { // Do not try to bind text to statement when database is not available if(!gravityDB_opened && !gravityDB_open()) @@ -1180,10 +1173,10 @@ static enum db_result domain_in_list(const char *domain, sqlite3_stmt *stmt, con sqlite3_clear_bindings(stmt); return LIST_NOT_AVAILABLE; } - else if(rc != SQLITE_ROW) + else if(rc != SQLITE_ROW && rc != SQLITE_DONE) { - // Any return code that is neither SQLITE_BUSY not SQLITE_ROW - // is a real error we should log + // Any return code that is neither SQLITE_BUSY nor SQLITE_ROW or + // SQLITE_DONE is an error we should log logg("domain_in_list(\"%s\", %p, %s): Failed to perform step: %s", domain, stmt, listname, sqlite3_errstr(rc)); sqlite3_reset(stmt); @@ -1191,8 +1184,10 @@ static enum db_result domain_in_list(const char *domain, sqlite3_stmt *stmt, con return LIST_NOT_AVAILABLE; } - // Get result of query "SELECT EXISTS(...)" - const int result = sqlite3_column_int(stmt, 0); + // Get result of query (if available) + const int result = (rc == SQLITE_ROW) ? sqlite3_column_int(stmt, 0) : -1; + if(domain_id != NULL) + *domain_id = result; if(config.debug & DEBUG_DATABASE) logg("domain_in_list(\"%s\", %p, %s): %d", domain, stmt, listname, result); @@ -1210,7 +1205,7 @@ static enum db_result domain_in_list(const char *domain, sqlite3_stmt *stmt, con // Return if domain was found in current table // SELECT EXISTS(...) either returns 0 (false) or 1 (true). - return (result == 1) ? FOUND : NOT_FOUND; + return (rc == SQLITE_ROW) ? FOUND : NOT_FOUND; } void gravityDB_reload_groups(clientsData* client) @@ -1268,17 +1263,7 @@ enum db_result in_whitelist(const char *domain, DNSCacheData *dns_cache, clients // We have to check both the exact whitelist (using a prepared database statement) // as well the compiled regex whitelist filters to check if the current domain is // whitelisted. - enum db_result on_whitelist = domain_in_list(domain, stmt, "whitelist"); - - // For performance reasons, the regex evaluations is executed only if the - // exact whitelist lookup does not deliver a positive match. This is an - // optimization as the database lookup will most likely hit (a) more domains - // and (b) will be faster (given a sufficiently large number of regex - // whitelisting filters). - if(on_whitelist == NOT_FOUND) - on_whitelist = match_regex(domain, dns_cache, client->id, REGEX_WHITELIST, false) != -1; - - return on_whitelist; + return domain_in_list(domain, stmt, "whitelist", &dns_cache->domainlist_id); } enum db_result in_gravity(const char *domain, clientsData *client) @@ -1306,10 +1291,10 @@ enum db_result in_gravity(const char *domain, clientsData *client) if(stmt == NULL) stmt = gravity_stmt->get(gravity_stmt, client->id); - return domain_in_list(domain, stmt, "gravity"); + return domain_in_list(domain, stmt, "gravity", NULL); } -enum db_result in_blacklist(const char *domain, clientsData *client) +enum db_result in_blacklist(const char *domain, DNSCacheData *dns_cache, clientsData *client) { // If list statement is not ready and cannot be initialized (e.g. no // access to the database), we return false to prevent an FTL crash @@ -1334,7 +1319,7 @@ enum db_result in_blacklist(const char *domain, clientsData *client) if(stmt == NULL) stmt = blacklist_stmt->get(blacklist_stmt, client->id); - return domain_in_list(domain, stmt, "blacklist"); + return domain_in_list(domain, stmt, "blacklist", &dns_cache->domainlist_id); } bool in_auditlist(const char *domain) @@ -1345,7 +1330,7 @@ bool in_auditlist(const char *domain) return false; // We check the domain_audit table for the given domain - return domain_in_list(domain, auditlist_stmt, "auditlist") == FOUND; + return domain_in_list(domain, auditlist_stmt, "auditlist", NULL) == FOUND; } bool gravityDB_get_regex_client_groups(clientsData* client, const unsigned int numregex, const regexData *regex, diff --git a/src/database/gravity-db.h b/src/database/gravity-db.h index 6c4b9f00..84beb1c8 100644 --- a/src/database/gravity-db.h +++ b/src/database/gravity-db.h @@ -31,7 +31,7 @@ void gravityDB_finalizeTable(void); int gravityDB_count(const enum gravity_tables list); enum db_result in_gravity(const char *domain, clientsData *client); -enum db_result in_blacklist(const char *domain, clientsData *client); +enum db_result in_blacklist(const char *domain, DNSCacheData *dns_cache, clientsData *client); enum db_result in_whitelist(const char *domain, DNSCacheData *dns_cache, clientsData *client); bool in_auditlist(const char *domain); diff --git a/src/database/query-table.c b/src/database/query-table.c index 6c7986ab..3b50b66f 100644 --- a/src/database/query-table.c +++ b/src/database/query-table.c @@ -373,6 +373,9 @@ int DB_save_queries(sqlite3 *db) sqlite3_bind_null(query_stmt, 7); } + const int cacheID = findCacheID(query->domainID, query->clientID, query->type); + DNSCacheData *cache = getDNSCache(cacheID, true); + // ADDITIONAL_INFO if(query->status == QUERY_GRAVITY_CNAME || query->status == QUERY_REGEX_CNAME || @@ -389,37 +392,29 @@ int DB_save_queries(sqlite3 *db) sqlite3_bind_text(addinfo_stmt, 2, cname, len, SQLITE_STATIC); if(sqlite3_step(addinfo_stmt) != SQLITE_DONE) { - logg("Encountered error while trying to store addinfo in long-term database"); + logg("Encountered error while trying to store addinfo in long-term database (CNAME)"); error = true; break; } sqlite3_clear_bindings(addinfo_stmt); sqlite3_reset(addinfo_stmt); } - else if(query->status == QUERY_REGEX) + else if(cache != NULL && cache->domainlist_id > -1) { - // Restore regex ID if applicable - const int cacheID = findCacheID(query->domainID, query->clientID, query->type); - DNSCacheData *cache = getDNSCache(cacheID, true); - if(cache != NULL) - { - sqlite3_bind_int(query_stmt, 8, ADDINFO_REGEX_ID); - sqlite3_bind_int(query_stmt, 9, cache->black_regex_idx); + sqlite3_bind_int(query_stmt, 8, ADDINFO_REGEX_ID); + sqlite3_bind_int(query_stmt, 9, cache->domainlist_id); - // Execute prepared addinfo statement and check if successful - sqlite3_bind_int(addinfo_stmt, 1, ADDINFO_REGEX_ID); - sqlite3_bind_int(addinfo_stmt, 2, cache->black_regex_idx); - if(sqlite3_step(addinfo_stmt) != SQLITE_DONE) - { - logg("Encountered error while trying to store addinfo in long-term database"); - error = true; - break; - } - sqlite3_clear_bindings(addinfo_stmt); - sqlite3_reset(addinfo_stmt); + // Execute prepared addinfo statement and check if successful + sqlite3_bind_int(addinfo_stmt, 1, ADDINFO_REGEX_ID); + sqlite3_bind_int(addinfo_stmt, 2, cache->domainlist_id); + if(sqlite3_step(addinfo_stmt) != SQLITE_DONE) + { + logg("Encountered error while trying to store addinfo in long-term database (domainlist_id)"); + error = true; + break; } - else - sqlite3_bind_null(query_stmt, 8); + sqlite3_clear_bindings(addinfo_stmt); + sqlite3_reset(addinfo_stmt); } else { @@ -933,16 +928,17 @@ void DB_read_queries(void) query->CNAME_domainID = CNAMEdomainID; } } - else if(status == QUERY_REGEX) + else if(sqlite3_column_bytes(stmt, 7) != 0) { - // QUERY_REGEX: Set ID regex which was the reason for blocking + // Set ID of the domainlist entry that was the reason for permitting/blocking this query + // We assume the value in this field is said ID when it is not a CNAME-related domain + // (checked above) and the value of additional_info is not NULL (0 bytes storage size) const int cacheID = findCacheID(query->domainID, query->clientID, query->type); DNSCacheData *cache = getDNSCache(cacheID, true); // Only load if - // a) we have a chace entry - // b) the value of additional_info is not NULL (0 bytes storage size) - if(cache != NULL && sqlite3_column_bytes(stmt, 7) != 0) - cache->black_regex_idx = sqlite3_column_int(stmt, 7); + // a) we have a cache entry + if(cache != NULL) + cache->domainlist_id = sqlite3_column_int(stmt, 7); } // Increment status counters, we first have to add one to the count of diff --git a/src/datastructure.c b/src/datastructure.c index d7bc1cfd..af6b1cc1 100644 --- a/src/datastructure.c +++ b/src/datastructure.c @@ -358,6 +358,7 @@ int findCacheID(int domainID, int clientID, enum query_types query_type) dns_cache->clientID = clientID; dns_cache->query_type = query_type; dns_cache->force_reply = 0u; + dns_cache->domainlist_id = -1; // -1 = not set // Increase counter by one counters->dns_cache_size++; diff --git a/src/datastructure.h b/src/datastructure.h index ee997001..98876cc1 100644 --- a/src/datastructure.h +++ b/src/datastructure.h @@ -104,7 +104,7 @@ typedef struct { enum query_types query_type; int domainID; int clientID; - int black_regex_idx; + int domainlist_id; } DNSCacheData; void strtolower(char *str); diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c index dddd4e89..dc2285b7 100644 --- a/src/dnsmasq_interface.c +++ b/src/dnsmasq_interface.c @@ -1089,7 +1089,7 @@ static bool check_domain_blocked(const char *domain, const int clientID, return false; // Check domains against exact blacklist - enum db_result blacklist = in_blacklist(domain, client); + enum db_result blacklist = in_blacklist(domain, dns_cache, client); if(blacklist == FOUND) { // Set new status @@ -1158,8 +1158,7 @@ static bool check_domain_blocked(const char *domain, const int clientID, // Check domain against blacklist regex filters // Skipped when the domain is whitelisted or blocked by exact blacklist or gravity - int regex_idx = 0; - if((regex_idx = match_regex(domain, dns_cache, client->id, REGEX_BLACKLIST, false)) > -1) + if(in_regex(domain, dns_cache, client-> id, REGEX_BLACKLIST) == FOUND) { // Set new status *new_status = QUERY_REGEX; @@ -1167,14 +1166,13 @@ static bool check_domain_blocked(const char *domain, const int clientID, // Mark domain as regex matched for this client set_dnscache_blockingstatus(dns_cache, client, REGEX_BLOCKED, domain); - dns_cache->black_regex_idx = regex_idx; // Regex may be overwriting reply type for this domain if(dns_cache->force_reply != REPLY_UNKNOWN) force_next_DNS_reply = dns_cache->force_reply; // Store ID of this regex (fork-private) - last_regex_idx = regex_idx; + last_regex_idx = dns_cache->domainlist_id; // We block this domain return true; @@ -1330,7 +1328,7 @@ static bool _FTL_check_blocking(int queryID, int domainID, int clientID, const c if(!query->flags.whitelisted) { force_next_DNS_reply = dns_cache->force_reply; - last_regex_idx = dns_cache->black_regex_idx; + last_regex_idx = dns_cache->domainlist_id; query_blocked(query, domain, client, QUERY_REGEX); return true; } @@ -1414,6 +1412,9 @@ static bool _FTL_check_blocking(int queryID, int domainID, int clientID, const c // Check whitelist (exact + regex) for match query->flags.whitelisted = in_whitelist(domainstr, dns_cache, client) == FOUND; + if(!query->flags.whitelisted) + query->flags.whitelisted = in_regex(domainstr, dns_cache, client->id, REGEX_WHITELIST) == FOUND; + // Check blacklist (exact + regex) and gravity for queried domain unsigned char new_status = QUERY_UNKNOWN; bool db_okay = true; @@ -1589,7 +1590,7 @@ bool _FTL_CNAME(const char *domain, const struct crec *cpp, const int id, const // Propagate ID of responsible regex up from the child to the parent domain if(parent_cache != NULL && child_cache != NULL) { - child_cache->black_regex_idx = parent_cache->black_regex_idx; + child_cache->domainlist_id = parent_cache->domainlist_id; } // Set status diff --git a/src/regex.c b/src/regex.c index 23f5c2bf..2a46dd70 100644 --- a/src/regex.c +++ b/src/regex.c @@ -302,8 +302,8 @@ static bool compile_regex(const char *regexin, const enum regex_type regexid, co return true; } -int match_regex(const char *input, DNSCacheData* dns_cache, const int clientID, - const enum regex_type regexid, const bool regextest) +static int match_regex(const char *input, DNSCacheData* dns_cache, const int clientID, + const enum regex_type regexid, const bool regextest) { int match_idx = -1; regexData *regex = get_regex_ptr(regexid); @@ -475,6 +475,24 @@ int match_regex(const char *input, DNSCacheData* dns_cache, const int clientID, return match_idx; } +enum db_result in_regex(const char *domain, DNSCacheData *dns_cache, const int clientID, const enum regex_type regexid) +{ + // For performance reasons, the regex evaluations is executed only if the + // exact whitelist lookup does not deliver a positive match. This is an + // optimization as the database lookup will most likely hit (a) more domains + // and (b) will be faster (given a sufficiently large number of regex + // whitelisting filters). + const int regex_id = match_regex(domain, dns_cache, clientID, regexid, false); + if(regex_id != -1) + { + // We found a match + dns_cache->domainlist_id = regex_id; + return FOUND; + } + + return NOT_FOUND; +} + static void free_regex(void) { // Return early if we don't use any regex filters diff --git a/src/regex_r.h b/src/regex_r.h index 6a44b4d5..6a63dcda 100644 --- a/src/regex_r.h +++ b/src/regex_r.h @@ -44,8 +44,7 @@ typedef struct { } regexData; unsigned int get_num_regex(const enum regex_type regexid) __attribute__((pure)); -int match_regex(const char *input, DNSCacheData* dns_cache, const int clientID, - const enum regex_type regexid, const bool regextest); +enum db_result in_regex(const char *domain, DNSCacheData *dns_cache, const int clientID, const enum regex_type regexid); void allocate_regex_client_enabled(clientsData *client, const int clientID); void reload_per_client_regex(clientsData *client); void read_regex_from_database(void); diff --git a/test/run.sh b/test/run.sh index 63d75d7f..17901625 100755 --- a/test/run.sh +++ b/test/run.sh @@ -108,6 +108,9 @@ if [[ $RET != 0 ]]; then echo "" echo -n "ptr.log: " curl_to_tricorder ./ptr.log + echo ""getallqueries + echo -n "getallqueries.log: " + curl_to_tricorder ./getallqueries.log echo "" fi diff --git a/test/test_suite.bats b/test/test_suite.bats index 1931e6b4..091c1466 100644 --- a/test/test_suite.bats +++ b/test/test_suite.bats @@ -171,13 +171,13 @@ run bash -c "grep -c 'Regex whitelist: Querying groups for client 127.0.0.4: \"SELECT id from vw_regex_whitelist WHERE group_id IN (4);\"' /var/log/pihole/FTL.log" printf "%s\n" "${lines[@]}" [[ ${lines[0]} == "1" ]] - run bash -c "grep -c 'get_client_querystr: SELECT EXISTS(SELECT domain from vw_whitelist WHERE domain = ? AND group_id IN (4));' /var/log/pihole/FTL.log" + run bash -c "grep -c 'get_client_querystr: SELECT id from vw_whitelist WHERE domain = ? AND group_id IN (4);' /var/log/pihole/FTL.log" printf "%s\n" "${lines[@]}" [[ ${lines[0]} != "0" ]] - run bash -c "grep -c 'get_client_querystr: SELECT EXISTS(SELECT domain from vw_blacklist WHERE domain = ? AND group_id IN (4));' /var/log/pihole/FTL.log" + run bash -c "grep -c 'get_client_querystr: SELECT id from vw_blacklist WHERE domain = ? AND group_id IN (4);' /var/log/pihole/FTL.log" printf "%s\n" "${lines[@]}" [[ ${lines[0]} != "0" ]] - run bash -c "grep -c 'get_client_querystr: SELECT EXISTS(SELECT domain from vw_gravity WHERE domain = ? AND group_id IN (4));' /var/log/pihole/FTL.log" + run bash -c "grep -c 'get_client_querystr: SELECT domain from vw_gravity WHERE domain = ? AND group_id IN (4);' /var/log/pihole/FTL.log" printf "%s\n" "${lines[@]}" [[ ${lines[0]} != "0" ]] run bash -c "grep -c 'Regex whitelist ([[:digit:]]*, DB ID [[:digit:]]*) .* NOT ENABLED for client 127.0.0.4' /var/log/pihole/FTL.log" @@ -203,13 +203,13 @@ run bash -c "grep -c 'Regex whitelist: Querying groups for client 127.0.0.5: \"SELECT id from vw_regex_whitelist WHERE group_id IN (4);\"' /var/log/pihole/FTL.log" printf "%s\n" "${lines[@]}" [[ ${lines[0]} == "1" ]] - run bash -c "grep -c 'get_client_querystr: SELECT EXISTS(SELECT domain from vw_whitelist WHERE domain = ? AND group_id IN (4));' /var/log/pihole/FTL.log" + run bash -c "grep -c 'get_client_querystr: SELECT id from vw_whitelist WHERE domain = ? AND group_id IN (4);' /var/log/pihole/FTL.log" printf "%s\n" "${lines[@]}" [[ ${lines[0]} != "0" ]] - run bash -c "grep -c 'get_client_querystr: SELECT EXISTS(SELECT domain from vw_blacklist WHERE domain = ? AND group_id IN (4));' /var/log/pihole/FTL.log" + run bash -c "grep -c 'get_client_querystr: SELECT id from vw_blacklist WHERE domain = ? AND group_id IN (4);' /var/log/pihole/FTL.log" printf "%s\n" "${lines[@]}" [[ ${lines[0]} != "0" ]] - run bash -c "grep -c 'get_client_querystr: SELECT EXISTS(SELECT domain from vw_gravity WHERE domain = ? AND group_id IN (4));' /var/log/pihole/FTL.log" + run bash -c "grep -c 'get_client_querystr: SELECT domain from vw_gravity WHERE domain = ? AND group_id IN (4);' /var/log/pihole/FTL.log" printf "%s\n" "${lines[@]}" [[ ${lines[0]} != "0" ]] run bash -c "grep -c 'Regex whitelist ([[:digit:]]*, DB ID [[:digit:]]*) .* NOT ENABLED for client 127.0.0.5' /var/log/pihole/FTL.log" @@ -241,13 +241,13 @@ run bash -c "grep -c 'Regex whitelist: Querying groups for client 127.0.0.6: \"SELECT id from vw_regex_whitelist WHERE group_id IN (5);\"' /var/log/pihole/FTL.log" printf "%s\n" "${lines[@]}" [[ ${lines[0]} == "1" ]] - run bash -c "grep -c 'get_client_querystr: SELECT EXISTS(SELECT domain from vw_whitelist WHERE domain = ? AND group_id IN (5));' /var/log/pihole/FTL.log" + run bash -c "grep -c 'get_client_querystr: SELECT id from vw_whitelist WHERE domain = ? AND group_id IN (5);' /var/log/pihole/FTL.log" printf "%s\n" "${lines[@]}" [[ ${lines[0]} == "1" ]] - run bash -c "grep -c 'get_client_querystr: SELECT EXISTS(SELECT domain from vw_blacklist WHERE domain = ? AND group_id IN (5));' /var/log/pihole/FTL.log" + run bash -c "grep -c 'get_client_querystr: SELECT id from vw_blacklist WHERE domain = ? AND group_id IN (5);' /var/log/pihole/FTL.log" printf "%s\n" "${lines[@]}" [[ ${lines[0]} == "1" ]] - run bash -c "grep -c 'get_client_querystr: SELECT EXISTS(SELECT domain from vw_gravity WHERE domain = ? AND group_id IN (5));' /var/log/pihole/FTL.log" + run bash -c "grep -c 'get_client_querystr: SELECT domain from vw_gravity WHERE domain = ? AND group_id IN (5);' /var/log/pihole/FTL.log" printf "%s\n" "${lines[@]}" [[ ${lines[0]} == "1" ]] run bash -c "grep -c 'Regex whitelist ([[:digit:]]*, DB ID [[:digit:]]*) .* NOT ENABLED for client 127.0.0.6' /var/log/pihole/FTL.log" @@ -537,22 +537,22 @@ # Here and below: Reply time is varying. don't test for a particular value (..."*"...) @test "Get all queries shows expected content" { - run bash -c 'echo ">getallqueries >quit" | nc -v 127.0.0.1 4711' + run bash -c 'echo ">getallqueries >quit" | nc -v 127.0.0.1 4711 | tee getallqueries.log' printf "%s\n" "${lines[@]}" [[ ${lines[1]} == *" TXT version.ftl 127.0.0.1 3 2 6 "*" N/A -1 N/A#0 \"\" \"0\""* ]] [[ ${lines[2]} == *" TXT version.bind 127.0.0.1 3 2 6 "*" N/A -1 N/A#0 \"\" \"1\""* ]] - [[ ${lines[3]} == *" A blacklisted.ftl 127.0.0.1 5 2 4 "*" N/A -1 N/A#0 \"\" \"2\""* ]] + [[ ${lines[3]} == *" A blacklisted.ftl 127.0.0.1 5 2 4 "*" N/A 5 N/A#0 \"\" \"2\""* ]] [[ ${lines[4]} == *" A gravity.ftl 127.0.0.1 1 2 4 "*" N/A -1 N/A#0 \"\" \"3\""* ]] [[ ${lines[5]} == *" A gravity.ftl 127.0.0.1 1 2 4 "*" N/A -1 N/A#0 \"\" \"4\""* ]] - [[ ${lines[6]} == *" A whitelisted.ftl 127.0.0.1 2 2 4 "*" N/A -1 127.0.0.1#5555 \"\" \"5\""* ]] - [[ ${lines[7]} == *" A gravity-whitelisted.ftl 127.0.0.1 2 2 4 "*" N/A -1 127.0.0.1#5555 \"\" \"6\""* ]] + [[ ${lines[6]} == *" A whitelisted.ftl 127.0.0.1 2 2 4 "*" N/A 1 127.0.0.1#5555 \"\" \"5\""* ]] + [[ ${lines[7]} == *" A gravity-whitelisted.ftl 127.0.0.1 2 2 4 "*" N/A 4 127.0.0.1#5555 \"\" \"6\""* ]] [[ ${lines[8]} == *" A regex5.ftl 127.0.0.1 4 2 4 "*" N/A 6 N/A#0 \"\" \"7\""* ]] [[ ${lines[9]} == *" A regexa.ftl 127.0.0.1 2 2 4 "*" N/A -1 127.0.0.1#5555 \"\" \"8\""* ]] - [[ ${lines[10]} == *" A regex1.ftl 127.0.0.1 2 2 4 "*" N/A -1 127.0.0.1#5555 \"\" \"9\""* ]] - [[ ${lines[11]} == *" A regex2.ftl 127.0.0.1 2 2 4 "*" N/A -1 127.0.0.1#5555 \"\" \"10\""* ]] + [[ ${lines[10]} == *" A regex1.ftl 127.0.0.1 2 2 4 "*" N/A 2 127.0.0.1#5555 \"\" \"9\""* ]] + [[ ${lines[11]} == *" A regex2.ftl 127.0.0.1 2 2 4 "*" N/A 3 127.0.0.1#5555 \"\" \"10\""* ]] [[ ${lines[12]} == *" A whitelisted.ftl 127.0.0.2 1 2 4 "*" N/A -1 N/A#0 \"\" \"11\""* ]] [[ ${lines[13]} == *" A regex1.ftl 127.0.0.2 4 2 4 "*" N/A 6 N/A#0 \"\" \"12\""* ]] - [[ ${lines[14]} == *" A regex1.ftl 127.0.0.1 3 2 4 "*" N/A -1 N/A#0 \"\" \"13\""* ]] + [[ ${lines[14]} == *" A regex1.ftl 127.0.0.1 3 2 4 "*" N/A 2 N/A#0 \"\" \"13\""* ]] [[ ${lines[15]} == *" A regex1.ftl 127.0.0.3 3 2 4 "*" N/A -1 N/A#0 \"\" \"14\""* ]] [[ ${lines[16]} == *" A blacklisted.ftl 127.0.0.2 2 2 4 "*" N/A -1 127.0.0.1#5555 \"\" \"15\""* ]] [[ ${lines[17]} == *" A blacklisted.ftl 127.0.0.3 3 2 4 "*" N/A -1 N/A#0 \"\" \"16\""* ]] From da335175538227e5590791c33de4e30879e8956f Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 20 Aug 2022 14:23:21 +0200 Subject: [PATCH 040/218] Add more debugging output to getstr() Signed-off-by: DL6ER --- src/dnsmasq_interface.c | 7 ++++--- src/regex.c | 6 +++--- src/regex_r.h | 2 +- src/shmem.c | 6 +++--- src/shmem.h | 3 ++- 5 files changed, 13 insertions(+), 11 deletions(-) diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c index dc2285b7..4c31c9f2 100644 --- a/src/dnsmasq_interface.c +++ b/src/dnsmasq_interface.c @@ -1158,7 +1158,7 @@ static bool check_domain_blocked(const char *domain, const int clientID, // Check domain against blacklist regex filters // Skipped when the domain is whitelisted or blocked by exact blacklist or gravity - if(in_regex(domain, dns_cache, client-> id, REGEX_BLACKLIST) == FOUND) + if(in_regex(domain, dns_cache, client-> id, REGEX_BLACKLIST)) { // Set new status *new_status = QUERY_REGEX; @@ -1409,11 +1409,12 @@ static bool _FTL_check_blocking(int queryID, int domainID, int clientID, const c domainstr = strdup(domainstr); const char *blockedDomain = domainstr; - // Check whitelist (exact + regex) for match + // Check exact whitelist for match query->flags.whitelisted = in_whitelist(domainstr, dns_cache, client) == FOUND; + // If not found: Check regex whitelist for match if(!query->flags.whitelisted) - query->flags.whitelisted = in_regex(domainstr, dns_cache, client->id, REGEX_WHITELIST) == FOUND; + query->flags.whitelisted = in_regex(domainstr, dns_cache, client->id, REGEX_WHITELIST); // Check blacklist (exact + regex) and gravity for queried domain unsigned char new_status = QUERY_UNKNOWN; diff --git a/src/regex.c b/src/regex.c index 2a46dd70..e6753023 100644 --- a/src/regex.c +++ b/src/regex.c @@ -475,7 +475,7 @@ static int match_regex(const char *input, DNSCacheData* dns_cache, const int cli return match_idx; } -enum db_result in_regex(const char *domain, DNSCacheData *dns_cache, const int clientID, const enum regex_type regexid) +bool in_regex(const char *domain, DNSCacheData *dns_cache, const int clientID, const enum regex_type regexid) { // For performance reasons, the regex evaluations is executed only if the // exact whitelist lookup does not deliver a positive match. This is an @@ -487,10 +487,10 @@ enum db_result in_regex(const char *domain, DNSCacheData *dns_cache, const int c { // We found a match dns_cache->domainlist_id = regex_id; - return FOUND; + return true; } - return NOT_FOUND; + return false; } static void free_regex(void) diff --git a/src/regex_r.h b/src/regex_r.h index 6a63dcda..7ec979f7 100644 --- a/src/regex_r.h +++ b/src/regex_r.h @@ -44,7 +44,7 @@ typedef struct { } regexData; unsigned int get_num_regex(const enum regex_type regexid) __attribute__((pure)); -enum db_result in_regex(const char *domain, DNSCacheData *dns_cache, const int clientID, const enum regex_type regexid); +bool in_regex(const char *domain, DNSCacheData *dns_cache, const int clientID, const enum regex_type regexid); void allocate_regex_client_enabled(clientsData *client, const int clientID); void reload_per_client_regex(clientsData *client); void read_regex_from_database(void); diff --git a/src/shmem.c b/src/shmem.c index b0fcbc6a..648b3f44 100644 --- a/src/shmem.c +++ b/src/shmem.c @@ -272,14 +272,14 @@ size_t addstr(const char *input) return (shmSettings->next_str_pos - len); } -const char *getstr(const size_t pos) +const char *_getstr(const size_t pos, const char *func, const int line, const char *file) { // Only access the string memory if this memory region has already been set if(pos < shmSettings->next_str_pos) return &((const char*)shm_strings.ptr)[pos]; else { - logg("WARN: Tried to access %zu but next_str_pos is %u", pos, shmSettings->next_str_pos); + logg("WARN: Tried to access %zu in %s() (%s:%i) but next_str_pos is %u", pos, func, file, line, shmSettings->next_str_pos); return ""; } } @@ -337,7 +337,7 @@ static void remap_shm(void) } // Obtain SHMEM lock -void _lock_shm(const char* func, const int line, const char * file) +void _lock_shm(const char *func, const int line, const char *file) { if(config.debug & DEBUG_LOCKS) logg("Waiting for SHM lock in %s() (%s:%i)", func, file, line); diff --git a/src/shmem.h b/src/shmem.h index 9c59bed3..4532c6d2 100644 --- a/src/shmem.h +++ b/src/shmem.h @@ -102,7 +102,8 @@ void _unlock_log(const char* func, const int line, const char * file); bool init_shmem(bool create_new); void destroy_shmem(void); size_t addstr(const char *str); -const char *getstr(const size_t pos); +#define getstr(pos) _getstr(pos, __FUNCTION__, __LINE__, __FILE__) +const char *_getstr(const size_t pos, const char *func, const int line, const char *file); /** * Escapes a string by replacing special characters, such as spaces From a709818764ee6b8d0f6a356b2398b6cefc50a96d Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 20 Aug 2022 14:50:57 +0200 Subject: [PATCH 041/218] Fix storing of API thread IDs Signed-off-by: DL6ER --- src/api/socket.c | 3 +-- src/daemon.c | 16 +++++++--------- src/daemon.h | 2 +- 3 files changed, 9 insertions(+), 12 deletions(-) diff --git a/src/api/socket.c b/src/api/socket.c index 1314b939..0f3b6554 100644 --- a/src/api/socket.c +++ b/src/api/socket.c @@ -263,7 +263,6 @@ void listen_telnet(const enum telnet_type type) for(unsigned int i = 0; i < MAX_API_THREADS; i++) { // Spawn telnet thread - pthread_t telnet_connection_thread; // Create a private copy of the socket fd for the child thread struct thread_info *tinfo = calloc(1, sizeof(struct thread_info)); if(!tinfo) @@ -273,7 +272,7 @@ void listen_telnet(const enum telnet_type type) tinfo->tid = i; tinfo->istelnet = (type == TELNETv4 || type == TELNETv6); tinfo->stype = stype; - if(pthread_create(&telnet_connection_thread, &attr, telnet_connection_handler_thread, (void*) tinfo) != 0) + if(pthread_create(&api_threads[i], &attr, telnet_connection_handler_thread, (void*) tinfo) != 0) { // Log the error code description logg("WARNING: Unable to open telnet processing thread: %s", strerror(errno)); diff --git a/src/daemon.c b/src/daemon.c index 9ba11f06..96890217 100644 --- a/src/daemon.c +++ b/src/daemon.c @@ -29,7 +29,7 @@ #include pthread_t threads[THREADS_MAX] = { 0 }; -pthread_t api_threads[TELNET_MAX][MAX_API_THREADS] = {{ 0 }}; +pthread_t api_threads[MAX_API_THREADS] = { 0 }; bool resolver_ready = false; void go_daemon(void) @@ -268,18 +268,16 @@ void cleanup(const int ret) // Do proper cleanup only if FTL started successfully if(resolver_ready) { + // Terminate threads terminate_threads(); // Cancel and join possibly still running API worker threads - for(enum telnet_type tt = 0; tt < TELNET_MAX; tt++) + for(unsigned int tid = 0; tid < MAX_API_THREADS; tid++) { - for(unsigned int tid = 0; tid < MAX_API_THREADS; tid++) - { - // Otherwise, cancel and join the thread - logg("Joining API worker thread %d/%d", tt,tid); - pthread_cancel(api_threads[tt][tid]); - pthread_join(api_threads[tt][tid], NULL); - } + // Otherwise, cancel and join the thread + logg("Joining API worker thread %d", tid); + pthread_cancel(api_threads[tid]); + pthread_join(api_threads[tid], NULL); } // Close database connection diff --git a/src/daemon.h b/src/daemon.h index edcd9227..cc91dd4d 100644 --- a/src/daemon.h +++ b/src/daemon.h @@ -13,7 +13,7 @@ #include "enums.h" extern pthread_t threads[THREADS_MAX]; #define MAX_API_THREADS 5 -extern pthread_t api_threads[TELNET_MAX][MAX_API_THREADS]; +extern pthread_t api_threads[MAX_API_THREADS]; void go_daemon(void); void savepid(void); From ff30d4a97361ffd301bd322edc9e0a4ddf2c5318 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 20 Aug 2022 14:30:48 +0200 Subject: [PATCH 042/218] Log memory access error details even when no debug flag is set Signed-off-by: DL6ER --- src/datastructure.c | 16 ++++++------- src/datastructure.h | 17 ++++++------- src/shmem.c | 58 ++++++++++++++++++++------------------------- 3 files changed, 43 insertions(+), 48 deletions(-) diff --git a/src/datastructure.c b/src/datastructure.c index af6b1cc1..e1aa775a 100644 --- a/src/datastructure.c +++ b/src/datastructure.c @@ -319,13 +319,13 @@ void change_clientcount(clientsData *client, int total, int blocked, int overTim } } -int findCacheID(int domainID, int clientID, enum query_types query_type) +int _findCacheID(int domainID, int clientID, enum query_types query_type, const char *func, int line, const char *file) { // Compare content of client against known client IP addresses for(int cacheID = 0; cacheID < counters->dns_cache_size; cacheID++) { // Get cache pointer - DNSCacheData* dns_cache = getDNSCache(cacheID, true); + DNSCacheData* dns_cache = _getDNSCache(cacheID, true, line, func, file); // Check if the returned pointer is valid before trying to access it if(dns_cache == NULL) @@ -343,7 +343,7 @@ int findCacheID(int domainID, int clientID, enum query_types query_type) const int cacheID = counters->dns_cache_size; // Get client pointer - DNSCacheData* dns_cache = getDNSCache(cacheID, false); + DNSCacheData* dns_cache = _getDNSCache(cacheID, false, line, func, file); if(dns_cache == NULL) { @@ -563,7 +563,7 @@ static const char *query_status_str[QUERY_STATUS_MAX] = { "SPECIAL_DOMAIN" }; -void _query_set_status(queriesData *query, const enum query_status new_status, const char *file, const int line) +void _query_set_status(queriesData *query, const enum query_status new_status, const char *func, const int line, const char *file) { // Debug logging if(config.debug & DEBUG_STATUS) @@ -571,14 +571,14 @@ void _query_set_status(queriesData *query, const enum query_status new_status, c const char *oldstr = query->status < QUERY_STATUS_MAX ? query_status_str[query->status] : "INVALID"; if(query->status == new_status) { - logg("Query %i: status unchanged: %s (%d) in %s:%i", - query->id, oldstr, query->status, short_path(file), line); + logg("Query %i: status unchanged: %s (%d) in %s() (%s:%i)", + query->id, oldstr, query->status, func, short_path(file), line); } else { const char *newstr = new_status < QUERY_STATUS_MAX ? query_status_str[new_status] : "INVALID"; - logg("Query %i: status changed: %s (%d) -> %s (%d) in %s:%i", - query->id, oldstr, query->status, newstr, new_status, short_path(file), line); + logg("Query %i: status changed: %s (%d) -> %s (%d) in %s() (%s:%i)", + query->id, oldstr, query->status, newstr, new_status, func, short_path(file), line); } } diff --git a/src/datastructure.h b/src/datastructure.h index 98876cc1..e1ef2779 100644 --- a/src/datastructure.h +++ b/src/datastructure.h @@ -113,13 +113,14 @@ int findQueryID(const int id); int findUpstreamID(const char * upstream, const in_port_t port); int findDomainID(const char *domain, const bool count); int findClientID(const char *client, const bool count, const bool aliasclient); -int findCacheID(int domainID, int clientID, enum query_types query_type); +#define findCacheID(domainID, clientID, query_type) _findCacheID(domainID, clientID, query_type, __FUNCTION__, __LINE__, __FILE__) +int _findCacheID(int domainID, int clientID, enum query_types query_type, const char *func, const int line, const char *file); bool isValidIPv4(const char *addr); bool isValidIPv6(const char *addr); bool is_blocked(const enum query_status status) __attribute__ ((const)); -#define query_set_status(query, new_status) _query_set_status(query, new_status, __FILE__, __LINE__) -void _query_set_status(queriesData *query, const enum query_status new_status, const char *file, const int line); +#define query_set_status(query, new_status) _query_set_status(query, new_status, __FUNCTION__, __LINE__, __FILE__) +void _query_set_status(queriesData *query, const enum query_status new_status, const char *func, const int line, const char *file); void FTL_reload_all_domainlists(void); void FTL_reset_per_client_domain_data(void); @@ -135,14 +136,14 @@ const char *get_query_reply_str(const enum reply_type query) __attribute__ ((con // Pointer getter functions #define getQuery(queryID, checkMagic) _getQuery(queryID, checkMagic, __LINE__, __FUNCTION__, __FILE__) -queriesData* _getQuery(int queryID, bool checkMagic, int line, const char * function, const char * file); +queriesData* _getQuery(int queryID, bool checkMagic, int line, const char *func, const char *file); #define getClient(clientID, checkMagic) _getClient(clientID, checkMagic, __LINE__, __FUNCTION__, __FILE__) -clientsData* _getClient(int clientID, bool checkMagic, int line, const char * function, const char * file); +clientsData* _getClient(int clientID, bool checkMagic, int line, const char *func, const char *file); #define getDomain(domainID, checkMagic) _getDomain(domainID, checkMagic, __LINE__, __FUNCTION__, __FILE__) -domainsData* _getDomain(int domainID, bool checkMagic, int line, const char * function, const char * file); +domainsData* _getDomain(int domainID, bool checkMagic, int line, const char *func, const char *file); #define getUpstream(upstreamID, checkMagic) _getUpstream(upstreamID, checkMagic, __LINE__, __FUNCTION__, __FILE__) -upstreamsData* _getUpstream(int upstreamID, bool checkMagic, int line, const char * function, const char * file); +upstreamsData* _getUpstream(int upstreamID, bool checkMagic, int line, const char *func, const char *file); #define getDNSCache(cacheID, checkMagic) _getDNSCache(cacheID, checkMagic, __LINE__, __FUNCTION__, __FILE__) -DNSCacheData* _getDNSCache(int cacheID, bool checkMagic, int line, const char * function, const char * file); +DNSCacheData* _getDNSCache(int cacheID, bool checkMagic, int line, const char *func, const char *file); #endif //DATASTRUCTURE_H diff --git a/src/shmem.c b/src/shmem.c index 648b3f44..01d94b7b 100644 --- a/src/shmem.c +++ b/src/shmem.c @@ -999,16 +999,13 @@ void set_per_client_regex(const int clientID, const int regexID, const bool valu ((bool*) shm_per_client_regex.ptr)[id] = value; } -static inline bool check_range(int ID, int MAXID, const char* type, int line, const char * function, const char * file) +static inline bool check_range(int ID, int MAXID, const char* type, const char *func, int line, const char *file) { // Check bounds if(ID < 0 || ID > MAXID) { - if(config.debug) - { - logg("ERROR: Trying to access %s ID %i, but maximum is %i", type, ID, MAXID); - logg(" found in %s() (%s:%i)", function, file, line); - } + logg("ERROR: Trying to access %s ID %i, but maximum is %i", type, ID, MAXID); + logg(" found in %s() (%s:%i)", func, short_path(file), line); return false; } @@ -1016,16 +1013,13 @@ static inline bool check_range(int ID, int MAXID, const char* type, int line, co return true; } -static inline bool check_magic(int ID, bool checkMagic, unsigned char magic, const char* type, int line, const char * function, const char * file) +static inline bool check_magic(int ID, bool checkMagic, unsigned char magic, const char *type, const char *func, int line, const char *file) { // Check magic only if requested (skipped for new entries which are uninitialized) if(checkMagic && magic != MAGICBYTE) { - if(config.debug) - { - logg("ERROR: Trying to access %s ID %i, but magic byte is %x", type, ID, magic); - logg(" found in %s() (%s:%i)", function, file, line); - } + logg("ERROR: Trying to access %s ID %i, but magic byte is %x", type, ID, magic); + logg(" found in %s() (%s:%i)", func, short_path(file), line); return false; } @@ -1033,7 +1027,7 @@ static inline bool check_magic(int ID, bool checkMagic, unsigned char magic, con return true; } -queriesData* _getQuery(int queryID, bool checkMagic, int line, const char * function, const char * file) +queriesData* _getQuery(int queryID, bool checkMagic, int line, const char *func, const char *file) { // This does not exist, return a NULL pointer if(queryID == -1) @@ -1043,19 +1037,19 @@ queriesData* _getQuery(int queryID, bool checkMagic, int line, const char * func if(config.debug & DEBUG_LOCKS && !is_our_lock()) { logg("ERROR: Tried to obtain query pointer without lock in %s() (%s:%i)!", - function, file, line); + func, short_path(file), line); generate_backtrace(); return NULL; } - if(check_range(queryID, counters->queries_MAX, "query", line, function, file) && - check_magic(queryID, checkMagic, queries[queryID].magic, "query", line, function, file)) + if(check_range(queryID, counters->queries_MAX, "query", func, line, file) && + check_magic(queryID, checkMagic, queries[queryID].magic, "query", func, line, file)) return &queries[queryID]; else return NULL; } -clientsData* _getClient(int clientID, bool checkMagic, int line, const char * function, const char * file) +clientsData* _getClient(int clientID, bool checkMagic, int line, const char *func, const char *file) { // This does not exist, we return a NULL pointer if(clientID == -1) @@ -1065,19 +1059,19 @@ clientsData* _getClient(int clientID, bool checkMagic, int line, const char * fu if(config.debug & DEBUG_LOCKS && !is_our_lock()) { logg("ERROR: Tried to obtain client pointer without lock in %s() (%s:%i)!", - function, file, line); + func, short_path(file), line); generate_backtrace(); return NULL; } - if(check_range(clientID, counters->clients_MAX, "client", line, function, file) && - check_magic(clientID, checkMagic, clients[clientID].magic, "client", line, function, file)) + if(check_range(clientID, counters->clients_MAX, "client", func, line, file) && + check_magic(clientID, checkMagic, clients[clientID].magic, "client", func, line, file)) return &clients[clientID]; else return NULL; } -domainsData* _getDomain(int domainID, bool checkMagic, int line, const char * function, const char * file) +domainsData* _getDomain(int domainID, bool checkMagic, int line, const char *func, const char *file) { // This does not exist, we return a NULL pointer if(domainID == -1) @@ -1087,19 +1081,19 @@ domainsData* _getDomain(int domainID, bool checkMagic, int line, const char * fu if(config.debug & DEBUG_LOCKS && !is_our_lock()) { logg("ERROR: Tried to obtain domain pointer without lock in %s() (%s:%i)!", - function, file, line); + func, short_path(file), line); generate_backtrace(); return NULL; } - if(check_range(domainID, counters->domains_MAX, "domain", line, function, file) && - check_magic(domainID, checkMagic, domains[domainID].magic, "domain", line, function, file)) + if(check_range(domainID, counters->domains_MAX, "domain", func, line, file) && + check_magic(domainID, checkMagic, domains[domainID].magic, "domain", func, line, file)) return &domains[domainID]; else return NULL; } -upstreamsData* _getUpstream(int upstreamID, bool checkMagic, int line, const char * function, const char * file) +upstreamsData* _getUpstream(int upstreamID, bool checkMagic, int line, const char *func, const char *file) { // This does not exist, we return a NULL pointer if(upstreamID == -1) @@ -1109,19 +1103,19 @@ upstreamsData* _getUpstream(int upstreamID, bool checkMagic, int line, const cha if(config.debug & DEBUG_LOCKS && !is_our_lock()) { logg("ERROR: Tried to obtain upstream pointer without lock in %s() (%s:%i)!", - function, file, line); + func, short_path(file), line); generate_backtrace(); return NULL; } - if(check_range(upstreamID, counters->upstreams_MAX, "upstream", line, function, file) && - check_magic(upstreamID, checkMagic, upstreams[upstreamID].magic, "upstream", line, function, file)) + if(check_range(upstreamID, counters->upstreams_MAX, "upstream", func, line, file) && + check_magic(upstreamID, checkMagic, upstreams[upstreamID].magic, "upstream", func, line, file)) return &upstreams[upstreamID]; else return NULL; } -DNSCacheData* _getDNSCache(int cacheID, bool checkMagic, int line, const char * function, const char * file) +DNSCacheData* _getDNSCache(int cacheID, bool checkMagic, int line, const char *func, const char *file) { // This does not exist, we return a NULL pointer if(cacheID == -1) @@ -1131,13 +1125,13 @@ DNSCacheData* _getDNSCache(int cacheID, bool checkMagic, int line, const char * if(config.debug & DEBUG_LOCKS && !is_our_lock()) { logg("ERROR: Tried to obtain cache pointer without lock in %s() (%s:%i)!", - function, file, line); + func, short_path(file), line); generate_backtrace(); return NULL; } - if(check_range(cacheID, counters->dns_cache_MAX, "dns_cache", line, function, file) && - check_magic(cacheID, checkMagic, dns_cache[cacheID].magic, "dns_cache", line, function, file)) + if(check_range(cacheID, counters->dns_cache_MAX, "dns_cache", func, line, file) && + check_magic(cacheID, checkMagic, dns_cache[cacheID].magic, "dns_cache", func, line, file)) return &dns_cache[cacheID]; else return NULL; From 1ccfd9a081490a2ae5ca84321c1ec0af09cfffa2 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 20 Aug 2022 15:27:38 +0200 Subject: [PATCH 043/218] Ensure the API does not try to create DNS cache records for queries that don't have one Signed-off-by: DL6ER --- src/api/api.c | 2 +- src/database/query-table.c | 4 ++-- src/datastructure.c | 5 ++++- src/datastructure.h | 4 ++-- src/dnsmasq_interface.c | 6 +++--- 5 files changed, 12 insertions(+), 9 deletions(-) diff --git a/src/api/api.c b/src/api/api.c index 67da3334..441f1626 100644 --- a/src/api/api.c +++ b/src/api/api.c @@ -1012,7 +1012,7 @@ void getAllQueries(const char *client_message, const int sock, const bool isteln int domainlist_id = -1; if (config.privacylevel < PRIVACY_HIDE_DOMAINS) { - unsigned int cacheID = findCacheID(query->domainID, query->clientID, query->type); + unsigned int cacheID = findCacheID(query->domainID, query->clientID, query->type, false); DNSCacheData *dns_cache = getDNSCache(cacheID, true); if(dns_cache != NULL) domainlist_id = dns_cache->domainlist_id; diff --git a/src/database/query-table.c b/src/database/query-table.c index 3b50b66f..ed532066 100644 --- a/src/database/query-table.c +++ b/src/database/query-table.c @@ -373,7 +373,7 @@ int DB_save_queries(sqlite3 *db) sqlite3_bind_null(query_stmt, 7); } - const int cacheID = findCacheID(query->domainID, query->clientID, query->type); + const int cacheID = findCacheID(query->domainID, query->clientID, query->type, false); DNSCacheData *cache = getDNSCache(cacheID, true); // ADDITIONAL_INFO @@ -933,7 +933,7 @@ void DB_read_queries(void) // Set ID of the domainlist entry that was the reason for permitting/blocking this query // We assume the value in this field is said ID when it is not a CNAME-related domain // (checked above) and the value of additional_info is not NULL (0 bytes storage size) - const int cacheID = findCacheID(query->domainID, query->clientID, query->type); + const int cacheID = findCacheID(query->domainID, query->clientID, query->type, true); DNSCacheData *cache = getDNSCache(cacheID, true); // Only load if // a) we have a cache entry diff --git a/src/datastructure.c b/src/datastructure.c index e1aa775a..904e49ca 100644 --- a/src/datastructure.c +++ b/src/datastructure.c @@ -319,7 +319,7 @@ void change_clientcount(clientsData *client, int total, int blocked, int overTim } } -int _findCacheID(int domainID, int clientID, enum query_types query_type, const char *func, int line, const char *file) +int _findCacheID(const int domainID, const int clientID, const enum query_types query_type, const bool create_new, const char *func, int line, const char *file) { // Compare content of client against known client IP addresses for(int cacheID = 0; cacheID < counters->dns_cache_size; cacheID++) @@ -339,6 +339,9 @@ int _findCacheID(int domainID, int clientID, enum query_types query_type, const } } + if(!create_new) + return -1; + // Get ID of new cache entry const int cacheID = counters->dns_cache_size; diff --git a/src/datastructure.h b/src/datastructure.h index e1ef2779..a37db29c 100644 --- a/src/datastructure.h +++ b/src/datastructure.h @@ -113,8 +113,8 @@ int findQueryID(const int id); int findUpstreamID(const char * upstream, const in_port_t port); int findDomainID(const char *domain, const bool count); int findClientID(const char *client, const bool count, const bool aliasclient); -#define findCacheID(domainID, clientID, query_type) _findCacheID(domainID, clientID, query_type, __FUNCTION__, __LINE__, __FILE__) -int _findCacheID(int domainID, int clientID, enum query_types query_type, const char *func, const int line, const char *file); +#define findCacheID(domainID, clientID, query_type, create_new) _findCacheID(domainID, clientID, query_type, create_new, __FUNCTION__, __LINE__, __FILE__) +int _findCacheID(const int domainID, const int clientID, const enum query_types query_type, const bool create_new, const char *func, const int line, const char *file); bool isValidIPv4(const char *addr); bool isValidIPv6(const char *addr); diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c index 4c31c9f2..a652c44a 100644 --- a/src/dnsmasq_interface.c +++ b/src/dnsmasq_interface.c @@ -1249,7 +1249,7 @@ static bool _FTL_check_blocking(int queryID, int domainID, int clientID, const c } // Get cache pointer - unsigned int cacheID = findCacheID(domainID, clientID, query->type); + unsigned int cacheID = findCacheID(domainID, clientID, query->type, true); DNSCacheData *dns_cache = getDNSCache(cacheID, true); if(dns_cache == NULL) { @@ -1581,8 +1581,8 @@ bool _FTL_CNAME(const char *domain, const struct crec *cpp, const int id, const else if(query->status == QUERY_REGEX) { // Get parent and child DNS cache entries - const int parent_cacheID = findCacheID(parent_domainID, clientID, query->type); - const int child_cacheID = findCacheID(child_domainID, clientID, query->type); + const int parent_cacheID = findCacheID(parent_domainID, clientID, query->type, false); + const int child_cacheID = findCacheID(child_domainID, clientID, query->type, false); // Get cache pointers DNSCacheData *parent_cache = getDNSCache(parent_cacheID, true); From 79fcefd6348d2a7c719e4239347634d19cd5e483 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 21 Aug 2022 11:59:15 +0200 Subject: [PATCH 044/218] Add missing function attribute to syscall to ensure compatibility with gcc 11.2.0 (Ubuntu 11.2.0-19ubuntu1) Signed-off-by: DL6ER --- src/syscalls/fopen.c | 2 +- src/syscalls/syscalls.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/syscalls/fopen.c b/src/syscalls/fopen.c index d6416533..e8ca6dd9 100644 --- a/src/syscalls/fopen.c +++ b/src/syscalls/fopen.c @@ -15,7 +15,7 @@ static uint8_t already_writing = 0; #undef fopen -FILE *FTLfopen(const char *pathname, const char *mode, const char *file, const char *func, const int line) +FILE * __attribute__ ((__malloc__)) FTLfopen(const char *pathname, const char *mode, const char *file, const char *func, const int line) { FILE *file_ptr = 0; do diff --git a/src/syscalls/syscalls.h b/src/syscalls/syscalls.h index 1b2787cd..dfe1a178 100644 --- a/src/syscalls/syscalls.h +++ b/src/syscalls/syscalls.h @@ -45,7 +45,7 @@ ssize_t FTLsendto(int sockfd, void *buf, size_t len, int flags, const struct soc int FTLpthread_mutex_lock(pthread_mutex_t *__mutex, const char *file, const char *func, const int line); // Interrupt-safe file routines -FILE *FTLfopen(const char *pathname, const char *mode, const char *file, const char *func, const int line); +FILE *FTLfopen(const char *pathname, const char *mode, const char *file, const char *func, const int line) __attribute__ ((__malloc__)); // Syscall helpers void syscalls_report_error(const char *error, FILE *stream, const int _errno, const char *format, const char *func, const char *file, const int line); From 88763b3bcca5c3c45e3cca3354022f66bcc341f4 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 27 Aug 2022 12:58:56 +0200 Subject: [PATCH 045/218] Substitute "." from empty domain names passed by dnsmasq Signed-off-by: DL6ER --- src/dnsmasq_interface.c | 30 +++++++++++++++++++++--------- src/dnsmasq_interface.h | 2 +- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c index dddd4e89..f9573bd3 100644 --- a/src/dnsmasq_interface.c +++ b/src/dnsmasq_interface.c @@ -73,6 +73,7 @@ static void check_pihole_PTR(char *domain); #define query_set_dnssec(query, dnssec) _query_set_dnssec(query, dnssec, __FILE__, __LINE__) static void _query_set_dnssec(queriesData *query, const enum dnssec_status dnssec, const char *file, const int line); static char *get_ptrname(struct in_addr *addr); +static const char *check_dnsmasq_name(const char *name); // Static blocking metadata static const char *blockingreason = ""; @@ -96,23 +97,18 @@ static union mysockaddr last_server = {{ 0 }}; unsigned char* pihole_privacylevel = &config.privacylevel; const char *flagnames[] = {"F_IMMORTAL ", "F_NAMEP ", "F_REVERSE ", "F_FORWARD ", "F_DHCP ", "F_NEG ", "F_HOSTS ", "F_IPV4 ", "F_IPV6 ", "F_BIGNAME ", "F_NXDOMAIN ", "F_CNAME ", "F_DNSKEY ", "F_CONFIG ", "F_DS ", "F_DNSSECOK ", "F_UPSTREAM ", "F_RRNAME ", "F_SERVER ", "F_QUERY ", "F_NOERR ", "F_AUTH ", "F_DNSSEC ", "F_KEYTAG ", "F_SECSTAT ", "F_NO_RR ", "F_IPSET ", "F_NOEXTRA ", "F_SERVFAIL", "F_RCODE"}; -void FTL_hook(unsigned int flags, char *name, union all_addr *addr, char *arg, int id, unsigned short type, const char* file, const int line) +void FTL_hook(unsigned int flags, const char *name, union all_addr *addr, char *arg, int id, unsigned short type, const char* file, const int line) { // Extract filename from path const char *path = short_path(file); if(config.debug & DEBUG_FLAGS) { - logg("Processing FTL hook from %s:%d...", path, line); + logg("Processing FTL hook from %s:%d (name: \"%s\")...", path, line, name); print_flags(flags); } - // Special domain name handling - // 1. Substitute "(NULL)" if no name is available (should not happen) - // 2. Substitute "." if we are querying the root domain (e.g. DNSKEY) - if(!name) - name = (char*)"(NULL)"; - else if(!name[0]) - name = (char*)"."; + // Check domain name received from dnsmasq + name = check_dnsmasq_name(name); // Note: The order matters here! if((flags & F_QUERY) && (flags & F_FORWARD)) @@ -513,6 +509,9 @@ bool _FTL_new_query(const unsigned int flags, const char *name, break; } + // Check domain name received from dnsmasq + name = check_dnsmasq_name(name); + // If domain is "pi.hole" or the local hostname we skip analyzing this query // and, instead, immediately reply with the IP address - these queries are not further analyzed if(is_pihole_domain(name)) @@ -3340,3 +3339,16 @@ int check_struct_sizes(void) return result; } + +static const char *check_dnsmasq_name(const char *name) +{ + // Special domain name handling + if(!name) + // 1. Substitute "(NULL)" if no name is available (should not happen) + return "(NULL)"; + else if(!name[0]) + // 2. Substitute "." if we are querying the root domain (e.g. DNSKEY) + return "."; + // else + return name; +} diff --git a/src/dnsmasq_interface.h b/src/dnsmasq_interface.h index a35b7381..731a1188 100644 --- a/src/dnsmasq_interface.h +++ b/src/dnsmasq_interface.h @@ -18,7 +18,7 @@ extern unsigned char* pihole_privacylevel; enum protocol { TCP, UDP, INTERNAL }; -void FTL_hook(unsigned int flags, char *name, union all_addr *addr, char *arg, int id, unsigned short type, const char* file, const int line); +void FTL_hook(unsigned int flags, const char *name, union all_addr *addr, char *arg, int id, unsigned short type, const char* file, const int line); #define FTL_iface(iface, addr, addrfamily) _FTL_iface(iface, addr, addrfamily, __FILE__, __LINE__) void _FTL_iface(struct irec *recviface, const union all_addr *addr, const sa_family_t addrfamily, const char* file, const int line); From 04133fb3628c589c38c9bb606d09f517375cf90f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Wed, 31 Aug 2022 14:10:43 +0200 Subject: [PATCH 046/218] Log inaccesible adlists to message table MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- src/database/gravity-db.c | 40 ++++++++++++++++++++++++++++++++++++ src/database/gravity-db.h | 1 + src/database/message-table.c | 18 +++++++++++++++- src/database/message-table.h | 1 + src/datastructure.c | 3 +++ src/dnsmasq_interface.c | 1 + src/enums.h | 3 ++- 7 files changed, 65 insertions(+), 2 deletions(-) diff --git a/src/database/gravity-db.c b/src/database/gravity-db.c index 06551dfe..d23f1d77 100644 --- a/src/database/gravity-db.c +++ b/src/database/gravity-db.c @@ -22,6 +22,7 @@ // SQLite3 prepared statement vectors #include "../vector.h" // log_subnet_warning() +// logg_inaccessible_adlist #include "message-table.h" // getMACfromIP() #include "network-table.h" @@ -1407,3 +1408,42 @@ bool gravityDB_get_regex_client_groups(clientsData* client, const unsigned int n return true; } + +void check_inaccessible_adlists(void){ + + // check if any adlist was inaccessible in the last gravity run + // if so, gravity stored `status` in the adlist table with + // "3": List unavailable, Pi-hole used a local copy + // "4": List unavailable, there is no local copy available + + // Do not proceed when database is not available + if(!gravityDB_opened && !gravityDB_open()) + { + logg("check_inaccessible_adlists(): Gravity database not available"); + return; + } + + const char *querystr = "SELECT id, address FROM adlist WHERE status IN (3,4)"; + + // Prepare query + sqlite3_stmt *query_stmt; + int rc = sqlite3_prepare_v2(gravity_db, querystr, -1, &query_stmt, NULL); + if(rc != SQLITE_OK){ + logg("check_inaccessible_adlists(): %s - SQL error prepare: %s", querystr, sqlite3_errstr(rc)); + gravityDB_close(); + return; + } + + // Perform query + while((rc = sqlite3_step(query_stmt)) == SQLITE_ROW) + { + int id = sqlite3_column_int(query_stmt, 0); + const char *address = strdup((const char*)sqlite3_column_text(query_stmt, 1)); + + // log to the message table + logg_inaccessible_adlist(id, address); + } + + // Finalize statement + sqlite3_finalize(query_stmt); +} diff --git a/src/database/gravity-db.h b/src/database/gravity-db.h index 6c4b9f00..c17fc6f4 100644 --- a/src/database/gravity-db.h +++ b/src/database/gravity-db.h @@ -29,6 +29,7 @@ const char* gravityDB_getDomain(int *rowid); char* get_client_names_from_ids(const char *group_ids) __attribute__ ((malloc)); void gravityDB_finalizeTable(void); int gravityDB_count(const enum gravity_tables list); +void check_inaccessible_adlists(void); enum db_result in_gravity(const char *domain, clientsData *client); enum db_result in_blacklist(const char *domain, clientsData *client); diff --git a/src/database/message-table.c b/src/database/message-table.c index 317b849a..36a23855 100644 --- a/src/database/message-table.c +++ b/src/database/message-table.c @@ -27,7 +27,7 @@ #include "../gc.h" static const char *message_types[MAX_MESSAGE] = - { "REGEX", "SUBNET", "HOSTNAME", "DNSMASQ_CONFIG", "RATE_LIMIT", "DNSMASQ_WARN", "LOAD", "SHMEM", "DISK" }; + { "REGEX", "SUBNET", "HOSTNAME", "DNSMASQ_CONFIG", "RATE_LIMIT", "DNSMASQ_WARN", "LOAD", "SHMEM", "DISK", "ADLIST" }; static unsigned char message_blob_types[MAX_MESSAGE][5] = { @@ -94,6 +94,13 @@ static unsigned char message_blob_types[MAX_MESSAGE][5] = SQLITE_NULL, // Not used SQLITE_NULL // Not used }, + { // INACCESSIBLE_ADLIST_MESSAGE: The message column contains the corresponding adlist URL + SQLITE_INTEGER, // database index of the adlist (so the dashboard can show a link) + SQLITE_NULL, // not used + SQLITE_NULL, // not used + SQLITE_NULL, // not used + SQLITE_NULL // not used + }, }; // Create message table in the database bool create_message_table(sqlite3 *db) @@ -391,3 +398,12 @@ void log_resource_shortage(const double load, const int nprocs, const int shmem, add_message(DISK_MESSAGE, true, path, 2, disk, msg); } } + +void logg_inaccessible_adlist(const int dbindex, const char *address) +{ + // Log to FTL.log + logg("Adlist warning: Adlist with ID %d (%s) was inaccessible during last gravity run", dbindex, address); + + // Log to database + add_message(INACCESSIBLE_ADLIST_MESSAGE, false, address, 1, dbindex); +} diff --git a/src/database/message-table.h b/src/database/message-table.h index ece5b752..7d006a28 100644 --- a/src/database/message-table.h +++ b/src/database/message-table.h @@ -23,5 +23,6 @@ void logg_fatal_dnsmasq_message(const char *message); void logg_rate_limit_message(const char *clientIP, const unsigned int rate_limit_count); void logg_warn_dnsmasq_message(char *message); void log_resource_shortage(const double load, const int nprocs, const int shmem, const int disk, const char *path, const char *msg); +void logg_inaccessible_adlist(const int dbindex, const char *address); #endif //MESSAGETABLE_H diff --git a/src/datastructure.c b/src/datastructure.c index d7bc1cfd..2f7f9784 100644 --- a/src/datastructure.c +++ b/src/datastructure.c @@ -506,6 +506,9 @@ void FTL_reload_all_domainlists(void) // only after having called gravityDB_open() read_regex_from_database(); + // Check for inaccessible adlist URLs + check_inaccessible_adlists(); + // Reset FTL's internal DNS cache storing whether a specific domain // has already been validated for a specific user FTL_reset_per_client_domain_data(); diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c index f9573bd3..c3155bc9 100644 --- a/src/dnsmasq_interface.c +++ b/src/dnsmasq_interface.c @@ -1788,6 +1788,7 @@ void FTL_dnsmasq_reload(void) // Gravity database updates // - (Re-)open gravity database connection // - Get number of blocked domains + // - check adlist table for inaccessible adlists // - Read and compile regex filters (incl. per-client) // - Flush FTL's DNS cache set_event(RELOAD_GRAVITY); diff --git a/src/enums.h b/src/enums.h index 08982f56..cd4dd3fd 100644 --- a/src/enums.h +++ b/src/enums.h @@ -209,7 +209,8 @@ enum message_type { LOAD_MESSAGE, SHMEM_MESSAGE, DISK_MESSAGE, - MAX_MESSAGE + INACCESSIBLE_ADLIST_MESSAGE, + MAX_MESSAGE, } __attribute__ ((packed)); enum ptr_type { From 94a28040fabff5343657a6866ccb8b84f720b6d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Wed, 31 Aug 2022 17:32:07 +0200 Subject: [PATCH 047/218] Update gravity.db.sql MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- test/gravity.db.sql | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/test/gravity.db.sql b/test/gravity.db.sql index 9a73d811..212434f4 100644 --- a/test/gravity.db.sql +++ b/test/gravity.db.sql @@ -25,12 +25,16 @@ CREATE TABLE domainlist CREATE TABLE adlist ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - address TEXT UNIQUE NOT NULL, - enabled BOOLEAN NOT NULL DEFAULT 1, - date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), - date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), - comment TEXT + id INTEGER PRIMARY KEY AUTOINCREMENT, + address TEXT UNIQUE NOT NULL, + enabled BOOLEAN NOT NULL DEFAULT 1, + date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + comment TEXT, + date_updated INTEGER, + number INTEGER NOT NULL DEFAULT 0, + invalid_domains INTEGER NOT NULL DEFAULT 0, + status INTEGER NOT NULL DEFAULT 0 ); CREATE TABLE adlist_by_group @@ -208,7 +212,7 @@ INSERT INTO domainlist VALUES(14,3,'^regex-notA$;querytype=!A',1,1559928803,1559 /* Other special domains */ INSERT INTO domainlist VALUES(15,1,'blacklisted-group-disabled.com',1,1559928803,1559928803,'Entry disabled by a group'); -INSERT INTO adlist VALUES(1,'https://hosts-file.net/ad_servers.txt',1,1559928803,1559928803,'Migrated from /etc/pihole/adlists.list'); +INSERT INTO adlist VALUES(1,'https://hosts-file.net/ad_servers.txt',1,1559928803,1559928803,'Migrated from /etc/pihole/adlists.list',1559928803,2000,2,1); INSERT INTO gravity VALUES('whitelisted.ftl',1); INSERT INTO gravity VALUES('gravity.ftl',1); From 5b20a7bc839bd9baa7383ee0970228041bf81c44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Thu, 1 Sep 2022 08:34:23 +0200 Subject: [PATCH 048/218] Only check enabled adlists MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- src/database/gravity-db.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/database/gravity-db.c b/src/database/gravity-db.c index d23f1d77..396363c0 100644 --- a/src/database/gravity-db.c +++ b/src/database/gravity-db.c @@ -1423,7 +1423,7 @@ void check_inaccessible_adlists(void){ return; } - const char *querystr = "SELECT id, address FROM adlist WHERE status IN (3,4)"; + const char *querystr = "SELECT id, address FROM adlist WHERE status IN (3,4) AND enabled=1"; // Prepare query sqlite3_stmt *query_stmt; From 87f8ef717124af88c8f95a21f6e22e4e0c2b3f98 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Fri, 2 Sep 2022 19:10:52 +0100 Subject: [PATCH 049/218] Helps if we are attaching from the correct location... Signed-off-by: Adam Warner --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3303bc34..be2c7ce9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -164,4 +164,4 @@ jobs: uses: softprops/action-gh-release@v1 with: files: | - * + ${{ steps.download.outputs.download-path }}/* From d7c55191656fba968c0b7680e47774527d0104ac Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 4 Sep 2022 14:56:48 +0200 Subject: [PATCH 050/218] Tweak CNAME inspection to also work on abandonned paths. The most obvious example for such a configuration is a CNAME child ending without a record (NODATA). See FTL#1421. Signed-off-by: DL6ER --- src/dnsmasq/forward.c | 21 ++++++++++++++++++++- src/dnsmasq/rfc1035.c | 24 +++++++++++++----------- src/dnsmasq_interface.c | 23 +++++------------------ src/dnsmasq_interface.h | 4 ++-- 4 files changed, 40 insertions(+), 32 deletions(-) diff --git a/src/dnsmasq/forward.c b/src/dnsmasq/forward.c index 4c061227..bc36eafe 100644 --- a/src/dnsmasq/forward.c +++ b/src/dnsmasq/forward.c @@ -624,6 +624,9 @@ static size_t process_reply(struct dns_header *header, time_t now, struct server int munged = 0, is_sign; unsigned int rcode = RCODE(header); size_t plen; + /******** Pi-hole modification ********/ + unsigned char *pheader_copy = NULL; + /**************************************/ (void)ad_reqd; (void)do_bit; @@ -762,6 +765,16 @@ static size_t process_reply(struct dns_header *header, time_t now, struct server if (ret == 2) { cache_secure = 0; + // Make a private copy of the pheader to ensure + // we are not accidentially rewriting what is in + // the pheader when we're creating a crafted reply + // further below (when a query is to be blocked) + if (pheader) + { + pheader_copy = calloc(1, plen); + memcpy(pheader_copy, pheader, plen); + } + // Generate DNS packet for reply, a possibly existing pseudo header // will be restored later inside resize_packet() n = FTL_make_answer(header, ((char *) header) + 65536, n, &ede); @@ -813,7 +826,13 @@ static size_t process_reply(struct dns_header *header, time_t now, struct server /* the bogus-nxdomain stuff, doctor and NXDOMAIN->NODATA munging can all elide sections of the packet. Find the new length here and put back pseudoheader if it was removed. */ - n = resize_packet(header, n, pheader, plen); + n = resize_packet(header, n, pheader_copy ? pheader_copy : pheader, plen); + /******** Pi-hole modification ********/ + // The line above was modified to use + // pheader_copy instead of pheader + if(pheader_copy) + free(pheader_copy); + /**************************************/ if (pheader && ede != EDE_UNSET) { diff --git a/src/dnsmasq/rfc1035.c b/src/dnsmasq/rfc1035.c index 6d7490d3..9a9fc7a9 100644 --- a/src/dnsmasq/rfc1035.c +++ b/src/dnsmasq/rfc1035.c @@ -763,16 +763,6 @@ int extract_addresses(struct dns_header *header, size_t qlen, char *name, time_t } #endif - // ****************************** Pi-hole modification ****************************** - if(FTL_CNAME(name, cpp, daemon->log_display_id)) - { - // Found while processing a reply from upstream. We prevent cache insertion here - // This query is to be blocked as we found a blocked - // domain while walking the CNAME path. Log to pihole.log here - log_query(F_UPSTREAM, name, NULL, "blocked during CNAME inspection", 0); - return 2; - } - // ********************************************************************************** if (aqtype == T_CNAME) { @@ -804,6 +794,17 @@ int extract_addresses(struct dns_header *header, size_t qlen, char *name, time_t if (!extract_name(header, qlen, &p1, name, 1, 0)) return 0; + // ****************************** Pi-hole modification ****************************** + const char *src = cpp != NULL ? cpp->flags & F_BIGNAME ? cpp->name.bname->name : cpp->name.sname : NULL; + if(FTL_CNAME(name, src, daemon->log_display_id)) + { + // Found while processing a reply from upstream. We prevent cache insertion here + // This query is to be blocked as we found a blocked + // domain while walking the CNAME path. Log to pihole.log here + log_query(F_UPSTREAM, name, NULL, "blocked during CNAME inspection", 0); + return 2; + } + // ********************************************************************************** if (qtype != T_CNAME) goto cname_loop1; @@ -1859,7 +1860,8 @@ size_t answer_request(struct dns_header *header, char *limit, size_t qlen, log_query(crecp->flags & ~F_REVERSE, name, &crecp->addr, record_source(crecp->uid), 0); // ****************************** Pi-hole modification ****************************** - if(FTL_CNAME(name, crecp, daemon->log_display_id)) + const char *src = crecp != NULL ? crecp->flags & F_BIGNAME ? crecp->name.bname->name : crecp->name.sname : NULL; + if(FTL_CNAME(name, src, daemon->log_display_id)) { // Served from cache. This can happen if a domain hidden in the CNAME path // is only blocked for some but not all clients. In this case, the entire diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c index f9573bd3..66821f46 100644 --- a/src/dnsmasq_interface.c +++ b/src/dnsmasq_interface.c @@ -1474,13 +1474,10 @@ static bool _FTL_check_blocking(int queryID, int domainID, int clientID, const c } -bool _FTL_CNAME(const char *domain, const struct crec *cpp, const int id, const char* file, const int line) +bool _FTL_CNAME(const char *dst, const char *src, const int id, const char* file, const int line) { if(config.debug & DEBUG_QUERIES) - { - const char *src = cpp != NULL ? cpp->flags & F_BIGNAME ? cpp->name.bname->name : cpp->name.sname : NULL; - logg("FTL_CNAME called with: src = %s, dst = %s, id = %d", src, domain, id); - } + logg("FTL_CNAME called with: src = %s, dst = %s, id = %d", src, dst, id); // Does the user want to skip deep CNAME inspection? if(!config.cname_inspection) @@ -1493,10 +1490,6 @@ bool _FTL_CNAME(const char *domain, const struct crec *cpp, const int id, const // Lock shared memory lock_shm(); - // Get CNAME destination and source (if applicable) - const char *src = cpp != NULL ? cpp->flags & F_BIGNAME ? cpp->name.bname->name : cpp->name.sname : NULL; - const char *dst = domain; - // Save status and upstreamID in corresponding query identified by dnsmasq's ID const int queryID = findQueryID(id); if(queryID < 0) @@ -1534,7 +1527,7 @@ bool _FTL_CNAME(const char *domain, const struct crec *cpp, const int id, const // child_domain = Intermediate domain in CNAME path // This is the domain which was queried later in this chain - char *child_domain = strdup(domain); + char *child_domain = strdup(dst); // Convert to lowercase for matching strtolower(child_domain); const int child_domainID = findDomainID(child_domain, false); @@ -1603,12 +1596,7 @@ bool _FTL_CNAME(const char *domain, const struct crec *cpp, const int id, const // Debug logging for deep CNAME inspection (if enabled) if(config.debug & DEBUG_QUERIES) - { - if(src == NULL) - logg("Query %d: CNAME %s", id, dst); - else - logg("Query %d: CNAME %s ---> %s", id, src, dst); - } + logg("Query %d: CNAME %s ---> %s", id, src, dst); // Return result free(child_domain); @@ -2641,8 +2629,7 @@ static void _query_set_reply(const unsigned int flags, const enum reply_type rep if(config.debug & DEBUG_QUERIES) { const char *path = short_path(file); - logg("Set reply to %s (%d) in %s:%d", get_query_reply_str(query->reply), query->reply, - path, line); + logg("Set reply to %s (%d) in %s:%d", get_query_reply_str(new_reply), new_reply, path, line); if(query->reply != REPLY_UNKNOWN && query->reply != new_reply) logg("Reply of query %i was %s now changing to %s", query->id, get_query_reply_str(query->reply), get_query_reply_str(new_reply)); diff --git a/src/dnsmasq_interface.h b/src/dnsmasq_interface.h index 731a1188..fdd5ed69 100644 --- a/src/dnsmasq_interface.h +++ b/src/dnsmasq_interface.h @@ -34,8 +34,8 @@ void FTL_forwarding_retried(const struct server *server, const int oldID, const #define FTL_make_answer(header, limit, len, ede) _FTL_make_answer(header, limit, len, ede, __FILE__, __LINE__) size_t _FTL_make_answer(struct dns_header *header, char *limit, const size_t len, int *ede, const char* file, const int line); -#define FTL_CNAME(domain, cpp, id) _FTL_CNAME(domain, cpp, id, __FILE__, __LINE__) -bool _FTL_CNAME(const char *domain, const struct crec *cpp, const int id, const char* file, const int line); +#define FTL_CNAME(dst, src, id) _FTL_CNAME(dst, src, id, __FILE__, __LINE__) +bool _FTL_CNAME(const char *dst, const char *src, const int id, const char* file, const int line); unsigned int FTL_extract_question_flags(struct dns_header *header, const size_t qlen); void FTL_query_in_progress(const int id); From 84085bebcd604e70b4b49bca5787ac288b41e2b4 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 5 Sep 2022 19:27:18 +0200 Subject: [PATCH 051/218] Tests: CNAME inspection: NODATA CNAME targets are blocked Signed-off-by: DL6ER --- test/gravity.db.sql | 3 +- test/pdns/setup.sh | 5 +++ test/test_suite.bats | 102 +++++++++++++++++++++++++++---------------- 3 files changed, 71 insertions(+), 39 deletions(-) diff --git a/test/gravity.db.sql b/test/gravity.db.sql index 9a73d811..63ff28b3 100644 --- a/test/gravity.db.sql +++ b/test/gravity.db.sql @@ -212,8 +212,9 @@ INSERT INTO adlist VALUES(1,'https://hosts-file.net/ad_servers.txt',1,1559928803 INSERT INTO gravity VALUES('whitelisted.ftl',1); INSERT INTO gravity VALUES('gravity.ftl',1); +INSERT INTO gravity VALUES('gravity-aaaa.ftl',1); INSERT INTO gravity VALUES('gravity-whitelisted.ftl',1); -INSERT INTO info VALUES('gravity_count',3); +INSERT INTO info VALUES('gravity_count',4); INSERT INTO "group" VALUES(1,0,'Test group',1559928803,1559928803,'A disabled test group'); INSERT INTO domainlist_by_group VALUES(15,1); diff --git a/test/pdns/setup.sh b/test/pdns/setup.sh index a751f85e..ea53a51b 100644 --- a/test/pdns/setup.sh +++ b/test/pdns/setup.sh @@ -65,6 +65,7 @@ pdnsutil add-record ftl. regex-REPLYv4 AAAA fe80::2c01 pdnsutil add-record ftl. regex-REPLYv6 AAAA fe80::2c02 pdnsutil add-record ftl. regex-REPLYv46 AAAA fe80::2c03 pdnsutil add-record ftl. any AAAA fe80::3c01 +pdnsutil add-record ftl. gravity-aaaa AAAA fe80::4c01 # Create CNAME records pdnsutil add-record ftl. cname-1 CNAME gravity.ftl @@ -79,6 +80,10 @@ pdnsutil add-record ftl. cname-ok CNAME a.ftl # Create CNAME for SOA test domain pdnsutil add-record ftl. soa CNAME ftl +# Create CNAME for NODATA tests +pdnsutil add-record ftl. aaaa-cname CNAME gravity-aaaa.ftl +pdnsutil add-record ftl. a-cname CNAME gravity.ftl + # Create PTR records pdnsutil add-record ftl. ptr PTR ptr.ftl. diff --git a/test/test_suite.bats b/test/test_suite.bats index 1931e6b4..1678f164 100644 --- a/test/test_suite.bats +++ b/test/test_suite.bats @@ -384,6 +384,25 @@ [[ ${lines[1]} == "" ]] } +@test "CNAME inspection: NODATA CNAME targets are blocked" { + run bash -c "dig A a-cname.ftl @127.0.0.1 +short" + printf "%s\n" "${lines[@]}" + [[ ${lines[0]} == "0.0.0.0" ]] + [[ ${lines[1]} == "" ]] + run bash -c "dig AAAA a-cname.ftl @127.0.0.1 +short" + printf "%s\n" "${lines[@]}" + [[ ${lines[0]} == "::" ]] + [[ ${lines[1]} == "" ]] + run bash -c "dig A aaaa-cname.ftl @127.0.0.1 +short" + printf "%s\n" "${lines[@]}" + [[ ${lines[0]} == "0.0.0.0" ]] + [[ ${lines[1]} == "" ]] + run bash -c "dig AAAA aaaa-cname.ftl @127.0.0.1 +short" + printf "%s\n" "${lines[@]}" + [[ ${lines[0]} == "::" ]] + [[ ${lines[1]} == "" ]] +} + @test "DNSSEC: SECURE domain is resolved" { run bash -c "dig A sigok.verteiltesysteme.net @127.0.0.1" printf "%s\n" "${lines[@]}" @@ -399,11 +418,11 @@ @test "Statistics as expected" { run bash -c 'echo ">stats >quit" | nc -v 127.0.0.1 4711' printf "%s\n" "${lines[@]}" - [[ ${lines[1]} == "domains_being_blocked 3" ]] - [[ ${lines[2]} == "dns_queries_today 47" ]] - [[ ${lines[3]} == "ads_blocked_today 9" ]] + [[ ${lines[1]} == "domains_being_blocked 4" ]] + [[ ${lines[2]} == "dns_queries_today 51" ]] + [[ ${lines[3]} == "ads_blocked_today 13" ]] #[[ ${lines[4]} == "ads_percentage_today 7.792208" ]] - [[ ${lines[5]} == "unique_domains 35" ]] + [[ ${lines[5]} == "unique_domains 38" ]] [[ ${lines[6]} == "queries_forwarded 26" ]] [[ ${lines[7]} == "queries_cached 12" ]] # Clients ever seen is commented out as the CI may have @@ -411,11 +430,11 @@ # number of clients may not work in all cases #[[ ${lines[8]} == "clients_ever_seen 8" ]] #[[ ${lines[9]} == "unique_clients 8" ]] - [[ ${lines[10]} == "dns_queries_all_types 47" ]] + [[ ${lines[10]} == "dns_queries_all_types 51" ]] [[ ${lines[11]} == "reply_UNKNOWN 0" ]] [[ ${lines[12]} == "reply_NODATA 0" ]] [[ ${lines[13]} == "reply_NXDOMAIN 1" ]] - [[ ${lines[14]} == "reply_CNAME 3" ]] + [[ ${lines[14]} == "reply_CNAME 7" ]] [[ ${lines[15]} == "reply_IP 23" ]] [[ ${lines[16]} == "reply_DOMAIN 0" ]] [[ ${lines[17]} == "reply_RRNAME 5" ]] @@ -426,7 +445,7 @@ [[ ${lines[22]} == "reply_DNSSEC 5" ]] [[ ${lines[23]} == "reply_NONE 0" ]] [[ ${lines[24]} == "reply_BLOB 10" ]] - [[ ${lines[25]} == "dns_queries_all_replies 47" ]] + [[ ${lines[25]} == "dns_queries_all_replies 51" ]] [[ ${lines[26]} == "privacy_level 0" ]] [[ ${lines[27]} == "status enabled" ]] [[ ${lines[28]} == "" ]] @@ -441,7 +460,7 @@ @test "Top Clients" { run bash -c 'echo ">top-clients >quit" | nc -v 127.0.0.1 4711' printf "%s\n" "${lines[@]}" - [[ ${lines[1]} == "0 32 127.0.0.1 "* ]] + [[ ${lines[1]} == "0 36 127.0.0.1 "* ]] [[ ${lines[2]} == "1 5 :: "* ]] [[ ${lines[3]} == "2 4 127.0.0.3 "* ]] [[ ${lines[4]} == "3 3 127.0.0.2 "* ]] @@ -484,7 +503,10 @@ @test "Top Ads" { run bash -c 'echo ">top-ads (20) >quit" | nc -v 127.0.0.1 4711' printf "%s\n" "${lines[@]}" - [[ "${lines[@]}" == *" 4 gravity.ftl"* ]] + [[ "${lines[@]}" == *" 6 gravity.ftl"* ]] + [[ "${lines[@]}" == *" 2 a-cname.ftl"* ]] + [[ "${lines[@]}" == *" 2 aaaa-cname.ftl"* ]] + [[ "${lines[@]}" == *" 2 gravity-aaaa.ftl"* ]] [[ "${lines[@]}" == *" 1 blacklisted.ftl"* ]] [[ "${lines[@]}" == *" 1 whitelisted.ftl"* ]] [[ "${lines[@]}" == *" 1 regex5.ftl"* ]] @@ -492,7 +514,7 @@ [[ "${lines[@]}" == *" 1 cname-1.ftl"* ]] [[ "${lines[@]}" == *" 1 cname-7.ftl"* ]] [[ "${lines[@]}" == *" 1 use-application-dns.net"* ]] - [[ ${lines[9]} == "" ]] + [[ ${lines[12]} == "" ]] } @test "Domain auditing, approved domains are not shown" { @@ -504,33 +526,33 @@ @test "Upstream Destinations reported correctly" { run bash -c 'echo ">forward-dest >quit" | nc -v 127.0.0.1 4711' printf "%s\n" "${lines[@]}" - [[ ${lines[1]} == "-3 19.15 blocked blocked" ]] - [[ ${lines[2]} == "-2 25.53 cached cached" ]] + [[ ${lines[1]} == "-3 25.49 blocked blocked" ]] + [[ ${lines[2]} == "-2 23.53 cached cached" ]] [[ ${lines[3]} == "-1 0.00 other other" ]] - [[ ${lines[4]} == "0 51.06 127.0.0.1#5555 127.0.0.1#5555" ]] - [[ ${lines[5]} == "1 4.26 127.0.0.1#5554 127.0.0.1#5554" ]] + [[ ${lines[4]} == "0 47.06 127.0.0.1#5555 127.0.0.1#5555" ]] + [[ ${lines[5]} == "1 3.92 127.0.0.1#5554 127.0.0.1#5554" ]] [[ ${lines[6]} == "" ]] } @test "Query Types reported correctly" { run bash -c 'echo ">querytypes >quit" | nc -v 127.0.0.1 4711' printf "%s\n" "${lines[@]}" - [[ ${lines[1]} == "A (IPv4): 51.06" ]] - [[ ${lines[2]} == "AAAA (IPv6): 4.26" ]] - [[ ${lines[3]} == "ANY: 2.13" ]] - [[ ${lines[4]} == "SRV: 2.13" ]] - [[ ${lines[5]} == "SOA: 2.13" ]] - [[ ${lines[6]} == "PTR: 2.13" ]] - [[ ${lines[7]} == "TXT: 12.77" ]] - [[ ${lines[8]} == "NAPTR: 2.13" ]] - [[ ${lines[9]} == "MX: 2.13" ]] - [[ ${lines[10]} == "DS: 4.26" ]] + [[ ${lines[1]} == "A (IPv4): 50.98" ]] + [[ ${lines[2]} == "AAAA (IPv6): 7.84" ]] + [[ ${lines[3]} == "ANY: 1.96" ]] + [[ ${lines[4]} == "SRV: 1.96" ]] + [[ ${lines[5]} == "SOA: 1.96" ]] + [[ ${lines[6]} == "PTR: 1.96" ]] + [[ ${lines[7]} == "TXT: 11.76" ]] + [[ ${lines[8]} == "NAPTR: 1.96" ]] + [[ ${lines[9]} == "MX: 1.96" ]] + [[ ${lines[10]} == "DS: 3.92" ]] [[ ${lines[11]} == "RRSIG: 0.00" ]] - [[ ${lines[12]} == "DNSKEY: 6.38" ]] - [[ ${lines[13]} == "NS: 2.13" ]] - [[ ${lines[14]} == "OTHER: 2.13" ]] - [[ ${lines[15]} == "SVCB: 2.13" ]] - [[ ${lines[16]} == "HTTPS: 2.13" ]] + [[ ${lines[12]} == "DNSKEY: 5.88" ]] + [[ ${lines[13]} == "NS: 1.96" ]] + [[ ${lines[14]} == "OTHER: 1.96" ]] + [[ ${lines[15]} == "SVCB: 1.96" ]] + [[ ${lines[16]} == "HTTPS: 1.96" ]] [[ ${lines[17]} == "" ]] } @@ -579,14 +601,18 @@ [[ ${lines[38]} == *" HTTPS https.ftl 127.0.0.1 2 2 13 "*" N/A -1 127.0.0.1#5554 \"\" \"37\""* ]] [[ ${lines[39]} == *" A cname-1.ftl 127.0.0.1 9 2 3 "*" gravity.ftl -1 127.0.0.1#5555 \"\" \"38\""* ]] [[ ${lines[40]} == *" A cname-7.ftl 127.0.0.1 9 2 3 "*" gravity.ftl -1 127.0.0.1#5555 \"\" \"39\""* ]] - [[ ${lines[41]} == *" A sigok.verteiltesysteme.net 127.0.0.1 2 1 4 "*" N/A -1 127.0.0.1#5555 \"\" \"40\""* ]] - [[ ${lines[42]} == *" DS net :: 2 1 11 "*" N/A -1 127.0.0.1#5555 \"\" \"41\""* ]] - [[ ${lines[43]} == *" DNSKEY . :: 2 1 11 "*" N/A -1 127.0.0.1#5555 \"\" \"42\""* ]] - [[ ${lines[44]} == *" DS verteiltesysteme.net :: 2 1 11 "*" N/A -1 127.0.0.1#5555 \"\" \"43\""* ]] - [[ ${lines[45]} == *" DNSKEY net :: 2 1 11 "*" N/A -1 127.0.0.1#5555 \"\" \"44\""* ]] - [[ ${lines[46]} == *" DNSKEY verteiltesysteme.net :: 2 1 11 "*" N/A -1 127.0.0.1#5555 \"\" \"45\""* ]] - [[ ${lines[47]} == *" A sigfail.verteiltesysteme.net 127.0.0.1 2 3 4 "*" N/A -1 127.0.0.1#5555 \"DNSKEY missing\" \"46\""* ]] - [[ ${lines[48]} == "" ]] + [[ ${lines[41]} == *" A a-cname.ftl 127.0.0.1 9 2 3 "*" gravity.ftl -1 127.0.0.1#5555 \"\" \"40\""* ]] + [[ ${lines[42]} == *" AAAA a-cname.ftl 127.0.0.1 9 2 3 "*" gravity.ftl -1 127.0.0.1#5555 \"\" \"41\""* ]] + [[ ${lines[43]} == *" A aaaa-cname.ftl 127.0.0.1 9 2 3 "*" gravity-aaaa.ftl -1 127.0.0.1#5555 \"\" \"42\""* ]] + [[ ${lines[44]} == *" AAAA aaaa-cname.ftl 127.0.0.1 9 2 3 "*" gravity-aaaa.ftl -1 127.0.0.1#5555 \"\" \"43\""* ]] + [[ ${lines[45]} == *" A sigok.verteiltesysteme.net 127.0.0.1 2 1 4 "*" N/A -1 127.0.0.1#5555 \"\" \"44\""* ]] + [[ ${lines[46]} == *" DS net :: 2 1 11 "*" N/A -1 127.0.0.1#5555 \"\" \"45\""* ]] + [[ ${lines[47]} == *" DNSKEY . :: 2 1 11 "*" N/A -1 127.0.0.1#5555 \"\" \"46\""* ]] + [[ ${lines[48]} == *" DS verteiltesysteme.net :: 2 1 11 "*" N/A -1 127.0.0.1#5555 \"\" \"47\""* ]] + [[ ${lines[49]} == *" DNSKEY net :: 2 1 11 "*" N/A -1 127.0.0.1#5555 \"\" \"48\""* ]] + [[ ${lines[50]} == *" DNSKEY verteiltesysteme.net :: 2 1 11 "*" N/A -1 127.0.0.1#5555 \"\" \"49\""* ]] + [[ ${lines[51]} == *" A sigfail.verteiltesysteme.net 127.0.0.1 2 3 4 "*" N/A -1 127.0.0.1#5555 \"DNSKEY missing\" \"50\""* ]] + [[ ${lines[52]} == "" ]] } @test "Get all queries (domain filtered) shows expected content" { @@ -599,7 +625,7 @@ @test "Recent blocked shows expected content" { run bash -c 'echo ">recentBlocked >quit" | nc -v 127.0.0.1 4711' printf "%s\n" "${lines[@]}" - [[ ${lines[1]} == "cname-7.ftl" ]] + [[ ${lines[1]} == "aaaa-cname.ftl" ]] [[ ${lines[2]} == "" ]] } From 2a4d10e9f7e5d89a4c204c31c1458a4c666878d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Tue, 6 Sep 2022 19:54:07 +0200 Subject: [PATCH 052/218] Apply reviewer's suggestions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- src/database/gravity-db.c | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/src/database/gravity-db.c b/src/database/gravity-db.c index 396363c0..7ff8c6ac 100644 --- a/src/database/gravity-db.c +++ b/src/database/gravity-db.c @@ -1409,7 +1409,8 @@ bool gravityDB_get_regex_client_groups(clientsData* client, const unsigned int n return true; } -void check_inaccessible_adlists(void){ +void check_inaccessible_adlists(void) +{ // check if any adlist was inaccessible in the last gravity run // if so, gravity stored `status` in the adlist table with @@ -1434,16 +1435,16 @@ void check_inaccessible_adlists(void){ return; } - // Perform query - while((rc = sqlite3_step(query_stmt)) == SQLITE_ROW) - { - int id = sqlite3_column_int(query_stmt, 0); - const char *address = strdup((const char*)sqlite3_column_text(query_stmt, 1)); + // Perform query + while((rc = sqlite3_step(query_stmt)) == SQLITE_ROW) + { + int id = sqlite3_column_int(query_stmt, 0); + const char *address = (const char*)sqlite3_column_text(query_stmt, 1); - // log to the message table - logg_inaccessible_adlist(id, address); - } + // log to the message table + logg_inaccessible_adlist(id, address); + } - // Finalize statement - sqlite3_finalize(query_stmt); + // Finalize statement + sqlite3_finalize(query_stmt); } From 97cb69cabbdcfe28a80982dd8626b09ae87c3896 Mon Sep 17 00:00:00 2001 From: Simon Kelley Date: Fri, 18 Feb 2022 20:53:56 +0000 Subject: [PATCH 053/218] Fix missing reverse-records from --dynamic-host. Thanks to Sten Spans for spotting the bug. Signed-off-by: DL6ER --- src/dnsmasq/rfc1035.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/dnsmasq/rfc1035.c b/src/dnsmasq/rfc1035.c index 6d7490d3..bf7838a2 100644 --- a/src/dnsmasq/rfc1035.c +++ b/src/dnsmasq/rfc1035.c @@ -1609,7 +1609,7 @@ size_t answer_request(struct dns_header *header, char *limit, size_t qlen, if (addrlist) break; - else + else if (!(intr->flags & INP4)) while (intr->next && strcmp(intr->intr, intr->next->intr) == 0) intr = intr->next; } @@ -1624,7 +1624,7 @@ size_t answer_request(struct dns_header *header, char *limit, size_t qlen, if (addrlist) break; - else + else if (!(intr->flags & INP6)) while (intr->next && strcmp(intr->intr, intr->next->intr) == 0) intr = intr->next; } From 666fca0c77d34dc0ffadf240f868ec69f4e5e19a Mon Sep 17 00:00:00 2001 From: Simon Kelley Date: Tue, 22 Feb 2022 21:56:48 +0000 Subject: [PATCH 054/218] Fix longjump() compiler warnings. Signed-off-by: DL6ER --- src/dnsmasq/option.c | 36 +++++++++++++++--------------------- 1 file changed, 15 insertions(+), 21 deletions(-) diff --git a/src/dnsmasq/option.c b/src/dnsmasq/option.c index eb181ff3..bc08aadc 100644 --- a/src/dnsmasq/option.c +++ b/src/dnsmasq/option.c @@ -4978,26 +4978,20 @@ err: return 1; } -static void read_file(char *file, FILE *f, int hard_opt) +static void read_file(char *file, FILE *f, int hard_opt, int from_script) { volatile int lineno = 0; char *buff = daemon->namebuff; while (fgets(buff, MAXDNAME, f)) { - int white, i, script = 0; + int white, i; volatile int option; char *errmess, *p, *arg, *start; size_t len; - if (hard_opt == LOPT_CONF_SCRIPT) - { - hard_opt = 0; - script = 1; - } - option = (hard_opt == LOPT_REV_SERV) ? 0 : hard_opt; - + /* Memory allocation failure longjmps here if mem_recover == 1 */ if (option != 0 || hard_opt == LOPT_REV_SERV) { @@ -5005,7 +4999,7 @@ static void read_file(char *file, FILE *f, int hard_opt) continue; mem_recover = 1; } - + arg = NULL; lineno++; errmess = NULL; @@ -5111,7 +5105,7 @@ static void read_file(char *file, FILE *f, int hard_opt) if (errmess || !one_opt(option, arg, daemon->namebuff, _("error"), 0, hard_opt == LOPT_REV_SERV)) { - if (script) + if (from_script) sprintf(daemon->namebuff + strlen(daemon->namebuff), _(" in output from %s"), file); else sprintf(daemon->namebuff + strlen(daemon->namebuff), _(" at line %d of %s"), lineno, file); @@ -5157,8 +5151,14 @@ static int one_file(char *file, int hard_opt) hard_opt = 0; nofile_ok = 1; } - - if (hard_opt == 0 && strcmp(file, "-") == 0) + + if (hard_opt == LOPT_CONF_SCRIPT) + { + hard_opt = 0; + do_popen = 1; + } + + if (hard_opt == 0 && !do_popen && strcmp(file, "-") == 0) { if (read_stdin == 1) return 1; @@ -5171,12 +5171,6 @@ static int one_file(char *file, int hard_opt) /* ignore repeated files. */ struct stat statbuf; - if (hard_opt == LOPT_CONF_SCRIPT) - { - hard_opt = 0; - do_popen = 1; - } - if (hard_opt == 0 && stat(file, &statbuf) == 0) { struct fileread *r; @@ -5215,7 +5209,7 @@ static int one_file(char *file, int hard_opt) } } - read_file(file, f, do_popen ? LOPT_CONF_SCRIPT : hard_opt); + read_file(file, f, hard_opt, do_popen); if (do_popen) { @@ -5369,7 +5363,7 @@ void read_servers_file(void) } mark_servers(SERV_FROM_FILE); - read_file(daemon->servers_file, f, LOPT_REV_SERV); + read_file(daemon->servers_file, f, LOPT_REV_SERV, 0); fclose(f); cleanup_servers(); check_servers(0); From 5b67f1b14da77d09f21f9e7e7ea1b3984db7dd38 Mon Sep 17 00:00:00 2001 From: Simon Kelley Date: Thu, 24 Feb 2022 23:18:54 +0000 Subject: [PATCH 055/218] Fix memory leak when DBUS connection fails. Signed-off-by: DL6ER --- src/dnsmasq/dbus.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/dnsmasq/dbus.c b/src/dnsmasq/dbus.c index 0c55ea5e..bf6b6613 100644 --- a/src/dnsmasq/dbus.c +++ b/src/dnsmasq/dbus.c @@ -761,8 +761,11 @@ char *dbus_init(void) dbus_error_init (&dbus_error); if (!(connection = dbus_bus_get (DBUS_BUS_SYSTEM, &dbus_error))) - return NULL; - + { + dbus_error_free(&dbus_error); + return NULL; + } + dbus_connection_set_exit_on_disconnect(connection, FALSE); dbus_connection_set_watch_functions(connection, add_watch, remove_watch, NULL, NULL, NULL); From 285b6f31e88c086495bbcbc47925c36a71b2e195 Mon Sep 17 00:00:00 2001 From: Simon Kelley Date: Sat, 5 Mar 2022 18:07:07 +0000 Subject: [PATCH 056/218] Enhance --domain to accept, interface names for the address range. This allows hosts get a domain which relects the interface they are attached to in a way which doesn't require hard-coding addresses. Thanks to Sten Spans for the idea. Signed-off-by: DL6ER --- src/dnsmasq/dnsmasq.h | 2 ++ src/dnsmasq/domain.c | 31 +++++++++++++++++++++------- src/dnsmasq/network.c | 47 ++++++++++++++++++++++++++++++++++++++++++- src/dnsmasq/option.c | 10 +++++++-- 4 files changed, 80 insertions(+), 10 deletions(-) diff --git a/src/dnsmasq/dnsmasq.h b/src/dnsmasq/dnsmasq.h index ba7f6360..08d8b7ac 100644 --- a/src/dnsmasq/dnsmasq.h +++ b/src/dnsmasq/dnsmasq.h @@ -987,6 +987,8 @@ struct dhcp_bridge { struct cond_domain { char *domain, *prefix; /* prefix is text-prefix on domain name */ + char *interface; /* These two set when domain comes from interface. */ + struct addrlist *al; struct in_addr start, end; struct in6_addr start6, end6; int is6, indexed, prefixlen; diff --git a/src/dnsmasq/domain.c b/src/dnsmasq/domain.c index 71664334..a893ce5a 100644 --- a/src/dnsmasq/domain.c +++ b/src/dnsmasq/domain.c @@ -230,9 +230,17 @@ int is_rev_synth(int flag, union all_addr *addr, char *name) static int match_domain(struct in_addr addr, struct cond_domain *c) { - if (!c->is6 && - ntohl(addr.s_addr) >= ntohl(c->start.s_addr) && - ntohl(addr.s_addr) <= ntohl(c->end.s_addr)) + if (c->interface) + { + struct addrlist *al; + for (al = c->al; al; al = al->next) + if (!(al->flags & ADDRLIST_IPV6) && + is_same_net_prefix(addr, al->addr.addr4, al->prefixlen)) + return 1; + } + else if (!c->is6 && + ntohl(addr.s_addr) >= ntohl(c->start.s_addr) && + ntohl(addr.s_addr) <= ntohl(c->end.s_addr)) return 1; return 0; @@ -259,12 +267,21 @@ char *get_domain(struct in_addr addr) static int match_domain6(struct in6_addr *addr, struct cond_domain *c) { - u64 addrpart = addr6part(addr); - - if (c->is6) + + /* subnet from interface address. */ + if (c->interface) + { + struct addrlist *al; + for (al = c->al; al; al = al->next) + if (al->flags & ADDRLIST_IPV6 && + is_same_net6(addr, &al->addr.addr6, al->prefixlen)) + return 1; + } + else if (c->is6) { if (c->prefixlen >= 64) { + u64 addrpart = addr6part(addr); if (is_same_net6(addr, &c->start6, 64) && addrpart >= addr6part(&c->start6) && addrpart <= addr6part(&c->end6)) @@ -273,7 +290,7 @@ static int match_domain6(struct in6_addr *addr, struct cond_domain *c) else if (is_same_net6(addr, &c->start6, c->prefixlen)) return 1; } - + return 0; } diff --git a/src/dnsmasq/network.c b/src/dnsmasq/network.c index 5cae606c..239c3569 100644 --- a/src/dnsmasq/network.c +++ b/src/dnsmasq/network.c @@ -233,6 +233,7 @@ static int iface_allowed(struct iface_param *param, int if_index, char *label, union mysockaddr *addr, struct in_addr netmask, int prefixlen, int iface_flags) { struct irec *iface; + struct cond_domain *cond; int loopback; struct ifreq ifr; int tftp_ok = !!option_bool(OPT_TFTP); @@ -455,7 +456,37 @@ static int iface_allowed(struct iface_param *param, int if_index, char *label, } } } - + + /* Update addresses for domain=, */ + for (cond = daemon->cond_domain; cond; cond = cond->next) + if (cond->interface && strncmp(label, cond->interface, IF_NAMESIZE) == 0) + { + struct addrlist *al; + + if (param->spare) + { + al = param->spare; + param->spare = al->next; + } + else + al = whine_malloc(sizeof(struct addrlist)); + + if (addr->sa.sa_family == AF_INET) + { + al->addr.addr4 = addr->in.sin_addr; + al->flags = 0; + } + else + { + al->addr.addr6 = addr->in6.sin6_addr; + al->flags = ADDRLIST_IPV6; + } + + al->prefixlen = prefixlen; + al->next = cond->al; + cond->al = al; + } + /* check whether the interface IP has been added already we call this routine multiple times. */ for (iface = daemon->interfaces; iface; iface = iface->next) @@ -700,6 +731,7 @@ int enumerate_interfaces(int reset) int errsave, ret = 1; struct addrlist *addr, *tmp; struct interface_name *intname; + struct cond_domain *cond; struct irec *iface; #ifdef HAVE_AUTH struct auth_zone *zone; @@ -759,6 +791,19 @@ again: intname->addr = NULL; } + /* remove addresses stored against cond-domains. */ + for (cond = daemon->cond_domain; cond; cond = cond->next) + { + for (addr = cond->al; addr; addr = tmp) + { + tmp = addr->next; + addr->next = spare; + spare = addr; + } + + cond->al = NULL; + } + /* Remove list of addresses of local interfaces */ for (addr = daemon->interface_addrs; addr; addr = tmp) { diff --git a/src/dnsmasq/option.c b/src/dnsmasq/option.c index bc08aadc..db2e5b09 100644 --- a/src/dnsmasq/option.c +++ b/src/dnsmasq/option.c @@ -2496,9 +2496,15 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma else if (!inet_pton(AF_INET6, arg, &new->end6)) ret_err_free(gen_err, new); } - else + else if (option == 's') + { + /* subnet from interface. */ + new->interface = opt_string_alloc(comma); + new->al = NULL; + } + else ret_err_free(gen_err, new); - + if (option != 's' && prefstr) { if (!(new->prefix = canonicalise_opt(prefstr)) || From cd038bbb1318c122d340c4579e80dabd9d7f305d Mon Sep 17 00:00:00 2001 From: Simon Kelley Date: Tue, 22 Mar 2022 13:47:05 +0000 Subject: [PATCH 057/218] Add DNSMASQ_DATA_MISSING envvar to lease-change script. Signed-off-by: DL6ER --- src/dnsmasq/helper.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/dnsmasq/helper.c b/src/dnsmasq/helper.c index a6a9b28d..33e49250 100644 --- a/src/dnsmasq/helper.c +++ b/src/dnsmasq/helper.c @@ -430,6 +430,9 @@ int create_helper(int event_fd, int err_fd, uid_t uid, gid_t gid, long max_fd) end = extradata + data.ed_len; buf = extradata; + + lua_pushnumber(lua, data.ed_len == 0 ? 1 : 0); + lua_setfield(lua, -2, "data_missing"); if (!is6) buf = grab_extradata_lua(buf, end, "vendor_class"); @@ -608,6 +611,9 @@ int create_helper(int event_fd, int err_fd, uid_t uid, gid_t gid, long max_fd) end = extradata + data.ed_len; buf = extradata; + + if (data.ed_len == 0) + my_setenv("DNSMASQ_DATA_MISSING", "1", &err); if (!is6) buf = grab_extradata(buf, end, "DNSMASQ_VENDOR_CLASS", &err); From 73faa6cec655bb066333743262f59d70d229c1ee Mon Sep 17 00:00:00 2001 From: Simon Kelley Date: Thu, 31 Mar 2022 21:35:20 +0100 Subject: [PATCH 058/218] Fix write-after-free error in DHCPv6 code. CVE-2022-0934 refers. Signed-off-by: DL6ER --- src/dnsmasq/rfc3315.c | 48 ++++++++++++++++++++++++------------------- 1 file changed, 27 insertions(+), 21 deletions(-) diff --git a/src/dnsmasq/rfc3315.c b/src/dnsmasq/rfc3315.c index cee8382c..e218d263 100644 --- a/src/dnsmasq/rfc3315.c +++ b/src/dnsmasq/rfc3315.c @@ -33,9 +33,9 @@ struct state { unsigned int mac_len, mac_type; }; -static int dhcp6_maybe_relay(struct state *state, void *inbuff, size_t sz, +static int dhcp6_maybe_relay(struct state *state, unsigned char *inbuff, size_t sz, struct in6_addr *client_addr, int is_unicast, time_t now); -static int dhcp6_no_relay(struct state *state, int msg_type, void *inbuff, size_t sz, int is_unicast, time_t now); +static int dhcp6_no_relay(struct state *state, int msg_type, unsigned char *inbuff, size_t sz, int is_unicast, time_t now); static void log6_opts(int nest, unsigned int xid, void *start_opts, void *end_opts); static void log6_packet(struct state *state, char *type, struct in6_addr *addr, char *string); static void log6_quiet(struct state *state, char *type, struct in6_addr *addr, char *string); @@ -104,12 +104,12 @@ unsigned short dhcp6_reply(struct dhcp_context *context, int interface, char *if } /* This cost me blood to write, it will probably cost you blood to understand - srk. */ -static int dhcp6_maybe_relay(struct state *state, void *inbuff, size_t sz, +static int dhcp6_maybe_relay(struct state *state, unsigned char *inbuff, size_t sz, struct in6_addr *client_addr, int is_unicast, time_t now) { void *end = inbuff + sz; void *opts = inbuff + 34; - int msg_type = *((unsigned char *)inbuff); + int msg_type = *inbuff; unsigned char *outmsgtypep; void *opt; struct dhcp_vendor *vendor; @@ -259,15 +259,15 @@ static int dhcp6_maybe_relay(struct state *state, void *inbuff, size_t sz, return 1; } -static int dhcp6_no_relay(struct state *state, int msg_type, void *inbuff, size_t sz, int is_unicast, time_t now) +static int dhcp6_no_relay(struct state *state, int msg_type, unsigned char *inbuff, size_t sz, int is_unicast, time_t now) { void *opt; - int i, o, o1, start_opts; + int i, o, o1, start_opts, start_msg; struct dhcp_opt *opt_cfg; struct dhcp_netid *tagif; struct dhcp_config *config = NULL; struct dhcp_netid known_id, iface_id, v6_id; - unsigned char *outmsgtypep; + unsigned char outmsgtype; struct dhcp_vendor *vendor; struct dhcp_context *context_tmp; struct dhcp_mac *mac_opt; @@ -296,12 +296,13 @@ static int dhcp6_no_relay(struct state *state, int msg_type, void *inbuff, size_ v6_id.next = state->tags; state->tags = &v6_id; - /* copy over transaction-id, and save pointer to message type */ - if (!(outmsgtypep = put_opt6(inbuff, 4))) + start_msg = save_counter(-1); + /* copy over transaction-id */ + if (!put_opt6(inbuff, 4)) return 0; start_opts = save_counter(-1); - state->xid = outmsgtypep[3] | outmsgtypep[2] << 8 | outmsgtypep[1] << 16; - + state->xid = inbuff[3] | inbuff[2] << 8 | inbuff[1] << 16; + /* We're going to be linking tags from all context we use. mark them as unused so we don't link one twice and break the list */ for (context_tmp = state->context; context_tmp; context_tmp = context_tmp->current) @@ -347,7 +348,7 @@ static int dhcp6_no_relay(struct state *state, int msg_type, void *inbuff, size_ (msg_type == DHCP6REQUEST || msg_type == DHCP6RENEW || msg_type == DHCP6RELEASE || msg_type == DHCP6DECLINE)) { - *outmsgtypep = DHCP6REPLY; + outmsgtype = DHCP6REPLY; o1 = new_opt6(OPTION6_STATUS_CODE); put_opt6_short(DHCP6USEMULTI); put_opt6_string("Use multicast"); @@ -619,11 +620,11 @@ static int dhcp6_no_relay(struct state *state, int msg_type, void *inbuff, size_ struct dhcp_netid *solicit_tags; struct dhcp_context *c; - *outmsgtypep = DHCP6ADVERTISE; + outmsgtype = DHCP6ADVERTISE; if (opt6_find(state->packet_options, state->end, OPTION6_RAPID_COMMIT, 0)) { - *outmsgtypep = DHCP6REPLY; + outmsgtype = DHCP6REPLY; state->lease_allocate = 1; o = new_opt6(OPTION6_RAPID_COMMIT); end_opt6(o); @@ -809,7 +810,7 @@ static int dhcp6_no_relay(struct state *state, int msg_type, void *inbuff, size_ int start = save_counter(-1); /* set reply message type */ - *outmsgtypep = DHCP6REPLY; + outmsgtype = DHCP6REPLY; state->lease_allocate = 1; log6_quiet(state, "DHCPREQUEST", NULL, ignore ? _("ignored") : NULL); @@ -924,7 +925,7 @@ static int dhcp6_no_relay(struct state *state, int msg_type, void *inbuff, size_ int address_assigned = 0; /* set reply message type */ - *outmsgtypep = DHCP6REPLY; + outmsgtype = DHCP6REPLY; log6_quiet(state, msg_type == DHCP6RENEW ? "DHCPRENEW" : "DHCPREBIND", NULL, NULL); @@ -1057,7 +1058,7 @@ static int dhcp6_no_relay(struct state *state, int msg_type, void *inbuff, size_ int good_addr = 0; /* set reply message type */ - *outmsgtypep = DHCP6REPLY; + outmsgtype = DHCP6REPLY; log6_quiet(state, "DHCPCONFIRM", NULL, NULL); @@ -1121,7 +1122,7 @@ static int dhcp6_no_relay(struct state *state, int msg_type, void *inbuff, size_ log6_quiet(state, "DHCPINFORMATION-REQUEST", NULL, ignore ? _("ignored") : state->hostname); if (ignore) return 0; - *outmsgtypep = DHCP6REPLY; + outmsgtype = DHCP6REPLY; tagif = add_options(state, 1); break; } @@ -1130,7 +1131,7 @@ static int dhcp6_no_relay(struct state *state, int msg_type, void *inbuff, size_ case DHCP6RELEASE: { /* set reply message type */ - *outmsgtypep = DHCP6REPLY; + outmsgtype = DHCP6REPLY; log6_quiet(state, "DHCPRELEASE", NULL, NULL); @@ -1195,7 +1196,7 @@ static int dhcp6_no_relay(struct state *state, int msg_type, void *inbuff, size_ case DHCP6DECLINE: { /* set reply message type */ - *outmsgtypep = DHCP6REPLY; + outmsgtype = DHCP6REPLY; log6_quiet(state, "DHCPDECLINE", NULL, NULL); @@ -1275,7 +1276,12 @@ static int dhcp6_no_relay(struct state *state, int msg_type, void *inbuff, size_ } } - + + /* Fill in the message type. Note that we store the offset, + not a direct pointer, since the packet memory may have been + reallocated. */ + ((unsigned char *)(daemon->outpacket.iov_base))[start_msg] = outmsgtype; + log_tags(tagif, state->xid); log6_opts(0, state->xid, daemon->outpacket.iov_base + start_opts, daemon->outpacket.iov_base + save_counter(-1)); From 20ef05beb89f603dab2db99a11081d6e52a18b81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=A2=81=E5=BB=BA=E9=B9=8F?= Date: Mon, 18 Apr 2022 15:25:54 +0100 Subject: [PATCH 059/218] Add inode compare while checking resolv file change Fix a bug found on OpenWrt when IPv4/6 dual stack enabled: The resolv file is located on tmpfs whose mtime resolution is 1 second. If the resolv file is updated twice within one second dnsmasq may can't notice the second update. netifd updates the resolv file with method: write temp then move, so adding an inode check fixes this bug. Signed-off-by: DL6ER --- src/dnsmasq/dnsmasq.c | 3 ++- src/dnsmasq/dnsmasq.h | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/dnsmasq/dnsmasq.c b/src/dnsmasq/dnsmasq.c index add4dd97..02d0a4a6 100644 --- a/src/dnsmasq/dnsmasq.c +++ b/src/dnsmasq/dnsmasq.c @@ -1689,9 +1689,10 @@ static void poll_resolv(int force, int do_reload, time_t now) else { res->logged = 0; - if (force || (statbuf.st_mtime != res->mtime)) + if (force || (statbuf.st_mtime != res->mtime || statbuf.st_ino != res->ino)) { res->mtime = statbuf.st_mtime; + res->ino = statbuf.st_ino; if (difftime(statbuf.st_mtime, last_change) > 0.0) { last_change = statbuf.st_mtime; diff --git a/src/dnsmasq/dnsmasq.h b/src/dnsmasq/dnsmasq.h index 08d8b7ac..f1e12736 100644 --- a/src/dnsmasq/dnsmasq.h +++ b/src/dnsmasq/dnsmasq.h @@ -677,6 +677,7 @@ struct resolvc { struct resolvc *next; int is_default, logged; time_t mtime; + ino_t ino; char *name; #ifdef HAVE_INOTIFY int wd; /* inotify watch descriptor */ From 52540efcd9dc8a8a4e68aba5dd9b617d8dfd6478 Mon Sep 17 00:00:00 2001 From: Dominik Derigs Date: Sat, 2 Apr 2022 21:45:47 +0200 Subject: [PATCH 060/218] Also log upstream port for dnssec-retry Signed-off-by: DL6ER --- src/dnsmasq/forward.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/dnsmasq/forward.c b/src/dnsmasq/forward.c index 4c061227..0e05f6c1 100644 --- a/src/dnsmasq/forward.c +++ b/src/dnsmasq/forward.c @@ -537,8 +537,8 @@ static int forward_query(int udpfd, union mysockaddr *udpaddr, } #ifdef HAVE_DNSSEC else - log_query_mysockaddr(F_NOEXTRA | F_DNSSEC, daemon->namebuff, &srv->addr, - "dnssec-retry", (forward->flags & FREC_DNSKEY_QUERY) ? T_DNSKEY : T_DS); + log_query_mysockaddr(F_NOEXTRA | F_DNSSEC | F_SERVER, daemon->namebuff, &srv->addr, + (forward->flags & FREC_DNSKEY_QUERY) ? "dnssec-retry[DNSKEY]" : "dnssec-retry[DS]", 0); #endif srv->queries++; From 0619acad13adcf24b252d80da3e82aa747f4add7 Mon Sep 17 00:00:00 2001 From: Simon Kelley Date: Thu, 26 May 2022 14:49:10 +0100 Subject: [PATCH 061/218] Fix outdated comment. Signed-off-by: DL6ER --- src/dnsmasq/dnssec.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dnsmasq/dnssec.c b/src/dnsmasq/dnssec.c index 9965eea3..daf679e0 100644 --- a/src/dnsmasq/dnssec.c +++ b/src/dnsmasq/dnssec.c @@ -1851,7 +1851,7 @@ static int zone_status(char *name, int class, char *keyname, time_t now) STAT_NEED_DS need DS to complete validation (name is returned in keyname) daemon->rr_status points to a char array which corressponds to the RRs in the - answer and auth sections. This is set to 1 for each RR which is validated, and 0 for any which aren't. + answer and auth sections. This is set to >1 for each RR which is validated, and 0 for any which aren't. When validating replies to DS records, we're only interested in the NSEC{3} RRs in the auth section. Other RRs in that section missing sigs will not cause am INSECURE reply. We determine this mode From 6417ff310cf531d1708ed16e79404677edc05b13 Mon Sep 17 00:00:00 2001 From: Simon Kelley Date: Thu, 26 May 2022 16:40:44 +0100 Subject: [PATCH 062/218] Add the ability to specify destination port in DHCP-relay mode. This change also removes a previous bug where --dhcp-alternate-port would affect the port used to relay _to_ as well as the port being listened on. The new feature allows configuration to provide bug-for-bug compatibility, if required. Thanks to Damian Kaczkowski for the feature suggestion. Signed-off-by: DL6ER --- src/dnsmasq/dhcp-common.c | 11 +++++++++-- src/dnsmasq/dhcp.c | 2 +- src/dnsmasq/dnsmasq.h | 1 + src/dnsmasq/option.c | 10 ++++++++++ src/dnsmasq/rfc3315.c | 2 +- 5 files changed, 22 insertions(+), 4 deletions(-) diff --git a/src/dnsmasq/dhcp-common.c b/src/dnsmasq/dhcp-common.c index b33c5dfc..fa690fe5 100644 --- a/src/dnsmasq/dhcp-common.c +++ b/src/dnsmasq/dhcp-common.c @@ -1017,7 +1017,10 @@ void log_relay(int family, struct dhcp_relay *relay) { int broadcast = relay->server.addr4.s_addr == 0; inet_ntop(family, &relay->local, daemon->addrbuff, ADDRSTRLEN); - inet_ntop(family, &relay->server, daemon->namebuff, ADDRSTRLEN); + inet_ntop(family, &relay->server, daemon->namebuff, ADDRSTRLEN); + + if (family == AF_INET && relay->port != DHCP_SERVER_PORT) + sprintf(daemon->namebuff + strlen(daemon->namebuff), "#%u", relay->port); #ifdef HAVE_DHCP6 struct in6_addr multicast; @@ -1025,7 +1028,11 @@ void log_relay(int family, struct dhcp_relay *relay) inet_pton(AF_INET6, ALL_SERVERS, &multicast); if (family == AF_INET6) - broadcast = IN6_ARE_ADDR_EQUAL(&relay->server.addr6, &multicast); + { + broadcast = IN6_ARE_ADDR_EQUAL(&relay->server.addr6, &multicast); + if (relay->port != DHCPV6_SERVER_PORT) + sprintf(daemon->namebuff + strlen(daemon->namebuff), "#%u", relay->port); + } #endif diff --git a/src/dnsmasq/dhcp.c b/src/dnsmasq/dhcp.c index 6104c87d..8e9c606d 100644 --- a/src/dnsmasq/dhcp.c +++ b/src/dnsmasq/dhcp.c @@ -1121,7 +1121,7 @@ static int relay_upstream4(int iface_index, struct dhcp_packet *mess, size_t sz) to.sa.sa_family = AF_INET; to.in.sin_addr = relay->server.addr4; - to.in.sin_port = htons(daemon->dhcp_server_port); + to.in.sin_port = htons(relay->port); /* Broadcasting to server. */ if (relay->server.addr4.s_addr == 0) diff --git a/src/dnsmasq/dnsmasq.h b/src/dnsmasq/dnsmasq.h index f1e12736..1205a3bd 100644 --- a/src/dnsmasq/dnsmasq.h +++ b/src/dnsmasq/dnsmasq.h @@ -1097,6 +1097,7 @@ struct dhcp_relay { union all_addr local, server; char *interface; /* Allowable interface for replies from server, and dest for IPv6 multicast */ int iface_index; /* working - interface in which requests arrived, for return */ + int port; /* Port of relay we forward to. */ #ifdef HAVE_SCRIPT struct snoop_record { struct in6_addr client, prefix; diff --git a/src/dnsmasq/option.c b/src/dnsmasq/option.c index db2e5b09..a36dcfde 100644 --- a/src/dnsmasq/option.c +++ b/src/dnsmasq/option.c @@ -4337,6 +4337,11 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma { if (inet_pton(AF_INET, arg, &new->local)) { + char *hash = split_chr(two, '#'); + + if (!hash || !atoi_check16(hash, &new->port)) + new->port = DHCP_SERVER_PORT; + if (!inet_pton(AF_INET, two, &new->server)) { new->server.addr4.s_addr = 0; @@ -4355,6 +4360,11 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma #ifdef HAVE_DHCP6 else if (inet_pton(AF_INET6, arg, &new->local)) { + char *hash = split_chr(two, '#'); + + if (!hash || !atoi_check16(hash, &new->port)) + new->port = DHCPV6_SERVER_PORT; + if (!inet_pton(AF_INET6, two, &new->server)) { inet_pton(AF_INET6, ALL_SERVERS, &new->server.addr6); diff --git a/src/dnsmasq/rfc3315.c b/src/dnsmasq/rfc3315.c index e218d263..65331977 100644 --- a/src/dnsmasq/rfc3315.c +++ b/src/dnsmasq/rfc3315.c @@ -2170,7 +2170,7 @@ int relay_upstream6(int iface_index, ssize_t sz, to.sa.sa_family = AF_INET6; to.in6.sin6_addr = relay->server.addr6; - to.in6.sin6_port = htons(DHCPV6_SERVER_PORT); + to.in6.sin6_port = htons(relay->port); to.in6.sin6_flowinfo = 0; to.in6.sin6_scope_id = 0; From 1aa4087abe81157be31efc0b26422a79ce43c9fb Mon Sep 17 00:00:00 2001 From: Beniamino Galvani Date: Fri, 27 May 2022 21:16:18 +0100 Subject: [PATCH 063/218] Fix parsing of IPv6 addresses with peer from netlink. In the most common case, an IPv6 address doesn't have a peer and the IFA_ADDRESS netlink attribute contains the address itself. But if the address has a peer (typically for point to point links), then IFA_ADDRESS contains the peer address and IFA_LOCAL contains the address [1]. [1] https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/net/ipv6/addrconf.c?h=v5.17#n5030 Fix the parsing of IPv6 addresses with peers, as currently dnsmasq unsuccessfully tries to bind on the peer address. A simple reproducer is: dnsmasq --conf-file=/dev/null -i dummy1 -d --bind-dynamic & sleep 2 ip link add dummy1 type dummy ip link set dummy1 up ip addr add dev dummy1 fd01::1/64 peer fd01::2/64 ip addr add dev dummy1 fd01::42/64 sleep 2 ss -lnp | grep dnsmasq | grep fd01 Before the patch: dnsmasq: failed to create listening socket for fd01::2: Cannot assign requested address dnsmasq: failed to create listening socket for fd01::2: Cannot assign requested address udp UNCONN 0 [fd01::42]:53 [::]:* users:(("dnsmasq",pid=23947,fd=14)) tcp LISTEN 0 [fd01::42]:53 [::]:* users:(("dnsmasq",pid=23947,fd=15 After: udp UNCONN 0 [fd01::42]:53 [::]:* users:(("dnsmasq",pid=23973,fd=16)) udp UNCONN 0 [fd01::1]:53 [::]:* users:(("dnsmasq",pid=23973,fd=14)) tcp LISTEN 0 [fd01::42]:53 [::]:* users:(("dnsmasq",pid=23973,fd=17)) tcp LISTEN 0 [fd01::1]:53 [::]:* users:(("dnsmasq",pid=23973,fd=15)) Signed-off-by: DL6ER --- src/dnsmasq/netlink.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/dnsmasq/netlink.c b/src/dnsmasq/netlink.c index da829430..c156cde3 100644 --- a/src/dnsmasq/netlink.c +++ b/src/dnsmasq/netlink.c @@ -258,7 +258,16 @@ int iface_enumerate(int family, void *parm, int (*callback)()) while (RTA_OK(rta, len1)) { - if (rta->rta_type == IFA_ADDRESS) + /* + * Important comment: (from if_addr.h) + * IFA_ADDRESS is prefix address, rather than local interface address. + * It makes no difference for normally configured broadcast interfaces, + * but for point-to-point IFA_ADDRESS is DESTINATION address, + * local address is supplied in IFA_LOCAL attribute. + */ + if (rta->rta_type == IFA_LOCAL) + addrp = ((struct in6_addr *)(rta+1)); + else if (rta->rta_type == IFA_ADDRESS && !addrp) addrp = ((struct in6_addr *)(rta+1)); else if (rta->rta_type == IFA_CACHEINFO) { From 195d67a2828f597f51a9a09044a5584dd6c9ac2c Mon Sep 17 00:00:00 2001 From: Simon Kelley Date: Thu, 7 Jul 2022 20:56:07 +0100 Subject: [PATCH 064/218] Fix comment typo. Signed-off-by: DL6ER --- src/dnsmasq/hash-questions.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dnsmasq/hash-questions.c b/src/dnsmasq/hash-questions.c index 4b1dd919..adcf62c8 100644 --- a/src/dnsmasq/hash-questions.c +++ b/src/dnsmasq/hash-questions.c @@ -23,7 +23,7 @@ The hash used is SHA-256. If we're building with DNSSEC support, we use the Nettle cypto library. If not, we prefer not to - add a dependency on Nettle, and use a stand-alone implementaion. + add a dependency on Nettle, and use a stand-alone implementation. */ #include "dnsmasq.h" From ff92410f9113cbef5dd46c5cac1ea9586e9b30b0 Mon Sep 17 00:00:00 2001 From: Hugo Hakim Damer Date: Sat, 9 Jul 2022 19:45:51 +0000 Subject: [PATCH 065/218] Pass MUD URLs (RFC 8520) supplied via DHCPv6 to DHCP scripts Extract Manufacturer Usage Description (MUD) URL from DHCP Option 112 and make it available to DHCP scripts as DNSMASQ_MUD_URL. This expands on the initial support for Manufacturer Usage Description URLs that has been added in the previous commit for DHCPv4 by also supporting MUD URLs supplied using DHCPv6. See https://datatracker.ietf.org/doc/html/rfc8520#section-17.3 and https://datatracker.ietf.org/doc/html/rfc8520#section-10 Co-authored-by: Jasper Wiegratz Signed-off-by: DL6ER --- src/dnsmasq/dhcp6-protocol.h | 1 + src/dnsmasq/helper.c | 6 ++++-- src/dnsmasq/rfc3315.c | 10 ++++++++++ 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/src/dnsmasq/dhcp6-protocol.h b/src/dnsmasq/dhcp6-protocol.h index 332d5364..ce166037 100644 --- a/src/dnsmasq/dhcp6-protocol.h +++ b/src/dnsmasq/dhcp6-protocol.h @@ -63,6 +63,7 @@ #define OPTION6_FQDN 39 #define OPTION6_NTP_SERVER 56 #define OPTION6_CLIENT_MAC 79 +#define OPTION6_MUD_URL 112 #define NTP_SUBOPTION_SRV_ADDR 1 #define NTP_SUBOPTION_MC_ADDR 2 diff --git a/src/dnsmasq/helper.c b/src/dnsmasq/helper.c index 33e49250..42101b31 100644 --- a/src/dnsmasq/helper.c +++ b/src/dnsmasq/helper.c @@ -647,8 +647,10 @@ int create_helper(int event_fd, int err_fd, uid_t uid, gid_t gid, long max_fd) buf = grab_extradata(buf, end, "DNSMASQ_TAGS", &err); - if (is6) - buf = grab_extradata(buf, end, "DNSMASQ_RELAY_ADDRESS", &err); + if (is6) { + buf = grab_extradata(buf, end, "DNSMASQ_RELAY_ADDRESS", &err); + buf = grab_extradata(buf, end, "DNSMASQ_MUD_URL", &err); + } else { const char *giaddr = NULL; diff --git a/src/dnsmasq/rfc3315.c b/src/dnsmasq/rfc3315.c index 65331977..8d601c42 100644 --- a/src/dnsmasq/rfc3315.c +++ b/src/dnsmasq/rfc3315.c @@ -1934,6 +1934,16 @@ static void update_leases(struct state *state, struct dhcp_context *context, str lease_add_extradata(lease, (unsigned char *)daemon->addrbuff, state->link_address ? strlen(daemon->addrbuff) : 0, 0); + void *mud_opt; + if ((mud_opt = opt6_find(state->packet_options, state->end, OPTION6_MUD_URL, 1))) + { + lease_add_extradata(lease, opt6_ptr(mud_opt, 0), opt6_len(mud_opt), NULL); + } + else + { + lease_add_extradata(lease, NULL, 0, 0); + } + if ((class_opt = opt6_find(state->packet_options, state->end, OPTION6_USER_CLASS, 2))) { void *enc_opt, *enc_end = opt6_ptr(class_opt, opt6_len(class_opt)); From 08e9d2ddf823456a682d6d67a734e23add7c401d Mon Sep 17 00:00:00 2001 From: Kevin Yeich Date: Sat, 9 Jul 2022 19:46:36 +0000 Subject: [PATCH 066/218] Pass MUD URLs (RFC 8520) supplied via DHCPv4 to DHCP scripts Extract Manufacturer Usage Description (MUD) URL from DHCP Option 161 and make it available to DHCP scripts as DNSMASQ_MUD_URL. See https://datatracker.ietf.org/doc/html/rfc8520#section-17.3 and https://datatracker.ietf.org/doc/html/rfc8520#section-10 Co-authored-by: Jasper Wiegratz Signed-off-by: DL6ER --- src/dnsmasq/dhcp-protocol.h | 1 + src/dnsmasq/helper.c | 1 + src/dnsmasq/rfc2131.c | 9 +++++++++ 3 files changed, 11 insertions(+) diff --git a/src/dnsmasq/dhcp-protocol.h b/src/dnsmasq/dhcp-protocol.h index 75c9cd3c..e281143a 100644 --- a/src/dnsmasq/dhcp-protocol.h +++ b/src/dnsmasq/dhcp-protocol.h @@ -64,6 +64,7 @@ #define OPTION_SIP_SERVER 120 #define OPTION_VENDOR_IDENT 124 #define OPTION_VENDOR_IDENT_OPT 125 +#define OPTION_MUD_URL_V4 161 #define OPTION_END 255 #define SUBOPT_CIRCUIT_ID 1 diff --git a/src/dnsmasq/helper.c b/src/dnsmasq/helper.c index 42101b31..04e320fe 100644 --- a/src/dnsmasq/helper.c +++ b/src/dnsmasq/helper.c @@ -642,6 +642,7 @@ int create_helper(int event_fd, int err_fd, uid_t uid, gid_t gid, long max_fd) buf = grab_extradata(buf, end, "DNSMASQ_CIRCUIT_ID", &err); buf = grab_extradata(buf, end, "DNSMASQ_SUBSCRIBER_ID", &err); buf = grab_extradata(buf, end, "DNSMASQ_REMOTE_ID", &err); + buf = grab_extradata(buf, end, "DNSMASQ_MUD_URL", &err); buf = grab_extradata(buf, end, "DNSMASQ_REQUESTED_OPTIONS", &err); } diff --git a/src/dnsmasq/rfc2131.c b/src/dnsmasq/rfc2131.c index ecda2d3e..692293ec 100644 --- a/src/dnsmasq/rfc2131.c +++ b/src/dnsmasq/rfc2131.c @@ -1417,6 +1417,15 @@ size_t dhcp_reply(struct dhcp_context *context, char *iface_name, int int_index, add_extradata_opt(lease, NULL); } + if ((opt = option_find(mess, sz, OPTION_MUD_URL_V4, 1))) + { + add_extradata_opt(lease, opt); + } + else + { + add_extradata_opt(lease, NULL); + } + /* DNSMASQ_REQUESTED_OPTIONS */ if ((opt = option_find(mess, sz, OPTION_REQUESTED_OPTIONS, 1))) { From bb35e185dc105046f794ac0ace408184f8bf6ad6 Mon Sep 17 00:00:00 2001 From: Simon Kelley Date: Sun, 31 Jul 2022 11:04:12 +0100 Subject: [PATCH 067/218] Tidy last two commits. Signed-off-by: DL6ER --- src/dnsmasq/rfc2131.c | 10 ++-------- src/dnsmasq/rfc3315.c | 29 ++++++++++++----------------- 2 files changed, 14 insertions(+), 25 deletions(-) diff --git a/src/dnsmasq/rfc2131.c b/src/dnsmasq/rfc2131.c index 692293ec..00539096 100644 --- a/src/dnsmasq/rfc2131.c +++ b/src/dnsmasq/rfc2131.c @@ -1418,14 +1418,8 @@ size_t dhcp_reply(struct dhcp_context *context, char *iface_name, int int_index, } if ((opt = option_find(mess, sz, OPTION_MUD_URL_V4, 1))) - { - add_extradata_opt(lease, opt); - } - else - { - add_extradata_opt(lease, NULL); - } - + add_extradata_opt(lease, opt); + /* DNSMASQ_REQUESTED_OPTIONS */ if ((opt = option_find(mess, sz, OPTION_REQUESTED_OPTIONS, 1))) { diff --git a/src/dnsmasq/rfc3315.c b/src/dnsmasq/rfc3315.c index 8d601c42..c0557181 100644 --- a/src/dnsmasq/rfc3315.c +++ b/src/dnsmasq/rfc3315.c @@ -1879,23 +1879,23 @@ static void update_leases(struct state *state, struct dhcp_context *context, str #ifdef HAVE_SCRIPT if (daemon->lease_change_command) { - void *class_opt; + void *opt; lease->flags |= LEASE_CHANGED; free(lease->extradata); lease->extradata = NULL; lease->extradata_size = lease->extradata_len = 0; lease->vendorclass_count = 0; - if ((class_opt = opt6_find(state->packet_options, state->end, OPTION6_VENDOR_CLASS, 4))) + if ((opt = opt6_find(state->packet_options, state->end, OPTION6_VENDOR_CLASS, 4))) { - void *enc_opt, *enc_end = opt6_ptr(class_opt, opt6_len(class_opt)); + void *enc_opt, *enc_end = opt6_ptr(opt, opt6_len(opt)); lease->vendorclass_count++; /* send enterprise number first */ - sprintf(daemon->dhcp_buff2, "%u", opt6_uint(class_opt, 0, 4)); + sprintf(daemon->dhcp_buff2, "%u", opt6_uint(opt, 0, 4)); lease_add_extradata(lease, (unsigned char *)daemon->dhcp_buff2, strlen(daemon->dhcp_buff2), 0); - if (opt6_len(class_opt) >= 6) - for (enc_opt = opt6_ptr(class_opt, 4); enc_opt; enc_opt = opt6_next(enc_opt, enc_end)) + if (opt6_len(opt) >= 6) + for (enc_opt = opt6_ptr(opt, 4); enc_opt; enc_opt = opt6_next(enc_opt, enc_end)) { lease->vendorclass_count++; lease_add_extradata(lease, opt6_ptr(enc_opt, 0), opt6_len(enc_opt), 0); @@ -1934,20 +1934,15 @@ static void update_leases(struct state *state, struct dhcp_context *context, str lease_add_extradata(lease, (unsigned char *)daemon->addrbuff, state->link_address ? strlen(daemon->addrbuff) : 0, 0); - void *mud_opt; - if ((mud_opt = opt6_find(state->packet_options, state->end, OPTION6_MUD_URL, 1))) - { - lease_add_extradata(lease, opt6_ptr(mud_opt, 0), opt6_len(mud_opt), NULL); - } + if ((opt = opt6_find(state->packet_options, state->end, OPTION6_MUD_URL, 1))) + lease_add_extradata(lease, opt6_ptr(opt, 0), opt6_len(opt), 0); else - { - lease_add_extradata(lease, NULL, 0, 0); - } + lease_add_extradata(lease, NULL, 0, 0); - if ((class_opt = opt6_find(state->packet_options, state->end, OPTION6_USER_CLASS, 2))) + if ((opt = opt6_find(state->packet_options, state->end, OPTION6_USER_CLASS, 2))) { - void *enc_opt, *enc_end = opt6_ptr(class_opt, opt6_len(class_opt)); - for (enc_opt = opt6_ptr(class_opt, 0); enc_opt; enc_opt = opt6_next(enc_opt, enc_end)) + void *enc_opt, *enc_end = opt6_ptr(opt, opt6_len(opt)); + for (enc_opt = opt6_ptr(opt, 0); enc_opt; enc_opt = opt6_next(enc_opt, enc_end)) lease_add_extradata(lease, opt6_ptr(enc_opt, 0), opt6_len(enc_opt), 0); } } From 6e73951270c3eca31cfac69a5116d2d0fcedc820 Mon Sep 17 00:00:00 2001 From: Simon Kelley Date: Sun, 31 Jul 2022 11:33:05 +0100 Subject: [PATCH 068/218] Fix bit-rotted data handling code for LUA scripts. Signed-off-by: DL6ER --- src/dnsmasq/helper.c | 12 ++++++------ src/dnsmasq/rfc2131.c | 7 +++---- src/dnsmasq/rfc3315.c | 10 +++++----- 3 files changed, 14 insertions(+), 15 deletions(-) diff --git a/src/dnsmasq/helper.c b/src/dnsmasq/helper.c index 04e320fe..f33d312e 100644 --- a/src/dnsmasq/helper.c +++ b/src/dnsmasq/helper.c @@ -459,8 +459,10 @@ int create_helper(int event_fd, int err_fd, uid_t uid, gid_t gid, long max_fd) buf = grab_extradata_lua(buf, end, "circuit_id"); buf = grab_extradata_lua(buf, end, "subscriber_id"); buf = grab_extradata_lua(buf, end, "remote_id"); + buf = grab_extradata_lua(buf, end, "requested_options"); } + buf = grab_extradata_lua(buf, end, "mud_url"); buf = grab_extradata_lua(buf, end, "tags"); if (is6) @@ -642,16 +644,14 @@ int create_helper(int event_fd, int err_fd, uid_t uid, gid_t gid, long max_fd) buf = grab_extradata(buf, end, "DNSMASQ_CIRCUIT_ID", &err); buf = grab_extradata(buf, end, "DNSMASQ_SUBSCRIBER_ID", &err); buf = grab_extradata(buf, end, "DNSMASQ_REMOTE_ID", &err); - buf = grab_extradata(buf, end, "DNSMASQ_MUD_URL", &err); buf = grab_extradata(buf, end, "DNSMASQ_REQUESTED_OPTIONS", &err); } + buf = grab_extradata(buf, end, "DNSMASQ_MUD_URL", &err); buf = grab_extradata(buf, end, "DNSMASQ_TAGS", &err); - - if (is6) { - buf = grab_extradata(buf, end, "DNSMASQ_RELAY_ADDRESS", &err); - buf = grab_extradata(buf, end, "DNSMASQ_MUD_URL", &err); - } + + if (is6) + buf = grab_extradata(buf, end, "DNSMASQ_RELAY_ADDRESS", &err); else { const char *giaddr = NULL; diff --git a/src/dnsmasq/rfc2131.c b/src/dnsmasq/rfc2131.c index 00539096..7e7c1a59 100644 --- a/src/dnsmasq/rfc2131.c +++ b/src/dnsmasq/rfc2131.c @@ -1417,9 +1417,6 @@ size_t dhcp_reply(struct dhcp_context *context, char *iface_name, int int_index, add_extradata_opt(lease, NULL); } - if ((opt = option_find(mess, sz, OPTION_MUD_URL_V4, 1))) - add_extradata_opt(lease, opt); - /* DNSMASQ_REQUESTED_OPTIONS */ if ((opt = option_find(mess, sz, OPTION_REQUESTED_OPTIONS, 1))) { @@ -1437,7 +1434,9 @@ size_t dhcp_reply(struct dhcp_context *context, char *iface_name, int int_index, { add_extradata_opt(lease, NULL); } - + + add_extradata_opt(lease, option_find(mess, sz, OPTION_MUD_URL_V4, 1)); + /* space-concat tag set */ if (!tagif_netid) add_extradata_opt(lease, NULL); diff --git a/src/dnsmasq/rfc3315.c b/src/dnsmasq/rfc3315.c index c0557181..31260df3 100644 --- a/src/dnsmasq/rfc3315.c +++ b/src/dnsmasq/rfc3315.c @@ -1905,6 +1905,11 @@ static void update_leases(struct state *state, struct dhcp_context *context, str lease_add_extradata(lease, (unsigned char *)state->client_hostname, state->client_hostname ? strlen(state->client_hostname) : 0, 0); + if ((opt = opt6_find(state->packet_options, state->end, OPTION6_MUD_URL, 1))) + lease_add_extradata(lease, opt6_ptr(opt, 0), opt6_len(opt), 0); + else + lease_add_extradata(lease, NULL, 0, 0); + /* space-concat tag set */ if (!tagif && !context->netid.net) lease_add_extradata(lease, NULL, 0, 0); @@ -1934,11 +1939,6 @@ static void update_leases(struct state *state, struct dhcp_context *context, str lease_add_extradata(lease, (unsigned char *)daemon->addrbuff, state->link_address ? strlen(daemon->addrbuff) : 0, 0); - if ((opt = opt6_find(state->packet_options, state->end, OPTION6_MUD_URL, 1))) - lease_add_extradata(lease, opt6_ptr(opt, 0), opt6_len(opt), 0); - else - lease_add_extradata(lease, NULL, 0, 0); - if ((opt = opt6_find(state->packet_options, state->end, OPTION6_USER_CLASS, 2))) { void *enc_opt, *enc_end = opt6_ptr(opt, opt6_len(opt)); From 48efc989c89655dd8d8c6edbab34877179597447 Mon Sep 17 00:00:00 2001 From: Simon Kelley Date: Sun, 31 Jul 2022 12:15:38 +0100 Subject: [PATCH 069/218] Update man page on DHCP data provided to scripts. Provide requested options for DHCPv6 also. Signed-off-by: DL6ER --- src/dnsmasq/helper.c | 6 +++--- src/dnsmasq/rfc2131.c | 15 +++++---------- src/dnsmasq/rfc3315.c | 14 ++++++++++++++ 3 files changed, 22 insertions(+), 13 deletions(-) diff --git a/src/dnsmasq/helper.c b/src/dnsmasq/helper.c index f33d312e..a96788b3 100644 --- a/src/dnsmasq/helper.c +++ b/src/dnsmasq/helper.c @@ -459,9 +459,9 @@ int create_helper(int event_fd, int err_fd, uid_t uid, gid_t gid, long max_fd) buf = grab_extradata_lua(buf, end, "circuit_id"); buf = grab_extradata_lua(buf, end, "subscriber_id"); buf = grab_extradata_lua(buf, end, "remote_id"); - buf = grab_extradata_lua(buf, end, "requested_options"); } - + + buf = grab_extradata_lua(buf, end, "requested_options"); buf = grab_extradata_lua(buf, end, "mud_url"); buf = grab_extradata_lua(buf, end, "tags"); @@ -644,9 +644,9 @@ int create_helper(int event_fd, int err_fd, uid_t uid, gid_t gid, long max_fd) buf = grab_extradata(buf, end, "DNSMASQ_CIRCUIT_ID", &err); buf = grab_extradata(buf, end, "DNSMASQ_SUBSCRIBER_ID", &err); buf = grab_extradata(buf, end, "DNSMASQ_REMOTE_ID", &err); - buf = grab_extradata(buf, end, "DNSMASQ_REQUESTED_OPTIONS", &err); } + buf = grab_extradata(buf, end, "DNSMASQ_REQUESTED_OPTIONS", &err); buf = grab_extradata(buf, end, "DNSMASQ_MUD_URL", &err); buf = grab_extradata(buf, end, "DNSMASQ_TAGS", &err); diff --git a/src/dnsmasq/rfc2131.c b/src/dnsmasq/rfc2131.c index 7e7c1a59..2056cbad 100644 --- a/src/dnsmasq/rfc2131.c +++ b/src/dnsmasq/rfc2131.c @@ -1420,20 +1420,15 @@ size_t dhcp_reply(struct dhcp_context *context, char *iface_name, int int_index, /* DNSMASQ_REQUESTED_OPTIONS */ if ((opt = option_find(mess, sz, OPTION_REQUESTED_OPTIONS, 1))) { - int len = option_len(opt); + int i, len = option_len(opt); unsigned char *rop = option_ptr(opt, 0); - char *q = daemon->namebuff; - int i; + for (i = 0; i < len; i++) - { - q += snprintf(q, MAXDNAME - (q - daemon->namebuff), "%d%s", rop[i], i + 1 == len ? "" : ","); - } - lease_add_extradata(lease, (unsigned char *)daemon->namebuff, (q - daemon->namebuff), 0); + lease_add_extradata(lease, (unsigned char *)daemon->namebuff, + sprintf(daemon->namebuff, "%u", rop[i]), (i + 1) == len ? 0 : ','); } else - { - add_extradata_opt(lease, NULL); - } + lease_add_extradata(lease, NULL, 0, 0); add_extradata_opt(lease, option_find(mess, sz, OPTION_MUD_URL_V4, 1)); diff --git a/src/dnsmasq/rfc3315.c b/src/dnsmasq/rfc3315.c index 31260df3..866b8b82 100644 --- a/src/dnsmasq/rfc3315.c +++ b/src/dnsmasq/rfc3315.c @@ -1880,6 +1880,7 @@ static void update_leases(struct state *state, struct dhcp_context *context, str if (daemon->lease_change_command) { void *opt; + lease->flags |= LEASE_CHANGED; free(lease->extradata); lease->extradata = NULL; @@ -1905,6 +1906,19 @@ static void update_leases(struct state *state, struct dhcp_context *context, str lease_add_extradata(lease, (unsigned char *)state->client_hostname, state->client_hostname ? strlen(state->client_hostname) : 0, 0); + /* DNSMASQ_REQUESTED_OPTIONS */ + if ((opt = opt6_find(state->packet_options, state->end, OPTION6_ORO, 2))) + { + int i, len = opt6_len(opt)/2; + u16 *rop = opt6_ptr(opt, 0); + + for (i = 0; i < len; i++) + lease_add_extradata(lease, (unsigned char *)daemon->namebuff, + sprintf(daemon->namebuff, "%u", ntohs(rop[i])), (i + 1) == len ? 0 : ','); + } + else + lease_add_extradata(lease, NULL, 0, 0); + if ((opt = opt6_find(state->packet_options, state->end, OPTION6_MUD_URL, 1))) lease_add_extradata(lease, opt6_ptr(opt, 0), opt6_len(opt), 0); else From c13654fe7b09a881eb6dc112cfb7aecf88327964 Mon Sep 17 00:00:00 2001 From: "Bertie, Taylor" Date: Sun, 31 Jul 2022 17:20:21 +0100 Subject: [PATCH 070/218] Bound the value of UDP packet size in the EDNS0 header of forwarded queries to the configured or default value of edns-packet-max. There's no point letting a client set a larger value if we're unable to return the answer. Signed-off-by: DL6ER --- src/dnsmasq/forward.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/dnsmasq/forward.c b/src/dnsmasq/forward.c index 0e05f6c1..b318bc8b 100644 --- a/src/dnsmasq/forward.c +++ b/src/dnsmasq/forward.c @@ -1661,13 +1661,17 @@ void receive_query(struct listener *listen, time_t now) /* If the client provides an EDNS0 UDP size, use that to limit our reply. (bounded by the maximum configured). If no EDNS0, then it - defaults to 512 */ + defaults to 512. We write this value into the query packet too, so that + if it's forwarded, we don't specify a maximum size greater than we can handle. */ if (udp_size > daemon->edns_pktsz) udp_size = daemon->edns_pktsz; else if (udp_size < PACKETSZ) udp_size = PACKETSZ; /* Sanity check - can't reduce below default. RFC 6891 6.2.3 */ - } + pheader -= 6; /* ext_class */ + PUTSHORT(udp_size, pheader); /* Bounding forwarded queries to maximum configured */ + } + #ifdef HAVE_CONNTRACK #ifdef HAVE_AUTH if (!auth_dns || local_auth) From 92efa1234be2404eb9cf187f35dc6a8aaf5fab0b Mon Sep 17 00:00:00 2001 From: Simon Kelley Date: Mon, 8 Aug 2022 15:27:32 +0100 Subject: [PATCH 071/218] Fix bad interaction between --address=/#/ and --server=/some.domain/# This would return for queries in some.domain, rather than forwarding the query via the default server(s) read from /etc/resolv.conf. Signed-off-by: DL6ER --- src/dnsmasq/domain-match.c | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/dnsmasq/domain-match.c b/src/dnsmasq/domain-match.c index 3ec49b80..f7db0fe8 100644 --- a/src/dnsmasq/domain-match.c +++ b/src/dnsmasq/domain-match.c @@ -213,9 +213,13 @@ int lookup_domain(char *domain, int flags, int *lowout, int *highout) to continue generalising */ { /* We've matched a setting which says to use servers without a domain. - Continue the search with empty query */ + Continue the search with empty query. We set the F_SERVER flag + so that --address=/#/... doesn't match. */ if (daemon->serverarray[nlow]->flags & SERV_USE_RESOLV) - crop_query = qlen; + { + crop_query = qlen; + flags |= F_SERVER; + } else break; } @@ -299,7 +303,7 @@ int filter_servers(int seed, int flags, int *lowout, int *highout) for (i = nlow; i < nhigh && (daemon->serverarray[i]->flags & SERV_6ADDR); i++); - if (i != nlow && (flags & F_IPV6)) + if (!(flags & F_SERVER) && i != nlow && (flags & F_IPV6)) nhigh = i; else { @@ -307,7 +311,7 @@ int filter_servers(int seed, int flags, int *lowout, int *highout) for (i = nlow; i < nhigh && (daemon->serverarray[i]->flags & SERV_4ADDR); i++); - if (i != nlow && (flags & F_IPV4)) + if (!(flags & F_SERVER) && i != nlow && (flags & F_IPV4)) nhigh = i; else { @@ -315,7 +319,7 @@ int filter_servers(int seed, int flags, int *lowout, int *highout) for (i = nlow; i < nhigh && (daemon->serverarray[i]->flags & SERV_ALL_ZEROS); i++); - if (i != nlow && (flags & (F_IPV4 | F_IPV6))) + if (!(flags & F_SERVER) && i != nlow && (flags & (F_IPV4 | F_IPV6))) nhigh = i; else { From 6635d5df2c23da16c69e415a9a2d3af3b8c189f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Men=C5=A1=C3=ADk?= Date: Mon, 18 Jul 2022 13:30:07 +0200 Subject: [PATCH 072/218] Introduce whine_realloc Move few patters with whine_malloc, if (successful) copy+free, to a new whine_realloc. It should do the same thing, but with a help from OS it can avoid unnecessary copy and free if allocation of more data after current data is possible. Added few setting remanining space to 0, because realloc does not use calloc like whine_malloc does. There is no advantage of zeroing what we will immediately overwrite. Zero only remaining space. Signed-off-by: DL6ER --- src/dnsmasq/cache.c | 4 +--- src/dnsmasq/dnsmasq.h | 1 + src/dnsmasq/lease.c | 8 +------- src/dnsmasq/poll.c | 9 +++++---- src/dnsmasq/rrfilter.c | 16 ++++++---------- src/dnsmasq/util.c | 10 ++++++++++ 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/src/dnsmasq/cache.c b/src/dnsmasq/cache.c index cbec10de..0d110861 100644 --- a/src/dnsmasq/cache.c +++ b/src/dnsmasq/cache.c @@ -1683,10 +1683,8 @@ int cache_make_stat(struct txt_record *t) { /* expand buffer if necessary */ newlen = bytes_needed + 1 + bufflen - bytes_avail; - if (!(new = whine_malloc(newlen))) + if (!(new = whine_realloc(buff, newlen))) return 0; - memcpy(new, buff, bufflen); - free(buff); p = new + (p - buff); lenp = p - 1; buff = new; diff --git a/src/dnsmasq/dnsmasq.h b/src/dnsmasq/dnsmasq.h index 1205a3bd..496b5973 100644 --- a/src/dnsmasq/dnsmasq.h +++ b/src/dnsmasq/dnsmasq.h @@ -1425,6 +1425,7 @@ void *safe_malloc(size_t size); void safe_strncpy(char *dest, const char *src, size_t size); void safe_pipe(int *fd, int read_noblock); void *whine_malloc(size_t size); +void *whine_realloc(void *ptr, size_t size); int sa_len(union mysockaddr *addr); int sockaddr_isequal(const union mysockaddr *s1, const union mysockaddr *s2); int hostname_order(const char *a, const char *b); diff --git a/src/dnsmasq/lease.c b/src/dnsmasq/lease.c index 81477d54..8a7b9756 100644 --- a/src/dnsmasq/lease.c +++ b/src/dnsmasq/lease.c @@ -1180,17 +1180,11 @@ void lease_add_extradata(struct dhcp_lease *lease, unsigned char *data, unsigned if ((lease->extradata_size - lease->extradata_len) < (len + 1)) { size_t newsz = lease->extradata_len + len + 100; - unsigned char *new = whine_malloc(newsz); + unsigned char *new = whine_realloc(lease->extradata, newsz); if (!new) return; - if (lease->extradata) - { - memcpy(new, lease->extradata, lease->extradata_len); - free(lease->extradata); - } - lease->extradata = new; lease->extradata_size = newsz; } diff --git a/src/dnsmasq/poll.c b/src/dnsmasq/poll.c index 29b33a0c..0e5964db 100644 --- a/src/dnsmasq/poll.c +++ b/src/dnsmasq/poll.c @@ -105,14 +105,15 @@ void poll_listen(int fd, short event) arrsize = (arrsize == 0) ? 64 : arrsize * 2; - if (!(new = whine_malloc(arrsize * sizeof(struct pollfd)))) + if (!(new = whine_realloc(pollfds, arrsize * sizeof(struct pollfd)))) return; if (pollfds) { - memcpy(new, pollfds, i * sizeof(struct pollfd)); - memcpy(&new[i+1], &pollfds[i], (nfds - i) * sizeof(struct pollfd)); - free(pollfds); + memmove(&new[i+1], &new[i], (nfds - i) * sizeof(struct pollfd)); + /* clear remaining space with zeroes. */ + if (nfds+1 < arrsize) + memset(new+nfds+1, 0, arrsize-nfds-1); } pollfds = new; diff --git a/src/dnsmasq/rrfilter.c b/src/dnsmasq/rrfilter.c index f02f5a5e..42d9c210 100644 --- a/src/dnsmasq/rrfilter.c +++ b/src/dnsmasq/rrfilter.c @@ -159,7 +159,7 @@ static int check_rrs(unsigned char *p, struct dns_header *header, size_t plen, i /* mode may be remove EDNS0 or DNSSEC RRs or remove A or AAAA from answer section. */ size_t rrfilter(struct dns_header *header, size_t plen, int mode) { - static unsigned char **rrs; + static unsigned char **rrs = NULL; static int rr_sz = 0; unsigned char *p = (unsigned char *)(header+1); @@ -339,15 +339,11 @@ int expand_workspace(unsigned char ***wkspc, int *szp, int new) return 0; new += 5; - - if (!(p = whine_malloc(new * sizeof(unsigned char *)))) - return 0; - - if (old != 0 && *wkspc) - { - memcpy(p, *wkspc, old * sizeof(unsigned char *)); - free(*wkspc); - } + + if (!(p = whine_realloc(*wkspc, new * sizeof(unsigned char *)))) + return 0; + + memset(p+old, 0, new-old); *wkspc = p; *szp = new; diff --git a/src/dnsmasq/util.c b/src/dnsmasq/util.c index ae514d29..140a3543 100644 --- a/src/dnsmasq/util.c +++ b/src/dnsmasq/util.c @@ -336,6 +336,16 @@ void *whine_malloc(size_t size) return ret; } +void *whine_realloc(void *ptr, size_t size) +{ + void *ret = realloc(ptr, size); + + if (!ret) + my_syslog(LOG_ERR, _("failed to reallocate %d bytes"), (int) size); + + return ret; +} + int sockaddr_isequal(const union mysockaddr *s1, const union mysockaddr *s2) { if (s1->sa.sa_family == s2->sa.sa_family) From e29c3b66103dbb248a5728029ac09613c1302741 Mon Sep 17 00:00:00 2001 From: Simon Kelley Date: Thu, 11 Aug 2022 17:04:54 +0100 Subject: [PATCH 073/218] Simplify realloc use in poll.c Signed-off-by: DL6ER --- src/dnsmasq/poll.c | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/src/dnsmasq/poll.c b/src/dnsmasq/poll.c index 0e5964db..bbb9009b 100644 --- a/src/dnsmasq/poll.c +++ b/src/dnsmasq/poll.c @@ -96,9 +96,7 @@ void poll_listen(int fd, short event) pollfds[i].events |= event; else { - if (arrsize != nfds) - memmove(&pollfds[i+1], &pollfds[i], (nfds - i) * sizeof(struct pollfd)); - else + if (arrsize == nfds) { /* Array too small, extend. */ struct pollfd *new; @@ -108,17 +106,11 @@ void poll_listen(int fd, short event) if (!(new = whine_realloc(pollfds, arrsize * sizeof(struct pollfd)))) return; - if (pollfds) - { - memmove(&new[i+1], &new[i], (nfds - i) * sizeof(struct pollfd)); - /* clear remaining space with zeroes. */ - if (nfds+1 < arrsize) - memset(new+nfds+1, 0, arrsize-nfds-1); - } - pollfds = new; } - + + memmove(&pollfds[i+1], &pollfds[i], (nfds - i) * sizeof(struct pollfd)); + pollfds[i].fd = fd; pollfds[i].events = event; nfds++; From a95263f6f6f5160eaf081fc5782a0644adfea122 Mon Sep 17 00:00:00 2001 From: Simon Kelley Date: Mon, 5 Sep 2022 18:04:35 +0100 Subject: [PATCH 074/218] Tweak packet dump code to make port numbers more accurate. Also add query-ids with log-queries=extra. Signed-off-by: DL6ER --- src/dnsmasq/dhcp.c | 15 +++-- src/dnsmasq/dhcp6.c | 12 ++-- src/dnsmasq/dnsmasq.h | 6 +- src/dnsmasq/dump.c | 125 +++++++++++++++++++++++++++--------------- src/dnsmasq/forward.c | 32 +++++------ src/dnsmasq/radv.c | 4 +- src/dnsmasq/rfc3315.c | 2 +- src/dnsmasq/tftp.c | 8 +-- 8 files changed, 120 insertions(+), 84 deletions(-) diff --git a/src/dnsmasq/dhcp.c b/src/dnsmasq/dhcp.c index 8e9c606d..42d819f0 100644 --- a/src/dnsmasq/dhcp.c +++ b/src/dnsmasq/dhcp.c @@ -177,8 +177,7 @@ void dhcp_packet(time_t now, int pxe_fd) return; #ifdef HAVE_DUMPFILE - dump_packet(DUMP_DHCP, (void *)daemon->dhcp_packet.iov_base, sz, (union mysockaddr *)&dest, NULL, - pxe_fd ? PXE_PORT : daemon->dhcp_server_port); + dump_packet_udp(DUMP_DHCP, (void *)daemon->dhcp_packet.iov_base, sz, (union mysockaddr *)&dest, NULL, fd); #endif #if defined (HAVE_LINUX_NETWORK) @@ -464,8 +463,8 @@ void dhcp_packet(time_t now, int pxe_fd) dest.sin_addr = mess->yiaddr; dest.sin_port = htons(daemon->dhcp_client_port); - dump_packet(DUMP_DHCP, (void *)iov.iov_base, iov.iov_len, NULL, - (union mysockaddr *)&dest, daemon->dhcp_server_port); + dump_packet_udp(DUMP_DHCP, (void *)iov.iov_base, iov.iov_len, NULL, + (union mysockaddr *)&dest, fd); #endif send_via_bpf(mess, iov.iov_len, iface_addr, &ifr); @@ -478,8 +477,8 @@ void dhcp_packet(time_t now, int pxe_fd) #endif #ifdef HAVE_DUMPFILE - dump_packet(DUMP_DHCP, (void *)iov.iov_base, iov.iov_len, NULL, - (union mysockaddr *)&dest, daemon->dhcp_server_port); + dump_packet_udp(DUMP_DHCP, (void *)iov.iov_base, iov.iov_len, NULL, + (union mysockaddr *)&dest, fd); #endif while(retry_send(sendmsg(fd, &msg, 0))); @@ -1147,8 +1146,8 @@ static int relay_upstream4(int iface_index, struct dhcp_packet *mess, size_t sz) fromsock.in.sin_port = htons(daemon->dhcp_server_port); fromsock.in.sin_addr = from.addr4; fromsock.sa.sa_family = AF_INET; - - dump_packet(DUMP_DHCP, (void *)mess, sz, &fromsock, &to, 0); + + dump_packet_udp(DUMP_DHCP, (void *)mess, sz, &fromsock, &to, -1); } #endif diff --git a/src/dnsmasq/dhcp6.c b/src/dnsmasq/dhcp6.c index edb87a4a..1c8c6794 100644 --- a/src/dnsmasq/dhcp6.c +++ b/src/dnsmasq/dhcp6.c @@ -119,8 +119,8 @@ void dhcp6_packet(time_t now) return; #ifdef HAVE_DUMPFILE - dump_packet(DUMP_DHCPV6, (void *)daemon->dhcp_packet.iov_base, sz, - (union mysockaddr *)&from, NULL, DHCPV6_SERVER_PORT); + dump_packet_udp(DUMP_DHCPV6, (void *)daemon->dhcp_packet.iov_base, sz, + (union mysockaddr *)&from, NULL, daemon->dhcp6fd); #endif for (cmptr = CMSG_FIRSTHDR(&msg); cmptr; cmptr = CMSG_NXTHDR(&msg, cmptr)) @@ -142,8 +142,8 @@ void dhcp6_packet(time_t now) if (relay_reply6(&from, sz, ifr.ifr_name)) { #ifdef HAVE_DUMPFILE - dump_packet(DUMP_DHCPV6, (void *)daemon->outpacket.iov_base, save_counter(-1), NULL, - (union mysockaddr *)&from, DHCPV6_SERVER_PORT); + dump_packet_udp(DUMP_DHCPV6, (void *)daemon->outpacket.iov_base, save_counter(-1), NULL, + (union mysockaddr *)&from, daemon->dhcp6fd); #endif while (retry_send(sendto(daemon->dhcp6fd, daemon->outpacket.iov_base, @@ -254,8 +254,8 @@ void dhcp6_packet(time_t now) from.sin6_port = htons(port); #ifdef HAVE_DUMPFILE - dump_packet(DUMP_DHCPV6, (void *)daemon->outpacket.iov_base, save_counter(-1), - NULL, (union mysockaddr *)&from, DHCPV6_SERVER_PORT); + dump_packet_udp(DUMP_DHCPV6, (void *)daemon->outpacket.iov_base, save_counter(-1), + NULL, (union mysockaddr *)&from, daemon->dhcp6fd); #endif while (retry_send(sendto(daemon->dhcp6fd, daemon->outpacket.iov_base, diff --git a/src/dnsmasq/dnsmasq.h b/src/dnsmasq/dnsmasq.h index 496b5973..5cdb11c0 100644 --- a/src/dnsmasq/dnsmasq.h +++ b/src/dnsmasq/dnsmasq.h @@ -1843,8 +1843,10 @@ int do_arp_script_run(void); /* dump.c */ #ifdef HAVE_DUMPFILE void dump_init(void); -void dump_packet(int mask, void *packet, size_t len, union mysockaddr *src, - union mysockaddr *dst, int port); +void dump_packet_udp(int mask, void *packet, size_t len, union mysockaddr *src, + union mysockaddr *dst, int fd); +void dump_packet_icmp(int mask, void *packet, size_t len, union mysockaddr *src, + union mysockaddr *dst); #endif /* domain-match.c */ diff --git a/src/dnsmasq/dump.c b/src/dnsmasq/dump.c index 7b15945c..57352a9d 100644 --- a/src/dnsmasq/dump.c +++ b/src/dnsmasq/dump.c @@ -21,6 +21,8 @@ #include static u32 packet_count; +static void do_dump_packet(int mask, void *packet, size_t len, + union mysockaddr *src, union mysockaddr *dst, int port, int proto); /* https://wiki.wireshark.org/Development/LibpcapFileFormat */ struct pcap_hdr_s { @@ -81,9 +83,45 @@ void dump_init(void) } } -/* port == -1 ->ICMPv6 */ -void dump_packet(int mask, void *packet, size_t len, - union mysockaddr *src, union mysockaddr *dst, int port) +void dump_packet_udp(int mask, void *packet, size_t len, + union mysockaddr *src, union mysockaddr *dst, int fd) +{ + union mysockaddr fd_addr; + socklen_t addr_len = sizeof(fd_addr); + + if (daemon->dumpfd != -1 && (mask & daemon->dump_mask)) + { + /* if fd is negative it carries a port number (negated) + which we use as a source or destination when not otherwise + specified so wireshark can ID the packet. + If both src and dst are specified, set this to -1 to avoid + a spurious getsockname() call. */ + int port = (fd < 0) ? -fd : -1; + + /* fd >= 0 is a file descriptor and the address of that file descriptor is used + in place of a NULL src or dst. */ + if (fd >= 0 && getsockname(fd, (struct sockaddr *)&fd_addr, &addr_len) != -1) + { + if (!src) + src = &fd_addr; + + if (!dst) + dst = &fd_addr; + } + + do_dump_packet(mask, packet, len, src, dst, port, IPPROTO_UDP); + } +} + +void dump_packet_icmp(int mask, void *packet, size_t len, + union mysockaddr *src, union mysockaddr *dst) +{ + if (daemon->dumpfd != -1 && (mask & daemon->dump_mask)) + do_dump_packet(mask, packet, len, src, dst, -1, IPPROTO_ICMP); +} + +static void do_dump_packet(int mask, void *packet, size_t len, + union mysockaddr *src, union mysockaddr *dst, int port, int proto) { struct ip ip; struct ip6_hdr ip6; @@ -100,13 +138,14 @@ void dump_packet(int mask, void *packet, size_t len, void *iphdr; size_t ipsz; int rc; + + /* if port != -1 it carries a port number + which we use as a source or destination when not otherwise + specified so wireshark can ID the packet. + If both src and dst are specified, set this to -1 to avoid + a spurious getsockname() call. */ + udp.uh_sport = udp.uh_dport = htons(port < 0 ? 0 : port); - if (daemon->dumpfd == -1 || !(mask & daemon->dump_mask)) - return; - - /* So wireshark can Id the packet. */ - udp.uh_sport = udp.uh_dport = htons(port); - if (src) family = src->sa.sa_family; else @@ -121,15 +160,12 @@ void dump_packet(int mask, void *packet, size_t len, ip6.ip6_vfc = 6 << 4; ip6.ip6_hops = 64; - if (port == -1) - { - ip6.ip6_plen = htons(len); - ip6.ip6_nxt = IPPROTO_ICMPV6; - } + if ((ip6.ip6_nxt = proto) == IPPROTO_UDP) + ip6.ip6_plen = htons(sizeof(struct udphdr) + len); else { - ip6.ip6_plen = htons(sizeof(struct udphdr) + len); - ip6.ip6_nxt = IPPROTO_UDP; + proto = ip6.ip6_nxt = IPPROTO_ICMPV6; + ip6.ip6_plen = htons(len); } if (src) @@ -161,15 +197,12 @@ void dump_packet(int mask, void *packet, size_t len, ip.ip_hl = sizeof(struct ip) / 4; ip.ip_ttl = IPDEFTTL; - if (port == -1) - { - ip.ip_len = htons(sizeof(struct ip) + len); - ip.ip_p = IPPROTO_ICMP; - } + if ((ip.ip_p = proto) == IPPROTO_UDP) + ip.ip_len = htons(sizeof(struct ip) + sizeof(struct udphdr) + len); else { - ip.ip_len = htons(sizeof(struct ip) + sizeof(struct udphdr) + len); - ip.ip_p = IPPROTO_UDP; + ip.ip_len = htons(sizeof(struct ip) + len); + proto = ip.ip_p = IPPROTO_ICMP; } if (src) @@ -191,7 +224,7 @@ void dump_packet(int mask, void *packet, size_t len, sum = (sum & 0xffff) + (sum >> 16); ip.ip_sum = (sum == 0xffff) ? sum : ~sum; - /* start UDP checksum */ + /* start UDP/ICMP checksum */ sum = ip.ip_src.s_addr & 0xffff; sum += (ip.ip_src.s_addr >> 16) & 0xffff; sum += ip.ip_dst.s_addr & 0xffff; @@ -201,25 +234,7 @@ void dump_packet(int mask, void *packet, size_t len, if (len & 1) ((unsigned char *)packet)[len] = 0; /* for checksum, in case length is odd. */ - if (port == -1) - { - /* ICMP - ICMPv6 packet is a superset of ICMP */ - struct icmp6_hdr *icmp = packet; - - /* See comment in UDP code below. */ - sum += htons((family == AF_INET6) ? IPPROTO_ICMPV6 : IPPROTO_ICMP); - sum += htons(len); - - icmp->icmp6_cksum = 0; - for (i = 0; i < (len + 1) / 2; i++) - sum += ((u16 *)packet)[i]; - while (sum >> 16) - sum = (sum & 0xffff) + (sum >> 16); - icmp->icmp6_cksum = (sum == 0xffff) ? sum : ~sum; - - pcap_header.incl_len = pcap_header.orig_len = ipsz + len; - } - else + if (proto == IPPROTO_UDP) { /* Add Remaining part of the pseudoheader. Note that though the IPv6 pseudoheader is very different to the IPv4 one, the @@ -241,7 +256,25 @@ void dump_packet(int mask, void *packet, size_t len, pcap_header.incl_len = pcap_header.orig_len = ipsz + sizeof(udp) + len; } - + else + { + /* ICMP - ICMPv6 packet is a superset of ICMP */ + struct icmp6_hdr *icmp = packet; + + /* See comment in UDP code above. */ + sum += htons(proto); + sum += htons(len); + + icmp->icmp6_cksum = 0; + for (i = 0; i < (len + 1) / 2; i++) + sum += ((u16 *)packet)[i]; + while (sum >> 16) + sum = (sum & 0xffff) + (sum >> 16); + icmp->icmp6_cksum = (sum == 0xffff) ? sum : ~sum; + + pcap_header.incl_len = pcap_header.orig_len = ipsz + len; + } + rc = gettimeofday(&time, NULL); pcap_header.ts_sec = time.tv_sec; pcap_header.ts_usec = time.tv_usec; @@ -249,9 +282,11 @@ void dump_packet(int mask, void *packet, size_t len, if (rc == -1 || !read_write(daemon->dumpfd, (void *)&pcap_header, sizeof(pcap_header), 0) || !read_write(daemon->dumpfd, iphdr, ipsz, 0) || - (port != -1 && !read_write(daemon->dumpfd, (void *)&udp, sizeof(udp), 0)) || + (proto == IPPROTO_UDP && !read_write(daemon->dumpfd, (void *)&udp, sizeof(udp), 0)) || !read_write(daemon->dumpfd, (void *)packet, len, 0)) my_syslog(LOG_ERR, _("failed to write packet dump")); + else if (option_bool(OPT_EXTRALOG)) + my_syslog(LOG_INFO, _("%u dumping packet %u mask 0x%04x"), daemon->log_display_id, ++packet_count, mask); else my_syslog(LOG_INFO, _("dumping packet %u mask 0x%04x"), ++packet_count, mask); diff --git a/src/dnsmasq/forward.c b/src/dnsmasq/forward.c index b318bc8b..9ec735a0 100644 --- a/src/dnsmasq/forward.c +++ b/src/dnsmasq/forward.c @@ -520,7 +520,7 @@ static int forward_query(int udpfd, union mysockaddr *udpaddr, if (errno == 0) { #ifdef HAVE_DUMPFILE - dump_packet(DUMP_UP_QUERY, (void *)header, plen, NULL, &srv->addr, daemon->port); + dump_packet_udp(DUMP_UP_QUERY, (void *)header, plen, NULL, &srv->addr, fd); #endif /* Keep info in case we want to re-send this packet */ @@ -861,8 +861,8 @@ static void dnssec_validate(struct frec *forward, struct dns_header *header, NULL, NULL, NULL); #ifdef HAVE_DUMPFILE if (STAT_ISEQUAL(status, STAT_BOGUS)) - dump_packet((forward->flags & (FREC_DNSKEY_QUERY | FREC_DS_QUERY)) ? DUMP_SEC_BOGUS : DUMP_BOGUS, - header, (size_t)plen, &forward->sentto->addr, NULL, daemon->port); + dump_packet_udp((forward->flags & (FREC_DNSKEY_QUERY | FREC_DS_QUERY)) ? DUMP_SEC_BOGUS : DUMP_BOGUS, + header, (size_t)plen, &forward->sentto->addr, NULL, -daemon->port); #endif } @@ -972,13 +972,13 @@ static void dnssec_validate(struct frec *forward, struct dns_header *header, set_outgoing_mark(orig, fd); #endif + server_send(server, fd, header, nn, 0); + server->queries++; #ifdef HAVE_DUMPFILE - dump_packet(DUMP_SEC_QUERY, (void *)header, (size_t)nn, NULL, &server->addr, daemon->port); + dump_packet_udp(DUMP_SEC_QUERY, (void *)header, (size_t)nn, NULL, &server->addr, fd); #endif log_query_mysockaddr(F_NOEXTRA | F_DNSSEC | F_SERVER, daemon->keyname, &server->addr, STAT_ISEQUAL(status, STAT_NEED_KEY) ? "dnssec-query[DNSKEY]" : "dnssec-query[DS]", 0); - server_send(server, fd, header, nn, 0); - server->queries++; return; } @@ -1067,16 +1067,16 @@ void reply_query(int fd, time_t now) if (difftime(now, server->pktsz_reduced) > UDP_TEST_TIME) server->edns_pktsz = daemon->edns_pktsz; -#ifdef HAVE_DUMPFILE - dump_packet((forward->flags & (FREC_DNSKEY_QUERY | FREC_DS_QUERY)) ? DUMP_SEC_REPLY : DUMP_UP_REPLY, - (void *)header, n, &serveraddr, NULL, daemon->port); -#endif - /* log_query gets called indirectly all over the place, so pass these in global variables - sorry. */ daemon->log_display_id = forward->frec_src.log_id; daemon->log_source_addr = &forward->frec_src.source; +#ifdef HAVE_DUMPFILE + dump_packet_udp((forward->flags & (FREC_DNSKEY_QUERY | FREC_DS_QUERY)) ? DUMP_SEC_REPLY : DUMP_UP_REPLY, + (void *)header, n, &serveraddr, NULL, fd); +#endif + if (daemon->ignore_addr && RCODE(header) == NOERROR && check_for_ignored_address(header, n)) return; @@ -1273,7 +1273,7 @@ static void return_reply(time_t now, struct frec *forward, struct dns_header *he header->id = htons(src->orig_id); #ifdef HAVE_DUMPFILE - dump_packet(DUMP_REPLY, daemon->packet, (size_t)nn, NULL, &src->source, daemon->port); + dump_packet_udp(DUMP_REPLY, daemon->packet, (size_t)nn, NULL, &src->source, src->fd); #endif #if defined(HAVE_CONNTRACK) && defined(HAVE_UBUS) @@ -1601,7 +1601,7 @@ void receive_query(struct listener *listen, time_t now) daemon->log_source_addr = &source_addr; #ifdef HAVE_DUMPFILE - dump_packet(DUMP_QUERY, daemon->packet, (size_t)n, &source_addr, NULL, daemon->port); + dump_packet_udp(DUMP_QUERY, daemon->packet, (size_t)n, &source_addr, NULL, listen->fd); #endif #ifdef HAVE_CONNTRACK @@ -1695,7 +1695,7 @@ void receive_query(struct listener *listen, time_t now) if (m >= 1) { #ifdef HAVE_DUMPFILE - dump_packet(DUMP_REPLY, daemon->packet, m, NULL, &source_addr, daemon->port); + dump_packet_udp(DUMP_REPLY, daemon->packet, m, NULL, &source_addr, listen->fd); #endif send_from(listen->fd, option_bool(OPT_NOWILD) || option_bool(OPT_CLEVERBIND), (char *)header, m, &source_addr, &dst_addr, if_index); @@ -1711,7 +1711,7 @@ void receive_query(struct listener *listen, time_t now) if (m >= 1) { #ifdef HAVE_DUMPFILE - dump_packet(DUMP_REPLY, daemon->packet, m, NULL, &source_addr, daemon->port); + dump_packet_udp(DUMP_REPLY, daemon->packet, m, NULL, &source_addr, listen->fd); #endif #if defined(HAVE_CONNTRACK) && defined(HAVE_UBUS) if (local_auth) @@ -1767,7 +1767,7 @@ void receive_query(struct listener *listen, time_t now) if (m >= 1) { #ifdef HAVE_DUMPFILE - dump_packet(DUMP_REPLY, daemon->packet, m, NULL, &source_addr, daemon->port); + dump_packet_udp(DUMP_REPLY, daemon->packet, m, NULL, &source_addr, listen->fd); #endif #if defined(HAVE_CONNTRACK) && defined(HAVE_UBUS) if (option_bool(OPT_CMARK_ALST_EN) && have_mark && ((u32)mark & daemon->allowlist_mask)) diff --git a/src/dnsmasq/radv.c b/src/dnsmasq/radv.c index 2b4326c0..c77b69ec 100644 --- a/src/dnsmasq/radv.c +++ b/src/dnsmasq/radv.c @@ -203,7 +203,7 @@ void icmp6_packet(time_t now) int opt_sz; #ifdef HAVE_DUMPFILE - dump_packet(DUMP_RA, (void *)packet, sz, (union mysockaddr *)&from, NULL, -1); + dump_packet_icmp(DUMP_RA, (void *)packet, sz, (union mysockaddr *)&from, NULL); #endif /* look for link-layer address option for logging */ @@ -560,7 +560,7 @@ static void send_ra_alias(time_t now, int iface, char *iface_name, struct in6_ad } #ifdef HAVE_DUMPFILE - dump_packet(DUMP_RA, (void *)daemon->outpacket.iov_base, save_counter(-1), NULL, (union mysockaddr *)&addr, -1); + dump_packet_icmp(DUMP_RA, (void *)daemon->outpacket.iov_base, save_counter(-1), NULL, (union mysockaddr *)&addr); #endif while (retry_send(sendto(daemon->icmp6fd, daemon->outpacket.iov_base, diff --git a/src/dnsmasq/rfc3315.c b/src/dnsmasq/rfc3315.c index 866b8b82..947bf048 100644 --- a/src/dnsmasq/rfc3315.c +++ b/src/dnsmasq/rfc3315.c @@ -2214,7 +2214,7 @@ int relay_upstream6(int iface_index, ssize_t sz, fromsock.in6.sin6_flowinfo = 0; fromsock.in6.sin6_scope_id = 0; - dump_packet(DUMP_DHCPV6, (void *)daemon->outpacket.iov_base, save_counter(-1), &fromsock, &to, 0); + dump_packet_udp(DUMP_DHCPV6, (void *)daemon->outpacket.iov_base, save_counter(-1), &fromsock, &to, -1); } #endif send_from(daemon->dhcp6fd, 0, daemon->outpacket.iov_base, save_counter(-1), &to, &from, 0); diff --git a/src/dnsmasq/tftp.c b/src/dnsmasq/tftp.c index bdf37a38..0861f37c 100644 --- a/src/dnsmasq/tftp.c +++ b/src/dnsmasq/tftp.c @@ -97,7 +97,7 @@ void tftp_request(struct listener *listen, time_t now) return; #ifdef HAVE_DUMPFILE - dump_packet(DUMP_TFTP, (void *)packet, len, (union mysockaddr *)&peer, NULL, TFTP_PORT); + dump_packet_udp(DUMP_TFTP, (void *)packet, len, (union mysockaddr *)&peer, NULL, listen->tftpfd); #endif /* Can always get recvd interface for IPv6 */ @@ -488,7 +488,7 @@ void tftp_request(struct listener *listen, time_t now) send_from(transfer->sockfd, !option_bool(OPT_SINGLE_PORT), packet, len, &peer, &addra, if_index); #ifdef HAVE_DUMPFILE - dump_packet(DUMP_TFTP, (void *)packet, len, NULL, (union mysockaddr *)&peer, TFTP_PORT); + dump_packet_udp(DUMP_TFTP, (void *)packet, len, NULL, (union mysockaddr *)&peer, transfer->sockfd); #endif if (is_err) @@ -610,7 +610,7 @@ void check_tftp_listeners(time_t now) while(retry_send(sendto(transfer->sockfd, daemon->packet, len, 0, &peer.sa, sa_len(&peer)))); #ifdef HAVE_DUMPFILE - dump_packet(DUMP_TFTP, (void *)daemon->packet, len, NULL, (union mysockaddr *)&peer, TFTP_PORT); + dump_packet_udp(DUMP_TFTP, (void *)daemon->packet, len, NULL, (union mysockaddr *)&peer, transfer->sockfd); #endif } } @@ -650,7 +650,7 @@ void check_tftp_listeners(time_t now) send_from(transfer->sockfd, !option_bool(OPT_SINGLE_PORT), daemon->packet, len, &transfer->peer, &transfer->source, transfer->if_index); #ifdef HAVE_DUMPFILE - dump_packet(DUMP_TFTP, (void *)daemon->packet, len, NULL, (union mysockaddr *)&transfer->peer, TFTP_PORT); + dump_packet_udp(DUMP_TFTP, (void *)daemon->packet, len, NULL, (union mysockaddr *)&transfer->peer, transfer->sockfd); #endif } From fbfbd64e9b23f01b17e1d0887cce309e06f4b6ff Mon Sep 17 00:00:00 2001 From: Simon Kelley Date: Tue, 6 Sep 2022 15:35:54 +0100 Subject: [PATCH 075/218] Free sockets awaiting upstream DNS replies ASAP. Once we have a good answer, close the socket so that the fd can be reused during DNSSEC validation and we don't have to read and discard more replies from other servers. Signed-off-by: DL6ER --- src/dnsmasq/forward.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/dnsmasq/forward.c b/src/dnsmasq/forward.c index 9ec735a0..2c4d41b0 100644 --- a/src/dnsmasq/forward.c +++ b/src/dnsmasq/forward.c @@ -1166,6 +1166,13 @@ void reply_query(int fd, time_t now) } forward->sentto = server; + + /* We have a good answer, and will now validate it or return it. + It may be some time before this the validation completes, but we don't need + any more answers, so close the socket(s) on which we were expecting + answers, to conserve file descriptors, and to save work reading and + discarding answers for other upstreams. */ + free_rfds(&forward->rfds); #ifdef HAVE_DNSSEC if ((forward->sentto->flags & SERV_DO_DNSSEC) && From 5d921cea731a3f592491e93a240582187a9385f1 Mon Sep 17 00:00:00 2001 From: Simon Kelley Date: Tue, 6 Sep 2022 15:40:42 +0100 Subject: [PATCH 076/218] Fix DHCPv6 relay to use a more sensble source address. Tweak things so that packets relayed towards a server have source address on the server-facing network, not the client-facing network. Thanks to Luis Thomas for spotting this and initial patch. Signed-off-by: DL6ER --- src/dnsmasq/rfc3315.c | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/src/dnsmasq/rfc3315.c b/src/dnsmasq/rfc3315.c index 947bf048..87544816 100644 --- a/src/dnsmasq/rfc3315.c +++ b/src/dnsmasq/rfc3315.c @@ -2181,10 +2181,7 @@ int relay_upstream6(int iface_index, ssize_t sz, if (relay->iface_index != 0 && relay->iface_index == iface_index) { union mysockaddr to; - union all_addr from; - - /* source address == relay address */ - from.addr6 = relay->local.addr6; + memcpy(&header[2], &relay->local.addr6, IN6ADDRSZ); to.sa.sa_family = AF_INET6; @@ -2206,18 +2203,11 @@ int relay_upstream6(int iface_index, ssize_t sz, } #ifdef HAVE_DUMPFILE - { - union mysockaddr fromsock; - fromsock.in6.sin6_port = htons(DHCPV6_SERVER_PORT); - fromsock.in6.sin6_addr = from.addr6; - fromsock.sa.sa_family = AF_INET6; - fromsock.in6.sin6_flowinfo = 0; - fromsock.in6.sin6_scope_id = 0; - - dump_packet_udp(DUMP_DHCPV6, (void *)daemon->outpacket.iov_base, save_counter(-1), &fromsock, &to, -1); - } + dump_packet_udp(DUMP_DHCPV6, (void *)daemon->outpacket.iov_base, save_counter(-1), NULL, &to, daemon->dhcp6fd); #endif - send_from(daemon->dhcp6fd, 0, daemon->outpacket.iov_base, save_counter(-1), &to, &from, 0); + + while (retry_send(sendto(daemon->dhcp6fd, (void *)daemon->outpacket.iov_base, save_counter(-1), + 0, (struct sockaddr *)&to, sa_len(&to)))); if (option_bool(OPT_LOG_OPTS)) { From 07bf1418b06cc250a1e0e661b4be81004a0b6893 Mon Sep 17 00:00:00 2001 From: Simon Kelley Date: Tue, 6 Sep 2022 18:08:39 +0100 Subject: [PATCH 077/218] Add source address to RA packet dumps. Signed-off-by: DL6ER --- src/dnsmasq/radv.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/dnsmasq/radv.c b/src/dnsmasq/radv.c index c77b69ec..5820f4a6 100644 --- a/src/dnsmasq/radv.c +++ b/src/dnsmasq/radv.c @@ -560,7 +560,13 @@ static void send_ra_alias(time_t now, int iface, char *iface_name, struct in6_ad } #ifdef HAVE_DUMPFILE - dump_packet_icmp(DUMP_RA, (void *)daemon->outpacket.iov_base, save_counter(-1), NULL, (union mysockaddr *)&addr); + { + struct sockaddr_in6 src; + src.sin6_family = AF_INET6; + src.sin6_addr = parm.link_local; + + dump_packet_icmp(DUMP_RA, (void *)daemon->outpacket.iov_base, save_counter(-1), (union mysockaddr *)&src, (union mysockaddr *)&addr); + } #endif while (retry_send(sendto(daemon->icmp6fd, daemon->outpacket.iov_base, From bd4e2b28d044cdd6d90ab76fca767043c552d62c Mon Sep 17 00:00:00 2001 From: Simon Kelley Date: Fri, 19 Aug 2022 13:28:00 +0100 Subject: [PATCH 078/218] Fix logic when a SERVFAIL reply is received after good replt for DNSSEC. If we get a SERVFAIL or REFUSED answer to a DNSSEC query for which we already have a good answer, just ignore it. Signed-off-by: DL6ER --- src/dnsmasq/forward.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/dnsmasq/forward.c b/src/dnsmasq/forward.c index 2c4d41b0..e3a91b24 100644 --- a/src/dnsmasq/forward.c +++ b/src/dnsmasq/forward.c @@ -1095,12 +1095,15 @@ void reply_query(int fd, time_t now) size_t nn = 0; #ifdef HAVE_DNSSEC - /* DNSSEC queries have a copy of the original query stashed. - The query MAY have got a good answer, and be awaiting + /* The query MAY have got a good answer, and be awaiting the results of further queries, in which case The Stash contains something else and we don't need to retry anyway. */ - if ((forward->flags & (FREC_DNSKEY_QUERY | FREC_DS_QUERY)) && !forward->blocking_query) + if (forward->blocking_query) + return; + + if (forward->flags & (FREC_DNSKEY_QUERY | FREC_DS_QUERY)) { + /* DNSSEC queries have a copy of the original query stashed. */ blockdata_retrieve(forward->stash, forward->stash_len, (void *)header); nn = forward->stash_len; udp_size = daemon->edns_pktsz; From 8e0b86742cd7fd4e3eaabf34f7bed31fb4ad95fd Mon Sep 17 00:00:00 2001 From: Simon Kelley Date: Tue, 6 Sep 2022 22:40:06 +0100 Subject: [PATCH 079/218] Add DHCPv4 option 108 "ipv6-only" to the options table. Signed-off-by: DL6ER --- src/dnsmasq/dhcp-common.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/dnsmasq/dhcp-common.c b/src/dnsmasq/dhcp-common.c index fa690fe5..545311ed 100644 --- a/src/dnsmasq/dhcp-common.c +++ b/src/dnsmasq/dhcp-common.c @@ -685,6 +685,7 @@ const struct opttab_t { { "client-machine-id", 97, 0 }, { "posix-timezone", 100, OT_NAME }, /* RFC 4833, Sec. 2 */ { "tzdb-timezone", 101, OT_NAME }, /* RFC 4833, Sec. 2 */ + { "ipv6-only", 108, 4 | OT_DEC }, /* RFC 8925 */ { "subnet-select", 118, OT_INTERNAL }, { "domain-search", 119, OT_RFC1035_NAME }, { "sip-server", 120, 0 }, From 4dfc8d66fe31895764359956841f6a27d77c5f0a Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 8 Sep 2022 14:50:02 +0200 Subject: [PATCH 080/218] Update dnsmasq version to v2.87rc1 Signed-off-by: DL6ER --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0e374db5..53e96556 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -11,6 +11,6 @@ cmake_minimum_required(VERSION 2.8.12) project(PIHOLE_FTL C) -set(DNSMASQ_VERSION pi-hole-2.87test8) +set(DNSMASQ_VERSION pi-hole-v2.87rc1) add_subdirectory(src) From bea963088283631eb484973314e462e23b1029b8 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 11 Sep 2022 11:31:08 +0200 Subject: [PATCH 081/218] Apply suggestions from code review Co-authored-by: yubiuser Signed-off-by: DL6ER --- src/api/api.c | 2 +- src/database/gravity-db.c | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/api/api.c b/src/api/api.c index 441f1626..2e807f33 100644 --- a/src/api/api.c +++ b/src/api/api.c @@ -1008,7 +1008,7 @@ void getAllQueries(const char *client_message, const int sock, const bool isteln CNAME_domain = getCNAMEDomainString(query); } - // Get ID of blocking regex, if applicable and permitted by privacy settings + // Get domainlist table ID, if applicable and permitted by privacy settings int domainlist_id = -1; if (config.privacylevel < PRIVACY_HIDE_DOMAINS) { diff --git a/src/database/gravity-db.c b/src/database/gravity-db.c index d4c5d6d0..f242995e 100644 --- a/src/database/gravity-db.c +++ b/src/database/gravity-db.c @@ -1204,7 +1204,6 @@ static enum db_result domain_in_list(const char *domain, sqlite3_stmt *stmt, con sqlite3_clear_bindings(stmt); // Return if domain was found in current table - // SELECT EXISTS(...) either returns 0 (false) or 1 (true). return (rc == SQLITE_ROW) ? FOUND : NOT_FOUND; } From 8035ae42f8fd5b4d24a9017c474cdfec4a92d007 Mon Sep 17 00:00:00 2001 From: a1346054 <36859588+a1346054@users.noreply.github.com> Date: Mon, 12 Sep 2022 22:27:28 +0000 Subject: [PATCH 082/218] Replace nonstandard `which` `command -v` is the standardized replacement for `which` Signed-off-by: a1346054 <36859588+a1346054@users.noreply.github.com> --- build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sh b/build.sh index 5d9fab3f..e2fd1b56 100755 --- a/build.sh +++ b/build.sh @@ -49,7 +49,7 @@ cmake --build . -- -j $(nproc) # Otherwise, we simply copy the binary one level up if [[ -n "${install}" ]]; then echo "Installing pihole-FTL" - SUDO=$(which sudo) + SUDO=$(command -v sudo) ${SUDO} cmake --install . else echo "Copying compiled pihole-FTL binary to repository root" From 5dc4163fbad91c6ea69b26e92255637b83576447 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 17 Sep 2022 08:52:34 +0200 Subject: [PATCH 083/218] Fix handling of dnssec-retry queries. Signed-off-by: DL6ER --- src/dnsmasq_interface.c | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c index a3e508fa..9c864c43 100644 --- a/src/dnsmasq_interface.c +++ b/src/dnsmasq_interface.c @@ -134,10 +134,29 @@ void FTL_hook(unsigned int flags, const char *name, union all_addr *addr, char * // derive the real query type from the arg string unsigned short qtype = type; if(strcmp(arg, "dnssec-query[DNSKEY]") == 0) + { qtype = T_DNSKEY; + arg = (char*)"dnssec-query"; + } else if(strcmp(arg, "dnssec-query[DS]") == 0) + { qtype = T_DS; - arg = (char*)"dnssec-query"; + arg = (char*)"dnssec-query"; + } + else if(strcmp(arg, "dnssec-retry[DNSKEY]") == 0) + { + qtype = T_DNSKEY; + arg = (char*)"dnssec-retry"; + } + else if(strcmp(arg, "dnssec-retry[DS]") == 0) + { + qtype = T_DS; + arg = (char*)"dnssec-retry"; + } + else + { + arg = (char*)"dnssec-unknown"; + } _FTL_new_query(flags, name, NULL, arg, qtype, id, &edns, INTERNAL, file, line); // forwarded upstream (type is used to store the upstream port) From 2cd071547d439a12258f8bdd18b7b355ea6565d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sat, 17 Sep 2022 17:08:45 +0200 Subject: [PATCH 084/218] Only log disk space warning when usage increased MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- src/gc.c | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/src/gc.c b/src/gc.c index b2eff794..d2f36a33 100644 --- a/src/gc.c +++ b/src/gc.c @@ -75,17 +75,21 @@ time_t get_rate_limit_turnaround(const unsigned int rate_limit_count) return (time_t)config.rate_limit.interval*how_often - (time(NULL) - lastRateLimitCleaner); } -static void check_space(const char *file) +static int check_space(const char *file, int LastUsage) { if(config.check.disk == 0) - return; + return 0; int perc = 0; char buffer[64] = { 0 }; // Warn if space usage at the device holding the corresponding file - // exceeds the configured threshold - if((perc = get_filepath_usage(file, buffer)) > config.check.disk) + // exceeds the configured threshold and current usage is higher than + // usage in the last run (to prevent log spam) + perc = get_filepath_usage(file, buffer); + if(perc > config.check.disk && perc > LastUsage ) log_resource_shortage(-1.0, 0, -1, perc, file, buffer); + + return perc; } static void check_load(void) @@ -118,6 +122,10 @@ void *GC_thread(void *val) lastRateLimitCleaner = time(NULL); time_t lastResourceCheck = 0; + // Remember disk usage + int LastLogStorageUsage = 0; + int LastDBStorageUsage = 0; + // Run as long as this thread is not canceled while(!killed) { @@ -138,8 +146,12 @@ void *GC_thread(void *val) if(now - lastResourceCheck >= RCinterval) { check_load(); - check_space(FTLfiles.FTL_db); - check_space(FTLfiles.log); + int DBStorageUsage = check_space(FTLfiles.FTL_db, LastDBStorageUsage); + int LogStorageUsage = check_space(FTLfiles.log, LastLogStorageUsage); + + LastLogStorageUsage = LogStorageUsage; + LastDBStorageUsage = DBStorageUsage; + lastResourceCheck = now; } From 8750bf59bd355d7513a36df6c5e0d7da911aeb72 Mon Sep 17 00:00:00 2001 From: yubiuser Date: Sun, 18 Sep 2022 08:58:33 +0200 Subject: [PATCH 085/218] No intermediate variables Co-authored-by: DL6ER Signed-off-by: yubiuser --- src/gc.c | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/gc.c b/src/gc.c index d2f36a33..c10c5d60 100644 --- a/src/gc.c +++ b/src/gc.c @@ -146,12 +146,8 @@ void *GC_thread(void *val) if(now - lastResourceCheck >= RCinterval) { check_load(); - int DBStorageUsage = check_space(FTLfiles.FTL_db, LastDBStorageUsage); - int LogStorageUsage = check_space(FTLfiles.log, LastLogStorageUsage); - - LastLogStorageUsage = LogStorageUsage; - LastDBStorageUsage = DBStorageUsage; - + LastDBStorageUsage = check_space(FTLfiles.FTL_db, LastDBStorageUsage); + LastLogStorageUsage = check_space(FTLfiles.log, LastLogStorageUsage); lastResourceCheck = now; } From 9aa7506056e7168e52c4eade9321ad93537844d5 Mon Sep 17 00:00:00 2001 From: yubiuser Date: Sun, 18 Sep 2022 13:11:54 +0200 Subject: [PATCH 086/218] Revert "Do not run built workflow on dependabot pushes and PRs" --- .github/workflows/build.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index be2c7ce9..68563772 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -4,10 +4,7 @@ on: push: branches: - '**' - - '!dependabot/**' pull_request: - branches-ignore: - - 'dependabot/**' release: types: [published] From 8e4a7cda0539a06002cba06a6ab16667806d9b00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sun, 18 Sep 2022 13:49:09 +0200 Subject: [PATCH 087/218] Separate dependabot branch names with a hypthen to work around the dir depth restriction in the built workflow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- .github/dependabot.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 09e98c92..55f79c21 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -10,3 +10,6 @@ updates: target-branch: development reviewers: - "pi-hole/ftl-maintainers" + pull-request-branch-name: + # Separate sections of the branch name with a hyphen + separator: "-" From 597e98e7731eb5ff3b7b61a54057a10be47c8d6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Wed, 21 Sep 2022 12:02:42 +0200 Subject: [PATCH 088/218] Don't create FTL.port MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- src/api/socket.c | 23 ----------------------- src/api/socket.h | 1 - src/config.c | 6 ------ src/daemon.c | 5 ----- test/run.sh | 10 +++++----- test/test_suite.bats | 6 ------ 6 files changed, 5 insertions(+), 46 deletions(-) diff --git a/src/api/socket.c b/src/api/socket.c index e7312cfe..c5c34d55 100644 --- a/src/api/socket.c +++ b/src/api/socket.c @@ -32,29 +32,6 @@ // reattempt at connection succeeds. #define BACKLOG 5 -void saveport(int port) -{ - FILE *f; - // Open "w" for truncation/creating file - if((f = fopen(FTLfiles.port, "w")) == NULL) - { - // Opening failed (permissions, path does not exist, etc.) - logg("WARNING: Unable to write used port to file"); - logg(" (API might not find the port)"); - } - else if(port > 0) - { - // Save port to file - fprintf(f, "%i", port); - fclose(f); - } - else - { - // FTL is terminating: Leave file truncated - fclose(f); - } -} - static int bind_to_telnet_socket(const enum telnet_type type, const char *stype) { const int socketdescriptor = socket(type == TELNET_SOCK ? AF_LOCAL : (type == TELNETv4 ? AF_INET : AF_INET6), SOCK_STREAM, 0); diff --git a/src/api/socket.h b/src/api/socket.h index 4ee4404b..50b14f62 100644 --- a/src/api/socket.h +++ b/src/api/socket.h @@ -20,7 +20,6 @@ struct thread_info { const char *stype; }; -void saveport(int port); void close_unix_socket(bool unlink_file); void seom(const int sock, const bool istelnet); #define ssend(sock, format, ...) _ssend(sock, __FILE__, __FUNCTION__, __LINE__, format, ##__VA_ARGS__) diff --git a/src/config.c b/src/config.c index 798b7ddb..eae5cdb8 100644 --- a/src/config.c +++ b/src/config.c @@ -14,8 +14,6 @@ #include "log.h" // nice() #include -// saveport() -#include "api/socket.h" // argv_dnsmasq #include "args.h" @@ -340,10 +338,6 @@ void read_FTLconf(void) // PIDFILE getpath(fp, "PIDFILE", "/run/pihole-FTL.pid", &FTLfiles.pid); - // PORTFILE - getpath(fp, "PORTFILE", "/run/pihole-FTL.port", &FTLfiles.port); - saveport(config.port); - // SOCKETFILE getpath(fp, "SOCKETFILE", "/run/pihole/FTL.sock", &FTLfiles.socketfile); diff --git a/src/daemon.c b/src/daemon.c index d2589423..9624165a 100644 --- a/src/daemon.c +++ b/src/daemon.c @@ -14,8 +14,6 @@ #include "log.h" // sleepms() #include "timers.h" -// saveport() -#include "api/socket.h" // gravityDB_close() #include "database/gravity-db.h" // destroy_shmem() @@ -275,9 +273,6 @@ void cleanup(const int ret) unlock_shm(); } - // Empty API port file, port 0 = truncate file - saveport(0); - // Remove PID file removepid(); diff --git a/test/run.sh b/test/run.sh index 17901625..b3fcd883 100755 --- a/test/run.sh +++ b/test/run.sh @@ -15,7 +15,7 @@ fi while pidof -s pihole-FTL > /dev/null; do pid="$(pidof -s pihole-FTL)" echo "Terminating running pihole-FTL process with PID ${pid}" - kill $pid + kill "$pid" sleep 1 done @@ -26,8 +26,8 @@ rm -f /etc/pihole/gravity.db /etc/pihole/pihole-FTL.db /var/log/pihole/pihole.lo mkdir -p /home/pihole /etc/pihole /run/pihole /var/log/pihole echo "" > /var/log/pihole/FTL.log echo "" > /var/log/pihole/pihole.log -touch /run/pihole-FTL.pid /run/pihole-FTL.port dig.log ptr.log -chown pihole:pihole /etc/pihole /run/pihole /var/log/pihole/pihole.log /var/log/pihole/FTL.log /run/pihole-FTL.pid /run/pihole-FTL.port +touch /run/pihole-FTL.pid dig.log ptr.log +chown pihole:pihole /etc/pihole /run/pihole /var/log/pihole/pihole.log /var/log/pihole/FTL.log /run/pihole-FTL.pid # Copy binary into a location the new user pihole can access cp ./pihole-FTL /home/pihole/pihole-FTL @@ -115,10 +115,10 @@ if [[ $RET != 0 ]]; then fi # Kill pihole-FTL after having completed tests -kill $(pidof pihole-FTL) +kill "$(pidof pihole-FTL)" # Restore umask -umask $OLDUMASK +umask "$OLDUMASK" # Remove copied file rm /home/pihole/pihole-FTL diff --git a/test/test_suite.bats b/test/test_suite.bats index 081be7cc..e19b1cc3 100644 --- a/test/test_suite.bats +++ b/test/test_suite.bats @@ -1141,12 +1141,6 @@ [[ ${lines[0]} == "2" ]] } -@test "Port file exists and contains expected API port" { - run bash -c 'cat /run/pihole-FTL.port' - printf "%s\n" "${lines[@]}" - [[ ${lines[0]} == "4711" ]] -} - @test "LUA: Interpreter returns FTL version" { run bash -c './pihole-FTL lua -e "print(pihole.ftl_version())"' printf "%s\n" "${lines[@]}" From 9f9744dbccfa867851bda6265d76a06216322edf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Wed, 21 Sep 2022 12:06:59 +0200 Subject: [PATCH 089/218] Remove FTLtest.sh MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- FTLtest.sh | 41 ----------------------------------------- 1 file changed, 41 deletions(-) delete mode 100755 FTLtest.sh diff --git a/FTLtest.sh b/FTLtest.sh deleted file mode 100755 index b8a78b86..00000000 --- a/FTLtest.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env bash -# Pi-hole: A black hole for Internet advertisements -# (c) 2017 Pi-hole, LLC (https://pi-hole.net) -# Network-wide ad blocking via your own hardware. -# -# Simple speed test bench for FTL -# -# This file is copyright under the latest version of the EUPL. -# Please see LICENSE file for your rights under this license. - -function GetFTLData { - # Open connection to FTL - exec 3<>/dev/tcp/localhost/"$(cat /run/pihole-FTL.port)" - - # Test if connection is open - if { true >&3; } 2> /dev/null; then - # Send command to FTL - echo -e ">$1" >&3 - echo -e ">quit" >&3 - - # Read all input - if [ "$1" == "stats" ]; then - cat <&3 - else - cat <&3 &> /dev/null - fi - - # Close connection - exec 3>&- - exec 3<&- - fi -} - -echo "Getting statistics data (output to stdout)" -time GetFTLData "stats" - -echo "Getting over time data (output to /dev/null)" -time GetFTLData "overTime" - -echo "Getting all queries (output to /dev/null)" -time GetFTLData "getallqueries" From 8e3b67a0cd97d14c003e0d6208b01dfce2e140e7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 24 Sep 2022 10:17:29 +0000 Subject: [PATCH 090/218] Bump actions/stale from 5.1.1 to 6.0.0 Bumps [actions/stale](https://github.com/actions/stale) from 5.1.1 to 6.0.0. - [Release notes](https://github.com/actions/stale/releases) - [Changelog](https://github.com/actions/stale/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/stale/compare/v5.1.1...v6.0.0) --- updated-dependencies: - dependency-name: actions/stale dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/stale.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index bce8bbea..1610c6fc 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -13,7 +13,7 @@ jobs: issues: write steps: - - uses: actions/stale@v5.1.1 + - uses: actions/stale@v6.0.0 with: repo-token: ${{ secrets.GITHUB_TOKEN }} days-before-stale: 30 From d74f5e1b0e342f093726555bfa02fed9d9f1bb30 Mon Sep 17 00:00:00 2001 From: Simon Kelley Date: Fri, 9 Sep 2022 12:53:49 +0100 Subject: [PATCH 091/218] Fix a problem in overload handling. Sending the same query repeatedly to a dnsmasq instance which doesn't get replies from upstream will eventually hit the hard limit on frec_src structures and start gettin REFUSED replies. This is OK, except that since the queries are no longer being forwarded, an upstream server coming back doesn't reset the situation. If there is any other traffic, frec allocation will eventually delete the timed-out frec and get things moving again, but that's not guaranteed. To fix this we explicitly delete the frec once timed out in this case. Thanks to Filip Jenicek for noticing and characterising this problem. Signed-off-by: DL6ER --- src/dnsmasq/forward.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/dnsmasq/forward.c b/src/dnsmasq/forward.c index af9e0da0..f8235c8c 100644 --- a/src/dnsmasq/forward.c +++ b/src/dnsmasq/forward.c @@ -247,6 +247,14 @@ static int forward_query(int udpfd, union mysockaddr *udpaddr, if (!daemon->free_frec_src) { query_full(now, NULL); + /* This is tricky; if we're blasted with the same query + over and over, we'll end up taking this path each time + and never resetting until the frec gets deleted by + aging followed by the receipt of a different query. This + is a bit of a DoS vuln. Avoid by explicitly deleting the + frec once it expires. */ + if (difftime(now, forward->time) >= TIMEOUT) + free_frec(forward); goto reply; } From bf69b6a01b850681eb72f6c90647b1f516772fc4 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 26 Sep 2022 19:15:48 +0200 Subject: [PATCH 092/218] Update embedded dnsmasq to v2.87 Signed-off-by: DL6ER --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 53e96556..6ab1cc45 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -11,6 +11,6 @@ cmake_minimum_required(VERSION 2.8.12) project(PIHOLE_FTL C) -set(DNSMASQ_VERSION pi-hole-v2.87rc1) +set(DNSMASQ_VERSION pi-hole-v2.87) add_subdirectory(src) From b2c3d4725ccb4ae26321139802a6d2218c7af727 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 1 Oct 2022 13:27:00 +0200 Subject: [PATCH 093/218] Fix audit log SQL query Signed-off-by: DL6ER --- src/api/api.c | 2 +- src/database/gravity-db.c | 15 +++++++-------- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/src/api/api.c b/src/api/api.c index 2e807f33..a5f54e4e 100644 --- a/src/api/api.c +++ b/src/api/api.c @@ -293,7 +293,7 @@ void getTopDomains(const char *client_message, const int sock, const bool isteln continue; // Skip this domain if already audited - if(audit && in_auditlist(getstr(domain->domainpos)) > 0) + if(audit && in_auditlist(getstr(domain->domainpos)) == FOUND) { if(config.debug & DEBUG_API) logg("API: %s has been audited.", getstr(domain->domainpos)); diff --git a/src/database/gravity-db.c b/src/database/gravity-db.c index c9ae89e1..ebea1c4b 100644 --- a/src/database/gravity-db.c +++ b/src/database/gravity-db.c @@ -155,14 +155,13 @@ bool gravityDB_open(void) // also other domains starting in google.de, like // abcgoogle.de rc = sqlite3_prepare_v3(gravity_db, - "SELECT EXISTS(" - "SELECT domain, " - "CASE WHEN substr(domain, 1, 1) = '*' " // Does the database string start in '*' ? - "THEN '*' || substr(:input, - length(domain) + 1) " // If so: Crop the input domain and prepend '*' - "ELSE :input " // If not: Use input domain directly for comparison - "END matcher " - "FROM domain_audit WHERE matcher = domain" // Match where (modified) domain equals the database domain - ");", -1, SQLITE_PREPARE_PERSISTENT, &auditlist_stmt, NULL); + "SELECT domain, " + "CASE WHEN substr(domain, 1, 1) = '*' " // Does the database string start in '*' ? + "THEN '*' || substr(:input, - length(domain) + 1) " // If so: Crop the input domain and prepend '*' + "ELSE :input " // If not: Use input domain directly for comparison + "END matcher " + "FROM domain_audit WHERE matcher = domain" // Match where (modified) domain equals the database domain + ";", -1, SQLITE_PREPARE_PERSISTENT, &auditlist_stmt, NULL); if( rc != SQLITE_OK ) { From df0eeec53e6adfb7662699a4279071d35942b4fb Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 1 Oct 2022 13:34:01 +0200 Subject: [PATCH 094/218] Tests: Domain auditing, there are some non-approved domains after the tests Signed-off-by: DL6ER --- test/test_suite.bats | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/test_suite.bats b/test/test_suite.bats index e19b1cc3..a67df65f 100644 --- a/test/test_suite.bats +++ b/test/test_suite.bats @@ -523,6 +523,12 @@ [[ ${lines[@]} != *"google.com"* ]] } +@test "Domain auditing, ten non-approved domains are shown" { + run bash -c 'echo ">top-domains for audit >quit" | nc -v 127.0.0.1 4711 | wc -l' + printf "%s\n" "${lines[@]}" + [[ ${lines[1]} == "10" ]] +} + @test "Upstream Destinations reported correctly" { run bash -c 'echo ">forward-dest >quit" | nc -v 127.0.0.1 4711' printf "%s\n" "${lines[@]}" From bcdf65397d31b75d3d0562719a6641c4b3dc13d4 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 3 Oct 2022 15:03:47 +0200 Subject: [PATCH 095/218] Fix printed error message when binding fails Signed-off-by: DL6ER --- src/api/socket.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/api/socket.c b/src/api/socket.c index c5c34d55..93f03809 100644 --- a/src/api/socket.c +++ b/src/api/socket.c @@ -113,14 +113,14 @@ static int bind_to_telnet_socket(const enum telnet_type type, const char *stype) // Bind to socket if(bind(socketdescriptor, (struct sockaddr *) address, addrlen) < 0) { - logg("Error listening on Unix socket %s: %s (%i)", FTLfiles.socketfile, strerror(errno), errno); + logg("Error binding to %s telnet socket: %s (%i)", stype, strerror(errno), errno); return -1; } // The listen system call allows the process to listen on the socket for connections if(listen(socketdescriptor, BACKLOG) == -1) { - logg("Error listening on %s socket: %s (%i)", stype, strerror(errno), errno); + logg("Error listening on %s telnet socket: %s (%i)", stype, strerror(errno), errno); return -1; } From ef8c6d6929e16d347fb16c509b5bc203a804668b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 8 Oct 2022 10:19:16 +0000 Subject: [PATCH 096/218] Bump actions/stale from 6.0.0 to 6.0.1 Bumps [actions/stale](https://github.com/actions/stale) from 6.0.0 to 6.0.1. - [Release notes](https://github.com/actions/stale/releases) - [Changelog](https://github.com/actions/stale/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/stale/compare/v6.0.0...v6.0.1) --- updated-dependencies: - dependency-name: actions/stale dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/stale.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 1610c6fc..27db3ded 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -13,7 +13,7 @@ jobs: issues: write steps: - - uses: actions/stale@v6.0.0 + - uses: actions/stale@v6.0.1 with: repo-token: ${{ secrets.GITHUB_TOKEN }} days-before-stale: 30 From e24e162da4b9320ba45b82929a4bfa5903a6e006 Mon Sep 17 00:00:00 2001 From: RD WebDesign Date: Tue, 11 Oct 2022 00:53:27 -0300 Subject: [PATCH 097/218] Fix comment Signed-off-by: RD WebDesign --- src/database/gravity-db.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/database/gravity-db.c b/src/database/gravity-db.c index ebea1c4b..5a6e0ac4 100644 --- a/src/database/gravity-db.c +++ b/src/database/gravity-db.c @@ -152,7 +152,7 @@ bool gravityDB_open(void) // BUT NOT google.de itself // Example 3: *google.de // matches 'google.de' and all of its subdomains but - // also other domains starting in google.de, like + // also other domains ending in google.de, like // abcgoogle.de rc = sqlite3_prepare_v3(gravity_db, "SELECT domain, " @@ -1398,7 +1398,7 @@ void check_inaccessible_adlists(void) // check if any adlist was inaccessible in the last gravity run // if so, gravity stored `status` in the adlist table with // "3": List unavailable, Pi-hole used a local copy - // "4": List unavailable, there is no local copy available + // "4": List unavailable, there is no local copy available // Do not proceed when database is not available if(!gravityDB_opened && !gravityDB_open()) @@ -1408,7 +1408,7 @@ void check_inaccessible_adlists(void) } const char *querystr = "SELECT id, address FROM adlist WHERE status IN (3,4) AND enabled=1"; - + // Prepare query sqlite3_stmt *query_stmt; int rc = sqlite3_prepare_v2(gravity_db, querystr, -1, &query_stmt, NULL); From 46180a3b69b7d6e118332995614e9b5fd360d2bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sat, 15 Oct 2022 14:55:38 +0200 Subject: [PATCH 098/218] Switch SSH deploy action MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- .github/workflows/build.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 68563772..d518611d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -142,10 +142,10 @@ jobs: working-directory: ${{steps.download.outputs.download-path}} - name: Install SSH Key - uses: shimataro/ssh-key-action@v2.3.1 + uses: benoitchantre/setup-ssh-authentication-action@1.0.0 with: - key: ${{ secrets.SSH_KEY }} - known_hosts: ${{ secrets.KNOWN_HOSTS }} + private-key: ${{ secrets.SSH_KEY }} + known-hosts: ${{ secrets.KNOWN_HOSTS }} - name: Transfer Builds to Pi-hole server for pihole checkout env: From 226492d2a270dfa071e7a2b750b3b500b7eb7af2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 15 Oct 2022 16:03:23 +0000 Subject: [PATCH 099/218] Bump actions/checkout from 3.0.2 to 3.1.0 Bumps [actions/checkout](https://github.com/actions/checkout) from 3.0.2 to 3.1.0. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v3.0.2...v3.1.0) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 6 +++--- .github/workflows/codespell.yml | 2 +- .github/workflows/sync-back-to-dev.yml | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d518611d..fdd878f3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -24,7 +24,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v3.1.0 - name: "Calculate required variables" id: variables @@ -85,7 +85,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v3.0.2 + uses: actions/checkout@v3.1.0 - name: "Fix ownership of repository" run: chown -R root . @@ -128,7 +128,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v3.1.0 - name: Get Binaries built in previous jobs uses: actions/download-artifact@v3.0.0 diff --git a/.github/workflows/codespell.yml b/.github/workflows/codespell.yml index 69f6cc38..4930242c 100644 --- a/.github/workflows/codespell.yml +++ b/.github/workflows/codespell.yml @@ -10,7 +10,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3.0.2 + uses: actions/checkout@v3.1.0 - name: Spell-Checking uses: codespell-project/actions-codespell@master diff --git a/.github/workflows/sync-back-to-dev.yml b/.github/workflows/sync-back-to-dev.yml index dd66ebdc..f689ae36 100644 --- a/.github/workflows/sync-back-to-dev.yml +++ b/.github/workflows/sync-back-to-dev.yml @@ -11,7 +11,7 @@ jobs: name: Syncing branches steps: - name: Checkout - uses: actions/checkout@v3.0.2 + uses: actions/checkout@v3.1.0 - name: Opening pull request id: pull uses: tretuna/sync-branches@1.4.0 From fde80ee8a913573f0f589f6b2a7eb724101c4776 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sun, 16 Oct 2022 17:45:04 +0200 Subject: [PATCH 100/218] Remove deprecated set-output in build.yml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- .github/workflows/build.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index fdd878f3..36820b29 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -32,9 +32,9 @@ jobs: GIT_TAG=${{ github.event.release.tag_name }} # If GIT_TAG is set then GIT BRANCH should be "master", else set it from GITHUB_REF GIT_BRANCH=$([ -n "${GIT_TAG}" ] && echo "master" || echo "${GITHUB_REF#refs/*/}") - echo ::set-output name=GIT_BRANCH::${GIT_BRANCH} - echo ::set-output name=GIT_TAG::${GIT_TAG} - echo ::set-output name=OUTPUT_DIR::${GIT_TAG:-${GIT_BRANCH}} + echo "GIT_BRANCH=${GIT_BRANCH}" >> $GITHUB_OUTPUT + echo "GIT_TAG=${GIT_TAG}" >> $GITHUB_OUTPUT + echo "OUTPUT_DIR=${GIT_TAG:-${GIT_BRANCH}}" >> $GITHUB_OUTPUT - name: "Check git branch name depth" env: From 0a2fd03ea17cec598b62a50ed0e483fa16d02063 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 22 Oct 2022 10:16:21 +0000 Subject: [PATCH 101/218] Bump actions/upload-artifact from 3.1.0 to 3.1.1 Bumps [actions/upload-artifact](https://github.com/actions/upload-artifact) from 3.1.0 to 3.1.1. - [Release notes](https://github.com/actions/upload-artifact/releases) - [Commits](https://github.com/actions/upload-artifact/compare/v3.1.0...v3.1.1) --- updated-dependencies: - dependency-name: actions/upload-artifact dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 36820b29..14346413 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -116,7 +116,7 @@ jobs: - name: Upload artifacts to job for later processing if: github.event_name != 'pull_request' - uses: actions/upload-artifact@v3.1.0 + uses: actions/upload-artifact@v3.1.1 with: name: tmp-binary-storage path: '${{ matrix.bin_name }}*' From 9953317c5537eaf235ef0a155393d0b3d3c8c0eb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 22 Oct 2022 10:16:24 +0000 Subject: [PATCH 102/218] Bump actions/download-artifact from 3.0.0 to 3.0.1 Bumps [actions/download-artifact](https://github.com/actions/download-artifact) from 3.0.0 to 3.0.1. - [Release notes](https://github.com/actions/download-artifact/releases) - [Commits](https://github.com/actions/download-artifact/compare/v3.0.0...v3.0.1) --- updated-dependencies: - dependency-name: actions/download-artifact dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 36820b29..94f5ea54 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -131,7 +131,7 @@ jobs: uses: actions/checkout@v3.1.0 - name: Get Binaries built in previous jobs - uses: actions/download-artifact@v3.0.0 + uses: actions/download-artifact@v3.0.1 id: download with: name: tmp-binary-storage From 615b8b627947b3b52181011b7dc885ce4c75faa3 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 2 Nov 2022 20:33:37 +0000 Subject: [PATCH 103/218] Work around long-standing bug in gcc 12.x reported in #1424 Signed-off-by: DL6ER --- src/api/api.c | 2 +- src/args.c | 14 +++++++------- src/args.h | 12 ++++++------ src/datastructure.c | 2 +- src/datastructure.h | 2 +- 5 files changed, 16 insertions(+), 16 deletions(-) diff --git a/src/api/api.c b/src/api/api.c index a5f54e4e..5408150d 100644 --- a/src/api/api.c +++ b/src/api/api.c @@ -1550,7 +1550,7 @@ static bool listInterfaces(struct if_info **head, char default_iface[IF_NAMESIZE while ((dp = readdir(dfd)) != NULL) { // Skip "." and ".." - if(!dp->d_name || strcmp(dp->d_name, ".") == 0 || strcmp(dp->d_name, "..") == 0) + if(strcmp(dp->d_name, ".") == 0 || strcmp(dp->d_name, "..") == 0) continue; // Create new interface record diff --git a/src/args.c b/src/args.c index 823a9dab..94bab36b 100644 --- a/src/args.c +++ b/src/args.c @@ -403,26 +403,26 @@ void parse_args(int argc, char* argv[]) #define COL_PURPLE "\x1b[95m" // bright foreground color #define COL_CYAN "\x1b[96m" // bright foreground color -static inline bool __attribute__ ((const)) is_term(void) +static inline bool __attribute__ ((pure)) is_term(void) { // test whether STDOUT refers to a terminal return isatty(fileno(stdout)) == 1; } // Returns green [✓] -const char __attribute__ ((const)) *cli_tick(void) +const char __attribute__ ((pure)) *cli_tick(void) { return is_term() ? "["COL_GREEN"✓"COL_NC"]" : "[✓]"; } // Returns red [✗] -const char __attribute__ ((const)) *cli_cross(void) +const char __attribute__ ((pure)) *cli_cross(void) { return is_term() ? "["COL_RED"✗"COL_NC"]" : "[✗]"; } // Returns [i] -const char __attribute__ ((const)) *cli_info(void) +const char __attribute__ ((pure)) *cli_info(void) { return is_term() ? COL_BOLD"[i]"COL_NC : "[i]"; } @@ -434,19 +434,19 @@ const char __attribute__ ((const)) *cli_qst(void) } // Returns green "done!"" -const char __attribute__ ((const)) *cli_done(void) +const char __attribute__ ((pure)) *cli_done(void) { return is_term() ? COL_GREEN"done!"COL_NC : "done!"; } // Sets font to bold -const char __attribute__ ((const)) *cli_bold(void) +const char __attribute__ ((pure)) *cli_bold(void) { return is_term() ? COL_BOLD : ""; } // Resets font to normal -const char __attribute__ ((const)) *cli_normal(void) +const char __attribute__ ((pure)) *cli_normal(void) { return is_term() ? COL_NC : ""; } diff --git a/src/args.h b/src/args.h index fe031f9e..57371e4e 100644 --- a/src/args.h +++ b/src/args.h @@ -16,13 +16,13 @@ extern bool daemonmode, cli_mode, dnsmasq_debug; extern int argc_dnsmasq; extern const char ** argv_dnsmasq; -const char *cli_tick(void) __attribute__ ((const)); -const char *cli_cross(void) __attribute__ ((const)); -const char *cli_info(void) __attribute__ ((const)); +const char *cli_tick(void) __attribute__ ((pure)); +const char *cli_cross(void) __attribute__ ((pure)); +const char *cli_info(void) __attribute__ ((pure)); const char *cli_qst(void) __attribute__ ((const)); -const char *cli_done(void) __attribute__ ((const)); -const char *cli_bold(void) __attribute__ ((const)); -const char *cli_normal(void) __attribute__ ((const)); +const char *cli_done(void) __attribute__ ((pure)); +const char *cli_bold(void) __attribute__ ((pure)); +const char *cli_normal(void) __attribute__ ((pure)); // defined in dnsmasq_interface.c int check_struct_sizes(void); diff --git a/src/datastructure.c b/src/datastructure.c index 5b4d6deb..9a69b35c 100644 --- a/src/datastructure.c +++ b/src/datastructure.c @@ -41,7 +41,7 @@ void strtolower(char *str) } // creates a simple hash of a string that fits into a uint32_t -uint32_t hashStr(const char *s) +uint32_t __attribute__ ((pure)) hashStr(const char *s) { uint32_t hash = 0; // Jenkins' One-at-a-Time hash (http://www.burtleburtle.net/bob/hash/doobs.html) diff --git a/src/datastructure.h b/src/datastructure.h index a37db29c..9dd8c4a3 100644 --- a/src/datastructure.h +++ b/src/datastructure.h @@ -108,7 +108,7 @@ typedef struct { } DNSCacheData; void strtolower(char *str); -uint32_t hashStr(const char *s) __attribute__((const)); +uint32_t hashStr(const char *s) __attribute__((pure)); int findQueryID(const int id); int findUpstreamID(const char * upstream, const in_port_t port); int findDomainID(const char *domain, const bool count); From ca9f7c027fb78987a55fdc8433fe7019a1035796 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 2 Nov 2022 20:35:39 +0000 Subject: [PATCH 104/218] Remove create_new option in init_shmem() as it was always true Signed-off-by: DL6ER --- src/main.c | 2 +- src/shmem.c | 117 ++++++++++++++++++++++------------------------------ src/shmem.h | 5 +-- 3 files changed, 52 insertions(+), 72 deletions(-) diff --git a/src/main.c b/src/main.c index 0253a56b..a6717de2 100644 --- a/src/main.c +++ b/src/main.c @@ -60,7 +60,7 @@ int main (int argc, char* argv[]) handle_signals(); // Initialize shared memory - if(!init_shmem(true)) + if(!init_shmem()) { logg("Initialization of shared memory failed."); // Check if there is already a running FTL process diff --git a/src/shmem.c b/src/shmem.c index 01d94b7b..1f17d4a3 100644 --- a/src/shmem.c +++ b/src/shmem.c @@ -429,134 +429,118 @@ bool is_our_lock(void) return false; } -bool init_shmem(bool create_new) +bool init_shmem() { // Get kernel's page size pagesize = getpagesize(); /****************************** shared memory lock ******************************/ // Try to create shared memory object - shm_lock = create_shm(SHARED_LOCK_NAME, sizeof(ShmLock), create_new); + shm_lock = create_shm(SHARED_LOCK_NAME, sizeof(ShmLock)); if(shm_lock.ptr == NULL) return false; - shmLock = (ShmLock*) shm_lock.ptr; - if(create_new) - { - shmLock->lock.outer = create_mutex(); - shmLock->lock.inner = create_mutex(); - } + + shmLock = (ShmLock*)shm_lock.ptr; + shmLock->lock.outer = create_mutex(); + shmLock->lock.inner = create_mutex(); /****************************** shared counters struct ******************************/ // Try to create shared memory object - shm_counters = create_shm(SHARED_COUNTERS_NAME, sizeof(countersStruct), create_new); + shm_counters = create_shm(SHARED_COUNTERS_NAME, sizeof(countersStruct)); if(shm_counters.ptr == NULL) return false; + counters = (countersStruct*)shm_counters.ptr; /****************************** shared settings struct ******************************/ // Try to create shared memory object - shm_settings = create_shm(SHARED_SETTINGS_NAME, sizeof(ShmSettings), create_new); + shm_settings = create_shm(SHARED_SETTINGS_NAME, sizeof(ShmSettings)); if(shm_settings.ptr == NULL) return false; + shmSettings = (ShmSettings*)shm_settings.ptr; - if(create_new) - { - shmSettings->version = SHARED_MEMORY_VERSION; - shmSettings->global_shm_counter = 0; - } - else - { - if(shmSettings->version != SHARED_MEMORY_VERSION) - { - logg("Shared memory version mismatch, found %d, expected %d!", - shmSettings->version, SHARED_MEMORY_VERSION); - return false; - } - } + shmSettings->version = SHARED_MEMORY_VERSION; + shmSettings->global_shm_counter = 0; /****************************** shared strings buffer ******************************/ // Try to create shared memory object - shm_strings = create_shm(SHARED_STRINGS_NAME, STRINGS_ALLOC_STEP, create_new); + shm_strings = create_shm(SHARED_STRINGS_NAME, STRINGS_ALLOC_STEP); if(shm_strings.ptr == NULL) return false; - if(create_new) - { - counters->strings_MAX = shm_strings.size; - // Initialize shared string object with an empty string at position zero - ((char*)shm_strings.ptr)[0] = '\0'; - shmSettings->next_str_pos = 1; - } + counters->strings_MAX = shm_strings.size; + + // Initialize shared string object with an empty string at position zero + ((char*)shm_strings.ptr)[0] = '\0'; + shmSettings->next_str_pos = 1; /****************************** shared domains struct ******************************/ size_t size = get_optimal_object_size(sizeof(domainsData), 1); // Try to create shared memory object - shm_domains = create_shm(SHARED_DOMAINS_NAME, size*sizeof(domainsData), create_new); + shm_domains = create_shm(SHARED_DOMAINS_NAME, size*sizeof(domainsData)); if(shm_domains.ptr == NULL) return false; + domains = (domainsData*)shm_domains.ptr; - if(create_new) - counters->domains_MAX = size; + counters->domains_MAX = size; /****************************** shared clients struct ******************************/ size = get_optimal_object_size(sizeof(clientsData), 1); // Try to create shared memory object - shm_clients = create_shm(SHARED_CLIENTS_NAME, size*sizeof(clientsData), create_new); + shm_clients = create_shm(SHARED_CLIENTS_NAME, size*sizeof(clientsData)); if(shm_clients.ptr == NULL) return false; + clients = (clientsData*)shm_clients.ptr; - if(create_new) - counters->clients_MAX = size; + counters->clients_MAX = size; /****************************** shared upstreams struct ******************************/ size = get_optimal_object_size(sizeof(upstreamsData), 1); // Try to create shared memory object - shm_upstreams = create_shm(SHARED_UPSTREAMS_NAME, size*sizeof(upstreamsData), create_new); + shm_upstreams = create_shm(SHARED_UPSTREAMS_NAME, size*sizeof(upstreamsData)); if(shm_upstreams.ptr == NULL) return false; upstreams = (upstreamsData*)shm_upstreams.ptr; - if(create_new) - counters->upstreams_MAX = size; + + counters->upstreams_MAX = size; /****************************** shared queries struct ******************************/ // Try to create shared memory object - shm_queries = create_shm(SHARED_QUERIES_NAME, pagesize*sizeof(queriesData), create_new); + shm_queries = create_shm(SHARED_QUERIES_NAME, pagesize*sizeof(queriesData)); if(shm_queries.ptr == NULL) return false; queries = (queriesData*)shm_queries.ptr; - if(create_new) - counters->queries_MAX = pagesize; + + counters->queries_MAX = pagesize; /****************************** shared overTime struct ******************************/ size = get_optimal_object_size(sizeof(overTimeData), OVERTIME_SLOTS); // Try to create shared memory object - shm_overTime = create_shm(SHARED_OVERTIME_NAME, size*sizeof(overTimeData), create_new); + shm_overTime = create_shm(SHARED_OVERTIME_NAME, size*sizeof(overTimeData)); if(shm_overTime.ptr == NULL) return false; - if(create_new) - { - // set global pointer in overTime.c - overTime = (overTimeData*)shm_overTime.ptr; - } + + // set global pointer in overTime.c + overTime = (overTimeData*)shm_overTime.ptr; /****************************** shared DNS cache struct ******************************/ size = get_optimal_object_size(sizeof(DNSCacheData), 1); // Try to create shared memory object - shm_dns_cache = create_shm(SHARED_DNS_CACHE, size*sizeof(DNSCacheData), create_new); + shm_dns_cache = create_shm(SHARED_DNS_CACHE, size*sizeof(DNSCacheData)); if(shm_dns_cache.ptr == NULL) return false; + dns_cache = (DNSCacheData*)shm_dns_cache.ptr; - if(create_new) - counters->dns_cache_MAX = size; + counters->dns_cache_MAX = size; /****************************** shared per-client regex buffer ******************************/ size = pagesize; // Allocate one pagesize initially. This may be expanded later on // Try to create shared memory object - shm_per_client_regex = create_shm(SHARED_PER_CLIENT_REGEX, size, create_new); + shm_per_client_regex = create_shm(SHARED_PER_CLIENT_REGEX, size); if(shm_per_client_regex.ptr == NULL) return false; - if(create_new) - counters->per_client_regex_MAX = size; + + counters->per_client_regex_MAX = size; return true; } @@ -588,10 +572,9 @@ void destroy_shmem(void) /// /// \param name the name of the shared memory /// \param size the size to allocate -/// \param create_new true = delete old file, create new, false = connect to existing object or fail /// \return a structure with a pointer to the mounted shared memory. The pointer /// will always be valid, because if it failed FTL will have exited. -static SharedMemory create_shm(const char *name, const size_t size, bool create_new) +static SharedMemory create_shm(const char *name, const size_t size) { char df[64] = { 0 }; const int percentage = get_dev_shm_usage(df); @@ -608,21 +591,19 @@ static SharedMemory create_shm(const char *name, const size_t size, bool create_ .ptr = NULL }; - // O_RDWR: Open the object for read-write access (we need to be able to modify the locks) - // When creating a new shared memory object, we add to this - // - O_CREAT: Create the shared memory object if it does not exist. - // - O_EXCL: Return an error if a shared memory object with the given name already exists. - const int shm_oflags = create_new ? O_RDWR | O_CREAT | O_EXCL : O_RDWR; - - // Create the shared memory file in read/write mode with 600 permissions + // Create the shared memory file in read/write mode with 600 (u+rw) permissions + // and the following open flags: + // - O_RDWR: Open the object for read-write access (we need to be able to modify the locks) + // - O_CREAT: Create the shared memory object if it does not exist. + // - O_EXCL: Return an error if a shared memory object with the given name already exists. errno = 0; - const int fd = shm_open(sharedMemory.name, shm_oflags, S_IRUSR | S_IWUSR); + const int fd = shm_open(sharedMemory.name, O_RDWR | O_CREAT | O_EXCL, S_IRUSR | S_IWUSR); // Check for `shm_open` error if(fd == -1) { - logg("FATAL: create_shm(): Failed to %s shared memory object \"%s\": %s", - create_new ? "create" : "open", name, strerror(errno)); + logg("FATAL: create_shm(): Failed to create shared memory object \"%s\": %s", + name, strerror(errno)); return sharedMemory; } diff --git a/src/shmem.h b/src/shmem.h index 4532c6d2..86e4eada 100644 --- a/src/shmem.h +++ b/src/shmem.h @@ -57,10 +57,9 @@ extern countersStruct *counters; /// /// \param name the name of the shared memory /// \param size the size to allocate -/// \param create_new true = delete old file, create new, false = connect to existing object or fail /// \return a structure with a pointer to the mounted shared memory. The pointer /// will always be valid, because if it failed FTL will have exited. -static SharedMemory create_shm(const char *name, const size_t size, bool create_new); +static SharedMemory create_shm(const char *name, const size_t size); /// Reallocate shared memory /// @@ -99,7 +98,7 @@ void _unlock_log(const char* func, const int line, const char * file); /// Block until a lock can be obtained -bool init_shmem(bool create_new); +bool init_shmem(void); void destroy_shmem(void); size_t addstr(const char *str); #define getstr(pos) _getstr(pos, __FUNCTION__, __LINE__, __FILE__) From fc303aeba52cd2e742ea6cc2b62e3d6c8a5f13ac Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 2 Nov 2022 20:41:03 +0000 Subject: [PATCH 105/218] Read FTL config *before* initializing shared memory to ensure we are aware of a possible DEBUG_SHMEM flag when creating the shared memory segments Signed-off-by: DL6ER --- src/main.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main.c b/src/main.c index a6717de2..ab4c955a 100644 --- a/src/main.c +++ b/src/main.c @@ -55,6 +55,9 @@ int main (int argc, char* argv[]) logg("########## FTL started on %s! ##########", hostname()); log_FTL_version(false); + // Process pihole-FTL.conf + read_FTLconf(); + // Catch signals not handled by dnsmasq // We configure real-time signals later (after dnsmasq has forked) handle_signals(); @@ -68,9 +71,6 @@ int main (int argc, char* argv[]) return EXIT_FAILURE; } - // Process pihole-FTL.conf - read_FTLconf(); - // pihole-FTL should really be run as user "pihole" to not mess up with file permissions // print warning otherwise if(strcmp(username, "pihole") != 0) From 1d447acb5aaa3778405712f4e62efe13ed5f25f3 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 2 Nov 2022 20:49:01 +0000 Subject: [PATCH 106/218] Store and check shared memory ownership before resizing shared memory objects. The scenario we are resolving here is the following: FTL is already running and someone(/-thing) deletes all the shared memory objects. If another instance of FTL is then started, it doesn't know a previous process is running and creates new SHM objects for itself. Both processes can - in theory - run just fine without touching each other as they are both actually using *different* shared memory objects. You are right when you think this is bizarre. The reason is that the first one has the shared memory objects still mapped into memory, i.e., "deleting" them from disk only removes the visible file handles. The next FTL instance creates files with the very same name, however, they are also new and distict files, i.e., their memory will point elsewhere. Both instances can now run in parallel just fine *until* one of them needs to resize a shared memory objects. If one of the shared memory events now gets resized to a size larger than it was before BUT smaller than what the other FTL instance is expecting, the other instance will instantaneously crash with a SIGSEGV (Bus error). This commit resolves this by storing the PID of the SHM object creator in the settings object. Each FTL instance reloads this shared memory instance now before performing any potentially dangerous operation and checks if the shared memory files on disk are still owned by this process. If this is not the case, we are in serious trouble and exit immediately. This should allow the second instance (you could call it the "rightful owner" of the current existing SHM objects) a fairly good chance to never even notice this and continue to operate just fine. Signed-off-by: DL6ER --- src/shmem.c | 57 +++++++++++++++++++++++++++++++++++++++++++++++++++++ src/shmem.h | 1 + 2 files changed, 58 insertions(+) diff --git a/src/shmem.c b/src/shmem.c index 1f17d4a3..986211f5 100644 --- a/src/shmem.c +++ b/src/shmem.c @@ -28,6 +28,8 @@ #include "files.h" // log_resource_shortage() #include "database/message-table.h" +// check_running_FTL() +#include "procps.h" /// The version of shared memory used #define SHARED_MEMORY_VERSION 14 @@ -104,6 +106,7 @@ static ShmSettings *shmSettings = NULL; static int pagesize; static unsigned int local_shm_counter = 0; +static pid_t shmem_pid = 0; static size_t used_shmem = 0u; static size_t get_optimal_object_size(const size_t objsize, const size_t minsize); @@ -128,6 +131,56 @@ static int get_dev_shm_usage(char buffer[64]) return percentage; } +// Verify the PID stored during shared memory initialization is the same as ours +// (while we initialized the shared memory objects) +static void verify_shmem_pid(void) +{ + const int fd = shm_open(SHARED_SETTINGS_NAME, O_RDWR, S_IRUSR | S_IWUSR); + if(fd == -1) + { + logg("FATAL: verify_shmem_pid(): Failed to open shared memory object \"%s\": %s", + SHARED_SETTINGS_NAME, strerror(errno)); + exit(EXIT_FAILURE); + } + + // Get new pointer + void *shm = mmap(NULL, shm_settings.size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + + // Check for `mmap` error + if(shm == MAP_FAILED) + { + logg("FATAL: verify_shmem_pid(): Failed to map shared memory object \"%s\" (%i): %s", + SHARED_SETTINGS_NAME, fd, strerror(errno)); + exit(EXIT_FAILURE); + } + + // Close file descriptor after call to mmap + close(fd); + + // Unmap old memory region + if(munmap(shm_settings.ptr, shm_settings.size) < 0) + { + logg("FATAL: verify_shmem_pid(): Failed to unmap shared memory object \"%s\" (%i): %s", + SHARED_SETTINGS_NAME, fd, strerror(errno)); + exit(EXIT_FAILURE); + } + + // Update pointer + shmSettings = shm_settings.ptr = shm; + + // Compare the SHM's PID to the one we had when creating the SHM objects + if(shmSettings->pid == shmem_pid) + return; + + // If we reach here, we are in serious trouble. Terminating with error + // code is the most sensible thing we can do at this point + logg("FATAL: Shared memory is owned by a different process (PID %d)", + shmSettings->pid); + check_running_FTL(); + logg("Exiting now!"); + exit(EXIT_FAILURE); +} + // chown_shmem() changes the file ownership of a given shared memory object static bool chown_shmem(SharedMemory *sharedMemory, struct passwd *ent_pw) { @@ -461,6 +514,7 @@ bool init_shmem() shmSettings = (ShmSettings*)shm_settings.ptr; shmSettings->version = SHARED_MEMORY_VERSION; shmSettings->global_shm_counter = 0; + shmSettings->pid = shmem_pid = getpid(); /****************************** shared strings buffer ******************************/ // Try to create shared memory object @@ -867,6 +921,9 @@ static size_t get_optimal_object_size(const size_t objsize, const size_t minsize // Enlarge shared memory to be able to hold at least one new record void shm_ensure_size(void) { + // Verify shared memory ownership + verify_shmem_pid(); + if(counters->queries >= counters->queries_MAX-1) { // Have to reallocate shared memory diff --git a/src/shmem.h b/src/shmem.h index 86e4eada..909da726 100644 --- a/src/shmem.h +++ b/src/shmem.h @@ -26,6 +26,7 @@ typedef struct { typedef struct { int version; + pid_t pid; unsigned int global_shm_counter; unsigned int next_str_pos; } ShmSettings; From 1e39eddbf89bbd63d6ba92ab8181389bb3126d39 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 2 Nov 2022 20:52:09 +0000 Subject: [PATCH 107/218] Ensure DELAY_STARTUP can be interrupted by external signals (e.g. SIGTERM) when trying to restart FTL really early Signed-off-by: DL6ER --- src/daemon.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/daemon.c b/src/daemon.c index 9624165a..26fcee04 100644 --- a/src/daemon.c +++ b/src/daemon.c @@ -190,7 +190,12 @@ void delay_startup(void) // Sleep if requested by DELAY_STARTUP logg("Sleeping for %d seconds as requested by configuration ...", config.delay_startup); - sleep(config.delay_startup); + if(sleep(config.delay_startup) != 0) + { + logg("FATAL: Sleeping was interrupted by an external signal"); + cleanup(EXIT_FAILURE); + exit(EXIT_FAILURE); + } logg("Done sleeping, continuing startup of resolver..."); } From 612b31b7359746423a83ad8f15e102e131d18b30 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 2 Nov 2022 20:55:52 +0000 Subject: [PATCH 108/218] Write PID early on so systemd cannot be fooled during DELAY_STARTUP times. The PID in this file will later be overwritten after forking Signed-off-by: DL6ER --- src/main.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main.c b/src/main.c index ab4c955a..1f7727af 100644 --- a/src/main.c +++ b/src/main.c @@ -76,6 +76,10 @@ int main (int argc, char* argv[]) if(strcmp(username, "pihole") != 0) logg("WARNING: Starting pihole-FTL as user %s is not recommended", username); + // Write PID early on so systemd cannot be fooled during DELAY_STARTUP + // times. The PID in this file will later be overwritten after forking + savepid(); + // Delay startup (if requested) // Do this before reading the database to make this option not only // useful for interfaces that aren't ready but also for fake-hwclocks From a0909894c4316ddadc07af6979085fbdbaa59bff Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 3 Nov 2022 20:43:16 +0100 Subject: [PATCH 109/218] Adjust tests Signed-off-by: DL6ER --- src/dnsmasq_interface.c | 2 +- test/test_suite.bats | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c index 9c864c43..7506d168 100644 --- a/src/dnsmasq_interface.c +++ b/src/dnsmasq_interface.c @@ -3339,7 +3339,7 @@ int check_struct_sizes(void) result += check_one_struct("overTimeData", sizeof(overTimeData), 32, 24); result += check_one_struct("regexData", sizeof(regexData), 56, 44); result += check_one_struct("SharedMemory", sizeof(SharedMemory), 24, 12); - result += check_one_struct("ShmSettings", sizeof(ShmSettings), 12, 12); + result += check_one_struct("ShmSettings", sizeof(ShmSettings), 16, 16); result += check_one_struct("countersStruct", sizeof(countersStruct), 244, 244); result += check_one_struct("sqlite3_stmt_vec", sizeof(sqlite3_stmt_vec), 32, 16); diff --git a/test/test_suite.bats b/test/test_suite.bats index a67df65f..9ae69f2a 100644 --- a/test/test_suite.bats +++ b/test/test_suite.bats @@ -28,8 +28,8 @@ @test "Running a second instance is detected and prevented" { run bash -c 'su pihole -s /bin/sh -c "/home/pihole/pihole-FTL -f"' printf "%s\n" "${lines[@]}" - [[ ${lines[9]} == *"Initialization of shared memory failed." ]] - [[ ${lines[10]} == *"HINT: pihole-FTL is already running!"* ]] + [[ "${lines[@]}" == *"Initialization of shared memory failed."* ]] + [[ "${lines[@]}" == *"HINT: pihole-FTL is already running!"* ]] } @test "Starting tests without prior history" { From cd83c1108bf620c25d17ac4e053d06b14106fe18 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 5 Nov 2022 09:44:56 +0100 Subject: [PATCH 110/218] Include libnettle version in pihole-FTL -vv output Signed-off-by: DL6ER --- src/args.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/args.c b/src/args.c index 94bab36b..dee6e55f 100644 --- a/src/args.c +++ b/src/args.c @@ -13,6 +13,12 @@ #include "dnsmasq/dnsmasq.h" #undef __USE_XOPEN +#include +#if !defined(NETTLE_VERSION_MAJOR) +# define NETTLE_VERSION_MAJOR 2 +# define NETTLE_VERSION_MINOR 0 +#endif + #include "FTL.h" #include "args.h" #include "version.h" @@ -258,6 +264,10 @@ void parse_args(int argc, char* argv[]) printf("\n"); printf("******************************** LUA ********************************\n"); printf(LUA_COPYRIGHT"\n"); + printf("\n"); + printf("***************************** LIBNETTLE *****************************\n"); + printf("Version: %d.%d\n", NETTLE_VERSION_MAJOR, NETTLE_VERSION_MINOR); + printf("GMP: %s\n", NETTLE_USE_MINI_GMP ? "Mini" : "Full"); exit(EXIT_SUCCESS); } From 1429f9a25bbb1635e6cbaad0e0c6846291615c9c Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 5 Nov 2022 09:45:44 +0100 Subject: [PATCH 111/218] Do not check disk usage during tests Signed-off-by: DL6ER --- test/pihole-FTL.conf | 1 + 1 file changed, 1 insertion(+) diff --git a/test/pihole-FTL.conf b/test/pihole-FTL.conf index 2d562a67..1f941296 100644 --- a/test/pihole-FTL.conf +++ b/test/pihole-FTL.conf @@ -2,6 +2,7 @@ DEBUG_ALL=true RESOLVE_IPV4=no RESOLVE_IPV6=no CHECK_LOAD=false +CHECK_DISK=0 LOCAL_IPV4=10.100.0.10 LOCAL_IPV6=fe80::10 BLOCK_IPV4=10.100.0.11 From d318a8d020796c137c96025dc7015bfec51aa410 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 6 Nov 2022 08:46:56 +0100 Subject: [PATCH 112/218] Use ftl-build:v1.22 containers Signed-off-by: DL6ER --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 91652b08..e49cfd34 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -53,7 +53,7 @@ jobs: needs: smoke-tests - container: ghcr.io/pi-hole/ftl-build:v1.19-${{ matrix.arch }} + container: ghcr.io/pi-hole/ftl-build:v1.22-${{ matrix.arch }} strategy: fail-fast: false From 622b814393bda965cadf664313fa04ec4b809578 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Fri, 4 Nov 2022 16:55:49 +0100 Subject: [PATCH 113/218] Remove repo specific issue and pull request template MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- .github/ISSUE_TEMPLATE.md | 38 -------------------------------- .github/PULL_REQUEST_TEMPLATE.md | 18 --------------- 2 files changed, 56 deletions(-) delete mode 100644 .github/ISSUE_TEMPLATE.md delete mode 100644 .github/PULL_REQUEST_TEMPLATE.md diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md deleted file mode 100644 index cd1d1d32..00000000 --- a/.github/ISSUE_TEMPLATE.md +++ /dev/null @@ -1,38 +0,0 @@ - -**In raising this issue, I confirm the following (please check boxes, eg [X]) Failure to fill the template will close your issue:** - -- [] I have read and understood the [contributors guide](https://github.com/pi-hole/pi-hole/blob/master/CONTRIBUTING.md). -- [] The issue I am reporting can be *replicated* -- [] The issue I am reporting isn't a duplicate - -**How familiar are you with the codebase?:** - -_{replace this text with a number from 1 to 10, with 1 being not familiar, and 10 being very familiar}_ - ---- - -**[BUG | ISSUE] Expected Behaviour:** - - -**[BUG | ISSUE] Actual Behaviour:** - - -**[BUG | ISSUE] Steps to reproduce:** - -- -- -- -- - -**Log file output [if available]** - -`output` - -**Device specifics** - -Hardware Type: rPi, VPS, etc -OS: - - -_This template was created based on the work of [`udemy-dl`](https://github.com/nishad/udemy-dl/blob/master/LICENSE)._ - diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md deleted file mode 100644 index eaec3dc2..00000000 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ /dev/null @@ -1,18 +0,0 @@ -**By submitting this pull request, I confirm the following (please check boxes, eg [X]) _Failure to fill the template will close your PR_:** - -***Please submit all pull requests against the `development` branch. Failure to do so will delay or deny your request*** - -- [] I have read and understood the [contributors guide](https://github.com/pi-hole/pi-hole/blob/master/CONTRIBUTING.md). -- [] I have checked that [another pull request](https://github.com/pi-hole/FTL/pulls) for this purpose does not exist. -- [] I have considered, and confirmed that this submission will be valuable to others. -- [] I accept that this submission may not be used, and the pull request closed at the will of the maintainer. -- [] I give this submission freely, and claim no ownership to its content. - -**How familiar are you with the codebase?:** _{replace this text with a number from 1 to 10, with 1 being not familiar, and 10 being very familiar}_ - ---- - -_{replace this line with your pull request content}_ - - -_This template was created based on the work of [`udemy-dl`](https://github.com/nishad/udemy-dl/blob/master/LICENSE)._ From 027f9496ce5d0695ec0fbf8ac71660f6124397d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sun, 16 Oct 2022 14:40:31 +0200 Subject: [PATCH 114/218] Update README.MD MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- README.md | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 34876e72..add4fea6 100644 --- a/README.md +++ b/README.md @@ -26,15 +26,14 @@ FTLDNS (`pihole-FTL`) provides an interactive API and also generates statistics - **Interactive**: our API can be used to interface with your projects - **Insightful**: stats normally reserved inside of `dnsmasq` are made available so you can see what's really happening on your network - -# Official documentation +## Official documentation The official *FTL*DNS documentation can be found [here](https://docs.pi-hole.net/ftldns/). -# Installation +## Installation -FTLDNS (`pihole-FTL`) is installed by default when you choose to enable the Web interface when installing Pi-hole. +FTLDNS (`pihole-FTL`) is automatically installed when installing Pi-hole. -> IMPORTANT! +### IMPORTANT ->FTLDNS will _disable_ any existing installations of `dnsmasq`. This is because FTLDNS _is_ `dnsmasq` + Pi-hole's code, so both cannot run simultaneously. +>FTLDNS will *disable* any existing installations of `dnsmasq`. This is because FTLDNS *is* `dnsmasq` + Pi-hole's code, so both cannot run simultaneously. From 66ba5cfc4339ba0c9472966da1479b8337999ae0 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 9 Nov 2022 19:01:20 +0100 Subject: [PATCH 115/218] Tests: Use dnssec.works instead of verteiltesysteme.net Signed-off-by: DL6ER --- test/test_suite.bats | 78 ++++++++++++++++++++++---------------------- 1 file changed, 39 insertions(+), 39 deletions(-) diff --git a/test/test_suite.bats b/test/test_suite.bats index 9ae69f2a..95e85436 100644 --- a/test/test_suite.bats +++ b/test/test_suite.bats @@ -404,13 +404,13 @@ } @test "DNSSEC: SECURE domain is resolved" { - run bash -c "dig A sigok.verteiltesysteme.net @127.0.0.1" + run bash -c "dig A dnssec.works @127.0.0.1" printf "%s\n" "${lines[@]}" [[ ${lines[@]} == *"status: NOERROR"* ]] } @test "DNSSEC: BOGUS domain is rejected" { - run bash -c "dig A sigfail.verteiltesysteme.net @127.0.0.1" + run bash -c "dig A fail01.dnssec.works @127.0.0.1" printf "%s\n" "${lines[@]}" [[ ${lines[@]} == *"status: SERVFAIL"* ]] } @@ -419,18 +419,18 @@ run bash -c 'echo ">stats >quit" | nc -v 127.0.0.1 4711' printf "%s\n" "${lines[@]}" [[ ${lines[1]} == "domains_being_blocked 4" ]] - [[ ${lines[2]} == "dns_queries_today 51" ]] + [[ ${lines[2]} == "dns_queries_today 52" ]] [[ ${lines[3]} == "ads_blocked_today 13" ]] #[[ ${lines[4]} == "ads_percentage_today 7.792208" ]] - [[ ${lines[5]} == "unique_domains 38" ]] - [[ ${lines[6]} == "queries_forwarded 26" ]] + [[ ${lines[5]} == "unique_domains 37" ]] + [[ ${lines[6]} == "queries_forwarded 27" ]] [[ ${lines[7]} == "queries_cached 12" ]] # Clients ever seen is commented out as the CI may have # more devices in its ARP cache so testing against a fixed # number of clients may not work in all cases #[[ ${lines[8]} == "clients_ever_seen 8" ]] #[[ ${lines[9]} == "unique_clients 8" ]] - [[ ${lines[10]} == "dns_queries_all_types 51" ]] + [[ ${lines[10]} == "dns_queries_all_types 52" ]] [[ ${lines[11]} == "reply_UNKNOWN 0" ]] [[ ${lines[12]} == "reply_NODATA 0" ]] [[ ${lines[13]} == "reply_NXDOMAIN 1" ]] @@ -442,10 +442,10 @@ [[ ${lines[19]} == "reply_REFUSED 0" ]] [[ ${lines[20]} == "reply_NOTIMP 0" ]] [[ ${lines[21]} == "reply_OTHER 0" ]] - [[ ${lines[22]} == "reply_DNSSEC 5" ]] + [[ ${lines[22]} == "reply_DNSSEC 6" ]] [[ ${lines[23]} == "reply_NONE 0" ]] [[ ${lines[24]} == "reply_BLOB 10" ]] - [[ ${lines[25]} == "dns_queries_all_replies 51" ]] + [[ ${lines[25]} == "dns_queries_all_replies 52" ]] [[ ${lines[26]} == "privacy_level 0" ]] [[ ${lines[27]} == "status enabled" ]] [[ ${lines[28]} == "" ]] @@ -461,7 +461,7 @@ run bash -c 'echo ">top-clients >quit" | nc -v 127.0.0.1 4711' printf "%s\n" "${lines[@]}" [[ ${lines[1]} == "0 36 127.0.0.1 "* ]] - [[ ${lines[2]} == "1 5 :: "* ]] + [[ ${lines[2]} == "1 6 :: "* ]] [[ ${lines[3]} == "2 4 127.0.0.3 "* ]] [[ ${lines[4]} == "3 3 127.0.0.2 "* ]] [[ "${lines[@]}" == *"1 aliasclient-0 some-aliasclient"* ]] @@ -475,11 +475,12 @@ [[ "${lines[@]}" == *" 4 version.bind"* ]] [[ "${lines[@]}" == *" 4 regex1.ftl"* ]] [[ "${lines[@]}" == *" 3 a.ftl"* ]] + [[ "${lines[@]}" == *" 3 dnssec.works"* ]] [[ "${lines[@]}" == *" 2 blacklisted.ftl"* ]] [[ "${lines[@]}" == *" 2 aaaa.ftl"* ]] - [[ "${lines[@]}" == *" 2 net"* ]] - [[ "${lines[@]}" == *" 2 verteiltesysteme.net"* ]] + [[ "${lines[@]}" == *" 2 works"* ]] [[ "${lines[@]}" == *" 2 ftl"* ]] + [[ "${lines[@]}" == *" 2 fail01.dnssec.works"* ]] [[ "${lines[@]}" == *" 1 version.ftl"* ]] [[ "${lines[@]}" == *" 1 whitelisted.ftl"* ]] [[ "${lines[@]}" == *" 1 gravity-whitelisted.ftl"* ]] @@ -496,8 +497,6 @@ [[ "${lines[@]}" == *" 1 svcb.ftl"* ]] [[ "${lines[@]}" == *" 1 https.ftl"* ]] [[ "${lines[@]}" == *" 1 ."* ]] - [[ "${lines[@]}" == *" 1 sigok.verteiltesysteme.net"* ]] - [[ "${lines[@]}" == *" 1 sigfail.verteiltesysteme.net"* ]] } @test "Top Ads" { @@ -532,33 +531,33 @@ @test "Upstream Destinations reported correctly" { run bash -c 'echo ">forward-dest >quit" | nc -v 127.0.0.1 4711' printf "%s\n" "${lines[@]}" - [[ ${lines[1]} == "-3 25.49 blocked blocked" ]] - [[ ${lines[2]} == "-2 23.53 cached cached" ]] + [[ ${lines[1]} == "-3 25.00 blocked blocked" ]] + [[ ${lines[2]} == "-2 23.08 cached cached" ]] [[ ${lines[3]} == "-1 0.00 other other" ]] - [[ ${lines[4]} == "0 47.06 127.0.0.1#5555 127.0.0.1#5555" ]] - [[ ${lines[5]} == "1 3.92 127.0.0.1#5554 127.0.0.1#5554" ]] + [[ ${lines[4]} == "0 48.08 127.0.0.1#5555 127.0.0.1#5555" ]] + [[ ${lines[5]} == "1 3.85 127.0.0.1#5554 127.0.0.1#5554" ]] [[ ${lines[6]} == "" ]] } @test "Query Types reported correctly" { run bash -c 'echo ">querytypes >quit" | nc -v 127.0.0.1 4711' printf "%s\n" "${lines[@]}" - [[ ${lines[1]} == "A (IPv4): 50.98" ]] - [[ ${lines[2]} == "AAAA (IPv6): 7.84" ]] - [[ ${lines[3]} == "ANY: 1.96" ]] - [[ ${lines[4]} == "SRV: 1.96" ]] - [[ ${lines[5]} == "SOA: 1.96" ]] - [[ ${lines[6]} == "PTR: 1.96" ]] - [[ ${lines[7]} == "TXT: 11.76" ]] - [[ ${lines[8]} == "NAPTR: 1.96" ]] - [[ ${lines[9]} == "MX: 1.96" ]] - [[ ${lines[10]} == "DS: 3.92" ]] + [[ ${lines[1]} == "A (IPv4): 50.00" ]] + [[ ${lines[2]} == "AAAA (IPv6): 7.69" ]] + [[ ${lines[3]} == "ANY: 1.92" ]] + [[ ${lines[4]} == "SRV: 1.92" ]] + [[ ${lines[5]} == "SOA: 1.92" ]] + [[ ${lines[6]} == "PTR: 1.92" ]] + [[ ${lines[7]} == "TXT: 11.54" ]] + [[ ${lines[8]} == "NAPTR: 1.92" ]] + [[ ${lines[9]} == "MX: 1.92" ]] + [[ ${lines[10]} == "DS: 5.77" ]] [[ ${lines[11]} == "RRSIG: 0.00" ]] - [[ ${lines[12]} == "DNSKEY: 5.88" ]] - [[ ${lines[13]} == "NS: 1.96" ]] - [[ ${lines[14]} == "OTHER: 1.96" ]] - [[ ${lines[15]} == "SVCB: 1.96" ]] - [[ ${lines[16]} == "HTTPS: 1.96" ]] + [[ ${lines[12]} == "DNSKEY: 5.77" ]] + [[ ${lines[13]} == "NS: 1.92" ]] + [[ ${lines[14]} == "OTHER: 1.92" ]] + [[ ${lines[15]} == "SVCB: 1.92" ]] + [[ ${lines[16]} == "HTTPS: 1.92" ]] [[ ${lines[17]} == "" ]] } @@ -611,14 +610,15 @@ [[ ${lines[42]} == *" AAAA a-cname.ftl 127.0.0.1 9 2 3 "*" gravity.ftl -1 127.0.0.1#5555 \"\" \"41\""* ]] [[ ${lines[43]} == *" A aaaa-cname.ftl 127.0.0.1 9 2 3 "*" gravity-aaaa.ftl -1 127.0.0.1#5555 \"\" \"42\""* ]] [[ ${lines[44]} == *" AAAA aaaa-cname.ftl 127.0.0.1 9 2 3 "*" gravity-aaaa.ftl -1 127.0.0.1#5555 \"\" \"43\""* ]] - [[ ${lines[45]} == *" A sigok.verteiltesysteme.net 127.0.0.1 2 1 4 "*" N/A -1 127.0.0.1#5555 \"\" \"44\""* ]] - [[ ${lines[46]} == *" DS net :: 2 1 11 "*" N/A -1 127.0.0.1#5555 \"\" \"45\""* ]] + [[ ${lines[45]} == *" A dnssec.works 127.0.0.1 2 1 4 "*" N/A -1 127.0.0.1#5555 \"\" \"44\""* ]] + [[ ${lines[46]} == *" DS works :: 2 1 11 "*" N/A -1 127.0.0.1#5555 \"\" \"45\""* ]] [[ ${lines[47]} == *" DNSKEY . :: 2 1 11 "*" N/A -1 127.0.0.1#5555 \"\" \"46\""* ]] - [[ ${lines[48]} == *" DS verteiltesysteme.net :: 2 1 11 "*" N/A -1 127.0.0.1#5555 \"\" \"47\""* ]] - [[ ${lines[49]} == *" DNSKEY net :: 2 1 11 "*" N/A -1 127.0.0.1#5555 \"\" \"48\""* ]] - [[ ${lines[50]} == *" DNSKEY verteiltesysteme.net :: 2 1 11 "*" N/A -1 127.0.0.1#5555 \"\" \"49\""* ]] - [[ ${lines[51]} == *" A sigfail.verteiltesysteme.net 127.0.0.1 2 3 4 "*" N/A -1 127.0.0.1#5555 \"DNSKEY missing\" \"50\""* ]] - [[ ${lines[52]} == "" ]] + [[ ${lines[48]} == *" DS dnssec.works :: 2 1 11 "*" N/A -1 127.0.0.1#5555 \"\" \"47\""* ]] + [[ ${lines[49]} == *" DNSKEY works :: 2 1 11 "*" N/A -1 127.0.0.1#5555 \"\" \"48\""* ]] + [[ ${lines[50]} == *" DNSKEY dnssec.works :: 2 1 11 "*" N/A -1 127.0.0.1#5555 \"\" \"49\""* ]] + [[ ${lines[51]} == *" A fail01.dnssec.works 127.0.0.1 2 3 4 "*" N/A -1 127.0.0.1#5555 \"RRSIG missing\" \"50\""* ]] + [[ ${lines[52]} == *" DS fail01.dnssec.works :: 2 1 11 "*" N/A -1 127.0.0.1#5555 \"\" \"51\""* ]] + [[ ${lines[53]} == "" ]] } @test "Get all queries (domain filtered) shows expected content" { From f3bf5480c002eb6f77d893311a7f6642537dee9b Mon Sep 17 00:00:00 2001 From: Simon Kelley Date: Wed, 17 Aug 2022 15:33:15 +0100 Subject: [PATCH 116/218] Add --port-limit option. By default, when sending a query via random ports to multiple upstream servers or retrying a query dnsmasq will use a single random port for all the tries/retries. This option allows a larger number of ports to be used, which can increase robustness in certain network configurations. Note that increasing this to more than two or three can have security and resource implications and should only be done with understanding of those. Signed-off-by: DL6ER --- src/dnsmasq/dnsmasq.h | 1 + src/dnsmasq/forward.c | 55 +++++++++++++++++++++++++++++++++++-------- src/dnsmasq/option.c | 9 +++++++ 3 files changed, 55 insertions(+), 10 deletions(-) diff --git a/src/dnsmasq/dnsmasq.h b/src/dnsmasq/dnsmasq.h index 5cdb11c0..f7c26c64 100644 --- a/src/dnsmasq/dnsmasq.h +++ b/src/dnsmasq/dnsmasq.h @@ -1151,6 +1151,7 @@ extern struct daemon { int log_fac; /* log facility */ char *log_file; /* optional log file */ int max_logs; /* queue limit */ + int randport_limit; /* Maximum number of source ports for query. */ int cachesize, ftabsize; int port, query_port, min_port, max_port; unsigned long local_ttl, neg_ttl, max_ttl, min_cache_ttl, max_cache_ttl, auth_ttl, dhcp_ttl, use_dhcp_ttl; diff --git a/src/dnsmasq/forward.c b/src/dnsmasq/forward.c index f8235c8c..8d0144ad 100644 --- a/src/dnsmasq/forward.c +++ b/src/dnsmasq/forward.c @@ -2451,7 +2451,7 @@ int allocate_rfd(struct randfd_list **fdlp, struct server *serv) { static int finger = 0; int i, j = 0; - struct randfd_list *rfl; + struct randfd_list **up, *rfl, *found, **found_link; struct randfd *rfd = NULL; int fd = 0; @@ -2459,17 +2459,27 @@ int allocate_rfd(struct randfd_list **fdlp, struct server *serv) if (serv->sfd) return serv->sfd->fd; - /* existing suitable random port socket linked to this transaction? */ - for (rfl = *fdlp; rfl; rfl = rfl->next) + /* existing suitable random port socket linked to this transaction? + Find the last one in the list and count how many there are. */ + for (found = NULL, found_link = NULL, i = 0, up = fdlp, rfl = *fdlp; rfl; up = &rfl->next, rfl = rfl->next) if (server_isequal(serv, rfl->rfd->serv)) - return rfl->rfd->fd; + { + i++; + found = rfl; + found_link = up; + } - /* No. need new link. */ - if ((rfl = daemon->rfl_spare)) - daemon->rfl_spare = rfl->next; - else if (!(rfl = whine_malloc(sizeof(struct randfd_list)))) - return -1; - + /* We have the maximum number for this query already. Promote + the last one on the list to the head, to circulate them, + and return it. */ + if (found && i >= daemon->randport_limit) + { + *found_link = found->next; + found->next = *fdlp; + *fdlp = found; + return found->rfd->fd; + } + /* limit the number of sockets we have open to avoid starvation of (eg) TFTP. Once we have a reasonable number, randomness should be OK */ for (i = 0; i < daemon->numrrand; i++) @@ -2484,6 +2494,31 @@ int allocate_rfd(struct randfd_list **fdlp, struct server *serv) } break; } + + /* We've hit the global limit on sockets before hitting the query limit, + use an exsiting socket as source in that case. */ + if (!rfd && found) + { + *found_link = found->next; + found->next = *fdlp; + *fdlp = found; + return found->rfd->fd; + } + + /* No good existing. Need new link. */ + if ((rfl = daemon->rfl_spare)) + daemon->rfl_spare = rfl->next; + else if (!(rfl = whine_malloc(sizeof(struct randfd_list)))) + { + /* malloc failed, don't leak allocated sock */ + if (rfd) + { + close(rfd->fd); + rfd->refcount = 0; + } + + return -1; + } /* No free ones or cannot get new socket, grab an existing one */ if (!rfd) diff --git a/src/dnsmasq/option.c b/src/dnsmasq/option.c index a36dcfde..e96c4c08 100644 --- a/src/dnsmasq/option.c +++ b/src/dnsmasq/option.c @@ -185,6 +185,7 @@ struct myoption { #define LOPT_STRIP_MAC 372 #define LOPT_CONF_OPT 373 #define LOPT_CONF_SCRIPT 374 +#define LOPT_RANDPORT_LIM 375 #ifdef HAVE_GETOPT_LONG static const struct option opts[] = @@ -370,6 +371,7 @@ static const struct myoption opts[] = { "log-debug", 0, 0, LOPT_LOG_DEBUG }, { "umbrella", 2, 0, LOPT_UMBRELLA }, { "quiet-tftp", 0, 0, LOPT_QUIET_TFTP }, + { "port-limit", 1, 0, LOPT_RANDPORT_LIM }, { NULL, 0, 0, 0 } }; @@ -434,6 +436,7 @@ static struct { { 'P', ARG_ONE, "", gettext_noop("Maximum supported UDP packet size for EDNS.0 (defaults to %s)."), "*" }, { 'q', ARG_DUP, NULL, gettext_noop("Log DNS queries."), NULL }, { 'Q', ARG_ONE, "", gettext_noop("Force the originating port for upstream DNS queries."), NULL }, + { LOPT_RANDPORT_LIM, ARG_ONE, "#ports", gettext_noop("Set maximum number of random originating ports for a query."), NULL }, { 'R', OPT_NO_RESOLV, NULL, gettext_noop("Do NOT read resolv.conf."), NULL }, { 'r', ARG_DUP, "", gettext_noop("Specify path to resolv.conf (defaults to %s)."), RESOLVFILE }, { LOPT_SERVERS_FILE, ARG_ONE, "", gettext_noop("Specify path to file with server= options"), NULL }, @@ -3183,6 +3186,11 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma if (daemon->query_port == 0) daemon->osport = 1; break; + + case LOPT_RANDPORT_LIM: /* --port-limit */ + if (!atoi_check(arg, &daemon->randport_limit) || (daemon->randport_limit < 1)) + ret_err(gen_err); + break; case 'T': /* --local-ttl */ case LOPT_NEGTTL: /* --neg-ttl */ @@ -5498,6 +5506,7 @@ void read_opts(int argc, char **argv, char *compile_opts) daemon->soa_refresh = SOA_REFRESH; daemon->soa_retry = SOA_RETRY; daemon->soa_expiry = SOA_EXPIRY; + daemon->randport_limit = 1; #ifndef NO_ID add_txt("version.bind", "dnsmasq-" VERSION, 0 ); From c7a4c2dac403badbb081ed65bcc7dbfc99b005d9 Mon Sep 17 00:00:00 2001 From: Simon Kelley Date: Sun, 21 Aug 2022 18:07:17 +0100 Subject: [PATCH 117/218] Add --fast-dns-retry option. This gives dnsmasq the ability to originate retries for upstream DNS queries itself, rather than relying on the downstream client. This is most useful when doing DNSSEC over unreliable upstream network. It comes with some cost in memory usage and network bandwidth. Signed-off-by: DL6ER --- src/dnsmasq/dnsmasq.c | 13 ++-- src/dnsmasq/dnsmasq.h | 8 ++- src/dnsmasq/forward.c | 153 ++++++++++++++++++++++++++++++++---------- src/dnsmasq/option.c | 14 +++- src/dnsmasq/util.c | 9 +++ 5 files changed, 153 insertions(+), 44 deletions(-) diff --git a/src/dnsmasq/dnsmasq.c b/src/dnsmasq/dnsmasq.c index 02d0a4a6..0053d397 100644 --- a/src/dnsmasq/dnsmasq.c +++ b/src/dnsmasq/dnsmasq.c @@ -1068,19 +1068,20 @@ int main_dnsmasq (int argc, char **argv) while (!terminate) { - int timeout = -1; + int timeout = fast_retry(now); poll_reset(); /* Whilst polling for the dbus, or doing a tftp transfer, wake every quarter second */ - if (daemon->tftp_trans || - (option_bool(OPT_DBUS) && !daemon->dbus)) + if ((daemon->tftp_trans || (option_bool(OPT_DBUS) && !daemon->dbus)) && + (timeout == -1 || timeout > 250)) timeout = 250; - + /* Wake every second whilst waiting for DAD to complete */ - else if (is_dad_listeners()) + else if (is_dad_listeners() && + (timeout == -1 || timeout > 1000)) timeout = 1000; - + set_dns_listeners(); #ifdef HAVE_DBUS diff --git a/src/dnsmasq/dnsmasq.h b/src/dnsmasq/dnsmasq.h index f7c26c64..b0408b11 100644 --- a/src/dnsmasq/dnsmasq.h +++ b/src/dnsmasq/dnsmasq.h @@ -767,11 +767,12 @@ struct frec { unsigned short new_id; int forwardall, flags; time_t time; + u32 forward_timestamp; unsigned char *hash[HASH_SIZE]; -#ifdef HAVE_DNSSEC - int class, work_counter; struct blockdata *stash; /* Saved reply, whilst we validate */ size_t stash_len; +#ifdef HAVE_DNSSEC + int class, work_counter; struct frec *dependent; /* Query awaiting internally-generated DNSKEY or DS query */ struct frec *next_dependent; /* list of above. */ struct frec *blocking_query; /* Query which is blocking us. */ @@ -1196,6 +1197,7 @@ extern struct daemon { int dump_mask; unsigned long soa_sn, soa_refresh, soa_retry, soa_expiry; u32 metrics[__METRIC_MAX]; + int fast_retry_time; #ifdef HAVE_DNSSEC struct ds_config *ds; char *timestamp_file; @@ -1433,6 +1435,7 @@ int hostname_order(const char *a, const char *b); int hostname_isequal(const char *a, const char *b); int hostname_issubdomain(char *a, char *b); time_t dnsmasq_time(void); +u32 dnsmasq_milliseconds(void); int netmask_length(struct in_addr mask); int is_same_net(struct in_addr a, struct in_addr b, struct in_addr mask); int is_same_net_prefix(struct in_addr a, struct in_addr b, int prefix); @@ -1492,6 +1495,7 @@ int send_from(int fd, int nowild, char *packet, size_t len, void resend_query(void); int allocate_rfd(struct randfd_list **fdlp, struct server *serv); void free_rfds(struct randfd_list **fdlp); +int fast_retry(time_t now); /* network.c */ int indextoname(int fd, int index, char *name); diff --git a/src/dnsmasq/forward.c b/src/dnsmasq/forward.c index 8d0144ad..4dec5b5e 100644 --- a/src/dnsmasq/forward.c +++ b/src/dnsmasq/forward.c @@ -319,6 +319,13 @@ static int forward_query(int udpfd, union mysockaddr *udpaddr, goto reply; /* table full - flags == 0, return REFUSED */ + /* Keep copy of query if we're doing fast retry. */ + if (daemon->fast_retry_time != 0) + { + forward->stash = blockdata_alloc((char *)header, plen); + forward->stash_len = plen; + } + forward->frec_src.log_id = daemon->log_id; forward->frec_src.source = *udpaddr; forward->frec_src.orig_id = ntohs(header->id); @@ -366,14 +373,14 @@ static int forward_query(int udpfd, union mysockaddr *udpaddr, #ifdef HAVE_DNSSEC /* If we've already got an answer to this query, but we're awaiting keys for validation, there's no point retrying the query, retry the key query instead...... */ - if (forward->blocking_query) + while (forward->blocking_query) + forward = forward->blocking_query; + + if (forward->flags & (FREC_DNSKEY_QUERY | FREC_DS_QUERY)) { int is_sign; unsigned char *pheader; - while (forward->blocking_query) - forward = forward->blocking_query; - /* log_id should match previous DNSSEC query. */ daemon->log_display_id = forward->frec_src.log_id; @@ -563,7 +570,11 @@ static int forward_query(int udpfd, union mysockaddr *udpaddr, } if (forwarded || is_dnssec) - return 1; + { + if (daemon->fast_retry_time != 0) + forward->forward_timestamp = dnsmasq_milliseconds(); + return 1; + } /* could not send on, prepare to return */ header->id = htons(forward->frec_src.orig_id); @@ -602,6 +613,64 @@ static int forward_query(int udpfd, union mysockaddr *udpaddr, return 0; } +/* Check if any frecs need to do a retry, and action that if so. + Return time in milliseconds until he next retyr will be required, + or -1 if none. */ +int fast_retry(time_t now) +{ + struct frec *f; + int ret = -1; + + if (daemon->fast_retry_time != 0) + { + u32 millis = dnsmasq_milliseconds(); + + for (f = daemon->frec_list; f; f = f->next) + if (f->sentto && f->stash && difftime(now, f->time) < TIMEOUT) + { +#ifdef HAVE_DNSSEC + if (f->blocking_query) + continue; +#endif + /* t is milliseconds since last query sent. */ + int to_run, t = (int)(millis - f->forward_timestamp); + + if (t < daemon->fast_retry_time) + to_run = daemon->fast_retry_time - t; + else + { + unsigned char *udpsz; + unsigned short udp_size = PACKETSZ; /* default if no EDNS0 */ + struct dns_header *header = (struct dns_header *)daemon->packet; + + /* packet buffer overwritten */ + daemon->srv_save = NULL; + + blockdata_retrieve(f->stash, f->stash_len, (void *)header); + + /* UDP size already set in saved query. */ + if (find_pseudoheader(header, f->stash_len, NULL, &udpsz, NULL, NULL)) + GETSHORT(udp_size, udpsz); + + daemon->log_display_id = f->frec_src.log_id; + + if (option_bool(OPT_LOG)) + my_syslog(LOG_INFO, _("fast retry"), NULL); + + forward_query(-1, NULL, NULL, 0, header, f->stash_len, ((char *) header) + udp_size, now, f, + f->flags & FREC_AD_QUESTION, f->flags & FREC_DO_QUESTION); + + to_run = daemon->fast_retry_time; + } + + if (ret == -1 || ret > to_run) + ret = to_run; + } + + } + return ret; +} + static struct ipsets *domain_find_sets(struct ipsets *setlist, const char *domain) { /* Similar algorithm to search_servers. */ struct ipsets *ipset_pos, *ret = NULL; @@ -990,6 +1059,8 @@ static void dnssec_validate(struct frec *forward, struct dns_header *header, /* Save query for retransmission and de-dup */ new->stash = blockdata_alloc((char *)header, nn); new->stash_len = nn; + if (daemon->fast_retry_time != 0) + new->forward_timestamp = dnsmasq_milliseconds(); /* Don't resend this. */ daemon->srv_save = NULL; @@ -1107,7 +1178,7 @@ void reply_query(int fd, time_t now) if (daemon->ignore_addr && RCODE(header) == NOERROR && check_for_ignored_address(header, n)) return; - + /* Note: if we send extra options in the EDNS0 header, we can't recreate the query from the reply. */ if ((RCODE(header) == REFUSED || RCODE(header) == SERVFAIL) && @@ -1138,34 +1209,46 @@ void reply_query(int fd, time_t now) else #endif { - /* recreate query from reply */ - if ((pheader = find_pseudoheader(header, (size_t)n, &plen, &udpsz, &is_sign, NULL))) - GETSHORT(udp_size, udpsz); - - /* If the client provides an EDNS0 UDP size, use that to limit our reply. - (bounded by the maximum configured). If no EDNS0, then it - defaults to 512 */ - if (udp_size > daemon->edns_pktsz) - udp_size = daemon->edns_pktsz; - else if (udp_size < PACKETSZ) - udp_size = PACKETSZ; /* Sanity check - can't reduce below default. RFC 6891 6.2.3 */ - - if (!is_sign && - (nn = resize_packet(header, (size_t)n, pheader, plen)) && - (forward->flags & FREC_DO_QUESTION)) - add_do_bit(header, nn, (unsigned char *)pheader + plen); - - header->ancount = htons(0); - header->nscount = htons(0); - header->arcount = htons(0); - header->hb3 &= ~(HB3_QR | HB3_AA | HB3_TC); - header->hb4 &= ~(HB4_RA | HB4_RCODE | HB4_CD | HB4_AD); - if (forward->flags & FREC_CHECKING_DISABLED) - header->hb4 |= HB4_CD; - if (forward->flags & FREC_AD_QUESTION) - header->hb4 |= HB4_AD; + /* in fast retry mode, we have a copy of the query. */ + if (daemon->fast_retry_time != 0 && forward->stash) + { + blockdata_retrieve(forward->stash, forward->stash_len, (void *)header); + nn = forward->stash_len; + /* UDP size already set in saved query. */ + if (find_pseudoheader(header, (size_t)n, NULL, &udpsz, NULL, NULL)) + GETSHORT(udp_size, udpsz); + } + else + { + /* recreate query from reply */ + if ((pheader = find_pseudoheader(header, (size_t)n, &plen, &udpsz, &is_sign, NULL))) + GETSHORT(udp_size, udpsz); + + /* If the client provides an EDNS0 UDP size, use that to limit our reply. + (bounded by the maximum configured). If no EDNS0, then it + defaults to 512 */ + if (udp_size > daemon->edns_pktsz) + udp_size = daemon->edns_pktsz; + else if (udp_size < PACKETSZ) + udp_size = PACKETSZ; /* Sanity check - can't reduce below default. RFC 6891 6.2.3 */ + + if (!is_sign && + (nn = resize_packet(header, (size_t)n, pheader, plen)) && + (forward->flags & FREC_DO_QUESTION)) + add_do_bit(header, nn, (unsigned char *)pheader + plen); + + header->ancount = htons(0); + header->nscount = htons(0); + header->arcount = htons(0); + header->hb3 &= ~(HB3_QR | HB3_AA | HB3_TC); + header->hb4 &= ~(HB4_RA | HB4_RCODE | HB4_CD | HB4_AD); + if (forward->flags & FREC_CHECKING_DISABLED) + header->hb4 |= HB4_CD; + if (forward->flags & FREC_AD_QUESTION) + header->hb4 |= HB4_AD; + } } - + if (nn) { forward_query(-1, NULL, NULL, 0, header, nn, ((char *) header) + udp_size, now, forward, @@ -2644,13 +2727,13 @@ static void free_frec(struct frec *f) f->sentto = NULL; f->flags = 0; -#ifdef HAVE_DNSSEC if (f->stash) { blockdata_free(f->stash); f->stash = NULL; } - + +#ifdef HAVE_DNSSEC /* Anything we're waiting on is pointless now, too */ if (f->blocking_query) { diff --git a/src/dnsmasq/option.c b/src/dnsmasq/option.c index e96c4c08..0fe78c9f 100644 --- a/src/dnsmasq/option.c +++ b/src/dnsmasq/option.c @@ -186,6 +186,7 @@ struct myoption { #define LOPT_CONF_OPT 373 #define LOPT_CONF_SCRIPT 374 #define LOPT_RANDPORT_LIM 375 +#define LOPT_FAST_RETRY 376 #ifdef HAVE_GETOPT_LONG static const struct option opts[] = @@ -372,6 +373,7 @@ static const struct myoption opts[] = { "umbrella", 2, 0, LOPT_UMBRELLA }, { "quiet-tftp", 0, 0, LOPT_QUIET_TFTP }, { "port-limit", 1, 0, LOPT_RANDPORT_LIM }, + { "fast-dns-retry", 1, 0, LOPT_FAST_RETRY }, { NULL, 0, 0, 0 } }; @@ -450,6 +452,7 @@ static struct { { LOPT_MAXTTL, ARG_ONE, "", gettext_noop("Specify time-to-live in seconds for maximum TTL to send to clients."), NULL }, { LOPT_MAXCTTL, ARG_ONE, "", gettext_noop("Specify time-to-live ceiling for cache."), NULL }, { LOPT_MINCTTL, ARG_ONE, "", gettext_noop("Specify time-to-live floor for cache."), NULL }, + { LOPT_FAST_RETRY, ARG_ONE, "", gettext_noop("Retry DNS queries after this many milliseconds."), NULL}, { 'u', ARG_ONE, "", gettext_noop("Change to this user after startup. (defaults to %s)."), CHUSER }, { 'U', ARG_DUP, "set:,", gettext_noop("Map DHCP vendor class to tag."), NULL }, { 'v', 0, NULL, gettext_noop("Display dnsmasq version and copyright information."), NULL }, @@ -3226,6 +3229,15 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma daemon->local_ttl = (unsigned long)ttl; break; } + + case LOPT_FAST_RETRY: + { + int retry; + if (!atoi_check(arg, &retry) || retry < 500) + ret_err(gen_err); + daemon->fast_retry_time = retry; + break; + } #ifdef HAVE_DHCP case 'X': /* --dhcp-lease-max */ @@ -5507,7 +5519,7 @@ void read_opts(int argc, char **argv, char *compile_opts) daemon->soa_retry = SOA_RETRY; daemon->soa_expiry = SOA_EXPIRY; daemon->randport_limit = 1; - + #ifndef NO_ID add_txt("version.bind", "dnsmasq-" VERSION, 0 ); add_txt("authors.bind", "Simon Kelley", 0); diff --git a/src/dnsmasq/util.c b/src/dnsmasq/util.c index 140a3543..5165fd66 100644 --- a/src/dnsmasq/util.c +++ b/src/dnsmasq/util.c @@ -457,6 +457,15 @@ time_t dnsmasq_time(void) #endif } +u32 dnsmasq_milliseconds(void) +{ + struct timeval tv; + + gettimeofday(&tv, NULL); + + return (tv.tv_sec) * 1000 + (tv.tv_usec / 1000); +} + int netmask_length(struct in_addr mask) { int zero_count = 0; From cf06741178988fb5c5a477b6d4209f4956107e92 Mon Sep 17 00:00:00 2001 From: Simon Kelley Date: Mon, 29 Aug 2022 21:44:05 +0100 Subject: [PATCH 118/218] Add --use-stale-cache option. Signed-off-by: DL6ER --- src/dnsmasq/cache.c | 6 ++ src/dnsmasq/dnsmasq.c | 3 - src/dnsmasq/dnsmasq.h | 7 +- src/dnsmasq/forward.c | 197 ++++++++++++++++++++++++++++++------------ src/dnsmasq/option.c | 3 + src/dnsmasq/rfc1035.c | 90 +++++++++++++++---- 6 files changed, 226 insertions(+), 80 deletions(-) diff --git a/src/dnsmasq/cache.c b/src/dnsmasq/cache.c index 0d110861..bc01fea9 100644 --- a/src/dnsmasq/cache.c +++ b/src/dnsmasq/cache.c @@ -375,6 +375,10 @@ static int is_outdated_cname_pointer(struct crec *crecp) static int is_expired(time_t now, struct crec *crecp) { + /* Don't dump expired entries if we're using them, cache becomes strictly LRU in that case. */ + if (option_bool(OPT_STALE_CACHE)) + return 0; + if (crecp->flags & F_IMMORTAL) return 0; @@ -2124,6 +2128,8 @@ void _log_query(unsigned int flags, char *name, union all_addr *addr, char *arg, name = arg; verb = daemon->addrbuff; } + else if (flags & F_STALE) + source = "cached-stale"; else source = "cached"; diff --git a/src/dnsmasq/dnsmasq.c b/src/dnsmasq/dnsmasq.c index 0053d397..1762b887 100644 --- a/src/dnsmasq/dnsmasq.c +++ b/src/dnsmasq/dnsmasq.c @@ -2052,9 +2052,6 @@ static void check_dns_listeners(time_t now) FTL_TCP_worker_terminating(true); /**********************************************/ - shutdown(confd, SHUT_RDWR); - close(confd); - if (buff) free(buff); diff --git a/src/dnsmasq/dnsmasq.h b/src/dnsmasq/dnsmasq.h index b0408b11..0e243af1 100644 --- a/src/dnsmasq/dnsmasq.h +++ b/src/dnsmasq/dnsmasq.h @@ -286,7 +286,8 @@ struct event_desc { #define OPT_FILTER_AAAA 68 #define OPT_STRIP_ECS 69 #define OPT_STRIP_MAC 70 -#define OPT_LAST 71 +#define OPT_STALE_CACHE 71 +#define OPT_LAST 72 #define OPTION_BITS (sizeof(unsigned int)*8) #define OPTION_SIZE ( (OPT_LAST/OPTION_BITS)+((OPT_LAST%OPTION_BITS)!=0) ) @@ -520,6 +521,7 @@ struct crec { #define F_DOMAINSRV (1u<<28) #define F_RCODE (1u<<29) #define F_SRV (1u<<30) +#define F_STALE (1u<<31) #define UID_NONE 0 /* Values of uid in crecs with F_CONFIG bit set. */ @@ -1369,7 +1371,8 @@ void report_addresses(struct dns_header *header, size_t len, u32 mark); #endif size_t answer_request(struct dns_header *header, char *limit, size_t qlen, struct in_addr local_addr, struct in_addr local_netmask, - time_t now, int ad_reqd, int do_bit, int have_pseudoheader); + time_t now, int ad_reqd, int do_bit, int have_pseudoheader, + int *stale); int check_for_bogus_wildcard(struct dns_header *header, size_t qlen, char *name, time_t now); int check_for_ignored_address(struct dns_header *header, size_t qlen); diff --git a/src/dnsmasq/forward.c b/src/dnsmasq/forward.c index 4dec5b5e..3a50ed24 100644 --- a/src/dnsmasq/forward.c +++ b/src/dnsmasq/forward.c @@ -1232,11 +1232,6 @@ void reply_query(int fd, time_t now) else if (udp_size < PACKETSZ) udp_size = PACKETSZ; /* Sanity check - can't reduce below default. RFC 6891 6.2.3 */ - if (!is_sign && - (nn = resize_packet(header, (size_t)n, pheader, plen)) && - (forward->flags & FREC_DO_QUESTION)) - add_do_bit(header, nn, (unsigned char *)pheader + plen); - header->ancount = htons(0); header->nscount = htons(0); header->arcount = htons(0); @@ -1246,6 +1241,11 @@ void reply_query(int fd, time_t now) header->hb4 |= HB4_CD; if (forward->flags & FREC_AD_QUESTION) header->hb4 |= HB4_AD; + + if (!is_sign && + (nn = resize_packet(header, (size_t)n, pheader, plen)) && + (forward->flags & FREC_DO_QUESTION)) + add_do_bit(header, nn, (unsigned char *)pheader + plen); } } @@ -1392,10 +1392,6 @@ static void return_reply(time_t now, struct frec *forward, struct dns_header *he { header->id = htons(src->orig_id); -#ifdef HAVE_DUMPFILE - dump_packet_udp(DUMP_REPLY, daemon->packet, (size_t)nn, NULL, &src->source, src->fd); -#endif - #if defined(HAVE_CONNTRACK) && defined(HAVE_UBUS) if (option_bool(OPT_CMARK_ALST_EN)) { @@ -1409,14 +1405,20 @@ static void return_reply(time_t now, struct frec *forward, struct dns_header *he /* Pi-hole modification */ int first_ID = -1; - send_from(src->fd, option_bool(OPT_NOWILD) || option_bool (OPT_CLEVERBIND), daemon->packet, nn, - &src->source, &src->dest, src->iface); - - if (option_bool(OPT_EXTRALOG) && src != &forward->frec_src) + if (src->fd != -1) { - daemon->log_display_id = src->log_id; - daemon->log_source_addr = &src->source; - log_query(F_UPSTREAM, "query", NULL, "duplicate", 0); +#ifdef HAVE_DUMPFILE + dump_packet_udp(DUMP_REPLY, daemon->packet, (size_t)nn, NULL, &src->source, src->fd); +#endif + send_from(src->fd, option_bool(OPT_NOWILD) || option_bool (OPT_CLEVERBIND), daemon->packet, nn, + &src->source, &src->dest, src->iface); + + if (option_bool(OPT_EXTRALOG) && src != &forward->frec_src) + { + daemon->log_display_id = src->log_id; + daemon->log_source_addr = &src->source; + log_query(F_UPSTREAM, "query", NULL, "duplicate", 0); + } } /* Pi-hole modification */ FTL_multiple_replies(src->log_id, &first_ID); @@ -1512,7 +1514,7 @@ void receive_query(struct listener *listen, time_t now) /************ Pi-hole modification ************/ bool piholeblocked = false; /**********************************************/ - + /* packet buffer overwritten */ daemon->srv_save = NULL; @@ -1846,7 +1848,11 @@ void receive_query(struct listener *listen, time_t now) #endif else { + int stale; int ad_reqd = do_bit; + u16 hb3 = header->hb3, hb4 = header->hb4; + int fd = listen->fd; + /* RFC 6840 5.7 */ if (header->hb4 & HB4_AD) ad_reqd = 1; @@ -1882,7 +1888,7 @@ void receive_query(struct listener *listen, time_t now) /**********************************************/ m = answer_request(header, ((char *) header) + udp_size, (size_t)n, - dst_addr_4, netmask, now, ad_reqd, do_bit, have_pseudoheader); + dst_addr_4, netmask, now, ad_reqd, do_bit, have_pseudoheader, &stale); if (m >= 1) { @@ -1897,11 +1903,36 @@ void receive_query(struct listener *listen, time_t now) (char *)header, m, &source_addr, &dst_addr, if_index); daemon->metrics[METRIC_DNS_LOCAL_ANSWERED]++; } - else if (forward_query(listen->fd, &source_addr, &dst_addr, if_index, - header, (size_t)n, ((char *) header) + udp_size, now, NULL, ad_reqd, do_bit)) - daemon->metrics[METRIC_DNS_QUERIES_FORWARDED]++; - else - daemon->metrics[METRIC_DNS_LOCAL_ANSWERED]++; + + if (m == 0 || stale) + { + if (m != 0) + { + size_t plen; + + /* We answered with stale cache data, so forward the query anyway to + refresh that. Restore the query from the answer packet. */ + pheader = find_pseudoheader(header, (size_t)m, &plen, NULL, NULL, NULL); + + header->hb3 = hb3; + header->hb4 = hb4; + header->ancount = htons(0); + header->nscount = htons(0); + header->arcount = htons(0); + + m = resize_packet(header, m, pheader, plen); + + /* We've already answered the client, so don't send it the answer + when it comes back. */ + fd = -1; + } + + if (forward_query(fd, &source_addr, &dst_addr, if_index, + header, (size_t)n, ((char *) header) + udp_size, now, NULL, ad_reqd, do_bit)) + daemon->metrics[METRIC_DNS_QUERIES_FORWARDED]++; + else + daemon->metrics[METRIC_DNS_LOCAL_ANSWERED]++; + } } } @@ -2128,8 +2159,9 @@ unsigned char *tcp_request(int confd, time_t now, unsigned char *pheader; unsigned int mark = 0; int have_mark = 0; - int first, last; + int first, last, stale, do_stale = 0; unsigned int flags = 0; + u16 hb3, hb4; /************ Pi-hole modification ************/ bool piholeblocked = false; @@ -2188,13 +2220,37 @@ unsigned char *tcp_request(int confd, time_t now, { int ede = EDE_UNSET; - if (query_count == TCP_MAX_QUERIES || - !packet || - !read_write(confd, &c1, 1, 1) || !read_write(confd, &c2, 1, 1) || - !(size = c1 << 8 | c2) || - !read_write(confd, payload, size, 1)) - return packet; - + if (query_count == TCP_MAX_QUERIES) + return packet; + + if (do_stale) + { + size_t plen; + + /* We answered the last query with stale data. Now try and get fresh data. + Restore query from answer. */ + pheader = find_pseudoheader(header, m, &plen, NULL, NULL, NULL); + + header->hb3 = hb3; + header->hb4 = hb4; + header->ancount = htons(0); + header->nscount = htons(0); + header->arcount = htons(0); + + size = resize_packet(header, m, pheader, plen); + } + else + { + if (!read_write(confd, &c1, 1, 1) || !read_write(confd, &c2, 1, 1) || + !(size = c1 << 8 | c2) || + !read_write(confd, payload, size, 1)) + return packet; + + /* for stale-answer processing. */ + hb3 = header->hb3; + hb4 = header->hb4; + } + if (size < (int)sizeof(struct dns_header)) continue; @@ -2225,28 +2281,31 @@ unsigned char *tcp_request(int confd, time_t now, struct auth_zone *zone; #endif - log_query_mysockaddr(F_QUERY | F_FORWARD, daemon->namebuff, - &peer_addr, auth_dns ? "auth" : "query", qtype); - - piholeblocked = FTL_new_query(F_QUERY | F_FORWARD, daemon->namebuff, - &peer_addr, auth_dns ? "auth" : "query", qtype, daemon->log_display_id, &edns, TCP); - #ifdef HAVE_CONNTRACK is_single_query = 1; #endif - + + if (!do_stale) + { + log_query_mysockaddr(F_QUERY | F_FORWARD, daemon->namebuff, + &peer_addr, auth_dns ? "auth" : "query", qtype); + + piholeblocked = FTL_new_query(F_QUERY | F_FORWARD, daemon->namebuff, + &peer_addr, auth_dns ? "auth" : "query", qtype, daemon->log_display_id, &edns, TCP); + #ifdef HAVE_AUTH - /* find queries for zones we're authoritative for, and answer them directly */ - if (!auth_dns && !option_bool(OPT_LOCALISE)) - for (zone = daemon->auth_zones; zone; zone = zone->next) - if (in_zone(zone, daemon->namebuff, NULL)) - { - auth_dns = 1; - local_auth = 1; - break; - } + /* find queries for zones we're authoritative for, and answer them directly */ + if (!auth_dns && !option_bool(OPT_LOCALISE)) + for (zone = daemon->auth_zones; zone; zone = zone->next) + if (in_zone(zone, daemon->namebuff, NULL)) + { + auth_dns = 1; + local_auth = 1; + break; + } #endif + } } norebind = domain_no_rebind(daemon->namebuff); @@ -2326,11 +2385,14 @@ unsigned char *tcp_request(int confd, time_t now, else { /**********************************************/ + + if (do_stale) + m = 0; + else + /* m > 0 if answered from cache */ + m = answer_request(header, ((char *) header) + 65536, (size_t)size, + dst_addr_4, netmask, now, ad_reqd, do_bit, have_pseudoheader, &stale); - /* m > 0 if answered from cache */ - m = answer_request(header, ((char *) header) + 65536, (size_t)size, - dst_addr_4, netmask, now, ad_reqd, do_bit, have_pseudoheader); - /* Do this by steam now we're not in the select() loop */ check_log_writer(1); @@ -2451,6 +2513,9 @@ unsigned char *tcp_request(int confd, time_t now, /**********************************************/ } + if (do_stale) + break; + /* In case of local answer or no connections made. */ if (m == 0 && !piholeblocked) // Pi-hole modified to ensure we don't provide local answers when dropping the reply { @@ -2461,11 +2526,11 @@ unsigned char *tcp_request(int confd, time_t now, if (have_pseudoheader) { u16 swap = htons((u16)ede); - - if (ede != EDE_UNSET) - m = add_pseudoheader(header, m, ((unsigned char *) header) + 65536, daemon->edns_pktsz, EDNS0_OPTION_EDE, (unsigned char *)&swap, 2, do_bit, 0); - else - m = add_pseudoheader(header, m, ((unsigned char *) header) + 65536, daemon->edns_pktsz, 0, NULL, 0, do_bit, 0); + + if (ede != EDE_UNSET) + m = add_pseudoheader(header, m, ((unsigned char *) header) + 65536, daemon->edns_pktsz, EDNS0_OPTION_EDE, (unsigned char *)&swap, 2, do_bit, 0); + else + m = add_pseudoheader(header, m, ((unsigned char *) header) + 65536, daemon->edns_pktsz, 0, NULL, 0, do_bit, 0); } } @@ -2482,8 +2547,26 @@ unsigned char *tcp_request(int confd, time_t now, #endif if (!read_write(confd, packet, m + sizeof(u16), 0)) break; + + /* If we answered with stale data, this process will now try and get fresh data into + the cache then and cannot therefore accept new queries. Close the incoming + connection to signal that to the client. Then set do_stale and loop round + once more to try and get fresh data, after which we exit. */ + if (stale) + { + shutdown(confd, SHUT_RDWR); + close(confd); + do_stale = 1; + } } - + + /* If we ran once to get fresh data, confd is already closed. */ + if (!do_stale) + { + shutdown(confd, SHUT_RDWR); + close(confd); + } + return packet; } diff --git a/src/dnsmasq/option.c b/src/dnsmasq/option.c index 0fe78c9f..4df68b13 100644 --- a/src/dnsmasq/option.c +++ b/src/dnsmasq/option.c @@ -187,6 +187,7 @@ struct myoption { #define LOPT_CONF_SCRIPT 374 #define LOPT_RANDPORT_LIM 375 #define LOPT_FAST_RETRY 376 +#define LOPT_STALE_CACHE 377 #ifdef HAVE_GETOPT_LONG static const struct option opts[] = @@ -374,6 +375,7 @@ static const struct myoption opts[] = { "quiet-tftp", 0, 0, LOPT_QUIET_TFTP }, { "port-limit", 1, 0, LOPT_RANDPORT_LIM }, { "fast-dns-retry", 1, 0, LOPT_FAST_RETRY }, + { "use-stale-cache", 0, 0 , LOPT_STALE_CACHE }, { NULL, 0, 0, 0 } }; @@ -431,6 +433,7 @@ static struct { { 'M', ARG_DUP, "", gettext_noop("Specify BOOTP options to DHCP server."), NULL }, { 'n', OPT_NO_POLL, NULL, gettext_noop("Do NOT poll %s file, reload only on SIGHUP."), RESOLVFILE }, { 'N', OPT_NO_NEG, NULL, gettext_noop("Do NOT cache failed search results."), NULL }, + { LOPT_STALE_CACHE, OPT_STALE_CACHE, NULL, gettext_noop("Use expired cache data for faster reply."), NULL }, { 'o', OPT_ORDER, NULL, gettext_noop("Use nameservers strictly in the order given in %s."), RESOLVFILE }, { 'O', ARG_DUP, "", gettext_noop("Specify options to be sent to DHCP clients."), NULL }, { LOPT_FORCE, ARG_DUP, "", gettext_noop("DHCP option sent even if the client does not request it."), NULL}, diff --git a/src/dnsmasq/rfc1035.c b/src/dnsmasq/rfc1035.c index e9aeceff..3a0c03af 100644 --- a/src/dnsmasq/rfc1035.c +++ b/src/dnsmasq/rfc1035.c @@ -1373,8 +1373,15 @@ int add_resource_record(struct dns_header *header, char *limit, int *truncp, int #undef CHECK_LIMIT } +static int crec_isstale(struct crec *crecp, time_t now) +{ + return (!(crecp->flags & F_IMMORTAL)) && difftime(crecp->ttd, now) < 0; +} + static unsigned long crec_ttl(struct crec *crecp, time_t now) { + signed long ttl = difftime(crecp->ttd, now); + /* Return 0 ttl for DHCP entries, which might change before the lease expires, unless configured otherwise. */ @@ -1383,8 +1390,8 @@ static unsigned long crec_ttl(struct crec *crecp, time_t now) int conf_ttl = daemon->use_dhcp_ttl ? daemon->dhcp_ttl : daemon->local_ttl; /* Apply ceiling of actual lease length to configured TTL. */ - if (!(crecp->flags & F_IMMORTAL) && (crecp->ttd - now) < conf_ttl) - return crecp->ttd - now; + if (!(crecp->flags & F_IMMORTAL) && ttl < conf_ttl) + return ttl; return conf_ttl; } @@ -1393,9 +1400,13 @@ static unsigned long crec_ttl(struct crec *crecp, time_t now) if (crecp->flags & F_IMMORTAL) return crecp->ttd; + /* Stale cache entries. */ + if (ttl < 0) + return 0; + /* Return the Max TTL value if it is lower than the actual TTL */ - if (daemon->max_ttl == 0 || ((unsigned)(crecp->ttd - now) < daemon->max_ttl)) - return crecp->ttd - now; + if (daemon->max_ttl == 0 || ((unsigned)ttl < daemon->max_ttl)) + return ttl; else return daemon->max_ttl; } @@ -1408,7 +1419,8 @@ static int cache_validated(const struct crec *crecp) /* return zero if we can't answer from cache, or packet size if we can */ size_t answer_request(struct dns_header *header, char *limit, size_t qlen, struct in_addr local_addr, struct in_addr local_netmask, - time_t now, int ad_reqd, int do_bit, int have_pseudoheader) + time_t now, int ad_reqd, int do_bit, int have_pseudoheader, + int *stale) { char *name = daemon->namebuff; unsigned char *p, *ansp; @@ -1424,6 +1436,9 @@ size_t answer_request(struct dns_header *header, char *limit, size_t qlen, size_t len; int rd_bit = (header->hb3 & HB3_RD); + if (stale) + *stale = 0; + /* never answer queries with RD unset, to avoid cache snooping. */ if (ntohs(header->ancount) != 0 || ntohs(header->nscount) != 0 || @@ -1472,13 +1487,22 @@ size_t answer_request(struct dns_header *header, char *limit, size_t qlen, while (--count != 0 && (crecp = cache_find_by_name(NULL, name, now, F_CNAME | F_NXDOMAIN))) { char *cname_target; - + int stale_flag = 0; + + if (crec_isstale(crecp, now)) + { + if (stale) + *stale = 1; + + stale_flag = F_STALE; + } + if (crecp->flags & F_NXDOMAIN) { if (qtype == T_CNAME) { if (!dryrun) - log_query(crecp->flags, name, NULL, record_source(crecp->uid), 0); + log_query(stale_flag | crecp->flags, name, NULL, record_source(crecp->uid), 0); auth = 0; nxdomain = 1; ans = 1; @@ -1500,7 +1524,7 @@ size_t answer_request(struct dns_header *header, char *limit, size_t qlen, if (!dryrun) { - log_query(crecp->flags, name, NULL, record_source(crecp->uid), 0); + log_query(stale_flag | crecp->flags, name, NULL, record_source(crecp->uid), 0); if (add_resource_record(header, limit, &trunc, nameoffset, &ansp, crec_ttl(crecp, now), &nameoffset, T_CNAME, C_IN, "d", cname_target)) @@ -1669,22 +1693,33 @@ size_t answer_request(struct dns_header *header, char *limit, size_t qlen, { do { + int stale_flag = 0; + + if (crec_isstale(crecp, now)) + { + if (stale) + *stale = 1; + + stale_flag = F_STALE; + } + /* don't answer wildcard queries with data not from /etc/hosts or dhcp leases */ if (qtype == T_ANY && !(crecp->flags & (F_HOSTS | F_DHCP))) continue; + if (!(crecp->flags & F_DNSSECOK)) sec_data = 0; - + ans = 1; - + if (crecp->flags & F_NEG) { auth = 0; if (crecp->flags & F_NXDOMAIN) nxdomain = 1; if (!dryrun) - log_query(crecp->flags & ~F_FORWARD, name, &addr, NULL, 0); + log_query(stale_flag | (crecp->flags & ~F_FORWARD), name, &addr, NULL, 0); } else { @@ -1692,9 +1727,9 @@ size_t answer_request(struct dns_header *header, char *limit, size_t qlen, auth = 0; if (!dryrun) { - log_query(crecp->flags & ~F_FORWARD, cache_get_name(crecp), &addr, + log_query(stale_flag | (crecp->flags & ~F_FORWARD), cache_get_name(crecp), &addr, record_source(crecp->uid), 0); - + if (add_resource_record(header, limit, &trunc, nameoffset, &ansp, crec_ttl(crecp, now), NULL, T_PTR, C_IN, "d", cache_get_name(crecp))) @@ -1801,7 +1836,7 @@ size_t answer_request(struct dns_header *header, char *limit, size_t qlen, if ((crecp = cache_find_by_name(NULL, name, now, flag | F_NXDOMAIN | (dryrun ? F_NO_RR : 0)))) { int localise = 0; - + /* See if a putative address is on the network from which we received the query, is so we'll filter other answers. */ if (local_addr.s_addr != 0 && option_bool(OPT_LOCALISE) && flag == F_IPV4) @@ -1823,6 +1858,16 @@ size_t answer_request(struct dns_header *header, char *limit, size_t qlen, (rd_bit && (!do_bit || cache_validated(crecp)) )) do { + int stale_flag = 0; + + if (crec_isstale(crecp, now)) + { + if (stale) + *stale = 1; + + stale_flag = F_STALE; + } + /* don't answer wildcard queries with data not from /etc/hosts or DHCP leases */ if (qtype == T_ANY && !(crecp->flags & (F_HOSTS | F_DHCP | F_CONFIG))) @@ -1840,7 +1885,7 @@ size_t answer_request(struct dns_header *header, char *limit, size_t qlen, if (!dryrun) // Pi-hole modification: Added record_source(crecp->uid) such that the subroutines know // where the reply came from (e.g. gravity.list) - log_query(crecp->flags, name, NULL, record_source(crecp->uid), 0); + log_query(stale_flag | crecp->flags, name, NULL, record_source(crecp->uid), 0); } else { @@ -1857,7 +1902,7 @@ size_t answer_request(struct dns_header *header, char *limit, size_t qlen, ans = 1; if (!dryrun) { - log_query(crecp->flags & ~F_REVERSE, name, &crecp->addr, + log_query(stale_flag | (crecp->flags & ~F_REVERSE), name, &crecp->addr, record_source(crecp->uid), 0); // ****************************** Pi-hole modification ****************************** const char *src = crecp != NULL ? crecp->flags & F_BIGNAME ? crecp->name.bname->name : crecp->name.sname : NULL; @@ -1981,6 +2026,15 @@ size_t answer_request(struct dns_header *header, char *limit, size_t qlen, rd_bit && (!do_bit || (option_bool(OPT_DNSSEC_VALID) && !(crecp->flags & F_DNSSECOK)))) do { + int stale_flag = 0; + + if (crec_isstale(crecp, now)) + { + if (stale) + *stale = 1; + + stale_flag = F_STALE; + } /* don't answer wildcard queries with data not from /etc/hosts or dhcp leases, except for NXDOMAIN */ if (qtype == T_ANY && !(crecp->flags & (F_NXDOMAIN))) break; @@ -1996,12 +2050,12 @@ size_t answer_request(struct dns_header *header, char *limit, size_t qlen, if (crecp->flags & F_NXDOMAIN) nxdomain = 1; if (!dryrun) - log_query(crecp->flags, name, NULL, NULL, 0); + log_query(stale_flag | crecp->flags, name, NULL, NULL, 0); } else if (!dryrun) { char *target = blockdata_retrieve(crecp->addr.srv.target, crecp->addr.srv.targetlen, NULL); - log_query(crecp->flags, name, NULL, NULL, 0); + log_query(stale_flag | crecp->flags, name, NULL, NULL, 0); if (add_resource_record(header, limit, &trunc, nameoffset, &ansp, crec_ttl(crecp, now), NULL, T_SRV, C_IN, "sssd", From 00f7c4e95c2c761211f211380a826cf1da4b4b80 Mon Sep 17 00:00:00 2001 From: Simon Kelley Date: Tue, 6 Sep 2022 22:09:21 +0100 Subject: [PATCH 119/218] Remove fast-retry development logging. Signed-off-by: DL6ER --- src/dnsmasq/forward.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/dnsmasq/forward.c b/src/dnsmasq/forward.c index 3a50ed24..b3791b9f 100644 --- a/src/dnsmasq/forward.c +++ b/src/dnsmasq/forward.c @@ -654,9 +654,6 @@ int fast_retry(time_t now) daemon->log_display_id = f->frec_src.log_id; - if (option_bool(OPT_LOG)) - my_syslog(LOG_INFO, _("fast retry"), NULL); - forward_query(-1, NULL, NULL, 0, header, f->stash_len, ((char *) header) + udp_size, now, f, f->flags & FREC_AD_QUESTION, f->flags & FREC_DO_QUESTION); From d4065af553a0197e240a9589d85a11a79ec1caac Mon Sep 17 00:00:00 2001 From: Simon Kelley Date: Fri, 9 Sep 2022 15:56:54 +0100 Subject: [PATCH 120/218] Fix namebuff overwrite leading to wrong log after socket bind warning. Signed-off-by: DL6ER --- src/dnsmasq/forward.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/dnsmasq/forward.c b/src/dnsmasq/forward.c index b3791b9f..02e699fb 100644 --- a/src/dnsmasq/forward.c +++ b/src/dnsmasq/forward.c @@ -2579,12 +2579,12 @@ static int random_sock(struct server *s) return fd; if (s->interface[0] == 0) - (void)prettyprint_addr(&s->source_addr, daemon->namebuff); + (void)prettyprint_addr(&s->source_addr, daemon->addrbuff); else - strcpy(daemon->namebuff, s->interface); + safe_strncpy(daemon->addrbuff, s->interface, ADDRSTRLEN); my_syslog(LOG_ERR, _("failed to bind server socket to %s: %s"), - daemon->namebuff, strerror(errno)); + daemon->addrbuff, strerror(errno)); close(fd); } From 281063af0c1e1a317336f8493874a32de41c59a3 Mon Sep 17 00:00:00 2001 From: Simon Kelley Date: Fri, 9 Sep 2022 17:09:32 +0100 Subject: [PATCH 121/218] Second try at port-limit option. 1) It's expected to fail to bind a new source port when they are scarce, suppress warning in log in this case. 2) Optimse bind_local when max_port - min_port is small. There's no randomness in this case, so we try all possible source ports rather than poking at random ones for an arbitrary number of tries. 3) In allocate_rfd() handle the case that all available source ports are already open. In this case we need to pick an existing socket/port to use, such that it has a different port from any we already hold. This gives the required property that the set of ports utilised by any given query is set by --port-limit and we don't re-use any until we have port-limit different ones. Signed-off-by: DL6ER --- src/dnsmasq/dnsmasq.h | 1 + src/dnsmasq/forward.c | 46 +++++++++++++++++++++++-------------------- src/dnsmasq/network.c | 13 ++++++++++-- src/dnsmasq/util.c | 13 ++++++++++++ 4 files changed, 50 insertions(+), 23 deletions(-) diff --git a/src/dnsmasq/dnsmasq.h b/src/dnsmasq/dnsmasq.h index 0e243af1..b02dd17e 100644 --- a/src/dnsmasq/dnsmasq.h +++ b/src/dnsmasq/dnsmasq.h @@ -1434,6 +1434,7 @@ void *whine_malloc(size_t size); void *whine_realloc(void *ptr, size_t size); int sa_len(union mysockaddr *addr); int sockaddr_isequal(const union mysockaddr *s1, const union mysockaddr *s2); +int sockaddr_isnull(const union mysockaddr *s); int hostname_order(const char *a, const char *b); int hostname_isequal(const char *a, const char *b); int hostname_issubdomain(char *a, char *b); diff --git a/src/dnsmasq/forward.c b/src/dnsmasq/forward.c index 02e699fb..1c4d103a 100644 --- a/src/dnsmasq/forward.c +++ b/src/dnsmasq/forward.c @@ -2578,13 +2578,18 @@ static int random_sock(struct server *s) if (local_bind(fd, &s->source_addr, s->interface, s->ifindex, 0)) return fd; - if (s->interface[0] == 0) - (void)prettyprint_addr(&s->source_addr, daemon->addrbuff); - else - safe_strncpy(daemon->addrbuff, s->interface, ADDRSTRLEN); - - my_syslog(LOG_ERR, _("failed to bind server socket to %s: %s"), - daemon->addrbuff, strerror(errno)); + /* don't log errors due to running out of available ports, we handle those. */ + if (!sockaddr_isnull(&s->source_addr) || errno != EADDRINUSE) + { + if (s->interface[0] == 0) + (void)prettyprint_addr(&s->source_addr, daemon->addrbuff); + else + safe_strncpy(daemon->addrbuff, s->interface, ADDRSTRLEN); + + my_syslog(LOG_ERR, _("failed to bind server socket to %s: %s"), + daemon->addrbuff, strerror(errno)); + } + close(fd); } @@ -2658,16 +2663,6 @@ int allocate_rfd(struct randfd_list **fdlp, struct server *serv) break; } - /* We've hit the global limit on sockets before hitting the query limit, - use an exsiting socket as source in that case. */ - if (!rfd && found) - { - *found_link = found->next; - found->next = *fdlp; - *fdlp = found; - return found->rfd->fd; - } - /* No good existing. Need new link. */ if ((rfl = daemon->rfl_spare)) daemon->rfl_spare = rfl->next; @@ -2692,10 +2687,19 @@ int allocate_rfd(struct randfd_list **fdlp, struct server *serv) server_isequal(serv, daemon->randomsocks[i].serv) && daemon->randomsocks[i].refcount != 0xfffe) { - finger = i + 1; - rfd = &daemon->randomsocks[i]; - rfd->refcount++; - break; + struct randfd_list *rl; + /* Don't pick one we already have. */ + for (rl = *fdlp; rl; rl = rl->next) + if (rl->rfd == &daemon->randomsocks[i]) + break; + + if (!rl) + { + finger = i + 1; + rfd = &daemon->randomsocks[i]; + rfd->refcount++; + break; + } } } diff --git a/src/dnsmasq/network.c b/src/dnsmasq/network.c index 239c3569..468deb76 100644 --- a/src/dnsmasq/network.c +++ b/src/dnsmasq/network.c @@ -1391,7 +1391,7 @@ int local_bind(int fd, union mysockaddr *addr, char *intname, unsigned int ifind or both are set. Otherwise use the OS's random ephemeral port allocation by leaving port == 0 and tries == 1 */ ports_avail = daemon->max_port - daemon->min_port + 1; - tries = ports_avail < 30 ? 3 * ports_avail : 100; + tries = (ports_avail < 30) ? ports_avail : 100; port = htons(daemon->min_port + (rand16() % ports_avail)); } @@ -1420,7 +1420,16 @@ int local_bind(int fd, union mysockaddr *addr, char *intname, unsigned int ifind if (--tries == 0) return 0; - port = htons(daemon->min_port + (rand16() % ports_avail)); + /* For small ranges, do a systematic search, not a random one. */ + if (ports_avail < 30) + { + unsigned short hport = ntohs(port); + if (hport++ == daemon->max_port) + hport = daemon->min_port; + port = htons(hport); + } + else + port = htons(daemon->min_port + (rand16() % ports_avail)); } if (!is_tcp && ifindex > 0) diff --git a/src/dnsmasq/util.c b/src/dnsmasq/util.c index 5165fd66..e0ce67d3 100644 --- a/src/dnsmasq/util.c +++ b/src/dnsmasq/util.c @@ -364,6 +364,19 @@ int sockaddr_isequal(const union mysockaddr *s1, const union mysockaddr *s2) return 0; } +int sockaddr_isnull(const union mysockaddr *s) +{ + if (s->sa.sa_family == AF_INET && + s->in.sin_addr.s_addr == 0) + return 1; + + if (s->sa.sa_family == AF_INET6 && + IN6_IS_ADDR_UNSPECIFIED(&s->in6.sin6_addr)) + return 1; + + return 0; +} + int sa_len(union mysockaddr *addr) { #ifdef HAVE_SOCKADDR_SA_LEN From 10dcef3ea9ced5285346bd61e1b50163c735a47d Mon Sep 17 00:00:00 2001 From: Simon Kelley Date: Fri, 9 Sep 2022 18:18:46 +0100 Subject: [PATCH 122/218] Further optimisation of --port-limit. No longer try and fail to open every port when the port range is in complete use; go straight to re-using an existing socket. Die at startup if port range is smaller than --port-limit, since the code behaves badly in this case. Signed-off-by: DL6ER --- src/dnsmasq/config.h | 1 + src/dnsmasq/dnsmasq.c | 4 +++ src/dnsmasq/forward.c | 70 +++++++++++++++++++++++++++++++++++-------- src/dnsmasq/network.c | 4 +-- 4 files changed, 64 insertions(+), 15 deletions(-) diff --git a/src/dnsmasq/config.h b/src/dnsmasq/config.h index 18429bf3..112c157e 100644 --- a/src/dnsmasq/config.h +++ b/src/dnsmasq/config.h @@ -24,6 +24,7 @@ #define KEYBLOCK_LEN 40 /* choose to minimise fragmentation when storing DNSSEC keys */ #define DNSSEC_WORK 50 /* Max number of queries to validate one question */ #define TIMEOUT 10 /* drop UDP queries after TIMEOUT seconds */ +#define SMALL_PORT_RANGE 30 /* If DNS port range is smaller than this, use different allocation. */ #define FORWARD_TEST 1000 /* try all servers every 1000 queries */ #define FORWARD_TIME 600 /* or 10 minutes */ #define UDP_TEST_TIME 60 /* How often to reset our idea of max packet size. */ diff --git a/src/dnsmasq/dnsmasq.c b/src/dnsmasq/dnsmasq.c index 1762b887..1d02462c 100644 --- a/src/dnsmasq/dnsmasq.c +++ b/src/dnsmasq/dnsmasq.c @@ -272,6 +272,10 @@ int main_dnsmasq (int argc, char **argv) if (daemon->max_port < daemon->min_port) die(_("max_port cannot be smaller than min_port"), NULL, EC_BADCONF); + + if (daemon->max_port != 0 && + daemon->max_port - daemon->min_port + 1 < daemon->randport_limit) + die(_("port_limit must not be larger than available port range"), NULL, EC_BADCONF); now = dnsmasq_time(); diff --git a/src/dnsmasq/forward.c b/src/dnsmasq/forward.c index 1c4d103a..4ad41db0 100644 --- a/src/dnsmasq/forward.c +++ b/src/dnsmasq/forward.c @@ -2575,6 +2575,21 @@ static int random_sock(struct server *s) if ((fd = socket(s->source_addr.sa.sa_family, SOCK_DGRAM, 0)) != -1) { + /* We need to set IPV6ONLY so we can use the same ports + for IPv4 and IPV6, otherwise, in restriced port situations, + we can end up with all our available ports in use for + one address family, and the other address family cannot be used. */ + if (s->source_addr.sa.sa_family == AF_INET6) + { + int opt = 1; + + if (setsockopt(fd, IPPROTO_IPV6, IPV6_V6ONLY, &opt, sizeof(opt)) == -1) + { + close(fd); + return -1; + } + } + if (local_bind(fd, &s->source_addr, s->interface, s->ifindex, 0)) return fd; @@ -2619,9 +2634,22 @@ int allocate_rfd(struct randfd_list **fdlp, struct server *serv) { static int finger = 0; int i, j = 0; + int ports_full = 0; struct randfd_list **up, *rfl, *found, **found_link; struct randfd *rfd = NULL; int fd = 0; + int ports_avail = 0; + + /* We can't have more randomsocks for this AF available than ports in our port range, + so check that here, to avoid trying and failing to bind every port + in local_bind(), called from random_sock(). The actual check is below when + ports_avail != 0 */ + if (daemon->max_port != 0) + { + ports_avail = daemon->max_port - daemon->min_port + 1; + if (ports_avail >= SMALL_PORT_RANGE) + ports_avail = 0; + } /* If server has a pre-allocated fd, use that. */ if (serv->sfd) @@ -2647,22 +2675,38 @@ int allocate_rfd(struct randfd_list **fdlp, struct server *serv) *fdlp = found; return found->rfd->fd; } + + /* check for all available ports in use. */ + if (ports_avail != 0) + { + int ports_inuse; + + for (ports_inuse = 0, i = 0; i < daemon->numrrand; i++) + if (daemon->randomsocks[i].refcount != 0 && + daemon->randomsocks[i].serv->source_addr.sa.sa_family == serv->source_addr.sa.sa_family && + ++ports_inuse >= ports_avail) + { + ports_full = 1; + break; + } + } /* limit the number of sockets we have open to avoid starvation of (eg) TFTP. Once we have a reasonable number, randomness should be OK */ - for (i = 0; i < daemon->numrrand; i++) - if (daemon->randomsocks[i].refcount == 0) - { - if ((fd = random_sock(serv)) != -1) - { - rfd = &daemon->randomsocks[i]; - rfd->serv = serv; - rfd->fd = fd; - rfd->refcount = 1; - } - break; - } - + if (!ports_full) + for (i = 0; i < daemon->numrrand; i++) + if (daemon->randomsocks[i].refcount == 0) + { + if ((fd = random_sock(serv)) != -1) + { + rfd = &daemon->randomsocks[i]; + rfd->serv = serv; + rfd->fd = fd; + rfd->refcount = 1; + } + break; + } + /* No good existing. Need new link. */ if ((rfl = daemon->rfl_spare)) daemon->rfl_spare = rfl->next; diff --git a/src/dnsmasq/network.c b/src/dnsmasq/network.c index 468deb76..f2ea24cc 100644 --- a/src/dnsmasq/network.c +++ b/src/dnsmasq/network.c @@ -1391,7 +1391,7 @@ int local_bind(int fd, union mysockaddr *addr, char *intname, unsigned int ifind or both are set. Otherwise use the OS's random ephemeral port allocation by leaving port == 0 and tries == 1 */ ports_avail = daemon->max_port - daemon->min_port + 1; - tries = (ports_avail < 30) ? ports_avail : 100; + tries = (ports_avail < SMALL_PORT_RANGE) ? ports_avail : 100; port = htons(daemon->min_port + (rand16() % ports_avail)); } @@ -1421,7 +1421,7 @@ int local_bind(int fd, union mysockaddr *addr, char *intname, unsigned int ifind return 0; /* For small ranges, do a systematic search, not a random one. */ - if (ports_avail < 30) + if (ports_avail < SMALL_PORT_RANGE) { unsigned short hport = ntohs(port); if (hport++ == daemon->max_port) From 7fe84aeba449ea9a25dc00bee9020b76f996d34d Mon Sep 17 00:00:00 2001 From: Simon Kelley Date: Mon, 12 Sep 2022 14:35:40 +0100 Subject: [PATCH 123/218] Add GetServerMetrics method to DBus interface. Signed-off-by: DL6ER --- src/dnsmasq/dbus.c | 57 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/src/dnsmasq/dbus.c b/src/dnsmasq/dbus.c index bf6b6613..c0770f37 100644 --- a/src/dnsmasq/dbus.c +++ b/src/dnsmasq/dbus.c @@ -91,6 +91,9 @@ const char* introspection_xml_template = " \n" " \n" " \n" +" \n" +" \n" +" \n" " \n" "\n"; @@ -644,6 +647,56 @@ static DBusMessage *dbus_get_metrics(DBusMessage* message) return reply; } +static void add_dict_entry(DBusMessageIter *container, const char *key, const char *val) +{ + DBusMessageIter dict; + + dbus_message_iter_open_container(container, DBUS_TYPE_DICT_ENTRY, NULL, &dict); + dbus_message_iter_append_basic(&dict, DBUS_TYPE_STRING, &key); + dbus_message_iter_append_basic(&dict, DBUS_TYPE_STRING, &val); + dbus_message_iter_close_container(container, &dict); +} + +static void add_dict_int(DBusMessageIter *container, const char *key, const unsigned int val) +{ + snprintf(daemon->namebuff, MAXDNAME, "%u", val); + + add_dict_entry(container, key, daemon->namebuff); +} + +static DBusMessage *dbus_get_server_metrics(DBusMessage* message) +{ + DBusMessage *reply = dbus_message_new_method_return(message); + DBusMessageIter server_array, dict_array, dict, server_iter, dict_iter; + struct server *serv; + + dbus_message_iter_init_append(reply, &server_iter); + dbus_message_iter_open_container(&server_iter, DBUS_TYPE_ARRAY, "a{ss}", &server_array); + + for (serv = daemon->servers; serv; serv = serv->next) + { + unsigned int port; + + dbus_message_iter_open_container(&server_array, DBUS_TYPE_ARRAY, "{ss}", &dict_array); + + port = prettyprint_addr(&serv->addr, daemon->namebuff); + add_dict_entry(&dict_array, "address", daemon->namebuff); + + add_dict_int(&dict_array, "port", port); + add_dict_int(&dict_array, "queries", serv->queries); + add_dict_int(&dict_array, "failed_queries", serv->failed_queries); + + if (strlen(serv->domain) != 0) + add_dict_entry(&dict_array, "domain", serv->domain); + + dbus_message_iter_close_container(&server_array, &dict_array); + } + + dbus_message_iter_close_container(&server_iter, &server_array); + + return reply; +} + DBusHandlerResult message_handler(DBusConnection *connection, DBusMessage *message, void *user_data) @@ -719,6 +772,10 @@ DBusHandlerResult message_handler(DBusConnection *connection, { reply = dbus_get_metrics(message); } + else if (strcmp(method, "GetServerMetrics") == 0) + { + reply = dbus_get_server_metrics(message); + } else if (strcmp(method, "ClearCache") == 0) clear_cache = 1; else From 8f2d7b5302fca95d7fa85855cf1e94ad59b9b170 Mon Sep 17 00:00:00 2001 From: Simon Kelley Date: Mon, 12 Sep 2022 14:50:17 +0100 Subject: [PATCH 124/218] Add stale cache replies to metrics. Signed-off-by: DL6ER --- src/dnsmasq/cache.c | 2 ++ src/dnsmasq/forward.c | 2 +- src/dnsmasq/metrics.c | 1 + src/dnsmasq/metrics.h | 1 + 4 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/dnsmasq/cache.c b/src/dnsmasq/cache.c index bc01fea9..6ec70d54 100644 --- a/src/dnsmasq/cache.c +++ b/src/dnsmasq/cache.c @@ -1769,6 +1769,8 @@ void dump_cache(time_t now) daemon->cachesize, daemon->metrics[METRIC_DNS_CACHE_LIVE_FREED], daemon->metrics[METRIC_DNS_CACHE_INSERTED]); my_syslog(LOG_INFO, _("queries forwarded %u, queries answered locally %u"), daemon->metrics[METRIC_DNS_QUERIES_FORWARDED], daemon->metrics[METRIC_DNS_LOCAL_ANSWERED]); + if (option_bool(OPT_STALE_CACHE)) + my_syslog(LOG_INFO, _("queries answered from stale cache %u"), daemon->metrics[METRIC_DNS_STALE_ANSWERED]); #ifdef HAVE_AUTH my_syslog(LOG_INFO, _("queries for authoritative zones %u"), daemon->metrics[METRIC_DNS_AUTH_ANSWERED]); #endif diff --git a/src/dnsmasq/forward.c b/src/dnsmasq/forward.c index 4ad41db0..e2a0353d 100644 --- a/src/dnsmasq/forward.c +++ b/src/dnsmasq/forward.c @@ -1898,7 +1898,7 @@ void receive_query(struct listener *listen, time_t now) #endif send_from(listen->fd, option_bool(OPT_NOWILD) || option_bool(OPT_CLEVERBIND), (char *)header, m, &source_addr, &dst_addr, if_index); - daemon->metrics[METRIC_DNS_LOCAL_ANSWERED]++; + daemon->metrics[stale ? METRIC_DNS_STALE_ANSWERED : METRIC_DNS_LOCAL_ANSWERED]++; } if (m == 0 || stale) diff --git a/src/dnsmasq/metrics.c b/src/dnsmasq/metrics.c index 68735293..eeea0749 100644 --- a/src/dnsmasq/metrics.c +++ b/src/dnsmasq/metrics.c @@ -22,6 +22,7 @@ const char * metric_names[] = { "dns_queries_forwarded", "dns_auth_answered", "dns_local_answered", + "dns_stale_answered", "bootp", "pxe", "dhcp_ack", diff --git a/src/dnsmasq/metrics.h b/src/dnsmasq/metrics.h index df72ec68..f7c817b4 100644 --- a/src/dnsmasq/metrics.h +++ b/src/dnsmasq/metrics.h @@ -21,6 +21,7 @@ enum { METRIC_DNS_QUERIES_FORWARDED, METRIC_DNS_AUTH_ANSWERED, METRIC_DNS_LOCAL_ANSWERED, + METRIC_DNS_STALE_ANSWERED, METRIC_BOOTP, METRIC_PXE, METRIC_DHCPACK, From 11119f7ed94b40312f5cf1f030de405f029f3b8b Mon Sep 17 00:00:00 2001 From: Simon Kelley Date: Mon, 12 Sep 2022 15:28:46 +0100 Subject: [PATCH 125/218] Return EDE_STALE extended error when returning stale data from cache. Signed-off-by: DL6ER --- src/dnsmasq/forward.c | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/dnsmasq/forward.c b/src/dnsmasq/forward.c index e2a0353d..38fec096 100644 --- a/src/dnsmasq/forward.c +++ b/src/dnsmasq/forward.c @@ -614,7 +614,7 @@ static int forward_query(int udpfd, union mysockaddr *udpaddr, } /* Check if any frecs need to do a retry, and action that if so. - Return time in milliseconds until he next retyr will be required, + Return time in milliseconds until he next retry will be required, or -1 if none. */ int fast_retry(time_t now) { @@ -1813,6 +1813,13 @@ void receive_query(struct listener *listen, time_t now) if (m >= 1) { + if (stale && have_pseudoheader) + { + u16 swap = htons(EDE_STALE); + + m = add_pseudoheader(header, m, ((unsigned char *) header) + udp_size, daemon->edns_pktsz, + EDNS0_OPTION_EDE, (unsigned char *)&swap, 2, do_bit, 0); + } #ifdef HAVE_DUMPFILE dump_packet_udp(DUMP_REPLY, daemon->packet, m, NULL, &source_addr, listen->fd); #endif @@ -2530,6 +2537,12 @@ unsigned char *tcp_request(int confd, time_t now, m = add_pseudoheader(header, m, ((unsigned char *) header) + 65536, daemon->edns_pktsz, 0, NULL, 0, do_bit, 0); } } + else if (stale) + { + u16 swap = htons((u16)EDE_STALE); + + m = add_pseudoheader(header, m, ((unsigned char *) header) + 65536, daemon->edns_pktsz, EDNS0_OPTION_EDE, (unsigned char *)&swap, 2, do_bit, 0); + } check_log_writer(1); From ee8072555a71f6fc922a774950b440e72d5d387c Mon Sep 17 00:00:00 2001 From: Simon Kelley Date: Tue, 13 Sep 2022 09:36:08 +0100 Subject: [PATCH 126/218] Remove unused vars. Signed-off-by: DL6ER --- src/dnsmasq/dbus.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dnsmasq/dbus.c b/src/dnsmasq/dbus.c index c0770f37..e0089fc3 100644 --- a/src/dnsmasq/dbus.c +++ b/src/dnsmasq/dbus.c @@ -667,7 +667,7 @@ static void add_dict_int(DBusMessageIter *container, const char *key, const unsi static DBusMessage *dbus_get_server_metrics(DBusMessage* message) { DBusMessage *reply = dbus_message_new_method_return(message); - DBusMessageIter server_array, dict_array, dict, server_iter, dict_iter; + DBusMessageIter server_array, dict_array, server_iter; struct server *serv; dbus_message_iter_init_append(reply, &server_iter); From 039e2b3711183f988978de5519b55524fe1b8a9a Mon Sep 17 00:00:00 2001 From: Simon Kelley Date: Wed, 14 Sep 2022 16:16:08 +0100 Subject: [PATCH 127/218] Make fast-retry more configurable and do exponential backoff. Signed-off-by: DL6ER --- src/dnsmasq/config.h | 1 + src/dnsmasq/dnsmasq.h | 3 ++- src/dnsmasq/forward.c | 15 +++++++++------ src/dnsmasq/option.c | 30 ++++++++++++++++++++++-------- 4 files changed, 34 insertions(+), 15 deletions(-) diff --git a/src/dnsmasq/config.h b/src/dnsmasq/config.h index 112c157e..3f6dd6ba 100644 --- a/src/dnsmasq/config.h +++ b/src/dnsmasq/config.h @@ -61,6 +61,7 @@ #define SOA_EXPIRY 1209600 /* SOA expiry default */ #define LOOP_TEST_DOMAIN "test" /* domain for loop testing, "test" is reserved by RFC 2606 and won't therefore clash */ #define LOOP_TEST_TYPE T_TXT +#define DEFAULT_FAST_RETRY 1000 /* ms, default delay before fast retry */ /* compile-time options: uncomment below to enable or do eg. make COPTS=-DHAVE_BROKEN_RTC diff --git a/src/dnsmasq/dnsmasq.h b/src/dnsmasq/dnsmasq.h index b02dd17e..e17d13f1 100644 --- a/src/dnsmasq/dnsmasq.h +++ b/src/dnsmasq/dnsmasq.h @@ -770,6 +770,7 @@ struct frec { int forwardall, flags; time_t time; u32 forward_timestamp; + int forward_delay; unsigned char *hash[HASH_SIZE]; struct blockdata *stash; /* Saved reply, whilst we validate */ size_t stash_len; @@ -1199,7 +1200,7 @@ extern struct daemon { int dump_mask; unsigned long soa_sn, soa_refresh, soa_retry, soa_expiry; u32 metrics[__METRIC_MAX]; - int fast_retry_time; + int fast_retry_time, fast_retry_timeout; #ifdef HAVE_DNSSEC struct ds_config *ds; char *timestamp_file; diff --git a/src/dnsmasq/forward.c b/src/dnsmasq/forward.c index 38fec096..f4ef63ae 100644 --- a/src/dnsmasq/forward.c +++ b/src/dnsmasq/forward.c @@ -626,7 +626,7 @@ int fast_retry(time_t now) u32 millis = dnsmasq_milliseconds(); for (f = daemon->frec_list; f; f = f->next) - if (f->sentto && f->stash && difftime(now, f->time) < TIMEOUT) + if (f->sentto && f->stash && difftime(now, f->time) < daemon->fast_retry_timeout) { #ifdef HAVE_DNSSEC if (f->blocking_query) @@ -635,8 +635,8 @@ int fast_retry(time_t now) /* t is milliseconds since last query sent. */ int to_run, t = (int)(millis - f->forward_timestamp); - if (t < daemon->fast_retry_time) - to_run = daemon->fast_retry_time - t; + if (t < f->forward_delay) + to_run = f->forward_delay - t; else { unsigned char *udpsz; @@ -657,7 +657,7 @@ int fast_retry(time_t now) forward_query(-1, NULL, NULL, 0, header, f->stash_len, ((char *) header) + udp_size, now, f, f->flags & FREC_AD_QUESTION, f->flags & FREC_DO_QUESTION); - to_run = daemon->fast_retry_time; + to_run = f->forward_delay = 2 * f->forward_delay; } if (ret == -1 || ret > to_run) @@ -2962,8 +2962,11 @@ static struct frec *get_new_frec(time_t now, struct server *master, int force) } if (target) - target->time = now; - + { + target->time = now; + target->forward_delay = daemon->fast_retry_time; + } + return target; } diff --git a/src/dnsmasq/option.c b/src/dnsmasq/option.c index 4df68b13..33d6b64e 100644 --- a/src/dnsmasq/option.c +++ b/src/dnsmasq/option.c @@ -374,7 +374,7 @@ static const struct myoption opts[] = { "umbrella", 2, 0, LOPT_UMBRELLA }, { "quiet-tftp", 0, 0, LOPT_QUIET_TFTP }, { "port-limit", 1, 0, LOPT_RANDPORT_LIM }, - { "fast-dns-retry", 1, 0, LOPT_FAST_RETRY }, + { "fast-dns-retry", 2, 0, LOPT_FAST_RETRY }, { "use-stale-cache", 0, 0 , LOPT_STALE_CACHE }, { NULL, 0, 0, 0 } }; @@ -3234,14 +3234,28 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma } case LOPT_FAST_RETRY: - { - int retry; - if (!atoi_check(arg, &retry) || retry < 500) - ret_err(gen_err); - daemon->fast_retry_time = retry; - break; - } + daemon->fast_retry_timeout = TIMEOUT; + if (!arg) + daemon->fast_retry_time = DEFAULT_FAST_RETRY; + else + { + int retry; + + comma = split(arg); + if (!atoi_check(arg, &retry) || retry < 50) + ret_err(gen_err); + daemon->fast_retry_time = retry; + + if (comma) + { + if (!atoi_check(comma, &retry)) + ret_err(gen_err); + daemon->fast_retry_timeout = retry/1000; + } + } + break; + #ifdef HAVE_DHCP case 'X': /* --dhcp-lease-max */ if (!atoi_check(arg, &daemon->dhcp_max)) From 85fdfb0d791b23d5edec13c262e02b5b3cd25746 Mon Sep 17 00:00:00 2001 From: Simon Kelley Date: Thu, 15 Sep 2022 22:06:39 +0100 Subject: [PATCH 128/218] Add metric for queries which never see an answer. Signed-off-by: DL6ER --- src/dnsmasq/forward.c | 2 ++ src/dnsmasq/metrics.c | 1 + src/dnsmasq/metrics.h | 1 + 3 files changed, 4 insertions(+) diff --git a/src/dnsmasq/forward.c b/src/dnsmasq/forward.c index f4ef63ae..cdbdb909 100644 --- a/src/dnsmasq/forward.c +++ b/src/dnsmasq/forward.c @@ -2930,6 +2930,7 @@ static struct frec *get_new_frec(time_t now, struct server *master, int force) { if (difftime(now, f->time) >= 4*TIMEOUT) { + daemon->metrics[METRIC_DNS_UNANSWERED_QUERY]++; free_frec(f); target = f; } @@ -2951,6 +2952,7 @@ static struct frec *get_new_frec(time_t now, struct server *master, int force) if (!target && oldest && ((int)difftime(now, oldest->time)) >= TIMEOUT) { /* can't find empty one, use oldest if there is one and it's older than timeout */ + daemon->metrics[METRIC_DNS_UNANSWERED_QUERY]++; free_frec(oldest); target = oldest; } diff --git a/src/dnsmasq/metrics.c b/src/dnsmasq/metrics.c index eeea0749..68fe7c6f 100644 --- a/src/dnsmasq/metrics.c +++ b/src/dnsmasq/metrics.c @@ -23,6 +23,7 @@ const char * metric_names[] = { "dns_auth_answered", "dns_local_answered", "dns_stale_answered", + "dns_unanswered", "bootp", "pxe", "dhcp_ack", diff --git a/src/dnsmasq/metrics.h b/src/dnsmasq/metrics.h index f7c817b4..bc91adf6 100644 --- a/src/dnsmasq/metrics.h +++ b/src/dnsmasq/metrics.h @@ -22,6 +22,7 @@ enum { METRIC_DNS_AUTH_ANSWERED, METRIC_DNS_LOCAL_ANSWERED, METRIC_DNS_STALE_ANSWERED, + METRIC_DNS_UNANSWERED_QUERY, METRIC_BOOTP, METRIC_PXE, METRIC_DHCPACK, From 3d5b4e20ec5d47539507147dff213cb9f9c1e9c5 Mon Sep 17 00:00:00 2001 From: Simon Kelley Date: Thu, 15 Sep 2022 22:29:44 +0100 Subject: [PATCH 129/218] Count NXDOMAIN replies from each server. Signed-off-by: DL6ER --- src/dnsmasq/cache.c | 11 +++++++---- src/dnsmasq/dbus.c | 1 + src/dnsmasq/dnsmasq.h | 2 +- src/dnsmasq/forward.c | 3 +++ 4 files changed, 12 insertions(+), 5 deletions(-) diff --git a/src/dnsmasq/cache.c b/src/dnsmasq/cache.c index 6ec70d54..6f6ee764 100644 --- a/src/dnsmasq/cache.c +++ b/src/dnsmasq/cache.c @@ -1671,18 +1671,19 @@ int cache_make_stat(struct txt_record *t) { char *new, *lenp; int port, newlen, bytes_avail, bytes_needed; - unsigned int queries = 0, failed_queries = 0; + unsigned int queries = 0, failed_queries = 0, nxdomain_replies = 0; for (serv1 = serv; serv1; serv1 = serv1->next) if (!(serv1->flags & SERV_MARK) && sockaddr_isequal(&serv->addr, &serv1->addr)) { serv1->flags |= SERV_MARK; queries += serv1->queries; failed_queries += serv1->failed_queries; + nxdomain_replies += serv1->nxdomain_replies; } port = prettyprint_addr(&serv->addr, daemon->addrbuff); lenp = p++; /* length */ bytes_avail = bufflen - (p - buff ); - bytes_needed = snprintf(p, bytes_avail, "%s#%d %u %u", daemon->addrbuff, port, queries, failed_queries); + bytes_needed = snprintf(p, bytes_avail, "%s#%d %u %u %u", daemon->addrbuff, port, queries, failed_queries, nxdomain_replies); if (bytes_needed >= bytes_avail) { /* expand buffer if necessary */ @@ -1785,16 +1786,18 @@ void dump_cache(time_t now) if (!(serv->flags & SERV_MARK)) { int port; - unsigned int queries = 0, failed_queries = 0; + unsigned int queries = 0, failed_queries = 0, nxdomain_replies = 0; for (serv1 = serv; serv1; serv1 = serv1->next) if (!(serv1->flags & SERV_MARK) && sockaddr_isequal(&serv->addr, &serv1->addr)) { serv1->flags |= SERV_MARK; queries += serv1->queries; failed_queries += serv1->failed_queries; + nxdomain_replies += serv1->nxdomain_replies; } port = prettyprint_addr(&serv->addr, daemon->addrbuff); - my_syslog(LOG_INFO, _("server %s#%d: queries sent %u, retried or failed %u"), daemon->addrbuff, port, queries, failed_queries); + my_syslog(LOG_INFO, _("server %s#%d: queries sent %u, retried or failed %u, nxdomain replies %u"), + daemon->addrbuff, port, queries, failed_queries, nxdomain_replies); } if (option_bool(OPT_DEBUG) || option_bool(OPT_LOG)) diff --git a/src/dnsmasq/dbus.c b/src/dnsmasq/dbus.c index e0089fc3..43cae308 100644 --- a/src/dnsmasq/dbus.c +++ b/src/dnsmasq/dbus.c @@ -685,6 +685,7 @@ static DBusMessage *dbus_get_server_metrics(DBusMessage* message) add_dict_int(&dict_array, "port", port); add_dict_int(&dict_array, "queries", serv->queries); add_dict_int(&dict_array, "failed_queries", serv->failed_queries); + add_dict_int(&dict_array, "nxdomain", serv->nxdomain_replies); if (strlen(serv->domain) != 0) add_dict_entry(&dict_array, "domain", serv->domain); diff --git a/src/dnsmasq/dnsmasq.h b/src/dnsmasq/dnsmasq.h index e17d13f1..59868ab3 100644 --- a/src/dnsmasq/dnsmasq.h +++ b/src/dnsmasq/dnsmasq.h @@ -595,7 +595,7 @@ struct server { struct serverfd *sfd; int tcpfd, edns_pktsz; time_t pktsz_reduced; - unsigned int queries, failed_queries; + unsigned int queries, failed_queries, nxdomain_replies; time_t forwardtime; int forwardcount; #ifdef HAVE_LOOP diff --git a/src/dnsmasq/forward.c b/src/dnsmasq/forward.c index cdbdb909..7efcc2e2 100644 --- a/src/dnsmasq/forward.c +++ b/src/dnsmasq/forward.c @@ -914,6 +914,9 @@ static size_t process_reply(struct dns_header *header, time_t now, struct server n = add_pseudoheader(header, n, limit, daemon->edns_pktsz, EDNS0_OPTION_EDE, (unsigned char *)&swap, 2, do_bit, 1); } + if (RCODE(header) == NXDOMAIN) + server->nxdomain_replies++; + return n; } From 7cae4bd3c30ef423dcf1c29a7a0073d28522e4b6 Mon Sep 17 00:00:00 2001 From: Simon Kelley Date: Thu, 15 Sep 2022 22:43:08 +0100 Subject: [PATCH 130/218] Combine server stats from all records for the same server in DBUS method. The DBUS per-server stats method should combine the stats from different records (for different domains) in the same way at the logging code. Signed-off-by: DL6ER --- src/dnsmasq/dbus.c | 49 +++++++++++++++++++++++++++++----------------- 1 file changed, 31 insertions(+), 18 deletions(-) diff --git a/src/dnsmasq/dbus.c b/src/dnsmasq/dbus.c index 43cae308..782fdc96 100644 --- a/src/dnsmasq/dbus.c +++ b/src/dnsmasq/dbus.c @@ -672,27 +672,40 @@ static DBusMessage *dbus_get_server_metrics(DBusMessage* message) dbus_message_iter_init_append(reply, &server_iter); dbus_message_iter_open_container(&server_iter, DBUS_TYPE_ARRAY, "a{ss}", &server_array); + + /* sum counts from different records for same server */ + for (serv = daemon->servers; serv; serv = serv->next) + serv->flags &= ~SERV_MARK; for (serv = daemon->servers; serv; serv = serv->next) - { - unsigned int port; - - dbus_message_iter_open_container(&server_array, DBUS_TYPE_ARRAY, "{ss}", &dict_array); - - port = prettyprint_addr(&serv->addr, daemon->namebuff); - add_dict_entry(&dict_array, "address", daemon->namebuff); - - add_dict_int(&dict_array, "port", port); - add_dict_int(&dict_array, "queries", serv->queries); - add_dict_int(&dict_array, "failed_queries", serv->failed_queries); - add_dict_int(&dict_array, "nxdomain", serv->nxdomain_replies); - - if (strlen(serv->domain) != 0) - add_dict_entry(&dict_array, "domain", serv->domain); - - dbus_message_iter_close_container(&server_array, &dict_array); - } + if (!(serv->flags & SERV_MARK)) + { + unsigned int port; + unsigned int queries = 0, failed_queries = 0, nxdomain_replies = 0; + struct server *serv1; + for (serv1 = serv; serv1; serv1 = serv1->next) + if (!(serv1->flags & SERV_MARK) && sockaddr_isequal(&serv->addr, &serv1->addr)) + { + serv1->flags |= SERV_MARK; + queries += serv1->queries; + failed_queries += serv1->failed_queries; + nxdomain_replies += serv1->nxdomain_replies; + } + + dbus_message_iter_open_container(&server_array, DBUS_TYPE_ARRAY, "{ss}", &dict_array); + + port = prettyprint_addr(&serv->addr, daemon->namebuff); + add_dict_entry(&dict_array, "address", daemon->namebuff); + + add_dict_int(&dict_array, "port", port); + add_dict_int(&dict_array, "queries", serv->queries); + add_dict_int(&dict_array, "failed_queries", serv->failed_queries); + add_dict_int(&dict_array, "nxdomain", serv->nxdomain_replies); + + dbus_message_iter_close_container(&server_array, &dict_array); + } + dbus_message_iter_close_container(&server_iter, &server_array); return reply; From b9957e4ca17f01817069e32da452e5dd26bc522e Mon Sep 17 00:00:00 2001 From: Simon Kelley Date: Thu, 15 Sep 2022 23:22:02 +0100 Subject: [PATCH 131/218] Keep a per-DNS-server moving average of query latency. Signed-off-by: DL6ER --- src/dnsmasq/cache.c | 13 ++++++++----- src/dnsmasq/dbus.c | 7 ++++++- src/dnsmasq/dnsmasq.h | 1 + src/dnsmasq/forward.c | 9 +++++++-- 4 files changed, 22 insertions(+), 8 deletions(-) diff --git a/src/dnsmasq/cache.c b/src/dnsmasq/cache.c index 6f6ee764..935eef0b 100644 --- a/src/dnsmasq/cache.c +++ b/src/dnsmasq/cache.c @@ -1671,19 +1671,18 @@ int cache_make_stat(struct txt_record *t) { char *new, *lenp; int port, newlen, bytes_avail, bytes_needed; - unsigned int queries = 0, failed_queries = 0, nxdomain_replies = 0; + unsigned int queries = 0, failed_queries = 0; for (serv1 = serv; serv1; serv1 = serv1->next) if (!(serv1->flags & SERV_MARK) && sockaddr_isequal(&serv->addr, &serv1->addr)) { serv1->flags |= SERV_MARK; queries += serv1->queries; failed_queries += serv1->failed_queries; - nxdomain_replies += serv1->nxdomain_replies; } port = prettyprint_addr(&serv->addr, daemon->addrbuff); lenp = p++; /* length */ bytes_avail = bufflen - (p - buff ); - bytes_needed = snprintf(p, bytes_avail, "%s#%d %u %u %u", daemon->addrbuff, port, queries, failed_queries, nxdomain_replies); + bytes_needed = snprintf(p, bytes_avail, "%s#%d %u %u", daemon->addrbuff, port, queries, failed_queries); if (bytes_needed >= bytes_avail) { /* expand buffer if necessary */ @@ -1787,6 +1786,8 @@ void dump_cache(time_t now) { int port; unsigned int queries = 0, failed_queries = 0, nxdomain_replies = 0; + unsigned int sigma_latency = 0, count_latency = 0; + for (serv1 = serv; serv1; serv1 = serv1->next) if (!(serv1->flags & SERV_MARK) && sockaddr_isequal(&serv->addr, &serv1->addr)) { @@ -1794,10 +1795,12 @@ void dump_cache(time_t now) queries += serv1->queries; failed_queries += serv1->failed_queries; nxdomain_replies += serv1->nxdomain_replies; + sigma_latency += serv1->query_latency; + count_latency++; } port = prettyprint_addr(&serv->addr, daemon->addrbuff); - my_syslog(LOG_INFO, _("server %s#%d: queries sent %u, retried or failed %u, nxdomain replies %u"), - daemon->addrbuff, port, queries, failed_queries, nxdomain_replies); + my_syslog(LOG_INFO, _("server %s#%d: queries sent %u, retried or failed %u, nxdomain replies %u, avg. latency %ums"), + daemon->addrbuff, port, queries, failed_queries, nxdomain_replies, sigma_latency/count_latency); } if (option_bool(OPT_DEBUG) || option_bool(OPT_LOG)) diff --git a/src/dnsmasq/dbus.c b/src/dnsmasq/dbus.c index 782fdc96..ebd8d7f5 100644 --- a/src/dnsmasq/dbus.c +++ b/src/dnsmasq/dbus.c @@ -682,6 +682,8 @@ static DBusMessage *dbus_get_server_metrics(DBusMessage* message) { unsigned int port; unsigned int queries = 0, failed_queries = 0, nxdomain_replies = 0; + unsigned int sigma_latency = 0, count_latency = 0; + struct server *serv1; for (serv1 = serv; serv1; serv1 = serv1->next) @@ -691,6 +693,8 @@ static DBusMessage *dbus_get_server_metrics(DBusMessage* message) queries += serv1->queries; failed_queries += serv1->failed_queries; nxdomain_replies += serv1->nxdomain_replies; + sigma_latency += serv1->query_latency; + count_latency++; } dbus_message_iter_open_container(&server_array, DBUS_TYPE_ARRAY, "{ss}", &dict_array); @@ -702,7 +706,8 @@ static DBusMessage *dbus_get_server_metrics(DBusMessage* message) add_dict_int(&dict_array, "queries", serv->queries); add_dict_int(&dict_array, "failed_queries", serv->failed_queries); add_dict_int(&dict_array, "nxdomain", serv->nxdomain_replies); - + add_dict_int(&dict_array, "latency", sigma_latency/count_latency; + dbus_message_iter_close_container(&server_array, &dict_array); } diff --git a/src/dnsmasq/dnsmasq.h b/src/dnsmasq/dnsmasq.h index 59868ab3..03526cb3 100644 --- a/src/dnsmasq/dnsmasq.h +++ b/src/dnsmasq/dnsmasq.h @@ -596,6 +596,7 @@ struct server { int tcpfd, edns_pktsz; time_t pktsz_reduced; unsigned int queries, failed_queries, nxdomain_replies; + unsigned int query_latency, mma_latency; time_t forwardtime; int forwardcount; #ifdef HAVE_LOOP diff --git a/src/dnsmasq/forward.c b/src/dnsmasq/forward.c index 7efcc2e2..ea75aec7 100644 --- a/src/dnsmasq/forward.c +++ b/src/dnsmasq/forward.c @@ -571,8 +571,7 @@ static int forward_query(int udpfd, union mysockaddr *udpaddr, if (forwarded || is_dnssec) { - if (daemon->fast_retry_time != 0) - forward->forward_timestamp = dnsmasq_milliseconds(); + forward->forward_timestamp = dnsmasq_milliseconds(); return 1; } @@ -1286,6 +1285,12 @@ void reply_query(int fd, time_t now) answers, to conserve file descriptors, and to save work reading and discarding answers for other upstreams. */ free_rfds(&forward->rfds); + + /* calculate modified moving average of server latency */ + server->mma_latency += dnsmasq_milliseconds() - forward->forward_timestamp - server->query_latency; + /* denominator controls how many queries we average over. */ + server->query_latency = server->mma_latency/128; + #ifdef HAVE_DNSSEC if ((forward->sentto->flags & SERV_DO_DNSSEC) && From 861f529980509e5cb44c7fdaa87c2b3da7ee425d Mon Sep 17 00:00:00 2001 From: Simon Kelley Date: Thu, 15 Sep 2022 23:54:53 +0100 Subject: [PATCH 132/218] Tweak server-selection logic in the fast-retry case. Signed-off-by: DL6ER --- src/dnsmasq/forward.c | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/dnsmasq/forward.c b/src/dnsmasq/forward.c index ea75aec7..410d6c24 100644 --- a/src/dnsmasq/forward.c +++ b/src/dnsmasq/forward.c @@ -173,7 +173,7 @@ static int domain_no_rebind(char *domain) static int forward_query(int udpfd, union mysockaddr *udpaddr, union all_addr *dst_addr, unsigned int dst_iface, struct dns_header *header, size_t plen, char *limit, time_t now, - struct frec *forward, int ad_reqd, int do_bit) + struct frec *forward, int ad_reqd, int do_bit, int fast_retry) { unsigned int flags = 0; unsigned int fwd_flags = 0; @@ -405,7 +405,8 @@ static int forward_query(int udpfd, union mysockaddr *udpaddr, #endif { /* retry on existing query, from original source. Send to all available servers */ - forward->sentto->failed_queries++; + if (!fast_retry) + forward->sentto->failed_queries++; FTL_forwarding_retried(forward->sentto, forward->frec_src.log_id, daemon->log_display_id, false); @@ -415,13 +416,13 @@ static int forward_query(int udpfd, union mysockaddr *udpaddr, master = daemon->serverarray[first]; /* Forward to all available servers on retry of query from same host. */ - if (!option_bool(OPT_ORDER) && old_src) + if (!option_bool(OPT_ORDER) && old_src && !fast_retry) forward->forwardall = 1; else { start = forward->sentto->arrayposn; - if (option_bool(OPT_ORDER)) + if (option_bool(OPT_ORDER) && !fast_retry) { /* In strict order mode, there must be a server later in the list left to send to, otherwise without the forwardall mechanism, @@ -654,7 +655,7 @@ int fast_retry(time_t now) daemon->log_display_id = f->frec_src.log_id; forward_query(-1, NULL, NULL, 0, header, f->stash_len, ((char *) header) + udp_size, now, f, - f->flags & FREC_AD_QUESTION, f->flags & FREC_DO_QUESTION); + f->flags & FREC_AD_QUESTION, f->flags & FREC_DO_QUESTION, 1); to_run = f->forward_delay = 2 * f->forward_delay; } @@ -1251,7 +1252,7 @@ void reply_query(int fd, time_t now) if (nn) { forward_query(-1, NULL, NULL, 0, header, nn, ((char *) header) + udp_size, now, forward, - forward->flags & FREC_AD_QUESTION, forward->flags & FREC_DO_QUESTION); + forward->flags & FREC_AD_QUESTION, forward->flags & FREC_DO_QUESTION, 0); return; } } @@ -1940,7 +1941,7 @@ void receive_query(struct listener *listen, time_t now) } if (forward_query(fd, &source_addr, &dst_addr, if_index, - header, (size_t)n, ((char *) header) + udp_size, now, NULL, ad_reqd, do_bit)) + header, (size_t)n, ((char *) header) + udp_size, now, NULL, ad_reqd, do_bit, 0)) daemon->metrics[METRIC_DNS_QUERIES_FORWARDED]++; else daemon->metrics[METRIC_DNS_LOCAL_ANSWERED]++; From a59927835fc36721ed4b63a731c50946d8f68fb8 Mon Sep 17 00:00:00 2001 From: Simon Kelley Date: Fri, 16 Sep 2022 00:07:36 +0100 Subject: [PATCH 133/218] Split failed queries in retries in stat counting. Signed-off-by: DL6ER --- src/dnsmasq/cache.c | 7 ++++--- src/dnsmasq/dbus.c | 4 +++- src/dnsmasq/dnsmasq.h | 2 +- src/dnsmasq/forward.c | 4 +++- 4 files changed, 11 insertions(+), 6 deletions(-) diff --git a/src/dnsmasq/cache.c b/src/dnsmasq/cache.c index 935eef0b..356a2d67 100644 --- a/src/dnsmasq/cache.c +++ b/src/dnsmasq/cache.c @@ -1785,7 +1785,7 @@ void dump_cache(time_t now) if (!(serv->flags & SERV_MARK)) { int port; - unsigned int queries = 0, failed_queries = 0, nxdomain_replies = 0; + unsigned int queries = 0, failed_queries = 0, nxdomain_replies = 0, retrys = 0; unsigned int sigma_latency = 0, count_latency = 0; for (serv1 = serv; serv1; serv1 = serv1->next) @@ -1795,12 +1795,13 @@ void dump_cache(time_t now) queries += serv1->queries; failed_queries += serv1->failed_queries; nxdomain_replies += serv1->nxdomain_replies; + retrys += serv1->retrys; sigma_latency += serv1->query_latency; count_latency++; } port = prettyprint_addr(&serv->addr, daemon->addrbuff); - my_syslog(LOG_INFO, _("server %s#%d: queries sent %u, retried or failed %u, nxdomain replies %u, avg. latency %ums"), - daemon->addrbuff, port, queries, failed_queries, nxdomain_replies, sigma_latency/count_latency); + my_syslog(LOG_INFO, _("server %s#%d: queries sent %u, retried %u, failed %u, nxdomain replies %u, avg. latency %ums"), + daemon->addrbuff, port, queries, retrys, failed_queries, nxdomain_replies, sigma_latency/count_latency); } if (option_bool(OPT_DEBUG) || option_bool(OPT_LOG)) diff --git a/src/dnsmasq/dbus.c b/src/dnsmasq/dbus.c index ebd8d7f5..65b72211 100644 --- a/src/dnsmasq/dbus.c +++ b/src/dnsmasq/dbus.c @@ -681,7 +681,7 @@ static DBusMessage *dbus_get_server_metrics(DBusMessage* message) if (!(serv->flags & SERV_MARK)) { unsigned int port; - unsigned int queries = 0, failed_queries = 0, nxdomain_replies = 0; + unsigned int queries = 0, failed_queries = 0, nxdomain_replies = 0, retrys = 0; unsigned int sigma_latency = 0, count_latency = 0; struct server *serv1; @@ -693,6 +693,7 @@ static DBusMessage *dbus_get_server_metrics(DBusMessage* message) queries += serv1->queries; failed_queries += serv1->failed_queries; nxdomain_replies += serv1->nxdomain_replies; + retrys += serv1->retrys; sigma_latency += serv1->query_latency; count_latency++; } @@ -706,6 +707,7 @@ static DBusMessage *dbus_get_server_metrics(DBusMessage* message) add_dict_int(&dict_array, "queries", serv->queries); add_dict_int(&dict_array, "failed_queries", serv->failed_queries); add_dict_int(&dict_array, "nxdomain", serv->nxdomain_replies); + add_dict_int(&dict_array, "retries", serv->retrys); add_dict_int(&dict_array, "latency", sigma_latency/count_latency; dbus_message_iter_close_container(&server_array, &dict_array); diff --git a/src/dnsmasq/dnsmasq.h b/src/dnsmasq/dnsmasq.h index 03526cb3..f2bde13b 100644 --- a/src/dnsmasq/dnsmasq.h +++ b/src/dnsmasq/dnsmasq.h @@ -595,7 +595,7 @@ struct server { struct serverfd *sfd; int tcpfd, edns_pktsz; time_t pktsz_reduced; - unsigned int queries, failed_queries, nxdomain_replies; + unsigned int queries, failed_queries, nxdomain_replies, retrys; unsigned int query_latency, mma_latency; time_t forwardtime; int forwardcount; diff --git a/src/dnsmasq/forward.c b/src/dnsmasq/forward.c index 410d6c24..465aaafa 100644 --- a/src/dnsmasq/forward.c +++ b/src/dnsmasq/forward.c @@ -405,8 +405,10 @@ static int forward_query(int udpfd, union mysockaddr *udpaddr, #endif { /* retry on existing query, from original source. Send to all available servers */ - if (!fast_retry) + if (udpfd == -1 && !fast_retry) forward->sentto->failed_queries++; + else + forward->sentto->retrys++; FTL_forwarding_retried(forward->sentto, forward->frec_src.log_id, daemon->log_display_id, false); From 5935160b7b971bbeed443affff7bc482c5fab262 Mon Sep 17 00:00:00 2001 From: Simon Kelley Date: Fri, 16 Sep 2022 00:16:18 +0100 Subject: [PATCH 134/218] Initialise modified-moving-average latency calc better. Use the first value, rather than initialising at zero, which takes many queries to converge. Signed-off-by: DL6ER --- src/dnsmasq/forward.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/dnsmasq/forward.c b/src/dnsmasq/forward.c index 465aaafa..df8a6241 100644 --- a/src/dnsmasq/forward.c +++ b/src/dnsmasq/forward.c @@ -1290,7 +1290,10 @@ void reply_query(int fd, time_t now) free_rfds(&forward->rfds); /* calculate modified moving average of server latency */ - server->mma_latency += dnsmasq_milliseconds() - forward->forward_timestamp - server->query_latency; + if (server->query_latency == 0) + server->mma_latency = (dnsmasq_milliseconds() - forward->forward_timestamp) * 128; /* init */ + else + server->mma_latency += dnsmasq_milliseconds() - forward->forward_timestamp - server->query_latency; /* denominator controls how many queries we average over. */ server->query_latency = server->mma_latency/128; From 0fd2a136c635404db323bbcee1306dedb0a44248 Mon Sep 17 00:00:00 2001 From: Simon Kelley Date: Fri, 16 Sep 2022 09:35:44 +0100 Subject: [PATCH 135/218] Don't exclude stale-cache answers from "local answered" metric. Signed-off-by: DL6ER --- src/dnsmasq/forward.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/dnsmasq/forward.c b/src/dnsmasq/forward.c index df8a6241..d6216fee 100644 --- a/src/dnsmasq/forward.c +++ b/src/dnsmasq/forward.c @@ -1919,7 +1919,9 @@ void receive_query(struct listener *listen, time_t now) #endif send_from(listen->fd, option_bool(OPT_NOWILD) || option_bool(OPT_CLEVERBIND), (char *)header, m, &source_addr, &dst_addr, if_index); - daemon->metrics[stale ? METRIC_DNS_STALE_ANSWERED : METRIC_DNS_LOCAL_ANSWERED]++; + daemon->metrics[METRIC_DNS_LOCAL_ANSWERED]++; + if (stale) + daemon->metrics[METRIC_DNS_STALE_ANSWERED]++; } if (m == 0 || stale) From f4e359296d916ba4ef0f547928692cdbd7b57df5 Mon Sep 17 00:00:00 2001 From: Simon Kelley Date: Fri, 16 Sep 2022 12:07:01 +0100 Subject: [PATCH 136/218] Optimise cache code when stale caching in use. Exclude DNSSEC entries from stale caching. Signed-off-by: DL6ER --- src/dnsmasq/cache.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/dnsmasq/cache.c b/src/dnsmasq/cache.c index 356a2d67..4bd8434a 100644 --- a/src/dnsmasq/cache.c +++ b/src/dnsmasq/cache.c @@ -375,8 +375,9 @@ static int is_outdated_cname_pointer(struct crec *crecp) static int is_expired(time_t now, struct crec *crecp) { - /* Don't dump expired entries if we're using them, cache becomes strictly LRU in that case. */ - if (option_bool(OPT_STALE_CACHE)) + /* Don't dump expired entries if we're using them, cache becomes strictly LRU in that case. + Never use expired DS or DNSKEY entries. */ + if (option_bool(OPT_STALE_CACHE) && !(crecp->flags & (F_DS | F_DNSKEY))) return 0; if (crecp->flags & F_IMMORTAL) @@ -558,7 +559,7 @@ static struct crec *really_insert(char *name, union all_addr *addr, unsigned sho { struct crec *new, *target_crec = NULL; union bigname *big_name = NULL; - int freed_all = flags & F_REVERSE; + int freed_all = (flags & F_REVERSE); int free_avail = 0; unsigned int target_uid; @@ -633,8 +634,12 @@ static struct crec *really_insert(char *name, union all_addr *addr, unsigned sho { /* For DNSSEC records, uid holds class. */ free_avail = 1; /* Must be free space now. */ + + /* condition valid when stale-caching */ + if (difftime(now, new->ttd) < 0) + daemon->metrics[METRIC_DNS_CACHE_LIVE_FREED]++; + cache_scan_free(cache_get_name(new), &new->addr, new->uid, now, new->flags, NULL, NULL); - daemon->metrics[METRIC_DNS_CACHE_LIVE_FREED]++; } else { From 976e9321edb4a7664b36034da07a5ea5b1b0c5cc Mon Sep 17 00:00:00 2001 From: Simon Kelley Date: Fri, 16 Sep 2022 12:58:41 +0100 Subject: [PATCH 137/218] Add ClearMetrics Dbus method. Signed-off-by: DL6ER --- src/dnsmasq/dbus.c | 12 +++++++++--- src/dnsmasq/metrics.c | 20 ++++++++++++++++++++ src/dnsmasq/metrics.h | 1 + 3 files changed, 30 insertions(+), 3 deletions(-) diff --git a/src/dnsmasq/dbus.c b/src/dnsmasq/dbus.c index 65b72211..4fdad465 100644 --- a/src/dnsmasq/dbus.c +++ b/src/dnsmasq/dbus.c @@ -94,6 +94,8 @@ const char* introspection_xml_template = " \n" " \n" " \n" +" \n" +" \n" " \n" "\n"; @@ -708,13 +710,13 @@ static DBusMessage *dbus_get_server_metrics(DBusMessage* message) add_dict_int(&dict_array, "failed_queries", serv->failed_queries); add_dict_int(&dict_array, "nxdomain", serv->nxdomain_replies); add_dict_int(&dict_array, "retries", serv->retrys); - add_dict_int(&dict_array, "latency", sigma_latency/count_latency; - + add_dict_int(&dict_array, "latency", sigma_latency/count_latency); + dbus_message_iter_close_container(&server_array, &dict_array); } dbus_message_iter_close_container(&server_iter, &server_array); - + return reply; } @@ -797,6 +799,10 @@ DBusHandlerResult message_handler(DBusConnection *connection, { reply = dbus_get_server_metrics(message); } + else if (strcmp(method, "ClearMetrics") == 0) + { + clear_metrics(); + } else if (strcmp(method, "ClearCache") == 0) clear_cache = 1; else diff --git a/src/dnsmasq/metrics.c b/src/dnsmasq/metrics.c index 68fe7c6f..f3e6728a 100644 --- a/src/dnsmasq/metrics.c +++ b/src/dnsmasq/metrics.c @@ -44,3 +44,23 @@ const char * metric_names[] = { const char* get_metric_name(int i) { return metric_names[i]; } + +void clear_metrics(void) +{ + int i; + struct server *serv; + + for (i = 0; i < __METRIC_MAX; i++) + daemon->metrics[i] = 0; + + for (serv = daemon->servers; serv; serv = serv->next) + { + serv->queries = 0; + serv->failed_queries = 0; + serv->failed_queries = 0; + serv->retrys = 0; + serv->nxdomain_replies = 0; + serv->query_latency = 0; + } +} + diff --git a/src/dnsmasq/metrics.h b/src/dnsmasq/metrics.h index bc91adf6..6f62a406 100644 --- a/src/dnsmasq/metrics.h +++ b/src/dnsmasq/metrics.h @@ -43,3 +43,4 @@ enum { }; const char* get_metric_name(int); +void clear_metrics(void); From 5929989ff91f1571440431ca36d9f014da7a80fb Mon Sep 17 00:00:00 2001 From: Simon Kelley Date: Thu, 13 Oct 2022 15:02:54 +0100 Subject: [PATCH 138/218] Fix bug in --dynamic-host when interface has /16 IPv4 address. Signed-off-by: DL6ER --- src/dnsmasq/network.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dnsmasq/network.c b/src/dnsmasq/network.c index f2ea24cc..f5900a7b 100644 --- a/src/dnsmasq/network.c +++ b/src/dnsmasq/network.c @@ -362,7 +362,7 @@ static int iface_allowed(struct iface_param *param, int if_index, char *label, if (int_name->flags & INP4) { - if (netmask.s_addr == 0xffff) + if (netmask.s_addr == 0xffffffff) continue; newaddr.s_addr = (addr->in.sin_addr.s_addr & netmask.s_addr) | From d39829875f914c66daeaa61aa517e06bb6ddeef8 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 14 Oct 2022 17:56:30 +0200 Subject: [PATCH 139/218] Update embedded dnsmasq to v2.88test1 Signed-off-by: DL6ER --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6ab1cc45..62ce77e7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -11,6 +11,6 @@ cmake_minimum_required(VERSION 2.8.12) project(PIHOLE_FTL C) -set(DNSMASQ_VERSION pi-hole-v2.87) +set(DNSMASQ_VERSION pi-hole-v2.88test1) add_subdirectory(src) From 0dc053364b2b01298824c332724005dfedf2833e Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 14 Oct 2022 18:26:57 +0200 Subject: [PATCH 140/218] Locally blocked queries are not stale Signed-off-by: DL6ER --- src/dnsmasq/forward.c | 1 + src/dnsmasq_interface.c | 2 ++ 2 files changed, 3 insertions(+) diff --git a/src/dnsmasq/forward.c b/src/dnsmasq/forward.c index d6216fee..88e0cdd3 100644 --- a/src/dnsmasq/forward.c +++ b/src/dnsmasq/forward.c @@ -2387,6 +2387,7 @@ unsigned char *tcp_request(int confd, time_t now, if(piholeblocked) { int ede = EDE_UNSET; + stale = 0; // Generate DNS packet for reply m = FTL_make_answer(header, ((char *) header) + 65536, size, &ede); // The pseudoheader may contain important information such as EDNS0 version important for diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c index 7506d168..799451a6 100644 --- a/src/dnsmasq_interface.c +++ b/src/dnsmasq_interface.c @@ -173,6 +173,8 @@ void FTL_hook(unsigned int flags, const char *name, union all_addr *addr, char * // This is inspired by make_local_answer() size_t _FTL_make_answer(struct dns_header *header, char *limit, const size_t len, int *ede, const char *file, const int line) { + if(config.debug & DEBUG_FLAGS) + logg("FTL_make_answer() called from %s:%d", short_path(file), line); // Exit early if there are no questions in this query if(ntohs(header->qdcount) == 0) return 0; From 652c1e8a90959cb4e592425cb832c1b0a1e0d1e4 Mon Sep 17 00:00:00 2001 From: Dominik Derigs Date: Sun, 16 Oct 2022 21:08:30 +0100 Subject: [PATCH 141/218] Improve logging of DNS record source from --hostsdir files. Patch author Dominik Derigs with subsequent bugfixes and tweaks from Simon Kelley. Signed-off-by: DL6ER --- src/dnsmasq/cache.c | 9 ++-- src/dnsmasq/dnsmasq.h | 13 ++++- src/dnsmasq/inotify.c | 123 +++++++++++++++++++++++++++--------------- src/dnsmasq/option.c | 39 ++++++++------ 4 files changed, 119 insertions(+), 65 deletions(-) diff --git a/src/dnsmasq/cache.c b/src/dnsmasq/cache.c index 4bd8434a..d17a4489 100644 --- a/src/dnsmasq/cache.c +++ b/src/dnsmasq/cache.c @@ -1901,6 +1901,7 @@ void dump_cache(time_t now) char *record_source(unsigned int index) { struct hostsfile *ah; + struct dyndir *dd; if (index == SRC_CONFIG) return "config"; @@ -1912,9 +1913,11 @@ char *record_source(unsigned int index) return ah->fname; #ifdef HAVE_INOTIFY - for (ah = daemon->dynamic_dirs; ah; ah = ah->next) - if (ah->index == index) - return ah->fname; + /* Dynamic directories contain multiple files */ + for (dd = daemon->dynamic_dirs; dd; dd = dd->next) + for (ah = dd->files; ah; ah = ah->next) + if (ah->index == index) + return ah->fname; #endif return ""; diff --git a/src/dnsmasq/dnsmasq.h b/src/dnsmasq/dnsmasq.h index f2bde13b..a6c5cc18 100644 --- a/src/dnsmasq/dnsmasq.h +++ b/src/dnsmasq/dnsmasq.h @@ -699,10 +699,17 @@ struct hostsfile { struct hostsfile *next; int flags; char *fname; + unsigned int index; /* matches to cache entries for logging */ +}; + +struct dyndir { + struct dyndir *next; + struct hostsfile *files; + int flags; + char *dname; #ifdef HAVE_INOTIFY int wd; /* inotify watch descriptor */ #endif - unsigned int index; /* matches to cache entries for logging */ }; /* packet-dump flags */ @@ -1164,6 +1171,7 @@ extern struct daemon { u32 umbrella_org; u32 umbrella_asset; u8 umbrella_device[8]; + int host_index; struct hostsfile *addn_hosts; struct dhcp_context *dhcp, *dhcp6; struct ra_interface *ra_interfaces; @@ -1184,7 +1192,8 @@ extern struct daemon { int doing_ra, doing_dhcp6; struct dhcp_netid_list *dhcp_ignore, *dhcp_ignore_names, *dhcp_gen_names; struct dhcp_netid_list *force_broadcast, *bootp_dynamic; - struct hostsfile *dhcp_hosts_file, *dhcp_opts_file, *dynamic_dirs; + struct hostsfile *dhcp_hosts_file, *dhcp_opts_file; + struct dyndir *dynamic_dirs; int dhcp_max, tftp_max, tftp_mtu; int dhcp_server_port, dhcp_client_port; int start_tftp_port, end_tftp_port; diff --git a/src/dnsmasq/inotify.c b/src/dnsmasq/inotify.c index 5687e37c..6e1ba18c 100644 --- a/src/dnsmasq/inotify.c +++ b/src/dnsmasq/inotify.c @@ -133,81 +133,112 @@ void inotify_dnsmasq_init() } } +static struct hostsfile *dyndir_addhosts(struct dyndir *dd, char *path) +{ + /* Check if this file is already known in dd->files */ + struct hostsfile *ah = NULL; + for(ah = dd->files; ah; ah = ah->next) + if(ah && ah->fname && strcmp(path, ah->fname) == 0) + return ah; + + /* Not known, create new hostsfile record for this dyndir */ + struct hostsfile *newah = NULL; + if(!(newah = whine_malloc(sizeof(struct hostsfile)))) + return NULL; + + /* Add this file to the tip of the linked list */ + newah->next = dd->files; + dd->files = newah; + + /* Copy flags, set index and the full file path */ + newah->flags = dd->flags; + newah->index = daemon->host_index++; + newah->fname = path; + + return newah; +} + /* initialisation for dynamic-dir. Set inotify watch for each directory, and read pre-existing files */ void set_dynamic_inotify(int flag, int total_size, struct crec **rhash, int revhashsz) { - struct hostsfile *ah; - - for (ah = daemon->dynamic_dirs; ah; ah = ah->next) + struct dyndir *dd; + + for (dd = daemon->dynamic_dirs; dd; dd = dd->next) { DIR *dir_stream = NULL; struct dirent *ent; struct stat buf; - - if (!(ah->flags & flag)) + + if (!(dd->flags & flag)) continue; - - if (stat(ah->fname, &buf) == -1) + + if (stat(dd->dname, &buf) == -1) { my_syslog(LOG_ERR, _("bad dynamic directory %s: %s"), - ah->fname, strerror(errno)); + dd->dname, strerror(errno)); continue; } if (!(S_ISDIR(buf.st_mode))) { my_syslog(LOG_ERR, _("bad dynamic directory %s: %s"), - ah->fname, _("not a directory")); + dd->dname, _("not a directory")); continue; } - - if (!(ah->flags & AH_WD_DONE)) + + if (!(dd->flags & AH_WD_DONE)) { - ah->wd = inotify_add_watch(daemon->inotifyfd, ah->fname, IN_CLOSE_WRITE | IN_MOVED_TO); - ah->flags |= AH_WD_DONE; + dd->wd = inotify_add_watch(daemon->inotifyfd, dd->dname, IN_CLOSE_WRITE | IN_MOVED_TO | IN_DELETE); + dd->flags |= AH_WD_DONE; } /* Read contents of dir _after_ calling add_watch, in the hope of avoiding a race which misses files being added as we start */ - if (ah->wd == -1 || !(dir_stream = opendir(ah->fname))) + if (dd->wd == -1 || !(dir_stream = opendir(dd->dname))) { my_syslog(LOG_ERR, _("failed to create inotify for %s: %s"), - ah->fname, strerror(errno)); + dd->dname, strerror(errno)); continue; } while ((ent = readdir(dir_stream))) { - size_t lendir = strlen(ah->fname); + size_t lendir = strlen(dd->dname); size_t lenfile = strlen(ent->d_name); char *path; - + /* ignore emacs backups and dotfiles */ if (lenfile == 0 || ent->d_name[lenfile - 1] == '~' || (ent->d_name[0] == '#' && ent->d_name[lenfile - 1] == '#') || ent->d_name[0] == '.') continue; - + if ((path = whine_malloc(lendir + lenfile + 2))) { - strcpy(path, ah->fname); + struct hostsfile *ah; + + strcpy(path, dd->dname); strcat(path, "/"); strcat(path, ent->d_name); + + if (!(ah = dyndir_addhosts(dd, path))) + { + free(path); + continue; + } /* ignore non-regular files */ if (stat(path, &buf) != -1 && S_ISREG(buf.st_mode)) { - if (ah->flags & AH_HOSTS) + if (dd->flags & AH_HOSTS) total_size = read_hostsfile(path, ah->index, total_size, rhash, revhashsz); #ifdef HAVE_DHCP - else if (ah->flags & (AH_DHCP_HST | AH_DHCP_OPT)) - option_read_dynfile(path, ah->flags); + else if (dd->flags & (AH_DHCP_HST | AH_DHCP_OPT)) + option_read_dynfile(path, dd->flags); #endif } - - free(path); } } @@ -218,7 +249,7 @@ void set_dynamic_inotify(int flag, int total_size, struct crec **rhash, int revh int inotify_check(time_t now) { int hit = 0; - struct hostsfile *ah; + struct dyndir *dd; while (1) { @@ -249,36 +280,41 @@ int inotify_check(time_t now) if (res->wd == in->wd && strcmp(res->file, in->name) == 0) hit = 1; - for (ah = daemon->dynamic_dirs; ah; ah = ah->next) - if (ah->wd == in->wd) + for (dd = daemon->dynamic_dirs; dd; dd = dd->next) + if (dd->wd == in->wd) { - size_t lendir = strlen(ah->fname); + size_t lendir = strlen(dd->dname); char *path; - + if ((path = whine_malloc(lendir + in->len + 2))) { - strcpy(path, ah->fname); + struct hostsfile *ah = NULL; + + strcpy(path, dd->dname); strcat(path, "/"); strcat(path, in->name); my_syslog(LOG_INFO, _("inotify, new or changed file %s"), path); - if (ah->flags & AH_HOSTS) + if (dd->flags & AH_HOSTS) { - read_hostsfile(path, ah->index, 0, NULL, 0); -#ifdef HAVE_DHCP - if (daemon->dhcp || daemon->doing_dhcp6) + if ((ah = dyndir_addhosts(dd, path))) { - /* Propagate the consequences of loading a new dhcp-host */ - dhcp_update_configs(daemon->dhcp_conf); - lease_update_from_configs(); - lease_update_file(now); - lease_update_dns(1); - } + read_hostsfile(path, ah->index, 0, NULL, 0); +#ifdef HAVE_DHCP + if (daemon->dhcp || daemon->doing_dhcp6) + { + /* Propagate the consequences of loading a new dhcp-host */ + dhcp_update_configs(daemon->dhcp_conf); + lease_update_from_configs(); + lease_update_file(now); + lease_update_dns(1); + } #endif + } } #ifdef HAVE_DHCP - else if (ah->flags & AH_DHCP_HST) + else if (dd->flags & AH_DHCP_HST) { if (option_read_dynfile(path, AH_DHCP_HST)) { @@ -289,11 +325,12 @@ int inotify_check(time_t now) lease_update_dns(1); } } - else if (ah->flags & AH_DHCP_OPT) + else if (dd->flags & AH_DHCP_OPT) option_read_dynfile(path, AH_DHCP_OPT); #endif - free(path); + if (!ah) + free(path); } } } diff --git a/src/dnsmasq/option.c b/src/dnsmasq/option.c index 33d6b64e..70933c33 100644 --- a/src/dnsmasq/option.c +++ b/src/dnsmasq/option.c @@ -2166,15 +2166,11 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma case LOPT_DHCP_HOST: /* --dhcp-hostsfile */ case LOPT_DHCP_OPTS: /* --dhcp-optsfile */ - case LOPT_DHCP_INOTIFY: /* --dhcp-hostsdir */ - case LOPT_DHOPT_INOTIFY: /* --dhcp-optsdir */ - case LOPT_HOST_INOTIFY: /* --hostsdir */ case 'H': /* --addn-hosts */ { struct hostsfile *new = opt_malloc(sizeof(struct hostsfile)); - static unsigned int hosts_index = SRC_AH; new->fname = opt_string_alloc(arg); - new->index = hosts_index++; + new->index = daemon->host_index++; new->flags = 0; if (option == 'H') { @@ -2190,21 +2186,29 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma { new->next = daemon->dhcp_opts_file; daemon->dhcp_opts_file = new; - } - else - { - new->next = daemon->dynamic_dirs; - daemon->dynamic_dirs = new; - if (option == LOPT_DHCP_INOTIFY) - new->flags |= AH_DHCP_HST; - else if (option == LOPT_DHOPT_INOTIFY) - new->flags |= AH_DHCP_OPT; - else if (option == LOPT_HOST_INOTIFY) - new->flags |= AH_HOSTS; } break; } + + case LOPT_DHCP_INOTIFY: /* --dhcp-hostsdir */ + case LOPT_DHOPT_INOTIFY: /* --dhcp-optsdir */ + case LOPT_HOST_INOTIFY: /* --hostsdir */ + { + struct dyndir *new = opt_malloc(sizeof(struct dyndir)); + new->dname = opt_string_alloc(arg); + new->flags = 0; + new->next = daemon->dynamic_dirs; + daemon->dynamic_dirs = new; + if (option == LOPT_DHCP_INOTIFY) + new->flags |= AH_DHCP_HST; + else if (option == LOPT_DHOPT_INOTIFY) + new->flags |= AH_DHCP_OPT; + else if (option == LOPT_HOST_INOTIFY) + new->flags |= AH_HOSTS; + + break; + } case LOPT_AUTHSERV: /* --auth-server */ comma = split(arg); @@ -5536,7 +5540,8 @@ void read_opts(int argc, char **argv, char *compile_opts) daemon->soa_retry = SOA_RETRY; daemon->soa_expiry = SOA_EXPIRY; daemon->randport_limit = 1; - + daemon->host_index = SRC_AH; + #ifndef NO_ID add_txt("version.bind", "dnsmasq-" VERSION, 0 ); add_txt("authors.bind", "Simon Kelley", 0); From e8d19f71cca533f84e95f4937ab9e32598b11e7c Mon Sep 17 00:00:00 2001 From: Dominik Derigs Date: Sun, 16 Oct 2022 22:08:45 +0100 Subject: [PATCH 142/218] Enhance --hostdir so that records are automatically removed when re-reading. Initial patch from Dominik Derigs, re-written by Simon Kelley. Signed-off-by: DL6ER --- src/dnsmasq/cache.c | 21 +++++++++++++++++++++ src/dnsmasq/dnsmasq.h | 1 + src/dnsmasq/inotify.c | 6 +++++- 3 files changed, 27 insertions(+), 1 deletion(-) diff --git a/src/dnsmasq/cache.c b/src/dnsmasq/cache.c index d17a4489..e0e3326e 100644 --- a/src/dnsmasq/cache.c +++ b/src/dnsmasq/cache.c @@ -389,6 +389,27 @@ static int is_expired(time_t now, struct crec *crecp) return 1; } +/* Remove entries with a given UID from the cache */ +unsigned int cache_remove_uid(const unsigned int uid) +{ + int i; + unsigned int removed = 0; + struct crec *crecp, **up; + + for (i = 0; i < hash_size; i++) + for (crecp = hash_table[i], up = &hash_table[i]; crecp; crecp = crecp->hash_next) + if ((crecp->flags & (F_HOSTS | F_DHCP | F_CONFIG)) && crecp->uid == uid) + { + *up = crecp->hash_next; + free(crecp); + removed++; + } + else + up = &crecp->hash_next; + + return removed; +} + static struct crec *cache_scan_free(char *name, union all_addr *addr, unsigned short class, time_t now, unsigned int flags, struct crec **target_crec, unsigned int *target_uid) { diff --git a/src/dnsmasq/dnsmasq.h b/src/dnsmasq/dnsmasq.h index a6c5cc18..4f53405f 100644 --- a/src/dnsmasq/dnsmasq.h +++ b/src/dnsmasq/dnsmasq.h @@ -1333,6 +1333,7 @@ struct crec *cache_find_by_name(struct crec *crecp, char *name, time_t now, unsigned int prot); void cache_end_insert(void); void cache_start_insert(void); +unsigned int cache_remove_uid(const unsigned int uid); int cache_recv_insert(time_t now, int fd); struct crec *cache_insert(char *name, union all_addr *addr, unsigned short class, time_t now, unsigned long ttl, unsigned int flags); diff --git a/src/dnsmasq/inotify.c b/src/dnsmasq/inotify.c index 6e1ba18c..e257ec41 100644 --- a/src/dnsmasq/inotify.c +++ b/src/dnsmasq/inotify.c @@ -294,12 +294,16 @@ int inotify_check(time_t now) strcat(path, "/"); strcat(path, in->name); - my_syslog(LOG_INFO, _("inotify, new or changed file %s"), path); + my_syslog(LOG_INFO, _("inotify: new, removed or changed file %s"), path); if (dd->flags & AH_HOSTS) { if ((ah = dyndir_addhosts(dd, path))) { + const unsigned int removed = cache_remove_uid(ah->index); + if (removed > 0) + my_syslog(LOG_INFO, _("flushed %u outdated entries"), removed); + read_hostsfile(path, ah->index, 0, NULL, 0); #ifdef HAVE_DHCP if (daemon->dhcp || daemon->doing_dhcp6) From a1b66e89d204e517f653def9ea720a3ce925c9db Mon Sep 17 00:00:00 2001 From: Dominik Derigs Date: Sun, 16 Oct 2022 22:30:08 +0100 Subject: [PATCH 143/218] Handle multiple addresses when removing duplicates in host files. Signed-off-by: DL6ER --- src/dnsmasq/cache.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/dnsmasq/cache.c b/src/dnsmasq/cache.c index e0e3326e..d571cbec 100644 --- a/src/dnsmasq/cache.c +++ b/src/dnsmasq/cache.c @@ -1049,16 +1049,17 @@ struct crec *cache_find_by_addr(struct crec *crecp, union all_addr *addr, void add_hosts_entry(struct crec *cache, union all_addr *addr, int addrlen, unsigned int index, struct crec **rhash, int hashsz) { - struct crec *lookup = cache_find_by_name(NULL, cache_get_name(cache), 0, cache->flags & (F_IPV4 | F_IPV6)); int i; unsigned int j; + struct crec *lookup = NULL; /* Remove duplicates in hosts files. */ - if (lookup && (lookup->flags & F_HOSTS) && memcmp(&lookup->addr, addr, addrlen) == 0) - { - free(cache); - return; - } + while ((lookup = cache_find_by_name(lookup, cache_get_name(cache), 0, cache->flags & (F_IPV4 | F_IPV6)))) + if ((lookup->flags & F_HOSTS) && memcmp(&lookup->addr, addr, addrlen) == 0) + { + free(cache); + return; + } /* Ensure there is only one address -> name mapping (first one trumps) We do this by steam here, The entries are kept in hash chains, linked From 4cfb84ae993d65aa1e0e1bf8a328b50aeafacc19 Mon Sep 17 00:00:00 2001 From: Simon Kelley Date: Mon, 17 Oct 2022 21:15:43 +0100 Subject: [PATCH 144/218] Fix loss of DNS servers on config reload. A bug, introduced in 2.87, which could result in DNS servers being removed from the configuration when reloading server configuration from DBus, or re-reading /etc/resolv.conf Only servers from the same source should be replaced, but some servers from other sources (ie hard coded or another dynamic source) could mysteriously disappear. Signed-off-by: DL6ER --- src/dnsmasq/domain-match.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/dnsmasq/domain-match.c b/src/dnsmasq/domain-match.c index f7db0fe8..76a1109e 100644 --- a/src/dnsmasq/domain-match.c +++ b/src/dnsmasq/domain-match.c @@ -683,7 +683,9 @@ int add_update_server(int flags, serv->next = NULL; } break; - } + } + else + up = &serv->next; } if (serv) From cdf9d9d5ab04b4dd81a8e799b0838e95b5916061 Mon Sep 17 00:00:00 2001 From: Simon Kelley Date: Tue, 18 Oct 2022 16:06:48 +0100 Subject: [PATCH 145/218] Add --no-round-robin option. Signed-off-by: DL6ER --- src/dnsmasq/cache.c | 12 +++++++++--- src/dnsmasq/dnsmasq.h | 3 ++- src/dnsmasq/option.c | 3 +++ 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/src/dnsmasq/cache.c b/src/dnsmasq/cache.c index d571cbec..36d47812 100644 --- a/src/dnsmasq/cache.c +++ b/src/dnsmasq/cache.c @@ -234,7 +234,8 @@ static void cache_hash(struct crec *crecp) immortal entries are at the end of the hash-chain. This allows reverse searches and garbage collection to be optimised */ - struct crec **up = hash_bucket(cache_get_name(crecp)); + char *name = cache_get_name(crecp); + struct crec **up = hash_bucket(name); if (!(crecp->flags & F_REVERSE)) { @@ -245,6 +246,11 @@ static void cache_hash(struct crec *crecp) while (*up && !((*up)->flags & F_IMMORTAL)) up = &((*up)->hash_next); } + + /* Preserve order when inserting the same name multiple times. */ + while (*up && hostname_isequal(cache_get_name(*up), name)) + up = &((*up)->hash_next); + crecp->hash_next = *up; *up = crecp; } @@ -723,7 +729,7 @@ static struct crec *really_insert(char *name, union all_addr *addr, unsigned sho new->ttd = now + (time_t)ttl; new->next = new_chain; new_chain = new; - + return new; } @@ -901,7 +907,7 @@ int cache_find_non_terminal(char *name, time_t now) struct crec *cache_find_by_name(struct crec *crecp, char *name, time_t now, unsigned int prot) { struct crec *ans; - int no_rr = prot & F_NO_RR; + int no_rr = (prot & F_NO_RR) || option_bool(OPT_NORR); prot &= ~F_NO_RR; diff --git a/src/dnsmasq/dnsmasq.h b/src/dnsmasq/dnsmasq.h index 4f53405f..1fdb1cbc 100644 --- a/src/dnsmasq/dnsmasq.h +++ b/src/dnsmasq/dnsmasq.h @@ -287,7 +287,8 @@ struct event_desc { #define OPT_STRIP_ECS 69 #define OPT_STRIP_MAC 70 #define OPT_STALE_CACHE 71 -#define OPT_LAST 72 +#define OPT_NORR 72 +#define OPT_LAST 73 #define OPTION_BITS (sizeof(unsigned int)*8) #define OPTION_SIZE ( (OPT_LAST/OPTION_BITS)+((OPT_LAST%OPTION_BITS)!=0) ) diff --git a/src/dnsmasq/option.c b/src/dnsmasq/option.c index 70933c33..c4d41311 100644 --- a/src/dnsmasq/option.c +++ b/src/dnsmasq/option.c @@ -188,6 +188,7 @@ struct myoption { #define LOPT_RANDPORT_LIM 375 #define LOPT_FAST_RETRY 376 #define LOPT_STALE_CACHE 377 +#define LOPT_NORR 378 #ifdef HAVE_GETOPT_LONG static const struct option opts[] = @@ -240,6 +241,7 @@ static const struct myoption opts[] = { "localmx", 0, 0, 'L' }, { "local-ttl", 1, 0, 'T' }, { "no-negcache", 0, 0, 'N' }, + { "no-round-robin", 0, 0, LOPT_NORR }, { "addn-hosts", 1, 0, 'H' }, { "hostsdir", 1, 0, LOPT_HOST_INOTIFY }, { "query-port", 1, 0, 'Q' }, @@ -571,6 +573,7 @@ static struct { { LOPT_SCRIPT_TIME, OPT_LEASE_RENEW, NULL, gettext_noop("Call dhcp-script when lease expiry changes."), NULL }, { LOPT_UMBRELLA, ARG_ONE, "[=]", gettext_noop("Send Cisco Umbrella identifiers including remote IP."), NULL }, { LOPT_QUIET_TFTP, OPT_QUIET_TFTP, NULL, gettext_noop("Do not log routine TFTP."), NULL }, + { LOPT_NORR, OPT_NORR, NULL, gettext_noop("Suppress round-robin ordering of DNS records."), NULL }, { 0, 0, NULL, NULL, NULL } }; From 3a6dd32f9115ea083b8c6489d52b685167b33648 Mon Sep 17 00:00:00 2001 From: Simon Kelley Date: Thu, 27 Oct 2022 12:04:58 +0100 Subject: [PATCH 146/218] Fix in dhcpv4 rapid-commit code. 1) Cosmetic: don't log the tags twice. 2) Functional. If a host has an old lease for a different address, the rapid-commit will appear to work, but the old lease will not be removed and the new lease will not be recorded, so the client and server will have conflicting state, leading to problems later. Signed-off-by: Dominik Derigs --- src/dnsmasq/rfc2131.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/dnsmasq/rfc2131.c b/src/dnsmasq/rfc2131.c index 2056cbad..17e97b52 100644 --- a/src/dnsmasq/rfc2131.c +++ b/src/dnsmasq/rfc2131.c @@ -1153,15 +1153,22 @@ size_t dhcp_reply(struct dhcp_context *context, char *iface_name, int int_index, tagif_netid = run_tag_if(&context->netid); } - log_tags(tagif_netid, ntohl(mess->xid)); apply_delay(mess->xid, recvtime, tagif_netid); if (option_bool(OPT_RAPID_COMMIT) && option_find(mess, sz, OPTION_RAPID_COMMIT, 0)) { rapid_commit = 1; + /* If a lease exists for this host and another address, squash it. */ + if (lease && lease->addr.s_addr != mess->yiaddr.s_addr) + { + lease_prune(lease, now); + lease = NULL; + } goto rapid_commit; } + log_tags(tagif_netid, ntohl(mess->xid)); + daemon->metrics[METRIC_DHCPOFFER]++; log_packet("DHCPOFFER" , &mess->yiaddr, emac, emac_len, iface_name, NULL, NULL, mess->xid); From 779ba107fa5e302198bd5f4a226446aae3ee9b66 Mon Sep 17 00:00:00 2001 From: Dominik Derigs Date: Thu, 27 Oct 2022 12:36:38 +0100 Subject: [PATCH 147/218] Do not (try to) re-read deleted files inside a --hostsdir. Signed-off-by: Dominik Derigs --- src/dnsmasq/inotify.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/dnsmasq/inotify.c b/src/dnsmasq/inotify.c index e257ec41..522f6b72 100644 --- a/src/dnsmasq/inotify.c +++ b/src/dnsmasq/inotify.c @@ -293,8 +293,12 @@ int inotify_check(time_t now) strcpy(path, dd->dname); strcat(path, "/"); strcat(path, in->name); - - my_syslog(LOG_INFO, _("inotify: new, removed or changed file %s"), path); + + /* Is this is a deletion event? */ + if (in->mask & IN_DELETE) + my_syslog(LOG_INFO, _("inotify: %s (removed)"), path); + else + my_syslog(LOG_INFO, _("inotify: %s (new or modified)"), path); if (dd->flags & AH_HOSTS) { @@ -304,7 +308,9 @@ int inotify_check(time_t now) if (removed > 0) my_syslog(LOG_INFO, _("flushed %u outdated entries"), removed); - read_hostsfile(path, ah->index, 0, NULL, 0); + /* (Re-)load hostsfile only if this event isn't triggered by deletion */ + if (!(in->mask & IN_DELETE)) + read_hostsfile(path, ah->index, 0, NULL, 0); #ifdef HAVE_DHCP if (daemon->dhcp || daemon->doing_dhcp6) { From 8aba33f751aa861d954ad9015fe940b225359e09 Mon Sep 17 00:00:00 2001 From: Simon Kelley Date: Thu, 27 Oct 2022 13:14:45 +0100 Subject: [PATCH 148/218] Inotify: make "flushed" log message more understandable. Saying we've "flushed x outdated entries" is confusing, since the count is the total number of entries in the modified file, most of which are going to get added straight back when the file is re-read. The log now looks like dnsmasq: inotify: /tmp/dir/1 (new or modified) dnsmasq: inotify: flushed 1 addresses read from /tmp/dir/1 dnsmasq: read /tmp/dir/1 - 2 addresses which hopefully make it more obvious that /tmp/dir/1 contained one address before, and now contains two. Signed-off-by: Dominik Derigs --- src/dnsmasq/inotify.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/dnsmasq/inotify.c b/src/dnsmasq/inotify.c index 522f6b72..cc6d7bd0 100644 --- a/src/dnsmasq/inotify.c +++ b/src/dnsmasq/inotify.c @@ -296,9 +296,9 @@ int inotify_check(time_t now) /* Is this is a deletion event? */ if (in->mask & IN_DELETE) - my_syslog(LOG_INFO, _("inotify: %s (removed)"), path); - else - my_syslog(LOG_INFO, _("inotify: %s (new or modified)"), path); + my_syslog(LOG_INFO, _("inotify: %s removed"), path); + else + my_syslog(LOG_INFO, _("inotify: %s new or modified"), path); if (dd->flags & AH_HOSTS) { @@ -306,7 +306,7 @@ int inotify_check(time_t now) { const unsigned int removed = cache_remove_uid(ah->index); if (removed > 0) - my_syslog(LOG_INFO, _("flushed %u outdated entries"), removed); + my_syslog(LOG_INFO, _("inotify: flushed %u addresses read from %s"), removed, path); /* (Re-)load hostsfile only if this event isn't triggered by deletion */ if (!(in->mask & IN_DELETE)) From 2bf286322420dfdcc5b7bf3b67dfc62b03532599 Mon Sep 17 00:00:00 2001 From: Simon Kelley Date: Sun, 30 Oct 2022 15:40:20 +0000 Subject: [PATCH 149/218] Reconcile "names" and "address" counts when reading hostfiles. Signed-off-by: DL6ER --- src/dnsmasq/cache.c | 10 +++++----- src/dnsmasq/inotify.c | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/dnsmasq/cache.c b/src/dnsmasq/cache.c index 36d47812..5b3ceb43 100644 --- a/src/dnsmasq/cache.c +++ b/src/dnsmasq/cache.c @@ -190,7 +190,7 @@ void rehash(int size) else if (new_size <= hash_size || !(new = whine_malloc(new_size * sizeof(struct crec *)))) return; - for(i = 0; i < new_size; i++) + for (i = 0; i < new_size; i++) new[i] = NULL; old = hash_table; @@ -1170,7 +1170,7 @@ int read_hostsfile(char *filename, unsigned int index, int cache_size, struct cr { FILE *f = fopen(filename, "r"); char *token = daemon->namebuff, *domain_suffix = NULL; - int addr_count = 0, name_count = cache_size, lineno = 1; + int names_done = 0, name_count = cache_size, lineno = 1; unsigned int flags = 0; union all_addr addr; int atnl, addrlen = 0; @@ -1206,8 +1206,6 @@ int read_hostsfile(char *filename, unsigned int index, int cache_size, struct cr continue; } - addr_count++; - /* rehash every 1000 names. */ if (rhash && ((name_count - cache_size) > 1000)) { @@ -1239,6 +1237,7 @@ int read_hostsfile(char *filename, unsigned int index, int cache_size, struct cr cache->ttd = daemon->local_ttl; add_hosts_entry(cache, &addr, addrlen, index, rhash, hashsz); name_count++; + names_done++; } if ((cache = whine_malloc(SIZEOF_BARE_CREC + strlen(canon) + 1))) { @@ -1247,6 +1246,7 @@ int read_hostsfile(char *filename, unsigned int index, int cache_size, struct cr cache->ttd = daemon->local_ttl; add_hosts_entry(cache, &addr, addrlen, index, rhash, hashsz); name_count++; + names_done++; } free(canon); @@ -1263,7 +1263,7 @@ int read_hostsfile(char *filename, unsigned int index, int cache_size, struct cr if (rhash) rehash(name_count); - my_syslog(LOG_INFO, _("read %s - %d addresses"), filename, addr_count); + my_syslog(LOG_INFO, _("read %s - %d names"), filename, names_done); return name_count; } diff --git a/src/dnsmasq/inotify.c b/src/dnsmasq/inotify.c index cc6d7bd0..d3c8277b 100644 --- a/src/dnsmasq/inotify.c +++ b/src/dnsmasq/inotify.c @@ -306,7 +306,7 @@ int inotify_check(time_t now) { const unsigned int removed = cache_remove_uid(ah->index); if (removed > 0) - my_syslog(LOG_INFO, _("inotify: flushed %u addresses read from %s"), removed, path); + my_syslog(LOG_INFO, _("inotify: flushed %u names read from %s"), removed, path); /* (Re-)load hostsfile only if this event isn't triggered by deletion */ if (!(in->mask & IN_DELETE)) From 703ee7e2b0bddad0b0fdfa4f503b5a87cfa38aa8 Mon Sep 17 00:00:00 2001 From: Dominik Derigs Date: Sat, 5 Nov 2022 11:49:52 +0000 Subject: [PATCH 150/218] Allow domain names as well is IP addresses in --server options. Signed-off-by: DL6ER --- src/dnsmasq/dbus.c | 8 +- src/dnsmasq/dnsmasq.h | 13 +- src/dnsmasq/option.c | 312 +++++++++++++++++++++++++++++++----------- 3 files changed, 249 insertions(+), 84 deletions(-) diff --git a/src/dnsmasq/dbus.c b/src/dnsmasq/dbus.c index 4fdad465..5cbddd93 100644 --- a/src/dnsmasq/dbus.c +++ b/src/dnsmasq/dbus.c @@ -292,6 +292,11 @@ static DBusMessage* dbus_read_servers_ex(DBusMessage *message, int strings) u16 flags = 0; char interface[IF_NAMESIZE]; char *str_addr, *str_domain = NULL; + struct server_details sdetails = { 0 }; + sdetails.addr = &addr; + sdetails.source_addr = &source_addr; + sdetails.interface = interface; + sdetails.flags = &flags; if (strings) { @@ -375,7 +380,8 @@ static DBusMessage* dbus_read_servers_ex(DBusMessage *message, int strings) } /* parse the IP address */ - if ((addr_err = parse_server(str_addr, &addr, &source_addr, (char *) &interface, &flags))) + if ((addr_err = parse_server(str_addr, &sdetails, 0)) || + (addr_err = parse_server_addr(&sdetails))) { error = dbus_message_new_error_printf(message, DBUS_ERROR_INVALID_ARGS, "Invalid IP address '%s': %s", diff --git a/src/dnsmasq/dnsmasq.h b/src/dnsmasq/dnsmasq.h index 1fdb1cbc..bfc4bdd1 100644 --- a/src/dnsmasq/dnsmasq.h +++ b/src/dnsmasq/dnsmasq.h @@ -140,6 +140,7 @@ typedef unsigned long long u64; #include #include #include +#include #ifndef HAVE_LINUX_NETWORK # include #endif @@ -1304,6 +1305,14 @@ extern struct daemon { #endif } *daemon; +struct server_details { + union mysockaddr *addr, *source_addr; + struct addrinfo *hostinfo; + char *interface, *source, *scope_id, *interface_opt; + int serv_port, source_port, addr_type, scope_index, valid, resolved; + u16 *flags; +}; + /* cache.c */ void cache_init(void); void next_uid(struct crec *crecp); @@ -1496,8 +1505,8 @@ void read_servers_file(void); void set_option_bool(unsigned int opt); void reset_option_bool(unsigned int opt); struct hostsfile *expand_filelist(struct hostsfile *list); -char *parse_server(char *arg, union mysockaddr *addr, - union mysockaddr *source_addr, char *interface, u16 *flags); +char *parse_server(char *arg, struct server_details *sdetails, const int can_resolve); +char *parse_server_addr(struct server_details *sdetails); int option_read_dynfile(char *file, int flags); /* forward.c */ diff --git a/src/dnsmasq/option.c b/src/dnsmasq/option.c index c4d41311..d7af4879 100644 --- a/src/dnsmasq/option.c +++ b/src/dnsmasq/option.c @@ -859,117 +859,237 @@ static char *parse_mysockaddr(char *arg, union mysockaddr *addr) return NULL; } -char *parse_server(char *arg, union mysockaddr *addr, union mysockaddr *source_addr, char *interface, u16 *flags) +char *parse_server(char *arg, struct server_details *sdetails, const int can_resolve) { - int source_port = 0, serv_port = NAMESERVER_PORT; - char *portno, *source; - char *interface_opt = NULL; - int scope_index = 0; - char *scope_id; + sdetails->serv_port = NAMESERVER_PORT; + char *portno; + int ecode = 0; + struct addrinfo hints = { 0 }; - *interface = 0; + *sdetails->interface = 0; if (strcmp(arg, "#") == 0) { - if (flags) - *flags |= SERV_USE_RESOLV; + if (sdetails->flags) + *sdetails->flags |= SERV_USE_RESOLV; + sdetails->valid = 1; return NULL; } - if ((source = split_chr(arg, '@')) && /* is there a source. */ - (portno = split_chr(source, '#')) && - !atoi_check16(portno, &source_port)) + if ((sdetails->source = split_chr(arg, '@')) && /* is there a source. */ + (portno = split_chr(sdetails->source, '#')) && + !atoi_check16(portno, &sdetails->source_port)) return _("bad port"); if ((portno = split_chr(arg, '#')) && /* is there a port no. */ - !atoi_check16(portno, &serv_port)) + !atoi_check16(portno, &sdetails->serv_port)) return _("bad port"); - scope_id = split_chr(arg, '%'); + sdetails->scope_id = split_chr(arg, '%'); - if (source) { - interface_opt = split_chr(source, '@'); + if (sdetails->source) { + sdetails->interface_opt = split_chr(sdetails->source, '@'); - if (interface_opt) + if (sdetails->interface_opt) { #if defined(SO_BINDTODEVICE) - safe_strncpy(interface, source, IF_NAMESIZE); - source = interface_opt; + safe_strncpy(sdetails->interface, sdetails->source, IF_NAMESIZE); + sdetails->source = sdetails->interface_opt; #else return _("interface binding not supported"); #endif } } - if (inet_pton(AF_INET, arg, &addr->in.sin_addr) > 0) + if (inet_pton(AF_INET, arg, &sdetails->addr->in.sin_addr) > 0) + sdetails->addr_type = AF_INET; + else if (inet_pton(AF_INET6, arg, &sdetails->addr->in6.sin6_addr) > 0) + sdetails->addr_type = AF_INET6; + /* if the argument is neither an IPv4 not an IPv6 address, it might be a + hostname and we should try to resolve it to a suitable address. + However, we don't try this in domain_rev4/6 (can_resolve = 0) */ + else if (can_resolve) { - addr->in.sin_port = htons(serv_port); - addr->sa.sa_family = source_addr->sa.sa_family = AF_INET; + memset(&hints, 0, sizeof(hints)); + /* The AI_ADDRCONFIG flag ensures that then IPv4 addresses are returned in + the result only if the local system has at least one IPv4 address + configured, and IPv6 addresses are returned only if the local system + has at least one IPv6 address configured. The loopback address is not + considered for this case as valid as a configured address. This flag is + useful on, for example, IPv4-only systems, to ensure that getaddrinfo() + does not return IPv6 socket addresses that would always fail in + subsequent connect() or bind() attempts. */ + hints.ai_flags = AI_ADDRCONFIG; +#if defined(HAVE_IDN) && defined(AI_IDN) + /* If the AI_IDN flag is specified and we have glibc 2.3.4 or newer, then + the node name given in node is converted to IDN format if necessary. + The source encoding is that of the current locale. */ + hints.ai_flags |= AI_IDN; +#endif + /* The value AF_UNSPEC indicates that getaddrinfo() should return socket + addresses for any address family (either IPv4 or IPv6, for example) + that can be used with node and service "domain". */ + hints.ai_family = AF_UNSPEC; + + /* Get addresses suitable for sending datagrams or TCP connections. */ + hints.ai_socktype = 0; + + /* Get address associated with this hostname */ + ecode = getaddrinfo(arg, NULL, &hints, &sdetails->hostinfo); + if (ecode == 0) + { + /* The getaddrinfo() function allocated and initialized a linked list of + addrinfo structures, one for each network address that matches node + and service, subject to the restrictions imposed by our + above, and returns a pointer to the start of the list in . + The items in the linked list are linked by the field. */ + sdetails->valid = 1; + sdetails->resolved = 1; + return NULL; + } + else + { + /* Lookup failed, return human readable error string */ + if (ecode == EAI_AGAIN) + return _("Cannot resolve server name"); + else + return _((char*)gai_strerror(ecode)); + } + } + else + return _("bad address"); + + sdetails->valid = 1; + return NULL; +} + +char *parse_server_addr(struct server_details *sdetails) +{ + if (sdetails->addr_type == AF_INET) + { + sdetails->addr->in.sin_port = htons(sdetails->serv_port); + sdetails->addr->sa.sa_family = sdetails->source_addr->sa.sa_family = AF_INET; #ifdef HAVE_SOCKADDR_SA_LEN source_addr->in.sin_len = addr->in.sin_len = sizeof(struct sockaddr_in); #endif - source_addr->in.sin_addr.s_addr = INADDR_ANY; - source_addr->in.sin_port = htons(daemon->query_port); + sdetails->source_addr->in.sin_addr.s_addr = INADDR_ANY; + sdetails->source_addr->in.sin_port = htons(daemon->query_port); - if (source) + if (sdetails->source) { - if (flags) - *flags |= SERV_HAS_SOURCE; - source_addr->in.sin_port = htons(source_port); - if (!(inet_pton(AF_INET, source, &source_addr->in.sin_addr) > 0)) + if (sdetails->flags) + *sdetails->flags |= SERV_HAS_SOURCE; + sdetails->source_addr->in.sin_port = htons(sdetails->source_port); + if (inet_pton(AF_INET, sdetails->source, &sdetails->source_addr->in.sin_addr) == 0) { + if (inet_pton(AF_INET6, sdetails->source, &sdetails->source_addr->in6.sin6_addr) == 1) + { + sdetails->source_addr->sa.sa_family = AF_INET6; + /* When resolving a server IP by hostname, we can simply skip mismatching + server / source IP pairs. Otherwise, when an IP address is given directly, + this is a fatal error. */ + if(!sdetails->resolved) + return _("cannot use IPv4 server address with IPv6 source address"); + } + else + { #if defined(SO_BINDTODEVICE) - if (interface_opt) - return _("interface can only be specified once"); - - source_addr->in.sin_addr.s_addr = INADDR_ANY; - safe_strncpy(interface, source, IF_NAMESIZE); + if (sdetails->interface_opt) + return _("interface can only be specified once"); + + sdetails->source_addr->in.sin_addr.s_addr = INADDR_ANY; + safe_strncpy(sdetails->interface, sdetails->source, IF_NAMESIZE); #else - return _("interface binding not supported"); + return _("interface binding not supported"); #endif + } } } } - else if (inet_pton(AF_INET6, arg, &addr->in6.sin6_addr) > 0) + else if (sdetails->addr_type == AF_INET6) { - if (scope_id && (scope_index = if_nametoindex(scope_id)) == 0) + if (sdetails->scope_id && (sdetails->scope_index = if_nametoindex(sdetails->scope_id)) == 0) return _("bad interface name"); - - addr->in6.sin6_port = htons(serv_port); - addr->in6.sin6_scope_id = scope_index; - source_addr->in6.sin6_addr = in6addr_any; - source_addr->in6.sin6_port = htons(daemon->query_port); - source_addr->in6.sin6_scope_id = 0; - addr->sa.sa_family = source_addr->sa.sa_family = AF_INET6; - addr->in6.sin6_flowinfo = source_addr->in6.sin6_flowinfo = 0; + + sdetails->addr->in6.sin6_port = htons(sdetails->serv_port); + sdetails->addr->in6.sin6_scope_id = sdetails->scope_index; + sdetails->source_addr->in6.sin6_addr = in6addr_any; + sdetails->source_addr->in6.sin6_port = htons(daemon->query_port); + sdetails->source_addr->in6.sin6_scope_id = 0; + sdetails->addr->sa.sa_family = sdetails->source_addr->sa.sa_family = AF_INET6; + sdetails->addr->in6.sin6_flowinfo = sdetails->source_addr->in6.sin6_flowinfo = 0; #ifdef HAVE_SOCKADDR_SA_LEN - addr->in6.sin6_len = source_addr->in6.sin6_len = sizeof(addr->in6); + sdetails->addr->in6.sin6_len = sdetails->source_addr->in6.sin6_len = sizeof(addr->in6); #endif - if (source) + if (sdetails->source) { - if (flags) - *flags |= SERV_HAS_SOURCE; - source_addr->in6.sin6_port = htons(source_port); - if (inet_pton(AF_INET6, source, &source_addr->in6.sin6_addr) == 0) + if (sdetails->flags) + *sdetails->flags |= SERV_HAS_SOURCE; + sdetails->source_addr->in6.sin6_port = htons(sdetails->source_port); + if (inet_pton(AF_INET6, sdetails->source, &sdetails->source_addr->in6.sin6_addr) == 0) { + if (inet_pton(AF_INET, sdetails->source, &sdetails->source_addr->in.sin_addr) == 1) + { + sdetails->source_addr->sa.sa_family = AF_INET; + /* When resolving a server IP by hostname, we can simply skip mismatching + server / source IP pairs. Otherwise, when an IP address is given directly, + this is a fatal error. */ + if(!sdetails->resolved) + return _("cannot use IPv6 server address with IPv4 source address"); + } + else + { #if defined(SO_BINDTODEVICE) - if (interface_opt) - return _("interface can only be specified once"); - - source_addr->in6.sin6_addr = in6addr_any; - safe_strncpy(interface, source, IF_NAMESIZE); + if (sdetails->interface_opt) + return _("interface can only be specified once"); + + sdetails->source_addr->in6.sin6_addr = in6addr_any; + safe_strncpy(sdetails->interface, sdetails->source, IF_NAMESIZE); #else - return _("interface binding not supported"); + return _("interface binding not supported"); #endif + } } } } else return _("bad address"); - return NULL; } +static int parse_server_next(struct server_details *sdetails) +{ + /* Looping over resolved addresses? */ + if (sdetails->hostinfo) + { + /* Get address type */ + sdetails->addr_type = sdetails->hostinfo->ai_family; + + /* Get address */ + if (sdetails->addr_type == AF_INET) + memcpy(&sdetails->addr->in.sin_addr, + &((struct sockaddr_in *) sdetails->hostinfo->ai_addr)->sin_addr, + sizeof(sdetails->addr->in.sin_addr)); + else if (sdetails->addr_type == AF_INET6) + memcpy(&sdetails->addr->in6.sin6_addr, + &((struct sockaddr_in6 *) sdetails->hostinfo->ai_addr)->sin6_addr, + sizeof(sdetails->addr->in6.sin6_addr)); + + /* Iterate to the next available address */ + sdetails->valid = sdetails->hostinfo->ai_next != NULL; + sdetails->hostinfo = sdetails->hostinfo->ai_next; + return 1; + } + else if (sdetails->valid) + { + /* When using an IP address, we return the address only once */ + sdetails->valid = 0; + return 1; + } + /* Stop iterating here, we used all available addresses */ + return 0; +} + static char *domain_rev4(int from_file, char *server, struct in_addr *addr4, int size) { int i, j; @@ -980,10 +1100,16 @@ static char *domain_rev4(int from_file, char *server, struct in_addr *addr4, int union mysockaddr serv_addr, source_addr; char interface[IF_NAMESIZE+1]; int count = 1, rem, addrbytes, addrbits; - + struct server_details sdetails = { 0 }; + sdetails.addr = &serv_addr; + sdetails.source_addr = &source_addr; + sdetails.interface = interface; + sdetails.flags = &flags; + if (!server) flags = SERV_LITERAL_ADDRESS; - else if ((string = parse_server(server, &serv_addr, &source_addr, interface, &flags))) + else if ((string = parse_server(server, &sdetails, 0)) || + (string = parse_server_addr(&sdetails))) return string; if (from_file) @@ -1039,10 +1165,16 @@ static char *domain_rev6(int from_file, char *server, struct in6_addr *addr6, in union mysockaddr serv_addr, source_addr; char interface[IF_NAMESIZE+1]; int count = 1, rem, addrbytes, addrbits; + struct server_details sdetails = { 0 }; + sdetails.addr = &serv_addr; + sdetails.source_addr = &source_addr; + sdetails.interface = interface; + sdetails.flags = &flags; if (!server) flags = SERV_LITERAL_ADDRESS; - else if ((string = parse_server(server, &serv_addr, &source_addr, interface, &flags))) + else if ((string = parse_server(server, &sdetails, 0)) || + (string = parse_server_addr(&sdetails))) return string; if (from_file) @@ -2804,6 +2936,12 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma union mysockaddr serv_addr, source_addr; char interface[IF_NAMESIZE+1]; + struct server_details sdetails = { 0 }; + sdetails.addr = &serv_addr; + sdetails.source_addr = &source_addr; + sdetails.interface = interface; + sdetails.flags = &flags; + unhide_metas(arg); /* split the domain args, if any and skip to the end of them. */ @@ -2836,36 +2974,48 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma } else { - if ((err = parse_server(arg, &serv_addr, &source_addr, interface, &flags))) + if ((err = parse_server(arg, &sdetails, 1))) ret_err(err); } if (servers_only && option == 'S') flags |= SERV_FROM_FILE; - while (1) + while (parse_server_next(&sdetails)) { - /* server=//1.2.3.4 is special. */ - if (lastdomain) - { - if (strlen(domain) == 0) - flags |= SERV_FOR_NODOTS; - else - flags &= ~SERV_FOR_NODOTS; + if ((err = parse_server_addr(&sdetails))) + ret_err(err); - /* address=/#/ matches the same as without domain */ - if (option == 'A' && domain[0] == '#' && domain[1] == 0) - domain[0] = 0; + /* When source is set only use DNS records of the same type and skip all others */ + if (flags & SERV_HAS_SOURCE && sdetails.addr_type != sdetails.source_addr->sa.sa_family) + continue; + + while (1) + { + /* server=//1.2.3.4 is special. */ + if (lastdomain) + { + if (strlen(domain) == 0) + flags |= SERV_FOR_NODOTS; + else + flags &= ~SERV_FOR_NODOTS; + + /* address=/#/ matches the same as without domain */ + if (option == 'A' && domain[0] == '#' && domain[1] == 0) + domain[0] = 0; + } + + if (!add_update_server(flags, sdetails.addr, sdetails.source_addr, sdetails.interface, domain, &addr)) + ret_err(gen_err); + + if (!lastdomain || domain == lastdomain) + break; + + domain += strlen(domain) + 1; } - - if (!add_update_server(flags, &serv_addr, &source_addr, interface, domain, &addr)) - ret_err(gen_err); - - if (!lastdomain || domain == lastdomain) - break; - - domain += strlen(domain) + 1; } + if (sdetails.resolved) + freeaddrinfo(sdetails.hostinfo); break; } From 1b2612cff63ab2c86a86d7aa6815cf1343b91b83 Mon Sep 17 00:00:00 2001 From: Simon Kelley Date: Sun, 6 Nov 2022 20:18:27 +0000 Subject: [PATCH 151/218] Fix breakage of --local=/domain.name/1.2.3.4 in immediately previous commit. Signed-off-by: DL6ER --- src/dnsmasq/option.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/dnsmasq/option.c b/src/dnsmasq/option.c index d7af4879..d27752e7 100644 --- a/src/dnsmasq/option.c +++ b/src/dnsmasq/option.c @@ -2981,9 +2981,9 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma if (servers_only && option == 'S') flags |= SERV_FROM_FILE; - while (parse_server_next(&sdetails)) + while ((flags & SERV_LITERAL_ADDRESS) || parse_server_next(&sdetails)) { - if ((err = parse_server_addr(&sdetails))) + if (!(flags & SERV_LITERAL_ADDRESS) && (err = parse_server_addr(&sdetails))) ret_err(err); /* When source is set only use DNS records of the same type and skip all others */ @@ -3013,9 +3013,13 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma domain += strlen(domain) + 1; } + + if (flags & SERV_LITERAL_ADDRESS) + break; } - if (sdetails.resolved) - freeaddrinfo(sdetails.hostinfo); + + if (sdetails.resolved) + freeaddrinfo(sdetails.hostinfo); break; } From 891acaa301c59ab4863aac991dd1cf573783e9c9 Mon Sep 17 00:00:00 2001 From: Simon Kelley Date: Sun, 6 Nov 2022 20:56:43 +0000 Subject: [PATCH 152/218] Extend specifying DNS servers by domain-name to --rev-server Also Dbus SetDomainServers method. Revert getaddrinfo hints.ai_socktype to SOCK_DGRAM to eliminate duplicating every address three times for DGRAM, STREAM and RAW in the results. Signed-off-by: DL6ER --- src/dnsmasq/dbus.c | 2 +- src/dnsmasq/dnsmasq.h | 2 +- src/dnsmasq/option.c | 81 ++++++++++++++++++++++++++++--------------- 3 files changed, 55 insertions(+), 30 deletions(-) diff --git a/src/dnsmasq/dbus.c b/src/dnsmasq/dbus.c index 5cbddd93..870912a7 100644 --- a/src/dnsmasq/dbus.c +++ b/src/dnsmasq/dbus.c @@ -380,7 +380,7 @@ static DBusMessage* dbus_read_servers_ex(DBusMessage *message, int strings) } /* parse the IP address */ - if ((addr_err = parse_server(str_addr, &sdetails, 0)) || + if ((addr_err = parse_server(str_addr, &sdetails)) || (addr_err = parse_server_addr(&sdetails))) { error = dbus_message_new_error_printf(message, DBUS_ERROR_INVALID_ARGS, diff --git a/src/dnsmasq/dnsmasq.h b/src/dnsmasq/dnsmasq.h index bfc4bdd1..7833fa78 100644 --- a/src/dnsmasq/dnsmasq.h +++ b/src/dnsmasq/dnsmasq.h @@ -1505,7 +1505,7 @@ void read_servers_file(void); void set_option_bool(unsigned int opt); void reset_option_bool(unsigned int opt); struct hostsfile *expand_filelist(struct hostsfile *list); -char *parse_server(char *arg, struct server_details *sdetails, const int can_resolve); +char *parse_server(char *arg, struct server_details *sdetails); char *parse_server_addr(struct server_details *sdetails); int option_read_dynfile(char *file, int flags); diff --git a/src/dnsmasq/option.c b/src/dnsmasq/option.c index d27752e7..4b889826 100644 --- a/src/dnsmasq/option.c +++ b/src/dnsmasq/option.c @@ -859,7 +859,7 @@ static char *parse_mysockaddr(char *arg, union mysockaddr *addr) return NULL; } -char *parse_server(char *arg, struct server_details *sdetails, const int can_resolve) +char *parse_server(char *arg, struct server_details *sdetails) { sdetails->serv_port = NAMESERVER_PORT; char *portno; @@ -905,11 +905,10 @@ char *parse_server(char *arg, struct server_details *sdetails, const int can_res sdetails->addr_type = AF_INET; else if (inet_pton(AF_INET6, arg, &sdetails->addr->in6.sin6_addr) > 0) sdetails->addr_type = AF_INET6; - /* if the argument is neither an IPv4 not an IPv6 address, it might be a - hostname and we should try to resolve it to a suitable address. - However, we don't try this in domain_rev4/6 (can_resolve = 0) */ - else if (can_resolve) + else { + /* if the argument is neither an IPv4 not an IPv6 address, it might be a + hostname and we should try to resolve it to a suitable address. */ memset(&hints, 0, sizeof(hints)); /* The AI_ADDRCONFIG flag ensures that then IPv4 addresses are returned in the result only if the local system has at least one IPv4 address @@ -931,8 +930,10 @@ char *parse_server(char *arg, struct server_details *sdetails, const int can_res that can be used with node and service "domain". */ hints.ai_family = AF_UNSPEC; - /* Get addresses suitable for sending datagrams or TCP connections. */ - hints.ai_socktype = 0; + /* Get addresses suitable for sending datagrams. We assume that we can use the + same addresses for TCP connections. Settting this to zero gets each address + threes times, for SOCK_STREAM, SOCK_RAW and SOCK_DGRAM, which is not useful. */ + hints.ai_socktype = SOCK_DGRAM; /* Get address associated with this hostname */ ecode = getaddrinfo(arg, NULL, &hints, &sdetails->hostinfo); @@ -956,8 +957,6 @@ char *parse_server(char *arg, struct server_details *sdetails, const int can_res return _((char*)gai_strerror(ecode)); } } - else - return _("bad address"); sdetails->valid = 1; return NULL; @@ -988,7 +987,7 @@ char *parse_server_addr(struct server_details *sdetails) /* When resolving a server IP by hostname, we can simply skip mismatching server / source IP pairs. Otherwise, when an IP address is given directly, this is a fatal error. */ - if(!sdetails->resolved) + if (!sdetails->resolved) return _("cannot use IPv4 server address with IPv6 source address"); } else @@ -1108,20 +1107,19 @@ static char *domain_rev4(int from_file, char *server, struct in_addr *addr4, int if (!server) flags = SERV_LITERAL_ADDRESS; - else if ((string = parse_server(server, &sdetails, 0)) || - (string = parse_server_addr(&sdetails))) + else if ((string = parse_server(server, &sdetails))) return string; - + if (from_file) flags |= SERV_FROM_FILE; - + rem = size & 0x7; addrbytes = (32 - size) >> 3; addrbits = (32 - size) & 7; if (size > 32 || size < 1) return _("bad IPv4 prefix length"); - + /* Zero out last address bits according to CIDR mask */ ((u8 *)addr4)[3-addrbytes] &= ~((1 << addrbits)-1); @@ -1135,23 +1133,37 @@ static char *domain_rev4(int from_file, char *server, struct in_addr *addr4, int *domain = 0; string = domain; msize = size/8; - + for (j = (rem == 0) ? msize-1 : msize; j >= 0; j--) { int dig = ((unsigned char *)addr4)[j]; - + if (j == msize) dig += i; - + string += sprintf(string, "%d.", dig); } - + sprintf(string, "in-addr.arpa"); - if (!add_update_server(flags, &serv_addr, &source_addr, interface, domain, NULL)) - return _("error"); + if (flags & SERV_LITERAL_ADDRESS) + { + if (!add_update_server(flags, &serv_addr, &source_addr, interface, domain, NULL)) + return _("error"); + } + else + { + while (parse_server_next(&sdetails)) + { + if ((string = parse_server_addr(&sdetails))) + return string; + + if (!add_update_server(flags, &serv_addr, &source_addr, interface, domain, NULL)) + return _("error"); + } + } } - + return NULL; } @@ -1173,8 +1185,7 @@ static char *domain_rev6(int from_file, char *server, struct in6_addr *addr6, in if (!server) flags = SERV_LITERAL_ADDRESS; - else if ((string = parse_server(server, &sdetails, 0)) || - (string = parse_server_addr(&sdetails))) + else if ((string = parse_server(server, &sdetails))) return string; if (from_file) @@ -1215,10 +1226,24 @@ static char *domain_rev6(int from_file, char *server, struct in6_addr *addr6, in sprintf(string, "ip6.arpa"); - if (!add_update_server(flags, &serv_addr, &source_addr, interface, domain, NULL)) - return _("error"); + if (flags & SERV_LITERAL_ADDRESS) + { + if (!add_update_server(flags, &serv_addr, &source_addr, interface, domain, NULL)) + return _("error"); + } + else + { + while (parse_server_next(&sdetails)) + { + if ((string = parse_server_addr(&sdetails))) + return string; + + if (!add_update_server(flags, &serv_addr, &source_addr, interface, domain, NULL)) + return _("error"); + } + } } - + return NULL; } @@ -2974,7 +2999,7 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma } else { - if ((err = parse_server(arg, &sdetails, 1))) + if ((err = parse_server(arg, &sdetails))) ret_err(err); } From 801f034ca1b633abf88594e770645de8f478a182 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 7 Nov 2022 13:04:08 +0100 Subject: [PATCH 153/218] Add dnsmasq tag v2.88test2 Signed-off-by: DL6ER --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 62ce77e7..d8682052 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -11,6 +11,6 @@ cmake_minimum_required(VERSION 2.8.12) project(PIHOLE_FTL C) -set(DNSMASQ_VERSION pi-hole-v2.88test1) +set(DNSMASQ_VERSION pi-hole-v2.88test2) add_subdirectory(src) From 42c71058a94cfe2e37159fe75b10651676c60100 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 7 Nov 2022 13:38:02 +0100 Subject: [PATCH 154/218] Add support for dnsmasq flags F_SRV and F_STALE Signed-off-by: DL6ER --- src/dnsmasq_interface.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c index 799451a6..c1791cd8 100644 --- a/src/dnsmasq_interface.c +++ b/src/dnsmasq_interface.c @@ -95,7 +95,7 @@ static struct { static union mysockaddr last_server = {{ 0 }}; unsigned char* pihole_privacylevel = &config.privacylevel; -const char *flagnames[] = {"F_IMMORTAL ", "F_NAMEP ", "F_REVERSE ", "F_FORWARD ", "F_DHCP ", "F_NEG ", "F_HOSTS ", "F_IPV4 ", "F_IPV6 ", "F_BIGNAME ", "F_NXDOMAIN ", "F_CNAME ", "F_DNSKEY ", "F_CONFIG ", "F_DS ", "F_DNSSECOK ", "F_UPSTREAM ", "F_RRNAME ", "F_SERVER ", "F_QUERY ", "F_NOERR ", "F_AUTH ", "F_DNSSEC ", "F_KEYTAG ", "F_SECSTAT ", "F_NO_RR ", "F_IPSET ", "F_NOEXTRA ", "F_SERVFAIL", "F_RCODE"}; +const char *flagnames[] = {"F_IMMORTAL ", "F_NAMEP ", "F_REVERSE ", "F_FORWARD ", "F_DHCP ", "F_NEG ", "F_HOSTS ", "F_IPV4 ", "F_IPV6 ", "F_BIGNAME ", "F_NXDOMAIN ", "F_CNAME ", "F_DNSKEY ", "F_CONFIG ", "F_DS ", "F_DNSSECOK ", "F_UPSTREAM ", "F_RRNAME ", "F_SERVER ", "F_QUERY ", "F_NOERR ", "F_AUTH ", "F_DNSSEC ", "F_KEYTAG ", "F_SECSTAT ", "F_NO_RR ", "F_IPSET ", "F_NOEXTRA ", "F_SERVFAIL", "F_RCODE", "F_SRV", "F_STALE" }; void FTL_hook(unsigned int flags, const char *name, union all_addr *addr, char *arg, int id, unsigned short type, const char* file, const int line) { From 7ecd358f9638cfcc1a835754c2fd005210258e45 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 7 Nov 2022 13:56:44 +0100 Subject: [PATCH 155/218] Allow FTL to analyze stale cache replies. They are assigned to a new query type (17) Signed-off-by: DL6ER --- src/database/query-table.c | 1 + src/datastructure.c | 69 ++++++++++++++++++++++++++------------ src/dnsmasq_interface.c | 22 ++++++++---- src/enums.h | 1 + src/gc.c | 1 + 5 files changed, 66 insertions(+), 28 deletions(-) diff --git a/src/database/query-table.c b/src/database/query-table.c index ed532066..bf2bdaaf 100644 --- a/src/database/query-table.c +++ b/src/database/query-table.c @@ -988,6 +988,7 @@ void DB_read_queries(void) break; case QUERY_CACHE: // Cached or local config + case QUERY_CACHE_STALE: // Nothing to be done here break; diff --git a/src/datastructure.c b/src/datastructure.c index 9a69b35c..a2bf2a2f 100644 --- a/src/datastructure.c +++ b/src/datastructure.c @@ -530,6 +530,7 @@ bool __attribute__ ((const)) is_blocked(const enum query_status status) case QUERY_RETRIED: case QUERY_RETRIED_DNSSEC: case QUERY_IN_PROGRESS: + case QUERY_CACHE_STALE: case QUERY_STATUS_MAX: default: return false; @@ -549,32 +550,58 @@ bool __attribute__ ((const)) is_blocked(const enum query_status status) } } -static const char *query_status_str[QUERY_STATUS_MAX] = { - "UNKNOWN", - "GRAVITY", - "FORWARDED", - "CACHE", - "REGEX", - "BLACKLIST", - "EXTERNAL_BLOCKED_IP", - "EXTERNAL_BLOCKED_NULL", - "EXTERNAL_BLOCKED_NXRA", - "GRAVITY_CNAME", - "REGEX_CNAME", - "BLACKLIST_CNAME", - "RETRIED", - "RETRIED_DNSSEC", - "IN_PROGRESS", - "DBBUSY", - "SPECIAL_DOMAIN" -}; +static const char* __attribute__ ((const)) query_status_str(const enum query_status status) +{ + switch (status) + { + case QUERY_UNKNOWN: + return "UNKNOWN"; + case QUERY_GRAVITY: + return "GRAVITY"; + case QUERY_FORWARDED: + return "FORWARDED"; + case QUERY_CACHE: + return "CACHE"; + case QUERY_REGEX: + return "REGEX"; + case QUERY_BLACKLIST: + return "BLACKLIST"; + case QUERY_EXTERNAL_BLOCKED_IP: + return "EXTERNAL_BLOCKED_IP"; + case QUERY_EXTERNAL_BLOCKED_NULL: + return "EXTERNAL_BLOCKED_NULL"; + case QUERY_EXTERNAL_BLOCKED_NXRA: + return "EXTERNAL_BLOCKED_NXRA"; + case QUERY_GRAVITY_CNAME: + return "GRAVITY_CNAME"; + case QUERY_REGEX_CNAME: + return "REGEX_CNAME"; + case QUERY_BLACKLIST_CNAME: + return "BLACKLIST_CNAME"; + case QUERY_RETRIED: + return "RETRIED"; + case QUERY_RETRIED_DNSSEC: + return "RETRIED_DNSSEC"; + case QUERY_IN_PROGRESS: + return "IN_PROGRESS"; + case QUERY_DBBUSY: + return "DBBUSY"; + case QUERY_SPECIAL_DOMAIN: + return "SPECIAL_DOMAIN"; + case QUERY_CACHE_STALE: + return "CACHE_STALE"; + case QUERY_STATUS_MAX: + return NULL; + } + return NULL; +} void _query_set_status(queriesData *query, const enum query_status new_status, const char *func, const int line, const char *file) { // Debug logging if(config.debug & DEBUG_STATUS) { - const char *oldstr = query->status < QUERY_STATUS_MAX ? query_status_str[query->status] : "INVALID"; + const char *oldstr = query->status < QUERY_STATUS_MAX ? query_status_str(query->status) : "INVALID"; if(query->status == new_status) { logg("Query %i: status unchanged: %s (%d) in %s() (%s:%i)", @@ -582,7 +609,7 @@ void _query_set_status(queriesData *query, const enum query_status new_status, c } else { - const char *newstr = new_status < QUERY_STATUS_MAX ? query_status_str[new_status] : "INVALID"; + const char *newstr = new_status < QUERY_STATUS_MAX ? query_status_str(new_status) : "INVALID"; logg("Query %i: status changed: %s (%d) -> %s (%d) in %s() (%s:%i)", query->id, oldstr, query->status, newstr, new_status, func, short_path(file), line); } diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c index c1791cd8..49c8bc87 100644 --- a/src/dnsmasq_interface.c +++ b/src/dnsmasq_interface.c @@ -1932,6 +1932,9 @@ static void FTL_reply(const unsigned int flags, const char *name, const union al logg("***** Unknown cache query"); } + // Is this a stale reply? + const bool stale = flags & F_STALE; + // Possible debugging output if(config.debug & DEBUG_QUERIES) { @@ -1983,16 +1986,18 @@ static void FTL_reply(const unsigned int flags, const char *name, const union al if(cached || last_server.sa.sa_family == 0) // Log cache or upstream reply from unknown source - logg("**** got %s reply: %s is %s (ID %i, %s:%i)", - cached ? "cache" : "upstream", dispname, answer, id, file, line); + logg("**** got %s%s reply: %s is %s (ID %i, %s:%i)", + stale ? "stale ": "", cached ? "cache" : "upstream", + dispname, answer, id, file, line); else { char ip[ADDRSTRLEN+1] = { 0 }; in_port_t port = 0; mysockaddr_extract_ip_port(&last_server, ip, &port); // Log server which replied to our request - logg("**** got %s reply from %s#%d: %s is %s (ID %i, %s:%i)", - cached ? "cache" : "upstream", ip, port, dispname, answer, id, file, line); + logg("**** got %s%s reply from %s#%d: %s is %s (ID %i, %s:%i)", + stale ? "stale ": "", cached ? "cache" : "upstream", + ip, port, dispname, answer, id, file, line); } } @@ -2042,13 +2047,16 @@ static void FTL_reply(const unsigned int flags, const char *name, const union al return; } + // Determine query status (live or stale data?) + const enum query_status qs = stale ? QUERY_CACHE_STALE : QUERY_CACHE; + // This is either a reply served from cache or a blocked query (which appear // to be from cache because of flags containing F_HOSTS) if(cached) { // Set status of this query only if this is not a blocked query if(!is_blocked(query->status)) - query_set_status(query, QUERY_CACHE); + query_set_status(query, qs); // Detect if returned IP indicates that this query was blocked const enum query_status new_status = detect_blocked_IP(flags, addr, query, domain); @@ -2089,7 +2097,7 @@ static void FTL_reply(const unsigned int flags, const char *name, const union al // Answered from a custom (user provided) cache file or because // we're the authoritative DNS server (e.g. DHCP server and this // is our own domain) - query_set_status(query, QUERY_CACHE); + query_set_status(query, qs); // Save reply type and update individual reply counters query_set_reply(flags, 0, addr, query, response); @@ -3342,7 +3350,7 @@ int check_struct_sizes(void) result += check_one_struct("regexData", sizeof(regexData), 56, 44); result += check_one_struct("SharedMemory", sizeof(SharedMemory), 24, 12); result += check_one_struct("ShmSettings", sizeof(ShmSettings), 16, 16); - result += check_one_struct("countersStruct", sizeof(countersStruct), 244, 244); + result += check_one_struct("countersStruct", sizeof(countersStruct), 248, 248); result += check_one_struct("sqlite3_stmt_vec", sizeof(sqlite3_stmt_vec), 32, 16); if(result == 0) diff --git a/src/enums.h b/src/enums.h index cd4dd3fd..9ec7dc9d 100644 --- a/src/enums.h +++ b/src/enums.h @@ -46,6 +46,7 @@ enum query_status { QUERY_IN_PROGRESS, QUERY_DBBUSY, QUERY_SPECIAL_DOMAIN, + QUERY_CACHE_STALE, QUERY_STATUS_MAX } __attribute__ ((packed)); diff --git a/src/gc.c b/src/gc.c index c10c5d60..47734c11 100644 --- a/src/gc.c +++ b/src/gc.c @@ -215,6 +215,7 @@ void *GC_thread(void *val) // Adjusting counters is done below in moveOverTimeMemory() break; case QUERY_CACHE: + case QUERY_CACHE_STALE: // Answered from local cache _or_ local config break; case QUERY_GRAVITY: // Blocked by Pi-hole's blocking lists (fall through) From 0ae86d264182b46ee7dc8ca7a77fad6d272b03c6 Mon Sep 17 00:00:00 2001 From: Simon Kelley Date: Mon, 7 Nov 2022 22:01:28 +0000 Subject: [PATCH 156/218] Call freeaddrinfo() in domain_rev[46]() Signed-off-by: DL6ER --- src/dnsmasq/option.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/dnsmasq/option.c b/src/dnsmasq/option.c index 4b889826..ba893534 100644 --- a/src/dnsmasq/option.c +++ b/src/dnsmasq/option.c @@ -1161,6 +1161,9 @@ static char *domain_rev4(int from_file, char *server, struct in_addr *addr4, int if (!add_update_server(flags, &serv_addr, &source_addr, interface, domain, NULL)) return _("error"); } + + if (sdetails.resolved) + freeaddrinfo(sdetails.hostinfo); } } @@ -1241,6 +1244,9 @@ static char *domain_rev6(int from_file, char *server, struct in6_addr *addr6, in if (!add_update_server(flags, &serv_addr, &source_addr, interface, domain, NULL)) return _("error"); } + + if (sdetails.resolved) + freeaddrinfo(sdetails.hostinfo); } } From ed8d37bf0ba031ddfea5036021052669d86115f3 Mon Sep 17 00:00:00 2001 From: Simon Kelley Date: Mon, 7 Nov 2022 23:00:34 +0000 Subject: [PATCH 157/218] Make specifying nameservers by name work for DBus API. Signed-off-by: DL6ER --- src/dnsmasq/dbus.c | 74 ++++++++++++++++++++++++++++++++----------- src/dnsmasq/dnsmasq.h | 1 + src/dnsmasq/option.c | 3 +- 3 files changed, 59 insertions(+), 19 deletions(-) diff --git a/src/dnsmasq/dbus.c b/src/dnsmasq/dbus.c index 870912a7..ef807111 100644 --- a/src/dnsmasq/dbus.c +++ b/src/dnsmasq/dbus.c @@ -379,21 +379,6 @@ static DBusMessage* dbus_read_servers_ex(DBusMessage *message, int strings) strcpy(str_addr, str); } - /* parse the IP address */ - if ((addr_err = parse_server(str_addr, &sdetails)) || - (addr_err = parse_server_addr(&sdetails))) - { - error = dbus_message_new_error_printf(message, DBUS_ERROR_INVALID_ARGS, - "Invalid IP address '%s': %s", - str, addr_err); - break; - } - - /* 0.0.0.0 for server address == NULL, for Dbus */ - if (addr.in.sin_family == AF_INET && - addr.in.sin_addr.s_addr == 0) - flags |= SERV_LITERAL_ADDRESS; - if (strings) { char *p; @@ -407,7 +392,31 @@ static DBusMessage* dbus_read_servers_ex(DBusMessage *message, int strings) else p = NULL; - add_update_server(flags | SERV_FROM_DBUS, &addr, &source_addr, interface, str_domain, NULL); + if (strings && strlen(str_addr) == 0) + add_update_server(SERV_LITERAL_ADDRESS | SERV_FROM_DBUS, &addr, &source_addr, interface, str_domain, NULL); + else + { + if ((addr_err = parse_server(str_addr, &sdetails))) + { + error = dbus_message_new_error_printf(message, DBUS_ERROR_INVALID_ARGS, + "Invalid IP address '%s': %s", + str, addr_err); + break; + } + + while (parse_server_next(&sdetails)) + { + if ((addr_err = parse_server_addr(&sdetails))) + { + error = dbus_message_new_error_printf(message, DBUS_ERROR_INVALID_ARGS, + "Invalid IP address '%s': %s", + str, addr_err); + break; + } + + add_update_server(flags | SERV_FROM_DBUS, &addr, &source_addr, interface, str_domain, NULL); + } + } } while ((str_domain = p)); } else @@ -421,11 +430,40 @@ static DBusMessage* dbus_read_servers_ex(DBusMessage *message, int strings) if (dbus_message_iter_get_arg_type(&string_iter) == DBUS_TYPE_STRING) dbus_message_iter_get_basic(&string_iter, &str); dbus_message_iter_next (&string_iter); + + if ((addr_err = parse_server(str_addr, &sdetails))) + { + error = dbus_message_new_error_printf(message, DBUS_ERROR_INVALID_ARGS, + "Invalid IP address '%s': %s", + str, addr_err); + break; + } - add_update_server(flags | SERV_FROM_DBUS, &addr, &source_addr, interface, str, NULL); + while (parse_server_next(&sdetails)) + { + if ((addr_err = parse_server_addr(&sdetails))) + { + error = dbus_message_new_error_printf(message, DBUS_ERROR_INVALID_ARGS, + "Invalid IP address '%s': %s", + str, addr_err); + break; + } + + /* 0.0.0.0 for server address == NULL, for Dbus */ + if (addr.in.sin_family == AF_INET && + addr.in.sin_addr.s_addr == 0) + flags |= SERV_LITERAL_ADDRESS; + else + flags &= ~SERV_LITERAL_ADDRESS; + + add_update_server(flags | SERV_FROM_DBUS, &addr, &source_addr, interface, str, NULL); + } } while (dbus_message_iter_get_arg_type(&string_iter) == DBUS_TYPE_STRING); } - + + if (sdetails.resolved) + freeaddrinfo(sdetails.hostinfo); + /* jump to next element in outer array */ dbus_message_iter_next(&array_iter); } diff --git a/src/dnsmasq/dnsmasq.h b/src/dnsmasq/dnsmasq.h index 7833fa78..2121c85e 100644 --- a/src/dnsmasq/dnsmasq.h +++ b/src/dnsmasq/dnsmasq.h @@ -1507,6 +1507,7 @@ void reset_option_bool(unsigned int opt); struct hostsfile *expand_filelist(struct hostsfile *list); char *parse_server(char *arg, struct server_details *sdetails); char *parse_server_addr(struct server_details *sdetails); +int parse_server_next(struct server_details *sdetails); int option_read_dynfile(char *file, int flags); /* forward.c */ diff --git a/src/dnsmasq/option.c b/src/dnsmasq/option.c index ba893534..15d75bae 100644 --- a/src/dnsmasq/option.c +++ b/src/dnsmasq/option.c @@ -1053,10 +1053,11 @@ char *parse_server_addr(struct server_details *sdetails) } else return _("bad address"); + return NULL; } -static int parse_server_next(struct server_details *sdetails) +int parse_server_next(struct server_details *sdetails) { /* Looping over resolved addresses? */ if (sdetails->hostinfo) From 1449829f1d1c8f0f0fa8efc56c0de771036ee5c1 Mon Sep 17 00:00:00 2001 From: Simon Kelley Date: Mon, 7 Nov 2022 23:14:30 +0000 Subject: [PATCH 158/218] Fix --server with multiple domains. Signed-off-by: DL6ER --- src/dnsmasq/option.c | 37 ++++++++++++++++++++----------------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/src/dnsmasq/option.c b/src/dnsmasq/option.c index 15d75bae..cbf18d13 100644 --- a/src/dnsmasq/option.c +++ b/src/dnsmasq/option.c @@ -2961,7 +2961,7 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma case LOPT_LOCAL: /* --local */ case 'A': /* --address */ { - char *lastdomain = NULL, *domain = ""; + char *lastdomain = NULL, *domain = "", *cur_domain; u16 flags = 0; char *err; union all_addr addr; @@ -3012,9 +3012,12 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma if (servers_only && option == 'S') flags |= SERV_FROM_FILE; - + + cur_domain = domain; while ((flags & SERV_LITERAL_ADDRESS) || parse_server_next(&sdetails)) { + cur_domain = domain; + if (!(flags & SERV_LITERAL_ADDRESS) && (err = parse_server_addr(&sdetails))) ret_err(err); @@ -3026,24 +3029,24 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma { /* server=//1.2.3.4 is special. */ if (lastdomain) - { - if (strlen(domain) == 0) - flags |= SERV_FOR_NODOTS; - else - flags &= ~SERV_FOR_NODOTS; - - /* address=/#/ matches the same as without domain */ - if (option == 'A' && domain[0] == '#' && domain[1] == 0) - domain[0] = 0; - } - - if (!add_update_server(flags, sdetails.addr, sdetails.source_addr, sdetails.interface, domain, &addr)) + { + if (strlen(cur_domain) == 0) + flags |= SERV_FOR_NODOTS; + else + flags &= ~SERV_FOR_NODOTS; + + /* address=/#/ matches the same as without domain */ + if (option == 'A' && cur_domain[0] == '#' && cur_domain[1] == 0) + cur_domain[0] = 0; + } + + if (!add_update_server(flags, sdetails.addr, sdetails.source_addr, sdetails.interface, cur_domain, &addr)) ret_err(gen_err); - - if (!lastdomain || domain == lastdomain) + + if (!lastdomain || cur_domain == lastdomain) break; - domain += strlen(domain) + 1; + cur_domain += strlen(cur_domain) + 1; } if (flags & SERV_LITERAL_ADDRESS) From 2a94aef407631ea3afde888ae0f4cd0291947a8a Mon Sep 17 00:00:00 2001 From: Simon Kelley Date: Mon, 7 Nov 2022 23:24:31 +0000 Subject: [PATCH 159/218] Fix --server=/domain/# Signed-off-by: DL6ER --- src/dnsmasq/option.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/dnsmasq/option.c b/src/dnsmasq/option.c index cbf18d13..b511ef34 100644 --- a/src/dnsmasq/option.c +++ b/src/dnsmasq/option.c @@ -867,11 +867,13 @@ char *parse_server(char *arg, struct server_details *sdetails) struct addrinfo hints = { 0 }; *sdetails->interface = 0; - + sdetails->addr_type = AF_UNSPEC; + if (strcmp(arg, "#") == 0) { if (sdetails->flags) *sdetails->flags |= SERV_USE_RESOLV; + sdetails->addr_type = AF_LOCAL; sdetails->valid = 1; return NULL; } @@ -1051,7 +1053,7 @@ char *parse_server_addr(struct server_details *sdetails) } } } - else + else if (sdetails->addr_type != AF_LOCAL) return _("bad address"); return NULL; From 5556dd5d78d94162a27277df08698797c4a30ad5 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 8 Nov 2022 19:54:17 +0100 Subject: [PATCH 160/218] Add dnsmasq tag v2.88test3 Signed-off-by: DL6ER --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index d8682052..15a56bca 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -11,6 +11,6 @@ cmake_minimum_required(VERSION 2.8.12) project(PIHOLE_FTL C) -set(DNSMASQ_VERSION pi-hole-v2.88test2) +set(DNSMASQ_VERSION pi-hole-v2.88test3) add_subdirectory(src) From 4b798e8aecbd74171fe1626932a6585c0c2c7260 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 8 Nov 2022 20:03:56 +0100 Subject: [PATCH 161/218] Fix incorrect three-way merge happened when importing the stale-cache patch Signed-off-by: DL6ER --- src/dnsmasq/forward.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/dnsmasq/forward.c b/src/dnsmasq/forward.c index 88e0cdd3..986bdc58 100644 --- a/src/dnsmasq/forward.c +++ b/src/dnsmasq/forward.c @@ -1827,13 +1827,6 @@ void receive_query(struct listener *listen, time_t now) if (m >= 1) { - if (stale && have_pseudoheader) - { - u16 swap = htons(EDE_STALE); - - m = add_pseudoheader(header, m, ((unsigned char *) header) + udp_size, daemon->edns_pktsz, - EDNS0_OPTION_EDE, (unsigned char *)&swap, 2, do_bit, 0); - } #ifdef HAVE_DUMPFILE dump_packet_udp(DUMP_REPLY, daemon->packet, m, NULL, &source_addr, listen->fd); #endif @@ -1910,6 +1903,13 @@ void receive_query(struct listener *listen, time_t now) if (m >= 1) { + if (stale && have_pseudoheader) + { + u16 swap = htons(EDE_STALE); + + m = add_pseudoheader(header, m, ((unsigned char *) header) + udp_size, daemon->edns_pktsz, + EDNS0_OPTION_EDE, (unsigned char *)&swap, 2, do_bit, 0); + } #ifdef HAVE_DUMPFILE dump_packet_udp(DUMP_REPLY, daemon->packet, m, NULL, &source_addr, listen->fd); #endif From eec622c6873e10876a82c072c11b3e8867f1a7e5 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 10 Nov 2022 19:23:32 +0100 Subject: [PATCH 162/218] Add shortcut to compile an all-options build of dnsmasq inside FTL. Note that this does not include ubus as it is an OpenWRT-native thing that cannot be setup easily on any other distro. Signed-off-by: DL6ER --- src/CMakeLists.txt | 16 ++++++++++++ src/FindDBus.cmake | 59 +++++++++++++++++++++++++++++++++++++++++++ src/dnsmasq/config.h | 11 ++++++++ src/dnsmasq/dnsmasq.h | 7 ----- test/arch_test.sh | 6 ++++- test/test_suite.bats | 11 ++++++++ 6 files changed, 102 insertions(+), 8 deletions(-) create mode 100644 src/FindDBus.cmake diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 93317a31..00f0def0 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -217,6 +217,22 @@ else() message(STATUS "Building FTL with readline support: NO") endif() +# Do we want to compile an all-in FTL version? +if($ENV{CI_ARCH} STREQUAL "x86_64_full") + add_definitions(-DDNSMASQ_ALL_OPTS) + set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${CMAKE_CURRENT_SOURCE_DIR}) + find_package(DBus REQUIRED) + # Use results of find_package() call. + include_directories(${DBUS_INCLUDE_DIRS}) + target_link_libraries(pihole-FTL ${DBUS_LIBRARIES}) + find_library(LIBMNL mnl) + find_library(LIBNFTNL nftnl) + find_library(LIBNFTABLES nftables) + find_library(LIBNFNETLINK nfnetlink) + find_library(LIBNETFILTER_CONNTRACK netfilter_conntrack) + target_link_libraries(pihole-FTL ${LIBMNL} ${LIBNFTABLES} ${LIBNFTNL} ${LIBNFNETLINK} ${LIBNETFILTER_CONNTRACK}) +endif() + if(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT) set(CMAKE_INSTALL_PREFIX "/usr" CACHE PATH "..." FORCE) endif() diff --git a/src/FindDBus.cmake b/src/FindDBus.cmake new file mode 100644 index 00000000..4a1a1805 --- /dev/null +++ b/src/FindDBus.cmake @@ -0,0 +1,59 @@ +# - Try to find DBus +# Once done, this will define +# +# DBUS_FOUND - system has DBus +# DBUS_INCLUDE_DIRS - the DBus include directories +# DBUS_LIBRARIES - link these to use DBus +# +# Copyright (C) 2012 Raphael Kubo da Costa +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER AND ITS CONTRIBUTORS ``AS +# IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR ITS +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; +# OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR +# OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF +# ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +FIND_PACKAGE(PkgConfig) +PKG_CHECK_MODULES(PC_DBUS QUIET dbus-1) + +FIND_LIBRARY(DBUS_LIBRARIES + NAMES dbus-1 + HINTS ${PC_DBUS_LIBDIR} + ${PC_DBUS_LIBRARY_DIRS} +) + +FIND_PATH(DBUS_INCLUDE_DIR + NAMES dbus/dbus.h + HINTS ${PC_DBUS_INCLUDEDIR} + ${PC_DBUS_INCLUDE_DIRS} +) + +GET_FILENAME_COMPONENT(_DBUS_LIBRARY_DIR ${DBUS_LIBRARIES} PATH) +FIND_PATH(DBUS_ARCH_INCLUDE_DIR + NAMES dbus/dbus-arch-deps.h + HINTS ${PC_DBUS_INCLUDEDIR} + ${PC_DBUS_INCLUDE_DIRS} + ${_DBUS_LIBRARY_DIR} + ${DBUS_INCLUDE_DIR} + PATH_SUFFIXES include +) + +SET(DBUS_INCLUDE_DIRS ${DBUS_INCLUDE_DIR} ${DBUS_ARCH_INCLUDE_DIR}) + +INCLUDE(FindPackageHandleStandardArgs) +FIND_PACKAGE_HANDLE_STANDARD_ARGS(DBUS REQUIRED_VARS DBUS_INCLUDE_DIRS DBUS_LIBRARIES) diff --git a/src/dnsmasq/config.h b/src/dnsmasq/config.h index 18429bf3..6be54c9f 100644 --- a/src/dnsmasq/config.h +++ b/src/dnsmasq/config.h @@ -200,6 +200,17 @@ RESOLVFILE /* #define HAVE_DNSSEC */ /* #define HAVE_NFTSET */ +/* Pi-hole definitions */ +#define HAVE_LUASCRIPT +#define HAVE_IDN +#define HAVE_DNSSEC +#ifdef DNSMASQ_ALL_OPTS + #define HAVE_DBUS + #define HAVE_CONNTRACK + #define HAVE_NFTSET +#endif +/***********************/ + /* Default locations for important system files. */ #ifndef LEASEFILE diff --git a/src/dnsmasq/dnsmasq.h b/src/dnsmasq/dnsmasq.h index 5cdb11c0..946f75b0 100644 --- a/src/dnsmasq/dnsmasq.h +++ b/src/dnsmasq/dnsmasq.h @@ -14,13 +14,6 @@ along with this program. If not, see . */ -/* Pi-hole definitions */ -#define HAVE_DNSSEC -#define HAVE_DNSSEC_STATIC -#define HAVE_IDN -#define HAVE_LUASCRIPT -/***********************/ - #define COPYRIGHT "Copyright (c) 2000-2022 Simon Kelley" /* We do defines that influence behavior of stdio.h, so complain diff --git a/test/arch_test.sh b/test/arch_test.sh index 11b47b37..9416985e 100644 --- a/test/arch_test.sh +++ b/test/arch_test.sh @@ -76,7 +76,11 @@ check_static() { if [[ "${CI_ARCH}" == "x86_64" ]]; then check_machine "ELF64" "Advanced Micro Devices X86-64" - check_libs "[libm.so.6] [librt.so.1] [libpthread.so.0] [libc.so.6]" + if [[ "${GIT_TAG}" == "all-dependencies-test-build" ]]; then + check_libs "[libm.so.6] [librt.so.1] [libdbus-1.so.3] [libmnl.so.0] [libnftables.so.1] [libnftnl.so.11] [libnfnetlink.so.0] [libnetfilter_conntrack.so.3] [libpthread.so.0] [libc.so.6]" + else + check_libs "[libm.so.6] [librt.so.1] [libpthread.so.0] [libc.so.6]" + fi check_file "ELF 64-bit LSB shared object, x86-64, version 1 (SYSV), dynamically linked, interpreter /lib64/ld-linux-x86-64.so.2, for GNU/Linux 3.2.0, with debug_info, not stripped" elif [[ "${CI_ARCH}" == "x86_64-musl" ]]; then diff --git a/test/test_suite.bats b/test/test_suite.bats index 95e85436..c2c55608 100644 --- a/test/test_suite.bats +++ b/test/test_suite.bats @@ -11,6 +11,17 @@ [[ ${lines[6]} == "" ]] } +@test "dnsmasq options as expected" { + run bash -c './pihole-FTL -vv | grep "cryptohash"' + printf "%s\n" "${lines[@]}" + if [[ "${CI_ARCH}" == "x86_64_full" ]]; then + [[ ${lines[0]} == "Compile options: IPv6 GNU-getopt DBus no-UBus no-i18n IDN DHCP DHCPv6 Lua TFTP conntrack ipset nftset auth cryptohash DNSSEC loop-detect inotify dumpfile" ]] + else + [[ ${lines[0]} == "Compile options: IPv6 GNU-getopt no-DBus no-UBus no-i18n IDN DHCP DHCPv6 Lua TFTP no-conntrack ipset no-nftset auth cryptohash DNSSEC loop-detect inotify dumpfile" ]] + fi + [[ ${lines[1]} == "" ]] +} + @test "DNS server port is reported over Telnet API" { run bash -c 'echo ">dns-port >quit" | nc -v 127.0.0.1 4711' printf "%s\n" "${lines[@]}" From 57352c9e7c6070d0cf0fa7f1adba28a73d2ff14c Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 11 Nov 2022 09:16:44 +0100 Subject: [PATCH 163/218] Add building the full binary in the x86_64 container Signed-off-by: DL6ER --- .github/workflows/build.yml | 5 ++++- test/arch_test.sh | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e49cfd34..bd96f013 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -62,6 +62,9 @@ jobs: include: - arch: x86_64 bin_name: pihole-FTL-linux-x86_64 + - arch: x86_64 + arch_extra: _full + bin_name: pihole-FTL-linux-x86_64_full - arch: x86_64-musl bin_name: pihole-FTL-musl-linux-x86_64 - arch: x86_32 @@ -80,7 +83,7 @@ jobs: bin_name: pihole-FTL-aarch64-linux-gnu env: - CI_ARCH: ${{ matrix.arch }} + CI_ARCH: ${{ matrix.arch }}${{ matrix.arch_extra }} steps: - diff --git a/test/arch_test.sh b/test/arch_test.sh index 9416985e..6a8ef0cd 100644 --- a/test/arch_test.sh +++ b/test/arch_test.sh @@ -73,10 +73,10 @@ check_static() { echo "Static executable check: OK" } -if [[ "${CI_ARCH}" == "x86_64" ]]; then +if [[ "${CI_ARCH}" == "x86_64" || "${CI_ARCH}" == "x86_64_full" ]]; then check_machine "ELF64" "Advanced Micro Devices X86-64" - if [[ "${GIT_TAG}" == "all-dependencies-test-build" ]]; then + if [[ "${CI_ARCH}" == "x86_64_full" ]]; then check_libs "[libm.so.6] [librt.so.1] [libdbus-1.so.3] [libmnl.so.0] [libnftables.so.1] [libnftnl.so.11] [libnfnetlink.so.0] [libnetfilter_conntrack.so.3] [libpthread.so.0] [libc.so.6]" else check_libs "[libm.so.6] [librt.so.1] [libpthread.so.0] [libc.so.6]" From d098f05f5695501f05a98c8e1a2d5aa1ba4bc5f4 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 13 Nov 2022 16:29:23 +0100 Subject: [PATCH 164/218] Use ftl-build:v1.23 containers Signed-off-by: DL6ER --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index bd96f013..38c040e4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -53,7 +53,7 @@ jobs: needs: smoke-tests - container: ghcr.io/pi-hole/ftl-build:v1.22-${{ matrix.arch }} + container: ghcr.io/pi-hole/ftl-build:v1.23-${{ matrix.arch }} strategy: fail-fast: false From 80b1d3edba03cf3ea93cf684630247850ced1ca0 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 13 Nov 2022 18:20:27 +0100 Subject: [PATCH 165/218] Run tests for all x86_* architectures Signed-off-by: DL6ER --- test/run.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/run.sh b/test/run.sh index b3fcd883..e267a39b 100755 --- a/test/run.sh +++ b/test/run.sh @@ -1,7 +1,7 @@ #!/bin/bash -# Only run tests on x86_64, x86_64-musl, and x86_32 targets -if [[ ${CI} == "true" && "${CI_ARCH}" != "x86_64" && "${CI_ARCH}" != "x86_64-musl" && "${CI_ARCH}" != "x86_32" ]]; then +# Only run tests on x86_* targets (where the CI can natively run the binaries) +if [[ ${CI} == "true" && "${CI_ARCH}" != "x86_"* ]]; then echo "Skipping tests (CI_ARCH: ${CI_ARCH})!" exit 0 fi From a052113364bef2d0af11c383d1c221e0c5c8ed46 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 14 Nov 2022 22:21:10 +0100 Subject: [PATCH 166/218] Add "pihole-FTL --hash" printing the current git hash of the binary's source code Signed-off-by: DL6ER --- src/args.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/args.c b/src/args.c index dee6e55f..439edd56 100644 --- a/src/args.c +++ b/src/args.c @@ -285,6 +285,12 @@ void parse_args(int argc, char* argv[]) exit(EXIT_SUCCESS); } + if(strcmp(argv[i], "--hash") == 0) + { + printf("%s\n",GIT_HASH); + exit(EXIT_SUCCESS); + } + // Don't go into background if(strcmp(argv[i], "-f") == 0 || strcmp(argv[i], "no-daemon") == 0) From 64d5b3af4eca19b0aa74c04d8ae31fc70ab20e3f Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 14 Nov 2022 22:29:55 +0100 Subject: [PATCH 167/218] Fix CMake ENV var comparison to restore compilation in environments where this variable is unset Signed-off-by: DL6ER --- src/CMakeLists.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 00f0def0..8bf6e32d 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -218,7 +218,8 @@ else() endif() # Do we want to compile an all-in FTL version? -if($ENV{CI_ARCH} STREQUAL "x86_64_full") +if(DEFINED ENV{CI_ARCH}) + if($ENV{CI_ARCH} STREQUAL "x86_64_full") add_definitions(-DDNSMASQ_ALL_OPTS) set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${CMAKE_CURRENT_SOURCE_DIR}) find_package(DBus REQUIRED) @@ -231,6 +232,7 @@ if($ENV{CI_ARCH} STREQUAL "x86_64_full") find_library(LIBNFNETLINK nfnetlink) find_library(LIBNETFILTER_CONNTRACK netfilter_conntrack) target_link_libraries(pihole-FTL ${LIBMNL} ${LIBNFTABLES} ${LIBNFTNL} ${LIBNFNETLINK} ${LIBNETFILTER_CONNTRACK}) + endif() endif() if(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT) From b63a37281d05e0c048311f34ef2384ab49d021ce Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 14 Nov 2022 22:52:53 +0100 Subject: [PATCH 168/218] Force 8 digits to display git object names Signed-off-by: DL6ER --- src/gen_version.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/gen_version.cmake b/src/gen_version.cmake index 3d70e524..a4130332 100644 --- a/src/gen_version.cmake +++ b/src/gen_version.cmake @@ -24,7 +24,7 @@ if(DEFINED ENV{GIT_HASH}) set(GIT_HASH "$ENV{GIT_HASH}") else() execute_process( - COMMAND git --no-pager describe --always --dirty + COMMAND git --no-pager describe --always --abbrev=8 --dirty WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} OUTPUT_VARIABLE GIT_HASH ERROR_QUIET @@ -36,7 +36,7 @@ if(DEFINED ENV{GIT_VERSION}) set(GIT_VERSION "$ENV{GIT_VERSION}") else() execute_process( - COMMAND git --no-pager describe --tags --always --dirty + COMMAND git --no-pager describe --tags --always --abbrev=8 --dirty WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} OUTPUT_VARIABLE GIT_VERSION ERROR_QUIET From fc212c383a67b88f25727357909182ddd92ceb96 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 15 Nov 2022 19:33:47 +0100 Subject: [PATCH 169/218] Always ensure FTL messages are unique Signed-off-by: DL6ER --- src/database/message-table.c | 111 +++++++++++++++++------------------ 1 file changed, 54 insertions(+), 57 deletions(-) diff --git a/src/database/message-table.c b/src/database/message-table.c index 36a23855..2e6867c6 100644 --- a/src/database/message-table.c +++ b/src/database/message-table.c @@ -150,7 +150,7 @@ bool flush_message_table(void) return true; } -static bool add_message(const enum message_type type, const bool unique, +static bool add_message(const enum message_type type, const char *message, const int count,...) { bool okay = false; @@ -166,55 +166,52 @@ static bool add_message(const enum message_type type, const bool unique, return false; } - // Ensure there are no duplicates when adding host name or rate-limiting messages - if(unique) - { - sqlite3_stmt* stmt = NULL; - const char *querystr = "DELETE FROM message WHERE type = ?1 AND message = ?2"; - int rc = sqlite3_prepare_v2(db, querystr, -1, &stmt, NULL); - if( rc != SQLITE_OK ){ - logg("add_message(type=%u, message=%s) - SQL error prepare DELETE: %s", - type, message, sqlite3_errstr(rc)); - goto end_of_add_message; - } - - // Bind type to prepared statement - if((rc = sqlite3_bind_text(stmt, 1, message_types[type], -1, SQLITE_STATIC)) != SQLITE_OK) - { - logg("add_message(type=%u, message=%s) - Failed to bind type DELETE: %s", - type, message, sqlite3_errstr(rc)); - sqlite3_reset(stmt); - sqlite3_finalize(stmt); - goto end_of_add_message; - } - - // Bind message to prepared statement - if((rc = sqlite3_bind_text(stmt, 2, message, -1, SQLITE_STATIC)) != SQLITE_OK) - { - logg("add_message(type=%u, message=%s) - Failed to bind message DELETE: %s", - type, message, sqlite3_errstr(rc)); - sqlite3_reset(stmt); - sqlite3_finalize(stmt); - goto end_of_add_message; - } - - // Execute and finalize - if((rc = sqlite3_step(stmt)) != SQLITE_OK && rc != SQLITE_DONE) - { - logg("add_message(type=%u, message=%s) - SQL error step DELETE: %s", - type, message, sqlite3_errstr(rc)); - goto end_of_add_message; - } - sqlite3_clear_bindings(stmt); - sqlite3_reset(stmt); - sqlite3_finalize(stmt); + // Ensure there are no duplicates when adding messages + sqlite3_stmt* stmt = NULL; + const char *querystr = "DELETE FROM message WHERE type = ?1 AND message = ?2"; + int rc = sqlite3_prepare_v2(db, querystr, -1, &stmt, NULL); + if( rc != SQLITE_OK ){ + logg("add_message(type=%u, message=%s) - SQL error prepare DELETE: %s", + type, message, sqlite3_errstr(rc)); + goto end_of_add_message; } + // Bind type to prepared statement + if((rc = sqlite3_bind_text(stmt, 1, message_types[type], -1, SQLITE_STATIC)) != SQLITE_OK) + { + logg("add_message(type=%u, message=%s) - Failed to bind type DELETE: %s", + type, message, sqlite3_errstr(rc)); + sqlite3_reset(stmt); + sqlite3_finalize(stmt); + goto end_of_add_message; + } + + // Bind message to prepared statement + if((rc = sqlite3_bind_text(stmt, 2, message, -1, SQLITE_STATIC)) != SQLITE_OK) + { + logg("add_message(type=%u, message=%s) - Failed to bind message DELETE: %s", + type, message, sqlite3_errstr(rc)); + sqlite3_reset(stmt); + sqlite3_finalize(stmt); + goto end_of_add_message; + } + + // Execute and finalize + if((rc = sqlite3_step(stmt)) != SQLITE_OK && rc != SQLITE_DONE) + { + logg("add_message(type=%u, message=%s) - SQL error step DELETE: %s", + type, message, sqlite3_errstr(rc)); + goto end_of_add_message; + } + sqlite3_clear_bindings(stmt); + sqlite3_reset(stmt); + sqlite3_finalize(stmt); + stmt = NULL; + // Prepare SQLite statement - sqlite3_stmt* stmt = NULL; - const char *querystr = "INSERT INTO message (timestamp,type,message,blob1,blob2,blob3,blob4,blob5) " - "VALUES ((cast(strftime('%s', 'now') as int)),?,?,?,?,?,?,?);"; - int rc = sqlite3_prepare_v2(db, querystr, -1, &stmt, NULL); + querystr = "INSERT INTO message (timestamp,type,message,blob1,blob2,blob3,blob4,blob5) " + "VALUES ((cast(strftime('%s', 'now') as int)),?,?,?,?,?,?,?);"; + rc = sqlite3_prepare_v2(db, querystr, -1, &stmt, NULL); if( rc != SQLITE_OK ) { logg("add_message(type=%u, message=%s) - SQL error prepare: %s", @@ -317,7 +314,7 @@ void logg_regex_warning(const char *type, const char *warning, const int dbindex // Log to database only if not in CLI mode if(!cli_mode) - add_message(REGEX_MESSAGE, false, warning, 3, type, regex, dbindex); + add_message(REGEX_MESSAGE, warning, 3, type, regex, dbindex); } void logg_subnet_warning(const char *ip, const int matching_count, const char *matching_ids, @@ -332,7 +329,7 @@ void logg_subnet_warning(const char *ip, const int matching_count, const char *m // Log to database char *names = get_client_names_from_ids(matching_ids); - add_message(SUBNET_MESSAGE, false, ip, 5, matching_count, names, matching_ids, chosen_match_text, chosen_match_id); + add_message(SUBNET_MESSAGE, ip, 5, matching_count, names, matching_ids, chosen_match_text, chosen_match_id); free(names); } @@ -343,7 +340,7 @@ void logg_hostname_warning(const char *ip, const char *name, const unsigned int ip, name, pos); // Log to database - add_message(HOSTNAME_MESSAGE, true, ip, 2, name, (const int)pos); + add_message(HOSTNAME_MESSAGE, ip, 2, name, (const int)pos); } void logg_fatal_dnsmasq_message(const char *message) @@ -352,7 +349,7 @@ void logg_fatal_dnsmasq_message(const char *message) logg("FATAL ERROR in dnsmasq core: %s", message); // Log to database - add_message(DNSMASQ_CONFIG_MESSAGE, false, message, 0); + add_message(DNSMASQ_CONFIG_MESSAGE, message, 0); // FTL will dies after this point, so we should make sure to clean up // behind ourselves @@ -368,7 +365,7 @@ void logg_rate_limit_message(const char *clientIP, const unsigned int rate_limit clientIP, turnaround, turnaround == 1 ? "" : "s"); // Log to database - add_message(RATE_LIMIT_MESSAGE, true, clientIP, 2, config.rate_limit.count, config.rate_limit.interval); + add_message(RATE_LIMIT_MESSAGE, clientIP, 2, config.rate_limit.count, config.rate_limit.interval); } void logg_warn_dnsmasq_message(char *message) @@ -377,7 +374,7 @@ void logg_warn_dnsmasq_message(char *message) logg("WARNING in dnsmasq core: %s", message); // Log to database - add_message(DNSMASQ_WARN_MESSAGE, false, message, 0); + add_message(DNSMASQ_WARN_MESSAGE, message, 0); } void log_resource_shortage(const double load, const int nprocs, const int shmem, const int disk, const char *path, const char *msg) @@ -385,17 +382,17 @@ void log_resource_shortage(const double load, const int nprocs, const int shmem, if(load > 0.0) { logg("WARNING: Long-term load (15min avg) larger than number of processors: %.1f > %d", load, nprocs); - add_message(LOAD_MESSAGE, true, "excessive load", 2, load, nprocs); + add_message(LOAD_MESSAGE, "excessive load", 2, load, nprocs); } else if(shmem > -1) { logg("WARNING: RAM shortage (%s) ahead: %d%% is used (%s)", path, shmem, msg); - add_message(SHMEM_MESSAGE, true, path, 2, shmem, msg); + add_message(SHMEM_MESSAGE, path, 2, shmem, msg); } else if(disk > -1) { logg("WARNING: Disk shortage (%s) ahead: %d%% is used (%s)", path, disk, msg); - add_message(DISK_MESSAGE, true, path, 2, disk, msg); + add_message(DISK_MESSAGE, path, 2, disk, msg); } } @@ -405,5 +402,5 @@ void logg_inaccessible_adlist(const int dbindex, const char *address) logg("Adlist warning: Adlist with ID %d (%s) was inaccessible during last gravity run", dbindex, address); // Log to database - add_message(INACCESSIBLE_ADLIST_MESSAGE, false, address, 1, dbindex); + add_message(INACCESSIBLE_ADLIST_MESSAGE, address, 1, dbindex); } From af495664f9fc76b4f9dca48a8fc848e01cb356fe Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 15 Nov 2022 21:28:38 +0100 Subject: [PATCH 170/218] Adlist warning -> ADLIST WARNING Signed-off-by: DL6ER --- src/database/message-table.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/database/message-table.c b/src/database/message-table.c index 2e6867c6..ef1b28d5 100644 --- a/src/database/message-table.c +++ b/src/database/message-table.c @@ -399,7 +399,7 @@ void log_resource_shortage(const double load, const int nprocs, const int shmem, void logg_inaccessible_adlist(const int dbindex, const char *address) { // Log to FTL.log - logg("Adlist warning: Adlist with ID %d (%s) was inaccessible during last gravity run", dbindex, address); + logg("ADLIST WARNING: Adlist with ID %d (%s) was inaccessible during last gravity run", dbindex, address); // Log to database add_message(INACCESSIBLE_ADLIST_MESSAGE, address, 1, dbindex); From c689cf23ff066c0ca7faf1134656e723425f3402 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 16 Nov 2022 04:26:40 +0100 Subject: [PATCH 171/218] Add debugging output to verify_shmem_pid() Signed-off-by: DL6ER --- src/shmem.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/shmem.c b/src/shmem.c index 986211f5..948b6d01 100644 --- a/src/shmem.c +++ b/src/shmem.c @@ -146,6 +146,10 @@ static void verify_shmem_pid(void) // Get new pointer void *shm = mmap(NULL, shm_settings.size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + // Debug logging + if(config.debug & DEBUG_SHMEM) + logg("verify_shmem_pid(): updating pointer of shmSettings: %p -> %p", shm_settings.ptr, shm); + // Check for `mmap` error if(shm == MAP_FAILED) { From bb571050013f21c18e0847226349ecb4530842d9 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 16 Nov 2022 17:58:46 +0100 Subject: [PATCH 172/218] Update embedded SQLite3 engine to version 3.40.0 Signed-off-by: DL6ER --- src/database/shell.c | 5097 +++++++++++++++++++++++++++++++--------- src/database/sqlite3.c | 5003 +++++++++++++++++++++++++++------------ src/database/sqlite3.h | 119 +- 3 files changed, 7478 insertions(+), 2741 deletions(-) diff --git a/src/database/shell.c b/src/database/shell.c index 6b40debc..83c42640 100644 --- a/src/database/shell.c +++ b/src/database/shell.c @@ -34,6 +34,8 @@ /* This needs to come before any includes for MSVC compiler */ #define _CRT_SECURE_NO_WARNINGS #endif +typedef unsigned int u32; +typedef unsigned short int u16; /* ** Optionally #include a user-defined header, whereby compilation options @@ -55,6 +57,15 @@ # define SQLITE_OS_WINRT 0 #endif +/* +** If SQLITE_SHELL_FIDDLE is defined then the shell is modified +** somewhat for use as a WASM module in a web browser. This flag +** should only be used when building the "fiddle" web application, as +** the browser-mode build has much different user input requirements +** and this build mode rewires the user input subsystem to account for +** that. +*/ + /* ** Warning pragmas copied from msvc.h in the core. */ @@ -94,6 +105,14 @@ # define _LARGEFILE_SOURCE 1 #endif +#if defined(SQLITE_SHELL_FIDDLE) && !defined(_POSIX_SOURCE) +/* +** emcc requires _POSIX_SOURCE (or one of several similar defines) +** to expose strdup(). +*/ +# define _POSIX_SOURCE +#endif + #include #include #include @@ -249,20 +268,21 @@ static void setTextMode(FILE *file, int isOutput){ # define setTextMode(X,Y) #endif -/* -** When compiling with emcc (a.k.a. emscripten), we're building a -** WebAssembly (WASM) bundle and need to disable and rewire a few -** things. -*/ -#ifdef __EMSCRIPTEN__ -#define SQLITE_SHELL_WASM_MODE -#else -#undef SQLITE_SHELL_WASM_MODE -#endif - /* True if the timer is enabled */ static int enableTimer = 0; +/* A version of strcmp() that works with NULL values */ +static int cli_strcmp(const char *a, const char *b){ + if( a==0 ) a = ""; + if( b==0 ) b = ""; + return strcmp(a,b); +} +static int cli_strncmp(const char *a, const char *b, size_t n){ + if( a==0 ) a = ""; + if( b==0 ) b = ""; + return strncmp(a,b,n); +} + /* Return the current wall-clock time */ static sqlite3_int64 timeOfDay(void){ static sqlite3_vfs *clockVfs = 0; @@ -551,6 +571,7 @@ static void utf8_width_print(FILE *pOut, int w, const char *zUtf){ int i; int n; int aw = w<0 ? -w : w; + if( zUtf==0 ) zUtf = ""; for(i=n=0; zUtf[i]; i++){ if( (zUtf[i]&0xc0)!=0x80 ){ n++; @@ -694,7 +715,7 @@ static char *local_getline(char *zLine, FILE *in){ if( stdin_is_interactive && in==stdin ){ char *zTrans = sqlite3_win32_mbcs_to_utf8_v2(zLine, 0); if( zTrans ){ - int nTrans = strlen30(zTrans)+1; + i64 nTrans = strlen(zTrans)+1; if( nTrans>nLine ){ zLine = realloc(zLine, nTrans); shell_check_oom(zLine); @@ -721,7 +742,7 @@ static char *local_getline(char *zLine, FILE *in){ ** be freed by the caller or else passed back into this routine via the ** zPrior argument for reuse. */ -#ifndef SQLITE_SHELL_WASM_MODE +#ifndef SQLITE_SHELL_FIDDLE static char *one_input_line(FILE *in, char *zPrior, int isContinuation){ char *zPrompt; char *zResult; @@ -741,7 +762,7 @@ static char *one_input_line(FILE *in, char *zPrior, int isContinuation){ } return zResult; } -#endif /* !SQLITE_SHELL_WASM_MODE */ +#endif /* !SQLITE_SHELL_FIDDLE */ /* ** Return the value of a hexadecimal digit. Return -1 if the input @@ -830,9 +851,9 @@ static void freeText(ShellText *p){ ** quote character for zAppend. */ static void appendText(ShellText *p, const char *zAppend, char quote){ - int len; - int i; - int nAppend = strlen30(zAppend); + i64 len; + i64 i; + i64 nAppend = strlen30(zAppend); len = nAppend+p->n+1; if( quote ){ @@ -991,10 +1012,10 @@ static void shellAddSchemaName( const char *zName = (const char*)sqlite3_value_text(apVal[2]); sqlite3 *db = sqlite3_context_db_handle(pCtx); UNUSED_PARAMETER(nVal); - if( zIn!=0 && strncmp(zIn, "CREATE ", 7)==0 ){ + if( zIn!=0 && cli_strncmp(zIn, "CREATE ", 7)==0 ){ for(i=0; inInit>in.mx ) return 0; + c = RE_START-1; } if( pRe->nState<=(sizeof(aSpace)/(sizeof(aSpace[0])*2)) ){ @@ -3973,6 +4023,10 @@ static int re_match(ReCompiled *pRe, const unsigned char *zIn, int nIn){ if( pRe->aArg[x]==c ) re_add_state(pNext, x+1); break; } + case RE_OP_ATSTART: { + if( cPrev==RE_START ) re_add_state(pThis, x+1); + break; + } case RE_OP_ANY: { if( c!=0 ) re_add_state(pNext, x+1); break; @@ -4054,7 +4108,9 @@ static int re_match(ReCompiled *pRe, const unsigned char *zIn, int nIn){ } } for(i=0; inState; i++){ - if( pRe->aOp[pNext->aState[i]]==RE_OP_ACCEPT ){ rc = 1; break; } + int x = pNext->aState[i]; + while( pRe->aOp[x]==RE_OP_GOTO ) x += pRe->aArg[x]; + if( pRe->aOp[x]==RE_OP_ACCEPT ){ rc = 1; break; } } re_match_end: sqlite3_free(pToFree); @@ -4209,7 +4265,6 @@ static const char *re_subcompile_string(ReCompiled *p){ iStart = p->nState; switch( c ){ case '|': - case '$': case ')': { p->sIn.i--; return 0; @@ -4246,6 +4301,14 @@ static const char *re_subcompile_string(ReCompiled *p){ re_insert(p, iPrev, RE_OP_FORK, p->nState - iPrev+1); break; } + case '$': { + re_append(p, RE_OP_MATCH, RE_EOF); + break; + } + case '^': { + re_append(p, RE_OP_ATSTART, 0); + break; + } case '{': { int m = 0, n = 0; int sz, j; @@ -4264,6 +4327,7 @@ static const char *re_subcompile_string(ReCompiled *p){ if( m==0 ){ if( n==0 ) return "both m and n are zero in '{m,n}'"; re_insert(p, iPrev, RE_OP_FORK, sz+1); + iPrev++; n--; }else{ for(j=1; jsIn.i+1>=pRe->sIn.mx ){ - re_append(pRe, RE_OP_MATCH, RE_EOF); - re_append(pRe, RE_OP_ACCEPT, 0); - *ppRe = pRe; - }else if( pRe->sIn.i>=pRe->sIn.mx ){ + if( pRe->sIn.i>=pRe->sIn.mx ){ re_append(pRe, RE_OP_ACCEPT, 0); *ppRe = pRe; }else{ @@ -4470,6 +4530,67 @@ static void re_sql_func( } } +#if defined(SQLITE_DEBUG) +/* +** This function is used for testing and debugging only. It is only available +** if the SQLITE_DEBUG compile-time option is used. +** +** Compile a regular expression and then convert the compiled expression into +** text and return that text. +*/ +static void re_bytecode_func( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const char *zPattern; + const char *zErr; + ReCompiled *pRe; + sqlite3_str *pStr; + int i; + int n; + char *z; + + zPattern = (const char*)sqlite3_value_text(argv[0]); + if( zPattern==0 ) return; + zErr = re_compile(&pRe, zPattern, sqlite3_user_data(context)!=0); + if( zErr ){ + re_free(pRe); + sqlite3_result_error(context, zErr, -1); + return; + } + if( pRe==0 ){ + sqlite3_result_error_nomem(context); + return; + } + pStr = sqlite3_str_new(0); + if( pStr==0 ) goto re_bytecode_func_err; + if( pRe->nInit>0 ){ + sqlite3_str_appendf(pStr, "INIT "); + for(i=0; inInit; i++){ + sqlite3_str_appendf(pStr, "%02x", pRe->zInit[i]); + } + sqlite3_str_appendf(pStr, "\n"); + } + for(i=0; (unsigned)inState; i++){ + sqlite3_str_appendf(pStr, "%-8s %4d\n", + ReOpName[(unsigned char)pRe->aOp[i]], pRe->aArg[i]); + } + n = sqlite3_str_length(pStr); + z = sqlite3_str_finish(pStr); + if( n==0 ){ + sqlite3_free(z); + }else{ + sqlite3_result_text(context, z, n-1, sqlite3_free); + } + +re_bytecode_func_err: + re_free(pRe); +} + +#endif /* SQLITE_DEBUG */ + + /* ** Invoke this routine to register the regexp() function with the ** SQLite database connection. @@ -4494,12 +4615,19 @@ int sqlite3_regexp_init( rc = sqlite3_create_function(db, "regexpi", 2, SQLITE_UTF8|SQLITE_INNOCUOUS|SQLITE_DETERMINISTIC, (void*)db, re_sql_func, 0, 0); +#if defined(SQLITE_DEBUG) + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "regexp_bytecode", 1, + SQLITE_UTF8|SQLITE_INNOCUOUS|SQLITE_DETERMINISTIC, + 0, re_bytecode_func, 0, 0); + } +#endif /* SQLITE_DEBUG */ } return rc; } /************************* End ../ext/misc/regexp.c ********************/ -#ifndef SQLITE_SHELL_WASM_MODE +#ifndef SQLITE_SHELL_FIDDLE /************************* Begin ../ext/misc/fileio.c ******************/ /* ** 2014-06-13 @@ -6769,8 +6897,8 @@ SQLITE_EXTENSION_INIT1 #endif /* typedef sqlite3_int64 i64; */ /* typedef unsigned char u8; */ -typedef UINT32_TYPE u32; /* 4-byte unsigned integer */ -typedef UINT16_TYPE u16; /* 2-byte unsigned integer */ +/* typedef UINT32_TYPE u32; // 4-byte unsigned integer // */ +/* typedef UINT16_TYPE u16; // 2-byte unsigned integer // */ #define MIN(a,b) ((a)<(b) ? (a) : (b)) #if defined(SQLITE_COVERAGE_TEST) || defined(SQLITE_MUTATION_TEST) @@ -10051,6 +10179,10 @@ static char *idxAppendText(int *pRc, char *zIn, const char *zFmt, ...){ */ static int idxIdentifierRequiresQuotes(const char *zId){ int i; + int nId = STRLEN(zId); + + if( sqlite3_keyword_check(zId, nId) ) return 1; + for(i=0; zId[i]; i++){ if( !(zId[i]=='_') && !(zId[i]>='0' && zId[i]<='9') @@ -11277,7 +11409,12 @@ void sqlite3_expert_destroy(sqlite3expert *p){ /************************* End ../ext/expert/sqlite3expert.c ********************/ #if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_DBPAGE_VTAB) -/************************* Begin ../ext/misc/dbdata.c ******************/ +#define SQLITE_SHELL_HAVE_RECOVER 1 +#else +#define SQLITE_SHELL_HAVE_RECOVER 0 +#endif +#if SQLITE_SHELL_HAVE_RECOVER +/************************* Begin ../ext/recover/dbdata.c ******************/ /* ** 2019-04-17 ** @@ -11351,16 +11488,20 @@ void sqlite3_expert_destroy(sqlite3expert *p){ ** It contains one entry for each b-tree pointer between a parent and ** child page in the database. */ + #if !defined(SQLITEINT_H) /* #include "sqlite3ext.h" */ /* typedef unsigned char u8; */ +/* typedef unsigned int u32; */ #endif SQLITE_EXTENSION_INIT1 #include #include +#ifndef SQLITE_OMIT_VIRTUALTABLE + #define DBDATA_PADDING_BYTES 100 typedef struct DbdataTable DbdataTable; @@ -11382,11 +11523,12 @@ struct DbdataCursor { /* Only for the sqlite_dbdata table */ u8 *pRec; /* Buffer containing current record */ - int nRec; /* Size of pRec[] in bytes */ - int nHdr; /* Size of header in bytes */ + sqlite3_int64 nRec; /* Size of pRec[] in bytes */ + sqlite3_int64 nHdr; /* Size of header in bytes */ int iField; /* Current field number */ u8 *pHdrPtr; u8 *pPtr; + u32 enc; /* Text encoding */ sqlite3_int64 iIntkey; /* Integer key value */ }; @@ -11579,14 +11721,14 @@ static int dbdataClose(sqlite3_vtab_cursor *pCursor){ /* ** Utility methods to decode 16 and 32-bit big-endian unsigned integers. */ -static unsigned int get_uint16(unsigned char *a){ +static u32 get_uint16(unsigned char *a){ return (a[0]<<8)|a[1]; } -static unsigned int get_uint32(unsigned char *a){ - return ((unsigned int)a[0]<<24) - | ((unsigned int)a[1]<<16) - | ((unsigned int)a[2]<<8) - | ((unsigned int)a[3]); +static u32 get_uint32(unsigned char *a){ + return ((u32)a[0]<<24) + | ((u32)a[1]<<16) + | ((u32)a[2]<<8) + | ((u32)a[3]); } /* @@ -11601,7 +11743,7 @@ static unsigned int get_uint32(unsigned char *a){ */ static int dbdataLoadPage( DbdataCursor *pCsr, /* Cursor object */ - unsigned int pgno, /* Page number of page to load */ + u32 pgno, /* Page number of page to load */ u8 **ppPage, /* OUT: pointer to page buffer */ int *pnPage /* OUT: Size of (*ppPage) in bytes */ ){ @@ -11611,25 +11753,27 @@ static int dbdataLoadPage( *ppPage = 0; *pnPage = 0; - sqlite3_bind_int64(pStmt, 2, pgno); - if( SQLITE_ROW==sqlite3_step(pStmt) ){ - int nCopy = sqlite3_column_bytes(pStmt, 0); - if( nCopy>0 ){ - u8 *pPage; - pPage = (u8*)sqlite3_malloc64(nCopy + DBDATA_PADDING_BYTES); - if( pPage==0 ){ - rc = SQLITE_NOMEM; - }else{ - const u8 *pCopy = sqlite3_column_blob(pStmt, 0); - memcpy(pPage, pCopy, nCopy); - memset(&pPage[nCopy], 0, DBDATA_PADDING_BYTES); + if( pgno>0 ){ + sqlite3_bind_int64(pStmt, 2, pgno); + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + int nCopy = sqlite3_column_bytes(pStmt, 0); + if( nCopy>0 ){ + u8 *pPage; + pPage = (u8*)sqlite3_malloc64(nCopy + DBDATA_PADDING_BYTES); + if( pPage==0 ){ + rc = SQLITE_NOMEM; + }else{ + const u8 *pCopy = sqlite3_column_blob(pStmt, 0); + memcpy(pPage, pCopy, nCopy); + memset(&pPage[nCopy], 0, DBDATA_PADDING_BYTES); + } + *ppPage = pPage; + *pnPage = nCopy; } - *ppPage = pPage; - *pnPage = nCopy; } + rc2 = sqlite3_reset(pStmt); + if( rc==SQLITE_OK ) rc = rc2; } - rc2 = sqlite3_reset(pStmt); - if( rc==SQLITE_OK ) rc = rc2; return rc; } @@ -11638,17 +11782,30 @@ static int dbdataLoadPage( ** Read a varint. Put the value in *pVal and return the number of bytes. */ static int dbdataGetVarint(const u8 *z, sqlite3_int64 *pVal){ - sqlite3_int64 v = 0; + sqlite3_uint64 u = 0; int i; for(i=0; i<8; i++){ - v = (v<<7) + (z[i]&0x7f); - if( (z[i]&0x80)==0 ){ *pVal = v; return i+1; } + u = (u<<7) + (z[i]&0x7f); + if( (z[i]&0x80)==0 ){ *pVal = (sqlite3_int64)u; return i+1; } } - v = (v<<8) + (z[i]&0xff); - *pVal = v; + u = (u<<8) + (z[i]&0xff); + *pVal = (sqlite3_int64)u; return 9; } +/* +** Like dbdataGetVarint(), but set the output to 0 if it is less than 0 +** or greater than 0xFFFFFFFF. This can be used for all varints in an +** SQLite database except for key values in intkey tables. +*/ +static int dbdataGetVarintU32(const u8 *z, sqlite3_int64 *pVal){ + sqlite3_int64 val; + int nRet = dbdataGetVarint(z, &val); + if( val<0 || val>0xFFFFFFFF ) val = 0; + *pVal = val; + return nRet; +} + /* ** Return the number of bytes of space used by an SQLite value of type ** eType. @@ -11685,9 +11842,10 @@ static int dbdataValueBytes(int eType){ */ static void dbdataValue( sqlite3_context *pCtx, + u32 enc, int eType, u8 *pData, - int nData + sqlite3_int64 nData ){ if( eType>=0 && dbdataValueBytes(eType)<=nData ){ switch( eType ){ @@ -11729,7 +11887,19 @@ static void dbdataValue( default: { int n = ((eType-12) / 2); if( eType % 2 ){ - sqlite3_result_text(pCtx, (const char*)pData, n, SQLITE_TRANSIENT); + switch( enc ){ +#ifndef SQLITE_OMIT_UTF16 + case SQLITE_UTF16BE: + sqlite3_result_text16be(pCtx, (void*)pData, n, SQLITE_TRANSIENT); + break; + case SQLITE_UTF16LE: + sqlite3_result_text16le(pCtx, (void*)pData, n, SQLITE_TRANSIENT); + break; +#endif + default: + sqlite3_result_text(pCtx, (char*)pData, n, SQLITE_TRANSIENT); + break; + } }else{ sqlite3_result_blob(pCtx, pData, n, SQLITE_TRANSIENT); } @@ -11757,6 +11927,7 @@ static int dbdataNext(sqlite3_vtab_cursor *pCursor){ rc = dbdataLoadPage(pCsr, pCsr->iPgno, &pCsr->aPage, &pCsr->nPage); if( rc!=SQLITE_OK ) return rc; if( pCsr->aPage ) break; + if( pCsr->bOnePage ) return SQLITE_OK; pCsr->iPgno++; } pCsr->iCell = pTab->bPtr ? -2 : 0; @@ -11820,7 +11991,7 @@ static int dbdataNext(sqlite3_vtab_cursor *pCursor){ if( bNextPage || iOff>pCsr->nPage ){ bNextPage = 1; }else{ - iOff += dbdataGetVarint(&pCsr->aPage[iOff], &nPayload); + iOff += dbdataGetVarintU32(&pCsr->aPage[iOff], &nPayload); } /* If this is a leaf intkey cell, load the rowid */ @@ -11867,7 +12038,7 @@ static int dbdataNext(sqlite3_vtab_cursor *pCursor){ /* Load content from overflow pages */ if( nPayload>nLocal ){ sqlite3_int64 nRem = nPayload - nLocal; - unsigned int pgnoOvfl = get_uint32(&pCsr->aPage[iOff]); + u32 pgnoOvfl = get_uint32(&pCsr->aPage[iOff]); while( nRem>0 ){ u8 *aOvfl = 0; int nOvfl = 0; @@ -11887,7 +12058,8 @@ static int dbdataNext(sqlite3_vtab_cursor *pCursor){ } } - iHdr = dbdataGetVarint(pCsr->pRec, &nHdr); + iHdr = dbdataGetVarintU32(pCsr->pRec, &nHdr); + if( nHdr>nPayload ) nHdr = 0; pCsr->nHdr = nHdr; pCsr->pHdrPtr = &pCsr->pRec[iHdr]; pCsr->pPtr = &pCsr->pRec[pCsr->nHdr]; @@ -11901,7 +12073,7 @@ static int dbdataNext(sqlite3_vtab_cursor *pCursor){ if( pCsr->pHdrPtr>&pCsr->pRec[pCsr->nRec] ){ bNextPage = 1; }else{ - pCsr->pHdrPtr += dbdataGetVarint(pCsr->pHdrPtr, &iType); + pCsr->pHdrPtr += dbdataGetVarintU32(pCsr->pHdrPtr, &iType); pCsr->pPtr += dbdataValueBytes(iType); } } @@ -11940,6 +12112,18 @@ static int dbdataEof(sqlite3_vtab_cursor *pCursor){ return pCsr->aPage==0; } +/* +** Return true if nul-terminated string zSchema ends in "()". Or false +** otherwise. +*/ +static int dbdataIsFunction(const char *zSchema){ + size_t n = strlen(zSchema); + if( n>2 && zSchema[n-2]=='(' && zSchema[n-1]==')' ){ + return (int)n-2; + } + return 0; +} + /* ** Determine the size in pages of database zSchema (where zSchema is ** "main", "temp" or the name of an attached database) and set @@ -11950,10 +12134,16 @@ static int dbdataDbsize(DbdataCursor *pCsr, const char *zSchema){ DbdataTable *pTab = (DbdataTable*)pCsr->base.pVtab; char *zSql = 0; int rc, rc2; + int nFunc = 0; sqlite3_stmt *pStmt = 0; - zSql = sqlite3_mprintf("PRAGMA %Q.page_count", zSchema); + if( (nFunc = dbdataIsFunction(zSchema))>0 ){ + zSql = sqlite3_mprintf("SELECT %.*s(0)", nFunc, zSchema); + }else{ + zSql = sqlite3_mprintf("PRAGMA %Q.page_count", zSchema); + } if( zSql==0 ) return SQLITE_NOMEM; + rc = sqlite3_prepare_v2(pTab->db, zSql, -1, &pStmt, 0); sqlite3_free(zSql); if( rc==SQLITE_OK && sqlite3_step(pStmt)==SQLITE_ROW ){ @@ -11964,6 +12154,25 @@ static int dbdataDbsize(DbdataCursor *pCsr, const char *zSchema){ return rc; } +/* +** Attempt to figure out the encoding of the database by retrieving page 1 +** and inspecting the header field. If successful, set the pCsr->enc variable +** and return SQLITE_OK. Otherwise, return an SQLite error code. +*/ +static int dbdataGetEncoding(DbdataCursor *pCsr){ + int rc = SQLITE_OK; + int nPg1 = 0; + u8 *aPg1 = 0; + rc = dbdataLoadPage(pCsr, 1, &aPg1, &nPg1); + assert( rc!=SQLITE_OK || nPg1==0 || nPg1>=512 ); + if( rc==SQLITE_OK && nPg1>0 ){ + pCsr->enc = get_uint32(&aPg1[56]); + } + sqlite3_free(aPg1); + return rc; +} + + /* ** xFilter method for sqlite_dbdata and sqlite_dbptr. */ @@ -11981,19 +12190,28 @@ static int dbdataFilter( assert( pCsr->iPgno==1 ); if( idxNum & 0x01 ){ zSchema = (const char*)sqlite3_value_text(argv[0]); + if( zSchema==0 ) zSchema = ""; } if( idxNum & 0x02 ){ pCsr->iPgno = sqlite3_value_int(argv[(idxNum & 0x01)]); pCsr->bOnePage = 1; }else{ - pCsr->nPage = dbdataDbsize(pCsr, zSchema); rc = dbdataDbsize(pCsr, zSchema); } if( rc==SQLITE_OK ){ + int nFunc = 0; if( pTab->pStmt ){ pCsr->pStmt = pTab->pStmt; pTab->pStmt = 0; + }else if( (nFunc = dbdataIsFunction(zSchema))>0 ){ + char *zSql = sqlite3_mprintf("SELECT %.*s(?2)", nFunc, zSchema); + if( zSql==0 ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare_v2(pTab->db, zSql, -1, &pCsr->pStmt, 0); + sqlite3_free(zSql); + } }else{ rc = sqlite3_prepare_v2(pTab->db, "SELECT data FROM sqlite_dbpage(?) WHERE pgno=?", -1, @@ -12006,13 +12224,20 @@ static int dbdataFilter( }else{ pTab->base.zErrMsg = sqlite3_mprintf("%s", sqlite3_errmsg(pTab->db)); } + + /* Try to determine the encoding of the db by inspecting the header + ** field on page 1. */ + if( rc==SQLITE_OK ){ + rc = dbdataGetEncoding(pCsr); + } + if( rc==SQLITE_OK ){ rc = dbdataNext(pCursor); } return rc; } -/* +/* ** Return a column for the sqlite_dbdata or sqlite_dbptr table. */ static int dbdataColumn( @@ -12056,11 +12281,12 @@ static int dbdataColumn( case DBDATA_COLUMN_VALUE: { if( pCsr->iField<0 ){ sqlite3_result_int64(ctx, pCsr->iIntkey); - }else{ + }else if( &pCsr->pRec[pCsr->nRec] >= pCsr->pPtr ){ sqlite3_int64 iType; - dbdataGetVarint(pCsr->pHdrPtr, &iType); + dbdataGetVarintU32(pCsr->pHdrPtr, &iType); dbdataValue( - ctx, iType, pCsr->pPtr, &pCsr->pRec[pCsr->nRec] - pCsr->pPtr + ctx, pCsr->enc, iType, pCsr->pPtr, + &pCsr->pRec[pCsr->nRec] - pCsr->pPtr ); } break; @@ -12130,7 +12356,3121 @@ int sqlite3_dbdata_init( return sqlite3DbdataRegister(db); } -/************************* End ../ext/misc/dbdata.c ********************/ +#endif /* ifndef SQLITE_OMIT_VIRTUALTABLE */ + +/************************* End ../ext/recover/dbdata.c ********************/ +/************************* Begin ../ext/recover/sqlite3recover.h ******************/ +/* +** 2022-08-27 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains the public interface to the "recover" extension - +** an SQLite extension designed to recover data from corrupted database +** files. +*/ + +/* +** OVERVIEW: +** +** To use the API to recover data from a corrupted database, an +** application: +** +** 1) Creates an sqlite3_recover handle by calling either +** sqlite3_recover_init() or sqlite3_recover_init_sql(). +** +** 2) Configures the new handle using one or more calls to +** sqlite3_recover_config(). +** +** 3) Executes the recovery by repeatedly calling sqlite3_recover_step() on +** the handle until it returns something other than SQLITE_OK. If it +** returns SQLITE_DONE, then the recovery operation completed without +** error. If it returns some other non-SQLITE_OK value, then an error +** has occurred. +** +** 4) Retrieves any error code and English language error message using the +** sqlite3_recover_errcode() and sqlite3_recover_errmsg() APIs, +** respectively. +** +** 5) Destroys the sqlite3_recover handle and frees all resources +** using sqlite3_recover_finish(). +** +** The application may abandon the recovery operation at any point +** before it is finished by passing the sqlite3_recover handle to +** sqlite3_recover_finish(). This is not an error, but the final state +** of the output database, or the results of running the partial script +** delivered to the SQL callback, are undefined. +*/ + +#ifndef _SQLITE_RECOVER_H +#define _SQLITE_RECOVER_H + +/* #include "sqlite3.h" */ + +#ifdef __cplusplus +extern "C" { +#endif + +/* +** An instance of the sqlite3_recover object represents a recovery +** operation in progress. +** +** Constructors: +** +** sqlite3_recover_init() +** sqlite3_recover_init_sql() +** +** Destructor: +** +** sqlite3_recover_finish() +** +** Methods: +** +** sqlite3_recover_config() +** sqlite3_recover_errcode() +** sqlite3_recover_errmsg() +** sqlite3_recover_run() +** sqlite3_recover_step() +*/ +typedef struct sqlite3_recover sqlite3_recover; + +/* +** These two APIs attempt to create and return a new sqlite3_recover object. +** In both cases the first two arguments identify the (possibly +** corrupt) database to recover data from. The first argument is an open +** database handle and the second the name of a database attached to that +** handle (i.e. "main", "temp" or the name of an attached database). +** +** If sqlite3_recover_init() is used to create the new sqlite3_recover +** handle, then data is recovered into a new database, identified by +** string parameter zUri. zUri may be an absolute or relative file path, +** or may be an SQLite URI. If the identified database file already exists, +** it is overwritten. +** +** If sqlite3_recover_init_sql() is invoked, then any recovered data will +** be returned to the user as a series of SQL statements. Executing these +** SQL statements results in the same database as would have been created +** had sqlite3_recover_init() been used. For each SQL statement in the +** output, the callback function passed as the third argument (xSql) is +** invoked once. The first parameter is a passed a copy of the fourth argument +** to this function (pCtx) as its first parameter, and a pointer to a +** nul-terminated buffer containing the SQL statement formated as UTF-8 as +** the second. If the xSql callback returns any value other than SQLITE_OK, +** then processing is immediately abandoned and the value returned used as +** the recover handle error code (see below). +** +** If an out-of-memory error occurs, NULL may be returned instead of +** a valid handle. In all other cases, it is the responsibility of the +** application to avoid resource leaks by ensuring that +** sqlite3_recover_finish() is called on all allocated handles. +*/ +sqlite3_recover *sqlite3_recover_init( + sqlite3* db, + const char *zDb, + const char *zUri +); +sqlite3_recover *sqlite3_recover_init_sql( + sqlite3* db, + const char *zDb, + int (*xSql)(void*, const char*), + void *pCtx +); + +/* +** Configure an sqlite3_recover object that has just been created using +** sqlite3_recover_init() or sqlite3_recover_init_sql(). This function +** may only be called before the first call to sqlite3_recover_step() +** or sqlite3_recover_run() on the object. +** +** The second argument passed to this function must be one of the +** SQLITE_RECOVER_* symbols defined below. Valid values for the third argument +** depend on the specific SQLITE_RECOVER_* symbol in use. +** +** SQLITE_OK is returned if the configuration operation was successful, +** or an SQLite error code otherwise. +*/ +int sqlite3_recover_config(sqlite3_recover*, int op, void *pArg); + +/* +** SQLITE_RECOVER_LOST_AND_FOUND: +** The pArg argument points to a string buffer containing the name +** of a "lost-and-found" table in the output database, or NULL. If +** the argument is non-NULL and the database contains seemingly +** valid pages that cannot be associated with any table in the +** recovered part of the schema, data is extracted from these +** pages to add to the lost-and-found table. +** +** SQLITE_RECOVER_FREELIST_CORRUPT: +** The pArg value must actually be a pointer to a value of type +** int containing value 0 or 1 cast as a (void*). If this option is set +** (argument is 1) and a lost-and-found table has been configured using +** SQLITE_RECOVER_LOST_AND_FOUND, then is assumed that the freelist is +** corrupt and an attempt is made to recover records from pages that +** appear to be linked into the freelist. Otherwise, pages on the freelist +** are ignored. Setting this option can recover more data from the +** database, but often ends up "recovering" deleted records. The default +** value is 0 (clear). +** +** SQLITE_RECOVER_ROWIDS: +** The pArg value must actually be a pointer to a value of type +** int containing value 0 or 1 cast as a (void*). If this option is set +** (argument is 1), then an attempt is made to recover rowid values +** that are not also INTEGER PRIMARY KEY values. If this option is +** clear, then new rowids are assigned to all recovered rows. The +** default value is 1 (set). +** +** SQLITE_RECOVER_SLOWINDEXES: +** The pArg value must actually be a pointer to a value of type +** int containing value 0 or 1 cast as a (void*). If this option is clear +** (argument is 0), then when creating an output database, the recover +** module creates and populates non-UNIQUE indexes right at the end of the +** recovery operation - after all recoverable data has been inserted +** into the new database. This is faster overall, but means that the +** final call to sqlite3_recover_step() for a recovery operation may +** be need to create a large number of indexes, which may be very slow. +** +** Or, if this option is set (argument is 1), then non-UNIQUE indexes +** are created in the output database before it is populated with +** recovered data. This is slower overall, but avoids the slow call +** to sqlite3_recover_step() at the end of the recovery operation. +** +** The default option value is 0. +*/ +#define SQLITE_RECOVER_LOST_AND_FOUND 1 +#define SQLITE_RECOVER_FREELIST_CORRUPT 2 +#define SQLITE_RECOVER_ROWIDS 3 +#define SQLITE_RECOVER_SLOWINDEXES 4 + +/* +** Perform a unit of work towards the recovery operation. This function +** must normally be called multiple times to complete database recovery. +** +** If no error occurs but the recovery operation is not completed, this +** function returns SQLITE_OK. If recovery has been completed successfully +** then SQLITE_DONE is returned. If an error has occurred, then an SQLite +** error code (e.g. SQLITE_IOERR or SQLITE_NOMEM) is returned. It is not +** considered an error if some or all of the data cannot be recovered +** due to database corruption. +** +** Once sqlite3_recover_step() has returned a value other than SQLITE_OK, +** all further such calls on the same recover handle are no-ops that return +** the same non-SQLITE_OK value. +*/ +int sqlite3_recover_step(sqlite3_recover*); + +/* +** Run the recovery operation to completion. Return SQLITE_OK if successful, +** or an SQLite error code otherwise. Calling this function is the same +** as executing: +** +** while( SQLITE_OK==sqlite3_recover_step(p) ); +** return sqlite3_recover_errcode(p); +*/ +int sqlite3_recover_run(sqlite3_recover*); + +/* +** If an error has been encountered during a prior call to +** sqlite3_recover_step(), then this function attempts to return a +** pointer to a buffer containing an English language explanation of +** the error. If no error message is available, or if an out-of memory +** error occurs while attempting to allocate a buffer in which to format +** the error message, NULL is returned. +** +** The returned buffer remains valid until the sqlite3_recover handle is +** destroyed using sqlite3_recover_finish(). +*/ +const char *sqlite3_recover_errmsg(sqlite3_recover*); + +/* +** If this function is called on an sqlite3_recover handle after +** an error occurs, an SQLite error code is returned. Otherwise, SQLITE_OK. +*/ +int sqlite3_recover_errcode(sqlite3_recover*); + +/* +** Clean up a recovery object created by a call to sqlite3_recover_init(). +** The results of using a recovery object with any API after it has been +** passed to this function are undefined. +** +** This function returns the same value as sqlite3_recover_errcode(). +*/ +int sqlite3_recover_finish(sqlite3_recover*); + + +#ifdef __cplusplus +} /* end of the 'extern "C"' block */ +#endif + +#endif /* ifndef _SQLITE_RECOVER_H */ + +/************************* End ../ext/recover/sqlite3recover.h ********************/ +/************************* Begin ../ext/recover/sqlite3recover.c ******************/ +/* +** 2022-08-27 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +*/ + + +/* #include "sqlite3recover.h" */ +#include +#include + +#ifndef SQLITE_OMIT_VIRTUALTABLE + +/* +** Declaration for public API function in file dbdata.c. This may be called +** with NULL as the final two arguments to register the sqlite_dbptr and +** sqlite_dbdata virtual tables with a database handle. +*/ +#ifdef _WIN32 + +#endif +int sqlite3_dbdata_init(sqlite3*, char**, const sqlite3_api_routines*); + +/* typedef unsigned int u32; */ +/* typedef unsigned char u8; */ +/* typedef sqlite3_int64 i64; */ + +typedef struct RecoverTable RecoverTable; +typedef struct RecoverColumn RecoverColumn; + +/* +** When recovering rows of data that can be associated with table +** definitions recovered from the sqlite_schema table, each table is +** represented by an instance of the following object. +** +** iRoot: +** The root page in the original database. Not necessarily (and usually +** not) the same in the recovered database. +** +** zTab: +** Name of the table. +** +** nCol/aCol[]: +** aCol[] is an array of nCol columns. In the order in which they appear +** in the table. +** +** bIntkey: +** Set to true for intkey tables, false for WITHOUT ROWID. +** +** iRowidBind: +** Each column in the aCol[] array has associated with it the index of +** the bind parameter its values will be bound to in the INSERT statement +** used to construct the output database. If the table does has a rowid +** but not an INTEGER PRIMARY KEY column, then iRowidBind contains the +** index of the bind paramater to which the rowid value should be bound. +** Otherwise, it contains -1. If the table does contain an INTEGER PRIMARY +** KEY column, then the rowid value should be bound to the index associated +** with the column. +** +** pNext: +** All RecoverTable objects used by the recovery operation are allocated +** and populated as part of creating the recovered database schema in +** the output database, before any non-schema data are recovered. They +** are then stored in a singly-linked list linked by this variable beginning +** at sqlite3_recover.pTblList. +*/ +struct RecoverTable { + u32 iRoot; /* Root page in original database */ + char *zTab; /* Name of table */ + int nCol; /* Number of columns in table */ + RecoverColumn *aCol; /* Array of columns */ + int bIntkey; /* True for intkey, false for without rowid */ + int iRowidBind; /* If >0, bind rowid to INSERT here */ + RecoverTable *pNext; +}; + +/* +** Each database column is represented by an instance of the following object +** stored in the RecoverTable.aCol[] array of the associated table. +** +** iField: +** The index of the associated field within database records. Or -1 if +** there is no associated field (e.g. for virtual generated columns). +** +** iBind: +** The bind index of the INSERT statement to bind this columns values +** to. Or 0 if there is no such index (iff (iField<0)). +** +** bIPK: +** True if this is the INTEGER PRIMARY KEY column. +** +** zCol: +** Name of column. +** +** eHidden: +** A RECOVER_EHIDDEN_* constant value (see below for interpretation of each). +*/ +struct RecoverColumn { + int iField; /* Field in record on disk */ + int iBind; /* Binding to use in INSERT */ + int bIPK; /* True for IPK column */ + char *zCol; + int eHidden; +}; + +#define RECOVER_EHIDDEN_NONE 0 /* Normal database column */ +#define RECOVER_EHIDDEN_HIDDEN 1 /* Column is __HIDDEN__ */ +#define RECOVER_EHIDDEN_VIRTUAL 2 /* Virtual generated column */ +#define RECOVER_EHIDDEN_STORED 3 /* Stored generated column */ + +/* +** Bitmap object used to track pages in the input database. Allocated +** and manipulated only by the following functions: +** +** recoverBitmapAlloc() +** recoverBitmapFree() +** recoverBitmapSet() +** recoverBitmapQuery() +** +** nPg: +** Largest page number that may be stored in the bitmap. The range +** of valid keys is 1 to nPg, inclusive. +** +** aElem[]: +** Array large enough to contain a bit for each key. For key value +** iKey, the associated bit is the bit (iKey%32) of aElem[iKey/32]. +** In other words, the following is true if bit iKey is set, or +** false if it is clear: +** +** (aElem[iKey/32] & (1 << (iKey%32))) ? 1 : 0 +*/ +typedef struct RecoverBitmap RecoverBitmap; +struct RecoverBitmap { + i64 nPg; /* Size of bitmap */ + u32 aElem[1]; /* Array of 32-bit bitmasks */ +}; + +/* +** State variables (part of the sqlite3_recover structure) used while +** recovering data for tables identified in the recovered schema (state +** RECOVER_STATE_WRITING). +*/ +typedef struct RecoverStateW1 RecoverStateW1; +struct RecoverStateW1 { + sqlite3_stmt *pTbls; + sqlite3_stmt *pSel; + sqlite3_stmt *pInsert; + int nInsert; + + RecoverTable *pTab; /* Table currently being written */ + int nMax; /* Max column count in any schema table */ + sqlite3_value **apVal; /* Array of nMax values */ + int nVal; /* Number of valid entries in apVal[] */ + int bHaveRowid; + i64 iRowid; + i64 iPrevPage; + int iPrevCell; +}; + +/* +** State variables (part of the sqlite3_recover structure) used while +** recovering data destined for the lost and found table (states +** RECOVER_STATE_LOSTANDFOUND[123]). +*/ +typedef struct RecoverStateLAF RecoverStateLAF; +struct RecoverStateLAF { + RecoverBitmap *pUsed; + i64 nPg; /* Size of db in pages */ + sqlite3_stmt *pAllAndParent; + sqlite3_stmt *pMapInsert; + sqlite3_stmt *pMaxField; + sqlite3_stmt *pUsedPages; + sqlite3_stmt *pFindRoot; + sqlite3_stmt *pInsert; /* INSERT INTO lost_and_found ... */ + sqlite3_stmt *pAllPage; + sqlite3_stmt *pPageData; + sqlite3_value **apVal; + int nMaxField; +}; + +/* +** Main recover handle structure. +*/ +struct sqlite3_recover { + /* Copies of sqlite3_recover_init[_sql]() parameters */ + sqlite3 *dbIn; /* Input database */ + char *zDb; /* Name of input db ("main" etc.) */ + char *zUri; /* URI for output database */ + void *pSqlCtx; /* SQL callback context */ + int (*xSql)(void*,const char*); /* Pointer to SQL callback function */ + + /* Values configured by sqlite3_recover_config() */ + char *zStateDb; /* State database to use (or NULL) */ + char *zLostAndFound; /* Name of lost-and-found table (or NULL) */ + int bFreelistCorrupt; /* SQLITE_RECOVER_FREELIST_CORRUPT setting */ + int bRecoverRowid; /* SQLITE_RECOVER_ROWIDS setting */ + int bSlowIndexes; /* SQLITE_RECOVER_SLOWINDEXES setting */ + + int pgsz; + int detected_pgsz; + int nReserve; + u8 *pPage1Disk; + u8 *pPage1Cache; + + /* Error code and error message */ + int errCode; /* For sqlite3_recover_errcode() */ + char *zErrMsg; /* For sqlite3_recover_errmsg() */ + + int eState; + int bCloseTransaction; + + /* Variables used with eState==RECOVER_STATE_WRITING */ + RecoverStateW1 w1; + + /* Variables used with states RECOVER_STATE_LOSTANDFOUND[123] */ + RecoverStateLAF laf; + + /* Fields used within sqlite3_recover_run() */ + sqlite3 *dbOut; /* Output database */ + sqlite3_stmt *pGetPage; /* SELECT against input db sqlite_dbdata */ + RecoverTable *pTblList; /* List of tables recovered from schema */ +}; + +/* +** The various states in which an sqlite3_recover object may exist: +** +** RECOVER_STATE_INIT: +** The object is initially created in this state. sqlite3_recover_step() +** has yet to be called. This is the only state in which it is permitted +** to call sqlite3_recover_config(). +** +** RECOVER_STATE_WRITING: +** +** RECOVER_STATE_LOSTANDFOUND1: +** State to populate the bitmap of pages used by other tables or the +** database freelist. +** +** RECOVER_STATE_LOSTANDFOUND2: +** Populate the recovery.map table - used to figure out a "root" page +** for each lost page from in the database from which records are +** extracted. +** +** RECOVER_STATE_LOSTANDFOUND3: +** Populate the lost-and-found table itself. +*/ +#define RECOVER_STATE_INIT 0 +#define RECOVER_STATE_WRITING 1 +#define RECOVER_STATE_LOSTANDFOUND1 2 +#define RECOVER_STATE_LOSTANDFOUND2 3 +#define RECOVER_STATE_LOSTANDFOUND3 4 +#define RECOVER_STATE_SCHEMA2 5 +#define RECOVER_STATE_DONE 6 + + +/* +** Global variables used by this extension. +*/ +typedef struct RecoverGlobal RecoverGlobal; +struct RecoverGlobal { + const sqlite3_io_methods *pMethods; + sqlite3_recover *p; +}; +static RecoverGlobal recover_g; + +/* +** Use this static SQLite mutex to protect the globals during the +** first call to sqlite3_recover_step(). +*/ +#define RECOVER_MUTEX_ID SQLITE_MUTEX_STATIC_APP2 + + +/* +** Default value for SQLITE_RECOVER_ROWIDS (sqlite3_recover.bRecoverRowid). +*/ +#define RECOVER_ROWID_DEFAULT 1 + +/* +** Mutex handling: +** +** recoverEnterMutex() - Enter the recovery mutex +** recoverLeaveMutex() - Leave the recovery mutex +** recoverAssertMutexHeld() - Assert that the recovery mutex is held +*/ +#if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE==0 +# define recoverEnterMutex() +# define recoverLeaveMutex() +#else +static void recoverEnterMutex(void){ + sqlite3_mutex_enter(sqlite3_mutex_alloc(RECOVER_MUTEX_ID)); +} +static void recoverLeaveMutex(void){ + sqlite3_mutex_leave(sqlite3_mutex_alloc(RECOVER_MUTEX_ID)); +} +#endif +#if SQLITE_THREADSAFE+0>=1 && defined(SQLITE_DEBUG) +static void recoverAssertMutexHeld(void){ + assert( sqlite3_mutex_held(sqlite3_mutex_alloc(RECOVER_MUTEX_ID)) ); +} +#else +# define recoverAssertMutexHeld() +#endif + + +/* +** Like strlen(). But handles NULL pointer arguments. +*/ +static int recoverStrlen(const char *zStr){ + if( zStr==0 ) return 0; + return (int)(strlen(zStr)&0x7fffffff); +} + +/* +** This function is a no-op if the recover handle passed as the first +** argument already contains an error (if p->errCode!=SQLITE_OK). +** +** Otherwise, an attempt is made to allocate, zero and return a buffer nByte +** bytes in size. If successful, a pointer to the new buffer is returned. Or, +** if an OOM error occurs, NULL is returned and the handle error code +** (p->errCode) set to SQLITE_NOMEM. +*/ +static void *recoverMalloc(sqlite3_recover *p, i64 nByte){ + void *pRet = 0; + assert( nByte>0 ); + if( p->errCode==SQLITE_OK ){ + pRet = sqlite3_malloc64(nByte); + if( pRet ){ + memset(pRet, 0, nByte); + }else{ + p->errCode = SQLITE_NOMEM; + } + } + return pRet; +} + +/* +** Set the error code and error message for the recover handle passed as +** the first argument. The error code is set to the value of parameter +** errCode. +** +** Parameter zFmt must be a printf() style formatting string. The handle +** error message is set to the result of using any trailing arguments for +** parameter substitutions in the formatting string. +** +** For example: +** +** recoverError(p, SQLITE_ERROR, "no such table: %s", zTablename); +*/ +static int recoverError( + sqlite3_recover *p, + int errCode, + const char *zFmt, ... +){ + char *z = 0; + va_list ap; + va_start(ap, zFmt); + if( zFmt ){ + z = sqlite3_vmprintf(zFmt, ap); + va_end(ap); + } + sqlite3_free(p->zErrMsg); + p->zErrMsg = z; + p->errCode = errCode; + return errCode; +} + + +/* +** This function is a no-op if p->errCode is initially other than SQLITE_OK. +** In this case it returns NULL. +** +** Otherwise, an attempt is made to allocate and return a bitmap object +** large enough to store a bit for all page numbers between 1 and nPg, +** inclusive. The bitmap is initially zeroed. +*/ +static RecoverBitmap *recoverBitmapAlloc(sqlite3_recover *p, i64 nPg){ + int nElem = (nPg+1+31) / 32; + int nByte = sizeof(RecoverBitmap) + nElem*sizeof(u32); + RecoverBitmap *pRet = (RecoverBitmap*)recoverMalloc(p, nByte); + + if( pRet ){ + pRet->nPg = nPg; + } + return pRet; +} + +/* +** Free a bitmap object allocated by recoverBitmapAlloc(). +*/ +static void recoverBitmapFree(RecoverBitmap *pMap){ + sqlite3_free(pMap); +} + +/* +** Set the bit associated with page iPg in bitvec pMap. +*/ +static void recoverBitmapSet(RecoverBitmap *pMap, i64 iPg){ + if( iPg<=pMap->nPg ){ + int iElem = (iPg / 32); + int iBit = (iPg % 32); + pMap->aElem[iElem] |= (((u32)1) << iBit); + } +} + +/* +** Query bitmap object pMap for the state of the bit associated with page +** iPg. Return 1 if it is set, or 0 otherwise. +*/ +static int recoverBitmapQuery(RecoverBitmap *pMap, i64 iPg){ + int ret = 1; + if( iPg<=pMap->nPg && iPg>0 ){ + int iElem = (iPg / 32); + int iBit = (iPg % 32); + ret = (pMap->aElem[iElem] & (((u32)1) << iBit)) ? 1 : 0; + } + return ret; +} + +/* +** Set the recover handle error to the error code and message returned by +** calling sqlite3_errcode() and sqlite3_errmsg(), respectively, on database +** handle db. +*/ +static int recoverDbError(sqlite3_recover *p, sqlite3 *db){ + return recoverError(p, sqlite3_errcode(db), "%s", sqlite3_errmsg(db)); +} + +/* +** This function is a no-op if recover handle p already contains an error +** (if p->errCode!=SQLITE_OK). +** +** Otherwise, it attempts to prepare the SQL statement in zSql against +** database handle db. If successful, the statement handle is returned. +** Or, if an error occurs, NULL is returned and an error left in the +** recover handle. +*/ +static sqlite3_stmt *recoverPrepare( + sqlite3_recover *p, + sqlite3 *db, + const char *zSql +){ + sqlite3_stmt *pStmt = 0; + if( p->errCode==SQLITE_OK ){ + if( sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0) ){ + recoverDbError(p, db); + } + } + return pStmt; +} + +/* +** This function is a no-op if recover handle p already contains an error +** (if p->errCode!=SQLITE_OK). +** +** Otherwise, argument zFmt is used as a printf() style format string, +** along with any trailing arguments, to create an SQL statement. This +** SQL statement is prepared against database handle db and, if successful, +** the statment handle returned. Or, if an error occurs - either during +** the printf() formatting or when preparing the resulting SQL - an +** error code and message are left in the recover handle. +*/ +static sqlite3_stmt *recoverPreparePrintf( + sqlite3_recover *p, + sqlite3 *db, + const char *zFmt, ... +){ + sqlite3_stmt *pStmt = 0; + if( p->errCode==SQLITE_OK ){ + va_list ap; + char *z; + va_start(ap, zFmt); + z = sqlite3_vmprintf(zFmt, ap); + va_end(ap); + if( z==0 ){ + p->errCode = SQLITE_NOMEM; + }else{ + pStmt = recoverPrepare(p, db, z); + sqlite3_free(z); + } + } + return pStmt; +} + +/* +** Reset SQLite statement handle pStmt. If the call to sqlite3_reset() +** indicates that an error occurred, and there is not already an error +** in the recover handle passed as the first argument, set the error +** code and error message appropriately. +** +** This function returns a copy of the statement handle pointer passed +** as the second argument. +*/ +static sqlite3_stmt *recoverReset(sqlite3_recover *p, sqlite3_stmt *pStmt){ + int rc = sqlite3_reset(pStmt); + if( rc!=SQLITE_OK && rc!=SQLITE_CONSTRAINT && p->errCode==SQLITE_OK ){ + recoverDbError(p, sqlite3_db_handle(pStmt)); + } + return pStmt; +} + +/* +** Finalize SQLite statement handle pStmt. If the call to sqlite3_reset() +** indicates that an error occurred, and there is not already an error +** in the recover handle passed as the first argument, set the error +** code and error message appropriately. +*/ +static void recoverFinalize(sqlite3_recover *p, sqlite3_stmt *pStmt){ + sqlite3 *db = sqlite3_db_handle(pStmt); + int rc = sqlite3_finalize(pStmt); + if( rc!=SQLITE_OK && p->errCode==SQLITE_OK ){ + recoverDbError(p, db); + } +} + +/* +** This function is a no-op if recover handle p already contains an error +** (if p->errCode!=SQLITE_OK). A copy of p->errCode is returned in this +** case. +** +** Otherwise, execute SQL script zSql. If successful, return SQLITE_OK. +** Or, if an error occurs, leave an error code and message in the recover +** handle and return a copy of the error code. +*/ +static int recoverExec(sqlite3_recover *p, sqlite3 *db, const char *zSql){ + if( p->errCode==SQLITE_OK ){ + int rc = sqlite3_exec(db, zSql, 0, 0, 0); + if( rc ){ + recoverDbError(p, db); + } + } + return p->errCode; +} + +/* +** Bind the value pVal to parameter iBind of statement pStmt. Leave an +** error in the recover handle passed as the first argument if an error +** (e.g. an OOM) occurs. +*/ +static void recoverBindValue( + sqlite3_recover *p, + sqlite3_stmt *pStmt, + int iBind, + sqlite3_value *pVal +){ + if( p->errCode==SQLITE_OK ){ + int rc = sqlite3_bind_value(pStmt, iBind, pVal); + if( rc ) recoverError(p, rc, 0); + } +} + +/* +** This function is a no-op if recover handle p already contains an error +** (if p->errCode!=SQLITE_OK). NULL is returned in this case. +** +** Otherwise, an attempt is made to interpret zFmt as a printf() style +** formatting string and the result of using the trailing arguments for +** parameter substitution with it written into a buffer obtained from +** sqlite3_malloc(). If successful, a pointer to the buffer is returned. +** It is the responsibility of the caller to eventually free the buffer +** using sqlite3_free(). +** +** Or, if an error occurs, an error code and message is left in the recover +** handle and NULL returned. +*/ +static char *recoverMPrintf(sqlite3_recover *p, const char *zFmt, ...){ + va_list ap; + char *z; + va_start(ap, zFmt); + z = sqlite3_vmprintf(zFmt, ap); + va_end(ap); + if( p->errCode==SQLITE_OK ){ + if( z==0 ) p->errCode = SQLITE_NOMEM; + }else{ + sqlite3_free(z); + z = 0; + } + return z; +} + +/* +** This function is a no-op if recover handle p already contains an error +** (if p->errCode!=SQLITE_OK). Zero is returned in this case. +** +** Otherwise, execute "PRAGMA page_count" against the input database. If +** successful, return the integer result. Or, if an error occurs, leave an +** error code and error message in the sqlite3_recover handle and return +** zero. +*/ +static i64 recoverPageCount(sqlite3_recover *p){ + i64 nPg = 0; + if( p->errCode==SQLITE_OK ){ + sqlite3_stmt *pStmt = 0; + pStmt = recoverPreparePrintf(p, p->dbIn, "PRAGMA %Q.page_count", p->zDb); + if( pStmt ){ + sqlite3_step(pStmt); + nPg = sqlite3_column_int64(pStmt, 0); + } + recoverFinalize(p, pStmt); + } + return nPg; +} + +/* +** Implementation of SQL scalar function "read_i32". The first argument to +** this function must be a blob. The second a non-negative integer. This +** function reads and returns a 32-bit big-endian integer from byte +** offset (4*) of the blob. +** +** SELECT read_i32(, ) +*/ +static void recoverReadI32( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const unsigned char *pBlob; + int nBlob; + int iInt; + + assert( argc==2 ); + nBlob = sqlite3_value_bytes(argv[0]); + pBlob = (const unsigned char*)sqlite3_value_blob(argv[0]); + iInt = sqlite3_value_int(argv[1]) & 0xFFFF; + + if( (iInt+1)*4<=nBlob ){ + const unsigned char *a = &pBlob[iInt*4]; + i64 iVal = ((i64)a[0]<<24) + + ((i64)a[1]<<16) + + ((i64)a[2]<< 8) + + ((i64)a[3]<< 0); + sqlite3_result_int64(context, iVal); + } +} + +/* +** Implementation of SQL scalar function "page_is_used". This function +** is used as part of the procedure for locating orphan rows for the +** lost-and-found table, and it depends on those routines having populated +** the sqlite3_recover.laf.pUsed variable. +** +** The only argument to this function is a page-number. It returns true +** if the page has already been used somehow during data recovery, or false +** otherwise. +** +** SELECT page_is_used(); +*/ +static void recoverPageIsUsed( + sqlite3_context *pCtx, + int nArg, + sqlite3_value **apArg +){ + sqlite3_recover *p = (sqlite3_recover*)sqlite3_user_data(pCtx); + i64 pgno = sqlite3_value_int64(apArg[0]); + assert( nArg==1 ); + sqlite3_result_int(pCtx, recoverBitmapQuery(p->laf.pUsed, pgno)); +} + +/* +** The implementation of a user-defined SQL function invoked by the +** sqlite_dbdata and sqlite_dbptr virtual table modules to access pages +** of the database being recovered. +** +** This function always takes a single integer argument. If the argument +** is zero, then the value returned is the number of pages in the db being +** recovered. If the argument is greater than zero, it is a page number. +** The value returned in this case is an SQL blob containing the data for +** the identified page of the db being recovered. e.g. +** +** SELECT getpage(0); -- return number of pages in db +** SELECT getpage(4); -- return page 4 of db as a blob of data +*/ +static void recoverGetPage( + sqlite3_context *pCtx, + int nArg, + sqlite3_value **apArg +){ + sqlite3_recover *p = (sqlite3_recover*)sqlite3_user_data(pCtx); + i64 pgno = sqlite3_value_int64(apArg[0]); + sqlite3_stmt *pStmt = 0; + + assert( nArg==1 ); + if( pgno==0 ){ + i64 nPg = recoverPageCount(p); + sqlite3_result_int64(pCtx, nPg); + return; + }else{ + if( p->pGetPage==0 ){ + pStmt = p->pGetPage = recoverPreparePrintf( + p, p->dbIn, "SELECT data FROM sqlite_dbpage(%Q) WHERE pgno=?", p->zDb + ); + }else if( p->errCode==SQLITE_OK ){ + pStmt = p->pGetPage; + } + + if( pStmt ){ + sqlite3_bind_int64(pStmt, 1, pgno); + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + const u8 *aPg; + int nPg; + assert( p->errCode==SQLITE_OK ); + aPg = sqlite3_column_blob(pStmt, 0); + nPg = sqlite3_column_bytes(pStmt, 0); + if( pgno==1 && nPg==p->pgsz && 0==memcmp(p->pPage1Cache, aPg, nPg) ){ + aPg = p->pPage1Disk; + } + sqlite3_result_blob(pCtx, aPg, nPg-p->nReserve, SQLITE_TRANSIENT); + } + recoverReset(p, pStmt); + } + } + + if( p->errCode ){ + if( p->zErrMsg ) sqlite3_result_error(pCtx, p->zErrMsg, -1); + sqlite3_result_error_code(pCtx, p->errCode); + } +} + +/* +** Find a string that is not found anywhere in z[]. Return a pointer +** to that string. +** +** Try to use zA and zB first. If both of those are already found in z[] +** then make up some string and store it in the buffer zBuf. +*/ +static const char *recoverUnusedString( + const char *z, /* Result must not appear anywhere in z */ + const char *zA, const char *zB, /* Try these first */ + char *zBuf /* Space to store a generated string */ +){ + unsigned i = 0; + if( strstr(z, zA)==0 ) return zA; + if( strstr(z, zB)==0 ) return zB; + do{ + sqlite3_snprintf(20,zBuf,"(%s%u)", zA, i++); + }while( strstr(z,zBuf)!=0 ); + return zBuf; +} + +/* +** Implementation of scalar SQL function "escape_crnl". The argument passed to +** this function is the output of built-in function quote(). If the first +** character of the input is "'", indicating that the value passed to quote() +** was a text value, then this function searches the input for "\n" and "\r" +** characters and adds a wrapper similar to the following: +** +** replace(replace(, '\n', char(10), '\r', char(13)); +** +** Or, if the first character of the input is not "'", then a copy of the input +** is returned. +*/ +static void recoverEscapeCrnl( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const char *zText = (const char*)sqlite3_value_text(argv[0]); + if( zText && zText[0]=='\'' ){ + int nText = sqlite3_value_bytes(argv[0]); + int i; + char zBuf1[20]; + char zBuf2[20]; + const char *zNL = 0; + const char *zCR = 0; + int nCR = 0; + int nNL = 0; + + for(i=0; zText[i]; i++){ + if( zNL==0 && zText[i]=='\n' ){ + zNL = recoverUnusedString(zText, "\\n", "\\012", zBuf1); + nNL = (int)strlen(zNL); + } + if( zCR==0 && zText[i]=='\r' ){ + zCR = recoverUnusedString(zText, "\\r", "\\015", zBuf2); + nCR = (int)strlen(zCR); + } + } + + if( zNL || zCR ){ + int iOut = 0; + i64 nMax = (nNL > nCR) ? nNL : nCR; + i64 nAlloc = nMax * nText + (nMax+64)*2; + char *zOut = (char*)sqlite3_malloc64(nAlloc); + if( zOut==0 ){ + sqlite3_result_error_nomem(context); + return; + } + + if( zNL && zCR ){ + memcpy(&zOut[iOut], "replace(replace(", 16); + iOut += 16; + }else{ + memcpy(&zOut[iOut], "replace(", 8); + iOut += 8; + } + for(i=0; zText[i]; i++){ + if( zText[i]=='\n' ){ + memcpy(&zOut[iOut], zNL, nNL); + iOut += nNL; + }else if( zText[i]=='\r' ){ + memcpy(&zOut[iOut], zCR, nCR); + iOut += nCR; + }else{ + zOut[iOut] = zText[i]; + iOut++; + } + } + + if( zNL ){ + memcpy(&zOut[iOut], ",'", 2); iOut += 2; + memcpy(&zOut[iOut], zNL, nNL); iOut += nNL; + memcpy(&zOut[iOut], "', char(10))", 12); iOut += 12; + } + if( zCR ){ + memcpy(&zOut[iOut], ",'", 2); iOut += 2; + memcpy(&zOut[iOut], zCR, nCR); iOut += nCR; + memcpy(&zOut[iOut], "', char(13))", 12); iOut += 12; + } + + sqlite3_result_text(context, zOut, iOut, SQLITE_TRANSIENT); + sqlite3_free(zOut); + return; + } + } + + sqlite3_result_value(context, argv[0]); +} + +/* +** This function is a no-op if recover handle p already contains an error +** (if p->errCode!=SQLITE_OK). A copy of the error code is returned in +** this case. +** +** Otherwise, attempt to populate temporary table "recovery.schema" with the +** parts of the database schema that can be extracted from the input database. +** +** If no error occurs, SQLITE_OK is returned. Otherwise, an error code +** and error message are left in the recover handle and a copy of the +** error code returned. It is not considered an error if part of all of +** the database schema cannot be recovered due to corruption. +*/ +static int recoverCacheSchema(sqlite3_recover *p){ + return recoverExec(p, p->dbOut, + "WITH RECURSIVE pages(p) AS (" + " SELECT 1" + " UNION" + " SELECT child FROM sqlite_dbptr('getpage()'), pages WHERE pgno=p" + ")" + "INSERT INTO recovery.schema SELECT" + " max(CASE WHEN field=0 THEN value ELSE NULL END)," + " max(CASE WHEN field=1 THEN value ELSE NULL END)," + " max(CASE WHEN field=2 THEN value ELSE NULL END)," + " max(CASE WHEN field=3 THEN value ELSE NULL END)," + " max(CASE WHEN field=4 THEN value ELSE NULL END)" + "FROM sqlite_dbdata('getpage()') WHERE pgno IN (" + " SELECT p FROM pages" + ") GROUP BY pgno, cell" + ); +} + +/* +** If this recover handle is not in SQL callback mode (i.e. was not created +** using sqlite3_recover_init_sql()) of if an error has already occurred, +** this function is a no-op. Otherwise, issue a callback with SQL statement +** zSql as the parameter. +** +** If the callback returns non-zero, set the recover handle error code to +** the value returned (so that the caller will abandon processing). +*/ +static void recoverSqlCallback(sqlite3_recover *p, const char *zSql){ + if( p->errCode==SQLITE_OK && p->xSql ){ + int res = p->xSql(p->pSqlCtx, zSql); + if( res ){ + recoverError(p, SQLITE_ERROR, "callback returned an error - %d", res); + } + } +} + +/* +** Transfer the following settings from the input database to the output +** database: +** +** + page-size, +** + auto-vacuum settings, +** + database encoding, +** + user-version (PRAGMA user_version), and +** + application-id (PRAGMA application_id), and +*/ +static void recoverTransferSettings(sqlite3_recover *p){ + const char *aPragma[] = { + "encoding", + "page_size", + "auto_vacuum", + "user_version", + "application_id" + }; + int ii; + + /* Truncate the output database to 0 pages in size. This is done by + ** opening a new, empty, temp db, then using the backup API to clobber + ** any existing output db with a copy of it. */ + if( p->errCode==SQLITE_OK ){ + sqlite3 *db2 = 0; + int rc = sqlite3_open("", &db2); + if( rc!=SQLITE_OK ){ + recoverDbError(p, db2); + return; + } + + for(ii=0; iidbIn, "PRAGMA %Q.%s", p->zDb, zPrag); + if( p->errCode==SQLITE_OK && sqlite3_step(p1)==SQLITE_ROW ){ + const char *zArg = (const char*)sqlite3_column_text(p1, 0); + char *z2 = recoverMPrintf(p, "PRAGMA %s = %Q", zPrag, zArg); + recoverSqlCallback(p, z2); + recoverExec(p, db2, z2); + sqlite3_free(z2); + if( zArg==0 ){ + recoverError(p, SQLITE_NOMEM, 0); + } + } + recoverFinalize(p, p1); + } + recoverExec(p, db2, "CREATE TABLE t1(a); DROP TABLE t1;"); + + if( p->errCode==SQLITE_OK ){ + sqlite3 *db = p->dbOut; + sqlite3_backup *pBackup = sqlite3_backup_init(db, "main", db2, "main"); + if( pBackup ){ + sqlite3_backup_step(pBackup, -1); + p->errCode = sqlite3_backup_finish(pBackup); + }else{ + recoverDbError(p, db); + } + } + + sqlite3_close(db2); + } +} + +/* +** This function is a no-op if recover handle p already contains an error +** (if p->errCode!=SQLITE_OK). A copy of the error code is returned in +** this case. +** +** Otherwise, an attempt is made to open the output database, attach +** and create the schema of the temporary database used to store +** intermediate data, and to register all required user functions and +** virtual table modules with the output handle. +** +** If no error occurs, SQLITE_OK is returned. Otherwise, an error code +** and error message are left in the recover handle and a copy of the +** error code returned. +*/ +static int recoverOpenOutput(sqlite3_recover *p){ + struct Func { + const char *zName; + int nArg; + void (*xFunc)(sqlite3_context*,int,sqlite3_value **); + } aFunc[] = { + { "getpage", 1, recoverGetPage }, + { "page_is_used", 1, recoverPageIsUsed }, + { "read_i32", 2, recoverReadI32 }, + { "escape_crnl", 1, recoverEscapeCrnl }, + }; + + const int flags = SQLITE_OPEN_URI|SQLITE_OPEN_CREATE|SQLITE_OPEN_READWRITE; + sqlite3 *db = 0; /* New database handle */ + int ii; /* For iterating through aFunc[] */ + + assert( p->dbOut==0 ); + + if( sqlite3_open_v2(p->zUri, &db, flags, 0) ){ + recoverDbError(p, db); + } + + /* Register the sqlite_dbdata and sqlite_dbptr virtual table modules. + ** These two are registered with the output database handle - this + ** module depends on the input handle supporting the sqlite_dbpage + ** virtual table only. */ + if( p->errCode==SQLITE_OK ){ + p->errCode = sqlite3_dbdata_init(db, 0, 0); + } + + /* Register the custom user-functions with the output handle. */ + for(ii=0; p->errCode==SQLITE_OK && iierrCode = sqlite3_create_function(db, aFunc[ii].zName, + aFunc[ii].nArg, SQLITE_UTF8, (void*)p, aFunc[ii].xFunc, 0, 0 + ); + } + + p->dbOut = db; + return p->errCode; +} + +/* +** Attach the auxiliary database 'recovery' to the output database handle. +** This temporary database is used during the recovery process and then +** discarded. +*/ +static void recoverOpenRecovery(sqlite3_recover *p){ + char *zSql = recoverMPrintf(p, "ATTACH %Q AS recovery;", p->zStateDb); + recoverExec(p, p->dbOut, zSql); + recoverExec(p, p->dbOut, + "PRAGMA writable_schema = 1;" + "CREATE TABLE recovery.map(pgno INTEGER PRIMARY KEY, parent INT);" + "CREATE TABLE recovery.schema(type, name, tbl_name, rootpage, sql);" + ); + sqlite3_free(zSql); +} + + +/* +** This function is a no-op if recover handle p already contains an error +** (if p->errCode!=SQLITE_OK). +** +** Otherwise, argument zName must be the name of a table that has just been +** created in the output database. This function queries the output db +** for the schema of said table, and creates a RecoverTable object to +** store the schema in memory. The new RecoverTable object is linked into +** the list at sqlite3_recover.pTblList. +** +** Parameter iRoot must be the root page of table zName in the INPUT +** database. +*/ +static void recoverAddTable( + sqlite3_recover *p, + const char *zName, /* Name of table created in output db */ + i64 iRoot /* Root page of same table in INPUT db */ +){ + sqlite3_stmt *pStmt = recoverPreparePrintf(p, p->dbOut, + "PRAGMA table_xinfo(%Q)", zName + ); + + if( pStmt ){ + int iPk = -1; + int iBind = 1; + RecoverTable *pNew = 0; + int nCol = 0; + int nName = recoverStrlen(zName); + int nByte = 0; + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + nCol++; + nByte += (sqlite3_column_bytes(pStmt, 1)+1); + } + nByte += sizeof(RecoverTable) + nCol*sizeof(RecoverColumn) + nName+1; + recoverReset(p, pStmt); + + pNew = recoverMalloc(p, nByte); + if( pNew ){ + int i = 0; + int iField = 0; + char *csr = 0; + pNew->aCol = (RecoverColumn*)&pNew[1]; + pNew->zTab = csr = (char*)&pNew->aCol[nCol]; + pNew->nCol = nCol; + pNew->iRoot = iRoot; + memcpy(csr, zName, nName); + csr += nName+1; + + for(i=0; sqlite3_step(pStmt)==SQLITE_ROW; i++){ + int iPKF = sqlite3_column_int(pStmt, 5); + int n = sqlite3_column_bytes(pStmt, 1); + const char *z = (const char*)sqlite3_column_text(pStmt, 1); + const char *zType = (const char*)sqlite3_column_text(pStmt, 2); + int eHidden = sqlite3_column_int(pStmt, 6); + + if( iPk==-1 && iPKF==1 && !sqlite3_stricmp("integer", zType) ) iPk = i; + if( iPKF>1 ) iPk = -2; + pNew->aCol[i].zCol = csr; + pNew->aCol[i].eHidden = eHidden; + if( eHidden==RECOVER_EHIDDEN_VIRTUAL ){ + pNew->aCol[i].iField = -1; + }else{ + pNew->aCol[i].iField = iField++; + } + if( eHidden!=RECOVER_EHIDDEN_VIRTUAL + && eHidden!=RECOVER_EHIDDEN_STORED + ){ + pNew->aCol[i].iBind = iBind++; + } + memcpy(csr, z, n); + csr += (n+1); + } + + pNew->pNext = p->pTblList; + p->pTblList = pNew; + pNew->bIntkey = 1; + } + + recoverFinalize(p, pStmt); + + pStmt = recoverPreparePrintf(p, p->dbOut, "PRAGMA index_xinfo(%Q)", zName); + while( pStmt && sqlite3_step(pStmt)==SQLITE_ROW ){ + int iField = sqlite3_column_int(pStmt, 0); + int iCol = sqlite3_column_int(pStmt, 1); + + assert( iFieldnCol && iColnCol ); + pNew->aCol[iCol].iField = iField; + + pNew->bIntkey = 0; + iPk = -2; + } + recoverFinalize(p, pStmt); + + if( p->errCode==SQLITE_OK ){ + if( iPk>=0 ){ + pNew->aCol[iPk].bIPK = 1; + }else if( pNew->bIntkey ){ + pNew->iRowidBind = iBind++; + } + } + } +} + +/* +** This function is called after recoverCacheSchema() has cached those parts +** of the input database schema that could be recovered in temporary table +** "recovery.schema". This function creates in the output database copies +** of all parts of that schema that must be created before the tables can +** be populated. Specifically, this means: +** +** * all tables that are not VIRTUAL, and +** * UNIQUE indexes. +** +** If the recovery handle uses SQL callbacks, then callbacks containing +** the associated "CREATE TABLE" and "CREATE INDEX" statements are made. +** +** Additionally, records are added to the sqlite_schema table of the +** output database for any VIRTUAL tables. The CREATE VIRTUAL TABLE +** records are written directly to sqlite_schema, not actually executed. +** If the handle is in SQL callback mode, then callbacks are invoked +** with equivalent SQL statements. +*/ +static int recoverWriteSchema1(sqlite3_recover *p){ + sqlite3_stmt *pSelect = 0; + sqlite3_stmt *pTblname = 0; + + pSelect = recoverPrepare(p, p->dbOut, + "WITH dbschema(rootpage, name, sql, tbl, isVirtual, isIndex) AS (" + " SELECT rootpage, name, sql, " + " type='table', " + " sql LIKE 'create virtual%'," + " (type='index' AND (sql LIKE '%unique%' OR ?1))" + " FROM recovery.schema" + ")" + "SELECT rootpage, tbl, isVirtual, name, sql" + " FROM dbschema " + " WHERE tbl OR isIndex" + " ORDER BY tbl DESC, name=='sqlite_sequence' DESC" + ); + + pTblname = recoverPrepare(p, p->dbOut, + "SELECT name FROM sqlite_schema " + "WHERE type='table' ORDER BY rowid DESC LIMIT 1" + ); + + if( pSelect ){ + sqlite3_bind_int(pSelect, 1, p->bSlowIndexes); + while( sqlite3_step(pSelect)==SQLITE_ROW ){ + i64 iRoot = sqlite3_column_int64(pSelect, 0); + int bTable = sqlite3_column_int(pSelect, 1); + int bVirtual = sqlite3_column_int(pSelect, 2); + const char *zName = (const char*)sqlite3_column_text(pSelect, 3); + const char *zSql = (const char*)sqlite3_column_text(pSelect, 4); + char *zFree = 0; + int rc = SQLITE_OK; + + if( bVirtual ){ + zSql = (const char*)(zFree = recoverMPrintf(p, + "INSERT INTO sqlite_schema VALUES('table', %Q, %Q, 0, %Q)", + zName, zName, zSql + )); + } + rc = sqlite3_exec(p->dbOut, zSql, 0, 0, 0); + if( rc==SQLITE_OK ){ + recoverSqlCallback(p, zSql); + if( bTable && !bVirtual ){ + if( SQLITE_ROW==sqlite3_step(pTblname) ){ + const char *zTbl = (const char*)sqlite3_column_text(pTblname, 0); + recoverAddTable(p, zTbl, iRoot); + } + recoverReset(p, pTblname); + } + }else if( rc!=SQLITE_ERROR ){ + recoverDbError(p, p->dbOut); + } + sqlite3_free(zFree); + } + } + recoverFinalize(p, pSelect); + recoverFinalize(p, pTblname); + + return p->errCode; +} + +/* +** This function is called after the output database has been populated. It +** adds all recovered schema elements that were not created in the output +** database by recoverWriteSchema1() - everything except for tables and +** UNIQUE indexes. Specifically: +** +** * views, +** * triggers, +** * non-UNIQUE indexes. +** +** If the recover handle is in SQL callback mode, then equivalent callbacks +** are issued to create the schema elements. +*/ +static int recoverWriteSchema2(sqlite3_recover *p){ + sqlite3_stmt *pSelect = 0; + + pSelect = recoverPrepare(p, p->dbOut, + p->bSlowIndexes ? + "SELECT rootpage, sql FROM recovery.schema " + " WHERE type!='table' AND type!='index'" + : + "SELECT rootpage, sql FROM recovery.schema " + " WHERE type!='table' AND (type!='index' OR sql NOT LIKE '%unique%')" + ); + + if( pSelect ){ + while( sqlite3_step(pSelect)==SQLITE_ROW ){ + const char *zSql = (const char*)sqlite3_column_text(pSelect, 1); + int rc = sqlite3_exec(p->dbOut, zSql, 0, 0, 0); + if( rc==SQLITE_OK ){ + recoverSqlCallback(p, zSql); + }else if( rc!=SQLITE_ERROR ){ + recoverDbError(p, p->dbOut); + } + } + } + recoverFinalize(p, pSelect); + + return p->errCode; +} + +/* +** This function is a no-op if recover handle p already contains an error +** (if p->errCode!=SQLITE_OK). In this case it returns NULL. +** +** Otherwise, if the recover handle is configured to create an output +** database (was created by sqlite3_recover_init()), then this function +** prepares and returns an SQL statement to INSERT a new record into table +** pTab, assuming the first nField fields of a record extracted from disk +** are valid. +** +** For example, if table pTab is: +** +** CREATE TABLE name(a, b GENERATED ALWAYS AS (a+1) STORED, c, d, e); +** +** And nField is 4, then the SQL statement prepared and returned is: +** +** INSERT INTO (a, c, d) VALUES (?1, ?2, ?3); +** +** In this case even though 4 values were extracted from the input db, +** only 3 are written to the output, as the generated STORED column +** cannot be written. +** +** If the recover handle is in SQL callback mode, then the SQL statement +** prepared is such that evaluating it returns a single row containing +** a single text value - itself an SQL statement similar to the above, +** except with SQL literals in place of the variables. For example: +** +** SELECT 'INSERT INTO (a, c, d) VALUES (' +** || quote(?1) || ', ' +** || quote(?2) || ', ' +** || quote(?3) || ')'; +** +** In either case, it is the responsibility of the caller to eventually +** free the statement handle using sqlite3_finalize(). +*/ +static sqlite3_stmt *recoverInsertStmt( + sqlite3_recover *p, + RecoverTable *pTab, + int nField +){ + sqlite3_stmt *pRet = 0; + const char *zSep = ""; + const char *zSqlSep = ""; + char *zSql = 0; + char *zFinal = 0; + char *zBind = 0; + int ii; + int bSql = p->xSql ? 1 : 0; + + if( nField<=0 ) return 0; + + assert( nField<=pTab->nCol ); + + zSql = recoverMPrintf(p, "INSERT OR IGNORE INTO %Q(", pTab->zTab); + + if( pTab->iRowidBind ){ + assert( pTab->bIntkey ); + zSql = recoverMPrintf(p, "%z_rowid_", zSql); + if( bSql ){ + zBind = recoverMPrintf(p, "%zquote(?%d)", zBind, pTab->iRowidBind); + }else{ + zBind = recoverMPrintf(p, "%z?%d", zBind, pTab->iRowidBind); + } + zSqlSep = "||', '||"; + zSep = ", "; + } + + for(ii=0; iiaCol[ii].eHidden; + if( eHidden!=RECOVER_EHIDDEN_VIRTUAL + && eHidden!=RECOVER_EHIDDEN_STORED + ){ + assert( pTab->aCol[ii].iField>=0 && pTab->aCol[ii].iBind>=1 ); + zSql = recoverMPrintf(p, "%z%s%Q", zSql, zSep, pTab->aCol[ii].zCol); + + if( bSql ){ + zBind = recoverMPrintf(p, + "%z%sescape_crnl(quote(?%d))", zBind, zSqlSep, pTab->aCol[ii].iBind + ); + zSqlSep = "||', '||"; + }else{ + zBind = recoverMPrintf(p, "%z%s?%d", zBind, zSep, pTab->aCol[ii].iBind); + } + zSep = ", "; + } + } + + if( bSql ){ + zFinal = recoverMPrintf(p, "SELECT %Q || ') VALUES (' || %s || ')'", + zSql, zBind + ); + }else{ + zFinal = recoverMPrintf(p, "%s) VALUES (%s)", zSql, zBind); + } + + pRet = recoverPrepare(p, p->dbOut, zFinal); + sqlite3_free(zSql); + sqlite3_free(zBind); + sqlite3_free(zFinal); + + return pRet; +} + + +/* +** Search the list of RecoverTable objects at p->pTblList for one that +** has root page iRoot in the input database. If such an object is found, +** return a pointer to it. Otherwise, return NULL. +*/ +static RecoverTable *recoverFindTable(sqlite3_recover *p, u32 iRoot){ + RecoverTable *pRet = 0; + for(pRet=p->pTblList; pRet && pRet->iRoot!=iRoot; pRet=pRet->pNext); + return pRet; +} + +/* +** This function attempts to create a lost and found table within the +** output db. If successful, it returns a pointer to a buffer containing +** the name of the new table. It is the responsibility of the caller to +** eventually free this buffer using sqlite3_free(). +** +** If an error occurs, NULL is returned and an error code and error +** message left in the recover handle. +*/ +static char *recoverLostAndFoundCreate( + sqlite3_recover *p, /* Recover object */ + int nField /* Number of column fields in new table */ +){ + char *zTbl = 0; + sqlite3_stmt *pProbe = 0; + int ii = 0; + + pProbe = recoverPrepare(p, p->dbOut, + "SELECT 1 FROM sqlite_schema WHERE name=?" + ); + for(ii=-1; zTbl==0 && p->errCode==SQLITE_OK && ii<1000; ii++){ + int bFail = 0; + if( ii<0 ){ + zTbl = recoverMPrintf(p, "%s", p->zLostAndFound); + }else{ + zTbl = recoverMPrintf(p, "%s_%d", p->zLostAndFound, ii); + } + + if( p->errCode==SQLITE_OK ){ + sqlite3_bind_text(pProbe, 1, zTbl, -1, SQLITE_STATIC); + if( SQLITE_ROW==sqlite3_step(pProbe) ){ + bFail = 1; + } + recoverReset(p, pProbe); + } + + if( bFail ){ + sqlite3_clear_bindings(pProbe); + sqlite3_free(zTbl); + zTbl = 0; + } + } + recoverFinalize(p, pProbe); + + if( zTbl ){ + const char *zSep = 0; + char *zField = 0; + char *zSql = 0; + + zSep = "rootpgno INTEGER, pgno INTEGER, nfield INTEGER, id INTEGER, "; + for(ii=0; p->errCode==SQLITE_OK && iidbOut, zSql); + recoverSqlCallback(p, zSql); + sqlite3_free(zSql); + }else if( p->errCode==SQLITE_OK ){ + recoverError( + p, SQLITE_ERROR, "failed to create %s output table", p->zLostAndFound + ); + } + + return zTbl; +} + +/* +** Synthesize and prepare an INSERT statement to write to the lost_and_found +** table in the output database. The name of the table is zTab, and it has +** nField c* fields. +*/ +static sqlite3_stmt *recoverLostAndFoundInsert( + sqlite3_recover *p, + const char *zTab, + int nField +){ + int nTotal = nField + 4; + int ii; + char *zBind = 0; + sqlite3_stmt *pRet = 0; + + if( p->xSql==0 ){ + for(ii=0; iidbOut, "INSERT INTO %s VALUES(%s)", zTab, zBind + ); + }else{ + const char *zSep = ""; + for(ii=0; iidbOut, "SELECT 'INSERT INTO %s VALUES(' || %s || ')'", zTab, zBind + ); + } + + sqlite3_free(zBind); + return pRet; +} + +/* +** Input database page iPg contains data that will be written to the +** lost-and-found table of the output database. This function attempts +** to identify the root page of the tree that page iPg belonged to. +** If successful, it sets output variable (*piRoot) to the page number +** of the root page and returns SQLITE_OK. Otherwise, if an error occurs, +** an SQLite error code is returned and the final value of *piRoot +** undefined. +*/ +static int recoverLostAndFoundFindRoot( + sqlite3_recover *p, + i64 iPg, + i64 *piRoot +){ + RecoverStateLAF *pLaf = &p->laf; + + if( pLaf->pFindRoot==0 ){ + pLaf->pFindRoot = recoverPrepare(p, p->dbOut, + "WITH RECURSIVE p(pgno) AS (" + " SELECT ?" + " UNION" + " SELECT parent FROM recovery.map AS m, p WHERE m.pgno=p.pgno" + ") " + "SELECT p.pgno FROM p, recovery.map m WHERE m.pgno=p.pgno " + " AND m.parent IS NULL" + ); + } + if( p->errCode==SQLITE_OK ){ + sqlite3_bind_int64(pLaf->pFindRoot, 1, iPg); + if( sqlite3_step(pLaf->pFindRoot)==SQLITE_ROW ){ + *piRoot = sqlite3_column_int64(pLaf->pFindRoot, 0); + }else{ + *piRoot = iPg; + } + recoverReset(p, pLaf->pFindRoot); + } + return p->errCode; +} + +/* +** Recover data from page iPage of the input database and write it to +** the lost-and-found table in the output database. +*/ +static void recoverLostAndFoundOnePage(sqlite3_recover *p, i64 iPage){ + RecoverStateLAF *pLaf = &p->laf; + sqlite3_value **apVal = pLaf->apVal; + sqlite3_stmt *pPageData = pLaf->pPageData; + sqlite3_stmt *pInsert = pLaf->pInsert; + + int nVal = -1; + int iPrevCell = 0; + i64 iRoot = 0; + int bHaveRowid = 0; + i64 iRowid = 0; + int ii = 0; + + if( recoverLostAndFoundFindRoot(p, iPage, &iRoot) ) return; + sqlite3_bind_int64(pPageData, 1, iPage); + while( p->errCode==SQLITE_OK && SQLITE_ROW==sqlite3_step(pPageData) ){ + int iCell = sqlite3_column_int64(pPageData, 0); + int iField = sqlite3_column_int64(pPageData, 1); + + if( iPrevCell!=iCell && nVal>=0 ){ + /* Insert the new row */ + sqlite3_bind_int64(pInsert, 1, iRoot); /* rootpgno */ + sqlite3_bind_int64(pInsert, 2, iPage); /* pgno */ + sqlite3_bind_int(pInsert, 3, nVal); /* nfield */ + if( bHaveRowid ){ + sqlite3_bind_int64(pInsert, 4, iRowid); /* id */ + } + for(ii=0; iinMaxField ){ + sqlite3_value *pVal = sqlite3_column_value(pPageData, 2); + apVal[iField] = sqlite3_value_dup(pVal); + assert( iField==nVal || (nVal==-1 && iField==0) ); + nVal = iField+1; + if( apVal[iField]==0 ){ + recoverError(p, SQLITE_NOMEM, 0); + } + } + + iPrevCell = iCell; + } + recoverReset(p, pPageData); + + for(ii=0; iilaf; + if( p->errCode==SQLITE_OK ){ + if( pLaf->pInsert==0 ){ + return SQLITE_DONE; + }else{ + if( p->errCode==SQLITE_OK ){ + int res = sqlite3_step(pLaf->pAllPage); + if( res==SQLITE_ROW ){ + i64 iPage = sqlite3_column_int64(pLaf->pAllPage, 0); + if( recoverBitmapQuery(pLaf->pUsed, iPage)==0 ){ + recoverLostAndFoundOnePage(p, iPage); + } + }else{ + recoverReset(p, pLaf->pAllPage); + return SQLITE_DONE; + } + } + } + } + return SQLITE_OK; +} + +/* +** Initialize resources required in RECOVER_STATE_LOSTANDFOUND3 +** state - during which the lost-and-found table of the output database +** is populated with recovered data that can not be assigned to any +** recovered schema object. +*/ +static void recoverLostAndFound3Init(sqlite3_recover *p){ + RecoverStateLAF *pLaf = &p->laf; + + if( pLaf->nMaxField>0 ){ + char *zTab = 0; /* Name of lost_and_found table */ + + zTab = recoverLostAndFoundCreate(p, pLaf->nMaxField); + pLaf->pInsert = recoverLostAndFoundInsert(p, zTab, pLaf->nMaxField); + sqlite3_free(zTab); + + pLaf->pAllPage = recoverPreparePrintf(p, p->dbOut, + "WITH RECURSIVE seq(ii) AS (" + " SELECT 1 UNION ALL SELECT ii+1 FROM seq WHERE ii<%lld" + ")" + "SELECT ii FROM seq" , p->laf.nPg + ); + pLaf->pPageData = recoverPrepare(p, p->dbOut, + "SELECT cell, field, value " + "FROM sqlite_dbdata('getpage()') d WHERE d.pgno=? " + "UNION ALL " + "SELECT -1, -1, -1" + ); + + pLaf->apVal = (sqlite3_value**)recoverMalloc(p, + pLaf->nMaxField*sizeof(sqlite3_value*) + ); + } +} + +/* +** Initialize resources required in RECOVER_STATE_WRITING state - during which +** tables recovered from the schema of the input database are populated with +** recovered data. +*/ +static int recoverWriteDataInit(sqlite3_recover *p){ + RecoverStateW1 *p1 = &p->w1; + RecoverTable *pTbl = 0; + int nByte = 0; + + /* Figure out the maximum number of columns for any table in the schema */ + assert( p1->nMax==0 ); + for(pTbl=p->pTblList; pTbl; pTbl=pTbl->pNext){ + if( pTbl->nCol>p1->nMax ) p1->nMax = pTbl->nCol; + } + + /* Allocate an array of (sqlite3_value*) in which to accumulate the values + ** that will be written to the output database in a single row. */ + nByte = sizeof(sqlite3_value*) * (p1->nMax+1); + p1->apVal = (sqlite3_value**)recoverMalloc(p, nByte); + if( p1->apVal==0 ) return p->errCode; + + /* Prepare the SELECT to loop through schema tables (pTbls) and the SELECT + ** to loop through cells that appear to belong to a single table (pSel). */ + p1->pTbls = recoverPrepare(p, p->dbOut, + "SELECT rootpage FROM recovery.schema " + " WHERE type='table' AND (sql NOT LIKE 'create virtual%')" + " ORDER BY (tbl_name='sqlite_sequence') ASC" + ); + p1->pSel = recoverPrepare(p, p->dbOut, + "WITH RECURSIVE pages(page) AS (" + " SELECT ?1" + " UNION" + " SELECT child FROM sqlite_dbptr('getpage()'), pages " + " WHERE pgno=page" + ") " + "SELECT page, cell, field, value " + "FROM sqlite_dbdata('getpage()') d, pages p WHERE p.page=d.pgno " + "UNION ALL " + "SELECT 0, 0, 0, 0" + ); + + return p->errCode; +} + +/* +** Clean up resources allocated by recoverWriteDataInit() (stuff in +** sqlite3_recover.w1). +*/ +static void recoverWriteDataCleanup(sqlite3_recover *p){ + RecoverStateW1 *p1 = &p->w1; + int ii; + for(ii=0; iinVal; ii++){ + sqlite3_value_free(p1->apVal[ii]); + } + sqlite3_free(p1->apVal); + recoverFinalize(p, p1->pInsert); + recoverFinalize(p, p1->pTbls); + recoverFinalize(p, p1->pSel); + memset(p1, 0, sizeof(*p1)); +} + +/* +** Perform one step (sqlite3_recover_step()) of work for the connection +** passed as the only argument, which is guaranteed to be in +** RECOVER_STATE_WRITING state - during which tables recovered from the +** schema of the input database are populated with recovered data. +*/ +static int recoverWriteDataStep(sqlite3_recover *p){ + RecoverStateW1 *p1 = &p->w1; + sqlite3_stmt *pSel = p1->pSel; + sqlite3_value **apVal = p1->apVal; + + if( p->errCode==SQLITE_OK && p1->pTab==0 ){ + if( sqlite3_step(p1->pTbls)==SQLITE_ROW ){ + i64 iRoot = sqlite3_column_int64(p1->pTbls, 0); + p1->pTab = recoverFindTable(p, iRoot); + + recoverFinalize(p, p1->pInsert); + p1->pInsert = 0; + + /* If this table is unknown, return early. The caller will invoke this + ** function again and it will move on to the next table. */ + if( p1->pTab==0 ) return p->errCode; + + /* If this is the sqlite_sequence table, delete any rows added by + ** earlier INSERT statements on tables with AUTOINCREMENT primary + ** keys before recovering its contents. The p1->pTbls SELECT statement + ** is rigged to deliver "sqlite_sequence" last of all, so we don't + ** worry about it being modified after it is recovered. */ + if( sqlite3_stricmp("sqlite_sequence", p1->pTab->zTab)==0 ){ + recoverExec(p, p->dbOut, "DELETE FROM sqlite_sequence"); + recoverSqlCallback(p, "DELETE FROM sqlite_sequence"); + } + + /* Bind the root page of this table within the original database to + ** SELECT statement p1->pSel. The SELECT statement will then iterate + ** through cells that look like they belong to table pTab. */ + sqlite3_bind_int64(pSel, 1, iRoot); + + p1->nVal = 0; + p1->bHaveRowid = 0; + p1->iPrevPage = -1; + p1->iPrevCell = -1; + }else{ + return SQLITE_DONE; + } + } + assert( p->errCode!=SQLITE_OK || p1->pTab ); + + if( p->errCode==SQLITE_OK && sqlite3_step(pSel)==SQLITE_ROW ){ + RecoverTable *pTab = p1->pTab; + + i64 iPage = sqlite3_column_int64(pSel, 0); + int iCell = sqlite3_column_int(pSel, 1); + int iField = sqlite3_column_int(pSel, 2); + sqlite3_value *pVal = sqlite3_column_value(pSel, 3); + int bNewCell = (p1->iPrevPage!=iPage || p1->iPrevCell!=iCell); + + assert( bNewCell==0 || (iField==-1 || iField==0) ); + assert( bNewCell || iField==p1->nVal || p1->nVal==pTab->nCol ); + + if( bNewCell ){ + int ii = 0; + if( p1->nVal>=0 ){ + if( p1->pInsert==0 || p1->nVal!=p1->nInsert ){ + recoverFinalize(p, p1->pInsert); + p1->pInsert = recoverInsertStmt(p, pTab, p1->nVal); + p1->nInsert = p1->nVal; + } + if( p1->nVal>0 ){ + sqlite3_stmt *pInsert = p1->pInsert; + for(ii=0; iinCol; ii++){ + RecoverColumn *pCol = &pTab->aCol[ii]; + int iBind = pCol->iBind; + if( iBind>0 ){ + if( pCol->bIPK ){ + sqlite3_bind_int64(pInsert, iBind, p1->iRowid); + }else if( pCol->iFieldnVal ){ + recoverBindValue(p, pInsert, iBind, apVal[pCol->iField]); + } + } + } + if( p->bRecoverRowid && pTab->iRowidBind>0 && p1->bHaveRowid ){ + sqlite3_bind_int64(pInsert, pTab->iRowidBind, p1->iRowid); + } + if( SQLITE_ROW==sqlite3_step(pInsert) ){ + const char *z = (const char*)sqlite3_column_text(pInsert, 0); + recoverSqlCallback(p, z); + } + recoverReset(p, pInsert); + assert( p->errCode || pInsert ); + if( pInsert ) sqlite3_clear_bindings(pInsert); + } + } + + for(ii=0; iinVal; ii++){ + sqlite3_value_free(apVal[ii]); + apVal[ii] = 0; + } + p1->nVal = -1; + p1->bHaveRowid = 0; + } + + if( iPage!=0 ){ + if( iField<0 ){ + p1->iRowid = sqlite3_column_int64(pSel, 3); + assert( p1->nVal==-1 ); + p1->nVal = 0; + p1->bHaveRowid = 1; + }else if( iFieldnCol ){ + assert( apVal[iField]==0 ); + apVal[iField] = sqlite3_value_dup( pVal ); + if( apVal[iField]==0 ){ + recoverError(p, SQLITE_NOMEM, 0); + } + p1->nVal = iField+1; + } + p1->iPrevCell = iCell; + p1->iPrevPage = iPage; + } + }else{ + recoverReset(p, pSel); + p1->pTab = 0; + } + + return p->errCode; +} + +/* +** Initialize resources required by sqlite3_recover_step() in +** RECOVER_STATE_LOSTANDFOUND1 state - during which the set of pages not +** already allocated to a recovered schema element is determined. +*/ +static void recoverLostAndFound1Init(sqlite3_recover *p){ + RecoverStateLAF *pLaf = &p->laf; + sqlite3_stmt *pStmt = 0; + + assert( p->laf.pUsed==0 ); + pLaf->nPg = recoverPageCount(p); + pLaf->pUsed = recoverBitmapAlloc(p, pLaf->nPg); + + /* Prepare a statement to iterate through all pages that are part of any tree + ** in the recoverable part of the input database schema to the bitmap. And, + ** if !p->bFreelistCorrupt, add all pages that appear to be part of the + ** freelist. */ + pStmt = recoverPrepare( + p, p->dbOut, + "WITH trunk(pgno) AS (" + " SELECT read_i32(getpage(1), 8) AS x WHERE x>0" + " UNION" + " SELECT read_i32(getpage(trunk.pgno), 0) AS x FROM trunk WHERE x>0" + ")," + "trunkdata(pgno, data) AS (" + " SELECT pgno, getpage(pgno) FROM trunk" + ")," + "freelist(data, n, freepgno) AS (" + " SELECT data, min(16384, read_i32(data, 1)-1), pgno FROM trunkdata" + " UNION ALL" + " SELECT data, n-1, read_i32(data, 2+n) FROM freelist WHERE n>=0" + ")," + "" + "roots(r) AS (" + " SELECT 1 UNION ALL" + " SELECT rootpage FROM recovery.schema WHERE rootpage>0" + ")," + "used(page) AS (" + " SELECT r FROM roots" + " UNION" + " SELECT child FROM sqlite_dbptr('getpage()'), used " + " WHERE pgno=page" + ") " + "SELECT page FROM used" + " UNION ALL " + "SELECT freepgno FROM freelist WHERE NOT ?" + ); + if( pStmt ) sqlite3_bind_int(pStmt, 1, p->bFreelistCorrupt); + pLaf->pUsedPages = pStmt; +} + +/* +** Perform one step (sqlite3_recover_step()) of work for the connection +** passed as the only argument, which is guaranteed to be in +** RECOVER_STATE_LOSTANDFOUND1 state - during which the set of pages not +** already allocated to a recovered schema element is determined. +*/ +static int recoverLostAndFound1Step(sqlite3_recover *p){ + RecoverStateLAF *pLaf = &p->laf; + int rc = p->errCode; + if( rc==SQLITE_OK ){ + rc = sqlite3_step(pLaf->pUsedPages); + if( rc==SQLITE_ROW ){ + i64 iPg = sqlite3_column_int64(pLaf->pUsedPages, 0); + recoverBitmapSet(pLaf->pUsed, iPg); + rc = SQLITE_OK; + }else{ + recoverFinalize(p, pLaf->pUsedPages); + pLaf->pUsedPages = 0; + } + } + return rc; +} + +/* +** Initialize resources required by RECOVER_STATE_LOSTANDFOUND2 +** state - during which the pages identified in RECOVER_STATE_LOSTANDFOUND1 +** are sorted into sets that likely belonged to the same database tree. +*/ +static void recoverLostAndFound2Init(sqlite3_recover *p){ + RecoverStateLAF *pLaf = &p->laf; + + assert( p->laf.pAllAndParent==0 ); + assert( p->laf.pMapInsert==0 ); + assert( p->laf.pMaxField==0 ); + assert( p->laf.nMaxField==0 ); + + pLaf->pMapInsert = recoverPrepare(p, p->dbOut, + "INSERT OR IGNORE INTO recovery.map(pgno, parent) VALUES(?, ?)" + ); + pLaf->pAllAndParent = recoverPreparePrintf(p, p->dbOut, + "WITH RECURSIVE seq(ii) AS (" + " SELECT 1 UNION ALL SELECT ii+1 FROM seq WHERE ii<%lld" + ")" + "SELECT pgno, child FROM sqlite_dbptr('getpage()') " + " UNION ALL " + "SELECT NULL, ii FROM seq", p->laf.nPg + ); + pLaf->pMaxField = recoverPreparePrintf(p, p->dbOut, + "SELECT max(field)+1 FROM sqlite_dbdata('getpage') WHERE pgno = ?" + ); +} + +/* +** Perform one step (sqlite3_recover_step()) of work for the connection +** passed as the only argument, which is guaranteed to be in +** RECOVER_STATE_LOSTANDFOUND2 state - during which the pages identified +** in RECOVER_STATE_LOSTANDFOUND1 are sorted into sets that likely belonged +** to the same database tree. +*/ +static int recoverLostAndFound2Step(sqlite3_recover *p){ + RecoverStateLAF *pLaf = &p->laf; + if( p->errCode==SQLITE_OK ){ + int res = sqlite3_step(pLaf->pAllAndParent); + if( res==SQLITE_ROW ){ + i64 iChild = sqlite3_column_int(pLaf->pAllAndParent, 1); + if( recoverBitmapQuery(pLaf->pUsed, iChild)==0 ){ + sqlite3_bind_int64(pLaf->pMapInsert, 1, iChild); + sqlite3_bind_value(pLaf->pMapInsert, 2, + sqlite3_column_value(pLaf->pAllAndParent, 0) + ); + sqlite3_step(pLaf->pMapInsert); + recoverReset(p, pLaf->pMapInsert); + sqlite3_bind_int64(pLaf->pMaxField, 1, iChild); + if( SQLITE_ROW==sqlite3_step(pLaf->pMaxField) ){ + int nMax = sqlite3_column_int(pLaf->pMaxField, 0); + if( nMax>pLaf->nMaxField ) pLaf->nMaxField = nMax; + } + recoverReset(p, pLaf->pMaxField); + } + }else{ + recoverFinalize(p, pLaf->pAllAndParent); + pLaf->pAllAndParent =0; + return SQLITE_DONE; + } + } + return p->errCode; +} + +/* +** Free all resources allocated as part of sqlite3_recover_step() calls +** in one of the RECOVER_STATE_LOSTANDFOUND[123] states. +*/ +static void recoverLostAndFoundCleanup(sqlite3_recover *p){ + recoverBitmapFree(p->laf.pUsed); + p->laf.pUsed = 0; + sqlite3_finalize(p->laf.pUsedPages); + sqlite3_finalize(p->laf.pAllAndParent); + sqlite3_finalize(p->laf.pMapInsert); + sqlite3_finalize(p->laf.pMaxField); + sqlite3_finalize(p->laf.pFindRoot); + sqlite3_finalize(p->laf.pInsert); + sqlite3_finalize(p->laf.pAllPage); + sqlite3_finalize(p->laf.pPageData); + p->laf.pUsedPages = 0; + p->laf.pAllAndParent = 0; + p->laf.pMapInsert = 0; + p->laf.pMaxField = 0; + p->laf.pFindRoot = 0; + p->laf.pInsert = 0; + p->laf.pAllPage = 0; + p->laf.pPageData = 0; + sqlite3_free(p->laf.apVal); + p->laf.apVal = 0; +} + +/* +** Free all resources allocated as part of sqlite3_recover_step() calls. +*/ +static void recoverFinalCleanup(sqlite3_recover *p){ + RecoverTable *pTab = 0; + RecoverTable *pNext = 0; + + recoverWriteDataCleanup(p); + recoverLostAndFoundCleanup(p); + + for(pTab=p->pTblList; pTab; pTab=pNext){ + pNext = pTab->pNext; + sqlite3_free(pTab); + } + p->pTblList = 0; + sqlite3_finalize(p->pGetPage); + p->pGetPage = 0; + + { +#ifndef NDEBUG + int res = +#endif + sqlite3_close(p->dbOut); + assert( res==SQLITE_OK ); + } + p->dbOut = 0; +} + +/* +** Decode and return an unsigned 16-bit big-endian integer value from +** buffer a[]. +*/ +static u32 recoverGetU16(const u8 *a){ + return (((u32)a[0])<<8) + ((u32)a[1]); +} + +/* +** Decode and return an unsigned 32-bit big-endian integer value from +** buffer a[]. +*/ +static u32 recoverGetU32(const u8 *a){ + return (((u32)a[0])<<24) + (((u32)a[1])<<16) + (((u32)a[2])<<8) + ((u32)a[3]); +} + +/* +** Decode an SQLite varint from buffer a[]. Write the decoded value to (*pVal) +** and return the number of bytes consumed. +*/ +static int recoverGetVarint(const u8 *a, i64 *pVal){ + sqlite3_uint64 u = 0; + int i; + for(i=0; i<8; i++){ + u = (u<<7) + (a[i]&0x7f); + if( (a[i]&0x80)==0 ){ *pVal = (sqlite3_int64)u; return i+1; } + } + u = (u<<8) + (a[i]&0xff); + *pVal = (sqlite3_int64)u; + return 9; +} + +/* +** The second argument points to a buffer n bytes in size. If this buffer +** or a prefix thereof appears to contain a well-formed SQLite b-tree page, +** return the page-size in bytes. Otherwise, if the buffer does not +** appear to contain a well-formed b-tree page, return 0. +*/ +static int recoverIsValidPage(u8 *aTmp, const u8 *a, int n){ + u8 *aUsed = aTmp; + int nFrag = 0; + int nActual = 0; + int iFree = 0; + int nCell = 0; /* Number of cells on page */ + int iCellOff = 0; /* Offset of cell array in page */ + int iContent = 0; + int eType = 0; + int ii = 0; + + eType = (int)a[0]; + if( eType!=0x02 && eType!=0x05 && eType!=0x0A && eType!=0x0D ) return 0; + + iFree = (int)recoverGetU16(&a[1]); + nCell = (int)recoverGetU16(&a[3]); + iContent = (int)recoverGetU16(&a[5]); + if( iContent==0 ) iContent = 65536; + nFrag = (int)a[7]; + + if( iContent>n ) return 0; + + memset(aUsed, 0, n); + memset(aUsed, 0xFF, iContent); + + /* Follow the free-list. This is the same format for all b-tree pages. */ + if( iFree && iFree<=iContent ) return 0; + while( iFree ){ + int iNext = 0; + int nByte = 0; + if( iFree>(n-4) ) return 0; + iNext = recoverGetU16(&a[iFree]); + nByte = recoverGetU16(&a[iFree+2]); + if( iFree+nByte>n ) return 0; + if( iNext && iNextiContent ) return 0; + for(ii=0; iin ){ + return 0; + } + if( eType==0x05 || eType==0x02 ) nByte += 4; + nByte += recoverGetVarint(&a[iOff+nByte], &nPayload); + if( eType==0x0D ){ + i64 dummy = 0; + nByte += recoverGetVarint(&a[iOff+nByte], &dummy); + } + if( eType!=0x05 ){ + int X = (eType==0x0D) ? n-35 : (((n-12)*64/255)-23); + int M = ((n-12)*32/255)-23; + int K = M+((nPayload-M)%(n-4)); + + if( nPayloadn ){ + return 0; + } + for(iByte=iOff; iByte<(iOff+nByte); iByte++){ + if( aUsed[iByte]!=0 ){ + return 0; + } + aUsed[iByte] = 0xFF; + } + } + + nActual = 0; + for(ii=0; iipMethods!=&recover_methods ); + return pFd->pMethods->xClose(pFd); +} + +/* +** Write value v to buffer a[] as a 16-bit big-endian unsigned integer. +*/ +static void recoverPutU16(u8 *a, u32 v){ + a[0] = (v>>8) & 0x00FF; + a[1] = (v>>0) & 0x00FF; +} + +/* +** Write value v to buffer a[] as a 32-bit big-endian unsigned integer. +*/ +static void recoverPutU32(u8 *a, u32 v){ + a[0] = (v>>24) & 0x00FF; + a[1] = (v>>16) & 0x00FF; + a[2] = (v>>8) & 0x00FF; + a[3] = (v>>0) & 0x00FF; +} + +/* +** Detect the page-size of the database opened by file-handle pFd by +** searching the first part of the file for a well-formed SQLite b-tree +** page. If parameter nReserve is non-zero, then as well as searching for +** a b-tree page with zero reserved bytes, this function searches for one +** with nReserve reserved bytes at the end of it. +** +** If successful, set variable p->detected_pgsz to the detected page-size +** in bytes and return SQLITE_OK. Or, if no error occurs but no valid page +** can be found, return SQLITE_OK but leave p->detected_pgsz set to 0. Or, +** if an error occurs (e.g. an IO or OOM error), then an SQLite error code +** is returned. The final value of p->detected_pgsz is undefined in this +** case. +*/ +static int recoverVfsDetectPagesize( + sqlite3_recover *p, /* Recover handle */ + sqlite3_file *pFd, /* File-handle open on input database */ + u32 nReserve, /* Possible nReserve value */ + i64 nSz /* Size of database file in bytes */ +){ + int rc = SQLITE_OK; + const int nMin = 512; + const int nMax = 65536; + const int nMaxBlk = 4; + u32 pgsz = 0; + int iBlk = 0; + u8 *aPg = 0; + u8 *aTmp = 0; + int nBlk = 0; + + aPg = (u8*)sqlite3_malloc(2*nMax); + if( aPg==0 ) return SQLITE_NOMEM; + aTmp = &aPg[nMax]; + + nBlk = (nSz+nMax-1)/nMax; + if( nBlk>nMaxBlk ) nBlk = nMaxBlk; + + do { + for(iBlk=0; rc==SQLITE_OK && iBlk=((iBlk+1)*nMax)) ? nMax : (nSz % nMax); + memset(aPg, 0, nMax); + rc = pFd->pMethods->xRead(pFd, aPg, nByte, iBlk*nMax); + if( rc==SQLITE_OK ){ + int pgsz2; + for(pgsz2=(pgsz ? pgsz*2 : nMin); pgsz2<=nMax; pgsz2=pgsz2*2){ + int iOff; + for(iOff=0; iOff(u32)p->detected_pgsz ){ + p->detected_pgsz = pgsz; + p->nReserve = nReserve; + } + if( nReserve==0 ) break; + nReserve = 0; + }while( 1 ); + + p->detected_pgsz = pgsz; + sqlite3_free(aPg); + return rc; +} + +/* +** The xRead() method of the wrapper VFS. This is used to intercept calls +** to read page 1 of the input database. +*/ +static int recoverVfsRead(sqlite3_file *pFd, void *aBuf, int nByte, i64 iOff){ + int rc = SQLITE_OK; + if( pFd->pMethods==&recover_methods ){ + pFd->pMethods = recover_g.pMethods; + rc = pFd->pMethods->xRead(pFd, aBuf, nByte, iOff); + if( nByte==16 ){ + sqlite3_randomness(16, aBuf); + }else + if( rc==SQLITE_OK && iOff==0 && nByte>=108 ){ + /* Ensure that the database has a valid header file. The only fields + ** that really matter to recovery are: + ** + ** + Database page size (16-bits at offset 16) + ** + Size of db in pages (32-bits at offset 28) + ** + Database encoding (32-bits at offset 56) + ** + ** Also preserved are: + ** + ** + first freelist page (32-bits at offset 32) + ** + size of freelist (32-bits at offset 36) + ** + ** We also try to preserve the auto-vacuum, incr-value, user-version + ** and application-id fields - all 32 bit quantities at offsets + ** 52, 60, 64 and 68. All other fields are set to known good values. + ** + ** Byte offset 105 should also contain the page-size as a 16-bit + ** integer. + */ + const int aPreserve[] = {32, 36, 52, 60, 64, 68}; + u8 aHdr[108] = { + 0x53, 0x51, 0x4c, 0x69, 0x74, 0x65, 0x20, 0x66, + 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x20, 0x33, 0x00, + 0xFF, 0xFF, 0x01, 0x01, 0x00, 0x40, 0x20, 0x20, + 0x00, 0x00, 0x00, 0x00, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, + 0x00, 0x00, 0x10, 0x00, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x2e, 0x5b, 0x30, + + 0x0D, 0x00, 0x00, 0x00, 0x00, 0xFF, 0xFF, 0x00 + }; + u8 *a = (u8*)aBuf; + + u32 pgsz = recoverGetU16(&a[16]); + u32 nReserve = a[20]; + u32 enc = recoverGetU32(&a[56]); + u32 dbsz = 0; + i64 dbFileSize = 0; + int ii; + sqlite3_recover *p = recover_g.p; + + if( pgsz==0x01 ) pgsz = 65536; + rc = pFd->pMethods->xFileSize(pFd, &dbFileSize); + + if( rc==SQLITE_OK && p->detected_pgsz==0 ){ + rc = recoverVfsDetectPagesize(p, pFd, nReserve, dbFileSize); + } + if( p->detected_pgsz ){ + pgsz = p->detected_pgsz; + nReserve = p->nReserve; + } + + if( pgsz ){ + dbsz = dbFileSize / pgsz; + } + if( enc!=SQLITE_UTF8 && enc!=SQLITE_UTF16BE && enc!=SQLITE_UTF16LE ){ + enc = SQLITE_UTF8; + } + + sqlite3_free(p->pPage1Cache); + p->pPage1Cache = 0; + p->pPage1Disk = 0; + + p->pgsz = nByte; + p->pPage1Cache = (u8*)recoverMalloc(p, nByte*2); + if( p->pPage1Cache ){ + p->pPage1Disk = &p->pPage1Cache[nByte]; + memcpy(p->pPage1Disk, aBuf, nByte); + + recoverPutU32(&aHdr[28], dbsz); + recoverPutU32(&aHdr[56], enc); + recoverPutU16(&aHdr[105], pgsz-nReserve); + if( pgsz==65536 ) pgsz = 1; + recoverPutU16(&aHdr[16], pgsz); + aHdr[20] = nReserve; + for(ii=0; iipPage1Cache, aBuf, nByte); + }else{ + rc = p->errCode; + } + + } + pFd->pMethods = &recover_methods; + }else{ + rc = pFd->pMethods->xRead(pFd, aBuf, nByte, iOff); + } + return rc; +} + +/* +** Used to make sqlite3_io_methods wrapper methods less verbose. +*/ +#define RECOVER_VFS_WRAPPER(code) \ + int rc = SQLITE_OK; \ + if( pFd->pMethods==&recover_methods ){ \ + pFd->pMethods = recover_g.pMethods; \ + rc = code; \ + pFd->pMethods = &recover_methods; \ + }else{ \ + rc = code; \ + } \ + return rc; + +/* +** Methods of the wrapper VFS. All methods except for xRead() and xClose() +** simply uninstall the sqlite3_io_methods wrapper, invoke the equivalent +** method on the lower level VFS, then reinstall the wrapper before returning. +** Those that return an integer value use the RECOVER_VFS_WRAPPER macro. +*/ +static int recoverVfsWrite( + sqlite3_file *pFd, const void *aBuf, int nByte, i64 iOff +){ + RECOVER_VFS_WRAPPER ( + pFd->pMethods->xWrite(pFd, aBuf, nByte, iOff) + ); +} +static int recoverVfsTruncate(sqlite3_file *pFd, sqlite3_int64 size){ + RECOVER_VFS_WRAPPER ( + pFd->pMethods->xTruncate(pFd, size) + ); +} +static int recoverVfsSync(sqlite3_file *pFd, int flags){ + RECOVER_VFS_WRAPPER ( + pFd->pMethods->xSync(pFd, flags) + ); +} +static int recoverVfsFileSize(sqlite3_file *pFd, sqlite3_int64 *pSize){ + RECOVER_VFS_WRAPPER ( + pFd->pMethods->xFileSize(pFd, pSize) + ); +} +static int recoverVfsLock(sqlite3_file *pFd, int eLock){ + RECOVER_VFS_WRAPPER ( + pFd->pMethods->xLock(pFd, eLock) + ); +} +static int recoverVfsUnlock(sqlite3_file *pFd, int eLock){ + RECOVER_VFS_WRAPPER ( + pFd->pMethods->xUnlock(pFd, eLock) + ); +} +static int recoverVfsCheckReservedLock(sqlite3_file *pFd, int *pResOut){ + RECOVER_VFS_WRAPPER ( + pFd->pMethods->xCheckReservedLock(pFd, pResOut) + ); +} +static int recoverVfsFileControl(sqlite3_file *pFd, int op, void *pArg){ + RECOVER_VFS_WRAPPER ( + (pFd->pMethods ? pFd->pMethods->xFileControl(pFd, op, pArg) : SQLITE_NOTFOUND) + ); +} +static int recoverVfsSectorSize(sqlite3_file *pFd){ + RECOVER_VFS_WRAPPER ( + pFd->pMethods->xSectorSize(pFd) + ); +} +static int recoverVfsDeviceCharacteristics(sqlite3_file *pFd){ + RECOVER_VFS_WRAPPER ( + pFd->pMethods->xDeviceCharacteristics(pFd) + ); +} +static int recoverVfsShmMap( + sqlite3_file *pFd, int iPg, int pgsz, int bExtend, void volatile **pp +){ + RECOVER_VFS_WRAPPER ( + pFd->pMethods->xShmMap(pFd, iPg, pgsz, bExtend, pp) + ); +} +static int recoverVfsShmLock(sqlite3_file *pFd, int offset, int n, int flags){ + RECOVER_VFS_WRAPPER ( + pFd->pMethods->xShmLock(pFd, offset, n, flags) + ); +} +static void recoverVfsShmBarrier(sqlite3_file *pFd){ + if( pFd->pMethods==&recover_methods ){ + pFd->pMethods = recover_g.pMethods; + pFd->pMethods->xShmBarrier(pFd); + pFd->pMethods = &recover_methods; + }else{ + pFd->pMethods->xShmBarrier(pFd); + } +} +static int recoverVfsShmUnmap(sqlite3_file *pFd, int deleteFlag){ + RECOVER_VFS_WRAPPER ( + pFd->pMethods->xShmUnmap(pFd, deleteFlag) + ); +} + +static int recoverVfsFetch( + sqlite3_file *pFd, + sqlite3_int64 iOff, + int iAmt, + void **pp +){ + *pp = 0; + return SQLITE_OK; +} +static int recoverVfsUnfetch(sqlite3_file *pFd, sqlite3_int64 iOff, void *p){ + return SQLITE_OK; +} + +/* +** Install the VFS wrapper around the file-descriptor open on the input +** database for recover handle p. Mutex RECOVER_MUTEX_ID must be held +** when this function is called. +*/ +static void recoverInstallWrapper(sqlite3_recover *p){ + sqlite3_file *pFd = 0; + assert( recover_g.pMethods==0 ); + recoverAssertMutexHeld(); + sqlite3_file_control(p->dbIn, p->zDb, SQLITE_FCNTL_FILE_POINTER, (void*)&pFd); + assert( pFd==0 || pFd->pMethods!=&recover_methods ); + if( pFd && pFd->pMethods ){ + int iVersion = 1 + (pFd->pMethods->iVersion>1 && pFd->pMethods->xShmMap!=0); + recover_g.pMethods = pFd->pMethods; + recover_g.p = p; + recover_methods.iVersion = iVersion; + pFd->pMethods = &recover_methods; + } +} + +/* +** Uninstall the VFS wrapper that was installed around the file-descriptor open +** on the input database for recover handle p. Mutex RECOVER_MUTEX_ID must be +** held when this function is called. +*/ +static void recoverUninstallWrapper(sqlite3_recover *p){ + sqlite3_file *pFd = 0; + recoverAssertMutexHeld(); + sqlite3_file_control(p->dbIn, p->zDb,SQLITE_FCNTL_FILE_POINTER,(void*)&pFd); + if( pFd && pFd->pMethods ){ + pFd->pMethods = recover_g.pMethods; + recover_g.pMethods = 0; + recover_g.p = 0; + } +} + +/* +** This function does the work of a single sqlite3_recover_step() call. It +** is guaranteed that the handle is not in an error state when this +** function is called. +*/ +static void recoverStep(sqlite3_recover *p){ + assert( p && p->errCode==SQLITE_OK ); + switch( p->eState ){ + case RECOVER_STATE_INIT: + /* This is the very first call to sqlite3_recover_step() on this object. + */ + recoverSqlCallback(p, "BEGIN"); + recoverSqlCallback(p, "PRAGMA writable_schema = on"); + + recoverEnterMutex(); + recoverInstallWrapper(p); + + /* Open the output database. And register required virtual tables and + ** user functions with the new handle. */ + recoverOpenOutput(p); + + /* Open transactions on both the input and output databases. */ + recoverExec(p, p->dbIn, "PRAGMA writable_schema = on"); + recoverExec(p, p->dbIn, "BEGIN"); + if( p->errCode==SQLITE_OK ) p->bCloseTransaction = 1; + recoverExec(p, p->dbIn, "SELECT 1 FROM sqlite_schema"); + recoverTransferSettings(p); + recoverOpenRecovery(p); + recoverCacheSchema(p); + + recoverUninstallWrapper(p); + recoverLeaveMutex(); + + recoverExec(p, p->dbOut, "BEGIN"); + + recoverWriteSchema1(p); + p->eState = RECOVER_STATE_WRITING; + break; + + case RECOVER_STATE_WRITING: { + if( p->w1.pTbls==0 ){ + recoverWriteDataInit(p); + } + if( SQLITE_DONE==recoverWriteDataStep(p) ){ + recoverWriteDataCleanup(p); + if( p->zLostAndFound ){ + p->eState = RECOVER_STATE_LOSTANDFOUND1; + }else{ + p->eState = RECOVER_STATE_SCHEMA2; + } + } + break; + } + + case RECOVER_STATE_LOSTANDFOUND1: { + if( p->laf.pUsed==0 ){ + recoverLostAndFound1Init(p); + } + if( SQLITE_DONE==recoverLostAndFound1Step(p) ){ + p->eState = RECOVER_STATE_LOSTANDFOUND2; + } + break; + } + case RECOVER_STATE_LOSTANDFOUND2: { + if( p->laf.pAllAndParent==0 ){ + recoverLostAndFound2Init(p); + } + if( SQLITE_DONE==recoverLostAndFound2Step(p) ){ + p->eState = RECOVER_STATE_LOSTANDFOUND3; + } + break; + } + + case RECOVER_STATE_LOSTANDFOUND3: { + if( p->laf.pInsert==0 ){ + recoverLostAndFound3Init(p); + } + if( SQLITE_DONE==recoverLostAndFound3Step(p) ){ + p->eState = RECOVER_STATE_SCHEMA2; + } + break; + } + + case RECOVER_STATE_SCHEMA2: { + int rc = SQLITE_OK; + + recoverWriteSchema2(p); + p->eState = RECOVER_STATE_DONE; + + /* If no error has occurred, commit the write transaction on the output + ** database. Regardless of whether or not an error has occurred, make + ** an attempt to end the read transaction on the input database. */ + recoverExec(p, p->dbOut, "COMMIT"); + rc = sqlite3_exec(p->dbIn, "END", 0, 0, 0); + if( p->errCode==SQLITE_OK ) p->errCode = rc; + + recoverSqlCallback(p, "PRAGMA writable_schema = off"); + recoverSqlCallback(p, "COMMIT"); + p->eState = RECOVER_STATE_DONE; + recoverFinalCleanup(p); + break; + }; + + case RECOVER_STATE_DONE: { + /* no-op */ + break; + }; + } +} + + +/* +** This is a worker function that does the heavy lifting for both init +** functions: +** +** sqlite3_recover_init() +** sqlite3_recover_init_sql() +** +** All this function does is allocate space for the recover handle and +** take copies of the input parameters. All the real work is done within +** sqlite3_recover_run(). +*/ +sqlite3_recover *recoverInit( + sqlite3* db, + const char *zDb, + const char *zUri, /* Output URI for _recover_init() */ + int (*xSql)(void*, const char*),/* SQL callback for _recover_init_sql() */ + void *pSqlCtx /* Context arg for _recover_init_sql() */ +){ + sqlite3_recover *pRet = 0; + int nDb = 0; + int nUri = 0; + int nByte = 0; + + if( zDb==0 ){ zDb = "main"; } + + nDb = recoverStrlen(zDb); + nUri = recoverStrlen(zUri); + + nByte = sizeof(sqlite3_recover) + nDb+1 + nUri+1; + pRet = (sqlite3_recover*)sqlite3_malloc(nByte); + if( pRet ){ + memset(pRet, 0, nByte); + pRet->dbIn = db; + pRet->zDb = (char*)&pRet[1]; + pRet->zUri = &pRet->zDb[nDb+1]; + memcpy(pRet->zDb, zDb, nDb); + if( nUri>0 && zUri ) memcpy(pRet->zUri, zUri, nUri); + pRet->xSql = xSql; + pRet->pSqlCtx = pSqlCtx; + pRet->bRecoverRowid = RECOVER_ROWID_DEFAULT; + } + + return pRet; +} + +/* +** Initialize a recovery handle that creates a new database containing +** the recovered data. +*/ +sqlite3_recover *sqlite3_recover_init( + sqlite3* db, + const char *zDb, + const char *zUri +){ + return recoverInit(db, zDb, zUri, 0, 0); +} + +/* +** Initialize a recovery handle that returns recovered data in the +** form of SQL statements via a callback. +*/ +sqlite3_recover *sqlite3_recover_init_sql( + sqlite3* db, + const char *zDb, + int (*xSql)(void*, const char*), + void *pSqlCtx +){ + return recoverInit(db, zDb, 0, xSql, pSqlCtx); +} + +/* +** Return the handle error message, if any. +*/ +const char *sqlite3_recover_errmsg(sqlite3_recover *p){ + return (p && p->errCode!=SQLITE_NOMEM) ? p->zErrMsg : "out of memory"; +} + +/* +** Return the handle error code. +*/ +int sqlite3_recover_errcode(sqlite3_recover *p){ + return p ? p->errCode : SQLITE_NOMEM; +} + +/* +** Configure the handle. +*/ +int sqlite3_recover_config(sqlite3_recover *p, int op, void *pArg){ + int rc = SQLITE_OK; + if( p==0 ){ + rc = SQLITE_NOMEM; + }else if( p->eState!=RECOVER_STATE_INIT ){ + rc = SQLITE_MISUSE; + }else{ + switch( op ){ + case 789: + /* This undocumented magic configuration option is used to set the + ** name of the auxiliary database that is ATTACH-ed to the database + ** connection and used to hold state information during the + ** recovery process. This option is for debugging use only and + ** is subject to change or removal at any time. */ + sqlite3_free(p->zStateDb); + p->zStateDb = recoverMPrintf(p, "%s", (char*)pArg); + break; + + case SQLITE_RECOVER_LOST_AND_FOUND: { + const char *zArg = (const char*)pArg; + sqlite3_free(p->zLostAndFound); + if( zArg ){ + p->zLostAndFound = recoverMPrintf(p, "%s", zArg); + }else{ + p->zLostAndFound = 0; + } + break; + } + + case SQLITE_RECOVER_FREELIST_CORRUPT: + p->bFreelistCorrupt = *(int*)pArg; + break; + + case SQLITE_RECOVER_ROWIDS: + p->bRecoverRowid = *(int*)pArg; + break; + + case SQLITE_RECOVER_SLOWINDEXES: + p->bSlowIndexes = *(int*)pArg; + break; + + default: + rc = SQLITE_NOTFOUND; + break; + } + } + + return rc; +} + +/* +** Do a unit of work towards the recovery job. Return SQLITE_OK if +** no error has occurred but database recovery is not finished, SQLITE_DONE +** if database recovery has been successfully completed, or an SQLite +** error code if an error has occurred. +*/ +int sqlite3_recover_step(sqlite3_recover *p){ + if( p==0 ) return SQLITE_NOMEM; + if( p->errCode==SQLITE_OK ) recoverStep(p); + if( p->eState==RECOVER_STATE_DONE && p->errCode==SQLITE_OK ){ + return SQLITE_DONE; + } + return p->errCode; +} + +/* +** Do the configured recovery operation. Return SQLITE_OK if successful, or +** else an SQLite error code. +*/ +int sqlite3_recover_run(sqlite3_recover *p){ + while( SQLITE_OK==sqlite3_recover_step(p) ); + return sqlite3_recover_errcode(p); +} + + +/* +** Free all resources associated with the recover handle passed as the only +** argument. The results of using a handle with any sqlite3_recover_** +** API function after it has been passed to this function are undefined. +** +** A copy of the value returned by the first call made to sqlite3_recover_run() +** on this handle is returned, or SQLITE_OK if sqlite3_recover_run() has +** not been called on this handle. +*/ +int sqlite3_recover_finish(sqlite3_recover *p){ + int rc; + if( p==0 ){ + rc = SQLITE_NOMEM; + }else{ + recoverFinalCleanup(p); + if( p->bCloseTransaction && sqlite3_get_autocommit(p->dbIn)==0 ){ + rc = sqlite3_exec(p->dbIn, "END", 0, 0, 0); + if( p->errCode==SQLITE_OK ) p->errCode = rc; + } + rc = p->errCode; + sqlite3_free(p->zErrMsg); + sqlite3_free(p->zStateDb); + sqlite3_free(p->zLostAndFound); + sqlite3_free(p->pPage1Cache); + sqlite3_free(p); + } + return rc; +} + +#endif /* ifndef SQLITE_OMIT_VIRTUALTABLE */ + +/************************* End ../ext/recover/sqlite3recover.c ********************/ #endif #if defined(SQLITE_ENABLE_SESSION) @@ -12252,15 +15592,16 @@ struct ShellState { char *zNonce; /* Nonce for temporary safe-mode excapes */ EQPGraph sGraph; /* Information for the graphical EXPLAIN QUERY PLAN */ ExpertInfo expert; /* Valid if previous command was ".expert OPT..." */ -#ifdef SQLITE_SHELL_WASM_MODE +#ifdef SQLITE_SHELL_FIDDLE struct { const char * zInput; /* Input string from wasm/JS proxy */ const char * zPos; /* Cursor pos into zInput */ + const char * zDefaultDbName; /* Default name for db file */ } wasm; #endif }; -#ifdef SQLITE_SHELL_WASM_MODE +#ifdef SQLITE_SHELL_FIDDLE static ShellState shellState; #endif @@ -12592,10 +15933,23 @@ static void outputModePop(ShellState *p){ */ static void output_hex_blob(FILE *out, const void *pBlob, int nBlob){ int i; - char *zBlob = (char *)pBlob; - raw_printf(out,"X'"); - for(i=0; i> 4) ]; + zStr[i*2+1] = aHex[ (aBlob[i] & 0x0F) ]; + } + zStr[i*2] = '\0'; + + raw_printf(out,"X'%s'", zStr); + sqlite3_free(zStr); } /* @@ -12755,9 +16109,9 @@ static void output_c_string(FILE *out, const char *z){ /* ** Output the given string as a quoted according to JSON quoting rules. */ -static void output_json_string(FILE *out, const char *z, int n){ +static void output_json_string(FILE *out, const char *z, i64 n){ unsigned int c; - if( n<0 ) n = (int)strlen(z); + if( n<0 ) n = strlen(z); fputc('"', out); while( n-- ){ c = *(z++); @@ -12928,7 +16282,7 @@ static int safeModeAuth( UNUSED_PARAMETER(zA4); switch( op ){ case SQLITE_ATTACH: { -#ifndef SQLITE_SHELL_WASM_MODE +#ifndef SQLITE_SHELL_FIDDLE /* In WASM builds the filesystem is a virtual sandbox, so ** there's no harm in using ATTACH. */ failIfSafeMode(p, "cannot run ATTACH in safe mode"); @@ -13001,15 +16355,37 @@ static int shellAuth( ** ** This routine converts some CREATE TABLE statements for shadow tables ** in FTS3/4/5 into CREATE TABLE IF NOT EXISTS statements. +** +** If the schema statement in z[] contains a start-of-comment and if +** sqlite3_complete() returns false, try to terminate the comment before +** printing the result. https://sqlite.org/forum/forumpost/d7be961c5c */ static void printSchemaLine(FILE *out, const char *z, const char *zTail){ + char *zToFree = 0; if( z==0 ) return; if( zTail==0 ) return; + if( zTail[0]==';' && (strstr(z, "/*")!=0 || strstr(z,"--")!=0) ){ + const char *zOrig = z; + static const char *azTerm[] = { "", "*/", "\n" }; + int i; + for(i=0; iautoEQPtest ){ utf8_printf(p->out, "%d,%d,%s\n", iEqpId, p2, zText); } @@ -13083,14 +16461,14 @@ static EQPGraphRow *eqp_next_row(ShellState *p, int iEqpId, EQPGraphRow *pOld){ */ static void eqp_render_level(ShellState *p, int iEqpId){ EQPGraphRow *pRow, *pNext; - int n = strlen30(p->sGraph.zPrefix); + i64 n = strlen(p->sGraph.zPrefix); char *z; for(pRow = eqp_next_row(p, iEqpId, 0); pRow; pRow = pNext){ pNext = eqp_next_row(p, iEqpId, pRow); z = pRow->zText; utf8_printf(p->out, "%s%s%s\n", p->sGraph.zPrefix, pNext ? "|--" : "`--", z); - if( n<(int)sizeof(p->sGraph.zPrefix)-7 ){ + if( n<(i64)sizeof(p->sGraph.zPrefix)-7 ){ memcpy(&p->sGraph.zPrefix[n], pNext ? "| " : " ", 4); eqp_render_level(p, pRow->iEqpId); p->sGraph.zPrefix[n] = 0; @@ -13690,6 +17068,7 @@ static char *shell_error_context(const char *zSql, sqlite3 *db){ while( (zSql[len]&0xc0)==0x80 ) len--; } zCode = sqlite3_mprintf("%.*s", len, zSql); + shell_check_oom(zCode); for(i=0; zCode[i]; i++){ if( IsSpace(zSql[i]) ) zCode[i] = ' '; } if( iOffset<25 ){ zMsg = sqlite3_mprintf("\n %z\n %*s^--- error here", zCode, iOffset, ""); @@ -13806,7 +17185,7 @@ static void displayLinuxIoStats(FILE *out){ int i; for(i=0; icMode = p->mode; sqlite3_reset(pSql); return; @@ -14844,10 +18223,10 @@ static int expertDotCommand( int n; if( z[0]=='-' && z[1]=='-' ) z++; n = strlen30(z); - if( n>=2 && 0==strncmp(z, "-verbose", n) ){ + if( n>=2 && 0==cli_strncmp(z, "-verbose", n) ){ pState->expert.bVerbose = 1; } - else if( n>=2 && 0==strncmp(z, "-sample", n) ){ + else if( n>=2 && 0==cli_strncmp(z, "-sample", n) ){ if( i==(nArg-1) ){ raw_printf(stderr, "option requires an argument: %s\n", z); rc = SQLITE_ERROR; @@ -15195,18 +18574,20 @@ static int dump_callback(void *pArg, int nArg, char **azArg, char **azNotUsed){ zTable = azArg[0]; zType = azArg[1]; zSql = azArg[2]; + if( zTable==0 ) return 0; + if( zType==0 ) return 0; dataOnly = (p->shellFlgs & SHFLG_DumpDataOnly)!=0; noSys = (p->shellFlgs & SHFLG_DumpNoSys)!=0; - if( strcmp(zTable, "sqlite_sequence")==0 && !noSys ){ + if( cli_strcmp(zTable, "sqlite_sequence")==0 && !noSys ){ if( !dataOnly ) raw_printf(p->out, "DELETE FROM sqlite_sequence;\n"); }else if( sqlite3_strglob("sqlite_stat?", zTable)==0 && !noSys ){ if( !dataOnly ) raw_printf(p->out, "ANALYZE sqlite_schema;\n"); - }else if( strncmp(zTable, "sqlite_", 7)==0 ){ + }else if( cli_strncmp(zTable, "sqlite_", 7)==0 ){ return 0; }else if( dataOnly ){ /* no-op */ - }else if( strncmp(zSql, "CREATE VIRTUAL TABLE", 20)==0 ){ + }else if( cli_strncmp(zSql, "CREATE VIRTUAL TABLE", 20)==0 ){ char *zIns; if( !p->writableSchema ){ raw_printf(p->out, "PRAGMA writable_schema=ON;\n"); @@ -15224,7 +18605,7 @@ static int dump_callback(void *pArg, int nArg, char **azArg, char **azNotUsed){ printSchemaLine(p->out, zSql, ";\n"); } - if( strcmp(zType, "table")==0 ){ + if( cli_strcmp(zType, "table")==0 ){ ShellText sSelect; ShellText sTable; char **azCol; @@ -15342,7 +18723,7 @@ static int run_schema_dump_query( */ static const char *(azHelp[]) = { #if defined(SQLITE_HAVE_ZLIB) && !defined(SQLITE_OMIT_VIRTUALTABLE) \ - && !defined(SQLITE_SHELL_WASM_MODE) + && !defined(SQLITE_SHELL_FIDDLE) ".archive ... Manage SQL archives", " Each command must have exactly one of the following options:", " -c, --create Create a new archive", @@ -15368,7 +18749,7 @@ static const char *(azHelp[]) = { #ifndef SQLITE_OMIT_AUTHORIZATION ".auth ON|OFF Show authorizer callbacks", #endif -#ifndef SQLITE_SHELL_WASM_MODE +#ifndef SQLITE_SHELL_FIDDLE ".backup ?DB? FILE Backup DB (default \"main\") to FILE", " Options:", " --append Use the appendvfs", @@ -15376,18 +18757,18 @@ static const char *(azHelp[]) = { #endif ".bail on|off Stop after hitting an error. Default OFF", ".binary on|off Turn binary output on or off. Default OFF", -#ifndef SQLITE_SHELL_WASM_MODE +#ifndef SQLITE_SHELL_FIDDLE ".cd DIRECTORY Change the working directory to DIRECTORY", #endif ".changes on|off Show number of rows changed by SQL", -#ifndef SQLITE_SHELL_WASM_MODE +#ifndef SQLITE_SHELL_FIDDLE ".check GLOB Fail if output since .testcase does not match", ".clone NEWDB Clone data into NEWDB from the existing database", #endif ".connection [close] [#] Open or close an auxiliary database connection", ".databases List names and files of attached databases", ".dbconfig ?op? ?val? List or change sqlite3_db_config() options", -#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_DBPAGE_VTAB) +#if SQLITE_SHELL_HAVE_RECOVER ".dbinfo ?DB? Show status information about the database", #endif ".dump ?OBJECTS? Render database content as SQL", @@ -15406,11 +18787,11 @@ static const char *(azHelp[]) = { " trace Like \"full\" but enable \"PRAGMA vdbe_trace\"", #endif " trigger Like \"full\" but also show trigger bytecode", -#ifndef SQLITE_SHELL_WASM_MODE +#ifndef SQLITE_SHELL_FIDDLE ".excel Display the output of next command in spreadsheet", " --bom Put a UTF8 byte-order mark on intermediate file", #endif -#ifndef SQLITE_SHELL_WASM_MODE +#ifndef SQLITE_SHELL_FIDDLE ".exit ?CODE? Exit this program with return-code CODE", #endif ".expert EXPERIMENTAL. Suggest indexes for queries", @@ -15421,7 +18802,7 @@ static const char *(azHelp[]) = { ".fullschema ?--indent? Show schema and the content of sqlite_stat tables", ".headers on|off Turn display of headers on or off", ".help ?-all? ?PATTERN? Show help text for PATTERN", -#ifndef SQLITE_SHELL_WASM_MODE +#ifndef SQLITE_SHELL_FIDDLE ".import FILE TABLE Import data from FILE into TABLE", " Options:", " --ascii Use \\037 and \\036 as column and row separators", @@ -15450,10 +18831,10 @@ static const char *(azHelp[]) = { ".lint OPTIONS Report potential schema issues.", " Options:", " fkey-indexes Find missing foreign key indexes", -#if !defined(SQLITE_OMIT_LOAD_EXTENSION) && !defined(SQLITE_SHELL_WASM_MODE) +#if !defined(SQLITE_OMIT_LOAD_EXTENSION) && !defined(SQLITE_SHELL_FIDDLE) ".load FILE ?ENTRY? Load an extension library", #endif -#ifndef SQLITE_SHELL_WASM_MODE +#ifndef SQLITE_SHELL_FIDDLE ".log FILE|off Turn logging on or off. FILE can be stderr/stdout", #endif ".mode MODE ?OPTIONS? Set output mode", @@ -15468,7 +18849,7 @@ static const char *(azHelp[]) = { " line One value per line", " list Values delimited by \"|\"", " markdown Markdown table format", - " qbox Shorthand for \"box --width 60 --quote\"", + " qbox Shorthand for \"box --wrap 60 --quote\"", " quote Escape answers as for SQL", " table ASCII-art table", " tabs Tab-separated values", @@ -15480,11 +18861,11 @@ static const char *(azHelp[]) = { " --quote Quote output text as SQL literals", " --noquote Do not quote output text", " TABLE The name of SQL table used for \"insert\" mode", -#ifndef SQLITE_SHELL_WASM_MODE +#ifndef SQLITE_SHELL_FIDDLE ".nonce STRING Suspend safe mode for one command if nonce matches", #endif ".nullvalue STRING Use STRING in place of NULL values", -#ifndef SQLITE_SHELL_WASM_MODE +#ifndef SQLITE_SHELL_FIDDLE ".once ?OPTIONS? ?FILE? Output for the next SQL command only to FILE", " If FILE begins with '|' then open as a pipe", " --bom Put a UTF8 byte-order mark at the beginning", @@ -15506,7 +18887,7 @@ static const char *(azHelp[]) = { " --nofollow Do not follow symbolic links", " --readonly Open FILE readonly", " --zip FILE is a ZIP archive", -#ifndef SQLITE_SHELL_WASM_MODE +#ifndef SQLITE_SHELL_FIDDLE ".output ?FILE? Send output to FILE or stdout if FILE is omitted", " If FILE begins with '|' then open it as a pipe.", " Options:", @@ -15530,20 +18911,19 @@ static const char *(azHelp[]) = { " --reset Reset the count for each input and interrupt", #endif ".prompt MAIN CONTINUE Replace the standard prompts", -#ifndef SQLITE_SHELL_WASM_MODE +#ifndef SQLITE_SHELL_FIDDLE ".quit Exit this program", ".read FILE Read input from FILE or command output", " If FILE begins with \"|\", it is a command that generates the input.", #endif -#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_DBPAGE_VTAB) +#if SQLITE_SHELL_HAVE_RECOVER ".recover Recover as much data as possible from corrupt db.", - " --freelist-corrupt Assume the freelist is corrupt", - " --recovery-db NAME Store recovery metadata in database file NAME", + " --ignore-freelist Ignore pages that appear to be on db freelist", " --lost-and-found TABLE Alternative name for the lost-and-found table", " --no-rowids Do not attempt to recover rowid values", " that are not also INTEGER PRIMARY KEYs", #endif -#ifndef SQLITE_SHELL_WASM_MODE +#ifndef SQLITE_SHELL_FIDDLE ".restore ?DB? FILE Restore content of DB (default \"main\") from FILE", ".save ?OPTIONS? FILE Write database to FILE (an alias for .backup ...)", #endif @@ -15580,7 +18960,7 @@ static const char *(azHelp[]) = { " --sha3-384 Use the sha3-384 algorithm", " --sha3-512 Use the sha3-512 algorithm", " Any other argument is a LIKE pattern for tables to hash", -#if !defined(SQLITE_NOHAVE_SYSTEM) && !defined(SQLITE_SHELL_WASM_MODE) +#if !defined(SQLITE_NOHAVE_SYSTEM) && !defined(SQLITE_SHELL_FIDDLE) ".shell CMD ARGS... Run CMD ARGS... in a system shell", #endif ".show Show the current values for various settings", @@ -15589,11 +18969,11 @@ static const char *(azHelp[]) = { " on Turn on automatic stat display", " stmt Show statement stats", " vmstep Show the virtual machine step count only", -#if !defined(SQLITE_NOHAVE_SYSTEM) && !defined(SQLITE_SHELL_WASM_MODE) +#if !defined(SQLITE_NOHAVE_SYSTEM) && !defined(SQLITE_SHELL_FIDDLE) ".system CMD ARGS... Run CMD ARGS... in a system shell", #endif ".tables ?TABLE? List names of tables matching LIKE pattern TABLE", -#ifndef SQLITE_SHELL_WASM_MODE +#ifndef SQLITE_SHELL_FIDDLE ".testcase NAME Begin redirecting output to 'testcase-out.txt'", #endif ".testctrl CMD ... Run various sqlite3_test_control() operations", @@ -15643,9 +19023,9 @@ static int showHelp(FILE *out, const char *zPattern){ char *zPat; if( zPattern==0 || zPattern[0]=='0' - || strcmp(zPattern,"-a")==0 - || strcmp(zPattern,"-all")==0 - || strcmp(zPattern,"--all")==0 + || cli_strcmp(zPattern,"-a")==0 + || cli_strcmp(zPattern,"-all")==0 + || cli_strcmp(zPattern,"--all")==0 ){ /* Show all commands, but only one line per command */ if( zPattern==0 ) zPattern = ""; @@ -15882,7 +19262,7 @@ static unsigned char *readHexDb(ShellState *p, int *pnData){ iOffset = k; continue; } - if( strncmp(zLine, "| end ", 6)==0 ){ + if( cli_strncmp(zLine, "| end ", 6)==0 ){ break; } rc = sscanf(zLine,"| %d: %x %x %x %x %x %x %x %x %x %x %x %x %x %x %x %x", @@ -15910,7 +19290,7 @@ readHexDb_error: }else{ while( fgets(zLine, sizeof(zLine), p->in)!=0 ){ nLine++; - if(strncmp(zLine, "| end ", 6)==0 ) break; + if(cli_strncmp(zLine, "| end ", 6)==0 ) break; } p->lineno = nLine; } @@ -16002,28 +19382,28 @@ static void shellEscapeCrnl( const char *zText = (const char*)sqlite3_value_text(argv[0]); UNUSED_PARAMETER(argc); if( zText && zText[0]=='\'' ){ - int nText = sqlite3_value_bytes(argv[0]); - int i; + i64 nText = sqlite3_value_bytes(argv[0]); + i64 i; char zBuf1[20]; char zBuf2[20]; const char *zNL = 0; const char *zCR = 0; - int nCR = 0; - int nNL = 0; + i64 nCR = 0; + i64 nNL = 0; for(i=0; zText[i]; i++){ if( zNL==0 && zText[i]=='\n' ){ zNL = unused_string(zText, "\\n", "\\012", zBuf1); - nNL = (int)strlen(zNL); + nNL = strlen(zNL); } if( zCR==0 && zText[i]=='\r' ){ zCR = unused_string(zText, "\\r", "\\015", zBuf2); - nCR = (int)strlen(zCR); + nCR = strlen(zCR); } } if( zNL || zCR ){ - int iOut = 0; + i64 iOut = 0; i64 nMax = (nNL > nCR) ? nNL : nCR; i64 nAlloc = nMax * nText + (nMax+64)*2; char *zOut = (char*)sqlite3_malloc64(nAlloc); @@ -16146,11 +19526,11 @@ static void open_db(ShellState *p, int openFlags){ sqlite3_regexp_init(p->db, 0, 0); sqlite3_ieee_init(p->db, 0, 0); sqlite3_series_init(p->db, 0, 0); -#ifndef SQLITE_SHELL_WASM_MODE +#ifndef SQLITE_SHELL_FIDDLE sqlite3_fileio_init(p->db, 0, 0); sqlite3_completion_init(p->db, 0, 0); #endif -#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_DBPAGE_VTAB) +#if SQLITE_SHELL_HAVE_RECOVER sqlite3_dbdata_init(p->db, 0, 0); #endif #ifdef SQLITE_HAVE_ZLIB @@ -16264,8 +19644,8 @@ static char **readline_completion(const char *zText, int iStart, int iEnd){ ** Linenoise completion callback */ static void linenoise_completion(const char *zLine, linenoiseCompletions *lc){ - int nLine = strlen30(zLine); - int i, iStart; + i64 nLine = strlen(zLine); + i64 i, iStart; sqlite3_stmt *pStmt = 0; char *zSql; char zBuf[1000]; @@ -16403,11 +19783,11 @@ static void output_file_close(FILE *f){ */ static FILE *output_file_open(const char *zFile, int bTextMode){ FILE *f; - if( strcmp(zFile,"stdout")==0 ){ + if( cli_strcmp(zFile,"stdout")==0 ){ f = stdout; - }else if( strcmp(zFile, "stderr")==0 ){ + }else if( cli_strcmp(zFile, "stderr")==0 ){ f = stderr; - }else if( strcmp(zFile, "off")==0 ){ + }else if( cli_strcmp(zFile, "off")==0 ){ f = 0; }else{ f = fopen(zFile, bTextMode ? "w" : "wb"); @@ -16431,7 +19811,7 @@ static int sql_trace_callback( ShellState *p = (ShellState*)pArg; sqlite3_stmt *pStmt; const char *zSql; - int nSql; + i64 nSql; if( p->traceOut==0 ) return 0; if( mType==SQLITE_TRACE_CLOSE ){ utf8_printf(p->traceOut, "-- closing database connection\n"); @@ -16459,17 +19839,18 @@ static int sql_trace_callback( } } if( zSql==0 ) return 0; - nSql = strlen30(zSql); + nSql = strlen(zSql); + if( nSql>1000000000 ) nSql = 1000000000; while( nSql>0 && zSql[nSql-1]==';' ){ nSql--; } switch( mType ){ case SQLITE_TRACE_ROW: case SQLITE_TRACE_STMT: { - utf8_printf(p->traceOut, "%.*s;\n", nSql, zSql); + utf8_printf(p->traceOut, "%.*s;\n", (int)nSql, zSql); break; } case SQLITE_TRACE_PROFILE: { sqlite3_int64 nNanosec = *(sqlite3_int64*)pX; - utf8_printf(p->traceOut, "%.*s; -- %lld ns\n", nSql, zSql, nNanosec); + utf8_printf(p->traceOut, "%.*s; -- %lld ns\n", (int)nSql, zSql, nNanosec); break; } } @@ -17018,7 +20399,7 @@ static int shell_dbinfo_command(ShellState *p, int nArg, char **azArg){ } if( zDb==0 ){ zSchemaTab = sqlite3_mprintf("main.sqlite_schema"); - }else if( strcmp(zDb,"temp")==0 ){ + }else if( cli_strcmp(zDb,"temp")==0 ){ zSchemaTab = sqlite3_mprintf("%s", "sqlite_temp_schema"); }else{ zSchemaTab = sqlite3_mprintf("\"%w\".sqlite_schema", zDb); @@ -17034,8 +20415,7 @@ static int shell_dbinfo_command(ShellState *p, int nArg, char **azArg){ utf8_printf(p->out, "%-20s %u\n", "data version", iDataVersion); return 0; } -#endif /* !defined(SQLITE_OMIT_VIRTUALTABLE) - && defined(SQLITE_ENABLE_DBPAGE_VTAB) */ +#endif /* SQLITE_SHELL_HAVE_RECOVER */ /* ** Print the current sqlite3_errmsg() value to stderr and return 1. @@ -17152,7 +20532,7 @@ static int optionMatch(const char *zStr, const char *zOpt){ if( zStr[0]!='-' ) return 0; zStr++; if( zStr[0]=='-' ) zStr++; - return strcmp(zStr, zOpt)==0; + return cli_strcmp(zStr, zOpt)==0; } /* @@ -18308,364 +21688,16 @@ end_ar_command: *******************************************************************************/ #endif /* !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_HAVE_ZLIB) */ -#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_DBPAGE_VTAB) -/* -** If (*pRc) is not SQLITE_OK when this function is called, it is a no-op. -** Otherwise, the SQL statement or statements in zSql are executed using -** database connection db and the error code written to *pRc before -** this function returns. -*/ -static void shellExec(sqlite3 *db, int *pRc, const char *zSql){ - int rc = *pRc; - if( rc==SQLITE_OK ){ - char *zErr = 0; - rc = sqlite3_exec(db, zSql, 0, 0, &zErr); - if( rc!=SQLITE_OK ){ - raw_printf(stderr, "SQL error: %s\n", zErr); - } - sqlite3_free(zErr); - *pRc = rc; - } -} +#if SQLITE_SHELL_HAVE_RECOVER /* -** Like shellExec(), except that zFmt is a printf() style format string. +** This function is used as a callback by the recover extension. Simply +** print the supplied SQL statement to stdout. */ -static void shellExecPrintf(sqlite3 *db, int *pRc, const char *zFmt, ...){ - char *z = 0; - if( *pRc==SQLITE_OK ){ - va_list ap; - va_start(ap, zFmt); - z = sqlite3_vmprintf(zFmt, ap); - va_end(ap); - if( z==0 ){ - *pRc = SQLITE_NOMEM; - }else{ - shellExec(db, pRc, z); - } - sqlite3_free(z); - } -} - -/* -** If *pRc is not SQLITE_OK when this function is called, it is a no-op. -** Otherwise, an attempt is made to allocate, zero and return a pointer -** to a buffer nByte bytes in size. If an OOM error occurs, *pRc is set -** to SQLITE_NOMEM and NULL returned. -*/ -static void *shellMalloc(int *pRc, sqlite3_int64 nByte){ - void *pRet = 0; - if( *pRc==SQLITE_OK ){ - pRet = sqlite3_malloc64(nByte); - if( pRet==0 ){ - *pRc = SQLITE_NOMEM; - }else{ - memset(pRet, 0, nByte); - } - } - return pRet; -} - -/* -** If *pRc is not SQLITE_OK when this function is called, it is a no-op. -** Otherwise, zFmt is treated as a printf() style string. The result of -** formatting it along with any trailing arguments is written into a -** buffer obtained from sqlite3_malloc(), and pointer to which is returned. -** It is the responsibility of the caller to eventually free this buffer -** using a call to sqlite3_free(). -** -** If an OOM error occurs, (*pRc) is set to SQLITE_NOMEM and a NULL -** pointer returned. -*/ -static char *shellMPrintf(int *pRc, const char *zFmt, ...){ - char *z = 0; - if( *pRc==SQLITE_OK ){ - va_list ap; - va_start(ap, zFmt); - z = sqlite3_vmprintf(zFmt, ap); - va_end(ap); - if( z==0 ){ - *pRc = SQLITE_NOMEM; - } - } - return z; -} - - -/* -** When running the ".recover" command, each output table, and the special -** orphaned row table if it is required, is represented by an instance -** of the following struct. -*/ -typedef struct RecoverTable RecoverTable; -struct RecoverTable { - char *zQuoted; /* Quoted version of table name */ - int nCol; /* Number of columns in table */ - char **azlCol; /* Array of column lists */ - int iPk; /* Index of IPK column */ -}; - -/* -** Free a RecoverTable object allocated by recoverFindTable() or -** recoverOrphanTable(). -*/ -static void recoverFreeTable(RecoverTable *pTab){ - if( pTab ){ - sqlite3_free(pTab->zQuoted); - if( pTab->azlCol ){ - int i; - for(i=0; i<=pTab->nCol; i++){ - sqlite3_free(pTab->azlCol[i]); - } - sqlite3_free(pTab->azlCol); - } - sqlite3_free(pTab); - } -} - -/* -** This function is a no-op if (*pRc) is not SQLITE_OK when it is called. -** Otherwise, it allocates and returns a RecoverTable object based on the -** final four arguments passed to this function. It is the responsibility -** of the caller to eventually free the returned object using -** recoverFreeTable(). -*/ -static RecoverTable *recoverNewTable( - int *pRc, /* IN/OUT: Error code */ - const char *zName, /* Name of table */ - const char *zSql, /* CREATE TABLE statement */ - int bIntkey, - int nCol -){ - sqlite3 *dbtmp = 0; /* sqlite3 handle for testing CREATE TABLE */ - int rc = *pRc; - RecoverTable *pTab = 0; - - pTab = (RecoverTable*)shellMalloc(&rc, sizeof(RecoverTable)); - if( rc==SQLITE_OK ){ - int nSqlCol = 0; - int bSqlIntkey = 0; - sqlite3_stmt *pStmt = 0; - - rc = sqlite3_open("", &dbtmp); - if( rc==SQLITE_OK ){ - sqlite3_create_function(dbtmp, "shell_idquote", 1, SQLITE_UTF8, 0, - shellIdQuote, 0, 0); - } - if( rc==SQLITE_OK ){ - rc = sqlite3_exec(dbtmp, "PRAGMA writable_schema = on", 0, 0, 0); - } - if( rc==SQLITE_OK ){ - rc = sqlite3_exec(dbtmp, zSql, 0, 0, 0); - if( rc==SQLITE_ERROR ){ - rc = SQLITE_OK; - goto finished; - } - } - shellPreparePrintf(dbtmp, &rc, &pStmt, - "SELECT count(*) FROM pragma_table_info(%Q)", zName - ); - if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ - nSqlCol = sqlite3_column_int(pStmt, 0); - } - shellFinalize(&rc, pStmt); - - if( rc!=SQLITE_OK || nSqlColiPk to the index - ** of the column, where columns are 0-numbered from left to right. - ** Or, if this is a WITHOUT ROWID table or if there is no IPK column, - ** leave zPk as "_rowid_" and pTab->iPk at -2. */ - pTab->iPk = -2; - if( bIntkey ){ - shellPreparePrintf(dbtmp, &rc, &pPkFinder, - "SELECT cid, name FROM pragma_table_info(%Q) " - " WHERE pk=1 AND type='integer' COLLATE nocase" - " AND NOT EXISTS (SELECT cid FROM pragma_table_info(%Q) WHERE pk=2)" - , zName, zName - ); - if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pPkFinder) ){ - pTab->iPk = sqlite3_column_int(pPkFinder, 0); - zPk = (const char*)sqlite3_column_text(pPkFinder, 1); - if( zPk==0 ){ zPk = "_"; /* Defensive. Should never happen */ } - } - } - - pTab->zQuoted = shellMPrintf(&rc, "\"%w\"", zName); - pTab->azlCol = (char**)shellMalloc(&rc, sizeof(char*) * (nSqlCol+1)); - pTab->nCol = nSqlCol; - - if( bIntkey ){ - pTab->azlCol[0] = shellMPrintf(&rc, "\"%w\"", zPk); - }else{ - pTab->azlCol[0] = shellMPrintf(&rc, ""); - } - i = 1; - shellPreparePrintf(dbtmp, &rc, &pStmt, - "SELECT %Q || group_concat(shell_idquote(name), ', ') " - " FILTER (WHERE cid!=%d) OVER (ORDER BY %s cid) " - "FROM pragma_table_info(%Q)", - bIntkey ? ", " : "", pTab->iPk, - bIntkey ? "" : "(CASE WHEN pk=0 THEN 1000000 ELSE pk END), ", - zName - ); - while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ - const char *zText = (const char*)sqlite3_column_text(pStmt, 0); - pTab->azlCol[i] = shellMPrintf(&rc, "%s%s", pTab->azlCol[0], zText); - i++; - } - shellFinalize(&rc, pStmt); - - shellFinalize(&rc, pPkFinder); - } - } - - finished: - sqlite3_close(dbtmp); - *pRc = rc; - if( rc!=SQLITE_OK || (pTab && pTab->zQuoted==0) ){ - recoverFreeTable(pTab); - pTab = 0; - } - return pTab; -} - -/* -** This function is called to search the schema recovered from the -** sqlite_schema table of the (possibly) corrupt database as part -** of a ".recover" command. Specifically, for a table with root page -** iRoot and at least nCol columns. Additionally, if bIntkey is 0, the -** table must be a WITHOUT ROWID table, or if non-zero, not one of -** those. -** -** If a table is found, a (RecoverTable*) object is returned. Or, if -** no such table is found, but bIntkey is false and iRoot is the -** root page of an index in the recovered schema, then (*pbNoop) is -** set to true and NULL returned. Or, if there is no such table or -** index, NULL is returned and (*pbNoop) set to 0, indicating that -** the caller should write data to the orphans table. -*/ -static RecoverTable *recoverFindTable( - ShellState *pState, /* Shell state object */ - int *pRc, /* IN/OUT: Error code */ - int iRoot, /* Root page of table */ - int bIntkey, /* True for an intkey table */ - int nCol, /* Number of columns in table */ - int *pbNoop /* OUT: True if iRoot is root of index */ -){ - sqlite3_stmt *pStmt = 0; - RecoverTable *pRet = 0; - int bNoop = 0; - const char *zSql = 0; - const char *zName = 0; - - /* Search the recovered schema for an object with root page iRoot. */ - shellPreparePrintf(pState->db, pRc, &pStmt, - "SELECT type, name, sql FROM recovery.schema WHERE rootpage=%d", iRoot - ); - while( *pRc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ - const char *zType = (const char*)sqlite3_column_text(pStmt, 0); - if( bIntkey==0 && sqlite3_stricmp(zType, "index")==0 ){ - bNoop = 1; - break; - } - if( sqlite3_stricmp(zType, "table")==0 ){ - zName = (const char*)sqlite3_column_text(pStmt, 1); - zSql = (const char*)sqlite3_column_text(pStmt, 2); - if( zName!=0 && zSql!=0 ){ - pRet = recoverNewTable(pRc, zName, zSql, bIntkey, nCol); - break; - } - } - } - - shellFinalize(pRc, pStmt); - *pbNoop = bNoop; - return pRet; -} - -/* -** Return a RecoverTable object representing the orphans table. -*/ -static RecoverTable *recoverOrphanTable( - ShellState *pState, /* Shell state object */ - int *pRc, /* IN/OUT: Error code */ - const char *zLostAndFound, /* Base name for orphans table */ - int nCol /* Number of user data columns */ -){ - RecoverTable *pTab = 0; - if( nCol>=0 && *pRc==SQLITE_OK ){ - int i; - - /* This block determines the name of the orphan table. The prefered - ** name is zLostAndFound. But if that clashes with another name - ** in the recovered schema, try zLostAndFound_0, zLostAndFound_1 - ** and so on until a non-clashing name is found. */ - int iTab = 0; - char *zTab = shellMPrintf(pRc, "%s", zLostAndFound); - sqlite3_stmt *pTest = 0; - shellPrepare(pState->db, pRc, - "SELECT 1 FROM recovery.schema WHERE name=?", &pTest - ); - if( pTest ) sqlite3_bind_text(pTest, 1, zTab, -1, SQLITE_TRANSIENT); - while( *pRc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pTest) ){ - shellReset(pRc, pTest); - sqlite3_free(zTab); - zTab = shellMPrintf(pRc, "%s_%d", zLostAndFound, iTab++); - sqlite3_bind_text(pTest, 1, zTab, -1, SQLITE_TRANSIENT); - } - shellFinalize(pRc, pTest); - - pTab = (RecoverTable*)shellMalloc(pRc, sizeof(RecoverTable)); - if( pTab ){ - pTab->zQuoted = shellMPrintf(pRc, "\"%w\"", zTab); - pTab->nCol = nCol; - pTab->iPk = -2; - if( nCol>0 ){ - pTab->azlCol = (char**)shellMalloc(pRc, sizeof(char*) * (nCol+1)); - if( pTab->azlCol ){ - pTab->azlCol[nCol] = shellMPrintf(pRc, ""); - for(i=nCol-1; i>=0; i--){ - pTab->azlCol[i] = shellMPrintf(pRc, "%s, NULL", pTab->azlCol[i+1]); - } - } - } - - if( *pRc!=SQLITE_OK ){ - recoverFreeTable(pTab); - pTab = 0; - }else{ - raw_printf(pState->out, - "CREATE TABLE %s(rootpgno INTEGER, " - "pgno INTEGER, nfield INTEGER, id INTEGER", pTab->zQuoted - ); - for(i=0; iout, ", c%d", i); - } - raw_printf(pState->out, ");\n"); - } - } - sqlite3_free(zTab); - } - return pTab; +static int recoverSqlCb(void *pCtx, const char *zSql){ + ShellState *pState = (ShellState*)pCtx; + utf8_printf(pState->out, "%s;\n", zSql); + return SQLITE_OK; } /* @@ -18675,32 +21707,33 @@ static RecoverTable *recoverOrphanTable( */ static int recoverDatabaseCmd(ShellState *pState, int nArg, char **azArg){ int rc = SQLITE_OK; - sqlite3_stmt *pLoop = 0; /* Loop through all root pages */ - sqlite3_stmt *pPages = 0; /* Loop through all pages in a group */ - sqlite3_stmt *pCells = 0; /* Loop through all cells in a page */ - const char *zRecoveryDb = ""; /* Name of "recovery" database */ - const char *zLostAndFound = "lost_and_found"; - int i; - int nOrphan = -1; - RecoverTable *pOrphan = 0; - - int bFreelist = 1; /* 0 if --freelist-corrupt is specified */ + const char *zRecoveryDb = ""; /* Name of "recovery" database. Debug only */ + const char *zLAF = "lost_and_found"; + int bFreelist = 1; /* 0 if --ignore-freelist is specified */ int bRowids = 1; /* 0 if --no-rowids */ + sqlite3_recover *p = 0; + int i = 0; + for(i=1; idb, &rc, - /* Attach an in-memory database named 'recovery'. Create an indexed - ** cache of the sqlite_dbptr virtual table. */ - "PRAGMA writable_schema = on;" - "ATTACH %Q AS recovery;" - "DROP TABLE IF EXISTS recovery.dbptr;" - "DROP TABLE IF EXISTS recovery.freelist;" - "DROP TABLE IF EXISTS recovery.map;" - "DROP TABLE IF EXISTS recovery.schema;" - "CREATE TABLE recovery.freelist(pgno INTEGER PRIMARY KEY);", zRecoveryDb + p = sqlite3_recover_init_sql( + pState->db, "main", recoverSqlCb, (void*)pState ); - if( bFreelist ){ - shellExec(pState->db, &rc, - "WITH trunk(pgno) AS (" - " SELECT shell_int32(" - " (SELECT data FROM sqlite_dbpage WHERE pgno=1), 8) AS x " - " WHERE x>0" - " UNION" - " SELECT shell_int32(" - " (SELECT data FROM sqlite_dbpage WHERE pgno=trunk.pgno), 0) AS x " - " FROM trunk WHERE x>0" - ")," - "freelist(data, n, freepgno) AS (" - " SELECT data, min(16384, shell_int32(data, 1)-1), t.pgno " - " FROM trunk t, sqlite_dbpage s WHERE s.pgno=t.pgno" - " UNION ALL" - " SELECT data, n-1, shell_int32(data, 2+n) " - " FROM freelist WHERE n>=0" - ")" - "REPLACE INTO recovery.freelist SELECT freepgno FROM freelist;" - ); + sqlite3_recover_config(p, 789, (void*)zRecoveryDb); /* Debug use only */ + sqlite3_recover_config(p, SQLITE_RECOVER_LOST_AND_FOUND, (void*)zLAF); + sqlite3_recover_config(p, SQLITE_RECOVER_ROWIDS, (void*)&bRowids); + sqlite3_recover_config(p, SQLITE_RECOVER_FREELIST_CORRUPT,(void*)&bFreelist); + + sqlite3_recover_run(p); + if( sqlite3_recover_errcode(p)!=SQLITE_OK ){ + const char *zErr = sqlite3_recover_errmsg(p); + int errCode = sqlite3_recover_errcode(p); + raw_printf(stderr, "sql error: %s (%d)\n", zErr, errCode); } - - /* If this is an auto-vacuum database, add all pointer-map pages to - ** the freelist table. Do this regardless of whether or not - ** --freelist-corrupt was specified. */ - shellExec(pState->db, &rc, - "WITH ptrmap(pgno) AS (" - " SELECT 2 WHERE shell_int32(" - " (SELECT data FROM sqlite_dbpage WHERE pgno=1), 13" - " )" - " UNION ALL " - " SELECT pgno+1+(SELECT page_size FROM pragma_page_size)/5 AS pp " - " FROM ptrmap WHERE pp<=(SELECT page_count FROM pragma_page_count)" - ")" - "REPLACE INTO recovery.freelist SELECT pgno FROM ptrmap" - ); - - shellExec(pState->db, &rc, - "CREATE TABLE recovery.dbptr(" - " pgno, child, PRIMARY KEY(child, pgno)" - ") WITHOUT ROWID;" - "INSERT OR IGNORE INTO recovery.dbptr(pgno, child) " - " SELECT * FROM sqlite_dbptr" - " WHERE pgno NOT IN freelist AND child NOT IN freelist;" - - /* Delete any pointer to page 1. This ensures that page 1 is considered - ** a root page, regardless of how corrupt the db is. */ - "DELETE FROM recovery.dbptr WHERE child = 1;" - - /* Delete all pointers to any pages that have more than one pointer - ** to them. Such pages will be treated as root pages when recovering - ** data. */ - "DELETE FROM recovery.dbptr WHERE child IN (" - " SELECT child FROM recovery.dbptr GROUP BY child HAVING count(*)>1" - ");" - - /* Create the "map" table that will (eventually) contain instructions - ** for dealing with each page in the db that contains one or more - ** records. */ - "CREATE TABLE recovery.map(" - "pgno INTEGER PRIMARY KEY, maxlen INT, intkey, root INT" - ");" - - /* Populate table [map]. If there are circular loops of pages in the - ** database, the following adds all pages in such a loop to the map - ** as individual root pages. This could be handled better. */ - "WITH pages(i, maxlen) AS (" - " SELECT page_count, (" - " SELECT max(field+1) FROM sqlite_dbdata WHERE pgno=page_count" - " ) FROM pragma_page_count WHERE page_count>0" - " UNION ALL" - " SELECT i-1, (" - " SELECT max(field+1) FROM sqlite_dbdata WHERE pgno=i-1" - " ) FROM pages WHERE i>=2" - ")" - "INSERT INTO recovery.map(pgno, maxlen, intkey, root) " - " SELECT i, maxlen, NULL, (" - " WITH p(orig, pgno, parent) AS (" - " SELECT 0, i, (SELECT pgno FROM recovery.dbptr WHERE child=i)" - " UNION " - " SELECT i, p.parent, " - " (SELECT pgno FROM recovery.dbptr WHERE child=p.parent) FROM p" - " )" - " SELECT pgno FROM p WHERE (parent IS NULL OR pgno = orig)" - ") " - "FROM pages WHERE maxlen IS NOT NULL AND i NOT IN freelist;" - "UPDATE recovery.map AS o SET intkey = (" - " SELECT substr(data, 1, 1)==X'0D' FROM sqlite_dbpage WHERE pgno=o.pgno" - ");" - - /* Extract data from page 1 and any linked pages into table - ** recovery.schema. With the same schema as an sqlite_schema table. */ - "CREATE TABLE recovery.schema(type, name, tbl_name, rootpage, sql);" - "INSERT INTO recovery.schema SELECT " - " max(CASE WHEN field=0 THEN value ELSE NULL END)," - " max(CASE WHEN field=1 THEN value ELSE NULL END)," - " max(CASE WHEN field=2 THEN value ELSE NULL END)," - " max(CASE WHEN field=3 THEN value ELSE NULL END)," - " max(CASE WHEN field=4 THEN value ELSE NULL END)" - "FROM sqlite_dbdata WHERE pgno IN (" - " SELECT pgno FROM recovery.map WHERE root=1" - ")" - "GROUP BY pgno, cell;" - "CREATE INDEX recovery.schema_rootpage ON schema(rootpage);" - ); - - /* Open a transaction, then print out all non-virtual, non-"sqlite_%" - ** CREATE TABLE statements that extracted from the existing schema. */ - if( rc==SQLITE_OK ){ - sqlite3_stmt *pStmt = 0; - /* ".recover" might output content in an order which causes immediate - ** foreign key constraints to be violated. So disable foreign-key - ** constraint enforcement to prevent problems when running the output - ** script. */ - raw_printf(pState->out, "PRAGMA foreign_keys=OFF;\n"); - raw_printf(pState->out, "BEGIN;\n"); - raw_printf(pState->out, "PRAGMA writable_schema = on;\n"); - shellPrepare(pState->db, &rc, - "SELECT sql FROM recovery.schema " - "WHERE type='table' AND sql LIKE 'create table%'", &pStmt - ); - while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ - const char *zCreateTable = (const char*)sqlite3_column_text(pStmt, 0); - raw_printf(pState->out, "CREATE TABLE IF NOT EXISTS %s;\n", - &zCreateTable[12] - ); - } - shellFinalize(&rc, pStmt); - } - - /* Figure out if an orphan table will be required. And if so, how many - ** user columns it should contain */ - shellPrepare(pState->db, &rc, - "SELECT coalesce(max(maxlen), -2) FROM recovery.map WHERE root>1" - , &pLoop - ); - if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pLoop) ){ - nOrphan = sqlite3_column_int(pLoop, 0); - } - shellFinalize(&rc, pLoop); - pLoop = 0; - - shellPrepare(pState->db, &rc, - "SELECT pgno FROM recovery.map WHERE root=?", &pPages - ); - - shellPrepare(pState->db, &rc, - "SELECT max(field), group_concat(shell_escape_crnl(quote" - "(case when (? AND field<0) then NULL else value end)" - "), ', ')" - ", min(field) " - "FROM sqlite_dbdata WHERE pgno = ? AND field != ?" - "GROUP BY cell", &pCells - ); - - /* Loop through each root page. */ - shellPrepare(pState->db, &rc, - "SELECT root, intkey, max(maxlen) FROM recovery.map" - " WHERE root>1 GROUP BY root, intkey ORDER BY root=(" - " SELECT rootpage FROM recovery.schema WHERE name='sqlite_sequence'" - ")", &pLoop - ); - while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pLoop) ){ - int iRoot = sqlite3_column_int(pLoop, 0); - int bIntkey = sqlite3_column_int(pLoop, 1); - int nCol = sqlite3_column_int(pLoop, 2); - int bNoop = 0; - RecoverTable *pTab; - - assert( bIntkey==0 || bIntkey==1 ); - pTab = recoverFindTable(pState, &rc, iRoot, bIntkey, nCol, &bNoop); - if( bNoop || rc ) continue; - if( pTab==0 ){ - if( pOrphan==0 ){ - pOrphan = recoverOrphanTable(pState, &rc, zLostAndFound, nOrphan); - } - pTab = pOrphan; - if( pTab==0 ) break; - } - - if( 0==sqlite3_stricmp(pTab->zQuoted, "\"sqlite_sequence\"") ){ - raw_printf(pState->out, "DELETE FROM sqlite_sequence;\n"); - } - sqlite3_bind_int(pPages, 1, iRoot); - if( bRowids==0 && pTab->iPk<0 ){ - sqlite3_bind_int(pCells, 1, 1); - }else{ - sqlite3_bind_int(pCells, 1, 0); - } - sqlite3_bind_int(pCells, 3, pTab->iPk); - - while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pPages) ){ - int iPgno = sqlite3_column_int(pPages, 0); - sqlite3_bind_int(pCells, 2, iPgno); - while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pCells) ){ - int nField = sqlite3_column_int(pCells, 0); - int iMin = sqlite3_column_int(pCells, 2); - const char *zVal = (const char*)sqlite3_column_text(pCells, 1); - - RecoverTable *pTab2 = pTab; - if( pTab!=pOrphan && (iMin<0)!=bIntkey ){ - if( pOrphan==0 ){ - pOrphan = recoverOrphanTable(pState, &rc, zLostAndFound, nOrphan); - } - pTab2 = pOrphan; - if( pTab2==0 ) break; - } - - nField = nField+1; - if( pTab2==pOrphan ){ - raw_printf(pState->out, - "INSERT INTO %s VALUES(%d, %d, %d, %s%s%s);\n", - pTab2->zQuoted, iRoot, iPgno, nField, - iMin<0 ? "" : "NULL, ", zVal, pTab2->azlCol[nField] - ); - }else{ - raw_printf(pState->out, "INSERT INTO %s(%s) VALUES( %s );\n", - pTab2->zQuoted, pTab2->azlCol[nField], zVal - ); - } - } - shellReset(&rc, pCells); - } - shellReset(&rc, pPages); - if( pTab!=pOrphan ) recoverFreeTable(pTab); - } - shellFinalize(&rc, pLoop); - shellFinalize(&rc, pPages); - shellFinalize(&rc, pCells); - recoverFreeTable(pOrphan); - - /* The rest of the schema */ - if( rc==SQLITE_OK ){ - sqlite3_stmt *pStmt = 0; - shellPrepare(pState->db, &rc, - "SELECT sql, name FROM recovery.schema " - "WHERE sql NOT LIKE 'create table%'", &pStmt - ); - while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ - const char *zSql = (const char*)sqlite3_column_text(pStmt, 0); - if( sqlite3_strnicmp(zSql, "create virt", 11)==0 ){ - const char *zName = (const char*)sqlite3_column_text(pStmt, 1); - char *zPrint = shellMPrintf(&rc, - "INSERT INTO sqlite_schema VALUES('table', %Q, %Q, 0, %Q)", - zName, zName, zSql - ); - raw_printf(pState->out, "%s;\n", zPrint); - sqlite3_free(zPrint); - }else{ - raw_printf(pState->out, "%s;\n", zSql); - } - } - shellFinalize(&rc, pStmt); - } - - if( rc==SQLITE_OK ){ - raw_printf(pState->out, "PRAGMA writable_schema = off;\n"); - raw_printf(pState->out, "COMMIT;\n"); - } - sqlite3_exec(pState->db, "DETACH recovery", 0, 0, 0); + rc = sqlite3_recover_finish(p); return rc; } -#endif /* !(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_DBPAGE_VTAB) */ +#endif /* SQLITE_SHELL_HAVE_RECOVER */ /* @@ -19258,7 +22035,7 @@ static int do_meta_command(char *zLine, ShellState *p){ clearTempFile(p); #ifndef SQLITE_OMIT_AUTHORIZATION - if( c=='a' && strncmp(azArg[0], "auth", n)==0 ){ + if( c=='a' && cli_strncmp(azArg[0], "auth", n)==0 ){ if( nArg!=2 ){ raw_printf(stderr, "Usage: .auth ON|OFF\n"); rc = 1; @@ -19276,17 +22053,17 @@ static int do_meta_command(char *zLine, ShellState *p){ #endif #if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_HAVE_ZLIB) \ - && !defined(SQLITE_SHELL_WASM_MODE) - if( c=='a' && strncmp(azArg[0], "archive", n)==0 ){ + && !defined(SQLITE_SHELL_FIDDLE) + if( c=='a' && cli_strncmp(azArg[0], "archive", n)==0 ){ open_db(p, 0); failIfSafeMode(p, "cannot run .archive in safe mode"); rc = arDotCommand(p, 0, azArg, nArg); }else #endif -#ifndef SQLITE_SHELL_WASM_MODE - if( (c=='b' && n>=3 && strncmp(azArg[0], "backup", n)==0) - || (c=='s' && n>=3 && strncmp(azArg[0], "save", n)==0) +#ifndef SQLITE_SHELL_FIDDLE + if( (c=='b' && n>=3 && cli_strncmp(azArg[0], "backup", n)==0) + || (c=='s' && n>=3 && cli_strncmp(azArg[0], "save", n)==0) ){ const char *zDestFile = 0; const char *zDb = 0; @@ -19300,10 +22077,10 @@ static int do_meta_command(char *zLine, ShellState *p){ const char *z = azArg[j]; if( z[0]=='-' ){ if( z[1]=='-' ) z++; - if( strcmp(z, "-append")==0 ){ + if( cli_strcmp(z, "-append")==0 ){ zVfs = "apndvfs"; }else - if( strcmp(z, "-async")==0 ){ + if( cli_strcmp(z, "-async")==0 ){ bAsync = 1; }else { @@ -19353,9 +22130,9 @@ static int do_meta_command(char *zLine, ShellState *p){ } close_db(pDest); }else -#endif /* !defined(SQLITE_SHELL_WASM_MODE) */ +#endif /* !defined(SQLITE_SHELL_FIDDLE) */ - if( c=='b' && n>=3 && strncmp(azArg[0], "bail", n)==0 ){ + if( c=='b' && n>=3 && cli_strncmp(azArg[0], "bail", n)==0 ){ if( nArg==2 ){ bail_on_error = booleanValue(azArg[1]); }else{ @@ -19364,7 +22141,7 @@ static int do_meta_command(char *zLine, ShellState *p){ } }else - if( c=='b' && n>=3 && strncmp(azArg[0], "binary", n)==0 ){ + if( c=='b' && n>=3 && cli_strncmp(azArg[0], "binary", n)==0 ){ if( nArg==2 ){ if( booleanValue(azArg[1]) ){ setBinaryMode(p->out, 1); @@ -19380,12 +22157,12 @@ static int do_meta_command(char *zLine, ShellState *p){ /* The undocumented ".breakpoint" command causes a call to the no-op ** routine named test_breakpoint(). */ - if( c=='b' && n>=3 && strncmp(azArg[0], "breakpoint", n)==0 ){ + if( c=='b' && n>=3 && cli_strncmp(azArg[0], "breakpoint", n)==0 ){ test_breakpoint(); }else -#ifndef SQLITE_SHELL_WASM_MODE - if( c=='c' && strcmp(azArg[0],"cd")==0 ){ +#ifndef SQLITE_SHELL_FIDDLE + if( c=='c' && cli_strcmp(azArg[0],"cd")==0 ){ failIfSafeMode(p, "cannot run .cd in safe mode"); if( nArg==2 ){ #if defined(_WIN32) || defined(WIN32) @@ -19404,9 +22181,9 @@ static int do_meta_command(char *zLine, ShellState *p){ rc = 1; } }else -#endif /* !defined(SQLITE_SHELL_WASM_MODE) */ +#endif /* !defined(SQLITE_SHELL_FIDDLE) */ - if( c=='c' && n>=3 && strncmp(azArg[0], "changes", n)==0 ){ + if( c=='c' && n>=3 && cli_strncmp(azArg[0], "changes", n)==0 ){ if( nArg==2 ){ setOrClearFlag(p, SHFLG_CountChanges, azArg[1]); }else{ @@ -19415,12 +22192,12 @@ static int do_meta_command(char *zLine, ShellState *p){ } }else -#ifndef SQLITE_SHELL_WASM_MODE +#ifndef SQLITE_SHELL_FIDDLE /* Cancel output redirection, if it is currently set (by .testcase) ** Then read the content of the testcase-out.txt file and compare against ** azArg[1]. If there are differences, report an error and exit. */ - if( c=='c' && n>=3 && strncmp(azArg[0], "check", n)==0 ){ + if( c=='c' && n>=3 && cli_strncmp(azArg[0], "check", n)==0 ){ char *zRes = 0; output_reset(p); if( nArg!=2 ){ @@ -19440,10 +22217,10 @@ static int do_meta_command(char *zLine, ShellState *p){ } sqlite3_free(zRes); }else -#endif /* !defined(SQLITE_SHELL_WASM_MODE) */ +#endif /* !defined(SQLITE_SHELL_FIDDLE) */ -#ifndef SQLITE_SHELL_WASM_MODE - if( c=='c' && strncmp(azArg[0], "clone", n)==0 ){ +#ifndef SQLITE_SHELL_FIDDLE + if( c=='c' && cli_strncmp(azArg[0], "clone", n)==0 ){ failIfSafeMode(p, "cannot run .clone in safe mode"); if( nArg==2 ){ tryToClone(p, azArg[1]); @@ -19452,9 +22229,9 @@ static int do_meta_command(char *zLine, ShellState *p){ rc = 1; } }else -#endif /* !defined(SQLITE_SHELL_WASM_MODE) */ +#endif /* !defined(SQLITE_SHELL_FIDDLE) */ - if( c=='c' && strncmp(azArg[0], "connection", n)==0 ){ + if( c=='c' && cli_strncmp(azArg[0], "connection", n)==0 ){ if( nArg==1 ){ /* List available connections */ int i; @@ -19481,7 +22258,7 @@ static int do_meta_command(char *zLine, ShellState *p){ globalDb = p->db = p->pAuxDb->db; p->pAuxDb->db = 0; } - }else if( nArg==3 && strcmp(azArg[1], "close")==0 + }else if( nArg==3 && cli_strcmp(azArg[1], "close")==0 && IsDigit(azArg[2][0]) && azArg[2][1]==0 ){ int i = azArg[2][0] - '0'; if( i<0 || i>=ArraySize(p->aAuxDb) ){ @@ -19500,7 +22277,7 @@ static int do_meta_command(char *zLine, ShellState *p){ } }else - if( c=='d' && n>1 && strncmp(azArg[0], "databases", n)==0 ){ + if( c=='d' && n>1 && cli_strncmp(azArg[0], "databases", n)==0 ){ char **azName = 0; int nName = 0; sqlite3_stmt *pStmt; @@ -19539,7 +22316,7 @@ static int do_meta_command(char *zLine, ShellState *p){ sqlite3_free(azName); }else - if( c=='d' && n>=3 && strncmp(azArg[0], "dbconfig", n)==0 ){ + if( c=='d' && n>=3 && cli_strncmp(azArg[0], "dbconfig", n)==0 ){ static const struct DbConfigChoices { const char *zName; int op; @@ -19564,7 +22341,7 @@ static int do_meta_command(char *zLine, ShellState *p){ int ii, v; open_db(p, 0); for(ii=0; ii1 && strcmp(azArg[1], aDbConfig[ii].zName)!=0 ) continue; + if( nArg>1 && cli_strcmp(azArg[1], aDbConfig[ii].zName)!=0 ) continue; if( nArg>=3 ){ sqlite3_db_config(p->db, aDbConfig[ii].op, booleanValue(azArg[2]), 0); } @@ -19578,18 +22355,18 @@ static int do_meta_command(char *zLine, ShellState *p){ } }else -#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_DBPAGE_VTAB) - if( c=='d' && n>=3 && strncmp(azArg[0], "dbinfo", n)==0 ){ +#if SQLITE_SHELL_HAVE_RECOVER + if( c=='d' && n>=3 && cli_strncmp(azArg[0], "dbinfo", n)==0 ){ rc = shell_dbinfo_command(p, nArg, azArg); }else - if( c=='r' && strncmp(azArg[0], "recover", n)==0 ){ + if( c=='r' && cli_strncmp(azArg[0], "recover", n)==0 ){ open_db(p, 0); rc = recoverDatabaseCmd(p, nArg, azArg); }else -#endif /* !(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_DBPAGE_VTAB) */ +#endif /* SQLITE_SHELL_HAVE_RECOVER */ - if( c=='d' && strncmp(azArg[0], "dump", n)==0 ){ + if( c=='d' && cli_strncmp(azArg[0], "dump", n)==0 ){ char *zLike = 0; char *zSql; int i; @@ -19602,7 +22379,7 @@ static int do_meta_command(char *zLine, ShellState *p){ if( azArg[i][0]=='-' ){ const char *z = azArg[i]+1; if( z[0]=='-' ) z++; - if( strcmp(z,"preserve-rowids")==0 ){ + if( cli_strcmp(z,"preserve-rowids")==0 ){ #ifdef SQLITE_OMIT_VIRTUALTABLE raw_printf(stderr, "The --preserve-rowids option is not compatible" " with SQLITE_OMIT_VIRTUALTABLE\n"); @@ -19613,13 +22390,13 @@ static int do_meta_command(char *zLine, ShellState *p){ ShellSetFlag(p, SHFLG_PreserveRowid); #endif }else - if( strcmp(z,"newlines")==0 ){ + if( cli_strcmp(z,"newlines")==0 ){ ShellSetFlag(p, SHFLG_Newlines); }else - if( strcmp(z,"data-only")==0 ){ + if( cli_strcmp(z,"data-only")==0 ){ ShellSetFlag(p, SHFLG_DumpDataOnly); }else - if( strcmp(z,"nosys")==0 ){ + if( cli_strcmp(z,"nosys")==0 ){ ShellSetFlag(p, SHFLG_DumpNoSys); }else { @@ -19701,7 +22478,7 @@ static int do_meta_command(char *zLine, ShellState *p){ p->shellFlgs = savedShellFlags; }else - if( c=='e' && strncmp(azArg[0], "echo", n)==0 ){ + if( c=='e' && cli_strncmp(azArg[0], "echo", n)==0 ){ if( nArg==2 ){ setOrClearFlag(p, SHFLG_Echo, azArg[1]); }else{ @@ -19710,22 +22487,22 @@ static int do_meta_command(char *zLine, ShellState *p){ } }else - if( c=='e' && strncmp(azArg[0], "eqp", n)==0 ){ + if( c=='e' && cli_strncmp(azArg[0], "eqp", n)==0 ){ if( nArg==2 ){ p->autoEQPtest = 0; if( p->autoEQPtrace ){ if( p->db ) sqlite3_exec(p->db, "PRAGMA vdbe_trace=OFF;", 0, 0, 0); p->autoEQPtrace = 0; } - if( strcmp(azArg[1],"full")==0 ){ + if( cli_strcmp(azArg[1],"full")==0 ){ p->autoEQP = AUTOEQP_full; - }else if( strcmp(azArg[1],"trigger")==0 ){ + }else if( cli_strcmp(azArg[1],"trigger")==0 ){ p->autoEQP = AUTOEQP_trigger; #ifdef SQLITE_DEBUG - }else if( strcmp(azArg[1],"test")==0 ){ + }else if( cli_strcmp(azArg[1],"test")==0 ){ p->autoEQP = AUTOEQP_on; p->autoEQPtest = 1; - }else if( strcmp(azArg[1],"trace")==0 ){ + }else if( cli_strcmp(azArg[1],"trace")==0 ){ p->autoEQP = AUTOEQP_full; p->autoEQPtrace = 1; open_db(p, 0); @@ -19741,8 +22518,8 @@ static int do_meta_command(char *zLine, ShellState *p){ } }else -#ifndef SQLITE_SHELL_WASM_MODE - if( c=='e' && strncmp(azArg[0], "exit", n)==0 ){ +#ifndef SQLITE_SHELL_FIDDLE + if( c=='e' && cli_strncmp(azArg[0], "exit", n)==0 ){ if( nArg>1 && (rc = (int)integerValue(azArg[1]))!=0 ) exit(rc); rc = 2; }else @@ -19750,10 +22527,10 @@ static int do_meta_command(char *zLine, ShellState *p){ /* The ".explain" command is automatic now. It is largely pointless. It ** retained purely for backwards compatibility */ - if( c=='e' && strncmp(azArg[0], "explain", n)==0 ){ + if( c=='e' && cli_strncmp(azArg[0], "explain", n)==0 ){ int val = 1; if( nArg>=2 ){ - if( strcmp(azArg[1],"auto")==0 ){ + if( cli_strcmp(azArg[1],"auto")==0 ){ val = 99; }else{ val = booleanValue(azArg[1]); @@ -19773,7 +22550,7 @@ static int do_meta_command(char *zLine, ShellState *p){ }else #ifndef SQLITE_OMIT_VIRTUALTABLE - if( c=='e' && strncmp(azArg[0], "expert", n)==0 ){ + if( c=='e' && cli_strncmp(azArg[0], "expert", n)==0 ){ if( p->bSafeMode ){ raw_printf(stderr, "Cannot run experimental commands such as \"%s\" in safe mode\n", @@ -19786,7 +22563,7 @@ static int do_meta_command(char *zLine, ShellState *p){ }else #endif - if( c=='f' && strncmp(azArg[0], "filectrl", n)==0 ){ + if( c=='f' && cli_strncmp(azArg[0], "filectrl", n)==0 ){ static const struct { const char *zCtrlName; /* Name of a test-control option */ int ctrlCode; /* Integer code for that option */ @@ -19816,7 +22593,7 @@ static int do_meta_command(char *zLine, ShellState *p){ zCmd = nArg>=2 ? azArg[1] : "help"; if( zCmd[0]=='-' - && (strcmp(zCmd,"--schema")==0 || strcmp(zCmd,"-schema")==0) + && (cli_strcmp(zCmd,"--schema")==0 || cli_strcmp(zCmd,"-schema")==0) && nArg>=4 ){ zSchema = azArg[2]; @@ -19832,7 +22609,7 @@ static int do_meta_command(char *zLine, ShellState *p){ } /* --help lists all file-controls */ - if( strcmp(zCmd,"help")==0 ){ + if( cli_strcmp(zCmd,"help")==0 ){ utf8_printf(p->out, "Available file-controls:\n"); for(i=0; iout, " .filectrl %s %s\n", @@ -19846,7 +22623,7 @@ static int do_meta_command(char *zLine, ShellState *p){ ** of the option name, or a numerical value. */ n2 = strlen30(zCmd); for(i=0; ishowHeader = booleanValue(azArg[1]); p->shellFlgs |= SHFLG_HeaderSet; @@ -19990,7 +22767,7 @@ static int do_meta_command(char *zLine, ShellState *p){ } }else - if( c=='h' && strncmp(azArg[0], "help", n)==0 ){ + if( c=='h' && cli_strncmp(azArg[0], "help", n)==0 ){ if( nArg>=2 ){ n = showHelp(p->out, azArg[1]); if( n==0 ){ @@ -20001,8 +22778,8 @@ static int do_meta_command(char *zLine, ShellState *p){ } }else -#ifndef SQLITE_SHELL_WASM_MODE - if( c=='i' && strncmp(azArg[0], "import", n)==0 ){ +#ifndef SQLITE_SHELL_FIDDLE + if( c=='i' && cli_strncmp(azArg[0], "import", n)==0 ){ char *zTable = 0; /* Insert data into this table */ char *zSchema = 0; /* within this schema (may default to "main") */ char *zFile = 0; /* Name of file to extra content from */ @@ -20042,18 +22819,18 @@ static int do_meta_command(char *zLine, ShellState *p){ showHelp(p->out, "import"); goto meta_command_exit; } - }else if( strcmp(z,"-v")==0 ){ + }else if( cli_strcmp(z,"-v")==0 ){ eVerbose++; - }else if( strcmp(z,"-schema")==0 && imode==MODE_Csv && strcmp(p->rowSeparator,SEP_CrLf)==0 ){ + if( nSep==2 && p->mode==MODE_Csv + && cli_strcmp(p->rowSeparator,SEP_CrLf)==0 + ){ /* When importing CSV (only), if the row separator is set to the ** default output row separator, change it to the default input ** row separator. This avoids having to maintain different input @@ -20292,10 +23071,10 @@ static int do_meta_command(char *zLine, ShellState *p){ sCtx.nRow, sCtx.nErr, sCtx.nLine-1); } }else -#endif /* !defined(SQLITE_SHELL_WASM_MODE) */ +#endif /* !defined(SQLITE_SHELL_FIDDLE) */ #ifndef SQLITE_UNTESTABLE - if( c=='i' && strncmp(azArg[0], "imposter", n)==0 ){ + if( c=='i' && cli_strncmp(azArg[0], "imposter", n)==0 ){ char *zSql; char *zCollist = 0; sqlite3_stmt *pStmt; @@ -20396,13 +23175,13 @@ static int do_meta_command(char *zLine, ShellState *p){ #endif /* !defined(SQLITE_OMIT_TEST_CONTROL) */ #ifdef SQLITE_ENABLE_IOTRACE - if( c=='i' && strncmp(azArg[0], "iotrace", n)==0 ){ + if( c=='i' && cli_strncmp(azArg[0], "iotrace", n)==0 ){ SQLITE_API extern void (SQLITE_CDECL *sqlite3IoTrace)(const char*, ...); if( iotrace && iotrace!=stdout ) fclose(iotrace); iotrace = 0; if( nArg<2 ){ sqlite3IoTrace = 0; - }else if( strcmp(azArg[1], "-")==0 ){ + }else if( cli_strcmp(azArg[1], "-")==0 ){ sqlite3IoTrace = iotracePrintf; iotrace = stdout; }else{ @@ -20418,7 +23197,7 @@ static int do_meta_command(char *zLine, ShellState *p){ }else #endif - if( c=='l' && n>=5 && strncmp(azArg[0], "limits", n)==0 ){ + if( c=='l' && n>=5 && cli_strncmp(azArg[0], "limits", n)==0 ){ static const struct { const char *zLimitName; /* Name of a limit */ int limitCode; /* Integer code for that limit */ @@ -20477,13 +23256,13 @@ static int do_meta_command(char *zLine, ShellState *p){ } }else - if( c=='l' && n>2 && strncmp(azArg[0], "lint", n)==0 ){ + if( c=='l' && n>2 && cli_strncmp(azArg[0], "lint", n)==0 ){ open_db(p, 0); lintDotCommand(p, azArg, nArg); }else -#if !defined(SQLITE_OMIT_LOAD_EXTENSION) && !defined(SQLITE_SHELL_WASM_MODE) - if( c=='l' && strncmp(azArg[0], "load", n)==0 ){ +#if !defined(SQLITE_OMIT_LOAD_EXTENSION) && !defined(SQLITE_SHELL_FIDDLE) + if( c=='l' && cli_strncmp(azArg[0], "load", n)==0 ){ const char *zFile, *zProc; char *zErrMsg = 0; failIfSafeMode(p, "cannot run .load in safe mode"); @@ -20504,8 +23283,8 @@ static int do_meta_command(char *zLine, ShellState *p){ }else #endif -#ifndef SQLITE_SHELL_WASM_MODE - if( c=='l' && strncmp(azArg[0], "log", n)==0 ){ +#ifndef SQLITE_SHELL_FIDDLE + if( c=='l' && cli_strncmp(azArg[0], "log", n)==0 ){ failIfSafeMode(p, "cannot run .log in safe mode"); if( nArg!=2 ){ raw_printf(stderr, "Usage: .log FILENAME\n"); @@ -20518,7 +23297,7 @@ static int do_meta_command(char *zLine, ShellState *p){ }else #endif - if( c=='m' && strncmp(azArg[0], "mode", n)==0 ){ + if( c=='m' && cli_strncmp(azArg[0], "mode", n)==0 ){ const char *zMode = 0; const char *zTabname = 0; int i, n2; @@ -20537,10 +23316,10 @@ static int do_meta_command(char *zLine, ShellState *p){ cmOpts.bQuote = 0; }else if( zMode==0 ){ zMode = z; - /* Apply defaults for qbox pseudo-mods. If that + /* Apply defaults for qbox pseudo-mode. If that * overwrites already-set values, user was informed of this. */ - if( strcmp(z, "qbox")==0 ){ + if( cli_strcmp(z, "qbox")==0 ){ ColModeOpts cmo = ColModeOpts_default_qbox; zMode = "box"; cmOpts = cmo; @@ -20579,58 +23358,58 @@ static int do_meta_command(char *zLine, ShellState *p){ zMode = modeDescr[p->mode]; } n2 = strlen30(zMode); - if( strncmp(zMode,"lines",n2)==0 ){ + if( cli_strncmp(zMode,"lines",n2)==0 ){ p->mode = MODE_Line; sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Row); - }else if( strncmp(zMode,"columns",n2)==0 ){ + }else if( cli_strncmp(zMode,"columns",n2)==0 ){ p->mode = MODE_Column; if( (p->shellFlgs & SHFLG_HeaderSet)==0 ){ p->showHeader = 1; } sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Row); p->cmOpts = cmOpts; - }else if( strncmp(zMode,"list",n2)==0 ){ + }else if( cli_strncmp(zMode,"list",n2)==0 ){ p->mode = MODE_List; sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Column); sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Row); - }else if( strncmp(zMode,"html",n2)==0 ){ + }else if( cli_strncmp(zMode,"html",n2)==0 ){ p->mode = MODE_Html; - }else if( strncmp(zMode,"tcl",n2)==0 ){ + }else if( cli_strncmp(zMode,"tcl",n2)==0 ){ p->mode = MODE_Tcl; sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Space); sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Row); - }else if( strncmp(zMode,"csv",n2)==0 ){ + }else if( cli_strncmp(zMode,"csv",n2)==0 ){ p->mode = MODE_Csv; sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Comma); sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_CrLf); - }else if( strncmp(zMode,"tabs",n2)==0 ){ + }else if( cli_strncmp(zMode,"tabs",n2)==0 ){ p->mode = MODE_List; sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Tab); - }else if( strncmp(zMode,"insert",n2)==0 ){ + }else if( cli_strncmp(zMode,"insert",n2)==0 ){ p->mode = MODE_Insert; set_table_name(p, zTabname ? zTabname : "table"); - }else if( strncmp(zMode,"quote",n2)==0 ){ + }else if( cli_strncmp(zMode,"quote",n2)==0 ){ p->mode = MODE_Quote; sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Comma); sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Row); - }else if( strncmp(zMode,"ascii",n2)==0 ){ + }else if( cli_strncmp(zMode,"ascii",n2)==0 ){ p->mode = MODE_Ascii; sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Unit); sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Record); - }else if( strncmp(zMode,"markdown",n2)==0 ){ + }else if( cli_strncmp(zMode,"markdown",n2)==0 ){ p->mode = MODE_Markdown; p->cmOpts = cmOpts; - }else if( strncmp(zMode,"table",n2)==0 ){ + }else if( cli_strncmp(zMode,"table",n2)==0 ){ p->mode = MODE_Table; p->cmOpts = cmOpts; - }else if( strncmp(zMode,"box",n2)==0 ){ + }else if( cli_strncmp(zMode,"box",n2)==0 ){ p->mode = MODE_Box; p->cmOpts = cmOpts; - }else if( strncmp(zMode,"count",n2)==0 ){ + }else if( cli_strncmp(zMode,"count",n2)==0 ){ p->mode = MODE_Count; - }else if( strncmp(zMode,"off",n2)==0 ){ + }else if( cli_strncmp(zMode,"off",n2)==0 ){ p->mode = MODE_Off; - }else if( strncmp(zMode,"json",n2)==0 ){ + }else if( cli_strncmp(zMode,"json",n2)==0 ){ p->mode = MODE_Json; }else{ raw_printf(stderr, "Error: mode should be one of: " @@ -20641,12 +23420,12 @@ static int do_meta_command(char *zLine, ShellState *p){ p->cMode = p->mode; }else -#ifndef SQLITE_SHELL_WASM_MODE - if( c=='n' && strcmp(azArg[0], "nonce")==0 ){ +#ifndef SQLITE_SHELL_FIDDLE + if( c=='n' && cli_strcmp(azArg[0], "nonce")==0 ){ if( nArg!=2 ){ raw_printf(stderr, "Usage: .nonce NONCE\n"); rc = 1; - }else if( p->zNonce==0 || strcmp(azArg[1],p->zNonce)!=0 ){ + }else if( p->zNonce==0 || cli_strcmp(azArg[1],p->zNonce)!=0 ){ raw_printf(stderr, "line %d: incorrect nonce: \"%s\"\n", p->lineno, azArg[1]); exit(1); @@ -20656,9 +23435,9 @@ static int do_meta_command(char *zLine, ShellState *p){ ** at the end of this procedure */ } }else -#endif /* !defined(SQLITE_SHELL_WASM_MODE) */ +#endif /* !defined(SQLITE_SHELL_FIDDLE) */ - if( c=='n' && strncmp(azArg[0], "nullvalue", n)==0 ){ + if( c=='n' && cli_strncmp(azArg[0], "nullvalue", n)==0 ){ if( nArg==2 ){ sqlite3_snprintf(sizeof(p->nullValue), p->nullValue, "%.*s", (int)ArraySize(p->nullValue)-1, azArg[1]); @@ -20668,7 +23447,7 @@ static int do_meta_command(char *zLine, ShellState *p){ } }else - if( c=='o' && strncmp(azArg[0], "open", n)==0 && n>=2 ){ + if( c=='o' && cli_strncmp(azArg[0], "open", n)==0 && n>=2 ){ const char *zFN = 0; /* Pointer to constant filename */ char *zNewFilename = 0; /* Name of the database file to open */ int iName = 1; /* Index in azArg[] of the filename */ @@ -20678,7 +23457,7 @@ static int do_meta_command(char *zLine, ShellState *p){ /* Check for command-line arguments */ for(iName=1; iNameszMax = integerValue(azArg[++iName]); #endif /* SQLITE_OMIT_DESERIALIZE */ }else -#endif /* !SQLITE_SHELL_WASM_MODE */ +#endif /* !SQLITE_SHELL_FIDDLE */ if( z[0]=='-' ){ utf8_printf(stderr, "unknown option: %s\n", z); rc = 1; @@ -20728,11 +23507,11 @@ static int do_meta_command(char *zLine, ShellState *p){ /* If a filename is specified, try to open it first */ if( zFN || p->openMode==SHELL_OPEN_HEXDB ){ if( newFlag && zFN && !p->bSafeMode ) shellDeleteFile(zFN); -#ifndef SQLITE_SHELL_WASM_MODE +#ifndef SQLITE_SHELL_FIDDLE if( p->bSafeMode && p->openMode!=SHELL_OPEN_HEXDB && zFN - && strcmp(zFN,":memory:")!=0 + && cli_strcmp(zFN,":memory:")!=0 ){ failIfSafeMode(p, "cannot open disk-based database files in safe mode"); } @@ -20761,10 +23540,11 @@ static int do_meta_command(char *zLine, ShellState *p){ } }else -#ifndef SQLITE_SHELL_WASM_MODE +#ifndef SQLITE_SHELL_FIDDLE if( (c=='o' - && (strncmp(azArg[0], "output", n)==0||strncmp(azArg[0], "once", n)==0)) - || (c=='e' && n==5 && strcmp(azArg[0],"excel")==0) + && (cli_strncmp(azArg[0], "output", n)==0 + || cli_strncmp(azArg[0], "once", n)==0)) + || (c=='e' && n==5 && cli_strcmp(azArg[0],"excel")==0) ){ char *zFile = 0; int bTxtMode = 0; @@ -20778,21 +23558,21 @@ static int do_meta_command(char *zLine, ShellState *p){ if( c=='e' ){ eMode = 'x'; bOnce = 2; - }else if( strncmp(azArg[0],"once",n)==0 ){ + }else if( cli_strncmp(azArg[0],"once",n)==0 ){ bOnce = 1; } for(i=1; iout, "ERROR: unknown option: \"%s\". Usage:\n", @@ -20865,7 +23645,7 @@ static int do_meta_command(char *zLine, ShellState *p){ }else{ p->out = output_file_open(zFile, bTxtMode); if( p->out==0 ){ - if( strcmp(zFile,"off")!=0 ){ + if( cli_strcmp(zFile,"off")!=0 ){ utf8_printf(stderr,"Error: cannot write to \"%s\"\n", zFile); } p->out = stdout; @@ -20877,16 +23657,16 @@ static int do_meta_command(char *zLine, ShellState *p){ } sqlite3_free(zFile); }else -#endif /* !defined(SQLITE_SHELL_WASM_MODE) */ +#endif /* !defined(SQLITE_SHELL_FIDDLE) */ - if( c=='p' && n>=3 && strncmp(azArg[0], "parameter", n)==0 ){ + if( c=='p' && n>=3 && cli_strncmp(azArg[0], "parameter", n)==0 ){ open_db(p,0); if( nArg<=1 ) goto parameter_syntax_error; /* .parameter clear ** Clear all bind parameters by dropping the TEMP table that holds them. */ - if( nArg==2 && strcmp(azArg[1],"clear")==0 ){ + if( nArg==2 && cli_strcmp(azArg[1],"clear")==0 ){ sqlite3_exec(p->db, "DROP TABLE IF EXISTS temp.sqlite_parameters;", 0, 0, 0); }else @@ -20894,7 +23674,7 @@ static int do_meta_command(char *zLine, ShellState *p){ /* .parameter list ** List all bind parameters. */ - if( nArg==2 && strcmp(azArg[1],"list")==0 ){ + if( nArg==2 && cli_strcmp(azArg[1],"list")==0 ){ sqlite3_stmt *pStmt = 0; int rx; int len = 0; @@ -20923,7 +23703,7 @@ static int do_meta_command(char *zLine, ShellState *p){ ** Make sure the TEMP table used to hold bind parameters exists. ** Create it if necessary. */ - if( nArg==2 && strcmp(azArg[1],"init")==0 ){ + if( nArg==2 && cli_strcmp(azArg[1],"init")==0 ){ bind_table_init(p); }else @@ -20933,7 +23713,7 @@ static int do_meta_command(char *zLine, ShellState *p){ ** VALUE can be in either SQL literal notation, or if not it will be ** understood to be a text string. */ - if( nArg==4 && strcmp(azArg[1],"set")==0 ){ + if( nArg==4 && cli_strcmp(azArg[1],"set")==0 ){ int rx; char *zSql; sqlite3_stmt *pStmt; @@ -20971,7 +23751,7 @@ static int do_meta_command(char *zLine, ShellState *p){ ** Remove the NAME binding from the parameter binding table, if it ** exists. */ - if( nArg==3 && strcmp(azArg[1],"unset")==0 ){ + if( nArg==3 && cli_strcmp(azArg[1],"unset")==0 ){ char *zSql = sqlite3_mprintf( "DELETE FROM temp.sqlite_parameters WHERE key=%Q", azArg[2]); shell_check_oom(zSql); @@ -20983,7 +23763,7 @@ static int do_meta_command(char *zLine, ShellState *p){ showHelp(p->out, "parameter"); }else - if( c=='p' && n>=3 && strncmp(azArg[0], "print", n)==0 ){ + if( c=='p' && n>=3 && cli_strncmp(azArg[0], "print", n)==0 ){ int i; for(i=1; i1 ) raw_printf(p->out, " "); @@ -20993,7 +23773,7 @@ static int do_meta_command(char *zLine, ShellState *p){ }else #ifndef SQLITE_OMIT_PROGRESS_CALLBACK - if( c=='p' && n>=3 && strncmp(azArg[0], "progress", n)==0 ){ + if( c=='p' && n>=3 && cli_strncmp(azArg[0], "progress", n)==0 ){ int i; int nn = 0; p->flgProgress = 0; @@ -21004,19 +23784,19 @@ static int do_meta_command(char *zLine, ShellState *p){ if( z[0]=='-' ){ z++; if( z[0]=='-' ) z++; - if( strcmp(z,"quiet")==0 || strcmp(z,"q")==0 ){ + if( cli_strcmp(z,"quiet")==0 || cli_strcmp(z,"q")==0 ){ p->flgProgress |= SHELL_PROGRESS_QUIET; continue; } - if( strcmp(z,"reset")==0 ){ + if( cli_strcmp(z,"reset")==0 ){ p->flgProgress |= SHELL_PROGRESS_RESET; continue; } - if( strcmp(z,"once")==0 ){ + if( cli_strcmp(z,"once")==0 ){ p->flgProgress |= SHELL_PROGRESS_ONCE; continue; } - if( strcmp(z,"limit")==0 ){ + if( cli_strcmp(z,"limit")==0 ){ if( i+1>=nArg ){ utf8_printf(stderr, "Error: missing argument on --limit\n"); rc = 1; @@ -21038,7 +23818,7 @@ static int do_meta_command(char *zLine, ShellState *p){ }else #endif /* SQLITE_OMIT_PROGRESS_CALLBACK */ - if( c=='p' && strncmp(azArg[0], "prompt", n)==0 ){ + if( c=='p' && cli_strncmp(azArg[0], "prompt", n)==0 ){ if( nArg >= 2) { strncpy(mainPrompt,azArg[1],(int)ArraySize(mainPrompt)-1); } @@ -21047,14 +23827,14 @@ static int do_meta_command(char *zLine, ShellState *p){ } }else -#ifndef SQLITE_SHELL_WASM_MODE - if( c=='q' && strncmp(azArg[0], "quit", n)==0 ){ +#ifndef SQLITE_SHELL_FIDDLE + if( c=='q' && cli_strncmp(azArg[0], "quit", n)==0 ){ rc = 2; }else #endif -#ifndef SQLITE_SHELL_WASM_MODE - if( c=='r' && n>=3 && strncmp(azArg[0], "read", n)==0 ){ +#ifndef SQLITE_SHELL_FIDDLE + if( c=='r' && n>=3 && cli_strncmp(azArg[0], "read", n)==0 ){ FILE *inSaved = p->in; int savedLineno = p->lineno; failIfSafeMode(p, "cannot run .read in safe mode"); @@ -21088,10 +23868,10 @@ static int do_meta_command(char *zLine, ShellState *p){ p->in = inSaved; p->lineno = savedLineno; }else -#endif /* !defined(SQLITE_SHELL_WASM_MODE) */ +#endif /* !defined(SQLITE_SHELL_FIDDLE) */ -#ifndef SQLITE_SHELL_WASM_MODE - if( c=='r' && n>=3 && strncmp(azArg[0], "restore", n)==0 ){ +#ifndef SQLITE_SHELL_FIDDLE + if( c=='r' && n>=3 && cli_strncmp(azArg[0], "restore", n)==0 ){ const char *zSrcFile; const char *zDb; sqlite3 *pSrc; @@ -21142,9 +23922,9 @@ static int do_meta_command(char *zLine, ShellState *p){ } close_db(pSrc); }else -#endif /* !defined(SQLITE_SHELL_WASM_MODE) */ +#endif /* !defined(SQLITE_SHELL_FIDDLE) */ - if( c=='s' && strncmp(azArg[0], "scanstats", n)==0 ){ + if( c=='s' && cli_strncmp(azArg[0], "scanstats", n)==0 ){ if( nArg==2 ){ p->scanstatsOn = (u8)booleanValue(azArg[1]); #ifndef SQLITE_ENABLE_STMT_SCANSTATUS @@ -21156,7 +23936,7 @@ static int do_meta_command(char *zLine, ShellState *p){ } }else - if( c=='s' && strncmp(azArg[0], "schema", n)==0 ){ + if( c=='s' && cli_strncmp(azArg[0], "schema", n)==0 ){ ShellText sSelect; ShellState data; char *zErrMsg = 0; @@ -21299,15 +24079,15 @@ static int do_meta_command(char *zLine, ShellState *p){ } }else - if( (c=='s' && n==11 && strncmp(azArg[0], "selecttrace", n)==0) - || (c=='t' && n==9 && strncmp(azArg[0], "treetrace", n)==0) + if( (c=='s' && n==11 && cli_strncmp(azArg[0], "selecttrace", n)==0) + || (c=='t' && n==9 && cli_strncmp(azArg[0], "treetrace", n)==0) ){ unsigned int x = nArg>=2 ? (unsigned int)integerValue(azArg[1]) : 0xffffffff; sqlite3_test_control(SQLITE_TESTCTRL_TRACEFLAGS, 1, &x); }else #if defined(SQLITE_ENABLE_SESSION) - if( c=='s' && strncmp(azArg[0],"session",n)==0 && n>=3 ){ + if( c=='s' && cli_strncmp(azArg[0],"session",n)==0 && n>=3 ){ struct AuxDb *pAuxDb = p->pAuxDb; OpenSession *pSession = &pAuxDb->aSession[0]; char **azCmd = &azArg[1]; @@ -21318,7 +24098,7 @@ static int do_meta_command(char *zLine, ShellState *p){ open_db(p, 0); if( nArg>=3 ){ for(iSes=0; iSesnSession; iSes++){ - if( strcmp(pAuxDb->aSession[iSes].zName, azArg[1])==0 ) break; + if( cli_strcmp(pAuxDb->aSession[iSes].zName, azArg[1])==0 ) break; } if( iSesnSession ){ pSession = &pAuxDb->aSession[iSes]; @@ -21334,7 +24114,7 @@ static int do_meta_command(char *zLine, ShellState *p){ ** Invoke the sqlite3session_attach() interface to attach a particular ** table so that it is never filtered. */ - if( strcmp(azCmd[0],"attach")==0 ){ + if( cli_strcmp(azCmd[0],"attach")==0 ){ if( nCmd!=2 ) goto session_syntax_error; if( pSession->p==0 ){ session_not_open: @@ -21352,7 +24132,9 @@ static int do_meta_command(char *zLine, ShellState *p){ ** .session patchset FILE ** Write a changeset or patchset into a file. The file is overwritten. */ - if( strcmp(azCmd[0],"changeset")==0 || strcmp(azCmd[0],"patchset")==0 ){ + if( cli_strcmp(azCmd[0],"changeset")==0 + || cli_strcmp(azCmd[0],"patchset")==0 + ){ FILE *out = 0; failIfSafeMode(p, "cannot run \".session %s\" in safe mode", azCmd[0]); if( nCmd!=2 ) goto session_syntax_error; @@ -21386,7 +24168,7 @@ static int do_meta_command(char *zLine, ShellState *p){ /* .session close ** Close the identified session */ - if( strcmp(azCmd[0], "close")==0 ){ + if( cli_strcmp(azCmd[0], "close")==0 ){ if( nCmd!=1 ) goto session_syntax_error; if( pAuxDb->nSession ){ session_close(pSession); @@ -21397,7 +24179,7 @@ static int do_meta_command(char *zLine, ShellState *p){ /* .session enable ?BOOLEAN? ** Query or set the enable flag */ - if( strcmp(azCmd[0], "enable")==0 ){ + if( cli_strcmp(azCmd[0], "enable")==0 ){ int ii; if( nCmd>2 ) goto session_syntax_error; ii = nCmd==1 ? -1 : booleanValue(azCmd[1]); @@ -21411,7 +24193,7 @@ static int do_meta_command(char *zLine, ShellState *p){ /* .session filter GLOB .... ** Set a list of GLOB patterns of table names to be excluded. */ - if( strcmp(azCmd[0], "filter")==0 ){ + if( cli_strcmp(azCmd[0], "filter")==0 ){ int ii, nByte; if( nCmd<2 ) goto session_syntax_error; if( pAuxDb->nSession ){ @@ -21436,7 +24218,7 @@ static int do_meta_command(char *zLine, ShellState *p){ /* .session indirect ?BOOLEAN? ** Query or set the indirect flag */ - if( strcmp(azCmd[0], "indirect")==0 ){ + if( cli_strcmp(azCmd[0], "indirect")==0 ){ int ii; if( nCmd>2 ) goto session_syntax_error; ii = nCmd==1 ? -1 : booleanValue(azCmd[1]); @@ -21450,7 +24232,7 @@ static int do_meta_command(char *zLine, ShellState *p){ /* .session isempty ** Determine if the session is empty */ - if( strcmp(azCmd[0], "isempty")==0 ){ + if( cli_strcmp(azCmd[0], "isempty")==0 ){ int ii; if( nCmd!=1 ) goto session_syntax_error; if( pAuxDb->nSession ){ @@ -21463,7 +24245,7 @@ static int do_meta_command(char *zLine, ShellState *p){ /* .session list ** List all currently open sessions */ - if( strcmp(azCmd[0],"list")==0 ){ + if( cli_strcmp(azCmd[0],"list")==0 ){ for(i=0; inSession; i++){ utf8_printf(p->out, "%d %s\n", i, pAuxDb->aSession[i].zName); } @@ -21473,13 +24255,13 @@ static int do_meta_command(char *zLine, ShellState *p){ ** Open a new session called NAME on the attached database DB. ** DB is normally "main". */ - if( strcmp(azCmd[0],"open")==0 ){ + if( cli_strcmp(azCmd[0],"open")==0 ){ char *zName; if( nCmd!=3 ) goto session_syntax_error; zName = azCmd[2]; if( zName[0]==0 ) goto session_syntax_error; for(i=0; inSession; i++){ - if( strcmp(pAuxDb->aSession[i].zName,zName)==0 ){ + if( cli_strcmp(pAuxDb->aSession[i].zName,zName)==0 ){ utf8_printf(stderr, "Session \"%s\" already exists\n", zName); goto meta_command_exit; } @@ -21510,15 +24292,15 @@ static int do_meta_command(char *zLine, ShellState *p){ #ifdef SQLITE_DEBUG /* Undocumented commands for internal testing. Subject to change ** without notice. */ - if( c=='s' && n>=10 && strncmp(azArg[0], "selftest-", 9)==0 ){ - if( strncmp(azArg[0]+9, "boolean", n-9)==0 ){ + if( c=='s' && n>=10 && cli_strncmp(azArg[0], "selftest-", 9)==0 ){ + if( cli_strncmp(azArg[0]+9, "boolean", n-9)==0 ){ int i, v; for(i=1; iout, "%s: %d 0x%x\n", azArg[i], v, v); } } - if( strncmp(azArg[0]+9, "integer", n-9)==0 ){ + if( cli_strncmp(azArg[0]+9, "integer", n-9)==0 ){ int i; sqlite3_int64 v; for(i=1; i=4 && strncmp(azArg[0],"selftest",n)==0 ){ + if( c=='s' && n>=4 && cli_strncmp(azArg[0],"selftest",n)==0 ){ int bIsInit = 0; /* True to initialize the SELFTEST table */ int bVerbose = 0; /* Verbose output */ int bSelftestExists; /* True if SELFTEST already exists */ @@ -21544,10 +24326,10 @@ static int do_meta_command(char *zLine, ShellState *p){ for(i=1; i0 ){ printf("%d: %s %s\n", tno, zOp, zSql); } - if( strcmp(zOp,"memo")==0 ){ + if( cli_strcmp(zOp,"memo")==0 ){ utf8_printf(p->out, "%s\n", zSql); }else - if( strcmp(zOp,"run")==0 ){ + if( cli_strcmp(zOp,"run")==0 ){ char *zErrMsg = 0; str.n = 0; str.z[0] = 0; @@ -21617,7 +24399,7 @@ static int do_meta_command(char *zLine, ShellState *p){ rc = 1; utf8_printf(p->out, "%d: error-code-%d: %s\n", tno, rc, zErrMsg); sqlite3_free(zErrMsg); - }else if( strcmp(zAns,str.z)!=0 ){ + }else if( cli_strcmp(zAns,str.z)!=0 ){ nErr++; rc = 1; utf8_printf(p->out, "%d: Expected: [%s]\n", tno, zAns); @@ -21637,7 +24419,7 @@ static int do_meta_command(char *zLine, ShellState *p){ utf8_printf(p->out, "%d errors out of %d tests\n", nErr, nTest); }else - if( c=='s' && strncmp(azArg[0], "separator", n)==0 ){ + if( c=='s' && cli_strncmp(azArg[0], "separator", n)==0 ){ if( nArg<2 || nArg>3 ){ raw_printf(stderr, "Usage: .separator COL ?ROW?\n"); rc = 1; @@ -21652,7 +24434,7 @@ static int do_meta_command(char *zLine, ShellState *p){ } }else - if( c=='s' && n>=4 && strncmp(azArg[0],"sha3sum",n)==0 ){ + if( c=='s' && n>=4 && cli_strncmp(azArg[0],"sha3sum",n)==0 ){ const char *zLike = 0; /* Which table to checksum. 0 means everything */ int i; /* Loop counter */ int bSchema = 0; /* Also hash the schema */ @@ -21670,15 +24452,15 @@ static int do_meta_command(char *zLine, ShellState *p){ if( z[0]=='-' ){ z++; if( z[0]=='-' ) z++; - if( strcmp(z,"schema")==0 ){ + if( cli_strcmp(z,"schema")==0 ){ bSchema = 1; }else - if( strcmp(z,"sha3-224")==0 || strcmp(z,"sha3-256")==0 - || strcmp(z,"sha3-384")==0 || strcmp(z,"sha3-512")==0 + if( cli_strcmp(z,"sha3-224")==0 || cli_strcmp(z,"sha3-256")==0 + || cli_strcmp(z,"sha3-384")==0 || cli_strcmp(z,"sha3-512")==0 ){ iSize = atoi(&z[5]); }else - if( strcmp(z,"debug")==0 ){ + if( cli_strcmp(z,"debug")==0 ){ bDebug = 1; }else { @@ -21718,20 +24500,20 @@ static int do_meta_command(char *zLine, ShellState *p){ const char *zTab = (const char*)sqlite3_column_text(pStmt,0); if( zTab==0 ) continue; if( zLike && sqlite3_strlike(zLike, zTab, 0)!=0 ) continue; - if( strncmp(zTab, "sqlite_",7)!=0 ){ + if( cli_strncmp(zTab, "sqlite_",7)!=0 ){ appendText(&sQuery,"SELECT * FROM ", 0); appendText(&sQuery,zTab,'"'); appendText(&sQuery," NOT INDEXED;", 0); - }else if( strcmp(zTab, "sqlite_schema")==0 ){ + }else if( cli_strcmp(zTab, "sqlite_schema")==0 ){ appendText(&sQuery,"SELECT type,name,tbl_name,sql FROM sqlite_schema" " ORDER BY name;", 0); - }else if( strcmp(zTab, "sqlite_sequence")==0 ){ + }else if( cli_strcmp(zTab, "sqlite_sequence")==0 ){ appendText(&sQuery,"SELECT name,seq FROM sqlite_sequence" " ORDER BY name;", 0); - }else if( strcmp(zTab, "sqlite_stat1")==0 ){ + }else if( cli_strcmp(zTab, "sqlite_stat1")==0 ){ appendText(&sQuery,"SELECT tbl,idx,stat FROM sqlite_stat1" " ORDER BY tbl,idx;", 0); - }else if( strcmp(zTab, "sqlite_stat4")==0 ){ + }else if( cli_strcmp(zTab, "sqlite_stat4")==0 ){ appendText(&sQuery, "SELECT * FROM ", 0); appendText(&sQuery, zTab, 0); appendText(&sQuery, " ORDER BY tbl, idx, rowid;\n", 0); @@ -21768,9 +24550,10 @@ static int do_meta_command(char *zLine, ShellState *p){ sqlite3_free(zSql); }else -#if !defined(SQLITE_NOHAVE_SYSTEM) && !defined(SQLITE_SHELL_WASM_MODE) +#if !defined(SQLITE_NOHAVE_SYSTEM) && !defined(SQLITE_SHELL_FIDDLE) if( c=='s' - && (strncmp(azArg[0], "shell", n)==0 || strncmp(azArg[0],"system",n)==0) + && (cli_strncmp(azArg[0], "shell", n)==0 + || cli_strncmp(azArg[0],"system",n)==0) ){ char *zCmd; int i, x; @@ -21789,9 +24572,9 @@ static int do_meta_command(char *zLine, ShellState *p){ sqlite3_free(zCmd); if( x ) raw_printf(stderr, "System command returns %d\n", x); }else -#endif /* !defined(SQLITE_NOHAVE_SYSTEM) && !defined(SQLITE_SHELL_WASM_MODE) */ +#endif /* !defined(SQLITE_NOHAVE_SYSTEM) && !defined(SQLITE_SHELL_FIDDLE) */ - if( c=='s' && strncmp(azArg[0], "show", n)==0 ){ + if( c=='s' && cli_strncmp(azArg[0], "show", n)==0 ){ static const char *azBool[] = { "off", "on", "trigger", "full"}; const char *zOut; int i; @@ -21844,11 +24627,11 @@ static int do_meta_command(char *zLine, ShellState *p){ p->pAuxDb->zDbFilename ? p->pAuxDb->zDbFilename : ""); }else - if( c=='s' && strncmp(azArg[0], "stats", n)==0 ){ + if( c=='s' && cli_strncmp(azArg[0], "stats", n)==0 ){ if( nArg==2 ){ - if( strcmp(azArg[1],"stmt")==0 ){ + if( cli_strcmp(azArg[1],"stmt")==0 ){ p->statsOn = 2; - }else if( strcmp(azArg[1],"vmstep")==0 ){ + }else if( cli_strcmp(azArg[1],"vmstep")==0 ){ p->statsOn = 3; }else{ p->statsOn = (u8)booleanValue(azArg[1]); @@ -21861,9 +24644,9 @@ static int do_meta_command(char *zLine, ShellState *p){ } }else - if( (c=='t' && n>1 && strncmp(azArg[0], "tables", n)==0) - || (c=='i' && (strncmp(azArg[0], "indices", n)==0 - || strncmp(azArg[0], "indexes", n)==0) ) + if( (c=='t' && n>1 && cli_strncmp(azArg[0], "tables", n)==0) + || (c=='i' && (cli_strncmp(azArg[0], "indices", n)==0 + || cli_strncmp(azArg[0], "indexes", n)==0) ) ){ sqlite3_stmt *pStmt; char **azResult; @@ -21969,9 +24752,9 @@ static int do_meta_command(char *zLine, ShellState *p){ sqlite3_free(azResult); }else -#ifndef SQLITE_SHELL_WASM_MODE +#ifndef SQLITE_SHELL_FIDDLE /* Begin redirecting output to the file "testcase-out.txt" */ - if( c=='t' && strcmp(azArg[0],"testcase")==0 ){ + if( c=='t' && cli_strcmp(azArg[0],"testcase")==0 ){ output_reset(p); p->out = output_file_open("testcase-out.txt", 0); if( p->out==0 ){ @@ -21983,10 +24766,10 @@ static int do_meta_command(char *zLine, ShellState *p){ sqlite3_snprintf(sizeof(p->zTestcase), p->zTestcase, "?"); } }else -#endif /* !defined(SQLITE_SHELL_WASM_MODE) */ +#endif /* !defined(SQLITE_SHELL_FIDDLE) */ #ifndef SQLITE_UNTESTABLE - if( c=='t' && n>=8 && strncmp(azArg[0], "testctrl", n)==0 ){ + if( c=='t' && n>=8 && cli_strncmp(azArg[0], "testctrl", n)==0 ){ static const struct { const char *zCtrlName; /* Name of a test-control option */ int ctrlCode; /* Integer code for that option */ @@ -22033,7 +24816,7 @@ static int do_meta_command(char *zLine, ShellState *p){ } /* --help lists all test-controls */ - if( strcmp(zCmd,"help")==0 ){ + if( cli_strcmp(zCmd,"help")==0 ){ utf8_printf(p->out, "Available test-controls:\n"); for(i=0; iout, " .testctrl %s %s\n", @@ -22047,7 +24830,7 @@ static int do_meta_command(char *zLine, ShellState *p){ ** of the option name, or a numerical value. */ n2 = strlen30(zCmd); for(i=0; i4 && strncmp(azArg[0], "timeout", n)==0 ){ + if( c=='t' && n>4 && cli_strncmp(azArg[0], "timeout", n)==0 ){ open_db(p, 0); sqlite3_busy_timeout(p->db, nArg>=2 ? (int)integerValue(azArg[1]) : 0); }else - if( c=='t' && n>=5 && strncmp(azArg[0], "timer", n)==0 ){ + if( c=='t' && n>=5 && cli_strncmp(azArg[0], "timer", n)==0 ){ if( nArg==2 ){ enableTimer = booleanValue(azArg[1]); if( enableTimer && !HAS_TIMER ){ @@ -22238,7 +25021,7 @@ static int do_meta_command(char *zLine, ShellState *p){ }else #ifndef SQLITE_OMIT_TRACE - if( c=='t' && strncmp(azArg[0], "trace", n)==0 ){ + if( c=='t' && cli_strncmp(azArg[0], "trace", n)==0 ){ int mType = 0; int jj; open_db(p, 0); @@ -22288,7 +25071,7 @@ static int do_meta_command(char *zLine, ShellState *p){ #endif /* !defined(SQLITE_OMIT_TRACE) */ #if defined(SQLITE_DEBUG) && !defined(SQLITE_OMIT_VIRTUALTABLE) - if( c=='u' && strncmp(azArg[0], "unmodule", n)==0 ){ + if( c=='u' && cli_strncmp(azArg[0], "unmodule", n)==0 ){ int ii; int lenOpt; char *zOpt; @@ -22301,7 +25084,7 @@ static int do_meta_command(char *zLine, ShellState *p){ zOpt = azArg[1]; if( zOpt[0]=='-' && zOpt[1]=='-' && zOpt[2]!=0 ) zOpt++; lenOpt = (int)strlen(zOpt); - if( lenOpt>=3 && strncmp(zOpt, "-allexcept",lenOpt)==0 ){ + if( lenOpt>=3 && cli_strncmp(zOpt, "-allexcept",lenOpt)==0 ){ assert( azArg[nArg]==0 ); sqlite3_drop_modules(p->db, nArg>2 ? (const char**)(azArg+2) : 0); }else{ @@ -22313,14 +25096,14 @@ static int do_meta_command(char *zLine, ShellState *p){ #endif #if SQLITE_USER_AUTHENTICATION - if( c=='u' && strncmp(azArg[0], "user", n)==0 ){ + if( c=='u' && cli_strncmp(azArg[0], "user", n)==0 ){ if( nArg<2 ){ raw_printf(stderr, "Usage: .user SUBCOMMAND ...\n"); rc = 1; goto meta_command_exit; } open_db(p, 0); - if( strcmp(azArg[1],"login")==0 ){ + if( cli_strcmp(azArg[1],"login")==0 ){ if( nArg!=4 ){ raw_printf(stderr, "Usage: .user login USER PASSWORD\n"); rc = 1; @@ -22332,7 +25115,7 @@ static int do_meta_command(char *zLine, ShellState *p){ utf8_printf(stderr, "Authentication failed for user %s\n", azArg[2]); rc = 1; } - }else if( strcmp(azArg[1],"add")==0 ){ + }else if( cli_strcmp(azArg[1],"add")==0 ){ if( nArg!=5 ){ raw_printf(stderr, "Usage: .user add USER PASSWORD ISADMIN\n"); rc = 1; @@ -22344,7 +25127,7 @@ static int do_meta_command(char *zLine, ShellState *p){ raw_printf(stderr, "User-Add failed: %d\n", rc); rc = 1; } - }else if( strcmp(azArg[1],"edit")==0 ){ + }else if( cli_strcmp(azArg[1],"edit")==0 ){ if( nArg!=5 ){ raw_printf(stderr, "Usage: .user edit USER PASSWORD ISADMIN\n"); rc = 1; @@ -22356,7 +25139,7 @@ static int do_meta_command(char *zLine, ShellState *p){ raw_printf(stderr, "User-Edit failed: %d\n", rc); rc = 1; } - }else if( strcmp(azArg[1],"delete")==0 ){ + }else if( cli_strcmp(azArg[1],"delete")==0 ){ if( nArg!=3 ){ raw_printf(stderr, "Usage: .user delete USER\n"); rc = 1; @@ -22375,7 +25158,7 @@ static int do_meta_command(char *zLine, ShellState *p){ }else #endif /* SQLITE_USER_AUTHENTICATION */ - if( c=='v' && strncmp(azArg[0], "version", n)==0 ){ + if( c=='v' && cli_strncmp(azArg[0], "version", n)==0 ){ utf8_printf(p->out, "SQLite %s %s\n" /*extra-version-info*/, sqlite3_libversion(), sqlite3_sourceid()); #if SQLITE_HAVE_ZLIB @@ -22394,7 +25177,7 @@ static int do_meta_command(char *zLine, ShellState *p){ #endif }else - if( c=='v' && strncmp(azArg[0], "vfsinfo", n)==0 ){ + if( c=='v' && cli_strncmp(azArg[0], "vfsinfo", n)==0 ){ const char *zDbName = nArg==2 ? azArg[1] : "main"; sqlite3_vfs *pVfs = 0; if( p->db ){ @@ -22408,7 +25191,7 @@ static int do_meta_command(char *zLine, ShellState *p){ } }else - if( c=='v' && strncmp(azArg[0], "vfslist", n)==0 ){ + if( c=='v' && cli_strncmp(azArg[0], "vfslist", n)==0 ){ sqlite3_vfs *pVfs; sqlite3_vfs *pCurrent = 0; if( p->db ){ @@ -22426,7 +25209,7 @@ static int do_meta_command(char *zLine, ShellState *p){ } }else - if( c=='v' && strncmp(azArg[0], "vfsname", n)==0 ){ + if( c=='v' && cli_strncmp(azArg[0], "vfsname", n)==0 ){ const char *zDbName = nArg==2 ? azArg[1] : "main"; char *zVfsName = 0; if( p->db ){ @@ -22438,12 +25221,12 @@ static int do_meta_command(char *zLine, ShellState *p){ } }else - if( c=='w' && strncmp(azArg[0], "wheretrace", n)==0 ){ + if( c=='w' && cli_strncmp(azArg[0], "wheretrace", n)==0 ){ unsigned int x = nArg>=2 ? (unsigned int)integerValue(azArg[1]) : 0xffffffff; sqlite3_test_control(SQLITE_TESTCTRL_TRACEFLAGS, 3, &x); }else - if( c=='w' && strncmp(azArg[0], "width", n)==0 ){ + if( c=='w' && cli_strncmp(azArg[0], "width", n)==0 ){ int j; assert( nArg<=ArraySize(azArg) ); p->nWidth = nArg-1; @@ -22621,10 +25404,10 @@ static int runOneSqlLine(ShellState *p, char *zSql, FILE *in, int startline){ if( zErrMsg==0 ){ zErrorType = "Error"; zErrorTail = sqlite3_errmsg(p->db); - }else if( strncmp(zErrMsg, "in prepare, ",12)==0 ){ + }else if( cli_strncmp(zErrMsg, "in prepare, ",12)==0 ){ zErrorType = "Parse error"; zErrorTail = &zErrMsg[12]; - }else if( strncmp(zErrMsg, "stepping, ", 10)==0 ){ + }else if( cli_strncmp(zErrMsg, "stepping, ", 10)==0 ){ zErrorType = "Runtime error"; zErrorTail = &zErrMsg[10]; }else{ @@ -22655,7 +25438,7 @@ static void echo_group_input(ShellState *p, const char *zDo){ if( ShellHasFlag(p, SHFLG_Echo) ) utf8_printf(p->out, "%s\n", zDo); } -#ifdef SQLITE_SHELL_WASM_MODE +#ifdef SQLITE_SHELL_FIDDLE /* ** Alternate one_input_line() impl for wasm mode. This is not in the primary impl ** because we need the global shellState and cannot access it from that function @@ -22666,7 +25449,7 @@ static char *one_input_line(FILE *in, char *zPrior, int isContinuation){ const char *zBegin = shellState.wasm.zPos; const char *z = zBegin; char *zLine = 0; - int nZ = 0; + i64 nZ = 0; UNUSED_PARAMETER(in); UNUSED_PARAMETER(isContinuation); @@ -22682,11 +25465,11 @@ static char *one_input_line(FILE *in, char *zPrior, int isContinuation){ shellState.wasm.zPos = z; zLine = realloc(zPrior, nZ+1); shell_check_oom(zLine); - memcpy(zLine, zBegin, (size_t)nZ); + memcpy(zLine, zBegin, nZ); zLine[nZ] = 0; return zLine; } -#endif /* SQLITE_SHELL_WASM_MODE */ +#endif /* SQLITE_SHELL_FIDDLE */ /* ** Read input from *in and process it. If *in==0 then input @@ -22700,12 +25483,12 @@ static char *one_input_line(FILE *in, char *zPrior, int isContinuation){ static int process_input(ShellState *p){ char *zLine = 0; /* A single input line */ char *zSql = 0; /* Accumulated SQL text */ - int nLine; /* Length of current line */ - int nSql = 0; /* Bytes of zSql[] used */ - int nAlloc = 0; /* Allocated zSql[] space */ + i64 nLine; /* Length of current line */ + i64 nSql = 0; /* Bytes of zSql[] used */ + i64 nAlloc = 0; /* Allocated zSql[] space */ int rc; /* Error code */ int errCnt = 0; /* Number of errors seen */ - int startline = 0; /* Line number for start of current input */ + i64 startline = 0; /* Line number for start of current input */ QuickScanState qss = QSS_Start; /* Accumulated line status (so far) */ if( p->inputNesting==MAX_INPUT_NESTING ){ @@ -22755,7 +25538,7 @@ static int process_input(ShellState *p){ continue; } /* No single-line dispositions remain; accumulate line(s). */ - nLine = strlen30(zLine); + nLine = strlen(zLine); if( nSql+nLine+2>=nAlloc ){ /* Grow buffer by half-again increments when big. */ nAlloc = nSql+(nSql>>1)+nLine+100; @@ -22763,7 +25546,7 @@ static int process_input(ShellState *p){ shell_check_oom(zSql); } if( nSql==0 ){ - int i; + i64 i; for(i=0; zLine[i] && IsSpace(zLine[i]); i++){} assert( nAlloc>0 && zSql!=0 ); memcpy(zSql, zLine+i, nLine+1-i); @@ -22863,7 +25646,7 @@ static char *find_home_dir(int clearFlag){ #endif /* !_WIN32_WCE */ if( home_dir ){ - int n = strlen30(home_dir) + 1; + i64 n = strlen(home_dir) + 1; char *z = malloc( n ); if( z ) memcpy(z, home_dir, n); home_dir = z; @@ -23069,7 +25852,7 @@ static char *cmdline_option_value(int argc, char **argv, int i){ # endif #endif -#ifdef SQLITE_SHELL_WASM_MODE +#ifdef SQLITE_SHELL_FIDDLE # define main fiddle_main #endif @@ -23083,7 +25866,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ sqlite3_int64 mem_main_enter = sqlite3_memory_used(); #endif char *zErrMsg = 0; -#ifdef SQLITE_SHELL_WASM_MODE +#ifdef SQLITE_SHELL_FIDDLE # define data shellState #else ShellState data; @@ -23103,9 +25886,10 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ setBinaryMode(stdin, 0); setvbuf(stderr, 0, _IONBF, 0); /* Make sure stderr is unbuffered */ -#ifdef SQLITE_SHELL_WASM_MODE +#ifdef SQLITE_SHELL_FIDDLE stdin_is_interactive = 0; stdout_is_console = 1; + data.wasm.zDefaultDbName = "/fiddle.sqlite3"; #else stdin_is_interactive = isatty(0); stdout_is_console = isatty(1); @@ -23133,7 +25917,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ #endif #if USE_SYSTEM_SQLITE+0!=1 - if( strncmp(sqlite3_sourceid(),SQLITE_SOURCE_ID,60)!=0 ){ + if( cli_strncmp(sqlite3_sourceid(),SQLITE_SOURCE_ID,60)!=0 ){ utf8_printf(stderr, "SQLite header and source version mismatch\n%s\n%s\n", sqlite3_sourceid(), SQLITE_SOURCE_ID); exit(1); @@ -23155,9 +25939,9 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ argv = argvToFree + argc; for(i=0; i70000 ) sz = 70000; @@ -23252,7 +26036,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ sqlite3_config(SQLITE_CONFIG_PAGECACHE, (n>0 && sz>0) ? malloc(n*sz) : 0, sz, n); data.shellFlgs |= SHFLG_Pagecache; - }else if( strcmp(z,"-lookaside")==0 ){ + }else if( cli_strcmp(z,"-lookaside")==0 ){ int n, sz; sz = (int)integerValue(cmdline_option_value(argc,argv,++i)); if( sz<0 ) sz = 0; @@ -23260,7 +26044,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ if( n<0 ) n = 0; sqlite3_config(SQLITE_CONFIG_LOOKASIDE, sz, n); if( sz*n==0 ) data.shellFlgs &= ~SHFLG_Lookaside; - }else if( strcmp(z,"-threadsafe")==0 ){ + }else if( cli_strcmp(z,"-threadsafe")==0 ){ int n; n = (int)integerValue(cmdline_option_value(argc,argv,++i)); switch( n ){ @@ -23269,7 +26053,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ default: sqlite3_config(SQLITE_CONFIG_SERIALIZED); break; } #ifdef SQLITE_ENABLE_VFSTRACE - }else if( strcmp(z,"-vfstrace")==0 ){ + }else if( cli_strcmp(z,"-vfstrace")==0 ){ extern int vfstrace_register( const char *zTraceName, const char *zOldVfsName, @@ -23280,50 +26064,50 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ vfstrace_register("trace",0,(int(*)(const char*,void*))fputs,stderr,1); #endif #ifdef SQLITE_ENABLE_MULTIPLEX - }else if( strcmp(z,"-multiplex")==0 ){ + }else if( cli_strcmp(z,"-multiplex")==0 ){ extern int sqlite3_multiple_initialize(const char*,int); sqlite3_multiplex_initialize(0, 1); #endif - }else if( strcmp(z,"-mmap")==0 ){ + }else if( cli_strcmp(z,"-mmap")==0 ){ sqlite3_int64 sz = integerValue(cmdline_option_value(argc,argv,++i)); sqlite3_config(SQLITE_CONFIG_MMAP_SIZE, sz, sz); #ifdef SQLITE_ENABLE_SORTER_REFERENCES - }else if( strcmp(z,"-sorterref")==0 ){ + }else if( cli_strcmp(z,"-sorterref")==0 ){ sqlite3_int64 sz = integerValue(cmdline_option_value(argc,argv,++i)); sqlite3_config(SQLITE_CONFIG_SORTERREF_SIZE, (int)sz); #endif - }else if( strcmp(z,"-vfs")==0 ){ + }else if( cli_strcmp(z,"-vfs")==0 ){ zVfs = cmdline_option_value(argc, argv, ++i); #ifdef SQLITE_HAVE_ZLIB - }else if( strcmp(z,"-zip")==0 ){ + }else if( cli_strcmp(z,"-zip")==0 ){ data.openMode = SHELL_OPEN_ZIPFILE; #endif - }else if( strcmp(z,"-append")==0 ){ + }else if( cli_strcmp(z,"-append")==0 ){ data.openMode = SHELL_OPEN_APPENDVFS; #ifndef SQLITE_OMIT_DESERIALIZE - }else if( strcmp(z,"-deserialize")==0 ){ + }else if( cli_strcmp(z,"-deserialize")==0 ){ data.openMode = SHELL_OPEN_DESERIALIZE; - }else if( strcmp(z,"-maxsize")==0 && i+10 ){ utf8_printf(stderr, "Error: cannot mix regular SQL or dot-commands" " with \"%s\"\n", z); @@ -23551,7 +26335,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ readStdin = 0; break; #endif - }else if( strcmp(z,"-safe")==0 ){ + }else if( cli_strcmp(z,"-safe")==0 ){ data.bSafeMode = data.bSafeModePersist = 1; }else{ utf8_printf(stderr,"%s: Error: unknown option: %s\n", Argv0, z); @@ -23634,7 +26418,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ rc = process_input(&data); } } -#ifndef SQLITE_SHELL_WASM_MODE +#ifndef SQLITE_SHELL_FIDDLE /* In WASM mode we have to leave the db state in place so that ** client code can "push" SQL into it after this call returns. */ free(azCmd); @@ -23669,38 +26453,38 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ (unsigned int)(sqlite3_memory_used()-mem_main_enter)); } #endif -#endif /* !SQLITE_SHELL_WASM_MODE */ +#endif /* !SQLITE_SHELL_FIDDLE */ return rc; } -#ifdef SQLITE_SHELL_WASM_MODE +#ifdef SQLITE_SHELL_FIDDLE /* Only for emcc experimentation purposes. */ int fiddle_experiment(int a,int b){ - return a + b; + return a + b; } -/* Only for emcc experimentation purposes. - - Define this function in JS using: - - emcc ... --js-library somefile.js - - containing: - -mergeInto(LibraryManager.library, { - my_foo: function(){ - console.debug("my_foo()",arguments); - } -}); +/* +** Returns a pointer to the current DB handle. */ -/*extern void my_foo(sqlite3 *);*/ -/* Only for emcc experimentation purposes. */ -sqlite3 * fiddle_the_db(){ - printf("fiddle_the_db(%p)\n", (const void*)globalDb); - /*my_foo(globalDb);*/ - return globalDb; +sqlite3 * fiddle_db_handle(){ + return globalDb; } + +/* +** Returns a pointer to the given DB name's VFS. If zDbName is 0 then +** "main" is assumed. Returns 0 if no db with the given name is +** open. +*/ +sqlite3_vfs * fiddle_db_vfs(const char *zDbName){ + sqlite3_vfs * pVfs = 0; + if(globalDb){ + sqlite3_file_control(globalDb, zDbName ? zDbName : "main", + SQLITE_FCNTL_VFS_POINTER, &pVfs); + } + return pVfs; +} + /* Only for emcc experimentation purposes. */ sqlite3 * fiddle_db_arg(sqlite3 *arg){ printf("fiddle_db_arg(%p)\n", (const void*)arg); @@ -23714,7 +26498,7 @@ sqlite3 * fiddle_db_arg(sqlite3 *arg){ ** portable enough to make real use of. */ void fiddle_interrupt(void){ - if(globalDb) sqlite3_interrupt(globalDb); + if( globalDb ) sqlite3_interrupt(globalDb); } /* @@ -23728,71 +26512,72 @@ const char * fiddle_db_filename(const char * zDbName){ } /* -** Closes, unlinks, and reopens the db using its current filename (or -** the default if the db is currently closed). It is assumed, for -** purposes of the fiddle build, that the file is in a transient -** virtual filesystem within the browser. +** Completely wipes out the contents of the currently-opened database +** but leaves its storage intact for reuse. */ void fiddle_reset_db(void){ - char *zFilename = 0; - if(0==globalDb){ - shellState.pAuxDb->zDbFilename = "/fiddle.sqlite3"; - }else{ - zFilename = - sqlite3_mprintf("%s", sqlite3_db_filename(globalDb, "main")); - shell_check_oom(zFilename); - close_db(globalDb); - shellDeleteFile(zFilename); - shellState.db = 0; - shellState.pAuxDb->zDbFilename = zFilename; + if( globalDb ){ + int rc = sqlite3_db_config(globalDb, SQLITE_DBCONFIG_RESET_DATABASE, 1, 0); + if( 0==rc ) rc = sqlite3_exec(globalDb, "VACUUM", 0, 0, 0); + sqlite3_db_config(globalDb, SQLITE_DBCONFIG_RESET_DATABASE, 0, 0); } - open_db(&shellState, 0); - sqlite3_free(zFilename); } /* -** Trivial exportable function for emscripten. Needs to be exported using: -** -** emcc ..flags... -sEXPORTED_FUNCTIONS=_fiddle_exec -sEXPORTED_RUNTIME_METHODS=ccall,cwrap -** -** (Note the underscore before the function name.) It processes zSql -** as if it were input to the sqlite3 shell and redirects all output -** to the wasm binding. +** Uses the current database's VFS xRead to stream the db file's +** contents out to the given callback. The callback gets a single +** chunk of size n (its 2nd argument) on each call and must return 0 +** on success, non-0 on error. This function returns 0 on success, +** SQLITE_NOTFOUND if no db is open, or propagates any other non-0 +** code from the callback. Note that this is not thread-friendly: it +** expects that it will be the only thread reading the db file and +** takes no measures to ensure that is the case. +*/ +int fiddle_export_db( int (*xCallback)(unsigned const char *zOut, int n) ){ + sqlite3_int64 nSize = 0; + sqlite3_int64 nPos = 0; + sqlite3_file * pFile = 0; + unsigned char buf[1024 * 8]; + int nBuf = (int)sizeof(buf); + int rc = shellState.db + ? sqlite3_file_control(shellState.db, "main", + SQLITE_FCNTL_FILE_POINTER, &pFile) + : SQLITE_NOTFOUND; + if( rc ) return rc; + rc = pFile->pMethods->xFileSize(pFile, &nSize); + if( rc ) return rc; + if(nSize % nBuf){ + /* DB size is not an even multiple of the buffer size. Reduce + ** buffer size so that we do not unduly inflate the db size when + ** exporting. */ + if(0 == nSize % 4096) nBuf = 4096; + else if(0 == nSize % 2048) nBuf = 2048; + else if(0 == nSize % 1024) nBuf = 1024; + else nBuf = 512; + } + for( ; 0==rc && nPospMethods->xRead(pFile, buf, nBuf, nPos); + if(SQLITE_IOERR_SHORT_READ == rc){ + rc = (nPos + nBuf) < nSize ? rc : 0/*assume EOF*/; + } + if( 0==rc ) rc = xCallback(buf, nBuf); + } + return rc; +} + +/* +** Trivial exportable function for emscripten. It processes zSql as if +** it were input to the sqlite3 shell and redirects all output to the +** wasm binding. fiddle_main() must have been called before this +** is called, or results are undefined. */ void fiddle_exec(const char * zSql){ - static int once = 0; - int rc = 0; - if(!once){ - /* Simulate an argv array for main() */ - static char * argv[] = {"fiddle", - "-bail", - "-safe"}; - rc = fiddle_main((int)(sizeof(argv)/sizeof(argv[0])), argv); - once = rc ? -1 : 1; - memset(&shellState.wasm, 0, sizeof(shellState.wasm)); - printf( - "SQLite version %s %.19s\n" /*extra-version-info*/, - sqlite3_libversion(), sqlite3_sourceid() - ); - puts("WASM shell"); - puts("Enter \".help\" for usage hints."); - if(once>0){ - fiddle_reset_db(); - } - if(shellState.db){ - printf("Connected to %s.\n", fiddle_db_filename(NULL)); - }else{ - fprintf(stderr,"ERROR initializing db!\n"); - return; - } - } - if(once<0){ - puts("DB init failed. Not executing SQL."); - }else if(zSql && *zSql){ + if(zSql && *zSql){ + if('.'==*zSql) puts(zSql); shellState.wasm.zInput = zSql; shellState.wasm.zPos = zSql; process_input(&shellState); - memset(&shellState.wasm, 0, sizeof(shellState.wasm)); + shellState.wasm.zInput = shellState.wasm.zPos = 0; } } -#endif /* SQLITE_SHELL_WASM_MODE */ +#endif /* SQLITE_SHELL_FIDDLE */ \ No newline at end of file diff --git a/src/database/sqlite3.c b/src/database/sqlite3.c index b8f98c7c..73c575ad 100644 --- a/src/database/sqlite3.c +++ b/src/database/sqlite3.c @@ -1,6 +1,6 @@ /****************************************************************************** ** This file is an amalgamation of many separate C source files from SQLite -** version 3.39.2. By combining all the individual C code files into this +** version 3.40.0. By combining all the individual C code files into this ** single large file, the entire code can be compiled as a single translation ** unit. This allows many compilers to do optimizations that would not be ** possible if the files were compiled separately. Performance improvements @@ -452,9 +452,9 @@ extern "C" { ** [sqlite3_libversion_number()], [sqlite3_sourceid()], ** [sqlite_version()] and [sqlite_source_id()]. */ -#define SQLITE_VERSION "3.39.2" -#define SQLITE_VERSION_NUMBER 3039002 -#define SQLITE_SOURCE_ID "2022-07-21 15:24:47 698edb77537b67c41adc68f9b892db56bcf9a55e00371a61420f3ddd668e6603" +#define SQLITE_VERSION "3.40.0" +#define SQLITE_VERSION_NUMBER 3040000 +#define SQLITE_SOURCE_ID "2022-11-16 12:10:08 89c459e766ea7e9165d0beeb124708b955a4950d0f4792f457465d71b158d318" /* ** CAPI3REF: Run-Time Library Version Numbers @@ -976,13 +976,17 @@ SQLITE_API int sqlite3_exec( ** ** SQLite uses one of these integer values as the second ** argument to calls it makes to the xLock() and xUnlock() methods -** of an [sqlite3_io_methods] object. +** of an [sqlite3_io_methods] object. These values are ordered from +** lest restrictive to most restrictive. +** +** The argument to xLock() is always SHARED or higher. The argument to +** xUnlock is either SHARED or NONE. */ -#define SQLITE_LOCK_NONE 0 -#define SQLITE_LOCK_SHARED 1 -#define SQLITE_LOCK_RESERVED 2 -#define SQLITE_LOCK_PENDING 3 -#define SQLITE_LOCK_EXCLUSIVE 4 +#define SQLITE_LOCK_NONE 0 /* xUnlock() only */ +#define SQLITE_LOCK_SHARED 1 /* xLock() or xUnlock() */ +#define SQLITE_LOCK_RESERVED 2 /* xLock() only */ +#define SQLITE_LOCK_PENDING 3 /* xLock() only */ +#define SQLITE_LOCK_EXCLUSIVE 4 /* xLock() only */ /* ** CAPI3REF: Synchronization Type Flags @@ -1060,7 +1064,14 @@ struct sqlite3_file { **
  • [SQLITE_LOCK_PENDING], or **
  • [SQLITE_LOCK_EXCLUSIVE]. ** -** xLock() increases the lock. xUnlock() decreases the lock. +** xLock() upgrades the database file lock. In other words, xLock() moves the +** database file lock in the direction NONE toward EXCLUSIVE. The argument to +** xLock() is always on of SHARED, RESERVED, PENDING, or EXCLUSIVE, never +** SQLITE_LOCK_NONE. If the database file lock is already at or above the +** requested lock, then the call to xLock() is a no-op. +** xUnlock() downgrades the database file lock to either SHARED or NONE. +* If the lock is already at or below the requested lock state, then the call +** to xUnlock() is a no-op. ** The xCheckReservedLock() method checks whether any database connection, ** either in this process or in some other process, is holding a RESERVED, ** PENDING, or EXCLUSIVE lock on the file. It returns true @@ -1165,9 +1176,8 @@ struct sqlite3_io_methods { ** opcode causes the xFileControl method to write the current state of ** the lock (one of [SQLITE_LOCK_NONE], [SQLITE_LOCK_SHARED], ** [SQLITE_LOCK_RESERVED], [SQLITE_LOCK_PENDING], or [SQLITE_LOCK_EXCLUSIVE]) -** into an integer that the pArg argument points to. This capability -** is used during testing and is only available when the SQLITE_TEST -** compile-time option is used. +** into an integer that the pArg argument points to. +** This capability is only available if SQLite is compiled with [SQLITE_DEBUG]. ** **
  • [[SQLITE_FCNTL_SIZE_HINT]] ** The [SQLITE_FCNTL_SIZE_HINT] opcode is used by SQLite to give the VFS @@ -1559,6 +1569,26 @@ typedef struct sqlite3_mutex sqlite3_mutex; */ typedef struct sqlite3_api_routines sqlite3_api_routines; +/* +** CAPI3REF: File Name +** +** Type [sqlite3_filename] is used by SQLite to pass filenames to the +** xOpen method of a [VFS]. It may be cast to (const char*) and treated +** as a normal, nul-terminated, UTF-8 buffer containing the filename, but +** may also be passed to special APIs such as: +** +**
      +**
    • sqlite3_filename_database() +**
    • sqlite3_filename_journal() +**
    • sqlite3_filename_wal() +**
    • sqlite3_uri_parameter() +**
    • sqlite3_uri_boolean() +**
    • sqlite3_uri_int64() +**
    • sqlite3_uri_key() +**
    +*/ +typedef const char *sqlite3_filename; + /* ** CAPI3REF: OS Interface Object ** @@ -1737,7 +1767,7 @@ struct sqlite3_vfs { sqlite3_vfs *pNext; /* Next registered VFS */ const char *zName; /* Name of this virtual file system */ void *pAppData; /* Pointer to application-specific data */ - int (*xOpen)(sqlite3_vfs*, const char *zName, sqlite3_file*, + int (*xOpen)(sqlite3_vfs*, sqlite3_filename zName, sqlite3_file*, int flags, int *pOutFlags); int (*xDelete)(sqlite3_vfs*, const char *zName, int syncDir); int (*xAccess)(sqlite3_vfs*, const char *zName, int flags, int *pResOut); @@ -2615,6 +2645,7 @@ struct sqlite3_mem_methods { **
      **
    • The [PRAGMA writable_schema=ON] statement. **
    • The [PRAGMA journal_mode=OFF] statement. +**
    • The [PRAGMA schema_version=N] statement. **
    • Writes to the [sqlite_dbpage] virtual table. **
    • Direct writes to [shadow tables]. **
    @@ -3730,6 +3761,9 @@ SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); **
    The database is opened [shared cache] enabled, overriding ** the default shared cache setting provided by ** [sqlite3_enable_shared_cache()].)^ +** The [use of shared cache mode is discouraged] and hence shared cache +** capabilities may be omitted from many builds of SQLite. In such cases, +** this option is a no-op. ** ** ^(
    [SQLITE_OPEN_PRIVATECACHE]
    **
    The database is opened [shared cache] disabled, overriding @@ -3745,7 +3779,7 @@ SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); ** to return an extended result code.
    ** ** [[OPEN_NOFOLLOW]] ^(
    [SQLITE_OPEN_NOFOLLOW]
    -**
    The database filename is not allowed to be a symbolic link
    +**
    The database filename is not allowed to contain a symbolic link
    ** )^ ** ** If the 3rd parameter to sqlite3_open_v2() is not one of the @@ -4004,10 +4038,10 @@ SQLITE_API int sqlite3_open_v2( ** ** See the [URI filename] documentation for additional information. */ -SQLITE_API const char *sqlite3_uri_parameter(const char *zFilename, const char *zParam); -SQLITE_API int sqlite3_uri_boolean(const char *zFile, const char *zParam, int bDefault); -SQLITE_API sqlite3_int64 sqlite3_uri_int64(const char*, const char*, sqlite3_int64); -SQLITE_API const char *sqlite3_uri_key(const char *zFilename, int N); +SQLITE_API const char *sqlite3_uri_parameter(sqlite3_filename z, const char *zParam); +SQLITE_API int sqlite3_uri_boolean(sqlite3_filename z, const char *zParam, int bDefault); +SQLITE_API sqlite3_int64 sqlite3_uri_int64(sqlite3_filename, const char*, sqlite3_int64); +SQLITE_API const char *sqlite3_uri_key(sqlite3_filename z, int N); /* ** CAPI3REF: Translate filenames @@ -4036,9 +4070,9 @@ SQLITE_API const char *sqlite3_uri_key(const char *zFilename, int N); ** return value from [sqlite3_db_filename()], then the result is ** undefined and is likely a memory access violation. */ -SQLITE_API const char *sqlite3_filename_database(const char*); -SQLITE_API const char *sqlite3_filename_journal(const char*); -SQLITE_API const char *sqlite3_filename_wal(const char*); +SQLITE_API const char *sqlite3_filename_database(sqlite3_filename); +SQLITE_API const char *sqlite3_filename_journal(sqlite3_filename); +SQLITE_API const char *sqlite3_filename_wal(sqlite3_filename); /* ** CAPI3REF: Database File Corresponding To A Journal @@ -4104,14 +4138,14 @@ SQLITE_API sqlite3_file *sqlite3_database_file_object(const char*); ** then the corresponding [sqlite3_module.xClose() method should also be ** invoked prior to calling sqlite3_free_filename(Y). */ -SQLITE_API char *sqlite3_create_filename( +SQLITE_API sqlite3_filename sqlite3_create_filename( const char *zDatabase, const char *zJournal, const char *zWal, int nParam, const char **azParam ); -SQLITE_API void sqlite3_free_filename(char*); +SQLITE_API void sqlite3_free_filename(sqlite3_filename); /* ** CAPI3REF: Error Codes And Messages @@ -5814,6 +5848,16 @@ SQLITE_API SQLITE_DEPRECATED int sqlite3_memory_alarm(void(*)(void*,sqlite3_int6 ** then the conversion is performed. Otherwise no conversion occurs. ** The [SQLITE_INTEGER | datatype] after conversion is returned.)^ ** +** ^(The sqlite3_value_encoding(X) interface returns one of [SQLITE_UTF8], +** [SQLITE_UTF16BE], or [SQLITE_UTF16LE] according to the current encoding +** of the value X, assuming that X has type TEXT.)^ If sqlite3_value_type(X) +** returns something other than SQLITE_TEXT, then the return value from +** sqlite3_value_encoding(X) is meaningless. ^Calls to +** sqlite3_value_text(X), sqlite3_value_text16(X), sqlite3_value_text16be(X), +** sqlite3_value_text16le(X), sqlite3_value_bytes(X), or +** sqlite3_value_bytes16(X) might change the encoding of the value X and +** thus change the return from subsequent calls to sqlite3_value_encoding(X). +** ** ^Within the [xUpdate] method of a [virtual table], the ** sqlite3_value_nochange(X) interface returns true if and only if ** the column corresponding to X is unchanged by the UPDATE operation @@ -5878,6 +5922,7 @@ SQLITE_API int sqlite3_value_type(sqlite3_value*); SQLITE_API int sqlite3_value_numeric_type(sqlite3_value*); SQLITE_API int sqlite3_value_nochange(sqlite3_value*); SQLITE_API int sqlite3_value_frombind(sqlite3_value*); +SQLITE_API int sqlite3_value_encoding(sqlite3_value*); /* ** CAPI3REF: Finding The Subtype Of SQL Values @@ -5931,7 +5976,7 @@ SQLITE_API void sqlite3_value_free(sqlite3_value*); ** ** ^The sqlite3_aggregate_context(C,N) routine returns a NULL pointer ** when first called if N is less than or equal to zero or if a memory -** allocate error occurs. +** allocation error occurs. ** ** ^(The amount of space allocated by sqlite3_aggregate_context(C,N) is ** determined by the N parameter on first successful call. Changing the @@ -6136,9 +6181,10 @@ typedef void (*sqlite3_destructor_type)(void*); ** of [SQLITE_UTF8], [SQLITE_UTF16], [SQLITE_UTF16BE], or [SQLITE_UTF16LE]. ** ^SQLite takes the text result from the application from ** the 2nd parameter of the sqlite3_result_text* interfaces. -** ^If the 3rd parameter to the sqlite3_result_text* interfaces -** is negative, then SQLite takes result text from the 2nd parameter -** through the first zero character. +** ^If the 3rd parameter to any of the sqlite3_result_text* interfaces +** other than sqlite3_result_text64() is negative, then SQLite computes +** the string length itself by searching the 2nd parameter for the first +** zero character. ** ^If the 3rd parameter to the sqlite3_result_text* interfaces ** is non-negative, then as many bytes (not characters) of the text ** pointed to by the 2nd parameter are taken as the application-defined @@ -6634,7 +6680,7 @@ SQLITE_API const char *sqlite3_db_name(sqlite3 *db, int N); **
  • [sqlite3_filename_wal()] ** */ -SQLITE_API const char *sqlite3_db_filename(sqlite3 *db, const char *zDbName); +SQLITE_API sqlite3_filename sqlite3_db_filename(sqlite3 *db, const char *zDbName); /* ** CAPI3REF: Determine if a database is read-only @@ -6771,7 +6817,7 @@ SQLITE_API void *sqlite3_rollback_hook(sqlite3*, void(*)(void *), void*); ** function C that is invoked prior to each autovacuum of the database ** file. ^The callback is passed a copy of the generic data pointer (P), ** the schema-name of the attached database that is being autovacuumed, -** the the size of the database file in pages, the number of free pages, +** the size of the database file in pages, the number of free pages, ** and the number of bytes per page, respectively. The callback should ** return the number of free pages that should be removed by the ** autovacuum. ^If the callback returns zero, then no autovacuum happens. @@ -6892,6 +6938,11 @@ SQLITE_API void *sqlite3_update_hook( ** to the same database. Sharing is enabled if the argument is true ** and disabled if the argument is false.)^ ** +** This interface is omitted if SQLite is compiled with +** [-DSQLITE_OMIT_SHARED_CACHE]. The [-DSQLITE_OMIT_SHARED_CACHE] +** compile-time option is recommended because the +** [use of shared cache mode is discouraged]. +** ** ^Cache sharing is enabled and disabled for an entire process. ** This is a change as of SQLite [version 3.5.0] ([dateof:3.5.0]). ** In prior versions of SQLite, @@ -6990,7 +7041,7 @@ SQLITE_API int sqlite3_db_release_memory(sqlite3*); ** ^The soft heap limit may not be greater than the hard heap limit. ** ^If the hard heap limit is enabled and if sqlite3_soft_heap_limit(N) ** is invoked with a value of N that is greater than the hard heap limit, -** the the soft heap limit is set to the value of the hard heap limit. +** the soft heap limit is set to the value of the hard heap limit. ** ^The soft heap limit is automatically enabled whenever the hard heap ** limit is enabled. ^When sqlite3_hard_heap_limit64(N) is invoked and ** the soft heap limit is outside the range of 1..N, then the soft heap @@ -9285,7 +9336,7 @@ typedef struct sqlite3_backup sqlite3_backup; ** if the application incorrectly accesses the destination [database connection] ** and so no error code is reported, but the operations may malfunction ** nevertheless. Use of the destination database connection while a -** backup is in progress might also also cause a mutex deadlock. +** backup is in progress might also cause a mutex deadlock. ** ** If running in [shared cache mode], the application must ** guarantee that the shared cache used by the destination database @@ -9713,7 +9764,7 @@ SQLITE_API int sqlite3_wal_checkpoint_v2( */ #define SQLITE_CHECKPOINT_PASSIVE 0 /* Do as much as possible w/o blocking */ #define SQLITE_CHECKPOINT_FULL 1 /* Wait for writers, then checkpoint */ -#define SQLITE_CHECKPOINT_RESTART 2 /* Like FULL but wait for for readers */ +#define SQLITE_CHECKPOINT_RESTART 2 /* Like FULL but wait for readers */ #define SQLITE_CHECKPOINT_TRUNCATE 3 /* Like RESTART but also truncate WAL */ /* @@ -13144,12 +13195,17 @@ struct fts5_api { /************** End of sqlite3.h *********************************************/ /************** Continuing where we left off in sqliteInt.h ******************/ +/* +** Reuse the STATIC_LRU for mutex access to sqlite3_temp_directory. +*/ +#define SQLITE_MUTEX_STATIC_TEMPDIR SQLITE_MUTEX_STATIC_VFS1 + /* ** Include the configuration header output by 'configure' if we're using the ** autoconf-based build */ #if defined(_HAVE_SQLITE_CONFIG_H) && !defined(SQLITECONFIG_H) -#include "config.h" +#include "sqlite_cfg.h" #define SQLITECONFIG_H 1 #endif @@ -14632,6 +14688,7 @@ typedef struct FuncDef FuncDef; typedef struct FuncDefHash FuncDefHash; typedef struct IdList IdList; typedef struct Index Index; +typedef struct IndexedExpr IndexedExpr; typedef struct IndexSample IndexSample; typedef struct KeyClass KeyClass; typedef struct KeyInfo KeyInfo; @@ -14697,6 +14754,7 @@ typedef struct With With; #define MASKBIT32(n) (((unsigned int)1)<<(n)) #define SMASKBIT32(n) ((n)<=31?((unsigned int)1)<<(n):0) #define ALLBITS ((Bitmask)-1) +#define TOPBIT (((Bitmask)1)<<(BMS-1)) /* A VList object records a mapping between parameters/variables/wildcards ** in the SQL statement (such as $abc, @pqr, or :xyz) and the integer @@ -14711,6 +14769,331 @@ typedef int VList; ** "BusyHandler" typedefs. vdbe.h also requires a few of the opaque ** pointer types (i.e. FuncDef) defined above. */ +/************** Include os.h in the middle of sqliteInt.h ********************/ +/************** Begin file os.h **********************************************/ +/* +** 2001 September 16 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This header file (together with is companion C source-code file +** "os.c") attempt to abstract the underlying operating system so that +** the SQLite library will work on both POSIX and windows systems. +** +** This header file is #include-ed by sqliteInt.h and thus ends up +** being included by every source file. +*/ +#ifndef _SQLITE_OS_H_ +#define _SQLITE_OS_H_ + +/* +** Attempt to automatically detect the operating system and setup the +** necessary pre-processor macros for it. +*/ +/************** Include os_setup.h in the middle of os.h *********************/ +/************** Begin file os_setup.h ****************************************/ +/* +** 2013 November 25 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains pre-processor directives related to operating system +** detection and/or setup. +*/ +#ifndef SQLITE_OS_SETUP_H +#define SQLITE_OS_SETUP_H + +/* +** Figure out if we are dealing with Unix, Windows, or some other operating +** system. +** +** After the following block of preprocess macros, all of +** +** SQLITE_OS_KV +** SQLITE_OS_OTHER +** SQLITE_OS_UNIX +** SQLITE_OS_WIN +** +** will defined to either 1 or 0. One of them will be 1. The others will be 0. +** If none of the macros are initially defined, then select either +** SQLITE_OS_UNIX or SQLITE_OS_WIN depending on the target platform. +** +** If SQLITE_OS_OTHER=1 is specified at compile-time, then the application +** must provide its own VFS implementation together with sqlite3_os_init() +** and sqlite3_os_end() routines. +*/ +#if !defined(SQLITE_OS_KV) && !defined(SQLITE_OS_OTHER) && \ + !defined(SQLITE_OS_UNIX) && !defined(SQLITE_OS_WIN) +# if defined(_WIN32) || defined(WIN32) || defined(__CYGWIN__) || \ + defined(__MINGW32__) || defined(__BORLANDC__) +# define SQLITE_OS_WIN 1 +# define SQLITE_OS_UNIX 0 +# else +# define SQLITE_OS_WIN 0 +# define SQLITE_OS_UNIX 1 +# endif +#endif +#if SQLITE_OS_OTHER+1>1 +# undef SQLITE_OS_KV +# define SQLITE_OS_KV 0 +# undef SQLITE_OS_UNIX +# define SQLITE_OS_UNIX 0 +# undef SQLITE_OS_WIN +# define SQLITE_OS_WIN 0 +#endif +#if SQLITE_OS_KV+1>1 +# undef SQLITE_OS_OTHER +# define SQLITE_OS_OTHER 0 +# undef SQLITE_OS_UNIX +# define SQLITE_OS_UNIX 0 +# undef SQLITE_OS_WIN +# define SQLITE_OS_WIN 0 +# define SQLITE_OMIT_LOAD_EXTENSION 1 +# define SQLITE_OMIT_WAL 1 +# define SQLITE_OMIT_DEPRECATED 1 +# undef SQLITE_TEMP_STORE +# define SQLITE_TEMP_STORE 3 /* Always use memory for temporary storage */ +# define SQLITE_DQS 0 +# define SQLITE_OMIT_SHARED_CACHE 1 +# define SQLITE_OMIT_AUTOINIT 1 +#endif +#if SQLITE_OS_UNIX+1>1 +# undef SQLITE_OS_KV +# define SQLITE_OS_KV 0 +# undef SQLITE_OS_OTHER +# define SQLITE_OS_OTHER 0 +# undef SQLITE_OS_WIN +# define SQLITE_OS_WIN 0 +#endif +#if SQLITE_OS_WIN+1>1 +# undef SQLITE_OS_KV +# define SQLITE_OS_KV 0 +# undef SQLITE_OS_OTHER +# define SQLITE_OS_OTHER 0 +# undef SQLITE_OS_UNIX +# define SQLITE_OS_UNIX 0 +#endif + + +#endif /* SQLITE_OS_SETUP_H */ + +/************** End of os_setup.h ********************************************/ +/************** Continuing where we left off in os.h *************************/ + +/* If the SET_FULLSYNC macro is not defined above, then make it +** a no-op +*/ +#ifndef SET_FULLSYNC +# define SET_FULLSYNC(x,y) +#endif + +/* Maximum pathname length. Note: FILENAME_MAX defined by stdio.h +*/ +#ifndef SQLITE_MAX_PATHLEN +# define SQLITE_MAX_PATHLEN FILENAME_MAX +#endif + +/* Maximum number of symlinks that will be resolved while trying to +** expand a filename in xFullPathname() in the VFS. +*/ +#ifndef SQLITE_MAX_SYMLINK +# define SQLITE_MAX_SYMLINK 200 +#endif + +/* +** The default size of a disk sector +*/ +#ifndef SQLITE_DEFAULT_SECTOR_SIZE +# define SQLITE_DEFAULT_SECTOR_SIZE 4096 +#endif + +/* +** Temporary files are named starting with this prefix followed by 16 random +** alphanumeric characters, and no file extension. They are stored in the +** OS's standard temporary file directory, and are deleted prior to exit. +** If sqlite is being embedded in another program, you may wish to change the +** prefix to reflect your program's name, so that if your program exits +** prematurely, old temporary files can be easily identified. This can be done +** using -DSQLITE_TEMP_FILE_PREFIX=myprefix_ on the compiler command line. +** +** 2006-10-31: The default prefix used to be "sqlite_". But then +** Mcafee started using SQLite in their anti-virus product and it +** started putting files with the "sqlite" name in the c:/temp folder. +** This annoyed many windows users. Those users would then do a +** Google search for "sqlite", find the telephone numbers of the +** developers and call to wake them up at night and complain. +** For this reason, the default name prefix is changed to be "sqlite" +** spelled backwards. So the temp files are still identified, but +** anybody smart enough to figure out the code is also likely smart +** enough to know that calling the developer will not help get rid +** of the file. +*/ +#ifndef SQLITE_TEMP_FILE_PREFIX +# define SQLITE_TEMP_FILE_PREFIX "etilqs_" +#endif + +/* +** The following values may be passed as the second argument to +** sqlite3OsLock(). The various locks exhibit the following semantics: +** +** SHARED: Any number of processes may hold a SHARED lock simultaneously. +** RESERVED: A single process may hold a RESERVED lock on a file at +** any time. Other processes may hold and obtain new SHARED locks. +** PENDING: A single process may hold a PENDING lock on a file at +** any one time. Existing SHARED locks may persist, but no new +** SHARED locks may be obtained by other processes. +** EXCLUSIVE: An EXCLUSIVE lock precludes all other locks. +** +** PENDING_LOCK may not be passed directly to sqlite3OsLock(). Instead, a +** process that requests an EXCLUSIVE lock may actually obtain a PENDING +** lock. This can be upgraded to an EXCLUSIVE lock by a subsequent call to +** sqlite3OsLock(). +*/ +#define NO_LOCK 0 +#define SHARED_LOCK 1 +#define RESERVED_LOCK 2 +#define PENDING_LOCK 3 +#define EXCLUSIVE_LOCK 4 + +/* +** File Locking Notes: (Mostly about windows but also some info for Unix) +** +** We cannot use LockFileEx() or UnlockFileEx() on Win95/98/ME because +** those functions are not available. So we use only LockFile() and +** UnlockFile(). +** +** LockFile() prevents not just writing but also reading by other processes. +** A SHARED_LOCK is obtained by locking a single randomly-chosen +** byte out of a specific range of bytes. The lock byte is obtained at +** random so two separate readers can probably access the file at the +** same time, unless they are unlucky and choose the same lock byte. +** An EXCLUSIVE_LOCK is obtained by locking all bytes in the range. +** There can only be one writer. A RESERVED_LOCK is obtained by locking +** a single byte of the file that is designated as the reserved lock byte. +** A PENDING_LOCK is obtained by locking a designated byte different from +** the RESERVED_LOCK byte. +** +** On WinNT/2K/XP systems, LockFileEx() and UnlockFileEx() are available, +** which means we can use reader/writer locks. When reader/writer locks +** are used, the lock is placed on the same range of bytes that is used +** for probabilistic locking in Win95/98/ME. Hence, the locking scheme +** will support two or more Win95 readers or two or more WinNT readers. +** But a single Win95 reader will lock out all WinNT readers and a single +** WinNT reader will lock out all other Win95 readers. +** +** The following #defines specify the range of bytes used for locking. +** SHARED_SIZE is the number of bytes available in the pool from which +** a random byte is selected for a shared lock. The pool of bytes for +** shared locks begins at SHARED_FIRST. +** +** The same locking strategy and +** byte ranges are used for Unix. This leaves open the possibility of having +** clients on win95, winNT, and unix all talking to the same shared file +** and all locking correctly. To do so would require that samba (or whatever +** tool is being used for file sharing) implements locks correctly between +** windows and unix. I'm guessing that isn't likely to happen, but by +** using the same locking range we are at least open to the possibility. +** +** Locking in windows is manditory. For this reason, we cannot store +** actual data in the bytes used for locking. The pager never allocates +** the pages involved in locking therefore. SHARED_SIZE is selected so +** that all locks will fit on a single page even at the minimum page size. +** PENDING_BYTE defines the beginning of the locks. By default PENDING_BYTE +** is set high so that we don't have to allocate an unused page except +** for very large databases. But one should test the page skipping logic +** by setting PENDING_BYTE low and running the entire regression suite. +** +** Changing the value of PENDING_BYTE results in a subtly incompatible +** file format. Depending on how it is changed, you might not notice +** the incompatibility right away, even running a full regression test. +** The default location of PENDING_BYTE is the first byte past the +** 1GB boundary. +** +*/ +#ifdef SQLITE_OMIT_WSD +# define PENDING_BYTE (0x40000000) +#else +# define PENDING_BYTE sqlite3PendingByte +#endif +#define RESERVED_BYTE (PENDING_BYTE+1) +#define SHARED_FIRST (PENDING_BYTE+2) +#define SHARED_SIZE 510 + +/* +** Wrapper around OS specific sqlite3_os_init() function. +*/ +SQLITE_PRIVATE int sqlite3OsInit(void); + +/* +** Functions for accessing sqlite3_file methods +*/ +SQLITE_PRIVATE void sqlite3OsClose(sqlite3_file*); +SQLITE_PRIVATE int sqlite3OsRead(sqlite3_file*, void*, int amt, i64 offset); +SQLITE_PRIVATE int sqlite3OsWrite(sqlite3_file*, const void*, int amt, i64 offset); +SQLITE_PRIVATE int sqlite3OsTruncate(sqlite3_file*, i64 size); +SQLITE_PRIVATE int sqlite3OsSync(sqlite3_file*, int); +SQLITE_PRIVATE int sqlite3OsFileSize(sqlite3_file*, i64 *pSize); +SQLITE_PRIVATE int sqlite3OsLock(sqlite3_file*, int); +SQLITE_PRIVATE int sqlite3OsUnlock(sqlite3_file*, int); +SQLITE_PRIVATE int sqlite3OsCheckReservedLock(sqlite3_file *id, int *pResOut); +SQLITE_PRIVATE int sqlite3OsFileControl(sqlite3_file*,int,void*); +SQLITE_PRIVATE void sqlite3OsFileControlHint(sqlite3_file*,int,void*); +#define SQLITE_FCNTL_DB_UNCHANGED 0xca093fa0 +SQLITE_PRIVATE int sqlite3OsSectorSize(sqlite3_file *id); +SQLITE_PRIVATE int sqlite3OsDeviceCharacteristics(sqlite3_file *id); +#ifndef SQLITE_OMIT_WAL +SQLITE_PRIVATE int sqlite3OsShmMap(sqlite3_file *,int,int,int,void volatile **); +SQLITE_PRIVATE int sqlite3OsShmLock(sqlite3_file *id, int, int, int); +SQLITE_PRIVATE void sqlite3OsShmBarrier(sqlite3_file *id); +SQLITE_PRIVATE int sqlite3OsShmUnmap(sqlite3_file *id, int); +#endif /* SQLITE_OMIT_WAL */ +SQLITE_PRIVATE int sqlite3OsFetch(sqlite3_file *id, i64, int, void **); +SQLITE_PRIVATE int sqlite3OsUnfetch(sqlite3_file *, i64, void *); + + +/* +** Functions for accessing sqlite3_vfs methods +*/ +SQLITE_PRIVATE int sqlite3OsOpen(sqlite3_vfs *, const char *, sqlite3_file*, int, int *); +SQLITE_PRIVATE int sqlite3OsDelete(sqlite3_vfs *, const char *, int); +SQLITE_PRIVATE int sqlite3OsAccess(sqlite3_vfs *, const char *, int, int *pResOut); +SQLITE_PRIVATE int sqlite3OsFullPathname(sqlite3_vfs *, const char *, int, char *); +#ifndef SQLITE_OMIT_LOAD_EXTENSION +SQLITE_PRIVATE void *sqlite3OsDlOpen(sqlite3_vfs *, const char *); +SQLITE_PRIVATE void sqlite3OsDlError(sqlite3_vfs *, int, char *); +SQLITE_PRIVATE void (*sqlite3OsDlSym(sqlite3_vfs *, void *, const char *))(void); +SQLITE_PRIVATE void sqlite3OsDlClose(sqlite3_vfs *, void *); +#endif /* SQLITE_OMIT_LOAD_EXTENSION */ +SQLITE_PRIVATE int sqlite3OsRandomness(sqlite3_vfs *, int, char *); +SQLITE_PRIVATE int sqlite3OsSleep(sqlite3_vfs *, int); +SQLITE_PRIVATE int sqlite3OsGetLastError(sqlite3_vfs*); +SQLITE_PRIVATE int sqlite3OsCurrentTimeInt64(sqlite3_vfs *, sqlite3_int64*); + +/* +** Convenience functions for opening and closing files using +** sqlite3_malloc() to obtain space for the file-handle structure. +*/ +SQLITE_PRIVATE int sqlite3OsOpenMalloc(sqlite3_vfs *, const char *, sqlite3_file **, int,int*); +SQLITE_PRIVATE void sqlite3OsCloseFree(sqlite3_file *); + +#endif /* _SQLITE_OS_H_ */ + +/************** End of os.h **************************************************/ +/************** Continuing where we left off in sqliteInt.h ******************/ /************** Include pager.h in the middle of sqliteInt.h *****************/ /************** Begin file pager.h *******************************************/ /* @@ -15555,48 +15938,48 @@ typedef struct VdbeOpList VdbeOpList; #define OP_Vacuum 5 #define OP_VFilter 6 /* jump, synopsis: iplan=r[P3] zplan='P4' */ #define OP_VUpdate 7 /* synopsis: data=r[P3@P2] */ -#define OP_Goto 8 /* jump */ -#define OP_Gosub 9 /* jump */ -#define OP_InitCoroutine 10 /* jump */ -#define OP_Yield 11 /* jump */ -#define OP_MustBeInt 12 /* jump */ -#define OP_Jump 13 /* jump */ -#define OP_Once 14 /* jump */ -#define OP_If 15 /* jump */ -#define OP_IfNot 16 /* jump */ -#define OP_IsNullOrType 17 /* jump, synopsis: if typeof(r[P1]) IN (P3,5) goto P2 */ -#define OP_IfNullRow 18 /* jump, synopsis: if P1.nullRow then r[P3]=NULL, goto P2 */ +#define OP_Init 8 /* jump, synopsis: Start at P2 */ +#define OP_Goto 9 /* jump */ +#define OP_Gosub 10 /* jump */ +#define OP_InitCoroutine 11 /* jump */ +#define OP_Yield 12 /* jump */ +#define OP_MustBeInt 13 /* jump */ +#define OP_Jump 14 /* jump */ +#define OP_Once 15 /* jump */ +#define OP_If 16 /* jump */ +#define OP_IfNot 17 /* jump */ +#define OP_IsType 18 /* jump, synopsis: if typeof(P1.P3) in P5 goto P2 */ #define OP_Not 19 /* same as TK_NOT, synopsis: r[P2]= !r[P1] */ -#define OP_SeekLT 20 /* jump, synopsis: key=r[P3@P4] */ -#define OP_SeekLE 21 /* jump, synopsis: key=r[P3@P4] */ -#define OP_SeekGE 22 /* jump, synopsis: key=r[P3@P4] */ -#define OP_SeekGT 23 /* jump, synopsis: key=r[P3@P4] */ -#define OP_IfNotOpen 24 /* jump, synopsis: if( !csr[P1] ) goto P2 */ -#define OP_IfNoHope 25 /* jump, synopsis: key=r[P3@P4] */ -#define OP_NoConflict 26 /* jump, synopsis: key=r[P3@P4] */ -#define OP_NotFound 27 /* jump, synopsis: key=r[P3@P4] */ -#define OP_Found 28 /* jump, synopsis: key=r[P3@P4] */ -#define OP_SeekRowid 29 /* jump, synopsis: intkey=r[P3] */ -#define OP_NotExists 30 /* jump, synopsis: intkey=r[P3] */ -#define OP_Last 31 /* jump */ -#define OP_IfSmaller 32 /* jump */ -#define OP_SorterSort 33 /* jump */ -#define OP_Sort 34 /* jump */ -#define OP_Rewind 35 /* jump */ -#define OP_SorterNext 36 /* jump */ -#define OP_Prev 37 /* jump */ -#define OP_Next 38 /* jump */ -#define OP_IdxLE 39 /* jump, synopsis: key=r[P3@P4] */ -#define OP_IdxGT 40 /* jump, synopsis: key=r[P3@P4] */ -#define OP_IdxLT 41 /* jump, synopsis: key=r[P3@P4] */ -#define OP_IdxGE 42 /* jump, synopsis: key=r[P3@P4] */ +#define OP_IfNullRow 20 /* jump, synopsis: if P1.nullRow then r[P3]=NULL, goto P2 */ +#define OP_SeekLT 21 /* jump, synopsis: key=r[P3@P4] */ +#define OP_SeekLE 22 /* jump, synopsis: key=r[P3@P4] */ +#define OP_SeekGE 23 /* jump, synopsis: key=r[P3@P4] */ +#define OP_SeekGT 24 /* jump, synopsis: key=r[P3@P4] */ +#define OP_IfNotOpen 25 /* jump, synopsis: if( !csr[P1] ) goto P2 */ +#define OP_IfNoHope 26 /* jump, synopsis: key=r[P3@P4] */ +#define OP_NoConflict 27 /* jump, synopsis: key=r[P3@P4] */ +#define OP_NotFound 28 /* jump, synopsis: key=r[P3@P4] */ +#define OP_Found 29 /* jump, synopsis: key=r[P3@P4] */ +#define OP_SeekRowid 30 /* jump, synopsis: intkey=r[P3] */ +#define OP_NotExists 31 /* jump, synopsis: intkey=r[P3] */ +#define OP_Last 32 /* jump */ +#define OP_IfSmaller 33 /* jump */ +#define OP_SorterSort 34 /* jump */ +#define OP_Sort 35 /* jump */ +#define OP_Rewind 36 /* jump */ +#define OP_SorterNext 37 /* jump */ +#define OP_Prev 38 /* jump */ +#define OP_Next 39 /* jump */ +#define OP_IdxLE 40 /* jump, synopsis: key=r[P3@P4] */ +#define OP_IdxGT 41 /* jump, synopsis: key=r[P3@P4] */ +#define OP_IdxLT 42 /* jump, synopsis: key=r[P3@P4] */ #define OP_Or 43 /* same as TK_OR, synopsis: r[P3]=(r[P1] || r[P2]) */ #define OP_And 44 /* same as TK_AND, synopsis: r[P3]=(r[P1] && r[P2]) */ -#define OP_RowSetRead 45 /* jump, synopsis: r[P3]=rowset(P1) */ -#define OP_RowSetTest 46 /* jump, synopsis: if r[P3] in rowset(P1) goto P2 */ -#define OP_Program 47 /* jump */ -#define OP_FkIfZero 48 /* jump, synopsis: if fkctr[P1]==0 goto P2 */ -#define OP_IfPos 49 /* jump, synopsis: if r[P1]>0 then r[P1]-=P3, goto P2 */ +#define OP_IdxGE 45 /* jump, synopsis: key=r[P3@P4] */ +#define OP_RowSetRead 46 /* jump, synopsis: r[P3]=rowset(P1) */ +#define OP_RowSetTest 47 /* jump, synopsis: if r[P3] in rowset(P1) goto P2 */ +#define OP_Program 48 /* jump */ +#define OP_FkIfZero 49 /* jump, synopsis: if fkctr[P1]==0 goto P2 */ #define OP_IsNull 50 /* jump, same as TK_ISNULL, synopsis: if r[P1]==NULL goto P2 */ #define OP_NotNull 51 /* jump, same as TK_NOTNULL, synopsis: if r[P1]!=NULL goto P2 */ #define OP_Ne 52 /* jump, same as TK_NE, synopsis: IF r[P3]!=r[P1] */ @@ -15606,12 +15989,12 @@ typedef struct VdbeOpList VdbeOpList; #define OP_Lt 56 /* jump, same as TK_LT, synopsis: IF r[P3]=r[P1] */ #define OP_ElseEq 58 /* jump, same as TK_ESCAPE */ -#define OP_IfNotZero 59 /* jump, synopsis: if r[P1]!=0 then r[P1]--, goto P2 */ -#define OP_DecrJumpZero 60 /* jump, synopsis: if (--r[P1])==0 goto P2 */ -#define OP_IncrVacuum 61 /* jump */ -#define OP_VNext 62 /* jump */ -#define OP_Filter 63 /* jump, synopsis: if key(P3@P4) not in filter(P1) goto P2 */ -#define OP_Init 64 /* jump, synopsis: Start at P2 */ +#define OP_IfPos 59 /* jump, synopsis: if r[P1]>0 then r[P1]-=P3, goto P2 */ +#define OP_IfNotZero 60 /* jump, synopsis: if r[P1]!=0 then r[P1]--, goto P2 */ +#define OP_DecrJumpZero 61 /* jump, synopsis: if (--r[P1])==0 goto P2 */ +#define OP_IncrVacuum 62 /* jump */ +#define OP_VNext 63 /* jump */ +#define OP_Filter 64 /* jump, synopsis: if key(P3@P4) not in filter(P1) goto P2 */ #define OP_PureFunc 65 /* synopsis: r[P3]=func(r[P2@NP]) */ #define OP_Function 66 /* synopsis: r[P3]=func(r[P2@NP]) */ #define OP_Return 67 @@ -15747,13 +16130,13 @@ typedef struct VdbeOpList VdbeOpList; #define OPFLG_OUT3 0x20 /* out3: P3 is an output */ #define OPFLG_INITIALIZER {\ /* 0 */ 0x00, 0x00, 0x00, 0x00, 0x10, 0x00, 0x01, 0x00,\ -/* 8 */ 0x01, 0x01, 0x01, 0x03, 0x03, 0x01, 0x01, 0x03,\ -/* 16 */ 0x03, 0x03, 0x01, 0x12, 0x09, 0x09, 0x09, 0x09,\ -/* 24 */ 0x01, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, 0x01,\ +/* 8 */ 0x01, 0x01, 0x01, 0x01, 0x03, 0x03, 0x01, 0x01,\ +/* 16 */ 0x03, 0x03, 0x01, 0x12, 0x01, 0x09, 0x09, 0x09,\ +/* 24 */ 0x09, 0x01, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09,\ /* 32 */ 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,\ -/* 40 */ 0x01, 0x01, 0x01, 0x26, 0x26, 0x23, 0x0b, 0x01,\ -/* 48 */ 0x01, 0x03, 0x03, 0x03, 0x0b, 0x0b, 0x0b, 0x0b,\ -/* 56 */ 0x0b, 0x0b, 0x01, 0x03, 0x03, 0x01, 0x01, 0x01,\ +/* 40 */ 0x01, 0x01, 0x01, 0x26, 0x26, 0x01, 0x23, 0x0b,\ +/* 48 */ 0x01, 0x01, 0x03, 0x03, 0x0b, 0x0b, 0x0b, 0x0b,\ +/* 56 */ 0x0b, 0x0b, 0x01, 0x03, 0x03, 0x03, 0x01, 0x01,\ /* 64 */ 0x01, 0x00, 0x00, 0x02, 0x02, 0x08, 0x00, 0x10,\ /* 72 */ 0x10, 0x10, 0x00, 0x10, 0x00, 0x10, 0x10, 0x00,\ /* 80 */ 0x00, 0x10, 0x10, 0x00, 0x00, 0x00, 0x02, 0x02,\ @@ -15845,6 +16228,7 @@ SQLITE_PRIVATE void sqlite3VdbeChangeP1(Vdbe*, int addr, int P1); SQLITE_PRIVATE void sqlite3VdbeChangeP2(Vdbe*, int addr, int P2); SQLITE_PRIVATE void sqlite3VdbeChangeP3(Vdbe*, int addr, int P3); SQLITE_PRIVATE void sqlite3VdbeChangeP5(Vdbe*, u16 P5); +SQLITE_PRIVATE void sqlite3VdbeTypeofColumn(Vdbe*, int); SQLITE_PRIVATE void sqlite3VdbeJumpHere(Vdbe*, int addr); SQLITE_PRIVATE void sqlite3VdbeJumpHereOrPopInst(Vdbe*, int addr); SQLITE_PRIVATE int sqlite3VdbeChangeToNoop(Vdbe*, int addr); @@ -15859,6 +16243,7 @@ SQLITE_PRIVATE void sqlite3VdbeAppendP4(Vdbe*, void *pP4, int p4type); SQLITE_PRIVATE void sqlite3VdbeSetP4KeyInfo(Parse*, Index*); SQLITE_PRIVATE void sqlite3VdbeUsesBtree(Vdbe*, int); SQLITE_PRIVATE VdbeOp *sqlite3VdbeGetOp(Vdbe*, int); +SQLITE_PRIVATE VdbeOp *sqlite3VdbeGetLastOp(Vdbe*); SQLITE_PRIVATE int sqlite3VdbeMakeLabel(Parse*); SQLITE_PRIVATE void sqlite3VdbeRunOnlyOnce(Vdbe*); SQLITE_PRIVATE void sqlite3VdbeReusable(Vdbe*); @@ -16207,297 +16592,6 @@ SQLITE_PRIVATE int sqlite3PCacheIsDirty(PCache *pCache); /************** End of pcache.h **********************************************/ /************** Continuing where we left off in sqliteInt.h ******************/ -/************** Include os.h in the middle of sqliteInt.h ********************/ -/************** Begin file os.h **********************************************/ -/* -** 2001 September 16 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -****************************************************************************** -** -** This header file (together with is companion C source-code file -** "os.c") attempt to abstract the underlying operating system so that -** the SQLite library will work on both POSIX and windows systems. -** -** This header file is #include-ed by sqliteInt.h and thus ends up -** being included by every source file. -*/ -#ifndef _SQLITE_OS_H_ -#define _SQLITE_OS_H_ - -/* -** Attempt to automatically detect the operating system and setup the -** necessary pre-processor macros for it. -*/ -/************** Include os_setup.h in the middle of os.h *********************/ -/************** Begin file os_setup.h ****************************************/ -/* -** 2013 November 25 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -****************************************************************************** -** -** This file contains pre-processor directives related to operating system -** detection and/or setup. -*/ -#ifndef SQLITE_OS_SETUP_H -#define SQLITE_OS_SETUP_H - -/* -** Figure out if we are dealing with Unix, Windows, or some other operating -** system. -** -** After the following block of preprocess macros, all of SQLITE_OS_UNIX, -** SQLITE_OS_WIN, and SQLITE_OS_OTHER will defined to either 1 or 0. One of -** the three will be 1. The other two will be 0. -*/ -#if defined(SQLITE_OS_OTHER) -# if SQLITE_OS_OTHER==1 -# undef SQLITE_OS_UNIX -# define SQLITE_OS_UNIX 0 -# undef SQLITE_OS_WIN -# define SQLITE_OS_WIN 0 -# else -# undef SQLITE_OS_OTHER -# endif -#endif -#if !defined(SQLITE_OS_UNIX) && !defined(SQLITE_OS_OTHER) -# define SQLITE_OS_OTHER 0 -# ifndef SQLITE_OS_WIN -# if defined(_WIN32) || defined(WIN32) || defined(__CYGWIN__) || \ - defined(__MINGW32__) || defined(__BORLANDC__) -# define SQLITE_OS_WIN 1 -# define SQLITE_OS_UNIX 0 -# else -# define SQLITE_OS_WIN 0 -# define SQLITE_OS_UNIX 1 -# endif -# else -# define SQLITE_OS_UNIX 0 -# endif -#else -# ifndef SQLITE_OS_WIN -# define SQLITE_OS_WIN 0 -# endif -#endif - -#endif /* SQLITE_OS_SETUP_H */ - -/************** End of os_setup.h ********************************************/ -/************** Continuing where we left off in os.h *************************/ - -/* If the SET_FULLSYNC macro is not defined above, then make it -** a no-op -*/ -#ifndef SET_FULLSYNC -# define SET_FULLSYNC(x,y) -#endif - -/* Maximum pathname length. Note: FILENAME_MAX defined by stdio.h -*/ -#ifndef SQLITE_MAX_PATHLEN -# define SQLITE_MAX_PATHLEN FILENAME_MAX -#endif - -/* Maximum number of symlinks that will be resolved while trying to -** expand a filename in xFullPathname() in the VFS. -*/ -#ifndef SQLITE_MAX_SYMLINK -# define SQLITE_MAX_SYMLINK 200 -#endif - -/* -** The default size of a disk sector -*/ -#ifndef SQLITE_DEFAULT_SECTOR_SIZE -# define SQLITE_DEFAULT_SECTOR_SIZE 4096 -#endif - -/* -** Temporary files are named starting with this prefix followed by 16 random -** alphanumeric characters, and no file extension. They are stored in the -** OS's standard temporary file directory, and are deleted prior to exit. -** If sqlite is being embedded in another program, you may wish to change the -** prefix to reflect your program's name, so that if your program exits -** prematurely, old temporary files can be easily identified. This can be done -** using -DSQLITE_TEMP_FILE_PREFIX=myprefix_ on the compiler command line. -** -** 2006-10-31: The default prefix used to be "sqlite_". But then -** Mcafee started using SQLite in their anti-virus product and it -** started putting files with the "sqlite" name in the c:/temp folder. -** This annoyed many windows users. Those users would then do a -** Google search for "sqlite", find the telephone numbers of the -** developers and call to wake them up at night and complain. -** For this reason, the default name prefix is changed to be "sqlite" -** spelled backwards. So the temp files are still identified, but -** anybody smart enough to figure out the code is also likely smart -** enough to know that calling the developer will not help get rid -** of the file. -*/ -#ifndef SQLITE_TEMP_FILE_PREFIX -# define SQLITE_TEMP_FILE_PREFIX "etilqs_" -#endif - -/* -** The following values may be passed as the second argument to -** sqlite3OsLock(). The various locks exhibit the following semantics: -** -** SHARED: Any number of processes may hold a SHARED lock simultaneously. -** RESERVED: A single process may hold a RESERVED lock on a file at -** any time. Other processes may hold and obtain new SHARED locks. -** PENDING: A single process may hold a PENDING lock on a file at -** any one time. Existing SHARED locks may persist, but no new -** SHARED locks may be obtained by other processes. -** EXCLUSIVE: An EXCLUSIVE lock precludes all other locks. -** -** PENDING_LOCK may not be passed directly to sqlite3OsLock(). Instead, a -** process that requests an EXCLUSIVE lock may actually obtain a PENDING -** lock. This can be upgraded to an EXCLUSIVE lock by a subsequent call to -** sqlite3OsLock(). -*/ -#define NO_LOCK 0 -#define SHARED_LOCK 1 -#define RESERVED_LOCK 2 -#define PENDING_LOCK 3 -#define EXCLUSIVE_LOCK 4 - -/* -** File Locking Notes: (Mostly about windows but also some info for Unix) -** -** We cannot use LockFileEx() or UnlockFileEx() on Win95/98/ME because -** those functions are not available. So we use only LockFile() and -** UnlockFile(). -** -** LockFile() prevents not just writing but also reading by other processes. -** A SHARED_LOCK is obtained by locking a single randomly-chosen -** byte out of a specific range of bytes. The lock byte is obtained at -** random so two separate readers can probably access the file at the -** same time, unless they are unlucky and choose the same lock byte. -** An EXCLUSIVE_LOCK is obtained by locking all bytes in the range. -** There can only be one writer. A RESERVED_LOCK is obtained by locking -** a single byte of the file that is designated as the reserved lock byte. -** A PENDING_LOCK is obtained by locking a designated byte different from -** the RESERVED_LOCK byte. -** -** On WinNT/2K/XP systems, LockFileEx() and UnlockFileEx() are available, -** which means we can use reader/writer locks. When reader/writer locks -** are used, the lock is placed on the same range of bytes that is used -** for probabilistic locking in Win95/98/ME. Hence, the locking scheme -** will support two or more Win95 readers or two or more WinNT readers. -** But a single Win95 reader will lock out all WinNT readers and a single -** WinNT reader will lock out all other Win95 readers. -** -** The following #defines specify the range of bytes used for locking. -** SHARED_SIZE is the number of bytes available in the pool from which -** a random byte is selected for a shared lock. The pool of bytes for -** shared locks begins at SHARED_FIRST. -** -** The same locking strategy and -** byte ranges are used for Unix. This leaves open the possibility of having -** clients on win95, winNT, and unix all talking to the same shared file -** and all locking correctly. To do so would require that samba (or whatever -** tool is being used for file sharing) implements locks correctly between -** windows and unix. I'm guessing that isn't likely to happen, but by -** using the same locking range we are at least open to the possibility. -** -** Locking in windows is manditory. For this reason, we cannot store -** actual data in the bytes used for locking. The pager never allocates -** the pages involved in locking therefore. SHARED_SIZE is selected so -** that all locks will fit on a single page even at the minimum page size. -** PENDING_BYTE defines the beginning of the locks. By default PENDING_BYTE -** is set high so that we don't have to allocate an unused page except -** for very large databases. But one should test the page skipping logic -** by setting PENDING_BYTE low and running the entire regression suite. -** -** Changing the value of PENDING_BYTE results in a subtly incompatible -** file format. Depending on how it is changed, you might not notice -** the incompatibility right away, even running a full regression test. -** The default location of PENDING_BYTE is the first byte past the -** 1GB boundary. -** -*/ -#ifdef SQLITE_OMIT_WSD -# define PENDING_BYTE (0x40000000) -#else -# define PENDING_BYTE sqlite3PendingByte -#endif -#define RESERVED_BYTE (PENDING_BYTE+1) -#define SHARED_FIRST (PENDING_BYTE+2) -#define SHARED_SIZE 510 - -/* -** Wrapper around OS specific sqlite3_os_init() function. -*/ -SQLITE_PRIVATE int sqlite3OsInit(void); - -/* -** Functions for accessing sqlite3_file methods -*/ -SQLITE_PRIVATE void sqlite3OsClose(sqlite3_file*); -SQLITE_PRIVATE int sqlite3OsRead(sqlite3_file*, void*, int amt, i64 offset); -SQLITE_PRIVATE int sqlite3OsWrite(sqlite3_file*, const void*, int amt, i64 offset); -SQLITE_PRIVATE int sqlite3OsTruncate(sqlite3_file*, i64 size); -SQLITE_PRIVATE int sqlite3OsSync(sqlite3_file*, int); -SQLITE_PRIVATE int sqlite3OsFileSize(sqlite3_file*, i64 *pSize); -SQLITE_PRIVATE int sqlite3OsLock(sqlite3_file*, int); -SQLITE_PRIVATE int sqlite3OsUnlock(sqlite3_file*, int); -SQLITE_PRIVATE int sqlite3OsCheckReservedLock(sqlite3_file *id, int *pResOut); -SQLITE_PRIVATE int sqlite3OsFileControl(sqlite3_file*,int,void*); -SQLITE_PRIVATE void sqlite3OsFileControlHint(sqlite3_file*,int,void*); -#define SQLITE_FCNTL_DB_UNCHANGED 0xca093fa0 -SQLITE_PRIVATE int sqlite3OsSectorSize(sqlite3_file *id); -SQLITE_PRIVATE int sqlite3OsDeviceCharacteristics(sqlite3_file *id); -#ifndef SQLITE_OMIT_WAL -SQLITE_PRIVATE int sqlite3OsShmMap(sqlite3_file *,int,int,int,void volatile **); -SQLITE_PRIVATE int sqlite3OsShmLock(sqlite3_file *id, int, int, int); -SQLITE_PRIVATE void sqlite3OsShmBarrier(sqlite3_file *id); -SQLITE_PRIVATE int sqlite3OsShmUnmap(sqlite3_file *id, int); -#endif /* SQLITE_OMIT_WAL */ -SQLITE_PRIVATE int sqlite3OsFetch(sqlite3_file *id, i64, int, void **); -SQLITE_PRIVATE int sqlite3OsUnfetch(sqlite3_file *, i64, void *); - - -/* -** Functions for accessing sqlite3_vfs methods -*/ -SQLITE_PRIVATE int sqlite3OsOpen(sqlite3_vfs *, const char *, sqlite3_file*, int, int *); -SQLITE_PRIVATE int sqlite3OsDelete(sqlite3_vfs *, const char *, int); -SQLITE_PRIVATE int sqlite3OsAccess(sqlite3_vfs *, const char *, int, int *pResOut); -SQLITE_PRIVATE int sqlite3OsFullPathname(sqlite3_vfs *, const char *, int, char *); -#ifndef SQLITE_OMIT_LOAD_EXTENSION -SQLITE_PRIVATE void *sqlite3OsDlOpen(sqlite3_vfs *, const char *); -SQLITE_PRIVATE void sqlite3OsDlError(sqlite3_vfs *, int, char *); -SQLITE_PRIVATE void (*sqlite3OsDlSym(sqlite3_vfs *, void *, const char *))(void); -SQLITE_PRIVATE void sqlite3OsDlClose(sqlite3_vfs *, void *); -#endif /* SQLITE_OMIT_LOAD_EXTENSION */ -SQLITE_PRIVATE int sqlite3OsRandomness(sqlite3_vfs *, int, char *); -SQLITE_PRIVATE int sqlite3OsSleep(sqlite3_vfs *, int); -SQLITE_PRIVATE int sqlite3OsGetLastError(sqlite3_vfs*); -SQLITE_PRIVATE int sqlite3OsCurrentTimeInt64(sqlite3_vfs *, sqlite3_int64*); - -/* -** Convenience functions for opening and closing files using -** sqlite3_malloc() to obtain space for the file-handle structure. -*/ -SQLITE_PRIVATE int sqlite3OsOpenMalloc(sqlite3_vfs *, const char *, sqlite3_file **, int,int*); -SQLITE_PRIVATE void sqlite3OsCloseFree(sqlite3_file *); - -#endif /* _SQLITE_OS_H_ */ - -/************** End of os.h **************************************************/ -/************** Continuing where we left off in sqliteInt.h ******************/ /************** Include mutex.h in the middle of sqliteInt.h *****************/ /************** Begin file mutex.h *******************************************/ /* @@ -16743,6 +16837,7 @@ struct Lookaside { #endif /* SQLITE_OMIT_TWOSIZE_LOOKASIDE */ void *pStart; /* First byte of available memory space */ void *pEnd; /* First byte past end of available space */ + void *pTrueEnd; /* True value of pEnd, when db->pnBytesFreed!=0 */ }; struct LookasideSlot { LookasideSlot *pNext; /* Next buffer in the list of free buffers */ @@ -17087,6 +17182,7 @@ struct sqlite3 { #define SQLITE_ReleaseReg 0x00400000 /* Use OP_ReleaseReg for testing */ #define SQLITE_FlttnUnionAll 0x00800000 /* Disable the UNION ALL flattener */ /* TH3 expects this value ^^^^^^^^^^ See flatten04.test */ +#define SQLITE_IndexedExpr 0x01000000 /* Pull exprs from index when able */ #define SQLITE_AllOpts 0xffffffff /* All optimizations */ /* @@ -17659,7 +17755,7 @@ struct Table { #ifndef SQLITE_OMIT_VIRTUALTABLE # define IsVirtual(X) ((X)->eTabType==TABTYP_VTAB) # define ExprIsVtab(X) \ - ((X)->op==TK_COLUMN && (X)->y.pTab!=0 && (X)->y.pTab->eTabType==TABTYP_VTAB) + ((X)->op==TK_COLUMN && (X)->y.pTab->eTabType==TABTYP_VTAB) #else # define IsVirtual(X) 0 # define ExprIsVtab(X) 0 @@ -17876,10 +17972,22 @@ struct UnpackedRecord { ** The Index.onError field determines whether or not the indexed columns ** must be unique and what to do if they are not. When Index.onError=OE_None, ** it means this is not a unique index. Otherwise it is a unique index -** and the value of Index.onError indicate the which conflict resolution -** algorithm to employ whenever an attempt is made to insert a non-unique +** and the value of Index.onError indicates which conflict resolution +** algorithm to employ when an attempt is made to insert a non-unique ** element. ** +** The colNotIdxed bitmask is used in combination with SrcItem.colUsed +** for a fast test to see if an index can serve as a covering index. +** colNotIdxed has a 1 bit for every column of the original table that +** is *not* available in the index. Thus the expression +** "colUsed & colNotIdxed" will be non-zero if the index is not a +** covering index. The most significant bit of of colNotIdxed will always +** be true (note-20221022-a). If a column beyond the 63rd column of the +** table is used, the "colUsed & colNotIdxed" test will always be non-zero +** and we have to assume either that the index is not covering, or use +** an alternative (slower) algorithm to determine whether or not +** the index is covering. +** ** While parsing a CREATE TABLE or CREATE INDEX statement in order to ** generate VDBE code (as opposed to parsing one read from an sqlite_schema ** table as part of parsing an existing database schema), transient instances @@ -17915,6 +18023,8 @@ struct Index { unsigned bNoQuery:1; /* Do not use this index to optimize queries */ unsigned bAscKeyBug:1; /* True if the bba7b69f9849b5bf bug applies */ unsigned bHasVCol:1; /* Index references one or more VIRTUAL columns */ + unsigned bHasExpr:1; /* Index contains an expression, either a literal + ** expression, or a reference to a VIRTUAL column */ #ifdef SQLITE_ENABLE_STAT4 int nSample; /* Number of elements in aSample[] */ int nSampleCol; /* Size of IndexSample.anEq[] and so on */ @@ -17923,7 +18033,7 @@ struct Index { tRowcnt *aiRowEst; /* Non-logarithmic stat1 data for this index */ tRowcnt nRowEst0; /* Non-logarithmic number of rows in the index */ #endif - Bitmask colNotIdxed; /* 0 for unindexed columns in pTab */ + Bitmask colNotIdxed; /* Unindexed columns in pTab */ }; /* @@ -18191,7 +18301,7 @@ struct Expr { #define EP_Reduced 0x004000 /* Expr struct EXPR_REDUCEDSIZE bytes only */ #define EP_Win 0x008000 /* Contains window functions */ #define EP_TokenOnly 0x010000 /* Expr struct EXPR_TOKENONLYSIZE bytes only */ -#define EP_MemToken 0x020000 /* Need to sqlite3DbFree() Expr.zToken */ + /* 0x020000 // Available for reuse */ #define EP_IfNullRow 0x040000 /* The TK_IF_NULL_ROW opcode */ #define EP_Unlikely 0x080000 /* unlikely() or likelihood() function */ #define EP_ConstFunc 0x100000 /* A SQLITE_FUNC_CONSTANT or _SLOCHNG function */ @@ -18376,6 +18486,14 @@ struct IdList { ** The SrcItem object represents a single term in the FROM clause of a query. ** The SrcList object is mostly an array of SrcItems. ** +** The jointype starts out showing the join type between the current table +** and the next table on the list. The parser builds the list this way. +** But sqlite3SrcListShiftJoinType() later shifts the jointypes so that each +** jointype expresses the join between the table and the previous table. +** +** In the colUsed field, the high-order bit (bit 63) is set if the table +** contains more than 63 columns and the 64-th or later column is used. +** ** Union member validity: ** ** u1.zIndexedBy fg.isIndexedBy && !fg.isTabFunc @@ -18415,14 +18533,14 @@ struct SrcItem { Expr *pOn; /* fg.isUsing==0 => The ON clause of a join */ IdList *pUsing; /* fg.isUsing==1 => The USING clause of a join */ } u3; - Bitmask colUsed; /* Bit N (1<62 */ union { char *zIndexedBy; /* Identifier from "INDEXED BY " clause */ ExprList *pFuncArg; /* Arguments to table-valued-function */ } u1; union { Index *pIBIndex; /* Index structure corresponding to u1.zIndexedBy */ - CteUse *pCteUse; /* CTE Usage info info fg.isCte is true */ + CteUse *pCteUse; /* CTE Usage info when fg.isCte is true */ } u2; }; @@ -18436,23 +18554,11 @@ struct OnOrUsing { }; /* -** The following structure describes the FROM clause of a SELECT statement. -** Each table or subquery in the FROM clause is a separate element of -** the SrcList.a[] array. +** This object represents one or more tables that are the source of +** content for an SQL statement. For example, a single SrcList object +** is used to hold the FROM clause of a SELECT statement. SrcList also +** represents the target tables for DELETE, INSERT, and UPDATE statements. ** -** With the addition of multiple database support, the following structure -** can also be used to describe a particular table such as the table that -** is modified by an INSERT, DELETE, or UPDATE statement. In standard SQL, -** such a table must be a simple name: ID. But in SQLite, the table can -** now be identified by a database name, a dot, then the table name: ID.ID. -** -** The jointype starts out showing the join type between the current table -** and the next table on the list. The parser builds the list this way. -** But sqlite3SrcListShiftJoinType() later shifts the jointypes so that each -** jointype expresses the join between the table and the previous table. -** -** In the colUsed field, the high-order bit (bit 63) is set if the table -** contains more than 63 columns and the 64-th or later column is used. */ struct SrcList { int nSrc; /* Number of tables or subqueries in the FROM clause */ @@ -18797,7 +18903,7 @@ struct SelectDest { int iSDParm2; /* A second parameter for the eDest disposal method */ int iSdst; /* Base register where results are written */ int nSdst; /* Number of registers allocated */ - char *zAffSdst; /* Affinity used when eDest==SRT_Set */ + char *zAffSdst; /* Affinity used for SRT_Set, SRT_Table, and similar */ ExprList *pOrderBy; /* Key columns for SRT_Queue and SRT_DistQueue */ }; @@ -18862,6 +18968,28 @@ struct TriggerPrg { # define DbMaskNonZero(M) (M)!=0 #endif +/* +** For each index X that has as one of its arguments either an expression +** or the name of a virtual generated column, and if X is in scope such that +** the value of the expression can simply be read from the index, then +** there is an instance of this object on the Parse.pIdxExpr list. +** +** During code generation, while generating code to evaluate expressions, +** this list is consulted and if a matching expression is found, the value +** is read from the index rather than being recomputed. +*/ +struct IndexedExpr { + Expr *pExpr; /* The expression contained in the index */ + int iDataCur; /* The data cursor associated with the index */ + int iIdxCur; /* The index cursor */ + int iIdxCol; /* The index column that contains value of pExpr */ + u8 bMaybeNullRow; /* True if we need an OP_IfNullRow check */ + IndexedExpr *pIENext; /* Next in a list of all indexed expressions */ +#ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS + const char *zIdxName; /* Name of index, used only for bytecode comments */ +#endif +}; + /* ** An instance of the ParseCleanup object specifies an operation that ** should be performed after parsing to deallocation resources obtained @@ -18903,7 +19031,7 @@ struct Parse { u8 hasCompound; /* Need to invoke convertCompoundSelectToSubquery() */ u8 okConstFactor; /* OK to factor out constants */ u8 disableLookaside; /* Number of times lookaside has been disabled */ - u8 disableVtab; /* Disable all virtual tables for this parse */ + u8 prepFlags; /* SQLITE_PREPARE_* flags */ u8 withinRJSubrtn; /* Nesting level for RIGHT JOIN body subroutines */ #if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST) u8 earlyCleanup; /* OOM inside sqlite3ParserAddCleanup() */ @@ -18920,6 +19048,7 @@ struct Parse { int nLabelAlloc; /* Number of slots in aLabel */ int *aLabel; /* Space to hold the labels */ ExprList *pConstExpr;/* Constant expressions */ + IndexedExpr *pIdxExpr;/* List of expressions used by active indexes */ Token constraintName;/* Name of the constraint currently being parsed */ yDbMask writeMask; /* Start a write transaction on these databases */ yDbMask cookieMask; /* Bitmask of schema verified databases */ @@ -19355,15 +19484,15 @@ struct Walker { struct RefSrcList *pRefSrcList; /* sqlite3ReferencesSrcList() */ int *aiCol; /* array of column indexes */ struct IdxCover *pIdxCover; /* Check for index coverage */ - struct IdxExprTrans *pIdxTrans; /* Convert idxed expr to column */ ExprList *pGroupBy; /* GROUP BY clause */ Select *pSelect; /* HAVING to WHERE clause ctx */ struct WindowRewrite *pRewrite; /* Window rewrite context */ struct WhereConst *pConst; /* WHERE clause constants */ struct RenameCtx *pRename; /* RENAME COLUMN context */ struct Table *pTab; /* Table of generated column */ + struct CoveringIndexCheck *pCovIdxCk; /* Check for covering index */ SrcItem *pSrcItem; /* A single FROM clause item */ - DbFixer *pFix; + DbFixer *pFix; /* See sqlite3FixSelect() */ } u; }; @@ -19669,6 +19798,7 @@ SQLITE_PRIVATE void *sqlite3DbReallocOrFree(sqlite3 *, void *, u64); SQLITE_PRIVATE void *sqlite3DbRealloc(sqlite3 *, void *, u64); SQLITE_PRIVATE void sqlite3DbFree(sqlite3*, void*); SQLITE_PRIVATE void sqlite3DbFreeNN(sqlite3*, void*); +SQLITE_PRIVATE void sqlite3DbNNFreeNN(sqlite3*, void*); SQLITE_PRIVATE int sqlite3MallocSize(const void*); SQLITE_PRIVATE int sqlite3DbMallocSize(sqlite3*, const void*); SQLITE_PRIVATE void *sqlite3PageMalloc(int); @@ -19689,12 +19819,16 @@ SQLITE_PRIVATE int sqlite3HeapNearlyFull(void); */ #ifdef SQLITE_USE_ALLOCA # define sqlite3StackAllocRaw(D,N) alloca(N) +# define sqlite3StackAllocRawNN(D,N) alloca(N) # define sqlite3StackAllocZero(D,N) memset(alloca(N), 0, N) # define sqlite3StackFree(D,P) +# define sqlite3StackFreeNN(D,P) #else # define sqlite3StackAllocRaw(D,N) sqlite3DbMallocRaw(D,N) +# define sqlite3StackAllocRawNN(D,N) sqlite3DbMallocRawNN(D,N) # define sqlite3StackAllocZero(D,N) sqlite3DbMallocZero(D,N) # define sqlite3StackFree(D,P) sqlite3DbFree(D,P) +# define sqlite3StackFreeNN(D,P) sqlite3DbFreeNN(D,P) #endif /* Do not allow both MEMSYS5 and MEMSYS3 to be defined together. If they @@ -20193,6 +20327,7 @@ SQLITE_PRIVATE int sqlite3FixSelect(DbFixer*, Select*); SQLITE_PRIVATE int sqlite3FixExpr(DbFixer*, Expr*); SQLITE_PRIVATE int sqlite3FixTriggerStep(DbFixer*, TriggerStep*); SQLITE_PRIVATE int sqlite3RealSameAsInt(double,sqlite3_int64); +SQLITE_PRIVATE i64 sqlite3RealToI64(double); SQLITE_PRIVATE void sqlite3Int64ToText(i64,char*); SQLITE_PRIVATE int sqlite3AtoF(const char *z, double*, int, u8); SQLITE_PRIVATE int sqlite3GetInt32(const char *, int*); @@ -20238,6 +20373,7 @@ SQLITE_PRIVATE int sqlite3VarintLen(u64 v); SQLITE_PRIVATE const char *sqlite3IndexAffinityStr(sqlite3*, Index*); +SQLITE_PRIVATE char *sqlite3TableAffinityStr(sqlite3*,const Table*); SQLITE_PRIVATE void sqlite3TableAffinity(Vdbe*, Table*, int); SQLITE_PRIVATE char sqlite3CompareAffinity(const Expr *pExpr, char aff2); SQLITE_PRIVATE int sqlite3IndexAffinityOk(const Expr *pExpr, char idx_affinity); @@ -20309,7 +20445,6 @@ SQLITE_PRIVATE const unsigned char sqlite3OpcodeProperty[]; SQLITE_PRIVATE const char sqlite3StrBINARY[]; SQLITE_PRIVATE const unsigned char sqlite3StdTypeLen[]; SQLITE_PRIVATE const char sqlite3StdTypeAffinity[]; -SQLITE_PRIVATE const char sqlite3StdTypeMap[]; SQLITE_PRIVATE const char *sqlite3StdType[]; SQLITE_PRIVATE const unsigned char sqlite3UpperToLower[]; SQLITE_PRIVATE const unsigned char *sqlite3aLTb; @@ -20753,6 +20888,10 @@ SQLITE_PRIVATE void sqlite3VectorErrorMsg(Parse*, Expr*); SQLITE_PRIVATE const char **sqlite3CompileOptions(int *pnOpt); #endif +#if SQLITE_OS_UNIX && defined(SQLITE_OS_KV_OPTIONAL) +SQLITE_PRIVATE int sqlite3KvvfsInit(void); +#endif + #endif /* SQLITEINT_H */ /************** End of sqliteInt.h *******************************************/ @@ -20984,7 +21123,7 @@ SQLITE_API extern int sqlite3_open_file_count; ** autoconf-based build */ #if defined(_HAVE_SQLITE_CONFIG_H) && !defined(SQLITECONFIG_H) -/* #include "config.h" */ +/* #include "sqlite_cfg.h" */ #define SQLITECONFIG_H 1 #endif @@ -21149,6 +21288,9 @@ static const char * const sqlite3azCompileOpt[] = { #ifdef SQLITE_DISABLE_SKIPAHEAD_DISTINCT "DISABLE_SKIPAHEAD_DISTINCT", #endif +#ifdef SQLITE_DQS + "DQS=" CTIMEOPT_VAL(SQLITE_DQS), +#endif #ifdef SQLITE_ENABLE_8_3_NAMES "ENABLE_8_3_NAMES=" CTIMEOPT_VAL(SQLITE_ENABLE_8_3_NAMES), #endif @@ -21639,9 +21781,6 @@ static const char * const sqlite3azCompileOpt[] = { #ifdef SQLITE_OMIT_XFER_OPT "OMIT_XFER_OPT", #endif -#ifdef SQLITE_PCACHE_SEPARATE_HEADER - "PCACHE_SEPARATE_HEADER", -#endif #ifdef SQLITE_PERFORMANCE_TRACE "PERFORMANCE_TRACE", #endif @@ -22121,10 +22260,6 @@ SQLITE_PRIVATE const char sqlite3StrBINARY[] = "BINARY"; ** ** sqlite3StdTypeAffinity[] The affinity associated with each entry ** in sqlite3StdType[]. -** -** sqlite3StdTypeMap[] The type value (as returned from -** sqlite3_column_type() or sqlite3_value_type()) -** for each entry in sqlite3StdType[]. */ SQLITE_PRIVATE const unsigned char sqlite3StdTypeLen[] = { 3, 4, 3, 7, 4, 4 }; SQLITE_PRIVATE const char sqlite3StdTypeAffinity[] = { @@ -22135,14 +22270,6 @@ SQLITE_PRIVATE const char sqlite3StdTypeAffinity[] = { SQLITE_AFF_REAL, SQLITE_AFF_TEXT }; -SQLITE_PRIVATE const char sqlite3StdTypeMap[] = { - 0, - SQLITE_BLOB, - SQLITE_INTEGER, - SQLITE_INTEGER, - SQLITE_FLOAT, - SQLITE_TEXT -}; SQLITE_PRIVATE const char *sqlite3StdType[] = { "ANY", "BLOB", @@ -22594,7 +22721,7 @@ struct DblquoteStr { */ struct Vdbe { sqlite3 *db; /* The database connection that owns this statement */ - Vdbe *pPrev,*pNext; /* Linked list of VDBEs with the same Vdbe.db */ + Vdbe **ppVPrev,*pVNext; /* Linked list of VDBEs with the same Vdbe.db */ Parse *pParse; /* Parsing context used to create this Vdbe */ ynVar nVar; /* Number of entries in aVar[] */ int nMem; /* Number of memory locations currently allocated */ @@ -23152,6 +23279,8 @@ SQLITE_API int sqlite3_db_status( sqlite3BtreeEnterAll(db); db->pnBytesFreed = &nByte; + assert( db->lookaside.pEnd==db->lookaside.pTrueEnd ); + db->lookaside.pEnd = db->lookaside.pStart; for(i=0; inDb; i++){ Schema *pSchema = db->aDb[i].pSchema; if( ALWAYS(pSchema!=0) ){ @@ -23177,6 +23306,7 @@ SQLITE_API int sqlite3_db_status( } } db->pnBytesFreed = 0; + db->lookaside.pEnd = db->lookaside.pTrueEnd; sqlite3BtreeLeaveAll(db); *pHighwater = 0; @@ -23194,9 +23324,12 @@ SQLITE_API int sqlite3_db_status( int nByte = 0; /* Used to accumulate return value */ db->pnBytesFreed = &nByte; - for(pVdbe=db->pVdbe; pVdbe; pVdbe=pVdbe->pNext){ + assert( db->lookaside.pEnd==db->lookaside.pTrueEnd ); + db->lookaside.pEnd = db->lookaside.pStart; + for(pVdbe=db->pVdbe; pVdbe; pVdbe=pVdbe->pVNext){ sqlite3VdbeDelete(pVdbe); } + db->lookaside.pEnd = db->lookaside.pTrueEnd; db->pnBytesFreed = 0; *pHighwater = 0; /* IMP: R-64479-57858 */ @@ -23532,7 +23665,7 @@ static void computeJD(DateTime *p){ p->iJD = (sqlite3_int64)((X1 + X2 + D + B - 1524.5 ) * 86400000); p->validJD = 1; if( p->validHMS ){ - p->iJD += p->h*3600000 + p->m*60000 + (sqlite3_int64)(p->s*1000); + p->iJD += p->h*3600000 + p->m*60000 + (sqlite3_int64)(p->s*1000 + 0.5); if( p->validTZ ){ p->iJD -= p->tz*60000; p->validYMD = 0; @@ -24041,7 +24174,7 @@ static int parseModifier( */ if( sqlite3_strnicmp(z, "weekday ", 8)==0 && sqlite3AtoF(&z[8], &r, sqlite3Strlen30(&z[8]), SQLITE_UTF8)>0 - && (n=(int)r)==r && n>=0 && r<7 ){ + && r>=0.0 && r<7.0 && (n=(int)r)==r ){ sqlite3_int64 Z; computeYMD_HMS(p); p->validTZ = 0; @@ -24722,9 +24855,11 @@ SQLITE_PRIVATE int sqlite3OsFileSize(sqlite3_file *id, i64 *pSize){ } SQLITE_PRIVATE int sqlite3OsLock(sqlite3_file *id, int lockType){ DO_OS_MALLOC_TEST(id); + assert( lockType>=SQLITE_LOCK_SHARED && lockType<=SQLITE_LOCK_EXCLUSIVE ); return id->pMethods->xLock(id, lockType); } SQLITE_PRIVATE int sqlite3OsUnlock(sqlite3_file *id, int lockType){ + assert( lockType==SQLITE_LOCK_NONE || lockType==SQLITE_LOCK_SHARED ); return id->pMethods->xUnlock(id, lockType); } SQLITE_PRIVATE int sqlite3OsCheckReservedLock(sqlite3_file *id, int *pResOut){ @@ -24839,6 +24974,7 @@ SQLITE_PRIVATE int sqlite3OsOpen( ** down into the VFS layer. Some SQLITE_OPEN_ flags (for example, ** SQLITE_OPEN_FULLMUTEX or SQLITE_OPEN_SHAREDCACHE) are blocked before ** reaching the VFS. */ + assert( zPath || (flags & SQLITE_OPEN_EXCLUSIVE) ); rc = pVfs->xOpen(pVfs, zPath, pFile, flags & 0x1087f7f, pFlagsOut); assert( rc==SQLITE_OK || pFile->pMethods==0 ); return rc; @@ -29057,18 +29193,34 @@ static void mallocWithAlarm(int n, void **pp){ *pp = p; } +/* +** Maximum size of any single memory allocation. +** +** This is not a limit on the total amount of memory used. This is +** a limit on the size parameter to sqlite3_malloc() and sqlite3_realloc(). +** +** The upper bound is slightly less than 2GiB: 0x7ffffeff == 2,147,483,391 +** This provides a 256-byte safety margin for defense against 32-bit +** signed integer overflow bugs when computing memory allocation sizes. +** Parnoid applications might want to reduce the maximum allocation size +** further for an even larger safety margin. 0x3fffffff or 0x0fffffff +** or even smaller would be reasonable upper bounds on the size of a memory +** allocations for most applications. +*/ +#ifndef SQLITE_MAX_ALLOCATION_SIZE +# define SQLITE_MAX_ALLOCATION_SIZE 2147483391 +#endif +#if SQLITE_MAX_ALLOCATION_SIZE>2147483391 +# error Maximum size for SQLITE_MAX_ALLOCATION_SIZE is 2147483391 +#endif + /* ** Allocate memory. This routine is like sqlite3_malloc() except that it ** assumes the memory subsystem has already been initialized. */ SQLITE_PRIVATE void *sqlite3Malloc(u64 n){ void *p; - if( n==0 || n>=0x7fffff00 ){ - /* A memory allocation of a number of bytes which is near the maximum - ** signed integer value might cause an integer overflow inside of the - ** xMalloc(). Hence we limit the maximum size to 0x7fffff00, giving - ** 255 bytes of overhead. SQLite itself will never use anything near - ** this amount. The only way to reach the limit is with sqlite3_malloc() */ + if( n==0 || n>SQLITE_MAX_ALLOCATION_SIZE ){ p = 0; }else if( sqlite3GlobalConfig.bMemstat ){ sqlite3_mutex_enter(mem0.mutex); @@ -29104,7 +29256,7 @@ SQLITE_API void *sqlite3_malloc64(sqlite3_uint64 n){ */ #ifndef SQLITE_OMIT_LOOKASIDE static int isLookaside(sqlite3 *db, const void *p){ - return SQLITE_WITHIN(p, db->lookaside.pStart, db->lookaside.pEnd); + return SQLITE_WITHIN(p, db->lookaside.pStart, db->lookaside.pTrueEnd); } #else #define isLookaside(A,B) 0 @@ -29128,18 +29280,16 @@ static int lookasideMallocSize(sqlite3 *db, const void *p){ SQLITE_PRIVATE int sqlite3DbMallocSize(sqlite3 *db, const void *p){ assert( p!=0 ); #ifdef SQLITE_DEBUG - if( db==0 || !isLookaside(db,p) ){ - if( db==0 ){ - assert( sqlite3MemdebugNoType(p, (u8)~MEMTYPE_HEAP) ); - assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) ); - }else{ - assert( sqlite3MemdebugHasType(p, (MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); - assert( sqlite3MemdebugNoType(p, (u8)~(MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); - } + if( db==0 ){ + assert( sqlite3MemdebugNoType(p, (u8)~MEMTYPE_HEAP) ); + assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) ); + }else if( !isLookaside(db,p) ){ + assert( sqlite3MemdebugHasType(p, (MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); + assert( sqlite3MemdebugNoType(p, (u8)~(MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); } #endif if( db ){ - if( ((uptr)p)<(uptr)(db->lookaside.pEnd) ){ + if( ((uptr)p)<(uptr)(db->lookaside.pTrueEnd) ){ #ifndef SQLITE_OMIT_TWOSIZE_LOOKASIDE if( ((uptr)p)>=(uptr)(db->lookaside.pMiddle) ){ assert( sqlite3_mutex_held(db->mutex) ); @@ -29195,14 +29345,11 @@ SQLITE_PRIVATE void sqlite3DbFreeNN(sqlite3 *db, void *p){ assert( db==0 || sqlite3_mutex_held(db->mutex) ); assert( p!=0 ); if( db ){ - if( db->pnBytesFreed ){ - measureAllocationSize(db, p); - return; - } if( ((uptr)p)<(uptr)(db->lookaside.pEnd) ){ #ifndef SQLITE_OMIT_TWOSIZE_LOOKASIDE if( ((uptr)p)>=(uptr)(db->lookaside.pMiddle) ){ LookasideSlot *pBuf = (LookasideSlot*)p; + assert( db->pnBytesFreed==0 ); #ifdef SQLITE_DEBUG memset(p, 0xaa, LOOKASIDE_SMALL); /* Trash freed content */ #endif @@ -29213,6 +29360,7 @@ SQLITE_PRIVATE void sqlite3DbFreeNN(sqlite3 *db, void *p){ #endif /* SQLITE_OMIT_TWOSIZE_LOOKASIDE */ if( ((uptr)p)>=(uptr)(db->lookaside.pStart) ){ LookasideSlot *pBuf = (LookasideSlot*)p; + assert( db->pnBytesFreed==0 ); #ifdef SQLITE_DEBUG memset(p, 0xaa, db->lookaside.szTrue); /* Trash freed content */ #endif @@ -29221,6 +29369,10 @@ SQLITE_PRIVATE void sqlite3DbFreeNN(sqlite3 *db, void *p){ return; } } + if( db->pnBytesFreed ){ + measureAllocationSize(db, p); + return; + } } assert( sqlite3MemdebugHasType(p, (MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); assert( sqlite3MemdebugNoType(p, (u8)~(MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); @@ -29228,6 +29380,43 @@ SQLITE_PRIVATE void sqlite3DbFreeNN(sqlite3 *db, void *p){ sqlite3MemdebugSetType(p, MEMTYPE_HEAP); sqlite3_free(p); } +SQLITE_PRIVATE void sqlite3DbNNFreeNN(sqlite3 *db, void *p){ + assert( db!=0 ); + assert( sqlite3_mutex_held(db->mutex) ); + assert( p!=0 ); + if( ((uptr)p)<(uptr)(db->lookaside.pEnd) ){ +#ifndef SQLITE_OMIT_TWOSIZE_LOOKASIDE + if( ((uptr)p)>=(uptr)(db->lookaside.pMiddle) ){ + LookasideSlot *pBuf = (LookasideSlot*)p; + assert( db->pnBytesFreed==0 ); +#ifdef SQLITE_DEBUG + memset(p, 0xaa, LOOKASIDE_SMALL); /* Trash freed content */ +#endif + pBuf->pNext = db->lookaside.pSmallFree; + db->lookaside.pSmallFree = pBuf; + return; + } +#endif /* SQLITE_OMIT_TWOSIZE_LOOKASIDE */ + if( ((uptr)p)>=(uptr)(db->lookaside.pStart) ){ + LookasideSlot *pBuf = (LookasideSlot*)p; + assert( db->pnBytesFreed==0 ); +#ifdef SQLITE_DEBUG + memset(p, 0xaa, db->lookaside.szTrue); /* Trash freed content */ +#endif + pBuf->pNext = db->lookaside.pFree; + db->lookaside.pFree = pBuf; + return; + } + } + if( db->pnBytesFreed ){ + measureAllocationSize(db, p); + return; + } + assert( sqlite3MemdebugHasType(p, (MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); + assert( sqlite3MemdebugNoType(p, (u8)~(MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); + sqlite3MemdebugSetType(p, MEMTYPE_HEAP); + sqlite3_free(p); +} SQLITE_PRIVATE void sqlite3DbFree(sqlite3 *db, void *p){ assert( db==0 || sqlite3_mutex_held(db->mutex) ); if( p ) sqlite3DbFreeNN(db, p); @@ -29563,8 +29752,13 @@ SQLITE_PRIVATE void *sqlite3OomFault(sqlite3 *db){ } DisableLookaside; if( db->pParse ){ + Parse *pParse; sqlite3ErrorMsg(db->pParse, "out of memory"); db->pParse->rc = SQLITE_NOMEM_BKPT; + for(pParse=db->pParse->pOuterParse; pParse; pParse = pParse->pOuterParse){ + pParse->nErr++; + pParse->rc = SQLITE_NOMEM; + } } } return 0; @@ -32334,16 +32528,41 @@ SQLITE_PRIVATE void sqlite3ShowWinFunc(const Window *p){ sqlite3TreeViewWinFunc( ** This structure is the current state of the generator. */ static SQLITE_WSD struct sqlite3PrngType { - unsigned char isInit; /* True if initialized */ - unsigned char i, j; /* State variables */ - unsigned char s[256]; /* State variables */ + u32 s[16]; /* 64 bytes of chacha20 state */ + u8 out[64]; /* Output bytes */ + u8 n; /* Output bytes remaining */ } sqlite3Prng; + +/* The RFC-7539 ChaCha20 block function +*/ +#define ROTL(a,b) (((a) << (b)) | ((a) >> (32 - (b)))) +#define QR(a, b, c, d) ( \ + a += b, d ^= a, d = ROTL(d,16), \ + c += d, b ^= c, b = ROTL(b,12), \ + a += b, d ^= a, d = ROTL(d, 8), \ + c += d, b ^= c, b = ROTL(b, 7)) +static void chacha_block(u32 *out, const u32 *in){ + int i; + u32 x[16]; + memcpy(x, in, 64); + for(i=0; i<10; i++){ + QR(x[0], x[4], x[ 8], x[12]); + QR(x[1], x[5], x[ 9], x[13]); + QR(x[2], x[6], x[10], x[14]); + QR(x[3], x[7], x[11], x[15]); + QR(x[0], x[5], x[10], x[15]); + QR(x[1], x[6], x[11], x[12]); + QR(x[2], x[7], x[ 8], x[13]); + QR(x[3], x[4], x[ 9], x[14]); + } + for(i=0; i<16; i++) out[i] = x[i]+in[i]; +} + /* ** Return N random bytes. */ SQLITE_API void sqlite3_randomness(int N, void *pBuf){ - unsigned char t; unsigned char *zBuf = pBuf; /* The "wsdPrng" macro will resolve to the pseudo-random number generator @@ -32373,53 +32592,46 @@ SQLITE_API void sqlite3_randomness(int N, void *pBuf){ sqlite3_mutex_enter(mutex); if( N<=0 || pBuf==0 ){ - wsdPrng.isInit = 0; + wsdPrng.s[0] = 0; sqlite3_mutex_leave(mutex); return; } /* Initialize the state of the random number generator once, - ** the first time this routine is called. The seed value does - ** not need to contain a lot of randomness since we are not - ** trying to do secure encryption or anything like that... - ** - ** Nothing in this file or anywhere else in SQLite does any kind of - ** encryption. The RC4 algorithm is being used as a PRNG (pseudo-random - ** number generator) not as an encryption device. + ** the first time this routine is called. */ - if( !wsdPrng.isInit ){ + if( wsdPrng.s[0]==0 ){ sqlite3_vfs *pVfs = sqlite3_vfs_find(0); - int i; - char k[256]; - wsdPrng.j = 0; - wsdPrng.i = 0; + static const u32 chacha20_init[] = { + 0x61707865, 0x3320646e, 0x79622d32, 0x6b206574 + }; + memcpy(&wsdPrng.s[0], chacha20_init, 16); if( NEVER(pVfs==0) ){ - memset(k, 0, sizeof(k)); + memset(&wsdPrng.s[4], 0, 44); }else{ - sqlite3OsRandomness(pVfs, 256, k); + sqlite3OsRandomness(pVfs, 44, (char*)&wsdPrng.s[4]); } - for(i=0; i<256; i++){ - wsdPrng.s[i] = (u8)i; - } - for(i=0; i<256; i++){ - wsdPrng.j += wsdPrng.s[i] + k[i]; - t = wsdPrng.s[wsdPrng.j]; - wsdPrng.s[wsdPrng.j] = wsdPrng.s[i]; - wsdPrng.s[i] = t; - } - wsdPrng.isInit = 1; + wsdPrng.s[15] = wsdPrng.s[12]; + wsdPrng.s[12] = 0; + wsdPrng.n = 0; } assert( N>0 ); - do{ - wsdPrng.i++; - t = wsdPrng.s[wsdPrng.i]; - wsdPrng.j += t; - wsdPrng.s[wsdPrng.i] = wsdPrng.s[wsdPrng.j]; - wsdPrng.s[wsdPrng.j] = t; - t += wsdPrng.s[wsdPrng.i]; - *(zBuf++) = wsdPrng.s[t]; - }while( --N ); + while( 1 /* exit by break */ ){ + if( N<=wsdPrng.n ){ + memcpy(zBuf, &wsdPrng.out[wsdPrng.n-N], N); + wsdPrng.n -= N; + break; + } + if( wsdPrng.n>0 ){ + memcpy(zBuf, wsdPrng.out, wsdPrng.n); + N -= wsdPrng.n; + zBuf += wsdPrng.n; + } + wsdPrng.s[12]++; + chacha_block((u32*)wsdPrng.out, wsdPrng.s); + wsdPrng.n = 64; + } sqlite3_mutex_leave(mutex); } @@ -33459,7 +33671,7 @@ SQLITE_PRIVATE void sqlite3ErrorMsg(Parse *pParse, const char *zFormat, ...){ va_list ap; sqlite3 *db = pParse->db; assert( db!=0 ); - assert( db->pParse==pParse ); + assert( db->pParse==pParse || db->pParse->pToplevel==pParse ); db->errByteOffset = -2; va_start(ap, zFormat); zMsg = sqlite3VMPrintf(db, zFormat, ap); @@ -35277,48 +35489,48 @@ SQLITE_PRIVATE const char *sqlite3OpcodeName(int i){ /* 5 */ "Vacuum" OpHelp(""), /* 6 */ "VFilter" OpHelp("iplan=r[P3] zplan='P4'"), /* 7 */ "VUpdate" OpHelp("data=r[P3@P2]"), - /* 8 */ "Goto" OpHelp(""), - /* 9 */ "Gosub" OpHelp(""), - /* 10 */ "InitCoroutine" OpHelp(""), - /* 11 */ "Yield" OpHelp(""), - /* 12 */ "MustBeInt" OpHelp(""), - /* 13 */ "Jump" OpHelp(""), - /* 14 */ "Once" OpHelp(""), - /* 15 */ "If" OpHelp(""), - /* 16 */ "IfNot" OpHelp(""), - /* 17 */ "IsNullOrType" OpHelp("if typeof(r[P1]) IN (P3,5) goto P2"), - /* 18 */ "IfNullRow" OpHelp("if P1.nullRow then r[P3]=NULL, goto P2"), + /* 8 */ "Init" OpHelp("Start at P2"), + /* 9 */ "Goto" OpHelp(""), + /* 10 */ "Gosub" OpHelp(""), + /* 11 */ "InitCoroutine" OpHelp(""), + /* 12 */ "Yield" OpHelp(""), + /* 13 */ "MustBeInt" OpHelp(""), + /* 14 */ "Jump" OpHelp(""), + /* 15 */ "Once" OpHelp(""), + /* 16 */ "If" OpHelp(""), + /* 17 */ "IfNot" OpHelp(""), + /* 18 */ "IsType" OpHelp("if typeof(P1.P3) in P5 goto P2"), /* 19 */ "Not" OpHelp("r[P2]= !r[P1]"), - /* 20 */ "SeekLT" OpHelp("key=r[P3@P4]"), - /* 21 */ "SeekLE" OpHelp("key=r[P3@P4]"), - /* 22 */ "SeekGE" OpHelp("key=r[P3@P4]"), - /* 23 */ "SeekGT" OpHelp("key=r[P3@P4]"), - /* 24 */ "IfNotOpen" OpHelp("if( !csr[P1] ) goto P2"), - /* 25 */ "IfNoHope" OpHelp("key=r[P3@P4]"), - /* 26 */ "NoConflict" OpHelp("key=r[P3@P4]"), - /* 27 */ "NotFound" OpHelp("key=r[P3@P4]"), - /* 28 */ "Found" OpHelp("key=r[P3@P4]"), - /* 29 */ "SeekRowid" OpHelp("intkey=r[P3]"), - /* 30 */ "NotExists" OpHelp("intkey=r[P3]"), - /* 31 */ "Last" OpHelp(""), - /* 32 */ "IfSmaller" OpHelp(""), - /* 33 */ "SorterSort" OpHelp(""), - /* 34 */ "Sort" OpHelp(""), - /* 35 */ "Rewind" OpHelp(""), - /* 36 */ "SorterNext" OpHelp(""), - /* 37 */ "Prev" OpHelp(""), - /* 38 */ "Next" OpHelp(""), - /* 39 */ "IdxLE" OpHelp("key=r[P3@P4]"), - /* 40 */ "IdxGT" OpHelp("key=r[P3@P4]"), - /* 41 */ "IdxLT" OpHelp("key=r[P3@P4]"), - /* 42 */ "IdxGE" OpHelp("key=r[P3@P4]"), + /* 20 */ "IfNullRow" OpHelp("if P1.nullRow then r[P3]=NULL, goto P2"), + /* 21 */ "SeekLT" OpHelp("key=r[P3@P4]"), + /* 22 */ "SeekLE" OpHelp("key=r[P3@P4]"), + /* 23 */ "SeekGE" OpHelp("key=r[P3@P4]"), + /* 24 */ "SeekGT" OpHelp("key=r[P3@P4]"), + /* 25 */ "IfNotOpen" OpHelp("if( !csr[P1] ) goto P2"), + /* 26 */ "IfNoHope" OpHelp("key=r[P3@P4]"), + /* 27 */ "NoConflict" OpHelp("key=r[P3@P4]"), + /* 28 */ "NotFound" OpHelp("key=r[P3@P4]"), + /* 29 */ "Found" OpHelp("key=r[P3@P4]"), + /* 30 */ "SeekRowid" OpHelp("intkey=r[P3]"), + /* 31 */ "NotExists" OpHelp("intkey=r[P3]"), + /* 32 */ "Last" OpHelp(""), + /* 33 */ "IfSmaller" OpHelp(""), + /* 34 */ "SorterSort" OpHelp(""), + /* 35 */ "Sort" OpHelp(""), + /* 36 */ "Rewind" OpHelp(""), + /* 37 */ "SorterNext" OpHelp(""), + /* 38 */ "Prev" OpHelp(""), + /* 39 */ "Next" OpHelp(""), + /* 40 */ "IdxLE" OpHelp("key=r[P3@P4]"), + /* 41 */ "IdxGT" OpHelp("key=r[P3@P4]"), + /* 42 */ "IdxLT" OpHelp("key=r[P3@P4]"), /* 43 */ "Or" OpHelp("r[P3]=(r[P1] || r[P2])"), /* 44 */ "And" OpHelp("r[P3]=(r[P1] && r[P2])"), - /* 45 */ "RowSetRead" OpHelp("r[P3]=rowset(P1)"), - /* 46 */ "RowSetTest" OpHelp("if r[P3] in rowset(P1) goto P2"), - /* 47 */ "Program" OpHelp(""), - /* 48 */ "FkIfZero" OpHelp("if fkctr[P1]==0 goto P2"), - /* 49 */ "IfPos" OpHelp("if r[P1]>0 then r[P1]-=P3, goto P2"), + /* 45 */ "IdxGE" OpHelp("key=r[P3@P4]"), + /* 46 */ "RowSetRead" OpHelp("r[P3]=rowset(P1)"), + /* 47 */ "RowSetTest" OpHelp("if r[P3] in rowset(P1) goto P2"), + /* 48 */ "Program" OpHelp(""), + /* 49 */ "FkIfZero" OpHelp("if fkctr[P1]==0 goto P2"), /* 50 */ "IsNull" OpHelp("if r[P1]==NULL goto P2"), /* 51 */ "NotNull" OpHelp("if r[P1]!=NULL goto P2"), /* 52 */ "Ne" OpHelp("IF r[P3]!=r[P1]"), @@ -35328,12 +35540,12 @@ SQLITE_PRIVATE const char *sqlite3OpcodeName(int i){ /* 56 */ "Lt" OpHelp("IF r[P3]=r[P1]"), /* 58 */ "ElseEq" OpHelp(""), - /* 59 */ "IfNotZero" OpHelp("if r[P1]!=0 then r[P1]--, goto P2"), - /* 60 */ "DecrJumpZero" OpHelp("if (--r[P1])==0 goto P2"), - /* 61 */ "IncrVacuum" OpHelp(""), - /* 62 */ "VNext" OpHelp(""), - /* 63 */ "Filter" OpHelp("if key(P3@P4) not in filter(P1) goto P2"), - /* 64 */ "Init" OpHelp("Start at P2"), + /* 59 */ "IfPos" OpHelp("if r[P1]>0 then r[P1]-=P3, goto P2"), + /* 60 */ "IfNotZero" OpHelp("if r[P1]!=0 then r[P1]--, goto P2"), + /* 61 */ "DecrJumpZero" OpHelp("if (--r[P1])==0 goto P2"), + /* 62 */ "IncrVacuum" OpHelp(""), + /* 63 */ "VNext" OpHelp(""), + /* 64 */ "Filter" OpHelp("if key(P3@P4) not in filter(P1) goto P2"), /* 65 */ "PureFunc" OpHelp("r[P3]=func(r[P2@NP])"), /* 66 */ "Function" OpHelp("r[P3]=func(r[P2@NP])"), /* 67 */ "Return" OpHelp(""), @@ -35462,6 +35674,981 @@ SQLITE_PRIVATE const char *sqlite3OpcodeName(int i){ #endif /************** End of opcodes.c *********************************************/ +/************** Begin file os_kv.c *******************************************/ +/* +** 2022-09-06 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains an experimental VFS layer that operates on a +** Key/Value storage engine where both keys and values must be pure +** text. +*/ +/* #include */ +#if SQLITE_OS_KV || (SQLITE_OS_UNIX && defined(SQLITE_OS_KV_OPTIONAL)) + +/***************************************************************************** +** Debugging logic +*/ + +/* SQLITE_KV_TRACE() is used for tracing calls to kvstorage routines. */ +#if 0 +#define SQLITE_KV_TRACE(X) printf X +#else +#define SQLITE_KV_TRACE(X) +#endif + +/* SQLITE_KV_LOG() is used for tracing calls to the VFS interface */ +#if 0 +#define SQLITE_KV_LOG(X) printf X +#else +#define SQLITE_KV_LOG(X) +#endif + + +/* +** Forward declaration of objects used by this VFS implementation +*/ +typedef struct KVVfsFile KVVfsFile; + +/* A single open file. There are only two files represented by this +** VFS - the database and the rollback journal. +*/ +struct KVVfsFile { + sqlite3_file base; /* IO methods */ + const char *zClass; /* Storage class */ + int isJournal; /* True if this is a journal file */ + unsigned int nJrnl; /* Space allocated for aJrnl[] */ + char *aJrnl; /* Journal content */ + int szPage; /* Last known page size */ + sqlite3_int64 szDb; /* Database file size. -1 means unknown */ +}; + +/* +** Methods for KVVfsFile +*/ +static int kvvfsClose(sqlite3_file*); +static int kvvfsReadDb(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst); +static int kvvfsReadJrnl(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst); +static int kvvfsWriteDb(sqlite3_file*,const void*,int iAmt, sqlite3_int64); +static int kvvfsWriteJrnl(sqlite3_file*,const void*,int iAmt, sqlite3_int64); +static int kvvfsTruncateDb(sqlite3_file*, sqlite3_int64 size); +static int kvvfsTruncateJrnl(sqlite3_file*, sqlite3_int64 size); +static int kvvfsSyncDb(sqlite3_file*, int flags); +static int kvvfsSyncJrnl(sqlite3_file*, int flags); +static int kvvfsFileSizeDb(sqlite3_file*, sqlite3_int64 *pSize); +static int kvvfsFileSizeJrnl(sqlite3_file*, sqlite3_int64 *pSize); +static int kvvfsLock(sqlite3_file*, int); +static int kvvfsUnlock(sqlite3_file*, int); +static int kvvfsCheckReservedLock(sqlite3_file*, int *pResOut); +static int kvvfsFileControlDb(sqlite3_file*, int op, void *pArg); +static int kvvfsFileControlJrnl(sqlite3_file*, int op, void *pArg); +static int kvvfsSectorSize(sqlite3_file*); +static int kvvfsDeviceCharacteristics(sqlite3_file*); + +/* +** Methods for sqlite3_vfs +*/ +static int kvvfsOpen(sqlite3_vfs*, const char *, sqlite3_file*, int , int *); +static int kvvfsDelete(sqlite3_vfs*, const char *zName, int syncDir); +static int kvvfsAccess(sqlite3_vfs*, const char *zName, int flags, int *); +static int kvvfsFullPathname(sqlite3_vfs*, const char *zName, int, char *zOut); +static void *kvvfsDlOpen(sqlite3_vfs*, const char *zFilename); +static int kvvfsRandomness(sqlite3_vfs*, int nByte, char *zOut); +static int kvvfsSleep(sqlite3_vfs*, int microseconds); +static int kvvfsCurrentTime(sqlite3_vfs*, double*); +static int kvvfsCurrentTimeInt64(sqlite3_vfs*, sqlite3_int64*); + +static sqlite3_vfs sqlite3OsKvvfsObject = { + 1, /* iVersion */ + sizeof(KVVfsFile), /* szOsFile */ + 1024, /* mxPathname */ + 0, /* pNext */ + "kvvfs", /* zName */ + 0, /* pAppData */ + kvvfsOpen, /* xOpen */ + kvvfsDelete, /* xDelete */ + kvvfsAccess, /* xAccess */ + kvvfsFullPathname, /* xFullPathname */ + kvvfsDlOpen, /* xDlOpen */ + 0, /* xDlError */ + 0, /* xDlSym */ + 0, /* xDlClose */ + kvvfsRandomness, /* xRandomness */ + kvvfsSleep, /* xSleep */ + kvvfsCurrentTime, /* xCurrentTime */ + 0, /* xGetLastError */ + kvvfsCurrentTimeInt64 /* xCurrentTimeInt64 */ +}; + +/* Methods for sqlite3_file objects referencing a database file +*/ +static sqlite3_io_methods kvvfs_db_io_methods = { + 1, /* iVersion */ + kvvfsClose, /* xClose */ + kvvfsReadDb, /* xRead */ + kvvfsWriteDb, /* xWrite */ + kvvfsTruncateDb, /* xTruncate */ + kvvfsSyncDb, /* xSync */ + kvvfsFileSizeDb, /* xFileSize */ + kvvfsLock, /* xLock */ + kvvfsUnlock, /* xUnlock */ + kvvfsCheckReservedLock, /* xCheckReservedLock */ + kvvfsFileControlDb, /* xFileControl */ + kvvfsSectorSize, /* xSectorSize */ + kvvfsDeviceCharacteristics, /* xDeviceCharacteristics */ + 0, /* xShmMap */ + 0, /* xShmLock */ + 0, /* xShmBarrier */ + 0, /* xShmUnmap */ + 0, /* xFetch */ + 0 /* xUnfetch */ +}; + +/* Methods for sqlite3_file objects referencing a rollback journal +*/ +static sqlite3_io_methods kvvfs_jrnl_io_methods = { + 1, /* iVersion */ + kvvfsClose, /* xClose */ + kvvfsReadJrnl, /* xRead */ + kvvfsWriteJrnl, /* xWrite */ + kvvfsTruncateJrnl, /* xTruncate */ + kvvfsSyncJrnl, /* xSync */ + kvvfsFileSizeJrnl, /* xFileSize */ + kvvfsLock, /* xLock */ + kvvfsUnlock, /* xUnlock */ + kvvfsCheckReservedLock, /* xCheckReservedLock */ + kvvfsFileControlJrnl, /* xFileControl */ + kvvfsSectorSize, /* xSectorSize */ + kvvfsDeviceCharacteristics, /* xDeviceCharacteristics */ + 0, /* xShmMap */ + 0, /* xShmLock */ + 0, /* xShmBarrier */ + 0, /* xShmUnmap */ + 0, /* xFetch */ + 0 /* xUnfetch */ +}; + +/****** Storage subsystem **************************************************/ +#include +#include +#include + +/* Forward declarations for the low-level storage engine +*/ +static int kvstorageWrite(const char*, const char *zKey, const char *zData); +static int kvstorageDelete(const char*, const char *zKey); +static int kvstorageRead(const char*, const char *zKey, char *zBuf, int nBuf); +#define KVSTORAGE_KEY_SZ 32 + +/* Expand the key name with an appropriate prefix and put the result +** zKeyOut[]. The zKeyOut[] buffer is assumed to hold at least +** KVSTORAGE_KEY_SZ bytes. +*/ +static void kvstorageMakeKey( + const char *zClass, + const char *zKeyIn, + char *zKeyOut +){ + sqlite3_snprintf(KVSTORAGE_KEY_SZ, zKeyOut, "kvvfs-%s-%s", zClass, zKeyIn); +} + +/* Write content into a key. zClass is the particular namespace of the +** underlying key/value store to use - either "local" or "session". +** +** Both zKey and zData are zero-terminated pure text strings. +** +** Return the number of errors. +*/ +static int kvstorageWrite( + const char *zClass, + const char *zKey, + const char *zData +){ + FILE *fd; + char zXKey[KVSTORAGE_KEY_SZ]; + kvstorageMakeKey(zClass, zKey, zXKey); + fd = fopen(zXKey, "wb"); + if( fd ){ + SQLITE_KV_TRACE(("KVVFS-WRITE %-15s (%d) %.50s%s\n", zXKey, + (int)strlen(zData), zData, + strlen(zData)>50 ? "..." : "")); + fputs(zData, fd); + fclose(fd); + return 0; + }else{ + return 1; + } +} + +/* Delete a key (with its corresponding data) from the key/value +** namespace given by zClass. If the key does not previously exist, +** this routine is a no-op. +*/ +static int kvstorageDelete(const char *zClass, const char *zKey){ + char zXKey[KVSTORAGE_KEY_SZ]; + kvstorageMakeKey(zClass, zKey, zXKey); + unlink(zXKey); + SQLITE_KV_TRACE(("KVVFS-DELETE %-15s\n", zXKey)); + return 0; +} + +/* Read the value associated with a zKey from the key/value namespace given +** by zClass and put the text data associated with that key in the first +** nBuf bytes of zBuf[]. The value might be truncated if zBuf is not large +** enough to hold it all. The value put into zBuf must always be zero +** terminated, even if it gets truncated because nBuf is not large enough. +** +** Return the total number of bytes in the data, without truncation, and +** not counting the final zero terminator. Return -1 if the key does +** not exist. +** +** If nBuf<=0 then this routine simply returns the size of the data without +** actually reading it. +*/ +static int kvstorageRead( + const char *zClass, + const char *zKey, + char *zBuf, + int nBuf +){ + FILE *fd; + struct stat buf; + char zXKey[KVSTORAGE_KEY_SZ]; + kvstorageMakeKey(zClass, zKey, zXKey); + if( access(zXKey, R_OK)!=0 + || stat(zXKey, &buf)!=0 + || !S_ISREG(buf.st_mode) + ){ + SQLITE_KV_TRACE(("KVVFS-READ %-15s (-1)\n", zXKey)); + return -1; + } + if( nBuf<=0 ){ + return (int)buf.st_size; + }else if( nBuf==1 ){ + zBuf[0] = 0; + SQLITE_KV_TRACE(("KVVFS-READ %-15s (%d)\n", zXKey, + (int)buf.st_size)); + return (int)buf.st_size; + } + if( nBuf > buf.st_size + 1 ){ + nBuf = buf.st_size + 1; + } + fd = fopen(zXKey, "rb"); + if( fd==0 ){ + SQLITE_KV_TRACE(("KVVFS-READ %-15s (-1)\n", zXKey)); + return -1; + }else{ + sqlite3_int64 n = fread(zBuf, 1, nBuf-1, fd); + fclose(fd); + zBuf[n] = 0; + SQLITE_KV_TRACE(("KVVFS-READ %-15s (%lld) %.50s%s\n", zXKey, + n, zBuf, n>50 ? "..." : "")); + return (int)n; + } +} + +/* +** An internal level of indirection which enables us to replace the +** kvvfs i/o methods with JavaScript implementations in WASM builds. +** Maintenance reminder: if this struct changes in any way, the JSON +** rendering of its structure must be updated in +** sqlite3_wasm_enum_json(). There are no binary compatibility +** concerns, so it does not need an iVersion member. This file is +** necessarily always compiled together with sqlite3_wasm_enum_json(), +** and JS code dynamically creates the mapping of members based on +** that JSON description. +*/ +typedef struct sqlite3_kvvfs_methods sqlite3_kvvfs_methods; +struct sqlite3_kvvfs_methods { + int (*xRead)(const char *zClass, const char *zKey, char *zBuf, int nBuf); + int (*xWrite)(const char *zClass, const char *zKey, const char *zData); + int (*xDelete)(const char *zClass, const char *zKey); + const int nKeySize; +}; + +/* +** This object holds the kvvfs I/O methods which may be swapped out +** for JavaScript-side implementations in WASM builds. In such builds +** it cannot be const, but in native builds it should be so that +** the compiler can hopefully optimize this level of indirection out. +** That said, kvvfs is intended primarily for use in WASM builds. +** +** Note that this is not explicitly flagged as static because the +** amalgamation build will tag it with SQLITE_PRIVATE. +*/ +#ifndef SQLITE_WASM +const +#endif +SQLITE_PRIVATE sqlite3_kvvfs_methods sqlite3KvvfsMethods = { +kvstorageRead, +kvstorageWrite, +kvstorageDelete, +KVSTORAGE_KEY_SZ +}; + +/****** Utility subroutines ************************************************/ + +/* +** Encode binary into the text encoded used to persist on disk. +** The output text is stored in aOut[], which must be at least +** nData+1 bytes in length. +** +** Return the actual length of the encoded text, not counting the +** zero terminator at the end. +** +** Encoding format +** --------------- +** +** * Non-zero bytes are encoded as upper-case hexadecimal +** +** * A sequence of one or more zero-bytes that are not at the +** beginning of the buffer are encoded as a little-endian +** base-26 number using a..z. "a" means 0. "b" means 1, +** "z" means 25. "ab" means 26. "ac" means 52. And so forth. +** +** * Because there is no overlap between the encoding characters +** of hexadecimal and base-26 numbers, it is always clear where +** one stops and the next begins. +*/ +static int kvvfsEncode(const char *aData, int nData, char *aOut){ + int i, j; + const unsigned char *a = (const unsigned char*)aData; + for(i=j=0; i>4]; + aOut[j++] = "0123456789ABCDEF"[c&0xf]; + }else{ + /* A sequence of 1 or more zeros is stored as a little-endian + ** base-26 number using a..z as the digits. So one zero is "b". + ** Two zeros is "c". 25 zeros is "z", 26 zeros is "ab", 27 is "bb", + ** and so forth. + */ + int k; + for(k=1; i+k0 ){ + aOut[j++] = 'a'+(k%26); + k /= 26; + } + } + } + aOut[j] = 0; + return j; +} + +static const signed char kvvfsHexValue[256] = { + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, -1, -1, -1, -1, -1, -1, + -1, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1 +}; + +/* +** Decode the text encoding back to binary. The binary content is +** written into pOut, which must be at least nOut bytes in length. +** +** The return value is the number of bytes actually written into aOut[]. +*/ +static int kvvfsDecode(const char *a, char *aOut, int nOut){ + int i, j; + int c; + const unsigned char *aIn = (const unsigned char*)a; + i = 0; + j = 0; + while( 1 ){ + c = kvvfsHexValue[aIn[i]]; + if( c<0 ){ + int n = 0; + int mult = 1; + c = aIn[i]; + if( c==0 ) break; + while( c>='a' && c<='z' ){ + n += (c - 'a')*mult; + mult *= 26; + c = aIn[++i]; + } + if( j+n>nOut ) return -1; + memset(&aOut[j], 0, n); + j += n; + c = aIn[i]; + if( c==0 ) break; + }else{ + aOut[j] = c<<4; + c = kvvfsHexValue[aIn[++i]]; + if( c<0 ) break; + aOut[j++] += c; + i++; + } + } + return j; +} + +/* +** Decode a complete journal file. Allocate space in pFile->aJrnl +** and store the decoding there. Or leave pFile->aJrnl set to NULL +** if an error is encountered. +** +** The first few characters of the text encoding will be a little-endian +** base-26 number (digits a..z) that is the total number of bytes +** in the decoded journal file image. This base-26 number is followed +** by a single space, then the encoding of the journal. The space +** separator is required to act as a terminator for the base-26 number. +*/ +static void kvvfsDecodeJournal( + KVVfsFile *pFile, /* Store decoding in pFile->aJrnl */ + const char *zTxt, /* Text encoding. Zero-terminated */ + int nTxt /* Bytes in zTxt, excluding zero terminator */ +){ + unsigned int n = 0; + int c, i, mult; + i = 0; + mult = 1; + while( (c = zTxt[i++])>='a' && c<='z' ){ + n += (zTxt[i] - 'a')*mult; + mult *= 26; + } + sqlite3_free(pFile->aJrnl); + pFile->aJrnl = sqlite3_malloc64( n ); + if( pFile->aJrnl==0 ){ + pFile->nJrnl = 0; + return; + } + pFile->nJrnl = n; + n = kvvfsDecode(zTxt+i, pFile->aJrnl, pFile->nJrnl); + if( nnJrnl ){ + sqlite3_free(pFile->aJrnl); + pFile->aJrnl = 0; + pFile->nJrnl = 0; + } +} + +/* +** Read or write the "sz" element, containing the database file size. +*/ +static sqlite3_int64 kvvfsReadFileSize(KVVfsFile *pFile){ + char zData[50]; + zData[0] = 0; + sqlite3KvvfsMethods.xRead(pFile->zClass, "sz", zData, sizeof(zData)-1); + return strtoll(zData, 0, 0); +} +static int kvvfsWriteFileSize(KVVfsFile *pFile, sqlite3_int64 sz){ + char zData[50]; + sqlite3_snprintf(sizeof(zData), zData, "%lld", sz); + return sqlite3KvvfsMethods.xWrite(pFile->zClass, "sz", zData); +} + +/****** sqlite3_io_methods methods ******************************************/ + +/* +** Close an kvvfs-file. +*/ +static int kvvfsClose(sqlite3_file *pProtoFile){ + KVVfsFile *pFile = (KVVfsFile *)pProtoFile; + + SQLITE_KV_LOG(("xClose %s %s\n", pFile->zClass, + pFile->isJournal ? "journal" : "db")); + sqlite3_free(pFile->aJrnl); + return SQLITE_OK; +} + +/* +** Read from the -journal file. +*/ +static int kvvfsReadJrnl( + sqlite3_file *pProtoFile, + void *zBuf, + int iAmt, + sqlite_int64 iOfst +){ + KVVfsFile *pFile = (KVVfsFile*)pProtoFile; + assert( pFile->isJournal ); + SQLITE_KV_LOG(("xRead('%s-journal',%d,%lld)\n", pFile->zClass, iAmt, iOfst)); + if( pFile->aJrnl==0 ){ + int szTxt = kvstorageRead(pFile->zClass, "jrnl", 0, 0); + char *aTxt; + if( szTxt<=4 ){ + return SQLITE_IOERR; + } + aTxt = sqlite3_malloc64( szTxt+1 ); + if( aTxt==0 ) return SQLITE_NOMEM; + kvstorageRead(pFile->zClass, "jrnl", aTxt, szTxt+1); + kvvfsDecodeJournal(pFile, aTxt, szTxt); + sqlite3_free(aTxt); + if( pFile->aJrnl==0 ) return SQLITE_IOERR; + } + if( iOfst+iAmt>pFile->nJrnl ){ + return SQLITE_IOERR_SHORT_READ; + } + memcpy(zBuf, pFile->aJrnl+iOfst, iAmt); + return SQLITE_OK; +} + +/* +** Read from the database file. +*/ +static int kvvfsReadDb( + sqlite3_file *pProtoFile, + void *zBuf, + int iAmt, + sqlite_int64 iOfst +){ + KVVfsFile *pFile = (KVVfsFile*)pProtoFile; + unsigned int pgno; + int got, n; + char zKey[30]; + char aData[133073]; + assert( iOfst>=0 ); + assert( iAmt>=0 ); + SQLITE_KV_LOG(("xRead('%s-db',%d,%lld)\n", pFile->zClass, iAmt, iOfst)); + if( iOfst+iAmt>=512 ){ + if( (iOfst % iAmt)!=0 ){ + return SQLITE_IOERR_READ; + } + if( (iAmt & (iAmt-1))!=0 || iAmt<512 || iAmt>65536 ){ + return SQLITE_IOERR_READ; + } + pFile->szPage = iAmt; + pgno = 1 + iOfst/iAmt; + }else{ + pgno = 1; + } + sqlite3_snprintf(sizeof(zKey), zKey, "%u", pgno); + got = sqlite3KvvfsMethods.xRead(pFile->zClass, zKey, aData, sizeof(aData)-1); + if( got<0 ){ + n = 0; + }else{ + aData[got] = 0; + if( iOfst+iAmt<512 ){ + int k = iOfst+iAmt; + aData[k*2] = 0; + n = kvvfsDecode(aData, &aData[2000], sizeof(aData)-2000); + if( n>=iOfst+iAmt ){ + memcpy(zBuf, &aData[2000+iOfst], iAmt); + n = iAmt; + }else{ + n = 0; + } + }else{ + n = kvvfsDecode(aData, zBuf, iAmt); + } + } + if( nzClass, iAmt, iOfst)); + if( iEnd>=0x10000000 ) return SQLITE_FULL; + if( pFile->aJrnl==0 || pFile->nJrnlaJrnl, iEnd); + if( aNew==0 ){ + return SQLITE_IOERR_NOMEM; + } + pFile->aJrnl = aNew; + if( pFile->nJrnlaJrnl+pFile->nJrnl, 0, iOfst-pFile->nJrnl); + } + pFile->nJrnl = iEnd; + } + memcpy(pFile->aJrnl+iOfst, zBuf, iAmt); + return SQLITE_OK; +} + +/* +** Write into the database file. +*/ +static int kvvfsWriteDb( + sqlite3_file *pProtoFile, + const void *zBuf, + int iAmt, + sqlite_int64 iOfst +){ + KVVfsFile *pFile = (KVVfsFile*)pProtoFile; + unsigned int pgno; + char zKey[30]; + char aData[131073]; + SQLITE_KV_LOG(("xWrite('%s-db',%d,%lld)\n", pFile->zClass, iAmt, iOfst)); + assert( iAmt>=512 && iAmt<=65536 ); + assert( (iAmt & (iAmt-1))==0 ); + assert( pFile->szPage<0 || pFile->szPage==iAmt ); + pFile->szPage = iAmt; + pgno = 1 + iOfst/iAmt; + sqlite3_snprintf(sizeof(zKey), zKey, "%u", pgno); + kvvfsEncode(zBuf, iAmt, aData); + if( sqlite3KvvfsMethods.xWrite(pFile->zClass, zKey, aData) ){ + return SQLITE_IOERR; + } + if( iOfst+iAmt > pFile->szDb ){ + pFile->szDb = iOfst + iAmt; + } + return SQLITE_OK; +} + +/* +** Truncate an kvvfs-file. +*/ +static int kvvfsTruncateJrnl(sqlite3_file *pProtoFile, sqlite_int64 size){ + KVVfsFile *pFile = (KVVfsFile *)pProtoFile; + SQLITE_KV_LOG(("xTruncate('%s-journal',%lld)\n", pFile->zClass, size)); + assert( size==0 ); + sqlite3KvvfsMethods.xDelete(pFile->zClass, "jrnl"); + sqlite3_free(pFile->aJrnl); + pFile->aJrnl = 0; + pFile->nJrnl = 0; + return SQLITE_OK; +} +static int kvvfsTruncateDb(sqlite3_file *pProtoFile, sqlite_int64 size){ + KVVfsFile *pFile = (KVVfsFile *)pProtoFile; + if( pFile->szDb>size + && pFile->szPage>0 + && (size % pFile->szPage)==0 + ){ + char zKey[50]; + unsigned int pgno, pgnoMax; + SQLITE_KV_LOG(("xTruncate('%s-db',%lld)\n", pFile->zClass, size)); + pgno = 1 + size/pFile->szPage; + pgnoMax = 2 + pFile->szDb/pFile->szPage; + while( pgno<=pgnoMax ){ + sqlite3_snprintf(sizeof(zKey), zKey, "%u", pgno); + sqlite3KvvfsMethods.xDelete(pFile->zClass, zKey); + pgno++; + } + pFile->szDb = size; + return kvvfsWriteFileSize(pFile, size) ? SQLITE_IOERR : SQLITE_OK; + } + return SQLITE_IOERR; +} + +/* +** Sync an kvvfs-file. +*/ +static int kvvfsSyncJrnl(sqlite3_file *pProtoFile, int flags){ + int i, n; + KVVfsFile *pFile = (KVVfsFile *)pProtoFile; + char *zOut; + SQLITE_KV_LOG(("xSync('%s-journal')\n", pFile->zClass)); + if( pFile->nJrnl<=0 ){ + return kvvfsTruncateJrnl(pProtoFile, 0); + } + zOut = sqlite3_malloc64( pFile->nJrnl*2 + 50 ); + if( zOut==0 ){ + return SQLITE_IOERR_NOMEM; + } + n = pFile->nJrnl; + i = 0; + do{ + zOut[i++] = 'a' + (n%26); + n /= 26; + }while( n>0 ); + zOut[i++] = ' '; + kvvfsEncode(pFile->aJrnl, pFile->nJrnl, &zOut[i]); + i = sqlite3KvvfsMethods.xWrite(pFile->zClass, "jrnl", zOut); + sqlite3_free(zOut); + return i ? SQLITE_IOERR : SQLITE_OK; +} +static int kvvfsSyncDb(sqlite3_file *pProtoFile, int flags){ + return SQLITE_OK; +} + +/* +** Return the current file-size of an kvvfs-file. +*/ +static int kvvfsFileSizeJrnl(sqlite3_file *pProtoFile, sqlite_int64 *pSize){ + KVVfsFile *pFile = (KVVfsFile *)pProtoFile; + SQLITE_KV_LOG(("xFileSize('%s-journal')\n", pFile->zClass)); + *pSize = pFile->nJrnl; + return SQLITE_OK; +} +static int kvvfsFileSizeDb(sqlite3_file *pProtoFile, sqlite_int64 *pSize){ + KVVfsFile *pFile = (KVVfsFile *)pProtoFile; + SQLITE_KV_LOG(("xFileSize('%s-db')\n", pFile->zClass)); + if( pFile->szDb>=0 ){ + *pSize = pFile->szDb; + }else{ + *pSize = kvvfsReadFileSize(pFile); + } + return SQLITE_OK; +} + +/* +** Lock an kvvfs-file. +*/ +static int kvvfsLock(sqlite3_file *pProtoFile, int eLock){ + KVVfsFile *pFile = (KVVfsFile *)pProtoFile; + assert( !pFile->isJournal ); + SQLITE_KV_LOG(("xLock(%s,%d)\n", pFile->zClass, eLock)); + + if( eLock!=SQLITE_LOCK_NONE ){ + pFile->szDb = kvvfsReadFileSize(pFile); + } + return SQLITE_OK; +} + +/* +** Unlock an kvvfs-file. +*/ +static int kvvfsUnlock(sqlite3_file *pProtoFile, int eLock){ + KVVfsFile *pFile = (KVVfsFile *)pProtoFile; + assert( !pFile->isJournal ); + SQLITE_KV_LOG(("xUnlock(%s,%d)\n", pFile->zClass, eLock)); + if( eLock==SQLITE_LOCK_NONE ){ + pFile->szDb = -1; + } + return SQLITE_OK; +} + +/* +** Check if another file-handle holds a RESERVED lock on an kvvfs-file. +*/ +static int kvvfsCheckReservedLock(sqlite3_file *pProtoFile, int *pResOut){ + SQLITE_KV_LOG(("xCheckReservedLock\n")); + *pResOut = 0; + return SQLITE_OK; +} + +/* +** File control method. For custom operations on an kvvfs-file. +*/ +static int kvvfsFileControlJrnl(sqlite3_file *pProtoFile, int op, void *pArg){ + SQLITE_KV_LOG(("xFileControl(%d) on journal\n", op)); + return SQLITE_NOTFOUND; +} +static int kvvfsFileControlDb(sqlite3_file *pProtoFile, int op, void *pArg){ + SQLITE_KV_LOG(("xFileControl(%d) on database\n", op)); + if( op==SQLITE_FCNTL_SYNC ){ + KVVfsFile *pFile = (KVVfsFile *)pProtoFile; + int rc = SQLITE_OK; + SQLITE_KV_LOG(("xSync('%s-db')\n", pFile->zClass)); + if( pFile->szDb>0 && 0!=kvvfsWriteFileSize(pFile, pFile->szDb) ){ + rc = SQLITE_IOERR; + } + return rc; + } + return SQLITE_NOTFOUND; +} + +/* +** Return the sector-size in bytes for an kvvfs-file. +*/ +static int kvvfsSectorSize(sqlite3_file *pFile){ + return 512; +} + +/* +** Return the device characteristic flags supported by an kvvfs-file. +*/ +static int kvvfsDeviceCharacteristics(sqlite3_file *pProtoFile){ + return 0; +} + +/****** sqlite3_vfs methods *************************************************/ + +/* +** Open an kvvfs file handle. +*/ +static int kvvfsOpen( + sqlite3_vfs *pProtoVfs, + const char *zName, + sqlite3_file *pProtoFile, + int flags, + int *pOutFlags +){ + KVVfsFile *pFile = (KVVfsFile*)pProtoFile; + if( zName==0 ) zName = ""; + SQLITE_KV_LOG(("xOpen(\"%s\")\n", zName)); + if( strcmp(zName, "local")==0 + || strcmp(zName, "session")==0 + ){ + pFile->isJournal = 0; + pFile->base.pMethods = &kvvfs_db_io_methods; + }else + if( strcmp(zName, "local-journal")==0 + || strcmp(zName, "session-journal")==0 + ){ + pFile->isJournal = 1; + pFile->base.pMethods = &kvvfs_jrnl_io_methods; + }else{ + return SQLITE_CANTOPEN; + } + if( zName[0]=='s' ){ + pFile->zClass = "session"; + }else{ + pFile->zClass = "local"; + } + pFile->aJrnl = 0; + pFile->nJrnl = 0; + pFile->szPage = -1; + pFile->szDb = -1; + return SQLITE_OK; +} + +/* +** Delete the file located at zPath. If the dirSync argument is true, +** ensure the file-system modifications are synced to disk before +** returning. +*/ +static int kvvfsDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){ + if( strcmp(zPath, "local-journal")==0 ){ + sqlite3KvvfsMethods.xDelete("local", "jrnl"); + }else + if( strcmp(zPath, "session-journal")==0 ){ + sqlite3KvvfsMethods.xDelete("session", "jrnl"); + } + return SQLITE_OK; +} + +/* +** Test for access permissions. Return true if the requested permission +** is available, or false otherwise. +*/ +static int kvvfsAccess( + sqlite3_vfs *pProtoVfs, + const char *zPath, + int flags, + int *pResOut +){ + SQLITE_KV_LOG(("xAccess(\"%s\")\n", zPath)); + if( strcmp(zPath, "local-journal")==0 ){ + *pResOut = sqlite3KvvfsMethods.xRead("local", "jrnl", 0, 0)>0; + }else + if( strcmp(zPath, "session-journal")==0 ){ + *pResOut = sqlite3KvvfsMethods.xRead("session", "jrnl", 0, 0)>0; + }else + if( strcmp(zPath, "local")==0 ){ + *pResOut = sqlite3KvvfsMethods.xRead("local", "sz", 0, 0)>0; + }else + if( strcmp(zPath, "session")==0 ){ + *pResOut = sqlite3KvvfsMethods.xRead("session", "sz", 0, 0)>0; + }else + { + *pResOut = 0; + } + SQLITE_KV_LOG(("xAccess returns %d\n",*pResOut)); + return SQLITE_OK; +} + +/* +** Populate buffer zOut with the full canonical pathname corresponding +** to the pathname in zPath. zOut is guaranteed to point to a buffer +** of at least (INST_MAX_PATHNAME+1) bytes. +*/ +static int kvvfsFullPathname( + sqlite3_vfs *pVfs, + const char *zPath, + int nOut, + char *zOut +){ + size_t nPath; +#ifdef SQLITE_OS_KV_ALWAYS_LOCAL + zPath = "local"; +#endif + nPath = strlen(zPath); + SQLITE_KV_LOG(("xFullPathname(\"%s\")\n", zPath)); + if( nOut +static int kvvfsCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *pTimeOut){ + static const sqlite3_int64 unixEpoch = 24405875*(sqlite3_int64)8640000; + struct timeval sNow; + (void)gettimeofday(&sNow, 0); /* Cannot fail given valid arguments */ + *pTimeOut = unixEpoch + 1000*(sqlite3_int64)sNow.tv_sec + sNow.tv_usec/1000; + return SQLITE_OK; +} +#endif /* SQLITE_OS_KV || SQLITE_OS_UNIX */ + +#if SQLITE_OS_KV +/* +** This routine is called initialize the KV-vfs as the default VFS. +*/ +SQLITE_API int sqlite3_os_init(void){ + return sqlite3_vfs_register(&sqlite3OsKvvfsObject, 1); +} +SQLITE_API int sqlite3_os_end(void){ + return SQLITE_OK; +} +#endif /* SQLITE_OS_KV */ + +#if SQLITE_OS_UNIX && defined(SQLITE_OS_KV_OPTIONAL) +SQLITE_PRIVATE int sqlite3KvvfsInit(void){ + return sqlite3_vfs_register(&sqlite3OsKvvfsObject, 0); +} +#endif + +/************** End of os_kv.c ***********************************************/ /************** Begin file os_unix.c *****************************************/ /* ** 2004 May 22 @@ -35552,13 +36739,13 @@ SQLITE_PRIVATE const char *sqlite3OpcodeName(int i){ /* ** standard include files. */ -#include -#include +#include /* amalgamator: keep */ +#include /* amalgamator: keep */ #include #include -#include +#include /* amalgamator: keep */ /* #include */ -#include +#include /* amalgamator: keep */ #include #if !defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0 # include @@ -41320,6 +42507,7 @@ static const char *unixTempFileDir(void){ static int unixGetTempname(int nBuf, char *zBuf){ const char *zDir; int iLimit = 0; + int rc = SQLITE_OK; /* It's odd to simulate an io-error here, but really this is just ** using the io-error infrastructure to test that SQLite handles this @@ -41328,18 +42516,26 @@ static int unixGetTempname(int nBuf, char *zBuf){ zBuf[0] = 0; SimulateIOError( return SQLITE_IOERR ); + sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_TEMPDIR)); zDir = unixTempFileDir(); - if( zDir==0 ) return SQLITE_IOERR_GETTEMPPATH; - do{ - u64 r; - sqlite3_randomness(sizeof(r), &r); - assert( nBuf>2 ); - zBuf[nBuf-2] = 0; - sqlite3_snprintf(nBuf, zBuf, "%s/"SQLITE_TEMP_FILE_PREFIX"%llx%c", - zDir, r, 0); - if( zBuf[nBuf-2]!=0 || (iLimit++)>10 ) return SQLITE_ERROR; - }while( osAccess(zBuf,0)==0 ); - return SQLITE_OK; + if( zDir==0 ){ + rc = SQLITE_IOERR_GETTEMPPATH; + }else{ + do{ + u64 r; + sqlite3_randomness(sizeof(r), &r); + assert( nBuf>2 ); + zBuf[nBuf-2] = 0; + sqlite3_snprintf(nBuf, zBuf, "%s/"SQLITE_TEMP_FILE_PREFIX"%llx%c", + zDir, r, 0); + if( zBuf[nBuf-2]!=0 || (iLimit++)>10 ){ + rc = SQLITE_ERROR; + break; + } + }while( osAccess(zBuf,0)==0 ); + } + sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_TEMPDIR)); + return rc; } #if SQLITE_ENABLE_LOCKING_STYLE && defined(__APPLE__) @@ -43514,8 +44710,16 @@ SQLITE_API int sqlite3_os_init(void){ /* Register all VFSes defined in the aVfs[] array */ for(i=0; i<(sizeof(aVfs)/sizeof(sqlite3_vfs)); i++){ +#ifdef SQLITE_DEFAULT_UNIX_VFS + sqlite3_vfs_register(&aVfs[i], + 0==strcmp(aVfs[i].zName,SQLITE_DEFAULT_UNIX_VFS)); +#else sqlite3_vfs_register(&aVfs[i], i==0); +#endif } +#ifdef SQLITE_OS_KV_OPTIONAL + sqlite3KvvfsInit(); +#endif unixBigLock = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_VFS1); #ifndef SQLITE_OMIT_WAL @@ -45478,10 +46682,12 @@ SQLITE_API int sqlite3_win32_set_directory8( const char *zValue /* New value for directory being set or reset */ ){ char **ppDirectory = 0; + int rc; #ifndef SQLITE_OMIT_AUTOINIT - int rc = sqlite3_initialize(); + rc = sqlite3_initialize(); if( rc ) return rc; #endif + sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_TEMPDIR)); if( type==SQLITE_WIN32_DATA_DIRECTORY_TYPE ){ ppDirectory = &sqlite3_data_directory; }else if( type==SQLITE_WIN32_TEMP_DIRECTORY_TYPE ){ @@ -45496,14 +46702,19 @@ SQLITE_API int sqlite3_win32_set_directory8( if( zValue && zValue[0] ){ zCopy = sqlite3_mprintf("%s", zValue); if ( zCopy==0 ){ - return SQLITE_NOMEM_BKPT; + rc = SQLITE_NOMEM_BKPT; + goto set_directory8_done; } } sqlite3_free(*ppDirectory); *ppDirectory = zCopy; - return SQLITE_OK; + rc = SQLITE_OK; + }else{ + rc = SQLITE_ERROR; } - return SQLITE_ERROR; +set_directory8_done: + sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_TEMPDIR)); + return rc; } /* @@ -48277,6 +49488,19 @@ static int winMakeEndInDirSep(int nBuf, char *zBuf){ return 0; } +/* +** If sqlite3_temp_directory is defined, take the mutex and return true. +** +** If sqlite3_temp_directory is NULL (undefined), omit the mutex and +** return false. +*/ +static int winTempDirDefined(void){ + sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_TEMPDIR)); + if( sqlite3_temp_directory!=0 ) return 1; + sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_TEMPDIR)); + return 0; +} + /* ** Create a temporary file name and store the resulting pointer into pzBuf. ** The pointer returned in pzBuf must be freed via sqlite3_free(). @@ -48313,20 +49537,23 @@ static int winGetTempname(sqlite3_vfs *pVfs, char **pzBuf){ */ nDir = nMax - (nPre + 15); assert( nDir>0 ); - if( sqlite3_temp_directory ){ + if( winTempDirDefined() ){ int nDirLen = sqlite3Strlen30(sqlite3_temp_directory); if( nDirLen>0 ){ if( !winIsDirSep(sqlite3_temp_directory[nDirLen-1]) ){ nDirLen++; } if( nDirLen>nDir ){ + sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_TEMPDIR)); sqlite3_free(zBuf); OSTRACE(("TEMP-FILENAME rc=SQLITE_ERROR\n")); return winLogError(SQLITE_ERROR, 0, "winGetTempname1", 0); } sqlite3_snprintf(nMax, zBuf, "%s", sqlite3_temp_directory); } + sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_TEMPDIR)); } + #if defined(__CYGWIN__) else{ static const char *azDirs[] = { @@ -49115,7 +50342,7 @@ static BOOL winIsVerbatimPathname( ** pathname into zOut[]. zOut[] will be at least pVfs->mxPathname ** bytes in size. */ -static int winFullPathname( +static int winFullPathnameNoMutex( sqlite3_vfs *pVfs, /* Pointer to vfs object */ const char *zRelative, /* Possibly relative input path */ int nFull, /* Size of output buffer in bytes */ @@ -49294,6 +50521,20 @@ static int winFullPathname( } #endif } +static int winFullPathname( + sqlite3_vfs *pVfs, /* Pointer to vfs object */ + const char *zRelative, /* Possibly relative input path */ + int nFull, /* Size of output buffer in bytes */ + char *zFull /* Output buffer */ +){ + int rc; + MUTEX_LOGIC( sqlite3_mutex *pMutex; ) + MUTEX_LOGIC( pMutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_TEMPDIR); ) + sqlite3_mutex_enter(pMutex); + rc = winFullPathnameNoMutex(pVfs, zRelative, nFull, zFull); + sqlite3_mutex_leave(pMutex); + return rc; +} #ifndef SQLITE_OMIT_LOAD_EXTENSION /* @@ -51082,12 +52323,20 @@ struct PCache { int sqlite3PcacheTrace = 2; /* 0: off 1: simple 2: cache dumps */ int sqlite3PcacheMxDump = 9999; /* Max cache entries for pcacheDump() */ # define pcacheTrace(X) if(sqlite3PcacheTrace){sqlite3DebugPrintf X;} - void pcacheDump(PCache *pCache){ - int N; - int i, j; - sqlite3_pcache_page *pLower; + static void pcachePageTrace(int i, sqlite3_pcache_page *pLower){ PgHdr *pPg; unsigned char *a; + int j; + pPg = (PgHdr*)pLower->pExtra; + printf("%3d: nRef %2d flgs %02x data ", i, pPg->nRef, pPg->flags); + a = (unsigned char *)pLower->pBuf; + for(j=0; j<12; j++) printf("%02x", a[j]); + printf(" ptr %p\n", pPg); + } + static void pcacheDump(PCache *pCache){ + int N; + int i; + sqlite3_pcache_page *pLower; if( sqlite3PcacheTrace<2 ) return; if( pCache->pCache==0 ) return; @@ -51096,21 +52345,32 @@ struct PCache { for(i=1; i<=N; i++){ pLower = sqlite3GlobalConfig.pcache2.xFetch(pCache->pCache, i, 0); if( pLower==0 ) continue; - pPg = (PgHdr*)pLower->pExtra; - printf("%3d: nRef %2d flgs %02x data ", i, pPg->nRef, pPg->flags); - a = (unsigned char *)pLower->pBuf; - for(j=0; j<12; j++) printf("%02x", a[j]); - printf("\n"); - if( pPg->pPage==0 ){ + pcachePageTrace(i, pLower); + if( ((PgHdr*)pLower)->pPage==0 ){ sqlite3GlobalConfig.pcache2.xUnpin(pCache->pCache, pLower, 0); } } } - #else +#else # define pcacheTrace(X) +# define pcachePageTrace(PGNO, X) # define pcacheDump(X) #endif +/* +** Return 1 if pPg is on the dirty list for pCache. Return 0 if not. +** This routine runs inside of assert() statements only. +*/ +#ifdef SQLITE_DEBUG +static int pageOnDirtyList(PCache *pCache, PgHdr *pPg){ + PgHdr *p; + for(p=pCache->pDirty; p; p=p->pDirtyNext){ + if( p==pPg ) return 1; + } + return 0; +} +#endif + /* ** Check invariants on a PgHdr entry. Return true if everything is OK. ** Return false if any invariant is violated. @@ -51129,8 +52389,13 @@ SQLITE_PRIVATE int sqlite3PcachePageSanity(PgHdr *pPg){ assert( pCache!=0 ); /* Every page has an associated PCache */ if( pPg->flags & PGHDR_CLEAN ){ assert( (pPg->flags & PGHDR_DIRTY)==0 );/* Cannot be both CLEAN and DIRTY */ - assert( pCache->pDirty!=pPg ); /* CLEAN pages not on dirty list */ - assert( pCache->pDirtyTail!=pPg ); + assert( !pageOnDirtyList(pCache, pPg) );/* CLEAN pages not on dirty list */ + }else{ + assert( (pPg->flags & PGHDR_DIRTY)!=0 );/* If not CLEAN must be DIRTY */ + assert( pPg->pDirtyNext==0 || pPg->pDirtyNext->pDirtyPrev==pPg ); + assert( pPg->pDirtyPrev==0 || pPg->pDirtyPrev->pDirtyNext==pPg ); + assert( pPg->pDirtyPrev!=0 || pCache->pDirty==pPg ); + assert( pageOnDirtyList(pCache, pPg) ); } /* WRITEABLE pages must also be DIRTY */ if( pPg->flags & PGHDR_WRITEABLE ){ @@ -51404,8 +52669,9 @@ SQLITE_PRIVATE sqlite3_pcache_page *sqlite3PcacheFetch( assert( createFlag==0 || pCache->eCreate==eCreate ); assert( createFlag==0 || eCreate==1+(!pCache->bPurgeable||!pCache->pDirty) ); pRes = sqlite3GlobalConfig.pcache2.xFetch(pCache->pCache, pgno, eCreate); - pcacheTrace(("%p.FETCH %d%s (result: %p)\n",pCache,pgno, + pcacheTrace(("%p.FETCH %d%s (result: %p) ",pCache,pgno, createFlag?" create":"",pRes)); + pcachePageTrace(pgno, pRes); return pRes; } @@ -51533,6 +52799,7 @@ SQLITE_PRIVATE void SQLITE_NOINLINE sqlite3PcacheRelease(PgHdr *p){ pcacheUnpin(p); }else{ pcacheManageDirtyList(p, PCACHE_DIRTYLIST_FRONT); + assert( sqlite3PcachePageSanity(p) ); } } } @@ -51576,6 +52843,7 @@ SQLITE_PRIVATE void sqlite3PcacheMakeDirty(PgHdr *p){ pcacheTrace(("%p.DIRTY %d\n",p->pCache,p->pgno)); assert( (p->flags & (PGHDR_DIRTY|PGHDR_CLEAN))==PGHDR_DIRTY ); pcacheManageDirtyList(p, PCACHE_DIRTYLIST_ADD); + assert( sqlite3PcachePageSanity(p) ); } assert( sqlite3PcachePageSanity(p) ); } @@ -51638,14 +52906,24 @@ SQLITE_PRIVATE void sqlite3PcacheClearSyncFlags(PCache *pCache){ */ SQLITE_PRIVATE void sqlite3PcacheMove(PgHdr *p, Pgno newPgno){ PCache *pCache = p->pCache; + sqlite3_pcache_page *pOther; assert( p->nRef>0 ); assert( newPgno>0 ); assert( sqlite3PcachePageSanity(p) ); pcacheTrace(("%p.MOVE %d -> %d\n",pCache,p->pgno,newPgno)); + pOther = sqlite3GlobalConfig.pcache2.xFetch(pCache->pCache, newPgno, 0); + if( pOther ){ + PgHdr *pXPage = (PgHdr*)pOther->pExtra; + assert( pXPage->nRef==0 ); + pXPage->nRef++; + pCache->nRefSum++; + sqlite3PcacheDrop(pXPage); + } sqlite3GlobalConfig.pcache2.xRekey(pCache->pCache, p->pPage, p->pgno,newPgno); p->pgno = newPgno; if( (p->flags&PGHDR_DIRTY) && (p->flags&PGHDR_NEED_SYNC) ){ pcacheManageDirtyList(p, PCACHE_DIRTYLIST_FRONT); + assert( sqlite3PcachePageSanity(p) ); } } @@ -51943,12 +53221,13 @@ SQLITE_PRIVATE void sqlite3PcacheIterateDirty(PCache *pCache, void (*xIter)(PgHd ** size can vary according to architecture, compile-time options, and ** SQLite library version number. ** -** If SQLITE_PCACHE_SEPARATE_HEADER is defined, then the extension is obtained -** using a separate memory allocation from the database page content. This -** seeks to overcome the "clownshoe" problem (also called "internal -** fragmentation" in academic literature) of allocating a few bytes more -** than a power of two with the memory allocator rounding up to the next -** power of two, and leaving the rounded-up space unused. +** Historical note: It used to be that if the SQLITE_PCACHE_SEPARATE_HEADER +** was defined, then the page content would be held in a separate memory +** allocation from the PgHdr1. This was intended to avoid clownshoe memory +** allocations. However, the btree layer needs a small (16-byte) overrun +** area after the page content buffer. The header serves as that overrun +** area. Therefore SQLITE_PCACHE_SEPARATE_HEADER was discontinued to avoid +** any possibility of a memory error. ** ** This module tracks pointers to PgHdr1 objects. Only pcache.c communicates ** with this module. Information is passed back and forth as PgHdr1 pointers. @@ -51993,30 +53272,40 @@ typedef struct PGroup PGroup; /* ** Each cache entry is represented by an instance of the following -** structure. Unless SQLITE_PCACHE_SEPARATE_HEADER is defined, a buffer of -** PgHdr1.pCache->szPage bytes is allocated directly before this structure -** in memory. +** structure. A buffer of PgHdr1.pCache->szPage bytes is allocated +** directly before this structure and is used to cache the page content. ** -** Note: Variables isBulkLocal and isAnchor were once type "u8". That works, +** When reading a corrupt database file, it is possible that SQLite might +** read a few bytes (no more than 16 bytes) past the end of the page buffer. +** It will only read past the end of the page buffer, never write. This +** object is positioned immediately after the page buffer to serve as an +** overrun area, so that overreads are harmless. +** +** Variables isBulkLocal and isAnchor were once type "u8". That works, ** but causes a 2-byte gap in the structure for most architectures (since ** pointers must be either 4 or 8-byte aligned). As this structure is located ** in memory directly after the associated page data, if the database is ** corrupt, code at the b-tree layer may overread the page buffer and ** read part of this structure before the corruption is detected. This ** can cause a valgrind error if the unitialized gap is accessed. Using u16 -** ensures there is no such gap, and therefore no bytes of unitialized memory -** in the structure. +** ensures there is no such gap, and therefore no bytes of uninitialized +** memory in the structure. +** +** The pLruNext and pLruPrev pointers form a double-linked circular list +** of all pages that are unpinned. The PGroup.lru element (which should be +** the only element on the list with PgHdr1.isAnchor set to 1) forms the +** beginning and the end of the list. */ struct PgHdr1 { - sqlite3_pcache_page page; /* Base class. Must be first. pBuf & pExtra */ - unsigned int iKey; /* Key value (page number) */ - u16 isBulkLocal; /* This page from bulk local storage */ - u16 isAnchor; /* This is the PGroup.lru element */ - PgHdr1 *pNext; /* Next in hash table chain */ - PCache1 *pCache; /* Cache that currently owns this page */ - PgHdr1 *pLruNext; /* Next in LRU list of unpinned pages */ - PgHdr1 *pLruPrev; /* Previous in LRU list of unpinned pages */ - /* NB: pLruPrev is only valid if pLruNext!=0 */ + sqlite3_pcache_page page; /* Base class. Must be first. pBuf & pExtra */ + unsigned int iKey; /* Key value (page number) */ + u16 isBulkLocal; /* This page from bulk local storage */ + u16 isAnchor; /* This is the PGroup.lru element */ + PgHdr1 *pNext; /* Next in hash table chain */ + PCache1 *pCache; /* Cache that currently owns this page */ + PgHdr1 *pLruNext; /* Next in circular LRU list of unpinned pages */ + PgHdr1 *pLruPrev; /* Previous in LRU list of unpinned pages */ + /* NB: pLruPrev is only valid if pLruNext!=0 */ }; /* @@ -52342,25 +53631,13 @@ static PgHdr1 *pcache1AllocPage(PCache1 *pCache, int benignMalloc){ pcache1LeaveMutex(pCache->pGroup); #endif if( benignMalloc ){ sqlite3BeginBenignMalloc(); } -#ifdef SQLITE_PCACHE_SEPARATE_HEADER - pPg = pcache1Alloc(pCache->szPage); - p = sqlite3Malloc(sizeof(PgHdr1) + pCache->szExtra); - if( !pPg || !p ){ - pcache1Free(pPg); - sqlite3_free(p); - pPg = 0; - } -#else pPg = pcache1Alloc(pCache->szAlloc); -#endif if( benignMalloc ){ sqlite3EndBenignMalloc(); } #ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT pcache1EnterMutex(pCache->pGroup); #endif if( pPg==0 ) return 0; -#ifndef SQLITE_PCACHE_SEPARATE_HEADER p = (PgHdr1 *)&((u8 *)pPg)[pCache->szPage]; -#endif p->page.pBuf = pPg; p->page.pExtra = &p[1]; p->isBulkLocal = 0; @@ -52384,9 +53661,6 @@ static void pcache1FreePage(PgHdr1 *p){ pCache->pFree = p; }else{ pcache1Free(p->page.pBuf); -#ifdef SQLITE_PCACHE_SEPARATE_HEADER - sqlite3_free(p); -#endif } (*pCache->pnPurgeable)--; } @@ -53027,23 +54301,26 @@ static void pcache1Rekey( PCache1 *pCache = (PCache1 *)p; PgHdr1 *pPage = (PgHdr1 *)pPg; PgHdr1 **pp; - unsigned int h; + unsigned int hOld, hNew; assert( pPage->iKey==iOld ); assert( pPage->pCache==pCache ); + assert( iOld!=iNew ); /* The page number really is changing */ pcache1EnterMutex(pCache->pGroup); - h = iOld%pCache->nHash; - pp = &pCache->apHash[h]; + assert( pcache1FetchNoMutex(p, iOld, 0)==pPage ); /* pPg really is iOld */ + hOld = iOld%pCache->nHash; + pp = &pCache->apHash[hOld]; while( (*pp)!=pPage ){ pp = &(*pp)->pNext; } *pp = pPage->pNext; - h = iNew%pCache->nHash; + assert( pcache1FetchNoMutex(p, iNew, 0)==0 ); /* iNew not in cache */ + hNew = iNew%pCache->nHash; pPage->iKey = iNew; - pPage->pNext = pCache->apHash[h]; - pCache->apHash[h] = pPage; + pPage->pNext = pCache->apHash[hNew]; + pCache->apHash[hNew] = pPage; if( iNew>pCache->iMaxKey ){ pCache->iMaxKey = iNew; } @@ -53150,9 +54427,6 @@ SQLITE_PRIVATE int sqlite3PcacheReleaseMemory(int nReq){ && p->isAnchor==0 ){ nFree += pcache1MemSize(p->page.pBuf); -#ifdef SQLITE_PCACHE_SEPARATE_HEADER - nFree += sqlite3MemSize(p); -#endif assert( PAGE_IS_UNPINNED(p) ); pcache1PinPage(p); pcache1RemoveFromHash(p, 1); @@ -59641,6 +60915,7 @@ static int pager_open_journal(Pager *pPager){ if( pPager->tempFile ){ flags |= (SQLITE_OPEN_DELETEONCLOSE|SQLITE_OPEN_TEMP_JOURNAL); + flags |= SQLITE_OPEN_EXCLUSIVE; nSpill = sqlite3Config.nStmtSpill; }else{ flags |= SQLITE_OPEN_MAIN_JOURNAL; @@ -59676,6 +60951,7 @@ static int pager_open_journal(Pager *pPager){ if( rc!=SQLITE_OK ){ sqlite3BitvecDestroy(pPager->pInJournal); pPager->pInJournal = 0; + pPager->journalOff = 0; }else{ assert( pPager->eState==PAGER_WRITER_LOCKED ); pPager->eState = PAGER_WRITER_CACHEMOD; @@ -66739,6 +68015,7 @@ SQLITE_PRIVATE int sqlite3BtreeHoldsAllMutexes(sqlite3 *db){ SQLITE_PRIVATE int sqlite3SchemaMutexHeld(sqlite3 *db, int iDb, Schema *pSchema){ Btree *p; assert( db!=0 ); + if( db->pVfs==0 && db->nDb==0 ) return 1; if( pSchema ) iDb = sqlite3SchemaToIndex(db, pSchema); assert( iDb>=0 && iDbnDb ); if( !sqlite3_mutex_held(db->mutex) ) return 0; @@ -68311,8 +69588,7 @@ static int defragmentPage(MemPage *pPage, int nMaxFrag){ assert( pPage->pBt->usableSize <= SQLITE_MAX_PAGE_SIZE ); assert( pPage->nOverflow==0 ); assert( sqlite3_mutex_held(pPage->pBt->mutex) ); - temp = 0; - src = data = pPage->aData; + data = pPage->aData; hdr = pPage->hdrOffset; cellOffset = pPage->cellOffset; nCell = pPage->nCell; @@ -68346,7 +69622,7 @@ static int defragmentPage(MemPage *pPage, int nMaxFrag){ if( iFree2+sz2 > usableSize ) return SQLITE_CORRUPT_PAGE(pPage); memmove(&data[iFree+sz+sz2], &data[iFree+sz], iFree2-(iFree+sz)); sz += sz2; - }else if( NEVER(iFree+sz>usableSize) ){ + }else if( iFree+sz>usableSize ){ return SQLITE_CORRUPT_PAGE(pPage); } @@ -68366,39 +69642,38 @@ static int defragmentPage(MemPage *pPage, int nMaxFrag){ cbrk = usableSize; iCellLast = usableSize - 4; iCellStart = get2byte(&data[hdr+5]); - for(i=0; iiCellLast ){ - return SQLITE_CORRUPT_PAGE(pPage); + if( nCell>0 ){ + temp = sqlite3PagerTempSpace(pPage->pBt->pPager); + memcpy(&temp[iCellStart], &data[iCellStart], usableSize - iCellStart); + src = temp; + for(i=0; iiCellLast ){ + return SQLITE_CORRUPT_PAGE(pPage); + } + assert( pc>=iCellStart && pc<=iCellLast ); + size = pPage->xCellSize(pPage, &src[pc]); + cbrk -= size; + if( cbrkusableSize ){ + return SQLITE_CORRUPT_PAGE(pPage); + } + assert( cbrk+size<=usableSize && cbrk>=iCellStart ); + testcase( cbrk+size==usableSize ); + testcase( pc+size==usableSize ); + put2byte(pAddr, cbrk); + memcpy(&data[cbrk], &src[pc], size); } - assert( pc>=iCellStart && pc<=iCellLast ); - size = pPage->xCellSize(pPage, &src[pc]); - cbrk -= size; - if( cbrkusableSize ){ - return SQLITE_CORRUPT_PAGE(pPage); - } - assert( cbrk+size<=usableSize && cbrk>=iCellStart ); - testcase( cbrk+size==usableSize ); - testcase( pc+size==usableSize ); - put2byte(pAddr, cbrk); - if( temp==0 ){ - if( cbrk==pc ) continue; - temp = sqlite3PagerTempSpace(pPage->pBt->pPager); - memcpy(&temp[iCellStart], &data[iCellStart], usableSize - iCellStart); - src = temp; - } - memcpy(&data[cbrk], &src[pc], size); } data[hdr+7] = 0; - defragment_out: +defragment_out: assert( pPage->nFree>=0 ); if( data[hdr+7]+cbrk-iCellFirst!=pPage->nFree ){ return SQLITE_CORRUPT_PAGE(pPage); @@ -68455,7 +69730,6 @@ static u8 *pageFindSlot(MemPage *pPg, int nByte, int *pRc){ ** fragmented bytes within the page. */ memcpy(&aData[iAddr], &aData[pc], 2); aData[hdr+7] += (u8)x; - testcase( pc+x>maxPC ); return &aData[pc]; }else if( x+pc > maxPC ){ /* This slot extends off the end of the usable part of the page */ @@ -68471,9 +69745,9 @@ static u8 *pageFindSlot(MemPage *pPg, int nByte, int *pRc){ iAddr = pc; pTmp = &aData[pc]; pc = get2byte(pTmp); - if( pc<=iAddr+size ){ + if( pc<=iAddr ){ if( pc ){ - /* The next slot in the chain is not past the end of the current slot */ + /* The next slot in the chain comes before the current slot */ *pRc = SQLITE_CORRUPT_PAGE(pPg); } return 0; @@ -68625,7 +69899,7 @@ static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){ iFreeBlk = 0; /* Shortcut for the case when the freelist is empty */ }else{ while( (iFreeBlk = get2byte(&data[iPtr]))pPage = pCur->apPage[pCur->iPage]; } testcase( pgno==0 ); - assert( pgno!=0 || rc==SQLITE_CORRUPT - || rc==SQLITE_IOERR_NOMEM - || rc==SQLITE_NOMEM ); + assert( pgno!=0 || rc!=SQLITE_OK ); return rc; } @@ -70545,6 +71817,9 @@ static int modifyPagePointer(MemPage *pPage, Pgno iFrom, Pgno iTo, u8 eType){ } } }else{ + if( pCell+4 > pPage->aData+pPage->pBt->usableSize ){ + return SQLITE_CORRUPT_PAGE(pPage); + } if( get4byte(pCell)==iFrom ){ put4byte(pCell, iTo); break; @@ -72051,8 +73326,6 @@ SQLITE_PRIVATE const void *sqlite3BtreePayloadFetch(BtCursor *pCur, u32 *pAmt){ ** vice-versa). */ static int moveToChild(BtCursor *pCur, u32 newPgno){ - BtShared *pBt = pCur->pBt; - assert( cursorOwnsBtShared(pCur) ); assert( pCur->eState==CURSOR_VALID ); assert( pCur->iPageapPage[pCur->iPage] = pCur->pPage; pCur->ix = 0; pCur->iPage++; - return getAndInitPage(pBt, newPgno, &pCur->pPage, pCur, pCur->curPagerFlags); + return getAndInitPage(pCur->pBt, newPgno, &pCur->pPage, pCur, + pCur->curPagerFlags); } #ifdef SQLITE_DEBUG @@ -72172,7 +73446,7 @@ static int moveToRoot(BtCursor *pCur){ } sqlite3BtreeClearCursor(pCur); } - rc = getAndInitPage(pCur->pBtree->pBt, pCur->pgnoRoot, &pCur->pPage, + rc = getAndInitPage(pCur->pBt, pCur->pgnoRoot, &pCur->pPage, 0, pCur->curPagerFlags); if( rc!=SQLITE_OK ){ pCur->eState = CURSOR_INVALID; @@ -72880,14 +74154,7 @@ static SQLITE_NOINLINE int btreeNext(BtCursor *pCur){ pPage = pCur->pPage; idx = ++pCur->ix; - if( !pPage->isInit || sqlite3FaultSim(412) ){ - /* The only known way for this to happen is for there to be a - ** recursive SQL function that does a DELETE operation as part of a - ** SELECT which deletes content out from under an active cursor - ** in a corrupt database file where the table being DELETE-ed from - ** has pages in common with the table being queried. See TH3 - ** module cov1/btree78.test testcase 220 (2018-06-08) for an - ** example. */ + if( NEVER(!pPage->isInit) || sqlite3FaultSim(412) ){ return SQLITE_CORRUPT_BKPT; } @@ -73063,8 +74330,8 @@ static int allocateBtreePage( assert( eMode==BTALLOC_ANY || (nearby>0 && IfNotOmitAV(pBt->autoVacuum)) ); pPage1 = pBt->pPage1; mxPage = btreePagecount(pBt); - /* EVIDENCE-OF: R-05119-02637 The 4-byte big-endian integer at offset 36 - ** stores stores the total number of pages on the freelist. */ + /* EVIDENCE-OF: R-21003-45125 The 4-byte big-endian integer at offset 36 + ** stores the total number of pages on the freelist. */ n = get4byte(&pPage1->aData[36]); testcase( n==mxPage-1 ); if( n>=mxPage ){ @@ -73813,12 +75080,6 @@ static void dropCell(MemPage *pPage, int idx, int sz, int *pRC){ assert( pPage->pBt->usableSize > (u32)(ptr-data) ); pc = get2byte(ptr); hdr = pPage->hdrOffset; -#if 0 /* Not required. Omit for efficiency */ - if( pcnCell*2 ){ - *pRC = SQLITE_CORRUPT_BKPT; - return; - } -#endif testcase( pc==(u32)get2byte(&data[hdr+5]) ); testcase( pc+sz==pPage->pBt->usableSize ); if( pc+sz > pPage->pBt->usableSize ){ @@ -74702,8 +75963,6 @@ static int balance_nonroot( Pgno pgno; /* Temp var to store a page number in */ u8 abDone[NB+2]; /* True after i'th new page is populated */ Pgno aPgno[NB+2]; /* Page numbers of new pages before shuffling */ - Pgno aPgOrder[NB+2]; /* Copy of aPgno[] used for sorting pages */ - u16 aPgFlags[NB+2]; /* flags field of new pages before shuffling */ CellArray b; /* Parsed information on cells being balanced */ memset(abDone, 0, sizeof(abDone)); @@ -75127,42 +76386,39 @@ static int balance_nonroot( ** of the table is closer to a linear scan through the file. That in turn ** helps the operating system to deliver pages from the disk more rapidly. ** - ** An O(n^2) insertion sort algorithm is used, but since n is never more - ** than (NB+2) (a small constant), that should not be a problem. + ** An O(N*N) sort algorithm is used, but since N is never more than NB+2 + ** (5), that is not a performance concern. ** ** When NB==3, this one optimization makes the database about 25% faster ** for large insertions and deletions. */ for(i=0; ipgno; - aPgFlags[i] = apNew[i]->pDbPage->flags; - for(j=0; jpgno; + assert( apNew[i]->pDbPage->flags & PGHDR_WRITEABLE ); + assert( apNew[i]->pDbPage->flags & PGHDR_DIRTY ); } - for(i=0; ipgno < apNew[iB]->pgno ) iB = j; } - pgno = aPgOrder[iBest]; - aPgOrder[iBest] = 0xffffffff; - if( iBest!=i ){ - if( iBest>i ){ - sqlite3PagerRekey(apNew[iBest]->pDbPage, pBt->nPage+iBest+1, 0); - } - sqlite3PagerRekey(apNew[i]->pDbPage, pgno, aPgFlags[iBest]); - apNew[i]->pgno = pgno; + + /* If apNew[i] has a page number that is bigger than any of the + ** subsequence apNew[i] entries, then swap apNew[i] with the subsequent + ** entry that has the smallest page number (which we know to be + ** entry apNew[iB]). + */ + if( iB!=i ){ + Pgno pgnoA = apNew[i]->pgno; + Pgno pgnoB = apNew[iB]->pgno; + Pgno pgnoTemp = (PENDING_BYTE/pBt->pageSize)+1; + u16 fgA = apNew[i]->pDbPage->flags; + u16 fgB = apNew[iB]->pDbPage->flags; + sqlite3PagerRekey(apNew[i]->pDbPage, pgnoTemp, fgB); + sqlite3PagerRekey(apNew[iB]->pDbPage, pgnoA, fgA); + sqlite3PagerRekey(apNew[i]->pDbPage, pgnoB, fgB); + apNew[i]->pgno = pgnoB; + apNew[iB]->pgno = pgnoA; } } @@ -75588,6 +76844,11 @@ static int balance(BtCursor *pCur){ }else{ break; } + }else if( sqlite3PagerPageRefcount(pPage->pDbPage)>1 ){ + /* The page being written is not a root page, and there is currently + ** more than one reference to it. This only happens if the page is one + ** of its own ancestor pages. Corruption. */ + rc = SQLITE_CORRUPT_BKPT; }else{ MemPage * const pParent = pCur->apPage[iPage-1]; int const iIdx = pCur->aiIdx[iPage-1]; @@ -79431,6 +80692,16 @@ SQLITE_PRIVATE int sqlite3RealSameAsInt(double r1, sqlite3_int64 i){ && i >= -2251799813685248LL && i < 2251799813685248LL); } +/* Convert a floating point value to its closest integer. Do so in +** a way that avoids 'outside the range of representable values' warnings +** from UBSAN. +*/ +SQLITE_PRIVATE i64 sqlite3RealToI64(double r){ + if( r<=(double)SMALLEST_INT64 ) return SMALLEST_INT64; + if( r>=(double)LARGEST_INT64) return LARGEST_INT64; + return (i64)r; +} + /* ** Convert pMem so that it has type MEM_Real or MEM_Int. ** Invalidate any prior representations. @@ -79452,7 +80723,7 @@ SQLITE_PRIVATE int sqlite3VdbeMemNumerify(Mem *pMem){ assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); rc = sqlite3AtoF(pMem->z, &pMem->u.r, pMem->n, pMem->enc); if( ((rc==0 || rc==1) && sqlite3Atoi64(pMem->z, &ix, pMem->n, pMem->enc)<=1) - || sqlite3RealSameAsInt(pMem->u.r, (ix = (i64)pMem->u.r)) + || sqlite3RealSameAsInt(pMem->u.r, (ix = sqlite3RealToI64(pMem->u.r))) ){ pMem->u.i = ix; MemSetTypeFlag(pMem, MEM_Int); @@ -79504,6 +80775,7 @@ SQLITE_PRIVATE int sqlite3VdbeMemCast(Mem *pMem, u8 aff, u8 encoding){ sqlite3ValueApplyAffinity(pMem, SQLITE_AFF_TEXT, encoding); assert( pMem->flags & MEM_Str || pMem->db->mallocFailed ); pMem->flags &= ~(MEM_Int|MEM_Real|MEM_IntReal|MEM_Blob|MEM_Zero); + if( encoding!=SQLITE_UTF8 ) pMem->n &= ~1; return sqlite3VdbeChangeEncoding(pMem, encoding); } } @@ -80639,6 +81911,9 @@ SQLITE_PRIVATE int sqlite3ValueBytes(sqlite3_value *pVal, u8 enc){ if( (p->flags & MEM_Str)!=0 && pVal->enc==enc ){ return p->n; } + if( (p->flags & MEM_Str)!=0 && enc!=SQLITE_UTF8 && pVal->enc!=SQLITE_UTF8 ){ + return p->n; + } if( (p->flags & MEM_Blob)!=0 ){ if( p->flags & MEM_Zero ){ return p->n + p->u.nZero; @@ -80684,10 +81959,10 @@ SQLITE_PRIVATE Vdbe *sqlite3VdbeCreate(Parse *pParse){ memset(&p->aOp, 0, sizeof(Vdbe)-offsetof(Vdbe,aOp)); p->db = db; if( db->pVdbe ){ - db->pVdbe->pPrev = p; + db->pVdbe->ppVPrev = &p->pVNext; } - p->pNext = db->pVdbe; - p->pPrev = 0; + p->pVNext = db->pVdbe; + p->ppVPrev = &db->pVdbe; db->pVdbe = p; assert( p->eVdbeState==VDBE_INIT_STATE ); p->pParse = pParse; @@ -80769,21 +82044,28 @@ SQLITE_PRIVATE int sqlite3VdbeUsesDoubleQuotedString( #endif /* -** Swap all content between two VDBE structures. +** Swap byte-code between two VDBE structures. +** +** This happens after pB was previously run and returned +** SQLITE_SCHEMA. The statement was then reprepared in pA. +** This routine transfers the new bytecode in pA over to pB +** so that pB can be run again. The old pB byte code is +** moved back to pA so that it will be cleaned up when pA is +** finalized. */ SQLITE_PRIVATE void sqlite3VdbeSwap(Vdbe *pA, Vdbe *pB){ - Vdbe tmp, *pTmp; + Vdbe tmp, *pTmp, **ppTmp; char *zTmp; assert( pA->db==pB->db ); tmp = *pA; *pA = *pB; *pB = tmp; - pTmp = pA->pNext; - pA->pNext = pB->pNext; - pB->pNext = pTmp; - pTmp = pA->pPrev; - pA->pPrev = pB->pPrev; - pB->pPrev = pTmp; + pTmp = pA->pVNext; + pA->pVNext = pB->pVNext; + pB->pVNext = pTmp; + ppTmp = pA->ppVPrev; + pA->ppVPrev = pB->ppVPrev; + pB->ppVPrev = ppTmp; zTmp = pA->zSql; pA->zSql = pB->zSql; pB->zSql = zTmp; @@ -81035,6 +82317,7 @@ SQLITE_PRIVATE int sqlite3VdbeAddFunctionCall( addr = sqlite3VdbeAddOp4(v, eCallCtx ? OP_PureFunc : OP_Function, p1, p2, p3, (char*)pCtx, P4_FUNCCTX); sqlite3VdbeChangeP5(v, eCallCtx & NC_SelfRef); + sqlite3MayAbort(pParse); return addr; } @@ -81103,7 +82386,7 @@ SQLITE_PRIVATE void sqlite3VdbeExplain(Parse *pParse, u8 bPush, const char *zFmt iThis = v->nOp; sqlite3VdbeAddOp4(v, OP_Explain, iThis, pParse->addrExplain, 0, zMsg, P4_DYNAMIC); - sqlite3ExplainBreakpoint(bPush?"PUSH":"", sqlite3VdbeGetOp(v,-1)->p4.z); + sqlite3ExplainBreakpoint(bPush?"PUSH":"", sqlite3VdbeGetLastOp(v)->p4.z); if( bPush){ pParse->addrExplain = iThis; } @@ -81370,6 +82653,7 @@ SQLITE_PRIVATE int sqlite3VdbeAssertMayAbort(Vdbe *v, int mayAbort){ || opcode==OP_VDestroy || opcode==OP_VCreate || opcode==OP_ParseSchema + || opcode==OP_Function || opcode==OP_PureFunc || ((opcode==OP_Halt || opcode==OP_HaltIfNull) && ((pOp->p1)!=SQLITE_OK && pOp->p2==OE_Abort)) ){ @@ -81460,8 +82744,8 @@ static void resolveP2Values(Vdbe *p, int *pMaxFuncArgs){ p->readOnly = 1; p->bIsReader = 0; pOp = &p->aOp[p->nOp-1]; - while(1){ - + assert( p->aOp[0].opcode==OP_Init ); + while( 1 /* Loop termates when it reaches the OP_Init opcode */ ){ /* Only JUMP opcodes and the short list of special opcodes in the switch ** below need to be considered. The mkopcodeh.tcl generator script groups ** all these opcodes together near the front of the opcode list. Skip @@ -81490,6 +82774,10 @@ static void resolveP2Values(Vdbe *p, int *pMaxFuncArgs){ p->bIsReader = 1; break; } + case OP_Init: { + assert( pOp->p2>=0 ); + goto resolve_p2_values_loop_exit; + } #ifndef SQLITE_OMIT_VIRTUALTABLE case OP_VUpdate: { if( pOp->p2>nMaxArgs ) nMaxArgs = pOp->p2; @@ -81522,11 +82810,12 @@ static void resolveP2Values(Vdbe *p, int *pMaxFuncArgs){ ** have non-negative values for P2. */ assert( (sqlite3OpcodeProperty[pOp->opcode]&OPFLG_JUMP)==0 || pOp->p2>=0); } - if( pOp==p->aOp ) break; + assert( pOp>p->aOp ); pOp--; } +resolve_p2_values_loop_exit: if( aLabel ){ - sqlite3DbFreeNN(p->db, pParse->aLabel); + sqlite3DbNNFreeNN(p->db, pParse->aLabel); pParse->aLabel = 0; } pParse->nLabel = 0; @@ -81775,15 +83064,19 @@ SQLITE_PRIVATE void sqlite3VdbeScanStatus( ** for a specific instruction. */ SQLITE_PRIVATE void sqlite3VdbeChangeOpcode(Vdbe *p, int addr, u8 iNewOpcode){ + assert( addr>=0 ); sqlite3VdbeGetOp(p,addr)->opcode = iNewOpcode; } SQLITE_PRIVATE void sqlite3VdbeChangeP1(Vdbe *p, int addr, int val){ + assert( addr>=0 ); sqlite3VdbeGetOp(p,addr)->p1 = val; } SQLITE_PRIVATE void sqlite3VdbeChangeP2(Vdbe *p, int addr, int val){ + assert( addr>=0 || p->db->mallocFailed ); sqlite3VdbeGetOp(p,addr)->p2 = val; } SQLITE_PRIVATE void sqlite3VdbeChangeP3(Vdbe *p, int addr, int val){ + assert( addr>=0 ); sqlite3VdbeGetOp(p,addr)->p3 = val; } SQLITE_PRIVATE void sqlite3VdbeChangeP5(Vdbe *p, u16 p5){ @@ -81791,6 +83084,18 @@ SQLITE_PRIVATE void sqlite3VdbeChangeP5(Vdbe *p, u16 p5){ if( p->nOp>0 ) p->aOp[p->nOp-1].p5 = p5; } +/* +** If the previous opcode is an OP_Column that delivers results +** into register iDest, then add the OPFLAG_TYPEOFARG flag to that +** opcode. +*/ +SQLITE_PRIVATE void sqlite3VdbeTypeofColumn(Vdbe *p, int iDest){ + VdbeOp *pOp = sqlite3VdbeGetLastOp(p); + if( pOp->p3==iDest && pOp->opcode==OP_Column ){ + pOp->p5 |= OPFLAG_TYPEOFARG; + } +} + /* ** Change the P2 operand of instruction addr so that it points to ** the address of the next instruction to be coded. @@ -81819,7 +83124,7 @@ SQLITE_PRIVATE void sqlite3VdbeJumpHereOrPopInst(Vdbe *p, int addr){ || p->aOp[addr].opcode==OP_FkIfZero ); assert( p->aOp[addr].p4type==0 ); #ifdef SQLITE_VDBE_COVERAGE - sqlite3VdbeGetOp(p,-1)->iSrcLine = 0; /* Erase VdbeCoverage() macros */ + sqlite3VdbeGetLastOp(p)->iSrcLine = 0; /* Erase VdbeCoverage() macros */ #endif p->nOp--; }else{ @@ -81833,8 +83138,9 @@ SQLITE_PRIVATE void sqlite3VdbeJumpHereOrPopInst(Vdbe *p, int addr){ ** the FuncDef is not ephermal, then do nothing. */ static void freeEphemeralFunction(sqlite3 *db, FuncDef *pDef){ + assert( db!=0 ); if( (pDef->funcFlags & SQLITE_FUNC_EPHEM)!=0 ){ - sqlite3DbFreeNN(db, pDef); + sqlite3DbNNFreeNN(db, pDef); } } @@ -81843,11 +83149,12 @@ static void freeEphemeralFunction(sqlite3 *db, FuncDef *pDef){ */ static SQLITE_NOINLINE void freeP4Mem(sqlite3 *db, Mem *p){ if( p->szMalloc ) sqlite3DbFree(db, p->zMalloc); - sqlite3DbFreeNN(db, p); + sqlite3DbNNFreeNN(db, p); } static SQLITE_NOINLINE void freeP4FuncCtx(sqlite3 *db, sqlite3_context *p){ + assert( db!=0 ); freeEphemeralFunction(db, p->pFunc); - sqlite3DbFreeNN(db, p); + sqlite3DbNNFreeNN(db, p); } static void freeP4(sqlite3 *db, int p4type, void *p4){ assert( db ); @@ -81860,7 +83167,7 @@ static void freeP4(sqlite3 *db, int p4type, void *p4){ case P4_INT64: case P4_DYNAMIC: case P4_INTARRAY: { - sqlite3DbFree(db, p4); + if( p4 ) sqlite3DbNNFreeNN(db, p4); break; } case P4_KEYINFO: { @@ -81899,6 +83206,7 @@ static void freeP4(sqlite3 *db, int p4type, void *p4){ */ static void vdbeFreeOpArray(sqlite3 *db, Op *aOp, int nOp){ assert( nOp>=0 ); + assert( db!=0 ); if( aOp ){ Op *pOp = &aOp[nOp-1]; while(1){ /* Exit via break */ @@ -81909,7 +83217,7 @@ static void vdbeFreeOpArray(sqlite3 *db, Op *aOp, int nOp){ if( pOp==aOp ) break; pOp--; } - sqlite3DbFreeNN(db, aOp); + sqlite3DbNNFreeNN(db, aOp); } } @@ -82140,13 +83448,13 @@ SQLITE_PRIVATE void sqlite3VdbeNoopComment(Vdbe *p, const char *zFormat, ...){ ** Set the value if the iSrcLine field for the previously coded instruction. */ SQLITE_PRIVATE void sqlite3VdbeSetLineNumber(Vdbe *v, int iLine){ - sqlite3VdbeGetOp(v,-1)->iSrcLine = iLine; + sqlite3VdbeGetLastOp(v)->iSrcLine = iLine; } #endif /* SQLITE_VDBE_COVERAGE */ /* -** Return the opcode for a given address. If the address is -1, then -** return the most recently inserted opcode. +** Return the opcode for a given address. The address must be non-negative. +** See sqlite3VdbeGetLastOp() to get the most recently added opcode. ** ** If a memory allocation error has occurred prior to the calling of this ** routine, then a pointer to a dummy VdbeOp will be returned. That opcode @@ -82162,9 +83470,6 @@ SQLITE_PRIVATE VdbeOp *sqlite3VdbeGetOp(Vdbe *p, int addr){ ** zeros, which is correct. MSVC generates a warning, nevertheless. */ static VdbeOp dummy; /* Ignore the MSVC warning about no initializer */ assert( p->eVdbeState==VDBE_INIT_STATE ); - if( addr<0 ){ - addr = p->nOp - 1; - } assert( (addr>=0 && addrnOp) || p->db->mallocFailed ); if( p->db->mallocFailed ){ return (VdbeOp*)&dummy; @@ -82173,6 +83478,12 @@ SQLITE_PRIVATE VdbeOp *sqlite3VdbeGetOp(Vdbe *p, int addr){ } } +/* Return the most recently added opcode +*/ +VdbeOp * sqlite3VdbeGetLastOp(Vdbe *p){ + return sqlite3VdbeGetOp(p, p->nOp - 1); +} + #if defined(SQLITE_ENABLE_EXPLAIN_COMMENTS) /* ** Return an integer value for one of the parameters to the opcode pOp @@ -82660,7 +83971,7 @@ static void releaseMemArray(Mem *p, int N){ sqlite3VdbeMemRelease(p); p->flags = MEM_Undefined; }else if( p->szMalloc ){ - sqlite3DbFreeNN(db, p->zMalloc); + sqlite3DbNNFreeNN(db, p->zMalloc); p->szMalloc = 0; p->flags = MEM_Undefined; } @@ -83652,7 +84963,7 @@ static void checkActiveVdbeCnt(sqlite3 *db){ if( p->readOnly==0 ) nWrite++; if( p->bIsReader ) nRead++; } - p = p->pNext; + p = p->pVNext; } assert( cnt==db->nVdbeActive ); assert( nWrite==db->nVdbeWrite ); @@ -84181,10 +85492,11 @@ SQLITE_PRIVATE void sqlite3VdbeDeleteAuxData(sqlite3 *db, AuxData **pp, int iOp, */ static void sqlite3VdbeClearObject(sqlite3 *db, Vdbe *p){ SubProgram *pSub, *pNext; + assert( db!=0 ); assert( p->db==0 || p->db==db ); if( p->aColName ){ releaseMemArray(p->aColName, p->nResColumn*COLNAME_N); - sqlite3DbFreeNN(db, p->aColName); + sqlite3DbNNFreeNN(db, p->aColName); } for(pSub=p->pProgram; pSub; pSub=pNext){ pNext = pSub->pNext; @@ -84193,11 +85505,11 @@ static void sqlite3VdbeClearObject(sqlite3 *db, Vdbe *p){ } if( p->eVdbeState!=VDBE_INIT_STATE ){ releaseMemArray(p->aVar, p->nVar); - if( p->pVList ) sqlite3DbFreeNN(db, p->pVList); - if( p->pFree ) sqlite3DbFreeNN(db, p->pFree); + if( p->pVList ) sqlite3DbNNFreeNN(db, p->pVList); + if( p->pFree ) sqlite3DbNNFreeNN(db, p->pFree); } vdbeFreeOpArray(db, p->aOp, p->nOp); - sqlite3DbFree(db, p->zSql); + if( p->zSql ) sqlite3DbNNFreeNN(db, p->zSql); #ifdef SQLITE_ENABLE_NORMALIZE sqlite3DbFree(db, p->zNormSql); { @@ -84227,20 +85539,17 @@ SQLITE_PRIVATE void sqlite3VdbeDelete(Vdbe *p){ assert( p!=0 ); db = p->db; + assert( db!=0 ); assert( sqlite3_mutex_held(db->mutex) ); sqlite3VdbeClearObject(db, p); if( db->pnBytesFreed==0 ){ - if( p->pPrev ){ - p->pPrev->pNext = p->pNext; - }else{ - assert( db->pVdbe==p ); - db->pVdbe = p->pNext; - } - if( p->pNext ){ - p->pNext->pPrev = p->pPrev; + assert( p->ppVPrev!=0 ); + *p->ppVPrev = p->pVNext; + if( p->pVNext ){ + p->pVNext->ppVPrev = p->ppVPrev; } } - sqlite3DbFreeNN(db, p); + sqlite3DbNNFreeNN(db, p); } /* @@ -85195,7 +86504,7 @@ SQLITE_PRIVATE int sqlite3VdbeRecordCompareWithSkip( assert( pPKey2->pKeyInfo->aSortFlags!=0 ); assert( pPKey2->pKeyInfo->nKeyField>0 ); assert( idx1<=szHdr1 || CORRUPT_DB ); - do{ + while( 1 /*exit-by-break*/ ){ u32 serial_type; /* RHS is an integer */ @@ -85205,7 +86514,7 @@ SQLITE_PRIVATE int sqlite3VdbeRecordCompareWithSkip( serial_type = aKey1[idx1]; testcase( serial_type==12 ); if( serial_type>=10 ){ - rc = +1; + rc = serial_type==10 ? -1 : +1; }else if( serial_type==0 ){ rc = -1; }else if( serial_type==7 ){ @@ -85230,7 +86539,7 @@ SQLITE_PRIVATE int sqlite3VdbeRecordCompareWithSkip( ** numbers). Types 10 and 11 are currently "reserved for future ** use", so it doesn't really matter what the results of comparing ** them to numberic values are. */ - rc = +1; + rc = serial_type==10 ? -1 : +1; }else if( serial_type==0 ){ rc = -1; }else{ @@ -85311,7 +86620,7 @@ SQLITE_PRIVATE int sqlite3VdbeRecordCompareWithSkip( /* RHS is null */ else{ serial_type = aKey1[idx1]; - rc = (serial_type!=0); + rc = (serial_type!=0 && serial_type!=10); } if( rc!=0 ){ @@ -85333,8 +86642,13 @@ SQLITE_PRIVATE int sqlite3VdbeRecordCompareWithSkip( if( i==pPKey2->nField ) break; pRhs++; d1 += sqlite3VdbeSerialTypeLen(serial_type); + if( d1>(unsigned)nKey1 ) break; idx1 += sqlite3VarintLen(serial_type); - }while( idx1<(unsigned)szHdr1 && d1<=(unsigned)nKey1 ); + if( idx1>=(unsigned)szHdr1 ){ + pPKey2->errCode = (u8)SQLITE_CORRUPT_BKPT; + return 0; /* Corrupt index */ + } + } /* No memory allocation is ever used on mem1. Prove this using ** the following assert(). If the assert() fails, it indicates a @@ -85735,7 +87049,7 @@ SQLITE_PRIVATE void sqlite3VdbeCountChanges(Vdbe *v){ */ SQLITE_PRIVATE void sqlite3ExpirePreparedStatements(sqlite3 *db, int iCode){ Vdbe *p; - for(p = db->pVdbe; p; p=p->pNext){ + for(p = db->pVdbe; p; p=p->pVNext){ p->expired = iCode+1; } } @@ -85856,13 +87170,14 @@ SQLITE_PRIVATE void sqlite3VtabImportErrmsg(Vdbe *p, sqlite3_vtab *pVtab){ ** the vdbeUnpackRecord() function found in vdbeapi.c. */ static void vdbeFreeUnpacked(sqlite3 *db, int nField, UnpackedRecord *p){ + assert( db!=0 ); if( p ){ int i; for(i=0; iaMem[i]; if( pMem->zMalloc ) sqlite3VdbeMemReleaseMalloc(pMem); } - sqlite3DbFreeNN(db, p); + sqlite3DbNNFreeNN(db, p); } } #endif /* SQLITE_ENABLE_PREUPDATE_HOOK */ @@ -85933,7 +87248,7 @@ SQLITE_PRIVATE void sqlite3VdbePreUpdateHook( for(i=0; inField; i++){ sqlite3VdbeMemRelease(&preupdate.aNew[i]); } - sqlite3DbFreeNN(db, preupdate.aNew); + sqlite3DbNNFreeNN(db, preupdate.aNew); } } #endif /* SQLITE_ENABLE_PREUPDATE_HOOK */ @@ -86050,7 +87365,9 @@ SQLITE_API int sqlite3_finalize(sqlite3_stmt *pStmt){ if( vdbeSafety(v) ) return SQLITE_MISUSE_BKPT; sqlite3_mutex_enter(db->mutex); checkProfileCallback(db, v); - rc = sqlite3VdbeFinalize(v); + assert( v->eVdbeState>=VDBE_READY_STATE ); + rc = sqlite3VdbeReset(v); + sqlite3VdbeDelete(v); rc = sqlite3ApiExit(db, rc); sqlite3LeaveMutexAndCloseZombie(db); } @@ -86258,6 +87575,9 @@ SQLITE_API int sqlite3_value_type(sqlite3_value* pVal){ #endif return aType[pVal->flags&MEM_AffMask]; } +SQLITE_API int sqlite3_value_encoding(sqlite3_value *pVal){ + return pVal->enc; +} /* Return true if a parameter to xUpdate represents an unchanged column */ SQLITE_API int sqlite3_value_nochange(sqlite3_value *pVal){ @@ -87372,7 +88692,7 @@ SQLITE_API const void *sqlite3_column_origin_name16(sqlite3_stmt *pStmt, int N){ ** The error code stored in database p->db is overwritten with the return ** value in any case. */ -static int vdbeUnbind(Vdbe *p, int i){ +static int vdbeUnbind(Vdbe *p, unsigned int i){ Mem *pVar; if( vdbeSafetyNotNull(p) ){ return SQLITE_MISUSE_BKPT; @@ -87385,12 +88705,11 @@ static int vdbeUnbind(Vdbe *p, int i){ "bind on a busy prepared statement: [%s]", p->zSql); return SQLITE_MISUSE_BKPT; } - if( i<1 || i>p->nVar ){ + if( i>=(unsigned int)p->nVar ){ sqlite3Error(p->db, SQLITE_RANGE); sqlite3_mutex_leave(p->db->mutex); return SQLITE_RANGE; } - i--; pVar = &p->aVar[i]; sqlite3VdbeMemRelease(pVar); pVar->flags = MEM_Null; @@ -87427,7 +88746,7 @@ static int bindText( Mem *pVar; int rc; - rc = vdbeUnbind(p, i); + rc = vdbeUnbind(p, (u32)(i-1)); if( rc==SQLITE_OK ){ if( zData!=0 ){ pVar = &p->aVar[i-1]; @@ -87476,7 +88795,7 @@ SQLITE_API int sqlite3_bind_blob64( SQLITE_API int sqlite3_bind_double(sqlite3_stmt *pStmt, int i, double rValue){ int rc; Vdbe *p = (Vdbe *)pStmt; - rc = vdbeUnbind(p, i); + rc = vdbeUnbind(p, (u32)(i-1)); if( rc==SQLITE_OK ){ sqlite3VdbeMemSetDouble(&p->aVar[i-1], rValue); sqlite3_mutex_leave(p->db->mutex); @@ -87489,7 +88808,7 @@ SQLITE_API int sqlite3_bind_int(sqlite3_stmt *p, int i, int iValue){ SQLITE_API int sqlite3_bind_int64(sqlite3_stmt *pStmt, int i, sqlite_int64 iValue){ int rc; Vdbe *p = (Vdbe *)pStmt; - rc = vdbeUnbind(p, i); + rc = vdbeUnbind(p, (u32)(i-1)); if( rc==SQLITE_OK ){ sqlite3VdbeMemSetInt64(&p->aVar[i-1], iValue); sqlite3_mutex_leave(p->db->mutex); @@ -87499,7 +88818,7 @@ SQLITE_API int sqlite3_bind_int64(sqlite3_stmt *pStmt, int i, sqlite_int64 iValu SQLITE_API int sqlite3_bind_null(sqlite3_stmt *pStmt, int i){ int rc; Vdbe *p = (Vdbe*)pStmt; - rc = vdbeUnbind(p, i); + rc = vdbeUnbind(p, (u32)(i-1)); if( rc==SQLITE_OK ){ sqlite3_mutex_leave(p->db->mutex); } @@ -87514,7 +88833,7 @@ SQLITE_API int sqlite3_bind_pointer( ){ int rc; Vdbe *p = (Vdbe*)pStmt; - rc = vdbeUnbind(p, i); + rc = vdbeUnbind(p, (u32)(i-1)); if( rc==SQLITE_OK ){ sqlite3VdbeMemSetPointer(&p->aVar[i-1], pPtr, zPTtype, xDestructor); sqlite3_mutex_leave(p->db->mutex); @@ -87592,7 +88911,7 @@ SQLITE_API int sqlite3_bind_value(sqlite3_stmt *pStmt, int i, const sqlite3_valu SQLITE_API int sqlite3_bind_zeroblob(sqlite3_stmt *pStmt, int i, int n){ int rc; Vdbe *p = (Vdbe *)pStmt; - rc = vdbeUnbind(p, i); + rc = vdbeUnbind(p, (u32)(i-1)); if( rc==SQLITE_OK ){ #ifndef SQLITE_OMIT_INCRBLOB sqlite3VdbeMemSetZeroBlob(&p->aVar[i-1], n); @@ -87752,7 +89071,7 @@ SQLITE_API sqlite3_stmt *sqlite3_next_stmt(sqlite3 *pDb, sqlite3_stmt *pStmt){ if( pStmt==0 ){ pNext = (sqlite3_stmt*)pDb->pVdbe; }else{ - pNext = (sqlite3_stmt*)((Vdbe*)pStmt)->pNext; + pNext = (sqlite3_stmt*)((Vdbe*)pStmt)->pVNext; } sqlite3_mutex_leave(pDb->mutex); return pNext; @@ -87777,8 +89096,11 @@ SQLITE_API int sqlite3_stmt_status(sqlite3_stmt *pStmt, int op, int resetFlag){ sqlite3_mutex_enter(db->mutex); v = 0; db->pnBytesFreed = (int*)&v; + assert( db->lookaside.pEnd==db->lookaside.pTrueEnd ); + db->lookaside.pEnd = db->lookaside.pStart; sqlite3VdbeDelete(pVdbe); db->pnBytesFreed = 0; + db->lookaside.pEnd = db->lookaside.pTrueEnd; sqlite3_mutex_leave(db->mutex); }else{ v = pVdbe->aCounter[op]; @@ -88618,7 +89940,8 @@ static VdbeCursor *allocateCursor( ** return false. */ static int alsoAnInt(Mem *pRec, double rValue, i64 *piValue){ - i64 iValue = (double)rValue; + i64 iValue; + iValue = sqlite3RealToI64(rValue); if( sqlite3RealSameAsInt(rValue,iValue) ){ *piValue = iValue; return 1; @@ -88780,17 +90103,18 @@ static u16 SQLITE_NOINLINE computeNumericType(Mem *pMem){ ** But it does set pMem->u.r and pMem->u.i appropriately. */ static u16 numericType(Mem *pMem){ - if( pMem->flags & (MEM_Int|MEM_Real|MEM_IntReal) ){ + assert( (pMem->flags & MEM_Null)==0 + || pMem->db==0 || pMem->db->mallocFailed ); + if( pMem->flags & (MEM_Int|MEM_Real|MEM_IntReal|MEM_Null) ){ testcase( pMem->flags & MEM_Int ); testcase( pMem->flags & MEM_Real ); testcase( pMem->flags & MEM_IntReal ); - return pMem->flags & (MEM_Int|MEM_Real|MEM_IntReal); - } - if( pMem->flags & (MEM_Str|MEM_Blob) ){ - testcase( pMem->flags & MEM_Str ); - testcase( pMem->flags & MEM_Blob ); - return computeNumericType(pMem); + return pMem->flags & (MEM_Int|MEM_Real|MEM_IntReal|MEM_Null); } + assert( pMem->flags & (MEM_Str|MEM_Blob) ); + testcase( pMem->flags & MEM_Str ); + testcase( pMem->flags & MEM_Blob ); + return computeNumericType(pMem); return 0; } @@ -90035,7 +91359,6 @@ case OP_Subtract: /* same as TK_MINUS, in1, in2, out3 */ case OP_Multiply: /* same as TK_STAR, in1, in2, out3 */ case OP_Divide: /* same as TK_SLASH, in1, in2, out3 */ case OP_Remainder: { /* same as TK_REM, in1, in2, out3 */ - u16 flags; /* Combined MEM_* flags from both inputs */ u16 type1; /* Numeric type of left operand */ u16 type2; /* Numeric type of right operand */ i64 iA; /* Integer value of left operand */ @@ -90044,12 +91367,12 @@ case OP_Remainder: { /* same as TK_REM, in1, in2, out3 */ double rB; /* Real value of right operand */ pIn1 = &aMem[pOp->p1]; - type1 = numericType(pIn1); + type1 = pIn1->flags; pIn2 = &aMem[pOp->p2]; - type2 = numericType(pIn2); + type2 = pIn2->flags; pOut = &aMem[pOp->p3]; - flags = pIn1->flags | pIn2->flags; if( (type1 & type2 & MEM_Int)!=0 ){ +int_math: iA = pIn1->u.i; iB = pIn2->u.i; switch( pOp->opcode ){ @@ -90071,9 +91394,12 @@ case OP_Remainder: { /* same as TK_REM, in1, in2, out3 */ } pOut->u.i = iB; MemSetTypeFlag(pOut, MEM_Int); - }else if( (flags & MEM_Null)!=0 ){ + }else if( ((type1 | type2) & MEM_Null)!=0 ){ goto arithmetic_result_is_null; }else{ + type1 = numericType(pIn1); + type2 = numericType(pIn2); + if( (type1 & type2 & MEM_Int)!=0 ) goto int_math; fp_math: rA = sqlite3VdbeRealValue(pIn1); rB = sqlite3VdbeRealValue(pIn2); @@ -90888,19 +92214,90 @@ case OP_IsNull: { /* same as TK_ISNULL, jump, in1 */ break; } -/* Opcode: IsNullOrType P1 P2 P3 * * -** Synopsis: if typeof(r[P1]) IN (P3,5) goto P2 +/* Opcode: IsType P1 P2 P3 P4 P5 +** Synopsis: if typeof(P1.P3) in P5 goto P2 +** +** Jump to P2 if the type of a column in a btree is one of the types specified +** by the P5 bitmask. +** +** P1 is normally a cursor on a btree for which the row decode cache is +** valid through at least column P3. In other words, there should have been +** a prior OP_Column for column P3 or greater. If the cursor is not valid, +** then this opcode might give spurious results. +** The the btree row has fewer than P3 columns, then use P4 as the +** datatype. +** +** If P1 is -1, then P3 is a register number and the datatype is taken +** from the value in that register. +** +** P5 is a bitmask of data types. SQLITE_INTEGER is the least significant +** (0x01) bit. SQLITE_FLOAT is the 0x02 bit. SQLITE_TEXT is 0x04. +** SQLITE_BLOB is 0x08. SQLITE_NULL is 0x10. +** +** Take the jump to address P2 if and only if the datatype of the +** value determined by P1 and P3 corresponds to one of the bits in the +** P5 bitmask. ** -** Jump to P2 if the value in register P1 is NULL or has a datatype P3. -** P3 is an integer which should be one of SQLITE_INTEGER, SQLITE_FLOAT, -** SQLITE_BLOB, SQLITE_NULL, or SQLITE_TEXT. */ -case OP_IsNullOrType: { /* jump, in1 */ - int doTheJump; - pIn1 = &aMem[pOp->p1]; - doTheJump = (pIn1->flags & MEM_Null)!=0 || sqlite3_value_type(pIn1)==pOp->p3; - VdbeBranchTaken( doTheJump, 2); - if( doTheJump ) goto jump_to_p2; +case OP_IsType: { /* jump */ + VdbeCursor *pC; + u16 typeMask; + u32 serialType; + + assert( pOp->p1>=(-1) && pOp->p1nCursor ); + assert( pOp->p1>=0 || (pOp->p3>=0 && pOp->p3<=(p->nMem+1 - p->nCursor)) ); + if( pOp->p1>=0 ){ + pC = p->apCsr[pOp->p1]; + assert( pC!=0 ); + assert( pOp->p3>=0 ); + if( pOp->p3nHdrParsed ){ + serialType = pC->aType[pOp->p3]; + if( serialType>=12 ){ + if( serialType&1 ){ + typeMask = 0x04; /* SQLITE_TEXT */ + }else{ + typeMask = 0x08; /* SQLITE_BLOB */ + } + }else{ + static const unsigned char aMask[] = { + 0x10, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x2, + 0x01, 0x01, 0x10, 0x10 + }; + testcase( serialType==0 ); + testcase( serialType==1 ); + testcase( serialType==2 ); + testcase( serialType==3 ); + testcase( serialType==4 ); + testcase( serialType==5 ); + testcase( serialType==6 ); + testcase( serialType==7 ); + testcase( serialType==8 ); + testcase( serialType==9 ); + testcase( serialType==10 ); + testcase( serialType==11 ); + typeMask = aMask[serialType]; + } + }else{ + typeMask = 1 << (pOp->p4.i - 1); + testcase( typeMask==0x01 ); + testcase( typeMask==0x02 ); + testcase( typeMask==0x04 ); + testcase( typeMask==0x08 ); + testcase( typeMask==0x10 ); + } + }else{ + assert( memIsValid(&aMem[pOp->p3]) ); + typeMask = 1 << (sqlite3_value_type((sqlite3_value*)&aMem[pOp->p3])-1); + testcase( typeMask==0x01 ); + testcase( typeMask==0x02 ); + testcase( typeMask==0x04 ); + testcase( typeMask==0x08 ); + testcase( typeMask==0x10 ); + } + VdbeBranchTaken( (typeMask & pOp->p5)!=0, 2); + if( typeMask & pOp->p5 ){ + goto jump_to_p2; + } break; } @@ -90943,11 +92340,14 @@ case OP_NotNull: { /* same as TK_NOTNULL, jump, in1 */ ** If it is, then set register P3 to NULL and jump immediately to P2. ** If P1 is not on a NULL row, then fall through without making any ** changes. +** +** If P1 is not an open cursor, then this opcode is a no-op. */ case OP_IfNullRow: { /* jump */ + VdbeCursor *pC; assert( pOp->p1>=0 && pOp->p1nCursor ); - assert( p->apCsr[pOp->p1]!=0 ); - if( p->apCsr[pOp->p1]->nullRow ){ + pC = p->apCsr[pOp->p1]; + if( ALWAYS(pC) && pC->nullRow ){ sqlite3VdbeMemSetNull(aMem + pOp->p3); goto jump_to_p2; } @@ -90998,7 +92398,7 @@ case OP_Offset: { /* out3 */ ** Interpret the data that cursor P1 points to as a structure built using ** the MakeRecord instruction. (See the MakeRecord opcode for additional ** information about the format of the data.) Extract the P2-th column -** from this record. If there are less that (P2+1) +** from this record. If there are less than (P2+1) ** values in the record, extract a NULL. ** ** The value extracted is stored in register P3. @@ -91007,10 +92407,12 @@ case OP_Offset: { /* out3 */ ** if the P4 argument is a P4_MEM use the value of the P4 argument as ** the result. ** -** If the OPFLAG_LENGTHARG and OPFLAG_TYPEOFARG bits are set on P5 then -** the result is guaranteed to only be used as the argument of a length() -** or typeof() function, respectively. The loading of large blobs can be -** skipped for length() and all content loading can be skipped for typeof(). +** If the OPFLAG_LENGTHARG bit is set in P5 then the result is guaranteed +** to only be used by the length() function or the equivalent. The content +** of large blobs is not loaded, thus saving CPU cycles. If the +** OPFLAG_TYPEOFARG bit is set then the result will only be used by the +** typeof() function or the IS NULL or IS NOT NULL operators or the +** equivalent. In this case, all content loading can be omitted. */ case OP_Column: { u32 p2; /* column number to retrieve */ @@ -92949,7 +94351,13 @@ case OP_SeekGT: { /* jump, in3, group */ r.aMem = &aMem[pOp->p3]; #ifdef SQLITE_DEBUG - { int i; for(i=0; i0 ) REGISTER_TRACE(pOp->p3+i, &r.aMem[i]); + } + } #endif r.eqSeen = 0; rc = sqlite3BtreeIndexMoveto(pC->uc.pCursor, &r, &res); @@ -93012,7 +94420,7 @@ seek_not_found: } -/* Opcode: SeekScan P1 P2 * * * +/* Opcode: SeekScan P1 P2 * * P5 ** Synopsis: Scan-ahead up to P1 rows ** ** This opcode is a prefix opcode to OP_SeekGE. In other words, this @@ -93022,8 +94430,8 @@ seek_not_found: ** This opcode uses the P1 through P4 operands of the subsequent ** OP_SeekGE. In the text that follows, the operands of the subsequent ** OP_SeekGE opcode are denoted as SeekOP.P1 through SeekOP.P4. Only -** the P1 and P2 operands of this opcode are also used, and are called -** This.P1 and This.P2. +** the P1, P2 and P5 operands of this opcode are also used, and are called +** This.P1, This.P2 and This.P5. ** ** This opcode helps to optimize IN operators on a multi-column index ** where the IN operator is on the later terms of the index by avoiding @@ -93033,29 +94441,51 @@ seek_not_found: ** ** The SeekGE.P3 and SeekGE.P4 operands identify an unpacked key which ** is the desired entry that we want the cursor SeekGE.P1 to be pointing -** to. Call this SeekGE.P4/P5 row the "target". +** to. Call this SeekGE.P3/P4 row the "target". ** ** If the SeekGE.P1 cursor is not currently pointing to a valid row, ** then this opcode is a no-op and control passes through into the OP_SeekGE. ** ** If the SeekGE.P1 cursor is pointing to a valid row, then that row ** might be the target row, or it might be near and slightly before the -** target row. This opcode attempts to position the cursor on the target -** row by, perhaps by invoking sqlite3BtreeStep() on the cursor -** between 0 and This.P1 times. +** target row, or it might be after the target row. If the cursor is +** currently before the target row, then this opcode attempts to position +** the cursor on or after the target row by invoking sqlite3BtreeStep() +** on the cursor between 1 and This.P1 times. ** -** There are three possible outcomes from this opcode:
      +** The This.P5 parameter is a flag that indicates what to do if the +** cursor ends up pointing at a valid row that is past the target +** row. If This.P5 is false (0) then a jump is made to SeekGE.P2. If +** This.P5 is true (non-zero) then a jump is made to This.P2. The P5==0 +** case occurs when there are no inequality constraints to the right of +** the IN constraing. The jump to SeekGE.P2 ends the loop. The P5!=0 case +** occurs when there are inequality constraints to the right of the IN +** operator. In that case, the This.P2 will point either directly to or +** to setup code prior to the OP_IdxGT or OP_IdxGE opcode that checks for +** loop terminate. ** -**
    1. If after This.P1 steps, the cursor is still pointing to a place that -** is earlier in the btree than the target row, then fall through -** into the subsquence OP_SeekGE opcode. +** Possible outcomes from this opcode:
        ** -**
      1. If the cursor is successfully moved to the target row by 0 or more -** sqlite3BtreeNext() calls, then jump to This.P2, which will land just -** past the OP_IdxGT or OP_IdxGE opcode that follows the OP_SeekGE. +**
      2. If the cursor is initally not pointed to any valid row, then +** fall through into the subsequent OP_SeekGE opcode. ** -**
      3. If the cursor ends up past the target row (indicating the the target -** row does not exist in the btree) then jump to SeekOP.P2. +**
      4. If the cursor is left pointing to a row that is before the target +** row, even after making as many as This.P1 calls to +** sqlite3BtreeNext(), then also fall through into OP_SeekGE. +** +**
      5. If the cursor is left pointing at the target row, either because it +** was at the target row to begin with or because one or more +** sqlite3BtreeNext() calls moved the cursor to the target row, +** then jump to This.P2.., +** +**
      6. If the cursor started out before the target row and a call to +** to sqlite3BtreeNext() moved the cursor off the end of the index +** (indicating that the target row definitely does not exist in the +** btree) then jump to SeekGE.P2, ending the loop. +** +**
      7. If the cursor ends up on a valid row that is past the target row +** (indicating that the target row does not exist in the btree) then +** jump to SeekOP.P2 if This.P5==0 or to This.P2 if This.P5>0. **
      */ case OP_SeekScan: { @@ -93066,14 +94496,25 @@ case OP_SeekScan: { assert( pOp[1].opcode==OP_SeekGE ); - /* pOp->p2 points to the first instruction past the OP_IdxGT that - ** follows the OP_SeekGE. */ + /* If pOp->p5 is clear, then pOp->p2 points to the first instruction past the + ** OP_IdxGT that follows the OP_SeekGE. Otherwise, it points to the first + ** opcode past the OP_SeekGE itself. */ assert( pOp->p2>=(int)(pOp-aOp)+2 ); - assert( aOp[pOp->p2-1].opcode==OP_IdxGT || aOp[pOp->p2-1].opcode==OP_IdxGE ); - testcase( aOp[pOp->p2-1].opcode==OP_IdxGE ); - assert( pOp[1].p1==aOp[pOp->p2-1].p1 ); - assert( pOp[1].p2==aOp[pOp->p2-1].p2 ); - assert( pOp[1].p3==aOp[pOp->p2-1].p3 ); +#ifdef SQLITE_DEBUG + if( pOp->p5==0 ){ + /* There are no inequality constraints following the IN constraint. */ + assert( pOp[1].p1==aOp[pOp->p2-1].p1 ); + assert( pOp[1].p2==aOp[pOp->p2-1].p2 ); + assert( pOp[1].p3==aOp[pOp->p2-1].p3 ); + assert( aOp[pOp->p2-1].opcode==OP_IdxGT + || aOp[pOp->p2-1].opcode==OP_IdxGE ); + testcase( aOp[pOp->p2-1].opcode==OP_IdxGE ); + }else{ + /* There are inequality constraints. */ + assert( pOp->p2==(int)(pOp-aOp)+2 ); + assert( aOp[pOp->p2-1].opcode==OP_SeekGE ); + } +#endif assert( pOp->p1>0 ); pC = p->apCsr[pOp[1].p1]; @@ -93107,8 +94548,9 @@ case OP_SeekScan: { while(1){ rc = sqlite3VdbeIdxKeyCompare(db, pC, &r, &res); if( rc ) goto abort_due_to_error; - if( res>0 ){ + if( res>0 && pOp->p5==0 ){ seekscan_search_fail: + /* Jump to SeekGE.P2, ending the loop */ #ifdef SQLITE_DEBUG if( db->flags&SQLITE_VdbeTrace ){ printf("... %d steps and then skip\n", pOp->p1 - nStep); @@ -93118,7 +94560,8 @@ case OP_SeekScan: { pOp++; goto jump_to_p2; } - if( res==0 ){ + if( res>=0 ){ + /* Jump to This.P2, bypassing the OP_SeekGE opcode */ #ifdef SQLITE_DEBUG if( db->flags&SQLITE_VdbeTrace ){ printf("... %d steps and then success\n", pOp->p1 - nStep); @@ -93194,12 +94637,16 @@ case OP_SeekHit: { /* Opcode: IfNotOpen P1 P2 * * * ** Synopsis: if( !csr[P1] ) goto P2 ** -** If cursor P1 is not open, jump to instruction P2. Otherwise, fall through. +** If cursor P1 is not open or if P1 is set to a NULL row using the +** OP_NullRow opcode, then jump to instruction P2. Otherwise, fall through. */ case OP_IfNotOpen: { /* jump */ + VdbeCursor *pCur; + assert( pOp->p1>=0 && pOp->p1nCursor ); - VdbeBranchTaken(p->apCsr[pOp->p1]==0, 2); - if( !p->apCsr[pOp->p1] ){ + pCur = p->apCsr[pOp->p1]; + VdbeBranchTaken(pCur==0 || pCur->nullRow, 2); + if( pCur==0 || pCur->nullRow ){ goto jump_to_p2_and_check_for_interrupt; } break; @@ -94390,7 +95837,9 @@ case OP_SorterNext: { /* jump */ case OP_Prev: /* jump */ assert( pOp->p1>=0 && pOp->p1nCursor ); - assert( pOp->p5aCounter) ); + assert( pOp->p5==0 + || pOp->p5==SQLITE_STMTSTATUS_FULLSCAN_STEP + || pOp->p5==SQLITE_STMTSTATUS_AUTOINDEX); pC = p->apCsr[pOp->p1]; assert( pC!=0 ); assert( pC->deferredMoveto==0 ); @@ -94403,7 +95852,9 @@ case OP_Prev: /* jump */ case OP_Next: /* jump */ assert( pOp->p1>=0 && pOp->p1nCursor ); - assert( pOp->p5aCounter) ); + assert( pOp->p5==0 + || pOp->p5==SQLITE_STMTSTATUS_FULLSCAN_STEP + || pOp->p5==SQLITE_STMTSTATUS_AUTOINDEX); pC = p->apCsr[pOp->p1]; assert( pC!=0 ); assert( pC->deferredMoveto==0 ); @@ -94610,10 +96061,10 @@ case OP_IdxRowid: { /* out2 */ ** of sqlite3VdbeCursorRestore() and sqlite3VdbeIdxRowid(). */ rc = sqlite3VdbeCursorRestore(pC); - /* sqlite3VbeCursorRestore() can only fail if the record has been deleted - ** out from under the cursor. That will never happens for an IdxRowid - ** or Seek opcode */ - if( NEVER(rc!=SQLITE_OK) ) goto abort_due_to_error; + /* sqlite3VdbeCursorRestore() may fail if the cursor has been disturbed + ** since it was last positioned and an error (e.g. OOM or an IO error) + ** occurs while trying to reposition it. */ + if( rc!=SQLITE_OK ) goto abort_due_to_error; if( !pC->nullRow ){ rowid = 0; /* Not needed. Only used to silence a warning. */ @@ -95515,7 +96966,7 @@ case OP_IfPos: { /* jump, in1 */ ** Synopsis: if r[P1]>0 then r[P2]=r[P1]+max(0,r[P3]) else r[P2]=(-1) ** ** This opcode performs a commonly used computation associated with -** LIMIT and OFFSET process. r[P1] holds the limit counter. r[P3] +** LIMIT and OFFSET processing. r[P1] holds the limit counter. r[P3] ** holds the offset counter. The opcode computes the combined value ** of the LIMIT and OFFSET and stores that value in r[P2]. The r[P2] ** value computed is the total number of rows that will need to be @@ -101113,6 +102564,8 @@ SQLITE_PRIVATE int sqlite3JournalOpen( ){ MemJournal *p = (MemJournal*)pJfd; + assert( zName || nSpill<0 || (flags & SQLITE_OPEN_EXCLUSIVE) ); + /* Zero the file-handle object. If nSpill was passed zero, initialize ** it using the sqlite3OsOpen() function of the underlying VFS. In this ** case none of the code in this module is executed as a result of calls @@ -101554,9 +103007,7 @@ static void resolveAlias( pExpr->y.pWin->pOwner = pExpr; } } - sqlite3ParserAddCleanup(pParse, - (void(*)(sqlite3*,void*))sqlite3ExprDelete, - pDup); + sqlite3ExprDeferredDelete(pParse, pDup); } } @@ -103650,9 +105101,8 @@ SQLITE_PRIVATE char sqlite3ExprAffinity(const Expr *pExpr){ if( op==TK_REGISTER ) op = pExpr->op2; if( op==TK_COLUMN || op==TK_AGG_COLUMN ){ assert( ExprUseYTab(pExpr) ); - if( pExpr->y.pTab ){ - return sqlite3TableColumnAffinity(pExpr->y.pTab, pExpr->iColumn); - } + assert( pExpr->y.pTab!=0 ); + return sqlite3TableColumnAffinity(pExpr->y.pTab, pExpr->iColumn); } if( op==TK_SELECT ){ assert( ExprUseXSelect(pExpr) ); @@ -103770,17 +105220,14 @@ SQLITE_PRIVATE CollSeq *sqlite3ExprCollSeq(Parse *pParse, const Expr *pExpr){ int op = p->op; if( op==TK_REGISTER ) op = p->op2; if( op==TK_AGG_COLUMN || op==TK_COLUMN || op==TK_TRIGGER ){ + int j; assert( ExprUseYTab(p) ); - if( p->y.pTab!=0 ){ - /* op==TK_REGISTER && p->y.pTab!=0 happens when pExpr was originally - ** a TK_COLUMN but was previously evaluated and cached in a register */ - int j = p->iColumn; - if( j>=0 ){ - const char *zColl = sqlite3ColumnColl(&p->y.pTab->aCol[j]); - pColl = sqlite3FindCollSeq(db, ENC(db), zColl, 0); - } - break; + assert( p->y.pTab!=0 ); + if( (j = p->iColumn)>=0 ){ + const char *zColl = sqlite3ColumnColl(&p->y.pTab->aCol[j]); + pColl = sqlite3FindCollSeq(db, ENC(db), zColl, 0); } + break; } if( op==TK_CAST || op==TK_UPLUS ){ p = p->pLeft; @@ -104365,7 +105812,9 @@ static void heightOfSelect(const Select *pSelect, int *pnHeight){ */ static void exprSetHeight(Expr *p){ int nHeight = p->pLeft ? p->pLeft->nHeight : 0; - if( p->pRight && p->pRight->nHeight>nHeight ) nHeight = p->pRight->nHeight; + if( NEVER(p->pRight) && p->pRight->nHeight>nHeight ){ + nHeight = p->pRight->nHeight; + } if( ExprUseXSelect(p) ){ heightOfSelect(p->x.pSelect, &nHeight); }else if( p->x.pList ){ @@ -104508,15 +105957,26 @@ SQLITE_PRIVATE void sqlite3ExprAttachSubtrees( sqlite3ExprDelete(db, pLeft); sqlite3ExprDelete(db, pRight); }else{ + assert( ExprUseXList(pRoot) ); + assert( pRoot->x.pSelect==0 ); if( pRight ){ pRoot->pRight = pRight; pRoot->flags |= EP_Propagate & pRight->flags; +#if SQLITE_MAX_EXPR_DEPTH>0 + pRoot->nHeight = pRight->nHeight+1; + }else{ + pRoot->nHeight = 1; +#endif } if( pLeft ){ pRoot->pLeft = pLeft; pRoot->flags |= EP_Propagate & pLeft->flags; +#if SQLITE_MAX_EXPR_DEPTH>0 + if( pLeft->nHeight>=pRoot->nHeight ){ + pRoot->nHeight = pLeft->nHeight+1; + } +#endif } - exprSetHeight(pRoot); } } @@ -104802,6 +106262,7 @@ SQLITE_PRIVATE void sqlite3ExprAssignVarNumber(Parse *pParse, Expr *pExpr, u32 n */ static SQLITE_NOINLINE void sqlite3ExprDeleteNN(sqlite3 *db, Expr *p){ assert( p!=0 ); + assert( db!=0 ); assert( !ExprUseUValue(p) || p->u.iValue>=0 ); assert( !ExprUseYWin(p) || !ExprUseYSub(p) ); assert( !ExprUseYWin(p) || p->y.pWin!=0 || db->mallocFailed ); @@ -104833,12 +106294,8 @@ static SQLITE_NOINLINE void sqlite3ExprDeleteNN(sqlite3 *db, Expr *p){ #endif } } - if( ExprHasProperty(p, EP_MemToken) ){ - assert( !ExprHasProperty(p, EP_IntValue) ); - sqlite3DbFree(db, p->u.zToken); - } if( !ExprHasProperty(p, EP_Static) ){ - sqlite3DbFreeNN(db, p); + sqlite3DbNNFreeNN(db, p); } } SQLITE_PRIVATE void sqlite3ExprDelete(sqlite3 *db, Expr *p){ @@ -104869,8 +106326,9 @@ SQLITE_PRIVATE void sqlite3ClearOnOrUsing(sqlite3 *db, OnOrUsing *p){ ** pExpr to the pParse->pConstExpr list with a register number of 0. */ SQLITE_PRIVATE void sqlite3ExprDeferredDelete(Parse *pParse, Expr *pExpr){ - pParse->pConstExpr = - sqlite3ExprListAppend(pParse, pParse->pConstExpr, pExpr); + sqlite3ParserAddCleanup(pParse, + (void(*)(sqlite3*,void*))sqlite3ExprDelete, + pExpr); } /* Invoke sqlite3RenameExprUnmap() and sqlite3ExprDelete() on the @@ -104944,7 +106402,6 @@ static int dupedExprStructSize(const Expr *p, int flags){ }else{ assert( !ExprHasProperty(p, EP_TokenOnly|EP_Reduced) ); assert( !ExprHasProperty(p, EP_OuterON) ); - assert( !ExprHasProperty(p, EP_MemToken) ); assert( !ExprHasVVAProperty(p, EP_NoReduce) ); if( p->pLeft || p->x.pList ){ nSize = EXPR_REDUCEDSIZE | EP_Reduced; @@ -105048,7 +106505,7 @@ static Expr *exprDup(sqlite3 *db, const Expr *p, int dupFlags, u8 **pzBuffer){ } /* Set the EP_Reduced, EP_TokenOnly, and EP_Static flags appropriately. */ - pNew->flags &= ~(EP_Reduced|EP_TokenOnly|EP_Static|EP_MemToken); + pNew->flags &= ~(EP_Reduced|EP_TokenOnly|EP_Static); pNew->flags |= nStructSize & (EP_Reduced|EP_TokenOnly); pNew->flags |= staticFlag; ExprClearVVAProperties(pNew); @@ -105624,12 +107081,13 @@ static SQLITE_NOINLINE void exprListDeleteNN(sqlite3 *db, ExprList *pList){ int i = pList->nExpr; struct ExprList_item *pItem = pList->a; assert( pList->nExpr>0 ); + assert( db!=0 ); do{ sqlite3ExprDelete(db, pItem->pExpr); - sqlite3DbFree(db, pItem->zEName); + if( pItem->zEName ) sqlite3DbNNFreeNN(db, pItem->zEName); pItem++; }while( --i>0 ); - sqlite3DbFreeNN(db, pList); + sqlite3DbNNFreeNN(db, pList); } SQLITE_PRIVATE void sqlite3ExprListDelete(sqlite3 *db, ExprList *pList){ if( pList ) exprListDeleteNN(db, pList); @@ -106807,6 +108265,7 @@ SQLITE_PRIVATE void sqlite3CodeRhsOfIN( sqlite3VdbeChangeP4(v, addr, (void *)pKeyInfo, P4_KEYINFO); } if( addrOnce ){ + sqlite3VdbeAddOp1(v, OP_NullRow, iTab); sqlite3VdbeJumpHere(v, addrOnce); /* Subroutine return */ assert( ExprUseYSub(pExpr) ); @@ -106920,7 +108379,7 @@ SQLITE_PRIVATE int sqlite3CodeSubselect(Parse *pParse, Expr *pExpr){ pLimit = sqlite3PExpr(pParse, TK_NE, sqlite3ExprDup(db, pSel->pLimit->pLeft, 0), pLimit); } - sqlite3ExprDelete(db, pSel->pLimit->pLeft); + sqlite3ExprDeferredDelete(pParse, pSel->pLimit->pLeft); pSel->pLimit->pLeft = pLimit; }else{ /* If there is no pre-existing limit add a limit of 1 */ @@ -107373,10 +108832,7 @@ SQLITE_PRIVATE void sqlite3ExprCodeGetColumnOfTable( ){ Column *pCol; assert( v!=0 ); - if( pTab==0 ){ - sqlite3VdbeAddOp3(v, OP_Column, iTabCur, iCol, regOut); - return; - } + assert( pTab!=0 ); if( iCol<0 || iCol==pTab->iPKey ){ sqlite3VdbeAddOp2(v, OP_Rowid, iTabCur, regOut); VdbeComment((v, "%s.rowid", pTab->zName)); @@ -107434,7 +108890,7 @@ SQLITE_PRIVATE int sqlite3ExprCodeGetColumn( assert( pParse->pVdbe!=0 ); sqlite3ExprCodeGetColumnOfTable(pParse->pVdbe, pTab, iTable, iColumn, iReg); if( p5 ){ - VdbeOp *pOp = sqlite3VdbeGetOp(pParse->pVdbe,-1); + VdbeOp *pOp = sqlite3VdbeGetLastOp(pParse->pVdbe); if( pOp->opcode==OP_Column ) pOp->p5 = p5; } return iReg; @@ -107503,7 +108959,7 @@ static int exprCodeVector(Parse *pParse, Expr *p, int *piFreeable){ ** so that a subsequent copy will not be merged into this one. */ static void setDoNotMergeFlagOnCopy(Vdbe *v){ - if( sqlite3VdbeGetOp(v, -1)->opcode==OP_Copy ){ + if( sqlite3VdbeGetLastOp(v)->opcode==OP_Copy ){ sqlite3VdbeChangeP5(v, 1); /* Tag trailing OP_Copy as not mergable */ } } @@ -107626,6 +109082,53 @@ static int exprCodeInlineFunction( return target; } +/* +** Check to see if pExpr is one of the indexed expressions on pParse->pIdxExpr. +** If it is, then resolve the expression by reading from the index and +** return the register into which the value has been read. If pExpr is +** not an indexed expression, then return negative. +*/ +static SQLITE_NOINLINE int sqlite3IndexedExprLookup( + Parse *pParse, /* The parsing context */ + Expr *pExpr, /* The expression to potentially bypass */ + int target /* Where to store the result of the expression */ +){ + IndexedExpr *p; + Vdbe *v; + for(p=pParse->pIdxExpr; p; p=p->pIENext){ + int iDataCur = p->iDataCur; + if( iDataCur<0 ) continue; + if( pParse->iSelfTab ){ + if( p->iDataCur!=pParse->iSelfTab-1 ) continue; + iDataCur = -1; + } + if( sqlite3ExprCompare(0, pExpr, p->pExpr, iDataCur)!=0 ) continue; + v = pParse->pVdbe; + assert( v!=0 ); + if( p->bMaybeNullRow ){ + /* If the index is on a NULL row due to an outer join, then we + ** cannot extract the value from the index. The value must be + ** computed using the original expression. */ + int addr = sqlite3VdbeCurrentAddr(v); + sqlite3VdbeAddOp3(v, OP_IfNullRow, p->iIdxCur, addr+3, target); + VdbeCoverage(v); + sqlite3VdbeAddOp3(v, OP_Column, p->iIdxCur, p->iIdxCol, target); + VdbeComment((v, "%s expr-column %d", p->zIdxName, p->iIdxCol)); + sqlite3VdbeGoto(v, 0); + p = pParse->pIdxExpr; + pParse->pIdxExpr = 0; + sqlite3ExprCode(pParse, pExpr, target); + pParse->pIdxExpr = p; + sqlite3VdbeJumpHere(v, addr+2); + }else{ + sqlite3VdbeAddOp3(v, OP_Column, p->iIdxCur, p->iIdxCol, target); + VdbeComment((v, "%s expr-column %d", p->zIdxName, p->iIdxCol)); + } + return target; + } + return -1; /* Not found */ +} + /* ** Generate code into the current Vdbe to evaluate the given @@ -107654,6 +109157,11 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) expr_code_doover: if( pExpr==0 ){ op = TK_NULL; + }else if( pParse->pIdxExpr!=0 + && !ExprHasProperty(pExpr, EP_Leaf) + && (r1 = sqlite3IndexedExprLookup(pParse, pExpr, target))>=0 + ){ + return r1; }else{ assert( !ExprHasVVAProperty(pExpr,EP_Immutable) ); op = pExpr->op; @@ -107674,7 +109182,7 @@ expr_code_doover: pCol->iSorterColumn, target); if( pCol->iColumn<0 ){ VdbeComment((v,"%s.rowid",pTab->zName)); - }else{ + }else if( ALWAYS(pTab!=0) ){ VdbeComment((v,"%s.%s", pTab->zName, pTab->aCol[pCol->iColumn].zCnName)); if( pTab->aCol[pCol->iColumn].affinity==SQLITE_AFF_REAL ){ @@ -107699,11 +109207,8 @@ expr_code_doover: int aff; iReg = sqlite3ExprCodeTarget(pParse, pExpr->pLeft,target); assert( ExprUseYTab(pExpr) ); - if( pExpr->y.pTab ){ - aff = sqlite3TableColumnAffinity(pExpr->y.pTab, pExpr->iColumn); - }else{ - aff = pExpr->affExpr; - } + assert( pExpr->y.pTab!=0 ); + aff = sqlite3TableColumnAffinity(pExpr->y.pTab, pExpr->iColumn); if( aff>SQLITE_AFF_BLOB ){ static const char zAff[] = "B\000C\000D\000E"; assert( SQLITE_AFF_BLOB=='A' ); @@ -107765,12 +109270,10 @@ expr_code_doover: } } assert( ExprUseYTab(pExpr) ); + assert( pExpr->y.pTab!=0 ); iReg = sqlite3ExprCodeGetColumn(pParse, pExpr->y.pTab, pExpr->iColumn, iTab, target, pExpr->op2); - if( pExpr->y.pTab==0 && pExpr->affExpr==SQLITE_AFF_REAL ){ - sqlite3VdbeAddOp1(v, OP_RealAffinity, iReg); - } return iReg; } case TK_INTEGER: { @@ -108269,6 +109772,21 @@ expr_code_doover: case TK_IF_NULL_ROW: { int addrINR; u8 okConstFactor = pParse->okConstFactor; + AggInfo *pAggInfo = pExpr->pAggInfo; + if( pAggInfo ){ + assert( pExpr->iAgg>=0 && pExpr->iAggnColumn ); + if( !pAggInfo->directMode ){ + inReg = pAggInfo->aCol[pExpr->iAgg].iMem; + break; + } + if( pExpr->pAggInfo->useSortingIdx ){ + sqlite3VdbeAddOp3(v, OP_Column, pAggInfo->sortingIdxPTab, + pAggInfo->aCol[pExpr->iAgg].iSorterColumn, + target); + inReg = target; + break; + } + } addrINR = sqlite3VdbeAddOp1(v, OP_IfNullRow, pExpr->iTable); /* Temporarily disable factoring of constant expressions, since ** even though expressions may appear to be constant, they are not @@ -108610,7 +110128,7 @@ SQLITE_PRIVATE int sqlite3ExprCodeExprList( if( inReg!=target+i ){ VdbeOp *pOp; if( copyOp==OP_Copy - && (pOp=sqlite3VdbeGetOp(v, -1))->opcode==OP_Copy + && (pOp=sqlite3VdbeGetLastOp(v))->opcode==OP_Copy && pOp->p1+pOp->p3+1==inReg && pOp->p2+pOp->p3+1==target+i && pOp->p5==0 /* The do-not-merge flag must be clear */ @@ -108809,6 +110327,7 @@ SQLITE_PRIVATE void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int assert( TK_ISNULL==OP_IsNull ); testcase( op==TK_ISNULL ); assert( TK_NOTNULL==OP_NotNull ); testcase( op==TK_NOTNULL ); r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); + sqlite3VdbeTypeofColumn(v, r1); sqlite3VdbeAddOp2(v, op, r1, dest); VdbeCoverageIf(v, op==TK_ISNULL); VdbeCoverageIf(v, op==TK_NOTNULL); @@ -108983,6 +110502,7 @@ SQLITE_PRIVATE void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int case TK_ISNULL: case TK_NOTNULL: { r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); + sqlite3VdbeTypeofColumn(v, r1); sqlite3VdbeAddOp2(v, op, r1, dest); testcase( op==TK_ISNULL ); VdbeCoverageIf(v, op==TK_ISNULL); testcase( op==TK_NOTNULL ); VdbeCoverageIf(v, op==TK_NOTNULL); @@ -109136,7 +110656,13 @@ SQLITE_PRIVATE int sqlite3ExprCompare( if( pB->op==TK_COLLATE && sqlite3ExprCompare(pParse, pA,pB->pLeft,iTab)<2 ){ return 1; } - return 2; + if( pA->op==TK_AGG_COLUMN && pB->op==TK_COLUMN + && pB->iTable<0 && pA->iTable==iTab + ){ + /* fall through */ + }else{ + return 2; + } } assert( !ExprHasProperty(pA, EP_IntValue) ); assert( !ExprHasProperty(pB, EP_IntValue) ); @@ -109438,10 +110964,10 @@ static int impliesNotNullRow(Walker *pWalker, Expr *pExpr){ assert( pLeft->op!=TK_COLUMN || ExprUseYTab(pLeft) ); assert( pRight->op!=TK_COLUMN || ExprUseYTab(pRight) ); if( (pLeft->op==TK_COLUMN - && pLeft->y.pTab!=0 + && ALWAYS(pLeft->y.pTab!=0) && IsVirtual(pLeft->y.pTab)) || (pRight->op==TK_COLUMN - && pRight->y.pTab!=0 + && ALWAYS(pRight->y.pTab!=0) && IsVirtual(pRight->y.pTab)) ){ return WRC_Prune; @@ -109646,6 +111172,7 @@ static int exprRefToSrcList(Walker *pWalker, Expr *pExpr){ SQLITE_PRIVATE int sqlite3ReferencesSrcList(Parse *pParse, Expr *pExpr, SrcList *pSrcList){ Walker w; struct RefSrcList x; + assert( pParse->db!=0 ); memset(&w, 0, sizeof(w)); memset(&x, 0, sizeof(x)); w.xExprCallback = exprRefToSrcList; @@ -109662,7 +111189,7 @@ SQLITE_PRIVATE int sqlite3ReferencesSrcList(Parse *pParse, Expr *pExpr, SrcList sqlite3WalkExpr(&w, pExpr->y.pWin->pFilter); } #endif - sqlite3DbFree(pParse->db, x.aiExclude); + if( x.aiExclude ) sqlite3DbNNFreeNN(pParse->db, x.aiExclude); if( w.eCode & 0x01 ){ return 1; }else if( w.eCode ){ @@ -109693,8 +111220,8 @@ static int agginfoPersistExprCb(Walker *pWalker, Expr *pExpr){ int iAgg = pExpr->iAgg; Parse *pParse = pWalker->pParse; sqlite3 *db = pParse->db; - assert( pExpr->op==TK_AGG_COLUMN || pExpr->op==TK_AGG_FUNCTION ); - if( pExpr->op==TK_AGG_COLUMN ){ + if( pExpr->op!=TK_AGG_FUNCTION ){ + assert( pExpr->op==TK_AGG_COLUMN || pExpr->op==TK_IF_NULL_ROW ); assert( iAgg>=0 && iAggnColumn ); if( pAggInfo->aCol[iAgg].pCExpr==pExpr ){ pExpr = sqlite3ExprDup(db, pExpr, 0); @@ -109704,6 +111231,7 @@ static int agginfoPersistExprCb(Walker *pWalker, Expr *pExpr){ } } }else{ + assert( pExpr->op==TK_AGG_FUNCTION ); assert( iAgg>=0 && iAggnFunc ); if( pAggInfo->aFunc[iAgg].pFExpr==pExpr ){ pExpr = sqlite3ExprDup(db, pExpr, 0); @@ -109774,10 +111302,12 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ assert( pNC->ncFlags & NC_UAggInfo ); switch( pExpr->op ){ + case TK_IF_NULL_ROW: case TK_AGG_COLUMN: case TK_COLUMN: { testcase( pExpr->op==TK_AGG_COLUMN ); testcase( pExpr->op==TK_COLUMN ); + testcase( pExpr->op==TK_IF_NULL_ROW ); /* Check to see if the column is in one of the tables in the FROM ** clause of the aggregate query */ if( ALWAYS(pSrcList!=0) ){ @@ -109795,8 +111325,10 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ int k; pCol = pAggInfo->aCol; for(k=0; knColumn; k++, pCol++){ - if( pCol->iTable==pExpr->iTable && - pCol->iColumn==pExpr->iColumn ){ + if( pCol->iTable==pExpr->iTable + && pCol->iColumn==pExpr->iColumn + && pExpr->op!=TK_IF_NULL_ROW + ){ break; } } @@ -109811,15 +111343,17 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ pCol->iMem = ++pParse->nMem; pCol->iSorterColumn = -1; pCol->pCExpr = pExpr; - if( pAggInfo->pGroupBy ){ + if( pAggInfo->pGroupBy && pExpr->op!=TK_IF_NULL_ROW ){ int j, n; ExprList *pGB = pAggInfo->pGroupBy; struct ExprList_item *pTerm = pGB->a; n = pGB->nExpr; for(j=0; jpExpr; - if( pE->op==TK_COLUMN && pE->iTable==pExpr->iTable && - pE->iColumn==pExpr->iColumn ){ + if( pE->op==TK_COLUMN + && pE->iTable==pExpr->iTable + && pE->iColumn==pExpr->iColumn + ){ pCol->iSorterColumn = j; break; } @@ -109836,7 +111370,9 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ */ ExprSetVVAProperty(pExpr, EP_NoReduce); pExpr->pAggInfo = pAggInfo; - pExpr->op = TK_AGG_COLUMN; + if( pExpr->op==TK_COLUMN ){ + pExpr->op = TK_AGG_COLUMN; + } pExpr->iAgg = (i16)k; break; } /* endif pExpr->iTable==pItem->iCursor */ @@ -113256,6 +114792,7 @@ static void analyzeVdbeCommentIndexWithColumnName( if( NEVER(i==XN_ROWID) ){ VdbeComment((v,"%s.rowid",pIdx->zName)); }else if( i==XN_EXPR ){ + assert( pIdx->bHasExpr ); VdbeComment((v,"%s.expr(%d)",pIdx->zName, k)); }else{ VdbeComment((v,"%s.%s", pIdx->zName, pIdx->pTable->aCol[i].zCnName)); @@ -115258,6 +116795,7 @@ SQLITE_PRIVATE int sqlite3DbMaskAllZero(yDbMask m){ SQLITE_PRIVATE void sqlite3FinishCoding(Parse *pParse){ sqlite3 *db; Vdbe *v; + int iDb, i; assert( pParse->pToplevel==0 ); db = pParse->db; @@ -115287,7 +116825,6 @@ SQLITE_PRIVATE void sqlite3FinishCoding(Parse *pParse){ if( pParse->bReturning ){ Returning *pReturning = pParse->u1.pReturning; int addrRewind; - int i; int reg; if( pReturning->nRetCol ){ @@ -115324,76 +116861,69 @@ SQLITE_PRIVATE void sqlite3FinishCoding(Parse *pParse){ ** transaction on each used database and to verify the schema cookie ** on each used database. */ - if( db->mallocFailed==0 - && (DbMaskNonZero(pParse->cookieMask) || pParse->pConstExpr) - ){ - int iDb, i; - assert( sqlite3VdbeGetOp(v, 0)->opcode==OP_Init ); - sqlite3VdbeJumpHere(v, 0); - assert( db->nDb>0 ); - iDb = 0; - do{ - Schema *pSchema; - if( DbMaskTest(pParse->cookieMask, iDb)==0 ) continue; - sqlite3VdbeUsesBtree(v, iDb); - pSchema = db->aDb[iDb].pSchema; - sqlite3VdbeAddOp4Int(v, - OP_Transaction, /* Opcode */ - iDb, /* P1 */ - DbMaskTest(pParse->writeMask,iDb), /* P2 */ - pSchema->schema_cookie, /* P3 */ - pSchema->iGeneration /* P4 */ - ); - if( db->init.busy==0 ) sqlite3VdbeChangeP5(v, 1); - VdbeComment((v, - "usesStmtJournal=%d", pParse->mayAbort && pParse->isMultiWrite)); - }while( ++iDbnDb ); + assert( pParse->nErr>0 || sqlite3VdbeGetOp(v, 0)->opcode==OP_Init ); + sqlite3VdbeJumpHere(v, 0); + assert( db->nDb>0 ); + iDb = 0; + do{ + Schema *pSchema; + if( DbMaskTest(pParse->cookieMask, iDb)==0 ) continue; + sqlite3VdbeUsesBtree(v, iDb); + pSchema = db->aDb[iDb].pSchema; + sqlite3VdbeAddOp4Int(v, + OP_Transaction, /* Opcode */ + iDb, /* P1 */ + DbMaskTest(pParse->writeMask,iDb), /* P2 */ + pSchema->schema_cookie, /* P3 */ + pSchema->iGeneration /* P4 */ + ); + if( db->init.busy==0 ) sqlite3VdbeChangeP5(v, 1); + VdbeComment((v, + "usesStmtJournal=%d", pParse->mayAbort && pParse->isMultiWrite)); + }while( ++iDbnDb ); #ifndef SQLITE_OMIT_VIRTUALTABLE - for(i=0; inVtabLock; i++){ - char *vtab = (char *)sqlite3GetVTable(db, pParse->apVtabLock[i]); - sqlite3VdbeAddOp4(v, OP_VBegin, 0, 0, 0, vtab, P4_VTAB); - } - pParse->nVtabLock = 0; + for(i=0; inVtabLock; i++){ + char *vtab = (char *)sqlite3GetVTable(db, pParse->apVtabLock[i]); + sqlite3VdbeAddOp4(v, OP_VBegin, 0, 0, 0, vtab, P4_VTAB); + } + pParse->nVtabLock = 0; #endif - /* Once all the cookies have been verified and transactions opened, - ** obtain the required table-locks. This is a no-op unless the - ** shared-cache feature is enabled. - */ - codeTableLocks(pParse); + /* Once all the cookies have been verified and transactions opened, + ** obtain the required table-locks. This is a no-op unless the + ** shared-cache feature is enabled. + */ + codeTableLocks(pParse); - /* Initialize any AUTOINCREMENT data structures required. - */ - sqlite3AutoincrementBegin(pParse); + /* Initialize any AUTOINCREMENT data structures required. + */ + sqlite3AutoincrementBegin(pParse); - /* Code constant expressions that where factored out of inner loops. - ** - ** The pConstExpr list might also contain expressions that we simply - ** want to keep around until the Parse object is deleted. Such - ** expressions have iConstExprReg==0. Do not generate code for - ** those expressions, of course. - */ - if( pParse->pConstExpr ){ - ExprList *pEL = pParse->pConstExpr; - pParse->okConstFactor = 0; - for(i=0; inExpr; i++){ - int iReg = pEL->a[i].u.iConstExprReg; - if( iReg>0 ){ - sqlite3ExprCode(pParse, pEL->a[i].pExpr, iReg); - } - } + /* Code constant expressions that where factored out of inner loops. + ** + ** The pConstExpr list might also contain expressions that we simply + ** want to keep around until the Parse object is deleted. Such + ** expressions have iConstExprReg==0. Do not generate code for + ** those expressions, of course. + */ + if( pParse->pConstExpr ){ + ExprList *pEL = pParse->pConstExpr; + pParse->okConstFactor = 0; + for(i=0; inExpr; i++){ + int iReg = pEL->a[i].u.iConstExprReg; + sqlite3ExprCode(pParse, pEL->a[i].pExpr, iReg); } - - if( pParse->bReturning ){ - Returning *pRet = pParse->u1.pReturning; - if( pRet->nRetCol ){ - sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pRet->iRetCur, pRet->nRetCol); - } - } - - /* Finally, jump back to the beginning of the executable code. */ - sqlite3VdbeGoto(v, 1); } + + if( pParse->bReturning ){ + Returning *pRet = pParse->u1.pReturning; + if( pRet->nRetCol ){ + sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pRet->iRetCur, pRet->nRetCol); + } + } + + /* Finally, jump back to the beginning of the executable code. */ + sqlite3VdbeGoto(v, 1); } /* Get the VDBE program ready for execution @@ -115578,7 +117108,7 @@ SQLITE_PRIVATE Table *sqlite3LocateTable( /* If zName is the not the name of a table in the schema created using ** CREATE, then check to see if it is the name of an virtual table that ** can be an eponymous virtual table. */ - if( pParse->disableVtab==0 && db->init.busy==0 ){ + if( (pParse->prepFlags & SQLITE_PREPARE_NO_VTAB)==0 && db->init.busy==0 ){ Module *pMod = (Module*)sqlite3HashFind(&db->aModule, zName); if( pMod==0 && sqlite3_strnicmp(zName, "pragma_", 7)==0 ){ pMod = sqlite3PragmaVtabRegister(db, zName); @@ -115591,7 +117121,7 @@ SQLITE_PRIVATE Table *sqlite3LocateTable( #endif if( flags & LOCATE_NOERR ) return 0; pParse->checkSchema = 1; - }else if( IsVirtual(p) && pParse->disableVtab ){ + }else if( IsVirtual(p) && (pParse->prepFlags & SQLITE_PREPARE_NO_VTAB)!=0 ){ p = 0; } @@ -115900,16 +117430,17 @@ SQLITE_PRIVATE void sqlite3DeleteColumnNames(sqlite3 *db, Table *pTable){ int i; Column *pCol; assert( pTable!=0 ); + assert( db!=0 ); if( (pCol = pTable->aCol)!=0 ){ for(i=0; inCol; i++, pCol++){ assert( pCol->zCnName==0 || pCol->hName==sqlite3StrIHash(pCol->zCnName) ); sqlite3DbFree(db, pCol->zCnName); } - sqlite3DbFree(db, pTable->aCol); + sqlite3DbNNFreeNN(db, pTable->aCol); if( IsOrdinaryTable(pTable) ){ sqlite3ExprListDelete(db, pTable->u.tab.pDfltList); } - if( db==0 || db->pnBytesFreed==0 ){ + if( db->pnBytesFreed==0 ){ pTable->aCol = 0; pTable->nCol = 0; if( IsOrdinaryTable(pTable) ){ @@ -115946,7 +117477,8 @@ static void SQLITE_NOINLINE deleteTable(sqlite3 *db, Table *pTable){ ** a Table object that was going to be marked ephemeral. So do not check ** that no lookaside memory is used in this case either. */ int nLookaside = 0; - if( db && !db->mallocFailed && (pTable->tabFlags & TF_Ephemeral)==0 ){ + assert( db!=0 ); + if( !db->mallocFailed && (pTable->tabFlags & TF_Ephemeral)==0 ){ nLookaside = sqlite3LookasideUsed(db, 0); } #endif @@ -115956,7 +117488,7 @@ static void SQLITE_NOINLINE deleteTable(sqlite3 *db, Table *pTable){ pNext = pIndex->pNext; assert( pIndex->pSchema==pTable->pSchema || (IsVirtual(pTable) && pIndex->idxType!=SQLITE_IDXTYPE_APPDEF) ); - if( (db==0 || db->pnBytesFreed==0) && !IsVirtual(pTable) ){ + if( db->pnBytesFreed==0 && !IsVirtual(pTable) ){ char *zName = pIndex->zName; TESTONLY ( Index *pOld = ) sqlite3HashInsert( &pIndex->pSchema->idxHash, zName, 0 @@ -115993,8 +117525,9 @@ static void SQLITE_NOINLINE deleteTable(sqlite3 *db, Table *pTable){ } SQLITE_PRIVATE void sqlite3DeleteTable(sqlite3 *db, Table *pTable){ /* Do not delete the table until the reference count reaches zero. */ + assert( db!=0 ); if( !pTable ) return; - if( ((!db || db->pnBytesFreed==0) && (--pTable->nTabRef)>0) ) return; + if( db->pnBytesFreed==0 && (--pTable->nTabRef)>0 ) return; deleteTable(db, pTable); } @@ -117398,7 +118931,8 @@ static int isDupColumn(Index *pIdx, int nKey, Index *pPk, int iCol){ /* Recompute the colNotIdxed field of the Index. ** ** colNotIdxed is a bitmask that has a 0 bit representing each indexed -** columns that are within the first 63 columns of the table. The +** columns that are within the first 63 columns of the table and a 1 for +** all other bits (all columns that are not in the index). The ** high-order bit of colNotIdxed is always 1. All unindexed columns ** of the table have a 1. ** @@ -117426,7 +118960,7 @@ static void recomputeColumnsNotIndexed(Index *pIdx){ } } pIdx->colNotIdxed = ~m; - assert( (pIdx->colNotIdxed>>63)==1 ); + assert( (pIdx->colNotIdxed>>63)==1 ); /* See note-20221022-a */ } /* @@ -118167,7 +119701,7 @@ create_view_fail: ** the columns of the view in the pTable structure. Return the number ** of errors. If an error is seen leave an error message in pParse->zErrMsg. */ -SQLITE_PRIVATE int sqlite3ViewGetColumnNames(Parse *pParse, Table *pTable){ +static SQLITE_NOINLINE int viewGetColumnNames(Parse *pParse, Table *pTable){ Table *pSelTab; /* A fake table from which we get the result set */ Select *pSel; /* Copy of the SELECT that implements the view */ int nErr = 0; /* Number of errors encountered */ @@ -118192,9 +119726,10 @@ SQLITE_PRIVATE int sqlite3ViewGetColumnNames(Parse *pParse, Table *pTable){ #ifndef SQLITE_OMIT_VIEW /* A positive nCol means the columns names for this view are - ** already known. + ** already known. This routine is not called unless either the + ** table is virtual or nCol is zero. */ - if( pTable->nCol>0 ) return 0; + assert( pTable->nCol<=0 ); /* A negative nCol is a special marker meaning that we are currently ** trying to compute the column names. If we enter this routine with @@ -118290,6 +119825,11 @@ SQLITE_PRIVATE int sqlite3ViewGetColumnNames(Parse *pParse, Table *pTable){ #endif /* SQLITE_OMIT_VIEW */ return nErr; } +SQLITE_PRIVATE int sqlite3ViewGetColumnNames(Parse *pParse, Table *pTable){ + assert( pTable!=0 ); + if( !IsVirtual(pTable) && pTable->nCol>0 ) return 0; + return viewGetColumnNames(pParse, pTable); +} #endif /* !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_VIRTUALTABLE) */ #ifndef SQLITE_OMIT_VIEW @@ -119155,7 +120695,7 @@ SQLITE_PRIVATE void sqlite3CreateIndex( } if( !IN_RENAME_OBJECT ){ if( !db->init.busy ){ - if( sqlite3FindTable(db, zName, 0)!=0 ){ + if( sqlite3FindTable(db, zName, pDb->zDbSName)!=0 ){ sqlite3ErrorMsg(pParse, "there is already a table named %s", zName); goto exit_create_index; } @@ -119308,6 +120848,7 @@ SQLITE_PRIVATE void sqlite3CreateIndex( j = XN_EXPR; pIndex->aiColumn[i] = XN_EXPR; pIndex->uniqNotNull = 0; + pIndex->bHasExpr = 1; }else{ j = pCExpr->iColumn; assert( j<=0x7fff ); @@ -119319,6 +120860,7 @@ SQLITE_PRIVATE void sqlite3CreateIndex( } if( pTab->aCol[j].colFlags & COLFLAG_VIRTUAL ){ pIndex->bHasVCol = 1; + pIndex->bHasExpr = 1; } } pIndex->aiColumn[i] = (i16)j; @@ -119808,12 +121350,13 @@ SQLITE_PRIVATE IdList *sqlite3IdListAppend(Parse *pParse, IdList *pList, Token * */ SQLITE_PRIVATE void sqlite3IdListDelete(sqlite3 *db, IdList *pList){ int i; + assert( db!=0 ); if( pList==0 ) return; assert( pList->eU4!=EU4_EXPR ); /* EU4_EXPR mode is not currently used */ for(i=0; inId; i++){ sqlite3DbFree(db, pList->a[i].zName); } - sqlite3DbFreeNN(db, pList); + sqlite3DbNNFreeNN(db, pList); } /* @@ -120016,11 +121559,12 @@ SQLITE_PRIVATE void sqlite3SrcListAssignCursors(Parse *pParse, SrcList *pList){ SQLITE_PRIVATE void sqlite3SrcListDelete(sqlite3 *db, SrcList *pList){ int i; SrcItem *pItem; + assert( db!=0 ); if( pList==0 ) return; for(pItem=pList->a, i=0; inSrc; i++, pItem++){ - if( pItem->zDatabase ) sqlite3DbFreeNN(db, pItem->zDatabase); - sqlite3DbFree(db, pItem->zName); - if( pItem->zAlias ) sqlite3DbFreeNN(db, pItem->zAlias); + if( pItem->zDatabase ) sqlite3DbNNFreeNN(db, pItem->zDatabase); + if( pItem->zName ) sqlite3DbNNFreeNN(db, pItem->zName); + if( pItem->zAlias ) sqlite3DbNNFreeNN(db, pItem->zAlias); if( pItem->fg.isIndexedBy ) sqlite3DbFree(db, pItem->u1.zIndexedBy); if( pItem->fg.isTabFunc ) sqlite3ExprListDelete(db, pItem->u1.pFuncArg); sqlite3DeleteTable(db, pItem->pTab); @@ -120031,7 +121575,7 @@ SQLITE_PRIVATE void sqlite3SrcListDelete(sqlite3 *db, SrcList *pList){ sqlite3ExprDelete(db, pItem->u3.pOn); } } - sqlite3DbFreeNN(db, pList); + sqlite3DbNNFreeNN(db, pList); } /* @@ -121283,19 +122827,21 @@ SQLITE_PRIVATE void sqlite3SchemaClear(void *p){ Hash temp2; HashElem *pElem; Schema *pSchema = (Schema *)p; + sqlite3 xdb; + memset(&xdb, 0, sizeof(xdb)); temp1 = pSchema->tblHash; temp2 = pSchema->trigHash; sqlite3HashInit(&pSchema->trigHash); sqlite3HashClear(&pSchema->idxHash); for(pElem=sqliteHashFirst(&temp2); pElem; pElem=sqliteHashNext(pElem)){ - sqlite3DeleteTrigger(0, (Trigger*)sqliteHashData(pElem)); + sqlite3DeleteTrigger(&xdb, (Trigger*)sqliteHashData(pElem)); } sqlite3HashClear(&temp2); sqlite3HashInit(&pSchema->tblHash); for(pElem=sqliteHashFirst(&temp1); pElem; pElem=sqliteHashNext(pElem)){ Table *pTab = sqliteHashData(pElem); - sqlite3DeleteTable(0, pTab); + sqlite3DeleteTable(&xdb, pTab); } sqlite3HashClear(&temp1); sqlite3HashClear(&pSchema->fkeyHash); @@ -121394,18 +122940,42 @@ SQLITE_PRIVATE void sqlite3CodeChangeCount(Vdbe *v, int regCounter, const char * ** 1) It is a virtual table and no implementation of the xUpdate method ** has been provided ** -** 2) It is a system table (i.e. sqlite_schema), this call is not +** 2) A trigger is currently being coded and the table is a virtual table +** that is SQLITE_VTAB_DIRECTONLY or if PRAGMA trusted_schema=OFF and +** the table is not SQLITE_VTAB_INNOCUOUS. +** +** 3) It is a system table (i.e. sqlite_schema), this call is not ** part of a nested parse and writable_schema pragma has not ** been specified ** -** 3) The table is a shadow table, the database connection is in +** 4) The table is a shadow table, the database connection is in ** defensive mode, and the current sqlite3_prepare() ** is for a top-level SQL statement. */ +static int vtabIsReadOnly(Parse *pParse, Table *pTab){ + if( sqlite3GetVTable(pParse->db, pTab)->pMod->pModule->xUpdate==0 ){ + return 1; + } + + /* Within triggers: + ** * Do not allow DELETE, INSERT, or UPDATE of SQLITE_VTAB_DIRECTONLY + ** virtual tables + ** * Only allow DELETE, INSERT, or UPDATE of non-SQLITE_VTAB_INNOCUOUS + ** virtual tables if PRAGMA trusted_schema=ON. + */ + if( pParse->pToplevel!=0 + && pTab->u.vtab.p->eVtabRisk > + ((pParse->db->flags & SQLITE_TrustedSchema)!=0) + ){ + sqlite3ErrorMsg(pParse, "unsafe use of virtual table \"%s\"", + pTab->zName); + } + return 0; +} static int tabIsReadOnly(Parse *pParse, Table *pTab){ sqlite3 *db; if( IsVirtual(pTab) ){ - return sqlite3GetVTable(pParse->db, pTab)->pMod->pModule->xUpdate==0; + return vtabIsReadOnly(pParse, pTab); } if( (pTab->tabFlags & (TF_Readonly|TF_Shadow))==0 ) return 0; db = pParse->db; @@ -121417,9 +122987,11 @@ static int tabIsReadOnly(Parse *pParse, Table *pTab){ } /* -** Check to make sure the given table is writable. If it is not -** writable, generate an error message and return 1. If it is -** writable return 0; +** Check to make sure the given table is writable. +** +** If pTab is not writable -> generate an error message and return 1. +** If pTab is writable but other errors have occurred -> return 1. +** If pTab is writable and no prior errors -> return 0; */ SQLITE_PRIVATE int sqlite3IsReadOnly(Parse *pParse, Table *pTab, int viewOk){ if( tabIsReadOnly(pParse, pTab) ){ @@ -121780,9 +123352,10 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( } for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ assert( pIdx->pSchema==pTab->pSchema ); - sqlite3VdbeAddOp2(v, OP_Clear, pIdx->tnum, iDb); if( IsPrimaryKeyIndex(pIdx) && !HasRowid(pTab) ){ - sqlite3VdbeChangeP3(v, -1, memCnt ? memCnt : -1); + sqlite3VdbeAddOp3(v, OP_Clear, pIdx->tnum, iDb, memCnt ? memCnt : -1); + }else{ + sqlite3VdbeAddOp2(v, OP_Clear, pIdx->tnum, iDb); } } }else @@ -121982,7 +123555,7 @@ delete_from_cleanup: sqlite3ExprListDelete(db, pOrderBy); sqlite3ExprDelete(db, pLimit); #endif - sqlite3DbFree(db, aToOpen); + if( aToOpen ) sqlite3DbNNFreeNN(db, aToOpen); return; } /* Make sure "isView" and other macros defined above are undefined. Otherwise @@ -123065,7 +124638,7 @@ static int patternCompare( ** c but in the other case and search the input string for either ** c or cx. */ - if( c<=0x80 ){ + if( c<0x80 ){ char zStop[3]; int bMatch; if( noCase ){ @@ -123148,7 +124721,13 @@ static int patternCompare( ** non-zero if there is no match. */ SQLITE_API int sqlite3_strglob(const char *zGlobPattern, const char *zString){ - return patternCompare((u8*)zGlobPattern, (u8*)zString, &globInfo, '['); + if( zString==0 ){ + return zGlobPattern!=0; + }else if( zGlobPattern==0 ){ + return 1; + }else { + return patternCompare((u8*)zGlobPattern, (u8*)zString, &globInfo, '['); + } } /* @@ -123156,7 +124735,13 @@ SQLITE_API int sqlite3_strglob(const char *zGlobPattern, const char *zString){ ** a miss - like strcmp(). */ SQLITE_API int sqlite3_strlike(const char *zPattern, const char *zStr, unsigned int esc){ - return patternCompare((u8*)zPattern, (u8*)zStr, &likeInfoNorm, esc); + if( zStr==0 ){ + return zPattern!=0; + }else if( zPattern==0 ){ + return 1; + }else{ + return patternCompare((u8*)zPattern, (u8*)zStr, &likeInfoNorm, esc); + } } /* @@ -126150,11 +127735,12 @@ SQLITE_PRIVATE void sqlite3FkDelete(sqlite3 *db, Table *pTab){ FKey *pNext; /* Copy of pFKey->pNextFrom */ assert( IsOrdinaryTable(pTab) ); + assert( db!=0 ); for(pFKey=pTab->u.tab.pFKey; pFKey; pFKey=pNext){ assert( db==0 || sqlite3SchemaMutexHeld(db, 0, pTab->pSchema) ); /* Remove the FK from the fkeyHash hash table. */ - if( !db || db->pnBytesFreed==0 ){ + if( db->pnBytesFreed==0 ){ if( pFKey->pPrevTo ){ pFKey->pPrevTo->pNextTo = pFKey->pNextTo; }else{ @@ -126284,6 +127870,7 @@ SQLITE_PRIVATE const char *sqlite3IndexAffinityStr(sqlite3 *db, Index *pIdx){ aff = SQLITE_AFF_INTEGER; }else{ assert( x==XN_EXPR ); + assert( pIdx->bHasExpr ); assert( pIdx->aColExpr!=0 ); aff = sqlite3ExprAffinity(pIdx->aColExpr->a[n].pExpr); } @@ -126297,6 +127884,28 @@ SQLITE_PRIVATE const char *sqlite3IndexAffinityStr(sqlite3 *db, Index *pIdx){ return pIdx->zColAff; } +/* +** Compute an affinity string for a table. Space is obtained +** from sqlite3DbMalloc(). The caller is responsible for freeing +** the space when done. +*/ +SQLITE_PRIVATE char *sqlite3TableAffinityStr(sqlite3 *db, const Table *pTab){ + char *zColAff; + zColAff = (char *)sqlite3DbMallocRaw(db, pTab->nCol+1); + if( zColAff ){ + int i, j; + for(i=j=0; inCol; i++){ + if( (pTab->aCol[i].colFlags & COLFLAG_VIRTUAL)==0 ){ + zColAff[j++] = pTab->aCol[i].affinity; + } + } + do{ + zColAff[j--] = 0; + }while( j>=0 && zColAff[j]<=SQLITE_AFF_BLOB ); + } + return zColAff; +} + /* ** Make changes to the evolving bytecode to do affinity transformations ** of values that are about to be gathered into a row for table pTab. @@ -126338,7 +127947,7 @@ SQLITE_PRIVATE const char *sqlite3IndexAffinityStr(sqlite3 *db, Index *pIdx){ ** Apply the type checking to that array of registers. */ SQLITE_PRIVATE void sqlite3TableAffinity(Vdbe *v, Table *pTab, int iReg){ - int i, j; + int i; char *zColAff; if( pTab->tabFlags & TF_Strict ){ if( iReg==0 ){ @@ -126347,7 +127956,7 @@ SQLITE_PRIVATE void sqlite3TableAffinity(Vdbe *v, Table *pTab, int iReg){ ** OP_MakeRecord is found */ VdbeOp *pPrev; sqlite3VdbeAppendP4(v, pTab, P4_TABLE); - pPrev = sqlite3VdbeGetOp(v, -1); + pPrev = sqlite3VdbeGetLastOp(v); assert( pPrev!=0 ); assert( pPrev->opcode==OP_MakeRecord || sqlite3VdbeDb(v)->mallocFailed ); pPrev->opcode = OP_TypeCheck; @@ -126361,22 +127970,11 @@ SQLITE_PRIVATE void sqlite3TableAffinity(Vdbe *v, Table *pTab, int iReg){ } zColAff = pTab->zColAff; if( zColAff==0 ){ - sqlite3 *db = sqlite3VdbeDb(v); - zColAff = (char *)sqlite3DbMallocRaw(0, pTab->nCol+1); + zColAff = sqlite3TableAffinityStr(0, pTab); if( !zColAff ){ - sqlite3OomFault(db); + sqlite3OomFault(sqlite3VdbeDb(v)); return; } - - for(i=j=0; inCol; i++){ - assert( pTab->aCol[i].affinity!=0 || sqlite3VdbeParser(v)->nErr>0 ); - if( (pTab->aCol[i].colFlags & COLFLAG_VIRTUAL)==0 ){ - zColAff[j++] = pTab->aCol[i].affinity; - } - } - do{ - zColAff[j--] = 0; - }while( j>=0 && zColAff[j]<=SQLITE_AFF_BLOB ); pTab->zColAff = zColAff; } assert( zColAff!=0 ); @@ -126385,7 +127983,7 @@ SQLITE_PRIVATE void sqlite3TableAffinity(Vdbe *v, Table *pTab, int iReg){ if( iReg ){ sqlite3VdbeAddOp4(v, OP_Affinity, iReg, i, 0, zColAff, i); }else{ - assert( sqlite3VdbeGetOp(v, -1)->opcode==OP_MakeRecord + assert( sqlite3VdbeGetLastOp(v)->opcode==OP_MakeRecord || sqlite3VdbeDb(v)->mallocFailed ); sqlite3VdbeChangeP4(v, -1, zColAff, i); } @@ -126471,7 +128069,7 @@ SQLITE_PRIVATE void sqlite3ComputeGeneratedColumns( */ sqlite3TableAffinity(pParse->pVdbe, pTab, iRegStore); if( (pTab->tabFlags & TF_HasStored)!=0 ){ - pOp = sqlite3VdbeGetOp(pParse->pVdbe,-1); + pOp = sqlite3VdbeGetLastOp(pParse->pVdbe); if( pOp->opcode==OP_Affinity ){ /* Change the OP_Affinity argument to '@' (NONE) for all stored ** columns. '@' is the no-op affinity and those columns have not @@ -127377,7 +128975,12 @@ SQLITE_PRIVATE void sqlite3Insert( sqlite3VdbeAddOp2(v, OP_SCopy, regFromSelect+k, iRegStore); } }else{ - sqlite3ExprCode(pParse, pList->a[k].pExpr, iRegStore); + Expr *pX = pList->a[k].pExpr; + int y = sqlite3ExprCodeTarget(pParse, pX, iRegStore); + if( y!=iRegStore ){ + sqlite3VdbeAddOp2(v, + ExprHasProperty(pX, EP_Subquery) ? OP_Copy : OP_SCopy, y, iRegStore); + } } } @@ -127514,7 +129117,9 @@ SQLITE_PRIVATE void sqlite3Insert( sqlite3GenerateConstraintChecks(pParse, pTab, aRegIdx, iDataCur, iIdxCur, regIns, 0, ipkColumn>=0, onError, endOfLoop, &isReplace, 0, pUpsert ); - sqlite3FkCheck(pParse, pTab, 0, regIns, 0, 0); + if( db->flags & SQLITE_ForeignKeys ){ + sqlite3FkCheck(pParse, pTab, 0, regIns, 0, 0); + } /* Set the OPFLAG_USESEEKRESULT flag if either (a) there are no REPLACE ** constraints or (b) there are no triggers and this table is not a @@ -127598,7 +129203,7 @@ insert_cleanup: sqlite3UpsertDelete(db, pUpsert); sqlite3SelectDelete(db, pSelect); sqlite3IdListDelete(db, pColumn); - sqlite3DbFree(db, aRegIdx); + if( aRegIdx ) sqlite3DbNNFreeNN(db, aRegIdx); } /* Make sure "isView" and other macros defined above are undefined. Otherwise @@ -129825,9 +131430,9 @@ struct sqlite3_api_routines { const char *(*filename_journal)(const char*); const char *(*filename_wal)(const char*); /* Version 3.32.0 and later */ - char *(*create_filename)(const char*,const char*,const char*, + const char *(*create_filename)(const char*,const char*,const char*, int,const char**); - void (*free_filename)(char*); + void (*free_filename)(const char*); sqlite3_file *(*database_file_object)(const char*); /* Version 3.34.0 and later */ int (*txn_state)(sqlite3*,const char*); @@ -129851,6 +131456,8 @@ struct sqlite3_api_routines { unsigned char *(*serialize)(sqlite3*,const char *,sqlite3_int64*, unsigned int); const char *(*db_name)(sqlite3*,int); + /* Version 3.40.0 and later */ + int (*value_encoding)(sqlite3_value*); }; /* @@ -130175,6 +131782,8 @@ typedef int (*sqlite3_loadext_entry)( #define sqlite3_serialize sqlite3_api->serialize #endif #define sqlite3_db_name sqlite3_api->db_name +/* Version 3.40.0 and later */ +#define sqlite3_value_encoding sqlite3_api->value_encoding #endif /* !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) */ #if !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) @@ -130687,7 +132296,9 @@ static const sqlite3_api_routines sqlite3Apis = { 0, 0, #endif - sqlite3_db_name + sqlite3_db_name, + /* Version 3.40.0 and later */ + sqlite3_value_type }; /* True if x is the directory separator character @@ -132704,6 +134315,7 @@ SQLITE_PRIVATE void sqlite3Pragma( ** */ case PragTyp_TEMP_STORE_DIRECTORY: { + sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_TEMPDIR)); if( !zRight ){ returnSingleText(v, sqlite3_temp_directory); }else{ @@ -132713,6 +134325,7 @@ SQLITE_PRIVATE void sqlite3Pragma( rc = sqlite3OsAccess(db->pVfs, zRight, SQLITE_ACCESS_READWRITE, &res); if( rc!=SQLITE_OK || res==0 ){ sqlite3ErrorMsg(pParse, "not a writable directory"); + sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_TEMPDIR)); goto pragma_out; } } @@ -132730,6 +134343,7 @@ SQLITE_PRIVATE void sqlite3Pragma( } #endif /* SQLITE_OMIT_WSD */ } + sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_TEMPDIR)); break; } @@ -132748,6 +134362,7 @@ SQLITE_PRIVATE void sqlite3Pragma( ** */ case PragTyp_DATA_STORE_DIRECTORY: { + sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_TEMPDIR)); if( !zRight ){ returnSingleText(v, sqlite3_data_directory); }else{ @@ -132757,6 +134372,7 @@ SQLITE_PRIVATE void sqlite3Pragma( rc = sqlite3OsAccess(db->pVfs, zRight, SQLITE_ACCESS_READWRITE, &res); if( rc!=SQLITE_OK || res==0 ){ sqlite3ErrorMsg(pParse, "not a writable directory"); + sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_TEMPDIR)); goto pragma_out; } } @@ -132768,6 +134384,7 @@ SQLITE_PRIVATE void sqlite3Pragma( } #endif /* SQLITE_OMIT_WSD */ } + sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_TEMPDIR)); break; } #endif @@ -133481,15 +135098,24 @@ SQLITE_PRIVATE void sqlite3Pragma( for(x=sqliteHashFirst(pTbls); x; x=sqliteHashNext(x)){ Table *pTab = sqliteHashData(x); Index *pIdx, *pPk; - Index *pPrior = 0; + Index *pPrior = 0; /* Previous index */ int loopTop; int iDataCur, iIdxCur; int r1 = -1; - int bStrict; + int bStrict; /* True for a STRICT table */ + int r2; /* Previous key for WITHOUT ROWID tables */ + int mxCol; /* Maximum non-virtual column number */ if( !IsOrdinaryTable(pTab) ) continue; if( pObjTab && pObjTab!=pTab ) continue; - pPk = HasRowid(pTab) ? 0 : sqlite3PrimaryKeyIndex(pTab); + if( isQuick || HasRowid(pTab) ){ + pPk = 0; + r2 = 0; + }else{ + pPk = sqlite3PrimaryKeyIndex(pTab); + r2 = sqlite3GetTempRange(pParse, pPk->nKeyCol); + sqlite3VdbeAddOp3(v, OP_Null, 1, r2, r2+pPk->nKeyCol-1); + } sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenRead, 0, 1, 0, &iDataCur, &iIdxCur); /* reg[7] counts the number of entries in the table. @@ -133503,52 +135129,157 @@ SQLITE_PRIVATE void sqlite3Pragma( assert( sqlite3NoTempsInRange(pParse,1,7+j) ); sqlite3VdbeAddOp2(v, OP_Rewind, iDataCur, 0); VdbeCoverage(v); loopTop = sqlite3VdbeAddOp2(v, OP_AddImm, 7, 1); - if( !isQuick ){ - /* Sanity check on record header decoding */ - sqlite3VdbeAddOp3(v, OP_Column, iDataCur, pTab->nNVCol-1,3); - sqlite3VdbeChangeP5(v, OPFLAG_TYPEOFARG); - VdbeComment((v, "(right-most column)")); + + /* Fetch the right-most column from the table. This will cause + ** the entire record header to be parsed and sanity checked. It + ** will also prepopulate the cursor column cache that is used + ** by the OP_IsType code, so it is a required step. + */ + mxCol = pTab->nCol-1; + while( mxCol>=0 + && ((pTab->aCol[mxCol].colFlags & COLFLAG_VIRTUAL)!=0 + || pTab->iPKey==mxCol) ) mxCol--; + if( mxCol>=0 ){ + sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, mxCol, 3); + sqlite3VdbeTypeofColumn(v, 3); } - /* Verify that all NOT NULL columns really are NOT NULL. At the - ** same time verify the type of the content of STRICT tables */ + + if( !isQuick ){ + if( pPk ){ + /* Verify WITHOUT ROWID keys are in ascending order */ + int a1; + char *zErr; + a1 = sqlite3VdbeAddOp4Int(v, OP_IdxGT, iDataCur, 0,r2,pPk->nKeyCol); + VdbeCoverage(v); + sqlite3VdbeAddOp1(v, OP_IsNull, r2); VdbeCoverage(v); + zErr = sqlite3MPrintf(db, + "row not in PRIMARY KEY order for %s", + pTab->zName); + sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, zErr, P4_DYNAMIC); + integrityCheckResultRow(v); + sqlite3VdbeJumpHere(v, a1); + sqlite3VdbeJumpHere(v, a1+1); + for(j=0; jnKeyCol; j++){ + sqlite3ExprCodeLoadIndexColumn(pParse, pPk, iDataCur, j, r2+j); + } + } + } + /* Verify datatypes for all columns: + ** + ** (1) NOT NULL columns may not contain a NULL + ** (2) Datatype must be exact for non-ANY columns in STRICT tables + ** (3) Datatype for TEXT columns in non-STRICT tables must be + ** NULL, TEXT, or BLOB. + ** (4) Datatype for numeric columns in non-STRICT tables must not + ** be a TEXT value that can be losslessly converted to numeric. + */ bStrict = (pTab->tabFlags & TF_Strict)!=0; for(j=0; jnCol; j++){ char *zErr; - Column *pCol = pTab->aCol + j; - int doError, jmp2; + Column *pCol = pTab->aCol + j; /* The column to be checked */ + int labelError; /* Jump here to report an error */ + int labelOk; /* Jump here if all looks ok */ + int p1, p3, p4; /* Operands to the OP_IsType opcode */ + int doTypeCheck; /* Check datatypes (besides NOT NULL) */ + if( j==pTab->iPKey ) continue; - if( pCol->notNull==0 && !bStrict ) continue; - doError = bStrict ? sqlite3VdbeMakeLabel(pParse) : 0; - sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, j, 3); - if( sqlite3VdbeGetOp(v,-1)->opcode==OP_Column ){ - sqlite3VdbeChangeP5(v, OPFLAG_TYPEOFARG); + if( bStrict ){ + doTypeCheck = pCol->eCType>COLTYPE_ANY; + }else{ + doTypeCheck = pCol->affinity>SQLITE_AFF_BLOB; } + if( pCol->notNull==0 && !doTypeCheck ) continue; + + /* Compute the operands that will be needed for OP_IsType */ + p4 = SQLITE_NULL; + if( pCol->colFlags & COLFLAG_VIRTUAL ){ + sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, j, 3); + p1 = -1; + p3 = 3; + }else{ + if( pCol->iDflt ){ + sqlite3_value *pDfltValue = 0; + sqlite3ValueFromExpr(db, sqlite3ColumnExpr(pTab,pCol), ENC(db), + pCol->affinity, &pDfltValue); + if( pDfltValue ){ + p4 = sqlite3_value_type(pDfltValue); + sqlite3ValueFree(pDfltValue); + } + } + p1 = iDataCur; + if( !HasRowid(pTab) ){ + testcase( j!=sqlite3TableColumnToStorage(pTab, j) ); + p3 = sqlite3TableColumnToIndex(sqlite3PrimaryKeyIndex(pTab), j); + }else{ + p3 = sqlite3TableColumnToStorage(pTab,j); + testcase( p3!=j); + } + } + + labelError = sqlite3VdbeMakeLabel(pParse); + labelOk = sqlite3VdbeMakeLabel(pParse); if( pCol->notNull ){ - jmp2 = sqlite3VdbeAddOp1(v, OP_NotNull, 3); VdbeCoverage(v); + /* (1) NOT NULL columns may not contain a NULL */ + int jmp2 = sqlite3VdbeAddOp4Int(v, OP_IsType, p1, labelOk, p3, p4); + sqlite3VdbeChangeP5(v, 0x0f); + VdbeCoverage(v); zErr = sqlite3MPrintf(db, "NULL value in %s.%s", pTab->zName, pCol->zCnName); sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, zErr, P4_DYNAMIC); - if( bStrict && pCol->eCType!=COLTYPE_ANY ){ - sqlite3VdbeGoto(v, doError); + if( doTypeCheck ){ + sqlite3VdbeGoto(v, labelError); + sqlite3VdbeJumpHere(v, jmp2); }else{ - integrityCheckResultRow(v); + /* VDBE byte code will fall thru */ } - sqlite3VdbeJumpHere(v, jmp2); } - if( (pTab->tabFlags & TF_Strict)!=0 - && pCol->eCType!=COLTYPE_ANY - ){ - jmp2 = sqlite3VdbeAddOp3(v, OP_IsNullOrType, 3, 0, - sqlite3StdTypeMap[pCol->eCType-1]); + if( bStrict && doTypeCheck ){ + /* (2) Datatype must be exact for non-ANY columns in STRICT tables*/ + static unsigned char aStdTypeMask[] = { + 0x1f, /* ANY */ + 0x18, /* BLOB */ + 0x11, /* INT */ + 0x11, /* INTEGER */ + 0x13, /* REAL */ + 0x14 /* TEXT */ + }; + sqlite3VdbeAddOp4Int(v, OP_IsType, p1, labelOk, p3, p4); + assert( pCol->eCType>=1 && pCol->eCType<=sizeof(aStdTypeMask) ); + sqlite3VdbeChangeP5(v, aStdTypeMask[pCol->eCType-1]); VdbeCoverage(v); zErr = sqlite3MPrintf(db, "non-%s value in %s.%s", sqlite3StdType[pCol->eCType-1], pTab->zName, pTab->aCol[j].zCnName); sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, zErr, P4_DYNAMIC); - sqlite3VdbeResolveLabel(v, doError); - integrityCheckResultRow(v); - sqlite3VdbeJumpHere(v, jmp2); + }else if( !bStrict && pCol->affinity==SQLITE_AFF_TEXT ){ + /* (3) Datatype for TEXT columns in non-STRICT tables must be + ** NULL, TEXT, or BLOB. */ + sqlite3VdbeAddOp4Int(v, OP_IsType, p1, labelOk, p3, p4); + sqlite3VdbeChangeP5(v, 0x1c); /* NULL, TEXT, or BLOB */ + VdbeCoverage(v); + zErr = sqlite3MPrintf(db, "NUMERIC value in %s.%s", + pTab->zName, pTab->aCol[j].zCnName); + sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, zErr, P4_DYNAMIC); + }else if( !bStrict && pCol->affinity>=SQLITE_AFF_NUMERIC ){ + /* (4) Datatype for numeric columns in non-STRICT tables must not + ** be a TEXT value that can be converted to numeric. */ + sqlite3VdbeAddOp4Int(v, OP_IsType, p1, labelOk, p3, p4); + sqlite3VdbeChangeP5(v, 0x1b); /* NULL, INT, FLOAT, or BLOB */ + VdbeCoverage(v); + if( p1>=0 ){ + sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, j, 3); + } + sqlite3VdbeAddOp4(v, OP_Affinity, 3, 1, 0, "C", P4_STATIC); + sqlite3VdbeAddOp4Int(v, OP_IsType, -1, labelOk, 3, p4); + sqlite3VdbeChangeP5(v, 0x1c); /* NULL, TEXT, or BLOB */ + VdbeCoverage(v); + zErr = sqlite3MPrintf(db, "TEXT value in %s.%s", + pTab->zName, pTab->aCol[j].zCnName); + sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, zErr, P4_DYNAMIC); } + sqlite3VdbeResolveLabel(v, labelError); + integrityCheckResultRow(v); + sqlite3VdbeResolveLabel(v, labelOk); } /* Verify CHECK constraints */ if( pTab->pCheck && (db->flags & SQLITE_IgnoreChecks)==0 ){ @@ -133636,6 +135367,9 @@ SQLITE_PRIVATE void sqlite3Pragma( integrityCheckResultRow(v); sqlite3VdbeJumpHere(v, addr); } + if( pPk ){ + sqlite3ReleaseTempRange(pParse, r2, pPk->nKeyCol); + } } } } @@ -133786,6 +135520,11 @@ SQLITE_PRIVATE void sqlite3Pragma( aOp[1].p2 = iCookie; aOp[1].p3 = sqlite3Atoi(zRight); aOp[1].p5 = 1; + if( iCookie==BTREE_SCHEMA_VERSION && (db->flags & SQLITE_Defensive)!=0 ){ + /* Do not allow the use of PRAGMA schema_version=VALUE in defensive + ** mode. Change the OP_SetCookie opcode into a no-op. */ + aOp[1].opcode = OP_Noop; + } }else{ /* Read the specified cookie value */ static const VdbeOpList readCookie[] = { @@ -135034,15 +136773,15 @@ SQLITE_PRIVATE void sqlite3ParseObjectReset(Parse *pParse){ assert( db->pParse==pParse ); assert( pParse->nested==0 ); #ifndef SQLITE_OMIT_SHARED_CACHE - sqlite3DbFree(db, pParse->aTableLock); + if( pParse->aTableLock ) sqlite3DbNNFreeNN(db, pParse->aTableLock); #endif while( pParse->pCleanup ){ ParseCleanup *pCleanup = pParse->pCleanup; pParse->pCleanup = pCleanup->pNext; pCleanup->xCleanup(db, pCleanup->pPtr); - sqlite3DbFreeNN(db, pCleanup); + sqlite3DbNNFreeNN(db, pCleanup); } - sqlite3DbFree(db, pParse->aLabel); + if( pParse->aLabel ) sqlite3DbNNFreeNN(db, pParse->aLabel); if( pParse->pConstExpr ){ sqlite3ExprListDelete(db, pParse->pConstExpr); } @@ -135165,7 +136904,7 @@ static int sqlite3Prepare( sParse.disableLookaside++; DisableLookaside; } - sParse.disableVtab = (prepFlags & SQLITE_PREPARE_NO_VTAB)!=0; + sParse.prepFlags = prepFlags & 0xff; /* Check to verify that it is possible to get a read lock on all ** database schemas. The inability to get a read lock indicates that @@ -135206,7 +136945,9 @@ static int sqlite3Prepare( } } - sqlite3VtabUnlockList(db); +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( db->pDisconnect ) sqlite3VtabUnlockList(db); +#endif if( nBytes>=0 && (nBytes==0 || zSql[nBytes-1]!=0) ){ char *zSqlCopy; @@ -135601,6 +137342,7 @@ struct SortCtx { ** If bFree==0, Leave the first Select object unfreed */ static void clearSelect(sqlite3 *db, Select *p, int bFree){ + assert( db!=0 ); while( p ){ Select *pPrior = p->pPrior; sqlite3ExprListDelete(db, p->pEList); @@ -135620,7 +137362,7 @@ static void clearSelect(sqlite3 *db, Select *p, int bFree){ sqlite3WindowUnlinkFromSelect(p->pWin); } #endif - if( bFree ) sqlite3DbFreeNN(db, p); + if( bFree ) sqlite3DbNNFreeNN(db, p); p = pPrior; bFree = 1; } @@ -136811,6 +138553,9 @@ static void selectInnerLoop( testcase( eDest==SRT_Fifo ); testcase( eDest==SRT_DistFifo ); sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nResultCol, r1+nPrefixReg); + if( pDest->zAffSdst ){ + sqlite3VdbeChangeP4(v, -1, pDest->zAffSdst, nResultCol); + } #ifndef SQLITE_OMIT_CTE if( eDest==SRT_DistFifo ){ /* If the destination is DistFifo, then cursor (iParm+1) is open @@ -137026,9 +138771,10 @@ SQLITE_PRIVATE KeyInfo *sqlite3KeyInfoAlloc(sqlite3 *db, int N, int X){ */ SQLITE_PRIVATE void sqlite3KeyInfoUnref(KeyInfo *p){ if( p ){ + assert( p->db!=0 ); assert( p->nRef>0 ); p->nRef--; - if( p->nRef==0 ) sqlite3DbFreeNN(p->db, p); + if( p->nRef==0 ) sqlite3DbNNFreeNN(p->db, p); } } @@ -137213,7 +138959,7 @@ static void generateSortTail( if( addrOnce ) sqlite3VdbeJumpHere(v, addrOnce); addr = 1 + sqlite3VdbeAddOp2(v, OP_SorterSort, iTab, addrBreak); VdbeCoverage(v); - codeOffset(v, p->iOffset, addrContinue); + assert( p->iLimit==0 && p->iOffset==0 ); sqlite3VdbeAddOp3(v, OP_SorterData, iTab, regSortOut, iSortTab); bSeq = 0; }else{ @@ -137221,6 +138967,9 @@ static void generateSortTail( codeOffset(v, p->iOffset, addrContinue); iSortTab = iTab; bSeq = 1; + if( p->iOffset>0 ){ + sqlite3VdbeAddOp2(v, OP_AddImm, p->iLimit, -1); + } } for(i=0, iCol=nKey+bSeq-1; ipPrior ){ - sqlite3SelectDelete(db, pSplit->pPrior); + sqlite3ParserAddCleanup(pParse, + (void(*)(sqlite3*,void*))sqlite3SelectDelete, pSplit->pPrior); } pSplit->pPrior = pPrior; pPrior->pNext = pSplit; @@ -139246,7 +140993,7 @@ static int multiSelectOrderBy( ** the left operands of a RIGHT JOIN. In either case, we need to potentially ** bypass the substituted expression with OP_IfNullRow. ** -** Suppose the original expression integer constant. Even though the table +** Suppose the original expression is an integer constant. Even though the table ** has the nullRow flag set, because the expression is an integer constant, ** it will not be NULLed out. So instead, we insert an OP_IfNullRow opcode ** that checks to see if the nullRow flag is set on the table. If the nullRow @@ -139272,6 +141019,7 @@ typedef struct SubstContext { int iNewTable; /* New table number */ int isOuterJoin; /* Add TK_IF_NULL_ROW opcodes on each replacement */ ExprList *pEList; /* Replacement expressions */ + ExprList *pCList; /* Collation sequences for replacement expr */ } SubstContext; /* Forward Declarations */ @@ -139313,9 +141061,10 @@ static Expr *substExpr( #endif { Expr *pNew; - Expr *pCopy = pSubst->pEList->a[pExpr->iColumn].pExpr; + int iColumn = pExpr->iColumn; + Expr *pCopy = pSubst->pEList->a[iColumn].pExpr; Expr ifNullRow; - assert( pSubst->pEList!=0 && pExpr->iColumnpEList->nExpr ); + assert( pSubst->pEList!=0 && iColumnpEList->nExpr ); assert( pExpr->pRight==0 ); if( sqlite3ExprIsVector(pCopy) ){ sqlite3VectorErrorMsg(pSubst->pParse, pCopy); @@ -139326,6 +141075,7 @@ static Expr *substExpr( ifNullRow.op = TK_IF_NULL_ROW; ifNullRow.pLeft = pCopy; ifNullRow.iTable = pSubst->iNewTable; + ifNullRow.iColumn = -99; ifNullRow.flags = EP_IfNullRow; pCopy = &ifNullRow; } @@ -139352,11 +141102,16 @@ static Expr *substExpr( /* Ensure that the expression now has an implicit collation sequence, ** just as it did when it was a column of a view or sub-query. */ - if( pExpr->op!=TK_COLUMN && pExpr->op!=TK_COLLATE ){ - CollSeq *pColl = sqlite3ExprCollSeq(pSubst->pParse, pExpr); - pExpr = sqlite3ExprAddCollateString(pSubst->pParse, pExpr, - (pColl ? pColl->zName : "BINARY") + { + CollSeq *pNat = sqlite3ExprCollSeq(pSubst->pParse, pExpr); + CollSeq *pColl = sqlite3ExprCollSeq(pSubst->pParse, + pSubst->pCList->a[iColumn].pExpr ); + if( pNat!=pColl || (pExpr->op!=TK_COLUMN && pExpr->op!=TK_COLLATE) ){ + pExpr = sqlite3ExprAddCollateString(pSubst->pParse, pExpr, + (pColl ? pColl->zName : "BINARY") + ); + } } ExprClearProperty(pExpr, EP_Collate); } @@ -139549,6 +141304,18 @@ static void renumberCursors( } #endif /* !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) */ +/* +** If pSel is not part of a compound SELECT, return a pointer to its +** expression list. Otherwise, return a pointer to the expression list +** of the leftmost SELECT in the compound. +*/ +static ExprList *findLeftmostExprlist(Select *pSel){ + while( pSel->pPrior ){ + pSel = pSel->pPrior; + } + return pSel->pEList; +} + #if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) /* ** This routine attempts to flatten subqueries as a performance optimization. @@ -139593,7 +141360,8 @@ static void renumberCursors( ** (3a) the subquery may not be a join and ** (3b) the FROM clause of the subquery may not contain a virtual ** table and -** (3c) the outer query may not be an aggregate. +** (**) Was: "The outer query may not have a GROUP BY." This case +** is now managed correctly ** (3d) the outer query may not be DISTINCT. ** See also (26) for restrictions on RIGHT JOIN. ** @@ -139650,6 +141418,8 @@ static void renumberCursors( ** (17g) either the subquery is the first element of the outer ** query or there are no RIGHT or FULL JOINs in any arm ** of the subquery. (This is a duplicate of condition (27b).) +** (17h) The corresponding result set expressions in all arms of the +** compound must have the same affinity. ** ** The parent and sub-query may contain WHERE clauses. Subject to ** rules (11), (13) and (14), they may also contain ORDER BY, @@ -139701,19 +141471,13 @@ static void renumberCursors( ** See also (3) for restrictions on LEFT JOIN. ** ** (27) The subquery may not contain a FULL or RIGHT JOIN unless it -** is the first element of the parent query. This must be the -** the case if: -** (27a) the subquery is not compound query, and +** is the first element of the parent query. Two subcases: +** (27a) the subquery is not a compound query. ** (27b) the subquery is a compound query and the RIGHT JOIN occurs ** in any arm of the compound query. (See also (17g).) ** ** (28) The subquery is not a MATERIALIZED CTE. ** -** (29) Either the subquery is not the right-hand operand of a join with an -** ON or USING clause nor the right-hand operand of a NATURAL JOIN, or -** the right-most table within the FROM clause of the subquery -** is not part of an outer join. -** ** ** In this routine, the "p" parameter is a pointer to the outer query. ** The subquery is p->pSrc->a[iFrom]. isAgg is true if the outer query @@ -139805,16 +141569,10 @@ static int flattenSubquery( ** ** which is not at all the same thing. ** - ** If the subquery is the right operand of a LEFT JOIN, then the outer - ** query cannot be an aggregate. (3c) This is an artifact of the way - ** aggregates are processed - there is no mechanism to determine if - ** the LEFT JOIN table should be all-NULL. - ** ** See also tickets #306, #350, and #3300. */ if( (pSubitem->fg.jointype & (JT_OUTER|JT_LTORJ))!=0 ){ if( pSubSrc->nSrc>1 /* (3a) */ - || isAgg /* (3c) */ || IsVirtual(pSubSrc->a[0].pTab) /* (3b) */ || (p->selFlags & SF_Distinct)!=0 /* (3d) */ || (pSubitem->fg.jointype & JT_RIGHT)!=0 /* (26) */ @@ -139823,15 +141581,6 @@ static int flattenSubquery( } isOuterJoin = 1; } -#ifdef SQLITE_EXTRA_IFNULLROW - else if( iFrom>0 && !isAgg ){ - /* Setting isOuterJoin to -1 causes OP_IfNullRow opcodes to be generated for - ** every reference to any result column from subquery in a join, even - ** though they are not necessary. This will stress-test the OP_IfNullRow - ** opcode. */ - isOuterJoin = -1; - } -#endif assert( pSubSrc->nSrc>0 ); /* True by restriction (7) */ if( iFrom>0 && (pSubSrc->a[0].fg.jointype & JT_LTORJ)!=0 ){ @@ -139841,41 +141590,13 @@ static int flattenSubquery( return 0; /* (28) */ } - /* Restriction (29): - ** - ** We do not want two constraints on the same term of the flattened - ** query where one constraint has EP_InnerON and the other is EP_OuterON. - ** To prevent this, one or the other of the following conditions must be - ** false: - ** - ** (29a) The right-most entry in the FROM clause of the subquery - ** must not be part of an outer join. - ** - ** (29b) The subquery itself must not be the right operand of a - ** NATURAL join or a join that as an ON or USING clause. - ** - ** These conditions are sufficient to keep an EP_OuterON from being - ** flattened into an EP_InnerON. Restrictions (3a) and (27a) prevent - ** an EP_InnerON from being flattened into an EP_OuterON. - */ - if( pSubSrc->nSrc>=2 - && (pSubSrc->a[pSubSrc->nSrc-1].fg.jointype & JT_OUTER)!=0 - ){ - if( (pSubitem->fg.jointype & JT_NATURAL)!=0 - || pSubitem->fg.isUsing - || NEVER(pSubitem->u3.pOn!=0) /* ON clause already shifted into WHERE */ - || pSubitem->fg.isOn - ){ - return 0; - } - } - /* Restriction (17): If the sub-query is a compound SELECT, then it must ** use only the UNION ALL operator. And none of the simple select queries ** that make up the compound SELECT are allowed to be aggregate or distinct ** queries. */ if( pSub->pPrior ){ + int ii; if( pSub->pOrderBy ){ return 0; /* Restriction (20) */ } @@ -139908,7 +141629,6 @@ static int flattenSubquery( /* Restriction (18). */ if( p->pOrderBy ){ - int ii; for(ii=0; iipOrderBy->nExpr; ii++){ if( p->pOrderBy->a[ii].u.x.iOrderByCol==0 ) return 0; } @@ -139917,6 +141637,21 @@ static int flattenSubquery( /* Restriction (23) */ if( (p->selFlags & SF_Recursive) ) return 0; + /* Restriction (17h) */ + for(ii=0; iipEList->nExpr; ii++){ + char aff; + assert( pSub->pEList->a[ii].pExpr!=0 ); + aff = sqlite3ExprAffinity(pSub->pEList->a[ii].pExpr); + for(pSub1=pSub->pPrior; pSub1; pSub1=pSub1->pPrior){ + assert( pSub1->pEList!=0 ); + assert( pSub1->pEList->nExpr>ii ); + assert( pSub1->pEList->a[ii].pExpr!=0 ); + if( sqlite3ExprAffinity(pSub1->pEList->a[ii].pExpr)!=aff ){ + return 0; + } + } + } + if( pSrc->nSrc>1 ){ if( pParse->nSelect>500 ) return 0; if( OptimizationDisabled(db, SQLITE_FlttnUnionAll) ) return 0; @@ -140150,6 +141885,7 @@ static int flattenSubquery( x.iNewTable = iNewParent; x.isOuterJoin = isOuterJoin; x.pEList = pSub->pEList; + x.pCList = findLeftmostExprlist(pSub); substSelect(&x, pParent, 0); } @@ -140169,7 +141905,7 @@ static int flattenSubquery( pSub->pLimit = 0; } - /* Recompute the SrcList_item.colUsed masks for the flattened + /* Recompute the SrcItem.colUsed masks for the flattened ** tables. */ for(i=0; ia[i+iFrom]); @@ -140559,6 +142295,13 @@ static int pushDownWindowCheck(Parse *pParse, Select *pSubq, Expr *pExpr){ ** be materialized. (This restriction is implemented in the calling ** routine.) ** +** (8) The subquery may not be a compound that uses UNION, INTERSECT, +** or EXCEPT. (We could, perhaps, relax this restriction to allow +** this case if none of the comparisons operators between left and +** right arms of the compound use a collation other than BINARY. +** But it is a lot of work to check that case for an obscure and +** minor optimization, so we omit it for now.) +** ** Return 0 if no changes are made and non-zero if one or more WHERE clause ** terms are duplicated into the subquery. */ @@ -140578,6 +142321,10 @@ static int pushDownWhereTerms( if( pSubq->pPrior ){ Select *pSel; for(pSel=pSubq; pSel; pSel=pSel->pPrior){ + u8 op = pSel->op; + assert( op==TK_ALL || op==TK_SELECT + || op==TK_UNION || op==TK_INTERSECT || op==TK_EXCEPT ); + if( op!=TK_ALL && op!=TK_SELECT ) return 0; /* restriction (8) */ if( pSel->pWin ) return 0; /* restriction (6b) */ } }else{ @@ -140632,6 +142379,7 @@ static int pushDownWhereTerms( x.iNewTable = pSrc->iCursor; x.isOuterJoin = 0; x.pEList = pSubq->pEList; + x.pCList = findLeftmostExprlist(pSubq); pNew = substExpr(&x, pNew); #ifndef SQLITE_OMIT_WINDOWFUNC if( pSubq->pWin && 0==pushDownWindowCheck(pParse, pSubq, pNew) ){ @@ -140735,6 +142483,7 @@ static Table *isSimpleCount(Select *p, AggInfo *pAggInfo){ || p->pSrc->nSrc!=1 || p->pSrc->a[0].pSelect || pAggInfo->nFunc!=1 + || p->pHaving ){ return 0; } @@ -141155,9 +142904,9 @@ SQLITE_PRIVATE void sqlite3SelectPopWith(Walker *pWalker, Select *p){ #endif /* -** The SrcList_item structure passed as the second argument represents a +** The SrcItem structure passed as the second argument represents a ** sub-query in the FROM clause of a SELECT statement. This function -** allocates and populates the SrcList_item.pTab object. If successful, +** allocates and populates the SrcItem.pTab object. If successful, ** SQLITE_OK is returned. Otherwise, if an OOM error is encountered, ** SQLITE_NOMEM. */ @@ -141990,7 +143739,7 @@ static void havingToWhere(Parse *pParse, Select *p){ /* ** Check to see if the pThis entry of pTabList is a self-join of a prior view. -** If it is, then return the SrcList_item for the prior view. If it is not, +** If it is, then return the SrcItem for the prior view. If it is not, ** then return 0. */ static SrcItem *isSelfJoinView( @@ -142608,7 +144357,10 @@ SQLITE_PRIVATE int sqlite3Select( } sqlite3SelectDestInit(&dest, SRT_EphemTab, pItem->iCursor); ExplainQueryPlan((pParse, 1, "MATERIALIZE %!S", pItem)); + dest.zAffSdst = sqlite3TableAffinityStr(db, pItem->pTab); sqlite3Select(pParse, pSub, &dest); + sqlite3DbFree(db, dest.zAffSdst); + dest.zAffSdst = 0; pItem->pTab->nRowLogEst = pSub->nSelectRow; if( onceAddr ) sqlite3VdbeJumpHere(v, onceAddr); sqlite3VdbeAddOp2(v, OP_Return, pItem->regReturn, topAddr+1); @@ -142728,7 +144480,7 @@ SQLITE_PRIVATE int sqlite3Select( if( (p->selFlags & SF_FixedLimit)==0 ){ p->nSelectRow = 320; /* 4 billion rows */ } - computeLimitRegisters(pParse, p, iEnd); + if( p->pLimit ) computeLimitRegisters(pParse, p, iEnd); if( p->iLimit==0 && sSort.addrSortIndex>=0 ){ sqlite3VdbeChangeOpcode(v, sSort.addrSortIndex, OP_SorterOpen); sSort.sortFlags |= SORTFLAG_UseSorter; @@ -142950,8 +144702,13 @@ SQLITE_PRIVATE int sqlite3Select( sqlite3TreeViewExprList(0, pMinMaxOrderBy, 0, "ORDERBY"); } for(ii=0; iinColumn; ii++){ - sqlite3DebugPrintf("agg-column[%d] iMem=%d\n", - ii, pAggInfo->aCol[ii].iMem); + struct AggInfo_col *pCol = &pAggInfo->aCol[ii]; + sqlite3DebugPrintf( + "agg-column[%d] pTab=%s iTable=%d iColumn=%d iMem=%d" + " iSorterColumn=%d\n", + ii, pCol->pTab ? pCol->pTab->zName : "NULL", + pCol->iTable, pCol->iColumn, pCol->iMem, + pCol->iSorterColumn); sqlite3TreeViewExpr(0, pAggInfo->aCol[ii].pCExpr, 0); } for(ii=0; iinFunc; ii++){ @@ -143029,7 +144786,7 @@ SQLITE_PRIVATE int sqlite3Select( sqlite3VdbeAddOp2(v, OP_Gosub, regReset, addrReset); SELECTTRACE(1,pParse,p,("WhereBegin\n")); pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, pGroupBy, pDistinct, - 0, (sDistinct.isTnct==2 ? WHERE_DISTINCTBY : WHERE_GROUPBY) + p, (sDistinct.isTnct==2 ? WHERE_DISTINCTBY : WHERE_GROUPBY) | (orderByGrp ? WHERE_SORTBYGROUP : 0) | distFlag, 0 ); if( pWInfo==0 ){ @@ -143072,15 +144829,15 @@ SQLITE_PRIVATE int sqlite3Select( regBase = sqlite3GetTempRange(pParse, nCol); sqlite3ExprCodeExprList(pParse, pGroupBy, regBase, 0, 0); j = nGroupBy; + pAggInfo->directMode = 1; for(i=0; inColumn; i++){ struct AggInfo_col *pCol = &pAggInfo->aCol[i]; if( pCol->iSorterColumn>=j ){ - int r1 = j + regBase; - sqlite3ExprCodeGetColumnOfTable(v, - pCol->pTab, pCol->iTable, pCol->iColumn, r1); + sqlite3ExprCode(pParse, pCol->pCExpr, j + regBase); j++; } } + pAggInfo->directMode = 0; regRecord = sqlite3GetTempReg(pParse); sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase, nCol, regRecord); sqlite3VdbeAddOp2(v, OP_SorterInsert, pAggInfo->sortingIdx, regRecord); @@ -143328,7 +145085,7 @@ SQLITE_PRIVATE int sqlite3Select( SELECTTRACE(1,pParse,p,("WhereBegin\n")); pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, pMinMaxOrderBy, - pDistinct, 0, minMaxFlag|distFlag, 0); + pDistinct, p, minMaxFlag|distFlag, 0); if( pWInfo==0 ){ goto select_end; } @@ -143972,6 +145729,23 @@ SQLITE_PRIVATE void sqlite3FinishTrigger( Vdbe *v; char *z; + /* If this is a new CREATE TABLE statement, and if shadow tables + ** are read-only, and the trigger makes a change to a shadow table, + ** then raise an error - do not allow the trigger to be created. */ + if( sqlite3ReadOnlyShadowTables(db) ){ + TriggerStep *pStep; + for(pStep=pTrig->step_list; pStep; pStep=pStep->pNext){ + if( pStep->zTarget!=0 + && sqlite3ShadowTableName(db, pStep->zTarget) + ){ + sqlite3ErrorMsg(pParse, + "trigger \"%s\" may not write to shadow table \"%s\"", + pTrig->zName, pStep->zTarget); + goto triggerfinish_cleanup; + } + } + } + /* Make an entry in the sqlite_schema table */ v = sqlite3GetVdbe(pParse); if( v==0 ) goto triggerfinish_cleanup; @@ -144795,7 +146569,7 @@ static TriggerPrg *codeRowTrigger( sSubParse.zAuthContext = pTrigger->zName; sSubParse.eTriggerOp = pTrigger->op; sSubParse.nQueryLoop = pParse->nQueryLoop; - sSubParse.disableVtab = pParse->disableVtab; + sSubParse.prepFlags = pParse->prepFlags; v = sqlite3GetVdbe(&sSubParse); if( v ){ @@ -145141,11 +146915,14 @@ static void updateVirtualTable( ** it has been converted into REAL. */ SQLITE_PRIVATE void sqlite3ColumnDefault(Vdbe *v, Table *pTab, int i, int iReg){ + Column *pCol; assert( pTab!=0 ); - if( !IsView(pTab) ){ + assert( pTab->nCol>i ); + pCol = &pTab->aCol[i]; + if( pCol->iDflt ){ sqlite3_value *pValue = 0; u8 enc = ENC(sqlite3VdbeDb(v)); - Column *pCol = &pTab->aCol[i]; + assert( !IsView(pTab) ); VdbeComment((v, "%s.%s", pTab->zName, pCol->zCnName)); assert( inCol ); sqlite3ValueFromExpr(sqlite3VdbeDb(v), @@ -145156,7 +146933,7 @@ SQLITE_PRIVATE void sqlite3ColumnDefault(Vdbe *v, Table *pTab, int i, int iReg){ } } #ifndef SQLITE_OMIT_FLOATING_POINT - if( pTab->aCol[i].affinity==SQLITE_AFF_REAL && !IsVirtual(pTab) ){ + if( pCol->affinity==SQLITE_AFF_REAL && !IsVirtual(pTab) ){ sqlite3VdbeAddOp1(v, OP_RealAffinity, iReg); } #endif @@ -146596,6 +148373,7 @@ SQLITE_PRIVATE int sqlite3UpsertAnalyzeTarget( if( pIdx->aiColumn[ii]==XN_EXPR ){ assert( pIdx->aColExpr!=0 ); assert( pIdx->aColExpr->nExpr>ii ); + assert( pIdx->bHasExpr ); pExpr = pIdx->aColExpr->a[ii].pExpr; if( pExpr->op!=TK_COLLATE ){ sCol[0].pLeft = pExpr; @@ -146909,6 +148687,7 @@ SQLITE_PRIVATE SQLITE_NOINLINE int sqlite3RunVacuum( int nDb; /* Number of attached databases */ const char *zDbMain; /* Schema name of database to vacuum */ const char *zOut; /* Name of output file */ + u32 pgflags = PAGER_SYNCHRONOUS_OFF; /* sync flags for output db */ if( !db->autoCommit ){ sqlite3SetString(pzErrMsg, db, "cannot VACUUM from within a transaction"); @@ -146980,12 +148759,17 @@ SQLITE_PRIVATE SQLITE_NOINLINE int sqlite3RunVacuum( goto end_of_vacuum; } db->mDbFlags |= DBFLAG_VacuumInto; + + /* For a VACUUM INTO, the pager-flags are set to the same values as + ** they are for the database being vacuumed, except that PAGER_CACHESPILL + ** is always set. */ + pgflags = db->aDb[iDb].safety_level | (db->flags & PAGER_FLAGS_MASK); } nRes = sqlite3BtreeGetRequestedReserve(pMain); sqlite3BtreeSetCacheSize(pTemp, db->aDb[iDb].pSchema->cache_size); sqlite3BtreeSetSpillSize(pTemp, sqlite3BtreeSetSpillSize(pMain,0)); - sqlite3BtreeSetPagerFlags(pTemp, PAGER_SYNCHRONOUS_OFF|PAGER_CACHESPILL); + sqlite3BtreeSetPagerFlags(pTemp, pgflags|PAGER_CACHESPILL); /* Begin a transaction and take an exclusive lock on the main database ** file. This is done before the sqlite3BtreeGetPageSize(pMain) call below, @@ -147498,7 +149282,8 @@ SQLITE_PRIVATE void sqlite3VtabUnlockList(sqlite3 *db){ */ SQLITE_PRIVATE void sqlite3VtabClear(sqlite3 *db, Table *p){ assert( IsVirtual(p) ); - if( !db || db->pnBytesFreed==0 ) vtabDisconnectAll(0, p); + assert( db!=0 ); + if( db->pnBytesFreed==0 ) vtabDisconnectAll(0, p); if( p->u.vtab.azArg ){ int i; for(i=0; iu.vtab.nArg; i++){ @@ -148298,7 +150083,7 @@ SQLITE_PRIVATE FuncDef *sqlite3VtabOverloadFunction( if( pExpr->op!=TK_COLUMN ) return pDef; assert( ExprUseYTab(pExpr) ); pTab = pExpr->y.pTab; - if( pTab==0 ) return pDef; + if( NEVER(pTab==0) ) return pDef; if( !IsVirtual(pTab) ) return pDef; pVtab = sqlite3GetVTable(db, pTab)->pVtab; assert( pVtab!=0 ); @@ -148905,7 +150690,7 @@ struct WhereAndInfo { ** between VDBE cursor numbers and bits of the bitmasks in WhereTerm. ** ** The VDBE cursor numbers are small integers contained in -** SrcList_item.iCursor and Expr.iTable fields. For any given WHERE +** SrcItem.iCursor and Expr.iTable fields. For any given WHERE ** clause, the cursor numbers might not begin with 0 and they might ** contain gaps in the numbering sequence. But we want to make maximum ** use of the bits in our bitmasks. This structure provides a mapping @@ -148976,20 +150761,6 @@ struct WhereLoopBuilder { # define SQLITE_QUERY_PLANNER_LIMIT_INCR 1000 #endif -/* -** Each instance of this object records a change to a single node -** in an expression tree to cause that node to point to a column -** of an index rather than an expression or a virtual column. All -** such transformations need to be undone at the end of WHERE clause -** processing. -*/ -typedef struct WhereExprMod WhereExprMod; -struct WhereExprMod { - WhereExprMod *pNext; /* Next translation on a list of them all */ - Expr *pExpr; /* The Expr node that was transformed */ - Expr orig; /* Original value of the Expr node */ -}; - /* ** The WHERE clause processing routine has two halves. The ** first part does the start of the WHERE loop and the second @@ -149005,10 +150776,10 @@ struct WhereInfo { SrcList *pTabList; /* List of tables in the join */ ExprList *pOrderBy; /* The ORDER BY clause or NULL */ ExprList *pResultSet; /* Result set of the query */ +#if WHERETRACE_ENABLED Expr *pWhere; /* The complete WHERE clause */ -#ifndef SQLITE_OMIT_VIRTUALTABLE - Select *pLimit; /* Used to access LIMIT expr/registers for vtabs */ #endif + Select *pSelect; /* The entire SELECT statement containing WHERE */ int aiCurOnePass[2]; /* OP_OpenWrite cursors for the ONEPASS opt */ int iContinue; /* Jump here to continue with next record */ int iBreak; /* Jump here to break out of the loop */ @@ -149027,7 +150798,6 @@ struct WhereInfo { int iTop; /* The very beginning of the WHERE loop */ int iEndWhere; /* End of the WHERE clause itself */ WhereLoop *pLoops; /* List of all WhereLoop objects */ - WhereExprMod *pExprMods; /* Expression modifications */ WhereMemBlock *pMemToFree;/* Memory to free when this object destroyed */ Bitmask revMask; /* Mask of ORDER BY terms that need reversing */ WhereClause sWC; /* Decomposition of the WHERE clause */ @@ -149175,6 +150945,7 @@ SQLITE_PRIVATE void sqlite3WhereTabFuncArgs(Parse*, SrcItem*, WhereClause*); #define WHERE_BLOOMFILTER 0x00400000 /* Consider using a Bloom-filter */ #define WHERE_SELFCULL 0x00800000 /* nOut reduced by extra WHERE terms */ #define WHERE_OMIT_OFFSET 0x01000000 /* Set offset counter to zero */ +#define WHERE_VIEWSCAN 0x02000000 /* A full-scan of a VIEW or subquery */ #endif /* !defined(SQLITE_WHEREINT_H) */ @@ -149783,7 +151554,8 @@ static int codeEqualityTerm( } sqlite3ExprDelete(db, pX); }else{ - aiMap = (int*)sqlite3DbMallocZero(pParse->db, sizeof(int)*nEq); + int n = sqlite3ExprVectorSize(pX->pLeft); + aiMap = (int*)sqlite3DbMallocZero(pParse->db, sizeof(int)*MAX(nEq,n)); eType = sqlite3FindInIndex(pParse, pX, IN_INDEX_LOOP, 0, aiMap, &iTab); } pX = pExpr; @@ -150053,7 +151825,7 @@ static void whereLikeOptimizationStringFixup( if( pTerm->wtFlags & TERM_LIKEOPT ){ VdbeOp *pOp; assert( pLevel->iLikeRepCntr>0 ); - pOp = sqlite3VdbeGetOp(v, -1); + pOp = sqlite3VdbeGetLastOp(v); assert( pOp!=0 ); assert( pOp->opcode==OP_String8 || pTerm->pWC->pWInfo->pParse->db->mallocFailed ); @@ -150377,143 +152149,6 @@ static void codeExprOrVector(Parse *pParse, Expr *p, int iReg, int nReg){ } } -/* An instance of the IdxExprTrans object carries information about a -** mapping from an expression on table columns into a column in an index -** down through the Walker. -*/ -typedef struct IdxExprTrans { - Expr *pIdxExpr; /* The index expression */ - int iTabCur; /* The cursor of the corresponding table */ - int iIdxCur; /* The cursor for the index */ - int iIdxCol; /* The column for the index */ - int iTabCol; /* The column for the table */ - WhereInfo *pWInfo; /* Complete WHERE clause information */ - sqlite3 *db; /* Database connection (for malloc()) */ -} IdxExprTrans; - -/* -** Preserve pExpr on the WhereETrans list of the WhereInfo. -*/ -static void preserveExpr(IdxExprTrans *pTrans, Expr *pExpr){ - WhereExprMod *pNew; - pNew = sqlite3DbMallocRaw(pTrans->db, sizeof(*pNew)); - if( pNew==0 ) return; - pNew->pNext = pTrans->pWInfo->pExprMods; - pTrans->pWInfo->pExprMods = pNew; - pNew->pExpr = pExpr; - memcpy(&pNew->orig, pExpr, sizeof(*pExpr)); -} - -/* The walker node callback used to transform matching expressions into -** a reference to an index column for an index on an expression. -** -** If pExpr matches, then transform it into a reference to the index column -** that contains the value of pExpr. -*/ -static int whereIndexExprTransNode(Walker *p, Expr *pExpr){ - IdxExprTrans *pX = p->u.pIdxTrans; - if( sqlite3ExprCompare(0, pExpr, pX->pIdxExpr, pX->iTabCur)==0 ){ - pExpr = sqlite3ExprSkipCollate(pExpr); - preserveExpr(pX, pExpr); - pExpr->affExpr = sqlite3ExprAffinity(pExpr); - pExpr->op = TK_COLUMN; - pExpr->iTable = pX->iIdxCur; - pExpr->iColumn = pX->iIdxCol; - testcase( ExprHasProperty(pExpr, EP_Unlikely) ); - ExprClearProperty(pExpr, EP_Skip|EP_Unlikely|EP_WinFunc|EP_Subrtn); - pExpr->y.pTab = 0; - return WRC_Prune; - }else{ - return WRC_Continue; - } -} - -#ifndef SQLITE_OMIT_GENERATED_COLUMNS -/* A walker node callback that translates a column reference to a table -** into a corresponding column reference of an index. -*/ -static int whereIndexExprTransColumn(Walker *p, Expr *pExpr){ - if( pExpr->op==TK_COLUMN ){ - IdxExprTrans *pX = p->u.pIdxTrans; - if( pExpr->iTable==pX->iTabCur && pExpr->iColumn==pX->iTabCol ){ - assert( ExprUseYTab(pExpr) && pExpr->y.pTab!=0 ); - preserveExpr(pX, pExpr); - pExpr->affExpr = sqlite3TableColumnAffinity(pExpr->y.pTab,pExpr->iColumn); - pExpr->iTable = pX->iIdxCur; - pExpr->iColumn = pX->iIdxCol; - pExpr->y.pTab = 0; - } - } - return WRC_Continue; -} -#endif /* SQLITE_OMIT_GENERATED_COLUMNS */ - -/* -** For an indexes on expression X, locate every instance of expression X -** in pExpr and change that subexpression into a reference to the appropriate -** column of the index. -** -** 2019-10-24: Updated to also translate references to a VIRTUAL column in -** the table into references to the corresponding (stored) column of the -** index. -*/ -static void whereIndexExprTrans( - Index *pIdx, /* The Index */ - int iTabCur, /* Cursor of the table that is being indexed */ - int iIdxCur, /* Cursor of the index itself */ - WhereInfo *pWInfo /* Transform expressions in this WHERE clause */ -){ - int iIdxCol; /* Column number of the index */ - ExprList *aColExpr; /* Expressions that are indexed */ - Table *pTab; - Walker w; - IdxExprTrans x; - aColExpr = pIdx->aColExpr; - if( aColExpr==0 && !pIdx->bHasVCol ){ - /* The index does not reference any expressions or virtual columns - ** so no translations are needed. */ - return; - } - pTab = pIdx->pTable; - memset(&w, 0, sizeof(w)); - w.u.pIdxTrans = &x; - x.iTabCur = iTabCur; - x.iIdxCur = iIdxCur; - x.pWInfo = pWInfo; - x.db = pWInfo->pParse->db; - for(iIdxCol=0; iIdxColnColumn; iIdxCol++){ - i16 iRef = pIdx->aiColumn[iIdxCol]; - if( iRef==XN_EXPR ){ - assert( aColExpr!=0 && aColExpr->a[iIdxCol].pExpr!=0 ); - x.pIdxExpr = aColExpr->a[iIdxCol].pExpr; - if( sqlite3ExprIsConstant(x.pIdxExpr) ) continue; - w.xExprCallback = whereIndexExprTransNode; -#ifndef SQLITE_OMIT_GENERATED_COLUMNS - }else if( iRef>=0 - && (pTab->aCol[iRef].colFlags & COLFLAG_VIRTUAL)!=0 - && ((pTab->aCol[iRef].colFlags & COLFLAG_HASCOLL)==0 - || sqlite3StrICmp(sqlite3ColumnColl(&pTab->aCol[iRef]), - sqlite3StrBINARY)==0) - ){ - /* Check to see if there are direct references to generated columns - ** that are contained in the index. Pulling the generated column - ** out of the index is an optimization only - the main table is always - ** available if the index cannot be used. To avoid unnecessary - ** complication, omit this optimization if the collating sequence for - ** the column is non-standard */ - x.iTabCol = iRef; - w.xExprCallback = whereIndexExprTransColumn; -#endif /* SQLITE_OMIT_GENERATED_COLUMNS */ - }else{ - continue; - } - x.iIdxCol = iIdxCol; - sqlite3WalkExpr(&w, pWInfo->pWhere); - sqlite3WalkExprList(&w, pWInfo->pOrderBy); - sqlite3WalkExprList(&w, pWInfo->pResultSet); - } -} - /* ** The pTruth expression is always true because it is the WHERE clause ** a partial index that is driving a query loop. Look through all of the @@ -150582,6 +152217,8 @@ static SQLITE_NOINLINE void filterPullDown( testcase( pTerm->wtFlags & TERM_VIRTUAL ); regRowid = sqlite3GetTempReg(pParse); regRowid = codeEqualityTerm(pParse, pTerm, pLevel, 0, 0, regRowid); + sqlite3VdbeAddOp2(pParse->pVdbe, OP_MustBeInt, regRowid, addrNxt); + VdbeCoverage(pParse->pVdbe); sqlite3VdbeAddOp4Int(pParse->pVdbe, OP_Filter, pLevel->regFilter, addrNxt, regRowid, 1); VdbeCoverage(pParse->pVdbe); @@ -150733,9 +152370,9 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( && pLoop->u.vtab.bOmitOffset ){ assert( pTerm->eOperator==WO_AUX ); - assert( pWInfo->pLimit!=0 ); - assert( pWInfo->pLimit->iOffset>0 ); - sqlite3VdbeAddOp2(v, OP_Integer, 0, pWInfo->pLimit->iOffset); + assert( pWInfo->pSelect!=0 ); + assert( pWInfo->pSelect->iOffset>0 ); + sqlite3VdbeAddOp2(v, OP_Integer, 0, pWInfo->pSelect->iOffset); VdbeComment((v,"Zero OFFSET counter")); } } @@ -150843,6 +152480,8 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( if( iRowidReg!=iReleaseReg ) sqlite3ReleaseTempReg(pParse, iReleaseReg); addrNxt = pLevel->addrNxt; if( pLevel->regFilter ){ + sqlite3VdbeAddOp2(v, OP_MustBeInt, iRowidReg, addrNxt); + VdbeCoverage(v); sqlite3VdbeAddOp4Int(v, OP_Filter, pLevel->regFilter, addrNxt, iRowidReg, 1); VdbeCoverage(v); @@ -151194,6 +152833,11 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( ** guess. */ addrSeekScan = sqlite3VdbeAddOp1(v, OP_SeekScan, (pIdx->aiRowLogEst[0]+9)/10); + if( pRangeStart ){ + sqlite3VdbeChangeP5(v, 1); + sqlite3VdbeChangeP2(v, addrSeekScan, sqlite3VdbeCurrentAddr(v)+1); + addrSeekScan = 0; + } VdbeCoverage(v); } sqlite3VdbeAddOp4Int(v, op, iIdxCur, addrNxt, regBase, nConstraint); @@ -151269,8 +152913,8 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( } nConstraint++; } - sqlite3DbFree(db, zStartAff); - sqlite3DbFree(db, zEndAff); + if( zStartAff ) sqlite3DbNNFreeNN(db, zStartAff); + if( zEndAff ) sqlite3DbNNFreeNN(db, zEndAff); /* Top of the loop body */ if( pLevel->p2==0 ) pLevel->p2 = sqlite3VdbeCurrentAddr(v); @@ -151332,27 +152976,6 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( } if( pLevel->iLeftJoin==0 ){ - /* If pIdx is an index on one or more expressions, then look through - ** all the expressions in pWInfo and try to transform matching expressions - ** into reference to index columns. Also attempt to translate references - ** to virtual columns in the table into references to (stored) columns - ** of the index. - ** - ** Do not do this for the RHS of a LEFT JOIN. This is because the - ** expression may be evaluated after OP_NullRow has been executed on - ** the cursor. In this case it is important to do the full evaluation, - ** as the result of the expression may not be NULL, even if all table - ** column values are. https://www.sqlite.org/src/info/7fa8049685b50b5a - ** - ** Also, do not do this when processing one index an a multi-index - ** OR clause, since the transformation will become invalid once we - ** move forward to the next index. - ** https://sqlite.org/src/info/4e8e4857d32d401f - */ - if( (pWInfo->wctrlFlags & (WHERE_OR_SUBCLAUSE|WHERE_RIGHT_JOIN))==0 ){ - whereIndexExprTrans(pIdx, iCur, iIdxCur, pWInfo); - } - /* If a partial index is driving the loop, try to eliminate WHERE clause ** terms from the query that must be true due to the WHERE clause of ** the partial index. @@ -151465,7 +153088,7 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( int nNotReady; /* The number of notReady tables */ SrcItem *origSrc; /* Original list of tables */ nNotReady = pWInfo->nLevel - iLevel - 1; - pOrTab = sqlite3StackAllocRaw(db, + pOrTab = sqlite3DbMallocRawNN(db, sizeof(*pOrTab)+ nNotReady*sizeof(pOrTab->a[0])); if( pOrTab==0 ) return notReady; pOrTab->nAlloc = (u8)(nNotReady + 1); @@ -151718,7 +153341,7 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( assert( pLevel->op==OP_Return ); pLevel->p2 = sqlite3VdbeCurrentAddr(v); - if( pWInfo->nLevel>1 ){ sqlite3StackFree(db, pOrTab); } + if( pWInfo->nLevel>1 ){ sqlite3DbFreeNN(db, pOrTab); } if( !untestedTerms ) disableTerm(pLevel, pTerm); }else #endif /* SQLITE_OMIT_OR_OPTIMIZATION */ @@ -152346,7 +153969,7 @@ static int isLikeOrGlob( if( pLeft->op!=TK_COLUMN || sqlite3ExprAffinity(pLeft)!=SQLITE_AFF_TEXT || (ALWAYS( ExprUseYTab(pLeft) ) - && pLeft->y.pTab + && ALWAYS(pLeft->y.pTab) && IsVirtual(pLeft->y.pTab)) /* Might be numeric */ ){ int isNum; @@ -152463,8 +154086,7 @@ static int isAuxiliaryVtabOperator( ** MATCH(expression,vtab_column) */ pCol = pList->a[1].pExpr; - assert( pCol->op!=TK_COLUMN || ExprUseYTab(pCol) ); - testcase( pCol->op==TK_COLUMN && pCol->y.pTab==0 ); + assert( pCol->op!=TK_COLUMN || (ExprUseYTab(pCol) && pCol->y.pTab!=0) ); if( ExprIsVtab(pCol) ){ for(i=0; ia[0].pExpr; assert( pCol->op!=TK_COLUMN || ExprUseYTab(pCol) ); - testcase( pCol->op==TK_COLUMN && pCol->y.pTab==0 ); + assert( pCol->op!=TK_COLUMN || (ExprUseYTab(pCol) && pCol->y.pTab!=0) ); if( ExprIsVtab(pCol) ){ sqlite3_vtab *pVtab; sqlite3_module *pMod; @@ -152514,13 +154136,12 @@ static int isAuxiliaryVtabOperator( int res = 0; Expr *pLeft = pExpr->pLeft; Expr *pRight = pExpr->pRight; - assert( pLeft->op!=TK_COLUMN || ExprUseYTab(pLeft) ); - testcase( pLeft->op==TK_COLUMN && pLeft->y.pTab==0 ); + assert( pLeft->op!=TK_COLUMN || (ExprUseYTab(pLeft) && pLeft->y.pTab!=0) ); if( ExprIsVtab(pLeft) ){ res++; } - assert( pRight==0 || pRight->op!=TK_COLUMN || ExprUseYTab(pRight) ); - testcase( pRight && pRight->op==TK_COLUMN && pRight->y.pTab==0 ); + assert( pRight==0 || pRight->op!=TK_COLUMN + || (ExprUseYTab(pRight) && pRight->y.pTab!=0) ); if( pRight && ExprIsVtab(pRight) ){ res++; SWAP(Expr*, pLeft, pRight); @@ -153069,6 +154690,7 @@ static SQLITE_NOINLINE int exprMightBeIndexed2( if( pIdx->aColExpr==0 ) continue; for(i=0; inKeyCol; i++){ if( pIdx->aiColumn[i]!=XN_EXPR ) continue; + assert( pIdx->bHasExpr ); if( sqlite3ExprCompareSkip(pExpr, pIdx->aColExpr->a[i].pExpr, iCur)==0 ){ aiCurCol[0] = iCur; aiCurCol[1] = XN_EXPR; @@ -153682,9 +155304,9 @@ static void whereAddLimitExpr( ** exist only so that they may be passed to the xBestIndex method of the ** single virtual table in the FROM clause of the SELECT. */ -SQLITE_PRIVATE void sqlite3WhereAddLimit(WhereClause *pWC, Select *p){ - assert( p==0 || (p->pGroupBy==0 && (p->selFlags & SF_Aggregate)==0) ); - if( (p && p->pLimit) /* 1 */ +SQLITE_PRIVATE void SQLITE_NOINLINE sqlite3WhereAddLimit(WhereClause *pWC, Select *p){ + assert( p!=0 && p->pLimit!=0 ); /* 1 -- checked by caller */ + if( p->pGroupBy==0 && (p->selFlags & (SF_Distinct|SF_Aggregate))==0 /* 2 */ && (p->pSrc->nSrc==1 && IsVirtual(p->pSrc->a[0].pTab)) /* 3 */ ){ @@ -154677,6 +156299,43 @@ static void whereTraceIndexInfoOutputs(sqlite3_index_info *p){ #define whereTraceIndexInfoOutputs(A) #endif +/* +** We know that pSrc is an operand of an outer join. Return true if +** pTerm is a constraint that is compatible with that join. +** +** pTerm must be EP_OuterON if pSrc is the right operand of an +** outer join. pTerm can be either EP_OuterON or EP_InnerON if pSrc +** is the left operand of a RIGHT join. +** +** See https://sqlite.org/forum/forumpost/206d99a16dd9212f +** for an example of a WHERE clause constraints that may not be used on +** the right table of a RIGHT JOIN because the constraint implies a +** not-NULL condition on the left table of the RIGHT JOIN. +*/ +static int constraintCompatibleWithOuterJoin( + const WhereTerm *pTerm, /* WHERE clause term to check */ + const SrcItem *pSrc /* Table we are trying to access */ +){ + assert( (pSrc->fg.jointype&(JT_LEFT|JT_LTORJ|JT_RIGHT))!=0 ); /* By caller */ + testcase( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))==JT_LEFT ); + testcase( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))==JT_LTORJ ); + testcase( ExprHasProperty(pTerm->pExpr, EP_OuterON) ) + testcase( ExprHasProperty(pTerm->pExpr, EP_InnerON) ); + if( !ExprHasProperty(pTerm->pExpr, EP_OuterON|EP_InnerON) + || pTerm->pExpr->w.iJoin != pSrc->iCursor + ){ + return 0; + } + if( (pSrc->fg.jointype & (JT_LEFT|JT_RIGHT))!=0 + && ExprHasProperty(pTerm->pExpr, EP_InnerON) + ){ + return 0; + } + return 1; +} + + + #ifndef SQLITE_OMIT_AUTOMATIC_INDEX /* ** Return TRUE if the WHERE clause term pTerm is of a form where it @@ -154692,16 +156351,10 @@ static int termCanDriveIndex( if( pTerm->leftCursor!=pSrc->iCursor ) return 0; if( (pTerm->eOperator & (WO_EQ|WO_IS))==0 ) return 0; assert( (pSrc->fg.jointype & JT_RIGHT)==0 ); - if( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))!=0 ){ - testcase( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))==JT_LEFT ); - testcase( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))==JT_LTORJ ); - testcase( ExprHasProperty(pTerm->pExpr, EP_OuterON) ) - testcase( ExprHasProperty(pTerm->pExpr, EP_InnerON) ); - if( !ExprHasProperty(pTerm->pExpr, EP_OuterON|EP_InnerON) - || pTerm->pExpr->w.iJoin != pSrc->iCursor - ){ - return 0; /* See tag-20191211-001 */ - } + if( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))!=0 + && !constraintCompatibleWithOuterJoin(pTerm,pSrc) + ){ + return 0; /* See https://sqlite.org/forum/forumpost/51e6959f61 */ } if( (pTerm->prereqRight & notReady)!=0 ) return 0; assert( (pTerm->eOperator & (WO_OR|WO_AND))==0 ); @@ -155113,22 +156766,10 @@ static sqlite3_index_info *allocateIndexInfo( assert( (pTerm->eOperator & (WO_OR|WO_AND))==0 ); assert( pTerm->u.x.leftColumn>=XN_ROWID ); assert( pTerm->u.x.leftColumnnCol ); - - /* tag-20191211-002: WHERE-clause constraints are not useful to the - ** right-hand table of a LEFT JOIN nor to the either table of a - ** RIGHT JOIN. See tag-20191211-001 for the - ** equivalent restriction for ordinary tables. */ - if( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))!=0 ){ - testcase( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))==JT_LEFT ); - testcase( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))==JT_RIGHT ); - testcase( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))==JT_LTORJ ); - testcase( ExprHasProperty(pTerm->pExpr, EP_OuterON) ); - testcase( ExprHasProperty(pTerm->pExpr, EP_InnerON) ); - if( !ExprHasProperty(pTerm->pExpr, EP_OuterON|EP_InnerON) - || pTerm->pExpr->w.iJoin != pSrc->iCursor - ){ - continue; - } + if( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))!=0 + && !constraintCompatibleWithOuterJoin(pTerm,pSrc) + ){ + continue; } nTerm++; pTerm->wtFlags |= TERM_OK; @@ -155501,7 +157142,7 @@ static int whereKeyStats( ** is larger than all samples in the array. */ tRowcnt iUpper, iGap; if( i>=pIdx->nSample ){ - iUpper = sqlite3LogEstToInt(pIdx->aiRowLogEst[0]); + iUpper = pIdx->nRowEst0; }else{ iUpper = aSample[i].anLt[iCol]; } @@ -156130,12 +157771,18 @@ static void whereLoopClearUnion(sqlite3 *db, WhereLoop *p){ } /* -** Deallocate internal memory used by a WhereLoop object +** Deallocate internal memory used by a WhereLoop object. Leave the +** object in an initialized state, as if it had been newly allocated. */ static void whereLoopClear(sqlite3 *db, WhereLoop *p){ - if( p->aLTerm!=p->aLTermSpace ) sqlite3DbFreeNN(db, p->aLTerm); + if( p->aLTerm!=p->aLTermSpace ){ + sqlite3DbFreeNN(db, p->aLTerm); + p->aLTerm = p->aLTermSpace; + p->nLSlot = ArraySize(p->aLTermSpace); + } whereLoopClearUnion(db, p); - whereLoopInit(p); + p->nLTerm = 0; + p->wsFlags = 0; } /* @@ -156159,7 +157806,9 @@ static int whereLoopResize(sqlite3 *db, WhereLoop *p, int n){ */ static int whereLoopXfer(sqlite3 *db, WhereLoop *pTo, WhereLoop *pFrom){ whereLoopClearUnion(db, pTo); - if( whereLoopResize(db, pTo, pFrom->nLTerm) ){ + if( pFrom->nLTerm > pTo->nLSlot + && whereLoopResize(db, pTo, pFrom->nLTerm) + ){ memset(pTo, 0, WHERE_LOOP_XFER_SZ); return SQLITE_NOMEM_BKPT; } @@ -156177,8 +157826,9 @@ static int whereLoopXfer(sqlite3 *db, WhereLoop *pTo, WhereLoop *pFrom){ ** Delete a WhereLoop object */ static void whereLoopDelete(sqlite3 *db, WhereLoop *p){ + assert( db!=0 ); whereLoopClear(db, p); - sqlite3DbFreeNN(db, p); + sqlite3DbNNFreeNN(db, p); } /* @@ -156186,30 +157836,19 @@ static void whereLoopDelete(sqlite3 *db, WhereLoop *p){ */ static void whereInfoFree(sqlite3 *db, WhereInfo *pWInfo){ assert( pWInfo!=0 ); + assert( db!=0 ); sqlite3WhereClauseClear(&pWInfo->sWC); while( pWInfo->pLoops ){ WhereLoop *p = pWInfo->pLoops; pWInfo->pLoops = p->pNextLoop; whereLoopDelete(db, p); } - assert( pWInfo->pExprMods==0 ); while( pWInfo->pMemToFree ){ WhereMemBlock *pNext = pWInfo->pMemToFree->pNext; - sqlite3DbFreeNN(db, pWInfo->pMemToFree); + sqlite3DbNNFreeNN(db, pWInfo->pMemToFree); pWInfo->pMemToFree = pNext; } - sqlite3DbFreeNN(db, pWInfo); -} - -/* Undo all Expr node modifications -*/ -static void whereUndoExprMods(WhereInfo *pWInfo){ - while( pWInfo->pExprMods ){ - WhereExprMod *p = pWInfo->pExprMods; - pWInfo->pExprMods = p->pNext; - memcpy(p->pExpr, &p->orig, sizeof(p->orig)); - sqlite3DbFree(pWInfo->pParse->db, p); - } + sqlite3DbNNFreeNN(db, pWInfo); } /* @@ -156776,32 +158415,11 @@ static int whereLoopAddBtreeIndex( ** to mix with a lower range bound from some other source */ if( pTerm->wtFlags & TERM_LIKEOPT && pTerm->eOperator==WO_LT ) continue; - /* tag-20191211-001: Do not allow constraints from the WHERE clause to - ** be used by the right table of a LEFT JOIN nor by the left table of a - ** RIGHT JOIN. Only constraints in the ON clause are allowed. - ** See tag-20191211-002 for the vtab equivalent. - ** - ** 2022-06-06: See https://sqlite.org/forum/forumpost/206d99a16dd9212f - ** for an example of a WHERE clause constraints that may not be used on - ** the right table of a RIGHT JOIN because the constraint implies a - ** not-NULL condition on the left table of the RIGHT JOIN. - ** - ** 2022-06-10: The same condition applies to termCanDriveIndex() above. - ** https://sqlite.org/forum/forumpost/51e6959f61 - */ - if( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))!=0 ){ - testcase( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))==JT_LEFT ); - testcase( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))==JT_RIGHT ); - testcase( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))==JT_LTORJ ); - testcase( ExprHasProperty(pTerm->pExpr, EP_OuterON) ) - testcase( ExprHasProperty(pTerm->pExpr, EP_InnerON) ); - if( !ExprHasProperty(pTerm->pExpr, EP_OuterON|EP_InnerON) - || pTerm->pExpr->w.iJoin != pSrc->iCursor - ){ - continue; - } + if( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))!=0 + && !constraintCompatibleWithOuterJoin(pTerm,pSrc) + ){ + continue; } - if( IsUniqueIndex(pProbe) && saved_nEq==pProbe->nKeyCol-1 ){ pBuilder->bldFlags1 |= SQLITE_BLDF1_UNIQUE; }else{ @@ -156812,7 +158430,11 @@ static int whereLoopAddBtreeIndex( pNew->u.btree.nBtm = saved_nBtm; pNew->u.btree.nTop = saved_nTop; pNew->nLTerm = saved_nLTerm; - if( whereLoopResize(db, pNew, pNew->nLTerm+1) ) break; /* OOM */ + if( pNew->nLTerm>=pNew->nLSlot + && whereLoopResize(db, pNew, pNew->nLTerm+1) + ){ + break; /* OOM while trying to enlarge the pNew->aLTerm array */ + } pNew->aLTerm[pNew->nLTerm++] = pTerm; pNew->prereq = (saved_prereq | pTerm->prereqRight) & ~pNew->maskSelf; @@ -156905,38 +158527,39 @@ static int whereLoopAddBtreeIndex( if( scan.iEquiv>1 ) pNew->wsFlags |= WHERE_TRANSCONS; }else if( eOp & WO_ISNULL ){ pNew->wsFlags |= WHERE_COLUMN_NULL; - }else if( eOp & (WO_GT|WO_GE) ){ - testcase( eOp & WO_GT ); - testcase( eOp & WO_GE ); - pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_BTM_LIMIT; - pNew->u.btree.nBtm = whereRangeVectorLen( - pParse, pSrc->iCursor, pProbe, saved_nEq, pTerm - ); - pBtm = pTerm; - pTop = 0; - if( pTerm->wtFlags & TERM_LIKEOPT ){ - /* Range constraints that come from the LIKE optimization are - ** always used in pairs. */ - pTop = &pTerm[1]; - assert( (pTop-(pTerm->pWC->a))pWC->nTerm ); - assert( pTop->wtFlags & TERM_LIKEOPT ); - assert( pTop->eOperator==WO_LT ); - if( whereLoopResize(db, pNew, pNew->nLTerm+1) ) break; /* OOM */ - pNew->aLTerm[pNew->nLTerm++] = pTop; - pNew->wsFlags |= WHERE_TOP_LIMIT; - pNew->u.btree.nTop = 1; - } }else{ - assert( eOp & (WO_LT|WO_LE) ); - testcase( eOp & WO_LT ); - testcase( eOp & WO_LE ); - pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_TOP_LIMIT; - pNew->u.btree.nTop = whereRangeVectorLen( + int nVecLen = whereRangeVectorLen( pParse, pSrc->iCursor, pProbe, saved_nEq, pTerm ); - pTop = pTerm; - pBtm = (pNew->wsFlags & WHERE_BTM_LIMIT)!=0 ? - pNew->aLTerm[pNew->nLTerm-2] : 0; + if( eOp & (WO_GT|WO_GE) ){ + testcase( eOp & WO_GT ); + testcase( eOp & WO_GE ); + pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_BTM_LIMIT; + pNew->u.btree.nBtm = nVecLen; + pBtm = pTerm; + pTop = 0; + if( pTerm->wtFlags & TERM_LIKEOPT ){ + /* Range constraints that come from the LIKE optimization are + ** always used in pairs. */ + pTop = &pTerm[1]; + assert( (pTop-(pTerm->pWC->a))pWC->nTerm ); + assert( pTop->wtFlags & TERM_LIKEOPT ); + assert( pTop->eOperator==WO_LT ); + if( whereLoopResize(db, pNew, pNew->nLTerm+1) ) break; /* OOM */ + pNew->aLTerm[pNew->nLTerm++] = pTop; + pNew->wsFlags |= WHERE_TOP_LIMIT; + pNew->u.btree.nTop = 1; + } + }else{ + assert( eOp & (WO_LT|WO_LE) ); + testcase( eOp & WO_LT ); + testcase( eOp & WO_LE ); + pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_TOP_LIMIT; + pNew->u.btree.nTop = nVecLen; + pTop = pTerm; + pBtm = (pNew->wsFlags & WHERE_BTM_LIMIT)!=0 ? + pNew->aLTerm[pNew->nLTerm-2] : 0; + } } /* At this point pNew->nOut is set to the number of rows expected to @@ -157180,6 +158803,94 @@ static int whereUsablePartialIndex( return 0; } +/* +** Structure passed to the whereIsCoveringIndex Walker callback. +*/ +struct CoveringIndexCheck { + Index *pIdx; /* The index */ + int iTabCur; /* Cursor number for the corresponding table */ +}; + +/* +** Information passed in is pWalk->u.pCovIdxCk. Call is pCk. +** +** If the Expr node references the table with cursor pCk->iTabCur, then +** make sure that column is covered by the index pCk->pIdx. We know that +** all columns less than 63 (really BMS-1) are covered, so we don't need +** to check them. But we do need to check any column at 63 or greater. +** +** If the index does not cover the column, then set pWalk->eCode to +** non-zero and return WRC_Abort to stop the search. +** +** If this node does not disprove that the index can be a covering index, +** then just return WRC_Continue, to continue the search. +*/ +static int whereIsCoveringIndexWalkCallback(Walker *pWalk, Expr *pExpr){ + int i; /* Loop counter */ + const Index *pIdx; /* The index of interest */ + const i16 *aiColumn; /* Columns contained in the index */ + u16 nColumn; /* Number of columns in the index */ + if( pExpr->op!=TK_COLUMN && pExpr->op!=TK_AGG_COLUMN ) return WRC_Continue; + if( pExpr->iColumn<(BMS-1) ) return WRC_Continue; + if( pExpr->iTable!=pWalk->u.pCovIdxCk->iTabCur ) return WRC_Continue; + pIdx = pWalk->u.pCovIdxCk->pIdx; + aiColumn = pIdx->aiColumn; + nColumn = pIdx->nColumn; + for(i=0; iiColumn ) return WRC_Continue; + } + pWalk->eCode = 1; + return WRC_Abort; +} + + +/* +** pIdx is an index that covers all of the low-number columns used by +** pWInfo->pSelect (columns from 0 through 62). But there are columns +** in pWInfo->pSelect beyond 62. This routine tries to answer the question +** of whether pIdx covers *all* columns in the query. +** +** Return 0 if pIdx is a covering index. Return non-zero if pIdx is +** not a covering index or if we are unable to determine if pIdx is a +** covering index. +** +** This routine is an optimization. It is always safe to return non-zero. +** But returning zero when non-zero should have been returned can lead to +** incorrect bytecode and assertion faults. +*/ +static SQLITE_NOINLINE u32 whereIsCoveringIndex( + WhereInfo *pWInfo, /* The WHERE clause context */ + Index *pIdx, /* Index that is being tested */ + int iTabCur /* Cursor for the table being indexed */ +){ + int i; + struct CoveringIndexCheck ck; + Walker w; + if( pWInfo->pSelect==0 ){ + /* We don't have access to the full query, so we cannot check to see + ** if pIdx is covering. Assume it is not. */ + return 1; + } + for(i=0; inColumn; i++){ + if( pIdx->aiColumn[i]>=BMS-1 ) break; + } + if( i>=pIdx->nColumn ){ + /* pIdx does not index any columns greater than 62, but we know from + ** colMask that columns greater than 62 are used, so this is not a + ** covering index */ + return 1; + } + ck.pIdx = pIdx; + ck.iTabCur = iTabCur; + memset(&w, 0, sizeof(w)); + w.xExprCallback = whereIsCoveringIndexWalkCallback; + w.xSelectCallback = sqlite3SelectWalkNoop; + w.u.pCovIdxCk = &ck; + w.eCode = 0; + sqlite3WalkSelect(&w, pWInfo->pSelect); + return w.eCode; +} + /* ** Add all WhereLoop objects for a single table of the join where the table ** is identified by pBuilder->pNew->iTab. That table is guaranteed to be @@ -157382,6 +159093,9 @@ static int whereLoopAddBtree( #else pNew->rRun = rSize + 16; #endif + if( IsView(pTab) || (pTab->tabFlags & TF_Ephemeral)!=0 ){ + pNew->wsFlags |= WHERE_VIEWSCAN; + } ApplyCostMultiplier(pNew->rRun, pTab->costMult); whereLoopOutputAdjust(pWC, pNew, rSize); rc = whereLoopInsert(pBuilder, pNew); @@ -157394,6 +159108,9 @@ static int whereLoopAddBtree( m = 0; }else{ m = pSrc->colUsed & pProbe->colNotIdxed; + if( m==TOPBIT ){ + m = whereIsCoveringIndex(pWInfo, pProbe, pSrc->iCursor); + } pNew->wsFlags = (m==0) ? (WHERE_IDX_ONLY|WHERE_INDEXED) : WHERE_INDEXED; } @@ -158108,7 +159825,13 @@ static int whereLoopAddAll(WhereLoopBuilder *pBuilder){ /* Loop over the tables in the join, from left to right */ pNew = pBuilder->pNew; - whereLoopInit(pNew); + + /* Verify that pNew has already been initialized */ + assert( pNew->nLTerm==0 ); + assert( pNew->wsFlags==0 ); + assert( pNew->nLSlot>=ArraySize(pNew->aLTermSpace) ); + assert( pNew->aLTerm!=0 ); + pBuilder->iPlanLimit = SQLITE_QUERY_PLANNER_LIMIT; for(iTab=0, pItem=pTabList->a; pItempParse; - db = pParse->db; nLoop = pWInfo->nLevel; /* TUNING: For simple queries, only the best path is tracked. ** For 2-way joins, the 5 best paths are followed. @@ -158655,7 +160376,7 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ /* Allocate and initialize space for aTo, aFrom and aSortCost[] */ nSpace = (sizeof(WherePath)+sizeof(WhereLoop*)*nLoop)*mxChoice*2; nSpace += sizeof(LogEst) * nOrderBy; - pSpace = sqlite3DbMallocRawNN(db, nSpace); + pSpace = sqlite3StackAllocRawNN(pParse->db, nSpace); if( pSpace==0 ) return SQLITE_NOMEM_BKPT; aTo = (WherePath*)pSpace; aFrom = aTo+mxChoice; @@ -158705,9 +160426,9 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ LogEst nOut; /* Rows visited by (pFrom+pWLoop) */ LogEst rCost; /* Cost of path (pFrom+pWLoop) */ LogEst rUnsorted; /* Unsorted cost of (pFrom+pWLoop) */ - i8 isOrdered = pFrom->isOrdered; /* isOrdered for (pFrom+pWLoop) */ + i8 isOrdered; /* isOrdered for (pFrom+pWLoop) */ Bitmask maskNew; /* Mask of src visited by (..) */ - Bitmask revMask = 0; /* Mask of rev-order loops for (..) */ + Bitmask revMask; /* Mask of rev-order loops for (..) */ if( (pWLoop->prereq & ~pFrom->maskLoop)!=0 ) continue; if( (pWLoop->maskSelf & pFrom->maskLoop)!=0 ) continue; @@ -158726,7 +160447,9 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ rUnsorted = sqlite3LogEstAdd(rUnsorted, pFrom->rUnsorted); nOut = pFrom->nRow + pWLoop->nOut; maskNew = pFrom->maskLoop | pWLoop->maskSelf; + isOrdered = pFrom->isOrdered; if( isOrdered<0 ){ + revMask = 0; isOrdered = wherePathSatisfiesOrderBy(pWInfo, pWInfo->pOrderBy, pFrom, pWInfo->wctrlFlags, iLoop, pWLoop, &revMask); @@ -158754,6 +160477,13 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ rUnsorted -= 2; /* TUNING: Slight bias in favor of no-sort plans */ } + /* TUNING: A full-scan of a VIEW or subquery in the outer loop + ** is not so bad. */ + if( iLoop==0 && (pWLoop->wsFlags & WHERE_VIEWSCAN)!=0 ){ + rCost += -10; + nOut += -30; + } + /* Check to see if pWLoop should be added to the set of ** mxChoice best-so-far paths. ** @@ -158904,7 +160634,7 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ if( nFrom==0 ){ sqlite3ErrorMsg(pParse, "no query solution"); - sqlite3DbFreeNN(db, pSpace); + sqlite3StackFreeNN(pParse->db, pSpace); return SQLITE_ERROR; } @@ -158986,7 +160716,7 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ pWInfo->nRowOut = pFrom->nRow; /* Free temporary memory and return success */ - sqlite3DbFreeNN(db, pSpace); + sqlite3StackFreeNN(pParse->db, pSpace); return SQLITE_OK; } @@ -159285,6 +161015,77 @@ static SQLITE_NOINLINE void whereCheckIfBloomFilterIsUseful( } } +/* +** This is an sqlite3ParserAddCleanup() callback that is invoked to +** free the Parse->pIdxExpr list when the Parse object is destroyed. +*/ +static void whereIndexedExprCleanup(sqlite3 *db, void *pObject){ + Parse *pParse = (Parse*)pObject; + while( pParse->pIdxExpr!=0 ){ + IndexedExpr *p = pParse->pIdxExpr; + pParse->pIdxExpr = p->pIENext; + sqlite3ExprDelete(db, p->pExpr); + sqlite3DbFreeNN(db, p); + } +} + +/* +** The index pIdx is used by a query and contains one or more expressions. +** In other words pIdx is an index on an expression. iIdxCur is the cursor +** number for the index and iDataCur is the cursor number for the corresponding +** table. +** +** This routine adds IndexedExpr entries to the Parse->pIdxExpr field for +** each of the expressions in the index so that the expression code generator +** will know to replace occurrences of the indexed expression with +** references to the corresponding column of the index. +*/ +static SQLITE_NOINLINE void whereAddIndexedExpr( + Parse *pParse, /* Add IndexedExpr entries to pParse->pIdxExpr */ + Index *pIdx, /* The index-on-expression that contains the expressions */ + int iIdxCur, /* Cursor number for pIdx */ + SrcItem *pTabItem /* The FROM clause entry for the table */ +){ + int i; + IndexedExpr *p; + Table *pTab; + assert( pIdx->bHasExpr ); + pTab = pIdx->pTable; + for(i=0; inColumn; i++){ + Expr *pExpr; + int j = pIdx->aiColumn[i]; + int bMaybeNullRow; + if( j==XN_EXPR ){ + pExpr = pIdx->aColExpr->a[i].pExpr; + testcase( pTabItem->fg.jointype & JT_LEFT ); + testcase( pTabItem->fg.jointype & JT_RIGHT ); + testcase( pTabItem->fg.jointype & JT_LTORJ ); + bMaybeNullRow = (pTabItem->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))!=0; + }else if( j>=0 && (pTab->aCol[j].colFlags & COLFLAG_VIRTUAL)!=0 ){ + pExpr = sqlite3ColumnExpr(pTab, &pTab->aCol[j]); + bMaybeNullRow = 0; + }else{ + continue; + } + if( sqlite3ExprIsConstant(pExpr) ) continue; + p = sqlite3DbMallocRaw(pParse->db, sizeof(IndexedExpr)); + if( p==0 ) break; + p->pIENext = pParse->pIdxExpr; + p->pExpr = sqlite3ExprDup(pParse->db, pExpr, 0); + p->iDataCur = pTabItem->iCursor; + p->iIdxCur = iIdxCur; + p->iIdxCol = i; + p->bMaybeNullRow = bMaybeNullRow; +#ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS + p->zIdxName = pIdx->zName; +#endif + pParse->pIdxExpr = p; + if( p->pIENext==0 ){ + sqlite3ParserAddCleanup(pParse, whereIndexedExprCleanup, pParse); + } + } +} + /* ** Generate the beginning of the loop used for WHERE clause processing. ** The return value is a pointer to an opaque structure that contains @@ -159379,7 +161180,7 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( Expr *pWhere, /* The WHERE clause */ ExprList *pOrderBy, /* An ORDER BY (or GROUP BY) clause, or NULL */ ExprList *pResultSet, /* Query result set. Req'd for DISTINCT */ - Select *pLimit, /* Use this LIMIT/OFFSET clause, if any */ + Select *pSelect, /* The entire SELECT statement */ u16 wctrlFlags, /* The WHERE_* flags defined in sqliteInt.h */ int iAuxArg /* If WHERE_OR_SUBCLAUSE is set, index cursor number ** If WHERE_USE_LIMIT, then the limit amount */ @@ -159448,7 +161249,9 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( pWInfo->pParse = pParse; pWInfo->pTabList = pTabList; pWInfo->pOrderBy = pOrderBy; +#if WHERETRACE_ENABLED pWInfo->pWhere = pWhere; +#endif pWInfo->pResultSet = pResultSet; pWInfo->aiCurOnePass[0] = pWInfo->aiCurOnePass[1] = -1; pWInfo->nLevel = nTabList; @@ -159456,9 +161259,7 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( pWInfo->wctrlFlags = wctrlFlags; pWInfo->iLimit = iAuxArg; pWInfo->savedNQueryLoop = pParse->nQueryLoop; -#ifndef SQLITE_OMIT_VIRTUALTABLE - pWInfo->pLimit = pLimit; -#endif + pWInfo->pSelect = pSelect; memset(&pWInfo->nOBSat, 0, offsetof(WhereInfo,sWC) - offsetof(WhereInfo,nOBSat)); memset(&pWInfo->a[0], 0, sizeof(WhereLoop)+nTabList*sizeof(WhereLevel)); @@ -159527,7 +161328,9 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( /* Analyze all of the subexpressions. */ sqlite3WhereExprAnalyze(pTabList, &pWInfo->sWC); - sqlite3WhereAddLimit(&pWInfo->sWC, pLimit); + if( pSelect && pSelect->pLimit ){ + sqlite3WhereAddLimit(&pWInfo->sWC, pSelect); + } if( pParse->nErr ) goto whereBeginError; /* Special case: WHERE terms that do not refer to any tables in the join @@ -159830,6 +161633,9 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( op = OP_ReopenIdx; }else{ iIndexCur = pParse->nTab++; + if( pIx->bHasExpr && OptimizationEnabled(db, SQLITE_IndexedExpr) ){ + whereAddIndexedExpr(pParse, pIx, iIndexCur, pTabItem); + } } pLevel->iIdxCur = iIndexCur; assert( pIx!=0 ); @@ -159952,8 +161758,6 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( /* Jump here if malloc fails */ whereBeginError: if( pWInfo ){ - testcase( pWInfo->pExprMods!=0 ); - whereUndoExprMods(pWInfo); pParse->nQueryLoop = pWInfo->savedNQueryLoop; whereInfoFree(db, pWInfo); } @@ -160172,7 +161976,6 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ } assert( pWInfo->nLevel<=pTabList->nSrc ); - if( pWInfo->pExprMods ) whereUndoExprMods(pWInfo); for(i=0, pLevel=pWInfo->a; inLevel; i++, pLevel++){ int k, last; VdbeOp *pOp, *pLastOp; @@ -160226,6 +162029,16 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ }else{ last = pWInfo->iEndWhere; } + if( pIdx->bHasExpr ){ + IndexedExpr *p = pParse->pIdxExpr; + while( p ){ + if( p->iIdxCur==pLevel->iIdxCur ){ + p->iDataCur = -1; + p->iIdxCur = -1; + } + p = p->pIENext; + } + } k = pLevel->addrBody + 1; #ifdef SQLITE_DEBUG if( db->flags & SQLITE_VdbeAddopTrace ){ @@ -161219,7 +163032,6 @@ static ExprList *exprListAppendList( for(i=0; inExpr; i++){ sqlite3 *db = pParse->db; Expr *pDup = sqlite3ExprDup(db, pAppend->a[i].pExpr, 0); - assert( pDup==0 || !ExprHasProperty(pDup, EP_MemToken) ); if( db->mallocFailed ){ sqlite3ExprDelete(db, pDup); break; @@ -162490,10 +164302,9 @@ static void windowCodeRangeTest( /* This block runs if reg1 is not NULL, but reg2 is. */ sqlite3VdbeJumpHere(v, addr); - sqlite3VdbeAddOp2(v, OP_IsNull, reg2, lbl); VdbeCoverage(v); - if( op==OP_Gt || op==OP_Ge ){ - sqlite3VdbeChangeP2(v, -1, addrDone); - } + sqlite3VdbeAddOp2(v, OP_IsNull, reg2, + (op==OP_Gt || op==OP_Ge) ? addrDone : lbl); + VdbeCoverage(v); } /* Register reg1 currently contains csr1.peerVal (the peer-value from csr1). @@ -170070,7 +171881,7 @@ SQLITE_PRIVATE int sqlite3RunParser(Parse *pParse, const char *zSql){ if( pParse->pNewTrigger && !IN_RENAME_OBJECT ){ sqlite3DeleteTrigger(db, pParse->pNewTrigger); } - if( pParse->pVList ) sqlite3DbFreeNN(db, pParse->pVList); + if( pParse->pVList ) sqlite3DbNNFreeNN(db, pParse->pVList); db->pParse = pParentParse; assert( nErr==0 || pParse->rc!=SQLITE_OK ); return nErr; @@ -171426,18 +173237,19 @@ static int setupLookaside(sqlite3 *db, void *pBuf, int sz, int cnt){ db->lookaside.bMalloced = pBuf==0 ?1:0; db->lookaside.nSlot = nBig+nSm; }else{ - db->lookaside.pStart = db; + db->lookaside.pStart = 0; #ifndef SQLITE_OMIT_TWOSIZE_LOOKASIDE db->lookaside.pSmallInit = 0; db->lookaside.pSmallFree = 0; - db->lookaside.pMiddle = db; + db->lookaside.pMiddle = 0; #endif /* SQLITE_OMIT_TWOSIZE_LOOKASIDE */ - db->lookaside.pEnd = db; + db->lookaside.pEnd = 0; db->lookaside.bDisable = 1; db->lookaside.sz = 0; db->lookaside.bMalloced = 0; db->lookaside.nSlot = 0; } + db->lookaside.pTrueEnd = db->lookaside.pEnd; assert( sqlite3LookasideUsed(db,0)==0 ); #endif /* SQLITE_OMIT_LOOKASIDE */ return SQLITE_OK; @@ -171516,6 +173328,7 @@ SQLITE_API int sqlite3_db_cacheflush(sqlite3 *db){ SQLITE_API int sqlite3_db_config(sqlite3 *db, int op, ...){ va_list ap; int rc; + sqlite3_mutex_enter(db->mutex); va_start(ap, op); switch( op ){ case SQLITE_DBCONFIG_MAINDBNAME: { @@ -171581,6 +173394,7 @@ SQLITE_API int sqlite3_db_config(sqlite3 *db, int op, ...){ } } va_end(ap); + sqlite3_mutex_leave(db->mutex); return rc; } @@ -173951,6 +175765,19 @@ static int openDatabase( goto opendb_out; } +#if SQLITE_OS_UNIX && defined(SQLITE_OS_KV_OPTIONAL) + /* Process magic filenames ":localStorage:" and ":sessionStorage:" */ + if( zFilename && zFilename[0]==':' ){ + if( strcmp(zFilename, ":localStorage:")==0 ){ + zFilename = "file:local?vfs=kvvfs"; + flags |= SQLITE_OPEN_URI; + }else if( strcmp(zFilename, ":sessionStorage:")==0 ){ + zFilename = "file:session?vfs=kvvfs"; + flags |= SQLITE_OPEN_URI; + } + } +#endif /* SQLITE_OS_UNIX && defined(SQLITE_OS_KV_OPTIONAL) */ + /* Parse the filename/URI argument ** ** Only allow sensible combinations of bits in the flags argument. @@ -173981,6 +175808,12 @@ static int openDatabase( sqlite3_free(zErrMsg); goto opendb_out; } + assert( db->pVfs!=0 ); +#if SQLITE_OS_KV || defined(SQLITE_OS_KV_OPTIONAL) + if( sqlite3_stricmp(db->pVfs->zName, "kvvfs")==0 ){ + db->temp_store = 2; + } +#endif /* Open the backend database driver */ rc = sqlite3BtreeOpen(db->pVfs, zOpen, db, &db->aDb[0].pBt, 0, @@ -175090,7 +176923,7 @@ static char *appendText(char *p, const char *z){ ** Memory layout must be compatible with that generated by the pager ** and expected by sqlite3_uri_parameter() and databaseName(). */ -SQLITE_API char *sqlite3_create_filename( +SQLITE_API const char *sqlite3_create_filename( const char *zDatabase, const char *zJournal, const char *zWal, @@ -175126,10 +176959,10 @@ SQLITE_API char *sqlite3_create_filename( ** error to call this routine with any parameter other than a pointer ** previously obtained from sqlite3_create_filename() or a NULL pointer. */ -SQLITE_API void sqlite3_free_filename(char *p){ +SQLITE_API void sqlite3_free_filename(const char *p){ if( p==0 ) return; - p = (char*)databaseName(p); - sqlite3_free(p - 4); + p = databaseName(p); + sqlite3_free((char*)p - 4); } @@ -175380,8 +177213,8 @@ SQLITE_API int sqlite3_snapshot_open( */ SQLITE_API int sqlite3_snapshot_recover(sqlite3 *db, const char *zDb){ int rc = SQLITE_ERROR; - int iDb; #ifndef SQLITE_OMIT_WAL + int iDb; #ifdef SQLITE_ENABLE_API_ARMOR if( !sqlite3SafetyCheckOk(db) ){ @@ -176936,7 +178769,7 @@ struct Fts3MultiSegReader { int nAdvance; /* How many seg-readers to advance */ Fts3SegFilter *pFilter; /* Pointer to filter object */ char *aBuffer; /* Buffer to merge doclists in */ - int nBuffer; /* Allocated size of aBuffer[] in bytes */ + i64 nBuffer; /* Allocated size of aBuffer[] in bytes */ int iColFilter; /* If >=0, filter for this column */ int bRestart; @@ -179632,7 +181465,7 @@ static int fts3TermSelectMerge( ** ** Similar padding is added in the fts3DoclistOrMerge() function. */ - pTS->aaOutput[0] = sqlite3_malloc(nDoclist + FTS3_VARINT_MAX + 1); + pTS->aaOutput[0] = sqlite3_malloc64((i64)nDoclist + FTS3_VARINT_MAX + 1); pTS->anOutput[0] = nDoclist; if( pTS->aaOutput[0] ){ memcpy(pTS->aaOutput[0], aDoclist, nDoclist); @@ -181120,7 +182953,7 @@ static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){ nDistance = iPrev - nMaxUndeferred; } - aOut = (char *)sqlite3_malloc(nPoslist+8); + aOut = (char *)sqlite3Fts3MallocZero(nPoslist+FTS3_BUFFER_PADDING); if( !aOut ){ sqlite3_free(aPoslist); return SQLITE_NOMEM; @@ -181489,7 +183322,7 @@ static int fts3EvalIncrPhraseNext( if( bEof==0 ){ int nList = 0; int nByte = a[p->nToken-1].nList; - char *aDoclist = sqlite3_malloc(nByte+FTS3_BUFFER_PADDING); + char *aDoclist = sqlite3_malloc64((i64)nByte+FTS3_BUFFER_PADDING); if( !aDoclist ) return SQLITE_NOMEM; memcpy(aDoclist, a[p->nToken-1].pList, nByte+1); memset(&aDoclist[nByte], 0, FTS3_BUFFER_PADDING); @@ -185725,7 +187558,7 @@ static int porterNext( if( n>c->nAllocated ){ char *pNew; c->nAllocated = n+20; - pNew = sqlite3_realloc(c->zToken, c->nAllocated); + pNew = sqlite3_realloc64(c->zToken, c->nAllocated); if( !pNew ) return SQLITE_NOMEM; c->zToken = pNew; } @@ -186477,7 +188310,7 @@ static int simpleNext( if( n>c->nTokenAllocated ){ char *pNew; c->nTokenAllocated = n+20; - pNew = sqlite3_realloc(c->pToken, c->nTokenAllocated); + pNew = sqlite3_realloc64(c->pToken, c->nTokenAllocated); if( !pNew ) return SQLITE_NOMEM; c->pToken = pNew; } @@ -187639,7 +189472,7 @@ static int fts3PendingListAppendVarint( /* Allocate or grow the PendingList as required. */ if( !p ){ - p = sqlite3_malloc(sizeof(*p) + 100); + p = sqlite3_malloc64(sizeof(*p) + 100); if( !p ){ return SQLITE_NOMEM; } @@ -187648,14 +189481,14 @@ static int fts3PendingListAppendVarint( p->nData = 0; } else if( p->nData+FTS3_VARINT_MAX+1>p->nSpace ){ - int nNew = p->nSpace * 2; - p = sqlite3_realloc(p, sizeof(*p) + nNew); + i64 nNew = p->nSpace * 2; + p = sqlite3_realloc64(p, sizeof(*p) + nNew); if( !p ){ sqlite3_free(*pp); *pp = 0; return SQLITE_NOMEM; } - p->nSpace = nNew; + p->nSpace = (int)nNew; p->aData = (char *)&p[1]; } @@ -188212,7 +190045,7 @@ SQLITE_PRIVATE int sqlite3Fts3ReadBlock( int nByte = sqlite3_blob_bytes(p->pSegments); *pnBlob = nByte; if( paBlob ){ - char *aByte = sqlite3_malloc(nByte + FTS3_NODE_PADDING); + char *aByte = sqlite3_malloc64((i64)nByte + FTS3_NODE_PADDING); if( !aByte ){ rc = SQLITE_NOMEM; }else{ @@ -188329,7 +190162,7 @@ static int fts3SegReaderNext( int nTerm = fts3HashKeysize(pElem); if( (nTerm+1)>pReader->nTermAlloc ){ sqlite3_free(pReader->zTerm); - pReader->zTerm = (char*)sqlite3_malloc((nTerm+1)*2); + pReader->zTerm = (char*)sqlite3_malloc64(((i64)nTerm+1)*2); if( !pReader->zTerm ) return SQLITE_NOMEM; pReader->nTermAlloc = (nTerm+1)*2; } @@ -188337,7 +190170,7 @@ static int fts3SegReaderNext( pReader->zTerm[nTerm] = '\0'; pReader->nTerm = nTerm; - aCopy = (char*)sqlite3_malloc(nCopy); + aCopy = (char*)sqlite3_malloc64(nCopy); if( !aCopy ) return SQLITE_NOMEM; memcpy(aCopy, pList->aData, nCopy); pReader->nNode = pReader->nDoclist = nCopy; @@ -188624,7 +190457,7 @@ SQLITE_PRIVATE int sqlite3Fts3SegReaderNew( nExtra = nRoot + FTS3_NODE_PADDING; } - pReader = (Fts3SegReader *)sqlite3_malloc(sizeof(Fts3SegReader) + nExtra); + pReader = (Fts3SegReader *)sqlite3_malloc64(sizeof(Fts3SegReader) + nExtra); if( !pReader ){ return SQLITE_NOMEM; } @@ -188716,7 +190549,7 @@ SQLITE_PRIVATE int sqlite3Fts3SegReaderPending( if( nElem==nAlloc ){ Fts3HashElem **aElem2; nAlloc += 16; - aElem2 = (Fts3HashElem **)sqlite3_realloc( + aElem2 = (Fts3HashElem **)sqlite3_realloc64( aElem, nAlloc*sizeof(Fts3HashElem *) ); if( !aElem2 ){ @@ -189050,7 +190883,7 @@ static int fts3NodeAddTerm( ** this is not expected to be a serious problem. */ assert( pTree->aData==(char *)&pTree[1] ); - pTree->aData = (char *)sqlite3_malloc(nReq); + pTree->aData = (char *)sqlite3_malloc64(nReq); if( !pTree->aData ){ return SQLITE_NOMEM; } @@ -189068,7 +190901,7 @@ static int fts3NodeAddTerm( if( isCopyTerm ){ if( pTree->nMalloczMalloc, nTerm*2); + char *zNew = sqlite3_realloc64(pTree->zMalloc, (i64)nTerm*2); if( !zNew ){ return SQLITE_NOMEM; } @@ -189094,7 +190927,7 @@ static int fts3NodeAddTerm( ** now. Instead, the term is inserted into the parent of pTree. If pTree ** has no parent, one is created here. */ - pNew = (SegmentNode *)sqlite3_malloc(sizeof(SegmentNode) + p->nNodeSize); + pNew = (SegmentNode *)sqlite3_malloc64(sizeof(SegmentNode) + p->nNodeSize); if( !pNew ){ return SQLITE_NOMEM; } @@ -189232,7 +191065,7 @@ static int fts3SegWriterAdd( ){ int nPrefix; /* Size of term prefix in bytes */ int nSuffix; /* Size of term suffix in bytes */ - int nReq; /* Number of bytes required on leaf page */ + i64 nReq; /* Number of bytes required on leaf page */ int nData; SegmentWriter *pWriter = *ppWriter; @@ -189241,13 +191074,13 @@ static int fts3SegWriterAdd( sqlite3_stmt *pStmt; /* Allocate the SegmentWriter structure */ - pWriter = (SegmentWriter *)sqlite3_malloc(sizeof(SegmentWriter)); + pWriter = (SegmentWriter *)sqlite3_malloc64(sizeof(SegmentWriter)); if( !pWriter ) return SQLITE_NOMEM; memset(pWriter, 0, sizeof(SegmentWriter)); *ppWriter = pWriter; /* Allocate a buffer in which to accumulate data */ - pWriter->aData = (char *)sqlite3_malloc(p->nNodeSize); + pWriter->aData = (char *)sqlite3_malloc64(p->nNodeSize); if( !pWriter->aData ) return SQLITE_NOMEM; pWriter->nSize = p->nNodeSize; @@ -189322,7 +191155,7 @@ static int fts3SegWriterAdd( ** the buffer to make it large enough. */ if( nReq>pWriter->nSize ){ - char *aNew = sqlite3_realloc(pWriter->aData, nReq); + char *aNew = sqlite3_realloc64(pWriter->aData, nReq); if( !aNew ) return SQLITE_NOMEM; pWriter->aData = aNew; pWriter->nSize = nReq; @@ -189347,7 +191180,7 @@ static int fts3SegWriterAdd( */ if( isCopyTerm ){ if( nTerm>pWriter->nMalloc ){ - char *zNew = sqlite3_realloc(pWriter->zMalloc, nTerm*2); + char *zNew = sqlite3_realloc64(pWriter->zMalloc, (i64)nTerm*2); if( !zNew ){ return SQLITE_NOMEM; } @@ -189655,12 +191488,12 @@ static void fts3ColumnFilter( static int fts3MsrBufferData( Fts3MultiSegReader *pMsr, /* Multi-segment-reader handle */ char *pList, - int nList + i64 nList ){ if( nList>pMsr->nBuffer ){ char *pNew; pMsr->nBuffer = nList*2; - pNew = (char *)sqlite3_realloc(pMsr->aBuffer, pMsr->nBuffer); + pNew = (char *)sqlite3_realloc64(pMsr->aBuffer, pMsr->nBuffer); if( !pNew ) return SQLITE_NOMEM; pMsr->aBuffer = pNew; } @@ -189716,7 +191549,7 @@ SQLITE_PRIVATE int sqlite3Fts3MsrIncrNext( fts3SegReaderSort(pMsr->apSegment, nMerge, j, xCmp); if( nList>0 && fts3SegReaderIsPending(apSegment[0]) ){ - rc = fts3MsrBufferData(pMsr, pList, nList+1); + rc = fts3MsrBufferData(pMsr, pList, (i64)nList+1); if( rc!=SQLITE_OK ) return rc; assert( (pMsr->aBuffer[nList] & 0xFE)==0x00 ); pList = pMsr->aBuffer; @@ -189853,11 +191686,11 @@ SQLITE_PRIVATE int sqlite3Fts3MsrIncrRestart(Fts3MultiSegReader *pCsr){ return SQLITE_OK; } -static int fts3GrowSegReaderBuffer(Fts3MultiSegReader *pCsr, int nReq){ +static int fts3GrowSegReaderBuffer(Fts3MultiSegReader *pCsr, i64 nReq){ if( nReq>pCsr->nBuffer ){ char *aNew; pCsr->nBuffer = nReq*2; - aNew = sqlite3_realloc(pCsr->aBuffer, pCsr->nBuffer); + aNew = sqlite3_realloc64(pCsr->aBuffer, pCsr->nBuffer); if( !aNew ){ return SQLITE_NOMEM; } @@ -189948,7 +191781,8 @@ SQLITE_PRIVATE int sqlite3Fts3SegReaderStep( ){ pCsr->nDoclist = apSegment[0]->nDoclist; if( fts3SegReaderIsPending(apSegment[0]) ){ - rc = fts3MsrBufferData(pCsr, apSegment[0]->aDoclist, pCsr->nDoclist); + rc = fts3MsrBufferData(pCsr, apSegment[0]->aDoclist, + (i64)pCsr->nDoclist); pCsr->aDoclist = pCsr->aBuffer; }else{ pCsr->aDoclist = apSegment[0]->aDoclist; @@ -190001,7 +191835,8 @@ SQLITE_PRIVATE int sqlite3Fts3SegReaderStep( nByte = sqlite3Fts3VarintLen(iDelta) + (isRequirePos?nList+1:0); - rc = fts3GrowSegReaderBuffer(pCsr, nByte+nDoclist+FTS3_NODE_PADDING); + rc = fts3GrowSegReaderBuffer(pCsr, + (i64)nByte+nDoclist+FTS3_NODE_PADDING); if( rc ) return rc; if( isFirst ){ @@ -190027,7 +191862,7 @@ SQLITE_PRIVATE int sqlite3Fts3SegReaderStep( fts3SegReaderSort(apSegment, nMerge, j, xCmp); } if( nDoclist>0 ){ - rc = fts3GrowSegReaderBuffer(pCsr, nDoclist+FTS3_NODE_PADDING); + rc = fts3GrowSegReaderBuffer(pCsr, (i64)nDoclist+FTS3_NODE_PADDING); if( rc ) return rc; memset(&pCsr->aBuffer[nDoclist], 0, FTS3_NODE_PADDING); pCsr->aDoclist = pCsr->aBuffer; @@ -190740,7 +192575,7 @@ struct NodeReader { static void blobGrowBuffer(Blob *pBlob, int nMin, int *pRc){ if( *pRc==SQLITE_OK && nMin>pBlob->nAlloc ){ int nAlloc = nMin; - char *a = (char *)sqlite3_realloc(pBlob->a, nAlloc); + char *a = (char *)sqlite3_realloc64(pBlob->a, nAlloc); if( a ){ pBlob->nAlloc = nAlloc; pBlob->a = a; @@ -191537,7 +193372,7 @@ static int fts3RepackSegdirLevel( if( nIdx>=nAlloc ){ int *aNew; nAlloc += 16; - aNew = sqlite3_realloc(aIdx, nAlloc*sizeof(int)); + aNew = sqlite3_realloc64(aIdx, nAlloc*sizeof(int)); if( !aNew ){ rc = SQLITE_NOMEM; break; @@ -191911,7 +193746,7 @@ SQLITE_PRIVATE int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ /* Allocate space for the cursor, filter and writer objects */ const int nAlloc = sizeof(*pCsr) + sizeof(*pFilter) + sizeof(*pWriter); - pWriter = (IncrmergeWriter *)sqlite3_malloc(nAlloc); + pWriter = (IncrmergeWriter *)sqlite3_malloc64(nAlloc); if( !pWriter ) return SQLITE_NOMEM; pFilter = (Fts3SegFilter *)&pWriter[1]; pCsr = (Fts3MultiSegReader *)&pFilter[1]; @@ -192547,7 +194382,7 @@ SQLITE_PRIVATE int sqlite3Fts3DeferredTokenList( return SQLITE_OK; } - pRet = (char *)sqlite3_malloc(p->pList->nData); + pRet = (char *)sqlite3_malloc64(p->pList->nData); if( !pRet ) return SQLITE_NOMEM; nSkip = sqlite3Fts3GetVarint(p->pList->aData, &dummy); @@ -192567,7 +194402,7 @@ SQLITE_PRIVATE int sqlite3Fts3DeferToken( int iCol /* Column that token must appear in (or -1) */ ){ Fts3DeferredToken *pDeferred; - pDeferred = sqlite3_malloc(sizeof(*pDeferred)); + pDeferred = sqlite3_malloc64(sizeof(*pDeferred)); if( !pDeferred ){ return SQLITE_NOMEM; } @@ -201282,7 +203117,7 @@ static int rtreeUpdate( rtreeReference(pRtree); assert(nData>=1); - cell.iRowid = 0; /* Used only to suppress a compiler warning */ + memset(&cell, 0, sizeof(cell)); /* Constraint handling. A write operation on an r-tree table may return ** SQLITE_CONSTRAINT for two reasons: @@ -204146,7 +205981,7 @@ static int geopolyUpdate( sqlite3_free(p); nChange = 1; } - for(jj=1; jjnAux; jj++){ + for(jj=1; jjdbRbu, "main", SQLITE_FCNTL_RBUCNT, (void*)p); if( p->zState==0 ){ const char *zFile = sqlite3_db_filename(p->dbRbu, "main"); - p->zState = rbuMPrintf(p, "file://%s-vacuum?modeof=%s", zFile, zFile); + p->zState = rbuMPrintf(p, "file:///%s-vacuum?modeof=%s", zFile, zFile); } } @@ -209122,32 +210988,7 @@ static void rbuMoveOalFile(sqlite3rbu *p){ } if( p->rc==SQLITE_OK ){ -#if defined(_WIN32_WCE) - { - LPWSTR zWideOal; - LPWSTR zWideWal; - - zWideOal = rbuWinUtf8ToUnicode(zOal); - if( zWideOal ){ - zWideWal = rbuWinUtf8ToUnicode(zWal); - if( zWideWal ){ - if( MoveFileW(zWideOal, zWideWal) ){ - p->rc = SQLITE_OK; - }else{ - p->rc = SQLITE_IOERR; - } - sqlite3_free(zWideWal); - }else{ - p->rc = SQLITE_IOERR_NOMEM; - } - sqlite3_free(zWideOal); - }else{ - p->rc = SQLITE_IOERR_NOMEM; - } - } -#else - p->rc = rename(zOal, zWal) ? SQLITE_IOERR : SQLITE_OK; -#endif + p->rc = p->xRename(p->pRenameArg, zOal, zWal); } if( p->rc!=SQLITE_OK @@ -209886,6 +211727,7 @@ static sqlite3rbu *openRbuHandle( /* Create the custom VFS. */ memset(p, 0, sizeof(sqlite3rbu)); + sqlite3rbu_rename_handler(p, 0, 0); rbuCreateVfs(p); /* Open the target, RBU and state databases */ @@ -210277,6 +212119,54 @@ SQLITE_API int sqlite3rbu_savestate(sqlite3rbu *p){ return rc; } +/* +** Default xRename callback for RBU. +*/ +static int xDefaultRename(void *pArg, const char *zOld, const char *zNew){ + int rc = SQLITE_OK; +#if defined(_WIN32_WCE) + { + LPWSTR zWideOld; + LPWSTR zWideNew; + + zWideOld = rbuWinUtf8ToUnicode(zOld); + if( zWideOld ){ + zWideNew = rbuWinUtf8ToUnicode(zNew); + if( zWideNew ){ + if( MoveFileW(zWideOld, zWideNew) ){ + rc = SQLITE_OK; + }else{ + rc = SQLITE_IOERR; + } + sqlite3_free(zWideNew); + }else{ + rc = SQLITE_IOERR_NOMEM; + } + sqlite3_free(zWideOld); + }else{ + rc = SQLITE_IOERR_NOMEM; + } + } +#else + rc = rename(zOld, zNew) ? SQLITE_IOERR : SQLITE_OK; +#endif + return rc; +} + +SQLITE_API void sqlite3rbu_rename_handler( + sqlite3rbu *pRbu, + void *pArg, + int (*xRename)(void *pArg, const char *zOld, const char *zNew) +){ + if( xRename ){ + pRbu->xRename = xRename; + pRbu->pRenameArg = pArg; + }else{ + pRbu->xRename = xDefaultRename; + pRbu->pRenameArg = 0; + } +} + /************************************************************************** ** Beginning of RBU VFS shim methods. The VFS shim modifies the behaviour ** of a standard VFS in the following ways: @@ -212407,12 +214297,18 @@ static int dbpageColumn( } case 1: { /* data */ DbPage *pDbPage = 0; - rc = sqlite3PagerGet(pCsr->pPager, pCsr->pgno, (DbPage**)&pDbPage, 0); - if( rc==SQLITE_OK ){ - sqlite3_result_blob(ctx, sqlite3PagerGetData(pDbPage), pCsr->szPage, - SQLITE_TRANSIENT); + if( pCsr->pgno==((PENDING_BYTE/pCsr->szPage)+1) ){ + /* The pending byte page. Assume it is zeroed out. Attempting to + ** request this page from the page is an SQLITE_CORRUPT error. */ + sqlite3_result_zeroblob(ctx, pCsr->szPage); + }else{ + rc = sqlite3PagerGet(pCsr->pPager, pCsr->pgno, (DbPage**)&pDbPage, 0); + if( rc==SQLITE_OK ){ + sqlite3_result_blob(ctx, sqlite3PagerGetData(pDbPage), pCsr->szPage, + SQLITE_TRANSIENT); + } + sqlite3PagerUnref(pDbPage); } - sqlite3PagerUnref(pDbPage); break; } default: { /* schema */ @@ -212421,7 +214317,7 @@ static int dbpageColumn( break; } } - return SQLITE_OK; + return rc; } static int dbpageRowid(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){ @@ -212481,11 +214377,12 @@ static int dbpageUpdate( pPager = sqlite3BtreePager(pBt); rc = sqlite3PagerGet(pPager, pgno, (DbPage**)&pDbPage, 0); if( rc==SQLITE_OK ){ - rc = sqlite3PagerWrite(pDbPage); - if( rc==SQLITE_OK ){ - memcpy(sqlite3PagerGetData(pDbPage), - sqlite3_value_blob(argv[3]), - szPage); + const void *pData = sqlite3_value_blob(argv[3]); + assert( pData!=0 || pTab->db->mallocFailed ); + if( pData + && (rc = sqlite3PagerWrite(pDbPage))==SQLITE_OK + ){ + memcpy(sqlite3PagerGetData(pDbPage), pData, szPage); } } sqlite3PagerUnref(pDbPage); @@ -212505,11 +214402,12 @@ static int dbpageBegin(sqlite3_vtab *pVtab){ DbpageTable *pTab = (DbpageTable *)pVtab; sqlite3 *db = pTab->db; int i; - for(i=0; inDb; i++){ + int rc = SQLITE_OK; + for(i=0; rc==SQLITE_OK && inDb; i++){ Btree *pBt = db->aDb[i].pBt; - if( pBt ) sqlite3BtreeBeginTrans(pBt, 1, 0); + if( pBt ) rc = sqlite3BtreeBeginTrans(pBt, 1, 0); } - return SQLITE_OK; + return rc; } @@ -219233,7 +221131,7 @@ static void sqlite3Fts5BufferAppendPrintf(int *, Fts5Buffer*, char *zFmt, ...); static char *sqlite3Fts5Mprintf(int *pRc, const char *zFmt, ...); #define fts5BufferZero(x) sqlite3Fts5BufferZero(x) -#define fts5BufferAppendVarint(a,b,c) sqlite3Fts5BufferAppendVarint(a,b,c) +#define fts5BufferAppendVarint(a,b,c) sqlite3Fts5BufferAppendVarint(a,b,(i64)c) #define fts5BufferFree(a) sqlite3Fts5BufferFree(a) #define fts5BufferAppendBlob(a,b,c,d) sqlite3Fts5BufferAppendBlob(a,b,c,d) #define fts5BufferSet(a,b,c,d) sqlite3Fts5BufferSet(a,b,c,d) @@ -231110,7 +233008,9 @@ static void fts5WriteAppendRowid( fts5BufferAppendVarint(&p->rc, &pPage->buf, iRowid); }else{ assert_nc( p->rc || iRowid>pWriter->iPrevRowid ); - fts5BufferAppendVarint(&p->rc, &pPage->buf, iRowid - pWriter->iPrevRowid); + fts5BufferAppendVarint(&p->rc, &pPage->buf, + (u64)iRowid - (u64)pWriter->iPrevRowid + ); } pWriter->iPrevRowid = iRowid; pWriter->bFirstRowidInDoclist = 0; @@ -231874,7 +233774,7 @@ static int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge){ static void fts5AppendRowid( Fts5Index *p, - i64 iDelta, + u64 iDelta, Fts5Iter *pUnused, Fts5Buffer *pBuf ){ @@ -231884,7 +233784,7 @@ static void fts5AppendRowid( static void fts5AppendPoslist( Fts5Index *p, - i64 iDelta, + u64 iDelta, Fts5Iter *pMulti, Fts5Buffer *pBuf ){ @@ -231959,10 +233859,10 @@ static void fts5MergeAppendDocid( } #endif -#define fts5MergeAppendDocid(pBuf, iLastRowid, iRowid) { \ - assert( (pBuf)->n!=0 || (iLastRowid)==0 ); \ - fts5BufferSafeAppendVarint((pBuf), (iRowid) - (iLastRowid)); \ - (iLastRowid) = (iRowid); \ +#define fts5MergeAppendDocid(pBuf, iLastRowid, iRowid) { \ + assert( (pBuf)->n!=0 || (iLastRowid)==0 ); \ + fts5BufferSafeAppendVarint((pBuf), (u64)(iRowid) - (u64)(iLastRowid)); \ + (iLastRowid) = (iRowid); \ } /* @@ -232233,7 +234133,7 @@ static void fts5SetupPrefixIter( int nMerge = 1; void (*xMerge)(Fts5Index*, Fts5Buffer*, int, Fts5Buffer*); - void (*xAppend)(Fts5Index*, i64, Fts5Iter*, Fts5Buffer*); + void (*xAppend)(Fts5Index*, u64, Fts5Iter*, Fts5Buffer*); if( p->pConfig->eDetail==FTS5_DETAIL_NONE ){ xMerge = fts5MergeRowidLists; xAppend = fts5AppendRowid; @@ -232272,7 +234172,7 @@ static void fts5SetupPrefixIter( Fts5SegIter *pSeg = &p1->aSeg[ p1->aFirst[1].iFirst ]; p1->xSetOutputs(p1, pSeg); if( p1->base.nData ){ - xAppend(p, p1->base.iRowid-iLastRowid, p1, &doclist); + xAppend(p, (u64)p1->base.iRowid-(u64)iLastRowid, p1, &doclist); iLastRowid = p1->base.iRowid; } } @@ -232320,7 +234220,7 @@ static void fts5SetupPrefixIter( iLastRowid = 0; } - xAppend(p, p1->base.iRowid-iLastRowid, p1, &doclist); + xAppend(p, (u64)p1->base.iRowid-(u64)iLastRowid, p1, &doclist); iLastRowid = p1->base.iRowid; } @@ -233299,6 +235199,7 @@ static int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum, int bUseCksum /* If this is a new term, query for it. Update cksum3 with the results. */ fts5TestTerm(p, &term, z, n, cksum2, &cksum3); + if( p->rc ) break; if( eDetail==FTS5_DETAIL_NONE ){ if( 0==fts5MultiIterIsEmpty(p, pIter) ){ @@ -236636,7 +238537,7 @@ static void fts5SourceIdFunc( ){ assert( nArg==0 ); UNUSED_PARAM2(nArg, apUnused); - sqlite3_result_text(pCtx, "fts5: 2022-07-21 15:24:47 698edb77537b67c41adc68f9b892db56bcf9a55e00371a61420f3ddd668e6603", -1, SQLITE_TRANSIENT); + sqlite3_result_text(pCtx, "fts5: 2022-11-16 12:10:08 89c459e766ea7e9165d0beeb124708b955a4950d0f4792f457465d71b158d318", -1, SQLITE_TRANSIENT); } /* @@ -241616,4 +243517,4 @@ SQLITE_API int sqlite3_stmt_init( /************** End of stmt.c ************************************************/ /* Return the source-id for this library */ SQLITE_API const char *sqlite3_sourceid(void){ return SQLITE_SOURCE_ID; } -/************************** End of sqlite3.c ******************************/ +/************************** End of sqlite3.c ******************************/ \ No newline at end of file diff --git a/src/database/sqlite3.h b/src/database/sqlite3.h index f0df724d..c46fd8c5 100644 --- a/src/database/sqlite3.h +++ b/src/database/sqlite3.h @@ -146,9 +146,9 @@ extern "C" { ** [sqlite3_libversion_number()], [sqlite3_sourceid()], ** [sqlite_version()] and [sqlite_source_id()]. */ -#define SQLITE_VERSION "3.39.2" -#define SQLITE_VERSION_NUMBER 3039002 -#define SQLITE_SOURCE_ID "2022-07-21 15:24:47 698edb77537b67c41adc68f9b892db56bcf9a55e00371a61420f3ddd668e6603" +#define SQLITE_VERSION "3.40.0" +#define SQLITE_VERSION_NUMBER 3040000 +#define SQLITE_SOURCE_ID "2022-11-16 12:10:08 89c459e766ea7e9165d0beeb124708b955a4950d0f4792f457465d71b158d318" /* ** CAPI3REF: Run-Time Library Version Numbers @@ -670,13 +670,17 @@ SQLITE_API int sqlite3_exec( ** ** SQLite uses one of these integer values as the second ** argument to calls it makes to the xLock() and xUnlock() methods -** of an [sqlite3_io_methods] object. +** of an [sqlite3_io_methods] object. These values are ordered from +** lest restrictive to most restrictive. +** +** The argument to xLock() is always SHARED or higher. The argument to +** xUnlock is either SHARED or NONE. */ -#define SQLITE_LOCK_NONE 0 -#define SQLITE_LOCK_SHARED 1 -#define SQLITE_LOCK_RESERVED 2 -#define SQLITE_LOCK_PENDING 3 -#define SQLITE_LOCK_EXCLUSIVE 4 +#define SQLITE_LOCK_NONE 0 /* xUnlock() only */ +#define SQLITE_LOCK_SHARED 1 /* xLock() or xUnlock() */ +#define SQLITE_LOCK_RESERVED 2 /* xLock() only */ +#define SQLITE_LOCK_PENDING 3 /* xLock() only */ +#define SQLITE_LOCK_EXCLUSIVE 4 /* xLock() only */ /* ** CAPI3REF: Synchronization Type Flags @@ -754,7 +758,14 @@ struct sqlite3_file { **
    2. [SQLITE_LOCK_PENDING], or **
    3. [SQLITE_LOCK_EXCLUSIVE]. ** -** xLock() increases the lock. xUnlock() decreases the lock. +** xLock() upgrades the database file lock. In other words, xLock() moves the +** database file lock in the direction NONE toward EXCLUSIVE. The argument to +** xLock() is always on of SHARED, RESERVED, PENDING, or EXCLUSIVE, never +** SQLITE_LOCK_NONE. If the database file lock is already at or above the +** requested lock, then the call to xLock() is a no-op. +** xUnlock() downgrades the database file lock to either SHARED or NONE. +* If the lock is already at or below the requested lock state, then the call +** to xUnlock() is a no-op. ** The xCheckReservedLock() method checks whether any database connection, ** either in this process or in some other process, is holding a RESERVED, ** PENDING, or EXCLUSIVE lock on the file. It returns true @@ -859,9 +870,8 @@ struct sqlite3_io_methods { ** opcode causes the xFileControl method to write the current state of ** the lock (one of [SQLITE_LOCK_NONE], [SQLITE_LOCK_SHARED], ** [SQLITE_LOCK_RESERVED], [SQLITE_LOCK_PENDING], or [SQLITE_LOCK_EXCLUSIVE]) -** into an integer that the pArg argument points to. This capability -** is used during testing and is only available when the SQLITE_TEST -** compile-time option is used. +** into an integer that the pArg argument points to. +** This capability is only available if SQLite is compiled with [SQLITE_DEBUG]. ** **
    4. [[SQLITE_FCNTL_SIZE_HINT]] ** The [SQLITE_FCNTL_SIZE_HINT] opcode is used by SQLite to give the VFS @@ -1253,6 +1263,26 @@ typedef struct sqlite3_mutex sqlite3_mutex; */ typedef struct sqlite3_api_routines sqlite3_api_routines; +/* +** CAPI3REF: File Name +** +** Type [sqlite3_filename] is used by SQLite to pass filenames to the +** xOpen method of a [VFS]. It may be cast to (const char*) and treated +** as a normal, nul-terminated, UTF-8 buffer containing the filename, but +** may also be passed to special APIs such as: +** +**
        +**
      • sqlite3_filename_database() +**
      • sqlite3_filename_journal() +**
      • sqlite3_filename_wal() +**
      • sqlite3_uri_parameter() +**
      • sqlite3_uri_boolean() +**
      • sqlite3_uri_int64() +**
      • sqlite3_uri_key() +**
      +*/ +typedef const char *sqlite3_filename; + /* ** CAPI3REF: OS Interface Object ** @@ -1431,7 +1461,7 @@ struct sqlite3_vfs { sqlite3_vfs *pNext; /* Next registered VFS */ const char *zName; /* Name of this virtual file system */ void *pAppData; /* Pointer to application-specific data */ - int (*xOpen)(sqlite3_vfs*, const char *zName, sqlite3_file*, + int (*xOpen)(sqlite3_vfs*, sqlite3_filename zName, sqlite3_file*, int flags, int *pOutFlags); int (*xDelete)(sqlite3_vfs*, const char *zName, int syncDir); int (*xAccess)(sqlite3_vfs*, const char *zName, int flags, int *pResOut); @@ -2309,6 +2339,7 @@ struct sqlite3_mem_methods { **
        **
      • The [PRAGMA writable_schema=ON] statement. **
      • The [PRAGMA journal_mode=OFF] statement. +**
      • The [PRAGMA schema_version=N] statement. **
      • Writes to the [sqlite_dbpage] virtual table. **
      • Direct writes to [shadow tables]. **
      @@ -3424,6 +3455,9 @@ SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); **
      The database is opened [shared cache] enabled, overriding ** the default shared cache setting provided by ** [sqlite3_enable_shared_cache()].)^ +** The [use of shared cache mode is discouraged] and hence shared cache +** capabilities may be omitted from many builds of SQLite. In such cases, +** this option is a no-op. ** ** ^(
      [SQLITE_OPEN_PRIVATECACHE]
      **
      The database is opened [shared cache] disabled, overriding @@ -3439,7 +3473,7 @@ SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); ** to return an extended result code.
      ** ** [[OPEN_NOFOLLOW]] ^(
      [SQLITE_OPEN_NOFOLLOW]
      -**
      The database filename is not allowed to be a symbolic link
      +**
      The database filename is not allowed to contain a symbolic link
      ** )^ ** ** If the 3rd parameter to sqlite3_open_v2() is not one of the @@ -3698,10 +3732,10 @@ SQLITE_API int sqlite3_open_v2( ** ** See the [URI filename] documentation for additional information. */ -SQLITE_API const char *sqlite3_uri_parameter(const char *zFilename, const char *zParam); -SQLITE_API int sqlite3_uri_boolean(const char *zFile, const char *zParam, int bDefault); -SQLITE_API sqlite3_int64 sqlite3_uri_int64(const char*, const char*, sqlite3_int64); -SQLITE_API const char *sqlite3_uri_key(const char *zFilename, int N); +SQLITE_API const char *sqlite3_uri_parameter(sqlite3_filename z, const char *zParam); +SQLITE_API int sqlite3_uri_boolean(sqlite3_filename z, const char *zParam, int bDefault); +SQLITE_API sqlite3_int64 sqlite3_uri_int64(sqlite3_filename, const char*, sqlite3_int64); +SQLITE_API const char *sqlite3_uri_key(sqlite3_filename z, int N); /* ** CAPI3REF: Translate filenames @@ -3730,9 +3764,9 @@ SQLITE_API const char *sqlite3_uri_key(const char *zFilename, int N); ** return value from [sqlite3_db_filename()], then the result is ** undefined and is likely a memory access violation. */ -SQLITE_API const char *sqlite3_filename_database(const char*); -SQLITE_API const char *sqlite3_filename_journal(const char*); -SQLITE_API const char *sqlite3_filename_wal(const char*); +SQLITE_API const char *sqlite3_filename_database(sqlite3_filename); +SQLITE_API const char *sqlite3_filename_journal(sqlite3_filename); +SQLITE_API const char *sqlite3_filename_wal(sqlite3_filename); /* ** CAPI3REF: Database File Corresponding To A Journal @@ -3798,14 +3832,14 @@ SQLITE_API sqlite3_file *sqlite3_database_file_object(const char*); ** then the corresponding [sqlite3_module.xClose() method should also be ** invoked prior to calling sqlite3_free_filename(Y). */ -SQLITE_API char *sqlite3_create_filename( +SQLITE_API sqlite3_filename sqlite3_create_filename( const char *zDatabase, const char *zJournal, const char *zWal, int nParam, const char **azParam ); -SQLITE_API void sqlite3_free_filename(char*); +SQLITE_API void sqlite3_free_filename(sqlite3_filename); /* ** CAPI3REF: Error Codes And Messages @@ -5508,6 +5542,16 @@ SQLITE_API SQLITE_DEPRECATED int sqlite3_memory_alarm(void(*)(void*,sqlite3_int6 ** then the conversion is performed. Otherwise no conversion occurs. ** The [SQLITE_INTEGER | datatype] after conversion is returned.)^ ** +** ^(The sqlite3_value_encoding(X) interface returns one of [SQLITE_UTF8], +** [SQLITE_UTF16BE], or [SQLITE_UTF16LE] according to the current encoding +** of the value X, assuming that X has type TEXT.)^ If sqlite3_value_type(X) +** returns something other than SQLITE_TEXT, then the return value from +** sqlite3_value_encoding(X) is meaningless. ^Calls to +** sqlite3_value_text(X), sqlite3_value_text16(X), sqlite3_value_text16be(X), +** sqlite3_value_text16le(X), sqlite3_value_bytes(X), or +** sqlite3_value_bytes16(X) might change the encoding of the value X and +** thus change the return from subsequent calls to sqlite3_value_encoding(X). +** ** ^Within the [xUpdate] method of a [virtual table], the ** sqlite3_value_nochange(X) interface returns true if and only if ** the column corresponding to X is unchanged by the UPDATE operation @@ -5572,6 +5616,7 @@ SQLITE_API int sqlite3_value_type(sqlite3_value*); SQLITE_API int sqlite3_value_numeric_type(sqlite3_value*); SQLITE_API int sqlite3_value_nochange(sqlite3_value*); SQLITE_API int sqlite3_value_frombind(sqlite3_value*); +SQLITE_API int sqlite3_value_encoding(sqlite3_value*); /* ** CAPI3REF: Finding The Subtype Of SQL Values @@ -5625,7 +5670,7 @@ SQLITE_API void sqlite3_value_free(sqlite3_value*); ** ** ^The sqlite3_aggregate_context(C,N) routine returns a NULL pointer ** when first called if N is less than or equal to zero or if a memory -** allocate error occurs. +** allocation error occurs. ** ** ^(The amount of space allocated by sqlite3_aggregate_context(C,N) is ** determined by the N parameter on first successful call. Changing the @@ -5830,9 +5875,10 @@ typedef void (*sqlite3_destructor_type)(void*); ** of [SQLITE_UTF8], [SQLITE_UTF16], [SQLITE_UTF16BE], or [SQLITE_UTF16LE]. ** ^SQLite takes the text result from the application from ** the 2nd parameter of the sqlite3_result_text* interfaces. -** ^If the 3rd parameter to the sqlite3_result_text* interfaces -** is negative, then SQLite takes result text from the 2nd parameter -** through the first zero character. +** ^If the 3rd parameter to any of the sqlite3_result_text* interfaces +** other than sqlite3_result_text64() is negative, then SQLite computes +** the string length itself by searching the 2nd parameter for the first +** zero character. ** ^If the 3rd parameter to the sqlite3_result_text* interfaces ** is non-negative, then as many bytes (not characters) of the text ** pointed to by the 2nd parameter are taken as the application-defined @@ -6328,7 +6374,7 @@ SQLITE_API const char *sqlite3_db_name(sqlite3 *db, int N); **
    5. [sqlite3_filename_wal()] ** */ -SQLITE_API const char *sqlite3_db_filename(sqlite3 *db, const char *zDbName); +SQLITE_API sqlite3_filename sqlite3_db_filename(sqlite3 *db, const char *zDbName); /* ** CAPI3REF: Determine if a database is read-only @@ -6465,7 +6511,7 @@ SQLITE_API void *sqlite3_rollback_hook(sqlite3*, void(*)(void *), void*); ** function C that is invoked prior to each autovacuum of the database ** file. ^The callback is passed a copy of the generic data pointer (P), ** the schema-name of the attached database that is being autovacuumed, -** the the size of the database file in pages, the number of free pages, +** the size of the database file in pages, the number of free pages, ** and the number of bytes per page, respectively. The callback should ** return the number of free pages that should be removed by the ** autovacuum. ^If the callback returns zero, then no autovacuum happens. @@ -6586,6 +6632,11 @@ SQLITE_API void *sqlite3_update_hook( ** to the same database. Sharing is enabled if the argument is true ** and disabled if the argument is false.)^ ** +** This interface is omitted if SQLite is compiled with +** [-DSQLITE_OMIT_SHARED_CACHE]. The [-DSQLITE_OMIT_SHARED_CACHE] +** compile-time option is recommended because the +** [use of shared cache mode is discouraged]. +** ** ^Cache sharing is enabled and disabled for an entire process. ** This is a change as of SQLite [version 3.5.0] ([dateof:3.5.0]). ** In prior versions of SQLite, @@ -6684,7 +6735,7 @@ SQLITE_API int sqlite3_db_release_memory(sqlite3*); ** ^The soft heap limit may not be greater than the hard heap limit. ** ^If the hard heap limit is enabled and if sqlite3_soft_heap_limit(N) ** is invoked with a value of N that is greater than the hard heap limit, -** the the soft heap limit is set to the value of the hard heap limit. +** the soft heap limit is set to the value of the hard heap limit. ** ^The soft heap limit is automatically enabled whenever the hard heap ** limit is enabled. ^When sqlite3_hard_heap_limit64(N) is invoked and ** the soft heap limit is outside the range of 1..N, then the soft heap @@ -8979,7 +9030,7 @@ typedef struct sqlite3_backup sqlite3_backup; ** if the application incorrectly accesses the destination [database connection] ** and so no error code is reported, but the operations may malfunction ** nevertheless. Use of the destination database connection while a -** backup is in progress might also also cause a mutex deadlock. +** backup is in progress might also cause a mutex deadlock. ** ** If running in [shared cache mode], the application must ** guarantee that the shared cache used by the destination database @@ -9407,7 +9458,7 @@ SQLITE_API int sqlite3_wal_checkpoint_v2( */ #define SQLITE_CHECKPOINT_PASSIVE 0 /* Do as much as possible w/o blocking */ #define SQLITE_CHECKPOINT_FULL 1 /* Wait for writers, then checkpoint */ -#define SQLITE_CHECKPOINT_RESTART 2 /* Like FULL but wait for for readers */ +#define SQLITE_CHECKPOINT_RESTART 2 /* Like FULL but wait for readers */ #define SQLITE_CHECKPOINT_TRUNCATE 3 /* Like RESTART but also truncate WAL */ /* @@ -12833,4 +12884,4 @@ struct fts5_api { #endif /* _FTS5_H */ -/******** End of fts5.h *********/ +/******** End of fts5.h *********/ \ No newline at end of file From c030b2d61048393883ea9258c7b56faf1bf2d4ea Mon Sep 17 00:00:00 2001 From: Simon Kelley Date: Sun, 13 Nov 2022 15:55:09 +0000 Subject: [PATCH 173/218] Handle known DNSSEC signature algorithms which are not supported. This fixes a confusion if certain algorithms are not supported because the version is the crypto library is too old. The validation should be treated the same as for a completely unknown algorithm, (ie return unverified answer) and not as a validation failure (ie return SERVFAIL). The algorithems affected are GOST and ED448. Signed-off-by: DL6ER --- src/dnsmasq/crypto.c | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/dnsmasq/crypto.c b/src/dnsmasq/crypto.c index 060e27ff..5a5de6f7 100644 --- a/src/dnsmasq/crypto.c +++ b/src/dnsmasq/crypto.c @@ -390,7 +390,12 @@ static int (*verify_func(int algo))(struct blockdata *key_data, unsigned int key return dnsmasq_ecdsa_verify; #if MIN_VERSION(3, 1) - case 15: case 16: + case 15: + return dnsmasq_eddsa_verify; +#endif + +#if MIN_VERSION(3, 6) + case 16: return dnsmasq_eddsa_verify; #endif } @@ -444,11 +449,17 @@ char *algo_digest_name(int algo) case 7: return "sha1"; /* RSASHA1-NSEC3-SHA1 */ case 8: return "sha256"; /* RSA/SHA-256 */ case 10: return "sha512"; /* RSA/SHA-512 */ +#if MIN_VERSION(3, 6) case 12: return "gosthash94"; /* ECC-GOST */ +#endif case 13: return "sha256"; /* ECDSAP256SHA256 */ case 14: return "sha384"; /* ECDSAP384SHA384 */ +#if MIN_VERSION(3, 1) case 15: return "null_hash"; /* ED25519 */ +# if MIN_VERSION(3, 6) case 16: return "null_hash"; /* ED448 */ +# endif +#endif default: return NULL; } } From b6e61c2da0882a637dca46289c19c0032b48840d Mon Sep 17 00:00:00 2001 From: Simon Kelley Date: Wed, 16 Nov 2022 15:54:43 +0000 Subject: [PATCH 174/218] Fix GOST signature algorithms for DNSSEC validation. Use CryptoPro version of the hash function. Handle the little-endian wire format of key data. Get the wire order of S and R correct. Note that Nettle version 3.6 or later is required for GOST support. Signed-off-by: DL6ER --- src/dnsmasq/crypto.c | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/dnsmasq/crypto.c b/src/dnsmasq/crypto.c index 5a5de6f7..d7990379 100644 --- a/src/dnsmasq/crypto.c +++ b/src/dnsmasq/crypto.c @@ -309,14 +309,14 @@ static int dnsmasq_gostdsa_verify(struct blockdata *key_data, unsigned int key_l mpz_init(y); } - mpz_import(x, 32 , 1, 1, 0, 0, p); - mpz_import(y, 32 , 1, 1, 0, 0, p + 32); + mpz_import(x, 32, -1, 1, 0, 0, p); + mpz_import(y, 32, -1, 1, 0, 0, p + 32); if (!ecc_point_set(gost_key, x, y)) - return 0; + return 0; - mpz_import(sig_struct->r, 32, 1, 1, 0, 0, sig); - mpz_import(sig_struct->s, 32, 1, 1, 0, 0, sig + 32); + mpz_import(sig_struct->s, 32, 1, 1, 0, 0, sig); + mpz_import(sig_struct->r, 32, 1, 1, 0, 0, sig + 32); return nettle_gostdsa_verify(gost_key, digest_len, digest, sig_struct); } @@ -430,7 +430,9 @@ char *ds_digest_name(int digest) { case 1: return "sha1"; case 2: return "sha256"; - case 3: return "gosthash94"; +#if MIN_VERSION(3, 6) + case 3: return "gosthash94cp"; +#endif case 4: return "sha384"; default: return NULL; } @@ -450,7 +452,7 @@ char *algo_digest_name(int algo) case 8: return "sha256"; /* RSA/SHA-256 */ case 10: return "sha512"; /* RSA/SHA-512 */ #if MIN_VERSION(3, 6) - case 12: return "gosthash94"; /* ECC-GOST */ + case 12: return "gosthash94cp"; /* ECC-GOST */ #endif case 13: return "sha256"; /* ECDSAP256SHA256 */ case 14: return "sha384"; /* ECDSAP384SHA384 */ From bc1acb24e8c5c36022c9b2d9d58c93ed15377232 Mon Sep 17 00:00:00 2001 From: Simon Kelley Date: Wed, 16 Nov 2022 16:49:30 +0000 Subject: [PATCH 175/218] Handle DS records for unsupported crypto algorithms correctly. Such a DS, as long as it is validated, should allow answers in the domain is attests to be returned as unvalidated, and not as a validation error. Signed-off-by: DL6ER --- src/dnsmasq/crypto.c | 2 +- src/dnsmasq/dnssec.c | 59 +++++++++++++++++++++++++++----------------- 2 files changed, 38 insertions(+), 23 deletions(-) diff --git a/src/dnsmasq/crypto.c b/src/dnsmasq/crypto.c index d7990379..2678683e 100644 --- a/src/dnsmasq/crypto.c +++ b/src/dnsmasq/crypto.c @@ -452,7 +452,7 @@ char *algo_digest_name(int algo) case 8: return "sha256"; /* RSA/SHA-256 */ case 10: return "sha512"; /* RSA/SHA-512 */ #if MIN_VERSION(3, 6) - case 12: return "gosthash94cp"; /* ECC-GOST */ + case 12: return "gosthash94cp"; /* ECC-GOST */ #endif case 13: return "sha256"; /* ECDSAP256SHA256 */ case 14: return "sha384"; /* ECDSAP384SHA384 */ diff --git a/src/dnsmasq/dnssec.c b/src/dnsmasq/dnssec.c index daf679e0..ce01b2e6 100644 --- a/src/dnsmasq/dnssec.c +++ b/src/dnsmasq/dnssec.c @@ -979,10 +979,13 @@ int dnssec_validate_by_ds(time_t now, struct dns_header *header, size_t plen, ch } /* The DNS packet is expected to contain the answer to a DS query - Put all DSs in the answer which are valid into the cache. + Put all DSs in the answer which are valid and have hash and signature algos + we support into the cache. Also handles replies which prove that there's no DS at this location, either because the zone is unsigned or this isn't a zone cut. These are cached too. + If none of the DS's are for supported algos, treat the answer as if + it's a proof of no DS at this location. RFC4035 para 5.2. return codes: STAT_OK At least one valid DS found and in cache. STAT_BOGUS no DS in reply or not signed, fails validation, bad packet. @@ -993,8 +996,8 @@ int dnssec_validate_by_ds(time_t now, struct dns_header *header, size_t plen, ch int dnssec_validate_ds(time_t now, struct dns_header *header, size_t plen, char *name, char *keyname, int class) { unsigned char *p = (unsigned char *)(header+1); - int qtype, qclass, rc, i, neganswer, nons, neg_ttl = 0; - int aclass, atype, rdlen; + int qtype, qclass, rc, i, neganswer, nons, neg_ttl = 0, found_supported = 0; + int aclass, atype, rdlen, flags; unsigned long ttl; union all_addr a; @@ -1065,14 +1068,22 @@ int dnssec_validate_ds(time_t now, struct dns_header *header, size_t plen, char algo = *p++; digest = *p++; - if ((key = blockdata_alloc((char*)p, rdlen - 4))) + if (!ds_digest_name(digest) || !ds_digest_name(digest)) + { + a.log.keytag = keytag; + a.log.algo = algo; + a.log.digest = digest; + log_query(F_NOEXTRA | F_KEYTAG | F_UPSTREAM, name, &a, "DS keytag %hu, algo %hu, digest %hu (not supported)", 0); + neg_ttl = ttl; + } + else if ((key = blockdata_alloc((char*)p, rdlen - 4))) { a.ds.digest = digest; a.ds.keydata = key; a.ds.algo = algo; a.ds.keytag = keytag; a.ds.keylen = rdlen - 4; - + if (!cache_insert(name, &a, class, now, ttl, F_FORWARD | F_DS | F_DNSSECOK)) { blockdata_free(key); @@ -1083,26 +1094,29 @@ int dnssec_validate_ds(time_t now, struct dns_header *header, size_t plen, char a.log.keytag = keytag; a.log.algo = algo; a.log.digest = digest; - if (ds_digest_name(digest) && algo_digest_name(algo)) - log_query(F_NOEXTRA | F_KEYTAG | F_UPSTREAM, name, &a, "DS keytag %hu, algo %hu, digest %hu", 0); - else - log_query(F_NOEXTRA | F_KEYTAG | F_UPSTREAM, name, &a, "DS keytag %hu, algo %hu, digest %hu (not supported)", 0); + log_query(F_NOEXTRA | F_KEYTAG | F_UPSTREAM, name, &a, "DS keytag %hu, algo %hu, digest %hu", 0); + found_supported = 1; } } p = psave; } + if (!ADD_RDLEN(header, p, plen, rdlen)) return STAT_BOGUS; /* bad packet */ } cache_end_insert(); + /* Fall through if no supported algo DS found. */ + if (found_supported) + return STAT_OK; } - else + + flags = F_FORWARD | F_DS | F_NEG | F_DNSSECOK; + + if (neganswer) { - int flags = F_FORWARD | F_DS | F_NEG | F_DNSSECOK; - if (RCODE(header) == NXDOMAIN) flags |= F_NXDOMAIN; @@ -1110,17 +1124,18 @@ int dnssec_validate_ds(time_t now, struct dns_header *header, size_t plen, char to store presence/absence of NS. */ if (nons) flags &= ~F_DNSSECOK; - - cache_start_insert(); - - /* Use TTL from NSEC for negative cache entries */ - if (!cache_insert(name, NULL, class, now, neg_ttl, flags)) - return STAT_BOGUS; - - cache_end_insert(); - - log_query(F_NOEXTRA | F_UPSTREAM, name, NULL, nons ? "no DS/cut" : "no DS", 0); } + + cache_start_insert(); + + /* Use TTL from NSEC for negative cache entries */ + if (!cache_insert(name, NULL, class, now, neg_ttl, flags)) + return STAT_BOGUS; + + cache_end_insert(); + + if (neganswer) + log_query(F_NOEXTRA | F_UPSTREAM, name, NULL, nons ? "no DS/cut" : "no DS", 0); return STAT_OK; } From b72d3b797c31f4daedab60992014ce35fbd58850 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 16 Nov 2022 21:45:42 +0100 Subject: [PATCH 176/218] Update embedded dnsmasq to v2.88rc1 Signed-off-by: DL6ER --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 15a56bca..a8e731d0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -11,6 +11,6 @@ cmake_minimum_required(VERSION 2.8.12) project(PIHOLE_FTL C) -set(DNSMASQ_VERSION pi-hole-v2.88test3) +set(DNSMASQ_VERSION pi-hole-v2.88rc1) add_subdirectory(src) From 78597ddf1ccde0164104535c9339227f7d459507 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 16 Nov 2022 21:32:58 +0100 Subject: [PATCH 177/218] Verify PID of shared memory without remapping global shmSettings object Signed-off-by: DL6ER --- src/shmem.c | 47 ++++++++++++++--------------------------------- 1 file changed, 14 insertions(+), 33 deletions(-) diff --git a/src/shmem.c b/src/shmem.c index 948b6d01..cb709ca3 100644 --- a/src/shmem.c +++ b/src/shmem.c @@ -135,51 +135,32 @@ static int get_dev_shm_usage(char buffer[64]) // (while we initialized the shared memory objects) static void verify_shmem_pid(void) { - const int fd = shm_open(SHARED_SETTINGS_NAME, O_RDWR, S_IRUSR | S_IWUSR); - if(fd == -1) + // Open shared memory settings object + const int settingsfd = shm_open(SHARED_SETTINGS_NAME, O_RDONLY, S_IRUSR | S_IWUSR); + if(settingsfd == -1) { logg("FATAL: verify_shmem_pid(): Failed to open shared memory object \"%s\": %s", - SHARED_SETTINGS_NAME, strerror(errno)); + SHARED_SETTINGS_NAME, strerror(errno)); exit(EXIT_FAILURE); } - // Get new pointer - void *shm = mmap(NULL, shm_settings.size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); - - // Debug logging - if(config.debug & DEBUG_SHMEM) - logg("verify_shmem_pid(): updating pointer of shmSettings: %p -> %p", shm_settings.ptr, shm); - - // Check for `mmap` error - if(shm == MAP_FAILED) + ShmSettings shms = { 0 }; + if(read(settingsfd, &shms, sizeof(shms)) != sizeof(shms)) { - logg("FATAL: verify_shmem_pid(): Failed to map shared memory object \"%s\" (%i): %s", - SHARED_SETTINGS_NAME, fd, strerror(errno)); + logg("FATAL: verify_shmem_pid(): Failed to read %zu bytes from shared memory object \"%s\": %s", + sizeof(shms), SHARED_SETTINGS_NAME, strerror(errno)); exit(EXIT_FAILURE); } - // Close file descriptor after call to mmap - close(fd); - - // Unmap old memory region - if(munmap(shm_settings.ptr, shm_settings.size) < 0) - { - logg("FATAL: verify_shmem_pid(): Failed to unmap shared memory object \"%s\" (%i): %s", - SHARED_SETTINGS_NAME, fd, strerror(errno)); - exit(EXIT_FAILURE); - } - - // Update pointer - shmSettings = shm_settings.ptr = shm; + close(settingsfd); // Compare the SHM's PID to the one we had when creating the SHM objects - if(shmSettings->pid == shmem_pid) + if(shms.pid == shmem_pid) return; // If we reach here, we are in serious trouble. Terminating with error // code is the most sensible thing we can do at this point - logg("FATAL: Shared memory is owned by a different process (PID %d)", - shmSettings->pid); + logg("FATAL: Shared memory is owned by a different process (PID %d)", shms.pid); check_running_FTL(); logg("Exiting now!"); exit(EXIT_FAILURE); @@ -786,6 +767,9 @@ static bool realloc_shm(SharedMemory *sharedMemory, const size_t size1, const si // TCP requests. if(resize) { + // Verify shared memory ownership + verify_shmem_pid(); + // Open shared memory object const int fd = shm_open(sharedMemory->name, O_RDWR, S_IRUSR | S_IWUSR); if(fd == -1) @@ -925,9 +909,6 @@ static size_t get_optimal_object_size(const size_t objsize, const size_t minsize // Enlarge shared memory to be able to hold at least one new record void shm_ensure_size(void) { - // Verify shared memory ownership - verify_shmem_pid(); - if(counters->queries >= counters->queries_MAX-1) { // Have to reallocate shared memory From fe4077ea9f4ff251c68a080b74463e2ab55120f4 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 17 Nov 2022 04:44:12 +0200 Subject: [PATCH 178/218] Increase default SQLite3 cache size from 2000 kiB to 16 MiB Signed-off-by: DL6ER --- src/CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 8bf6e32d..ef90d7c9 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -29,7 +29,8 @@ set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}) # SQLITE_ENABLE_DBPAGE_VTAB: Enables the SQLITE_DBPAGE virtual table. Warning: writing to the SQLITE_DBPAGE virtual table can very easily cause unrecoverably database corruption. # SQLITE_OMIT_DESERIALIZE: This option causes the the sqlite3_serialize() and sqlite3_deserialize() interfaces to be omitted from the build (was the default before 3.36.0) # HAVE_READLINE: Enable readline support to allow easy editing, history and auto-completion -set(SQLITE_DEFINES "-DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_DEFAULT_MEMSTATUS=0 -DSQLITE_OMIT_DEPRECATED -DSQLITE_OMIT_PROGRESS_CALLBACK -DSQLITE_DEFAULT_FOREIGN_KEYS=1 -DSQLITE_DQS=0 -DSQLITE_ENABLE_DBPAGE_VTAB -DSQLITE_OMIT_DESERIALIZE -DHAVE_READLINE") +# SQLITE_DEFAULT_CACHE_SIZE=-16384: Allow up to 16 MiB of cache to be used by SQLite3 (default is 2000 kiB) +set(SQLITE_DEFINES "-DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_DEFAULT_MEMSTATUS=0 -DSQLITE_OMIT_DEPRECATED -DSQLITE_OMIT_PROGRESS_CALLBACK -DSQLITE_DEFAULT_FOREIGN_KEYS=1 -DSQLITE_DQS=0 -DSQLITE_ENABLE_DBPAGE_VTAB -DSQLITE_OMIT_DESERIALIZE -DHAVE_READLINE -DSQLITE_DEFAULT_CACHE_SIZE=-16384") # Code hardening and debugging improvements # -fstack-protector-strong: The program will be resistant to having its stack overflowed From 576e766b593e86139454550c5f9121f1d5a3048e Mon Sep 17 00:00:00 2001 From: Simon Kelley Date: Thu, 17 Nov 2022 12:52:53 +0000 Subject: [PATCH 179/218] Fix struct hostinfo free code and BSD compile. The code added in6 c596f1cc1d92b2b90ef5ce043ace314eefa868b fails to free the returned datastructures from gethostinfo() because sdetails.hostinfo is used to loop through the addresses and ends up NULL. In some libc implementations this results in a SEGV when freeaddrinfo() is called. Also fix FTBFS under BSD. Thanks to Johnny S. Lee for the bug report. Signed-off-by: DL6ER --- src/dnsmasq/dbus.c | 4 ++-- src/dnsmasq/dnsmasq.h | 4 ++-- src/dnsmasq/option.c | 41 +++++++++++++++++++++++------------------ 3 files changed, 27 insertions(+), 22 deletions(-) diff --git a/src/dnsmasq/dbus.c b/src/dnsmasq/dbus.c index ef807111..fd5d1ca6 100644 --- a/src/dnsmasq/dbus.c +++ b/src/dnsmasq/dbus.c @@ -461,8 +461,8 @@ static DBusMessage* dbus_read_servers_ex(DBusMessage *message, int strings) } while (dbus_message_iter_get_arg_type(&string_iter) == DBUS_TYPE_STRING); } - if (sdetails.resolved) - freeaddrinfo(sdetails.hostinfo); + if (sdetails.orig_hostinfo) + freeaddrinfo(sdetails.orig_hostinfo); /* jump to next element in outer array */ dbus_message_iter_next(&array_iter); diff --git a/src/dnsmasq/dnsmasq.h b/src/dnsmasq/dnsmasq.h index c27b93f1..f91729cb 100644 --- a/src/dnsmasq/dnsmasq.h +++ b/src/dnsmasq/dnsmasq.h @@ -1300,9 +1300,9 @@ extern struct daemon { struct server_details { union mysockaddr *addr, *source_addr; - struct addrinfo *hostinfo; + struct addrinfo *hostinfo, *orig_hostinfo; char *interface, *source, *scope_id, *interface_opt; - int serv_port, source_port, addr_type, scope_index, valid, resolved; + int serv_port, source_port, addr_type, scope_index, valid; u16 *flags; }; diff --git a/src/dnsmasq/option.c b/src/dnsmasq/option.c index b511ef34..cccf4f66 100644 --- a/src/dnsmasq/option.c +++ b/src/dnsmasq/option.c @@ -864,11 +864,16 @@ char *parse_server(char *arg, struct server_details *sdetails) sdetails->serv_port = NAMESERVER_PORT; char *portno; int ecode = 0; - struct addrinfo hints = { 0 }; + struct addrinfo hints; + memset(&hints, 0, sizeof(struct addrinfo)); + *sdetails->interface = 0; sdetails->addr_type = AF_UNSPEC; + sdetails->valid = 0; + sdetails->hostinfo = sdetails->orig_hostinfo = NULL; + if (strcmp(arg, "#") == 0) { if (sdetails->flags) @@ -947,7 +952,7 @@ char *parse_server(char *arg, struct server_details *sdetails) above, and returns a pointer to the start of the list in . The items in the linked list are linked by the field. */ sdetails->valid = 1; - sdetails->resolved = 1; + sdetails->orig_hostinfo = sdetails->hostinfo; return NULL; } else @@ -971,7 +976,7 @@ char *parse_server_addr(struct server_details *sdetails) sdetails->addr->in.sin_port = htons(sdetails->serv_port); sdetails->addr->sa.sa_family = sdetails->source_addr->sa.sa_family = AF_INET; #ifdef HAVE_SOCKADDR_SA_LEN - source_addr->in.sin_len = addr->in.sin_len = sizeof(struct sockaddr_in); + sdetails->source_addr->in.sin_len = sdetails->addr->in.sin_len = sizeof(struct sockaddr_in); #endif sdetails->source_addr->in.sin_addr.s_addr = INADDR_ANY; sdetails->source_addr->in.sin_port = htons(daemon->query_port); @@ -989,7 +994,7 @@ char *parse_server_addr(struct server_details *sdetails) /* When resolving a server IP by hostname, we can simply skip mismatching server / source IP pairs. Otherwise, when an IP address is given directly, this is a fatal error. */ - if (!sdetails->resolved) + if (!sdetails->orig_hostinfo) return _("cannot use IPv4 server address with IPv6 source address"); } else @@ -1020,7 +1025,7 @@ char *parse_server_addr(struct server_details *sdetails) sdetails->addr->sa.sa_family = sdetails->source_addr->sa.sa_family = AF_INET6; sdetails->addr->in6.sin6_flowinfo = sdetails->source_addr->in6.sin6_flowinfo = 0; #ifdef HAVE_SOCKADDR_SA_LEN - sdetails->addr->in6.sin6_len = sdetails->source_addr->in6.sin6_len = sizeof(addr->in6); + sdetails->addr->in6.sin6_len = sdetails->source_addr->in6.sin6_len = sizeof(sdetails->addr->in6); #endif if (sdetails->source) { @@ -1035,7 +1040,7 @@ char *parse_server_addr(struct server_details *sdetails) /* When resolving a server IP by hostname, we can simply skip mismatching server / source IP pairs. Otherwise, when an IP address is given directly, this is a fatal error. */ - if(!sdetails->resolved) + if(!sdetails->orig_hostinfo) return _("cannot use IPv6 server address with IPv4 source address"); } else @@ -1102,12 +1107,12 @@ static char *domain_rev4(int from_file, char *server, struct in_addr *addr4, int union mysockaddr serv_addr, source_addr; char interface[IF_NAMESIZE+1]; int count = 1, rem, addrbytes, addrbits; - struct server_details sdetails = { 0 }; + struct server_details sdetails; sdetails.addr = &serv_addr; sdetails.source_addr = &source_addr; sdetails.interface = interface; sdetails.flags = &flags; - + if (!server) flags = SERV_LITERAL_ADDRESS; else if ((string = parse_server(server, &sdetails))) @@ -1165,8 +1170,8 @@ static char *domain_rev4(int from_file, char *server, struct in_addr *addr4, int return _("error"); } - if (sdetails.resolved) - freeaddrinfo(sdetails.hostinfo); + if (sdetails.orig_hostinfo) + freeaddrinfo(sdetails.orig_hostinfo); } } @@ -1183,12 +1188,12 @@ static char *domain_rev6(int from_file, char *server, struct in6_addr *addr6, in union mysockaddr serv_addr, source_addr; char interface[IF_NAMESIZE+1]; int count = 1, rem, addrbytes, addrbits; - struct server_details sdetails = { 0 }; + struct server_details sdetails; sdetails.addr = &serv_addr; sdetails.source_addr = &source_addr; sdetails.interface = interface; sdetails.flags = &flags; - + if (!server) flags = SERV_LITERAL_ADDRESS; else if ((string = parse_server(server, &sdetails))) @@ -1248,8 +1253,8 @@ static char *domain_rev6(int from_file, char *server, struct in6_addr *addr6, in return _("error"); } - if (sdetails.resolved) - freeaddrinfo(sdetails.hostinfo); + if (sdetails.orig_hostinfo) + freeaddrinfo(sdetails.orig_hostinfo); } } @@ -2970,12 +2975,12 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma union mysockaddr serv_addr, source_addr; char interface[IF_NAMESIZE+1]; - struct server_details sdetails = { 0 }; + struct server_details sdetails; sdetails.addr = &serv_addr; sdetails.source_addr = &source_addr; sdetails.interface = interface; sdetails.flags = &flags; - + unhide_metas(arg); /* split the domain args, if any and skip to the end of them. */ @@ -3055,8 +3060,8 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma break; } - if (sdetails.resolved) - freeaddrinfo(sdetails.hostinfo); + if (sdetails.orig_hostinfo) + freeaddrinfo(sdetails.orig_hostinfo); break; } From b7c858f6b596718ec72b105fde7575dcb07e8ba8 Mon Sep 17 00:00:00 2001 From: Simon Kelley Date: Thu, 17 Nov 2022 19:51:15 +0000 Subject: [PATCH 180/218] Fix SEGV on --local= added by immediately previous commit. Signed-off-by: DL6ER --- src/dnsmasq/option.c | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/dnsmasq/option.c b/src/dnsmasq/option.c index cccf4f66..0ad660fd 100644 --- a/src/dnsmasq/option.c +++ b/src/dnsmasq/option.c @@ -870,10 +870,7 @@ char *parse_server(char *arg, struct server_details *sdetails) *sdetails->interface = 0; sdetails->addr_type = AF_UNSPEC; - sdetails->valid = 0; - sdetails->hostinfo = sdetails->orig_hostinfo = NULL; - - + if (strcmp(arg, "#") == 0) { if (sdetails->flags) @@ -1108,6 +1105,8 @@ static char *domain_rev4(int from_file, char *server, struct in_addr *addr4, int char interface[IF_NAMESIZE+1]; int count = 1, rem, addrbytes, addrbits; struct server_details sdetails; + + memset(&sdetails, 0, sizeof(struct server_details)); sdetails.addr = &serv_addr; sdetails.source_addr = &source_addr; sdetails.interface = interface; @@ -1189,6 +1188,8 @@ static char *domain_rev6(int from_file, char *server, struct in6_addr *addr6, in char interface[IF_NAMESIZE+1]; int count = 1, rem, addrbytes, addrbits; struct server_details sdetails; + + memset(&sdetails, 0, sizeof(struct server_details)); sdetails.addr = &serv_addr; sdetails.source_addr = &source_addr; sdetails.interface = interface; @@ -2974,13 +2975,14 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma union all_addr addr; union mysockaddr serv_addr, source_addr; char interface[IF_NAMESIZE+1]; - struct server_details sdetails; + + memset(&sdetails, 0, sizeof(struct server_details)); sdetails.addr = &serv_addr; sdetails.source_addr = &source_addr; sdetails.interface = interface; sdetails.flags = &flags; - + unhide_metas(arg); /* split the domain args, if any and skip to the end of them. */ From 128096e7107f5a9cf2ee5e6d93af2d650e1bc713 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 19 Nov 2022 08:17:49 +0100 Subject: [PATCH 181/218] Update embedded dnsmasq to v2.88rc3 Signed-off-by: DL6ER --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index a8e731d0..95b08c70 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -11,6 +11,6 @@ cmake_minimum_required(VERSION 2.8.12) project(PIHOLE_FTL C) -set(DNSMASQ_VERSION pi-hole-v2.88rc1) +set(DNSMASQ_VERSION pi-hole-v2.88rc3) add_subdirectory(src) From 0fea2d586b5d8e683d1e441535c2ff207e5a32ff Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 20 Nov 2022 12:44:07 +0100 Subject: [PATCH 182/218] Add facility to embedd LUA scripts into FTL and run them during luainit Signed-off-by: DL6ER --- .gitignore | 3 + src/CMakeLists.txt | 1 + src/lua/ftl_lua.c | 24 ++- src/lua/ftl_lua.h | 2 - src/lua/lua.c | 4 +- src/lua/scripts/CMakeLists.txt | 35 ++++ src/lua/scripts/inspect.lua | 335 +++++++++++++++++++++++++++++++++ src/lua/scripts/scripts.h | 17 ++ 8 files changed, 414 insertions(+), 7 deletions(-) create mode 100644 src/lua/scripts/CMakeLists.txt create mode 100644 src/lua/scripts/inspect.lua create mode 100644 src/lua/scripts/scripts.h diff --git a/.gitignore b/.gitignore index 9cf9308d..87821469 100644 --- a/.gitignore +++ b/.gitignore @@ -26,3 +26,6 @@ tools/macvendor.db # Test Leftovers dig.log ptr.log + +# Compiled LUA scripts +/src/lua/scripts/*.hex diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index ef90d7c9..e7138179 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -250,5 +250,6 @@ add_subdirectory(api) add_subdirectory(database) add_subdirectory(dnsmasq) add_subdirectory(lua) +add_subdirectory(lua/scripts) add_subdirectory(tre-regex) add_subdirectory(syscalls) diff --git a/src/lua/ftl_lua.c b/src/lua/ftl_lua.c index 2da6cb22..e8130a1b 100644 --- a/src/lua/ftl_lua.c +++ b/src/lua/ftl_lua.c @@ -16,6 +16,7 @@ #include "../log.h" #include #include +#include "scripts/scripts.h" int run_lua_interpreter(const int argc, char **argv, bool dnsmasq_debug) { @@ -98,9 +99,28 @@ LUAMOD_API int luaopen_pihole(lua_State *L) { return LUA_YIELD; } +const char *script = inspect_lua; + +static bool ftl_lua_load_embedded_script(lua_State *L, const char *name, const char *script, const bool make_global) +{ + if (luaL_dostring(L, script) != 0) + { + const char *lua_err = lua_tostring(L, -1); + logg("LUA error: %s", lua_err); + return false; + } + + if(make_global) + { + /* Set global[name] = luaL_dostring return */ + lua_setglobal(L, name); + } + + return true; +} + // Load bundled libraries and make the available globally void ftl_lua_init(lua_State *L) { - if(dolibrary(L, "inspect") != LUA_OK) - printf("Failed to load library inspect.lua\n"); + ftl_lua_load_embedded_script(L, "inspect", inspect_lua, true); } diff --git a/src/lua/ftl_lua.h b/src/lua/ftl_lua.h index c2ef12ab..c91ab5c3 100644 --- a/src/lua/ftl_lua.h +++ b/src/lua/ftl_lua.h @@ -21,8 +21,6 @@ int run_luac(const int argc, char **argv); int lua_main (int argc, char **argv); int luac_main (int argc, char **argv); -extern int dolibrary (lua_State *L, const char *name); - void ftl_lua_init(lua_State *L); #endif //FTL_LUA_H \ No newline at end of file diff --git a/src/lua/lua.c b/src/lua/lua.c index fa52d252..63e591e4 100644 --- a/src/lua/lua.c +++ b/src/lua/lua.c @@ -194,9 +194,7 @@ static int dostring (lua_State *L, const char *s, const char *name) { ** Calls 'require(name)' and stores the result in a global variable ** with the given name. */ -/************** Pi-hole modification ***************/ -int dolibrary (lua_State *L, const char *name) { -/***************************************************/ +static int dolibrary (lua_State *L, const char *name) { int status; lua_getglobal(L, "require"); lua_pushstring(L, name); diff --git a/src/lua/scripts/CMakeLists.txt b/src/lua/scripts/CMakeLists.txt new file mode 100644 index 00000000..1278f8f9 --- /dev/null +++ b/src/lua/scripts/CMakeLists.txt @@ -0,0 +1,35 @@ +# Pi-hole: A black hole for Internet advertisements +# (c) 2022 Pi-hole, LLC (https://pi-hole.net) +# Network-wide ad blocking via your own hardware. +# +# FTL Engine +# /src/lua/scripts/CMakeList.txt +# +# This file is copyright under the latest version of the EUPL. +# Please see LICENSE file for your rights under this license. + +set(sources + inspect.lua.hex + scripts.h + ) + +# Compile files from raw/ into hex/ +find_program(RESOURCE_COMPILER xxd) +file(GLOB_RECURSE COMPILED_RESOURCES RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}/" "./*.lua") +foreach(INPUT_FILE ${COMPILED_RESOURCES}) + set(IN ${CMAKE_CURRENT_SOURCE_DIR}/${INPUT_FILE}) + set(OUTPUT_FILE ${CMAKE_CURRENT_SOURCE_DIR}/${INPUT_FILE}.hex) + add_custom_command( + OUTPUT ${INPUT_FILE}.hex + COMMAND ${RESOURCE_COMPILER} -i < ${IN} > ${OUTPUT_FILE} + COMMENT "Compiling ${INPUT_FILE} to binary" + VERBATIM) + list(APPEND COMPILED_RESOURCES ${OUTPUT_FILE}) +endforeach() + +# Ensure target lua_scripts is build before target lua +add_dependencies(lua lua_scripts) + +add_library(lua_scripts OBJECT ${sources}) +target_compile_options(lua_scripts PRIVATE ${EXTRAWARN}) +target_include_directories(lua_scripts PRIVATE ${PROJECT_SOURCE_DIR}/src) diff --git a/src/lua/scripts/inspect.lua b/src/lua/scripts/inspect.lua new file mode 100644 index 00000000..8d33a981 --- /dev/null +++ b/src/lua/scripts/inspect.lua @@ -0,0 +1,335 @@ +local inspect ={ + _VERSION = 'inspect.lua 3.1.0', + _URL = 'http://github.com/kikito/inspect.lua', + _DESCRIPTION = 'human-readable representations of tables', + _LICENSE = [[ + MIT LICENSE + + Copyright (c) 2013 Enrique García Cota + + Permission is hereby granted, free of charge, to any person obtaining a + copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to + permit persons to whom the Software is furnished to do so, subject to + the following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY + CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, + TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + ]] +} + +local tostring = tostring + +inspect.KEY = setmetatable({}, {__tostring = function() return 'inspect.KEY' end}) +inspect.METATABLE = setmetatable({}, {__tostring = function() return 'inspect.METATABLE' end}) + +local function rawpairs(t) + return next, t, nil +end + +-- Apostrophizes the string if it has quotes, but not aphostrophes +-- Otherwise, it returns a regular quoted string +local function smartQuote(str) + if str:match('"') and not str:match("'") then + return "'" .. str .. "'" + end + return '"' .. str:gsub('"', '\\"') .. '"' +end + +-- \a => '\\a', \0 => '\\0', 31 => '\31' +local shortControlCharEscapes = { + ["\a"] = "\\a", ["\b"] = "\\b", ["\f"] = "\\f", ["\n"] = "\\n", + ["\r"] = "\\r", ["\t"] = "\\t", ["\v"] = "\\v" +} +local longControlCharEscapes = {} -- \a => nil, \0 => \000, 31 => \031 +for i=0, 31 do + local ch = string.char(i) + if not shortControlCharEscapes[ch] then + shortControlCharEscapes[ch] = "\\"..i + longControlCharEscapes[ch] = string.format("\\%03d", i) + end +end + +local function escape(str) + return (str:gsub("\\", "\\\\") + :gsub("(%c)%f[0-9]", longControlCharEscapes) + :gsub("%c", shortControlCharEscapes)) +end + +local function isIdentifier(str) + return type(str) == 'string' and str:match( "^[_%a][_%a%d]*$" ) +end + +local function isSequenceKey(k, sequenceLength) + return type(k) == 'number' + and 1 <= k + and k <= sequenceLength + and math.floor(k) == k +end + +local defaultTypeOrders = { + ['number'] = 1, ['boolean'] = 2, ['string'] = 3, ['table'] = 4, + ['function'] = 5, ['userdata'] = 6, ['thread'] = 7 +} + +local function sortKeys(a, b) + local ta, tb = type(a), type(b) + + -- strings and numbers are sorted numerically/alphabetically + if ta == tb and (ta == 'string' or ta == 'number') then return a < b end + + local dta, dtb = defaultTypeOrders[ta], defaultTypeOrders[tb] + -- Two default types are compared according to the defaultTypeOrders table + if dta and dtb then return defaultTypeOrders[ta] < defaultTypeOrders[tb] + elseif dta then return true -- default types before custom ones + elseif dtb then return false -- custom types after default ones + end + + -- custom types are sorted out alphabetically + return ta < tb +end + +-- For implementation reasons, the behavior of rawlen & # is "undefined" when +-- tables aren't pure sequences. So we implement our own # operator. +local function getSequenceLength(t) + local len = 1 + local v = rawget(t,len) + while v ~= nil do + len = len + 1 + v = rawget(t,len) + end + return len - 1 +end + +local function getNonSequentialKeys(t) + local keys, keysLength = {}, 0 + local sequenceLength = getSequenceLength(t) + for k,_ in rawpairs(t) do + if not isSequenceKey(k, sequenceLength) then + keysLength = keysLength + 1 + keys[keysLength] = k + end + end + table.sort(keys, sortKeys) + return keys, keysLength, sequenceLength +end + +local function countTableAppearances(t, tableAppearances) + tableAppearances = tableAppearances or {} + + if type(t) == 'table' then + if not tableAppearances[t] then + tableAppearances[t] = 1 + for k,v in rawpairs(t) do + countTableAppearances(k, tableAppearances) + countTableAppearances(v, tableAppearances) + end + countTableAppearances(getmetatable(t), tableAppearances) + else + tableAppearances[t] = tableAppearances[t] + 1 + end + end + + return tableAppearances +end + +local copySequence = function(s) + local copy, len = {}, #s + for i=1, len do copy[i] = s[i] end + return copy, len +end + +local function makePath(path, ...) + local keys = {...} + local newPath, len = copySequence(path) + for i=1, #keys do + newPath[len + i] = keys[i] + end + return newPath +end + +local function processRecursive(process, item, path, visited) + if item == nil then return nil end + if visited[item] then return visited[item] end + + local processed = process(item, path) + if type(processed) == 'table' then + local processedCopy = {} + visited[item] = processedCopy + local processedKey + + for k,v in rawpairs(processed) do + processedKey = processRecursive(process, k, makePath(path, k, inspect.KEY), visited) + if processedKey ~= nil then + processedCopy[processedKey] = processRecursive(process, v, makePath(path, processedKey), visited) + end + end + + local mt = processRecursive(process, getmetatable(processed), makePath(path, inspect.METATABLE), visited) + if type(mt) ~= 'table' then mt = nil end -- ignore not nil/table __metatable field + setmetatable(processedCopy, mt) + processed = processedCopy + end + return processed +end + + + +------------------------------------------------------------------- + +local Inspector = {} +local Inspector_mt = {__index = Inspector} + +function Inspector:puts(...) + local args = {...} + local buffer = self.buffer + local len = #buffer + for i=1, #args do + len = len + 1 + buffer[len] = args[i] + end +end + +function Inspector:down(f) + self.level = self.level + 1 + f() + self.level = self.level - 1 +end + +function Inspector:tabify() + self:puts(self.newline, string.rep(self.indent, self.level)) +end + +function Inspector:alreadyVisited(v) + return self.ids[v] ~= nil +end + +function Inspector:getId(v) + local id = self.ids[v] + if not id then + local tv = type(v) + id = (self.maxIds[tv] or 0) + 1 + self.maxIds[tv] = id + self.ids[v] = id + end + return tostring(id) +end + +function Inspector:putKey(k) + if isIdentifier(k) then return self:puts(k) end + self:puts("[") + self:putValue(k) + self:puts("]") +end + +function Inspector:putTable(t) + if t == inspect.KEY or t == inspect.METATABLE then + self:puts(tostring(t)) + elseif self:alreadyVisited(t) then + self:puts('') + elseif self.level >= self.depth then + self:puts('{...}') + else + if self.tableAppearances[t] > 1 then self:puts('<', self:getId(t), '>') end + + local nonSequentialKeys, nonSequentialKeysLength, sequenceLength = getNonSequentialKeys(t) + local mt = getmetatable(t) + + self:puts('{') + self:down(function() + local count = 0 + for i=1, sequenceLength do + if count > 0 then self:puts(',') end + self:puts(' ') + self:putValue(t[i]) + count = count + 1 + end + + for i=1, nonSequentialKeysLength do + local k = nonSequentialKeys[i] + if count > 0 then self:puts(',') end + self:tabify() + self:putKey(k) + self:puts(' = ') + self:putValue(t[k]) + count = count + 1 + end + + if type(mt) == 'table' then + if count > 0 then self:puts(',') end + self:tabify() + self:puts(' = ') + self:putValue(mt) + end + end) + + if nonSequentialKeysLength > 0 or type(mt) == 'table' then -- result is multi-lined. Justify closing } + self:tabify() + elseif sequenceLength > 0 then -- array tables have one extra space before closing } + self:puts(' ') + end + + self:puts('}') + end +end + +function Inspector:putValue(v) + local tv = type(v) + + if tv == 'string' then + self:puts(smartQuote(escape(v))) + elseif tv == 'number' or tv == 'boolean' or tv == 'nil' or + tv == 'cdata' or tv == 'ctype' then + self:puts(tostring(v)) + elseif tv == 'table' then + self:putTable(v) + else + self:puts('<', tv, ' ', self:getId(v), '>') + end +end + +------------------------------------------------------------------- + +function inspect.inspect(root, options) + options = options or {} + + local depth = options.depth or math.huge + local newline = options.newline or '\n' + local indent = options.indent or ' ' + local process = options.process + + if process then + root = processRecursive(process, root, {}, {}) + end + + local inspector = setmetatable({ + depth = depth, + level = 0, + buffer = {}, + ids = {}, + maxIds = {}, + newline = newline, + indent = indent, + tableAppearances = countTableAppearances(root) + }, Inspector_mt) + + inspector:putValue(root) + + return table.concat(inspector.buffer) +end + +setmetatable(inspect, { __call = function(_, ...) return inspect.inspect(...) end }) + +return inspect + + diff --git a/src/lua/scripts/scripts.h b/src/lua/scripts/scripts.h new file mode 100644 index 00000000..2a78d5a4 --- /dev/null +++ b/src/lua/scripts/scripts.h @@ -0,0 +1,17 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2022 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* Embedded LUA scripts processor +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ +#ifndef LUA_SCRIPTS_H +#define LUA_SCRIPTS_H + +static const char inspect_lua[] = { +#include "inspect.lua.hex" +}; + +#endif // LUA_SCRIPTS_H \ No newline at end of file From 6b6c069b227a45ff7f47ff8a91198d32eeb911e8 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 20 Nov 2022 12:45:35 +0100 Subject: [PATCH 183/218] Remove downloading step of inspect.lua from the testing suite - the script is now embedded into FTL Signed-off-by: DL6ER --- test/run.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/test/run.sh b/test/run.sh index e267a39b..348980dc 100755 --- a/test/run.sh +++ b/test/run.sh @@ -60,10 +60,6 @@ bash test/pdns/setup.sh OLDUMASK=$(umask) umask 0022 -# Prepare LUA scripts -mkdir -p /opt/pihole/libs -wget -O /opt/pihole/libs/inspect.lua https://ftl.pi-hole.net/libraries/inspect.lua - # Start FTL if ! su pihole -s /bin/sh -c /home/pihole/pihole-FTL; then echo "pihole-FTL failed to start" From 395e0792d8db4a35138c9f85c101eda547f5540d Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 20 Nov 2022 13:53:41 +0100 Subject: [PATCH 184/218] Use luaL_loadbufferx to pass name directly to interpreter Signed-off-by: DL6ER --- src/lua/ftl_lua.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/lua/ftl_lua.c b/src/lua/ftl_lua.c index e8130a1b..72c371c3 100644 --- a/src/lua/ftl_lua.c +++ b/src/lua/ftl_lua.c @@ -99,14 +99,15 @@ LUAMOD_API int luaopen_pihole(lua_State *L) { return LUA_YIELD; } -const char *script = inspect_lua; - -static bool ftl_lua_load_embedded_script(lua_State *L, const char *name, const char *script, const bool make_global) +static bool ftl_lua_load_embedded_script(lua_State *L, const char *name, const char *script, const size_t script_len, const bool make_global) { - if (luaL_dostring(L, script) != 0) + // Explanation: + // luaL_dostring(L, script) expands to (luaL_loadstring(L, script) || lua_pcall(L, 0, LUA_MULTRET, 0)) + // luaL_loadstring(L, script) calls luaL_loadbuffer(L, s, strlen(s), s) + if (luaL_loadbufferx(L, script, script_len, name, NULL) || lua_pcall(L, 0, LUA_MULTRET, 0) != 0) { const char *lua_err = lua_tostring(L, -1); - logg("LUA error: %s", lua_err); + printf("LUA error while trying to import %s.lua: %s\n", name, lua_err); return false; } @@ -122,5 +123,5 @@ static bool ftl_lua_load_embedded_script(lua_State *L, const char *name, const c // Load bundled libraries and make the available globally void ftl_lua_init(lua_State *L) { - ftl_lua_load_embedded_script(L, "inspect", inspect_lua, true); + ftl_lua_load_embedded_script(L, "inspect", inspect_lua, sizeof(inspect_lua), true); } From 96e1c4407e33775918137a2f5fce61f67dff0fc5 Mon Sep 17 00:00:00 2001 From: Simon Kelley Date: Mon, 21 Nov 2022 16:40:53 +0000 Subject: [PATCH 185/218] Optimise readng large number --server options at start up. When re-reading upstream servers from /etc/resolv.conf or other sources that can change dnsmasq tries to avoid memory fragmentation by re-using existing records that are being re-read unchanged. This involves seaching all the server records for each new one installed. During startup this search is pointless, and can cause long start times with thousands of --server options because the work needed is O(n^2). Handle this case more intelligently. Thanks to Ye Zhou for spotting the problem and an initial patch. Signed-off-by: DL6ER --- src/dnsmasq/domain-match.c | 55 ++++++++++++++++++++++++-------------- src/dnsmasq/option.c | 5 +++- 2 files changed, 39 insertions(+), 21 deletions(-) diff --git a/src/dnsmasq/domain-match.c b/src/dnsmasq/domain-match.c index 76a1109e..bef460a1 100644 --- a/src/dnsmasq/domain-match.c +++ b/src/dnsmasq/domain-match.c @@ -546,11 +546,23 @@ static int order_qsort(const void *a, const void *b) return rc; } + +/* When loading large numbers of server=.... lines during startup, + there's no possibility that there will be server records that can be reused, but + searching a long list for each server added grows as O(n^2) and slows things down. + This flag is set only if is known there may be free server records that can be reused. + There's a call to mark_servers(0) in read_opts() to reset the flag before + main config read. */ + +static int maybe_free_servers = 0; + /* Must be called before add_update_server() to set daemon->servers_tail */ void mark_servers(int flag) { struct server *serv, **up; + maybe_free_servers = !!flag; + daemon->servers_tail = NULL; /* mark everything with argument flag */ @@ -667,27 +679,30 @@ int add_update_server(int flags, and move to the end of the list, for order. The entry found may already be at the end. */ struct server **up, *tmp; - - for (serv = daemon->servers, up = &daemon->servers; serv; serv = tmp) - { - tmp = serv->next; - if ((serv->flags & SERV_MARK) && - hostname_isequal(alloc_domain, serv->domain)) - { - /* Need to move down? */ - if (serv->next) - { - *up = serv->next; - daemon->servers_tail->next = serv; - daemon->servers_tail = serv; - serv->next = NULL; - } - break; - } - else - up = &serv->next; - } + serv = NULL; + + if (maybe_free_servers) + for (serv = daemon->servers, up = &daemon->servers; serv; serv = tmp) + { + tmp = serv->next; + if ((serv->flags & SERV_MARK) && + hostname_isequal(alloc_domain, serv->domain)) + { + /* Need to move down? */ + if (serv->next) + { + *up = serv->next; + daemon->servers_tail->next = serv; + daemon->servers_tail = serv; + serv->next = NULL; + } + break; + } + else + up = &serv->next; + } + if (serv) { free(alloc_domain); diff --git a/src/dnsmasq/option.c b/src/dnsmasq/option.c index 0ad660fd..d35745e5 100644 --- a/src/dnsmasq/option.c +++ b/src/dnsmasq/option.c @@ -5763,7 +5763,10 @@ void read_opts(int argc, char **argv, char *compile_opts) /******** Pi-hole modification ********/ add_txt("version.FTL", (char*)get_FTL_version(), 0 ); /**************************************/ - + + /* See comment above make_servers(). Optimises server-read code. */ + mark_servers(0); + while (1) { #ifdef HAVE_GETOPT_LONG From f74523d3910e6bceddab051e06a187135f0e5678 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Men=C5=A1=C3=ADk?= Date: Tue, 22 Nov 2022 22:51:11 +0000 Subject: [PATCH 186/218] fixup! Handle DS records for unsupported crypto algorithms. --- src/dnsmasq/dnssec.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dnsmasq/dnssec.c b/src/dnsmasq/dnssec.c index ce01b2e6..219ba9af 100644 --- a/src/dnsmasq/dnssec.c +++ b/src/dnsmasq/dnssec.c @@ -1068,7 +1068,7 @@ int dnssec_validate_ds(time_t now, struct dns_header *header, size_t plen, char algo = *p++; digest = *p++; - if (!ds_digest_name(digest) || !ds_digest_name(digest)) + if (!ds_digest_name(digest) || !algo_digest_name(algo)) { a.log.keytag = keytag; a.log.algo = algo; From 3033f5791e9ea049f31bb6b16efe1b4bfbaf1e64 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 23 Nov 2022 04:17:12 +0100 Subject: [PATCH 187/218] Update embedded dnsmasq to v2.88rc4 Signed-off-by: DL6ER --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 95b08c70..27b799d9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -11,6 +11,6 @@ cmake_minimum_required(VERSION 2.8.12) project(PIHOLE_FTL C) -set(DNSMASQ_VERSION pi-hole-v2.88rc3) +set(DNSMASQ_VERSION pi-hole-v2.88rc4) add_subdirectory(src) From 51b2b71756ceb43606a09a79ef14ce478511129b Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 16 Nov 2022 03:56:58 +0100 Subject: [PATCH 188/218] Print extended error code if anything in dbquery() fails. This will be helpful in a lot of cases where we'd only be logging 'disk I/O error' but a more specififc error is available Signed-off-by: DL6ER --- src/database/common.c | 114 ++++++++++++++++++++++++++++++++++++++++- src/database/common.h | 3 ++ src/database/shell.c | 2 +- src/database/sqlite3.c | 6 +-- src/database/sqlite3.h | 2 +- 5 files changed, 119 insertions(+), 8 deletions(-) diff --git a/src/database/common.c b/src/database/common.c index e8bfb5a1..33b5dff1 100644 --- a/src/database/common.c +++ b/src/database/common.c @@ -146,8 +146,8 @@ int dbquery(sqlite3* db, const char *format, ...) int rc = sqlite3_exec(db, query, NULL, NULL, NULL); if( rc != SQLITE_OK ){ - logg("ERROR: SQL query \"%s\" failed: %s", - query, sqlite3_errstr(rc)); + logg("ERROR: SQL query \"%s\" failed: %s (%s)", + query, sqlite3_errstr(rc), sqlite3ErrName(sqlite3_extended_errcode(db))); sqlite3_free(query); checkFTLDBrc(rc); return rc; @@ -634,3 +634,113 @@ const char *get_sqlite3_version(void) { return sqlite3_libversion(); } + +/* ---- copied from sqlite3.c where it is used as private debugging code ---- +** Return a static string containing the name corresponding to the error code +** specified in the argument. +*/ +const char *sqlite3ErrName(int rc){ + const char *zName = 0; + int i, origRc = rc; + for(i=0; i<2 && zName==0; i++, rc &= 0xff){ + switch( rc ){ + case SQLITE_OK: zName = "SQLITE_OK"; break; + case SQLITE_ERROR: zName = "SQLITE_ERROR"; break; + case SQLITE_ERROR_SNAPSHOT: zName = "SQLITE_ERROR_SNAPSHOT"; break; + case SQLITE_INTERNAL: zName = "SQLITE_INTERNAL"; break; + case SQLITE_PERM: zName = "SQLITE_PERM"; break; + case SQLITE_ABORT: zName = "SQLITE_ABORT"; break; + case SQLITE_ABORT_ROLLBACK: zName = "SQLITE_ABORT_ROLLBACK"; break; + case SQLITE_BUSY: zName = "SQLITE_BUSY"; break; + case SQLITE_BUSY_RECOVERY: zName = "SQLITE_BUSY_RECOVERY"; break; + case SQLITE_BUSY_SNAPSHOT: zName = "SQLITE_BUSY_SNAPSHOT"; break; + case SQLITE_LOCKED: zName = "SQLITE_LOCKED"; break; + case SQLITE_LOCKED_SHAREDCACHE: zName = "SQLITE_LOCKED_SHAREDCACHE";break; + case SQLITE_NOMEM: zName = "SQLITE_NOMEM"; break; + case SQLITE_READONLY: zName = "SQLITE_READONLY"; break; + case SQLITE_READONLY_RECOVERY: zName = "SQLITE_READONLY_RECOVERY"; break; + case SQLITE_READONLY_CANTINIT: zName = "SQLITE_READONLY_CANTINIT"; break; + case SQLITE_READONLY_ROLLBACK: zName = "SQLITE_READONLY_ROLLBACK"; break; + case SQLITE_READONLY_DBMOVED: zName = "SQLITE_READONLY_DBMOVED"; break; + case SQLITE_READONLY_DIRECTORY: zName = "SQLITE_READONLY_DIRECTORY";break; + case SQLITE_INTERRUPT: zName = "SQLITE_INTERRUPT"; break; + case SQLITE_IOERR: zName = "SQLITE_IOERR"; break; + case SQLITE_IOERR_READ: zName = "SQLITE_IOERR_READ"; break; + case SQLITE_IOERR_SHORT_READ: zName = "SQLITE_IOERR_SHORT_READ"; break; + case SQLITE_IOERR_WRITE: zName = "SQLITE_IOERR_WRITE"; break; + case SQLITE_IOERR_FSYNC: zName = "SQLITE_IOERR_FSYNC"; break; + case SQLITE_IOERR_DIR_FSYNC: zName = "SQLITE_IOERR_DIR_FSYNC"; break; + case SQLITE_IOERR_TRUNCATE: zName = "SQLITE_IOERR_TRUNCATE"; break; + case SQLITE_IOERR_FSTAT: zName = "SQLITE_IOERR_FSTAT"; break; + case SQLITE_IOERR_UNLOCK: zName = "SQLITE_IOERR_UNLOCK"; break; + case SQLITE_IOERR_RDLOCK: zName = "SQLITE_IOERR_RDLOCK"; break; + case SQLITE_IOERR_DELETE: zName = "SQLITE_IOERR_DELETE"; break; + case SQLITE_IOERR_NOMEM: zName = "SQLITE_IOERR_NOMEM"; break; + case SQLITE_IOERR_ACCESS: zName = "SQLITE_IOERR_ACCESS"; break; + case SQLITE_IOERR_CHECKRESERVEDLOCK: + zName = "SQLITE_IOERR_CHECKRESERVEDLOCK"; break; + case SQLITE_IOERR_LOCK: zName = "SQLITE_IOERR_LOCK"; break; + case SQLITE_IOERR_CLOSE: zName = "SQLITE_IOERR_CLOSE"; break; + case SQLITE_IOERR_DIR_CLOSE: zName = "SQLITE_IOERR_DIR_CLOSE"; break; + case SQLITE_IOERR_SHMOPEN: zName = "SQLITE_IOERR_SHMOPEN"; break; + case SQLITE_IOERR_SHMSIZE: zName = "SQLITE_IOERR_SHMSIZE"; break; + case SQLITE_IOERR_SHMLOCK: zName = "SQLITE_IOERR_SHMLOCK"; break; + case SQLITE_IOERR_SHMMAP: zName = "SQLITE_IOERR_SHMMAP"; break; + case SQLITE_IOERR_SEEK: zName = "SQLITE_IOERR_SEEK"; break; + case SQLITE_IOERR_DELETE_NOENT: zName = "SQLITE_IOERR_DELETE_NOENT";break; + case SQLITE_IOERR_MMAP: zName = "SQLITE_IOERR_MMAP"; break; + case SQLITE_IOERR_GETTEMPPATH: zName = "SQLITE_IOERR_GETTEMPPATH"; break; + case SQLITE_IOERR_CONVPATH: zName = "SQLITE_IOERR_CONVPATH"; break; + case SQLITE_CORRUPT: zName = "SQLITE_CORRUPT"; break; + case SQLITE_CORRUPT_VTAB: zName = "SQLITE_CORRUPT_VTAB"; break; + case SQLITE_NOTFOUND: zName = "SQLITE_NOTFOUND"; break; + case SQLITE_FULL: zName = "SQLITE_FULL"; break; + case SQLITE_CANTOPEN: zName = "SQLITE_CANTOPEN"; break; + case SQLITE_CANTOPEN_NOTEMPDIR: zName = "SQLITE_CANTOPEN_NOTEMPDIR";break; + case SQLITE_CANTOPEN_ISDIR: zName = "SQLITE_CANTOPEN_ISDIR"; break; + case SQLITE_CANTOPEN_FULLPATH: zName = "SQLITE_CANTOPEN_FULLPATH"; break; + case SQLITE_CANTOPEN_CONVPATH: zName = "SQLITE_CANTOPEN_CONVPATH"; break; + case SQLITE_CANTOPEN_SYMLINK: zName = "SQLITE_CANTOPEN_SYMLINK"; break; + case SQLITE_PROTOCOL: zName = "SQLITE_PROTOCOL"; break; + case SQLITE_EMPTY: zName = "SQLITE_EMPTY"; break; + case SQLITE_SCHEMA: zName = "SQLITE_SCHEMA"; break; + case SQLITE_TOOBIG: zName = "SQLITE_TOOBIG"; break; + case SQLITE_CONSTRAINT: zName = "SQLITE_CONSTRAINT"; break; + case SQLITE_CONSTRAINT_UNIQUE: zName = "SQLITE_CONSTRAINT_UNIQUE"; break; + case SQLITE_CONSTRAINT_TRIGGER: zName = "SQLITE_CONSTRAINT_TRIGGER";break; + case SQLITE_CONSTRAINT_FOREIGNKEY: + zName = "SQLITE_CONSTRAINT_FOREIGNKEY"; break; + case SQLITE_CONSTRAINT_CHECK: zName = "SQLITE_CONSTRAINT_CHECK"; break; + case SQLITE_CONSTRAINT_PRIMARYKEY: + zName = "SQLITE_CONSTRAINT_PRIMARYKEY"; break; + case SQLITE_CONSTRAINT_NOTNULL: zName = "SQLITE_CONSTRAINT_NOTNULL";break; + case SQLITE_CONSTRAINT_COMMITHOOK: + zName = "SQLITE_CONSTRAINT_COMMITHOOK"; break; + case SQLITE_CONSTRAINT_VTAB: zName = "SQLITE_CONSTRAINT_VTAB"; break; + case SQLITE_CONSTRAINT_FUNCTION: + zName = "SQLITE_CONSTRAINT_FUNCTION"; break; + case SQLITE_CONSTRAINT_ROWID: zName = "SQLITE_CONSTRAINT_ROWID"; break; + case SQLITE_MISMATCH: zName = "SQLITE_MISMATCH"; break; + case SQLITE_MISUSE: zName = "SQLITE_MISUSE"; break; + case SQLITE_NOLFS: zName = "SQLITE_NOLFS"; break; + case SQLITE_AUTH: zName = "SQLITE_AUTH"; break; + case SQLITE_FORMAT: zName = "SQLITE_FORMAT"; break; + case SQLITE_RANGE: zName = "SQLITE_RANGE"; break; + case SQLITE_NOTADB: zName = "SQLITE_NOTADB"; break; + case SQLITE_ROW: zName = "SQLITE_ROW"; break; + case SQLITE_NOTICE: zName = "SQLITE_NOTICE"; break; + case SQLITE_NOTICE_RECOVER_WAL: zName = "SQLITE_NOTICE_RECOVER_WAL";break; + case SQLITE_NOTICE_RECOVER_ROLLBACK: + zName = "SQLITE_NOTICE_RECOVER_ROLLBACK"; break; + case SQLITE_WARNING: zName = "SQLITE_WARNING"; break; + case SQLITE_WARNING_AUTOINDEX: zName = "SQLITE_WARNING_AUTOINDEX"; break; + case SQLITE_DONE: zName = "SQLITE_DONE"; break; + } + } + if( zName==0 ){ + static char zBuf[50]; + sqlite3_snprintf(sizeof(zBuf), zBuf, "SQLITE_UNKNOWN(%d)", origRc); + zName = zBuf; + } + return zName; +} diff --git a/src/database/common.h b/src/database/common.h index bacc5016..57026dd4 100644 --- a/src/database/common.h +++ b/src/database/common.h @@ -54,6 +54,9 @@ bool FTLDBerror(void) __attribute__ ((pure)); // Check SQLite3 non-success return codes for possible database corruption bool checkFTLDBrc(const int rc); +// Get human-readable *extended* error codes +const char *sqlite3ErrName(int rc); + // Database macros #define SQL_bool(db, ...) {\ int ret;\ diff --git a/src/database/shell.c b/src/database/shell.c index 83c42640..a5e82f70 100644 --- a/src/database/shell.c +++ b/src/database/shell.c @@ -26580,4 +26580,4 @@ void fiddle_exec(const char * zSql){ shellState.wasm.zInput = shellState.wasm.zPos = 0; } } -#endif /* SQLITE_SHELL_FIDDLE */ \ No newline at end of file +#endif /* SQLITE_SHELL_FIDDLE */ diff --git a/src/database/sqlite3.c b/src/database/sqlite3.c index 73c575ad..55f88efb 100644 --- a/src/database/sqlite3.c +++ b/src/database/sqlite3.c @@ -173885,8 +173885,7 @@ SQLITE_PRIVATE void sqlite3RollbackAll(sqlite3 *db, int tripCode){ ** Return a static string containing the name corresponding to the error code ** specified in the argument. */ -#if defined(SQLITE_NEED_ERR_NAME) -SQLITE_PRIVATE const char *sqlite3ErrName(int rc){ +SQLITE_API const char *sqlite3ErrName(int rc){ const char *zName = 0; int i, origRc = rc; for(i=0; i<2 && zName==0; i++, rc &= 0xff){ @@ -173991,7 +173990,6 @@ SQLITE_PRIVATE const char *sqlite3ErrName(int rc){ } return zName; } -#endif /* ** Return a static string that describes the kind of error specified in the @@ -243517,4 +243515,4 @@ SQLITE_API int sqlite3_stmt_init( /************** End of stmt.c ************************************************/ /* Return the source-id for this library */ SQLITE_API const char *sqlite3_sourceid(void){ return SQLITE_SOURCE_ID; } -/************************** End of sqlite3.c ******************************/ \ No newline at end of file +/************************** End of sqlite3.c ******************************/ diff --git a/src/database/sqlite3.h b/src/database/sqlite3.h index c46fd8c5..2859cc09 100644 --- a/src/database/sqlite3.h +++ b/src/database/sqlite3.h @@ -12884,4 +12884,4 @@ struct fts5_api { #endif /* _FTS5_H */ -/******** End of fts5.h *********/ \ No newline at end of file +/******** End of fts5.h *********/ From 5252aeb0778acd2bd60babd79e2b8f3e30fbcce8 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 25 Nov 2022 11:31:05 +0100 Subject: [PATCH 189/218] Reference sqlite3ErrName instead of copying it. This requires SQLite3 patching Signed-off-by: DL6ER --- patch/sqlite3.sh | 5 + ...int-FTL-version-in-interactive-shell.patch | 30 +++++ .../0002-make-sqlite3ErrName-public.patch | 22 ++++ src/database/common.c | 110 ------------------ src/database/common.h | 4 +- 5 files changed, 59 insertions(+), 112 deletions(-) create mode 100644 patch/sqlite3.sh create mode 100644 patch/sqlite3/0001-print-FTL-version-in-interactive-shell.patch create mode 100644 patch/sqlite3/0002-make-sqlite3ErrName-public.patch diff --git a/patch/sqlite3.sh b/patch/sqlite3.sh new file mode 100644 index 00000000..dd0c103a --- /dev/null +++ b/patch/sqlite3.sh @@ -0,0 +1,5 @@ +#!/bin/sh +set -e + +patch src/database/shell.c patches/sqlite3/0001-print-FTL-version-in-interactive-shell.patch +patch src/database/sqlite3.c patches/sqlite3/0002-make-sqlite3ErrName-public.patch diff --git a/patch/sqlite3/0001-print-FTL-version-in-interactive-shell.patch b/patch/sqlite3/0001-print-FTL-version-in-interactive-shell.patch new file mode 100644 index 00000000..02ed43bf --- /dev/null +++ b/patch/sqlite3/0001-print-FTL-version-in-interactive-shell.patch @@ -0,0 +1,30 @@ +diff --git a/src/database/shell.c b/src/database/shell.c +index 6280ebf6..a5e82f70 100644 +--- a/src/database/shell.c ++++ b/src/database/shell.c +@@ -126,6 +126,8 @@ typedef unsigned char u8; + #endif + #include + #include ++// print_FTL_version() ++#include "../log.h" + + #if !defined(_WIN32) && !defined(WIN32) + # include +@@ -25855,7 +25857,7 @@ static char *cmdline_option_value(int argc, char **argv, int i){ + #endif + + #if SQLITE_SHELL_IS_UTF8 +-int SQLITE_CDECL main(int argc, char **argv){ ++int SQLITE_CDECL sqlite3_shell_main(int argc, char **argv){ + #else + int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ + char **argv; +@@ -26377,6 +26379,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ + char *zHome; + char *zHistory; + int nHistory; ++ print_FTL_version(); + printf( + "SQLite version %s %.19s\n" /*extra-version-info*/ + "Enter \".help\" for usage hints.\n", diff --git a/patch/sqlite3/0002-make-sqlite3ErrName-public.patch b/patch/sqlite3/0002-make-sqlite3ErrName-public.patch new file mode 100644 index 00000000..92cf9a72 --- /dev/null +++ b/patch/sqlite3/0002-make-sqlite3ErrName-public.patch @@ -0,0 +1,22 @@ +diff --git a/src/database/sqlite3.c b/src/database/sqlite3.c +index 2763b1b4..55f88efb 100644 +--- a/src/database/sqlite3.c ++++ b/src/database/sqlite3.c +@@ -173885,8 +173885,7 @@ SQLITE_PRIVATE void sqlite3RollbackAll(sqlite3 *db, int tripCode){ + ** Return a static string containing the name corresponding to the error code + ** specified in the argument. + */ +-#if defined(SQLITE_NEED_ERR_NAME) +-SQLITE_PRIVATE const char *sqlite3ErrName(int rc){ ++SQLITE_API const char *sqlite3ErrName(int rc){ + const char *zName = 0; + int i, origRc = rc; + for(i=0; i<2 && zName==0; i++, rc &= 0xff){ +@@ -173991,7 +173990,6 @@ SQLITE_PRIVATE const char *sqlite3ErrName(int rc){ + } + return zName; + } +-#endif + + /* + ** Return a static string that describes the kind of error specified in the diff --git a/src/database/common.c b/src/database/common.c index 33b5dff1..1c9e01a2 100644 --- a/src/database/common.c +++ b/src/database/common.c @@ -634,113 +634,3 @@ const char *get_sqlite3_version(void) { return sqlite3_libversion(); } - -/* ---- copied from sqlite3.c where it is used as private debugging code ---- -** Return a static string containing the name corresponding to the error code -** specified in the argument. -*/ -const char *sqlite3ErrName(int rc){ - const char *zName = 0; - int i, origRc = rc; - for(i=0; i<2 && zName==0; i++, rc &= 0xff){ - switch( rc ){ - case SQLITE_OK: zName = "SQLITE_OK"; break; - case SQLITE_ERROR: zName = "SQLITE_ERROR"; break; - case SQLITE_ERROR_SNAPSHOT: zName = "SQLITE_ERROR_SNAPSHOT"; break; - case SQLITE_INTERNAL: zName = "SQLITE_INTERNAL"; break; - case SQLITE_PERM: zName = "SQLITE_PERM"; break; - case SQLITE_ABORT: zName = "SQLITE_ABORT"; break; - case SQLITE_ABORT_ROLLBACK: zName = "SQLITE_ABORT_ROLLBACK"; break; - case SQLITE_BUSY: zName = "SQLITE_BUSY"; break; - case SQLITE_BUSY_RECOVERY: zName = "SQLITE_BUSY_RECOVERY"; break; - case SQLITE_BUSY_SNAPSHOT: zName = "SQLITE_BUSY_SNAPSHOT"; break; - case SQLITE_LOCKED: zName = "SQLITE_LOCKED"; break; - case SQLITE_LOCKED_SHAREDCACHE: zName = "SQLITE_LOCKED_SHAREDCACHE";break; - case SQLITE_NOMEM: zName = "SQLITE_NOMEM"; break; - case SQLITE_READONLY: zName = "SQLITE_READONLY"; break; - case SQLITE_READONLY_RECOVERY: zName = "SQLITE_READONLY_RECOVERY"; break; - case SQLITE_READONLY_CANTINIT: zName = "SQLITE_READONLY_CANTINIT"; break; - case SQLITE_READONLY_ROLLBACK: zName = "SQLITE_READONLY_ROLLBACK"; break; - case SQLITE_READONLY_DBMOVED: zName = "SQLITE_READONLY_DBMOVED"; break; - case SQLITE_READONLY_DIRECTORY: zName = "SQLITE_READONLY_DIRECTORY";break; - case SQLITE_INTERRUPT: zName = "SQLITE_INTERRUPT"; break; - case SQLITE_IOERR: zName = "SQLITE_IOERR"; break; - case SQLITE_IOERR_READ: zName = "SQLITE_IOERR_READ"; break; - case SQLITE_IOERR_SHORT_READ: zName = "SQLITE_IOERR_SHORT_READ"; break; - case SQLITE_IOERR_WRITE: zName = "SQLITE_IOERR_WRITE"; break; - case SQLITE_IOERR_FSYNC: zName = "SQLITE_IOERR_FSYNC"; break; - case SQLITE_IOERR_DIR_FSYNC: zName = "SQLITE_IOERR_DIR_FSYNC"; break; - case SQLITE_IOERR_TRUNCATE: zName = "SQLITE_IOERR_TRUNCATE"; break; - case SQLITE_IOERR_FSTAT: zName = "SQLITE_IOERR_FSTAT"; break; - case SQLITE_IOERR_UNLOCK: zName = "SQLITE_IOERR_UNLOCK"; break; - case SQLITE_IOERR_RDLOCK: zName = "SQLITE_IOERR_RDLOCK"; break; - case SQLITE_IOERR_DELETE: zName = "SQLITE_IOERR_DELETE"; break; - case SQLITE_IOERR_NOMEM: zName = "SQLITE_IOERR_NOMEM"; break; - case SQLITE_IOERR_ACCESS: zName = "SQLITE_IOERR_ACCESS"; break; - case SQLITE_IOERR_CHECKRESERVEDLOCK: - zName = "SQLITE_IOERR_CHECKRESERVEDLOCK"; break; - case SQLITE_IOERR_LOCK: zName = "SQLITE_IOERR_LOCK"; break; - case SQLITE_IOERR_CLOSE: zName = "SQLITE_IOERR_CLOSE"; break; - case SQLITE_IOERR_DIR_CLOSE: zName = "SQLITE_IOERR_DIR_CLOSE"; break; - case SQLITE_IOERR_SHMOPEN: zName = "SQLITE_IOERR_SHMOPEN"; break; - case SQLITE_IOERR_SHMSIZE: zName = "SQLITE_IOERR_SHMSIZE"; break; - case SQLITE_IOERR_SHMLOCK: zName = "SQLITE_IOERR_SHMLOCK"; break; - case SQLITE_IOERR_SHMMAP: zName = "SQLITE_IOERR_SHMMAP"; break; - case SQLITE_IOERR_SEEK: zName = "SQLITE_IOERR_SEEK"; break; - case SQLITE_IOERR_DELETE_NOENT: zName = "SQLITE_IOERR_DELETE_NOENT";break; - case SQLITE_IOERR_MMAP: zName = "SQLITE_IOERR_MMAP"; break; - case SQLITE_IOERR_GETTEMPPATH: zName = "SQLITE_IOERR_GETTEMPPATH"; break; - case SQLITE_IOERR_CONVPATH: zName = "SQLITE_IOERR_CONVPATH"; break; - case SQLITE_CORRUPT: zName = "SQLITE_CORRUPT"; break; - case SQLITE_CORRUPT_VTAB: zName = "SQLITE_CORRUPT_VTAB"; break; - case SQLITE_NOTFOUND: zName = "SQLITE_NOTFOUND"; break; - case SQLITE_FULL: zName = "SQLITE_FULL"; break; - case SQLITE_CANTOPEN: zName = "SQLITE_CANTOPEN"; break; - case SQLITE_CANTOPEN_NOTEMPDIR: zName = "SQLITE_CANTOPEN_NOTEMPDIR";break; - case SQLITE_CANTOPEN_ISDIR: zName = "SQLITE_CANTOPEN_ISDIR"; break; - case SQLITE_CANTOPEN_FULLPATH: zName = "SQLITE_CANTOPEN_FULLPATH"; break; - case SQLITE_CANTOPEN_CONVPATH: zName = "SQLITE_CANTOPEN_CONVPATH"; break; - case SQLITE_CANTOPEN_SYMLINK: zName = "SQLITE_CANTOPEN_SYMLINK"; break; - case SQLITE_PROTOCOL: zName = "SQLITE_PROTOCOL"; break; - case SQLITE_EMPTY: zName = "SQLITE_EMPTY"; break; - case SQLITE_SCHEMA: zName = "SQLITE_SCHEMA"; break; - case SQLITE_TOOBIG: zName = "SQLITE_TOOBIG"; break; - case SQLITE_CONSTRAINT: zName = "SQLITE_CONSTRAINT"; break; - case SQLITE_CONSTRAINT_UNIQUE: zName = "SQLITE_CONSTRAINT_UNIQUE"; break; - case SQLITE_CONSTRAINT_TRIGGER: zName = "SQLITE_CONSTRAINT_TRIGGER";break; - case SQLITE_CONSTRAINT_FOREIGNKEY: - zName = "SQLITE_CONSTRAINT_FOREIGNKEY"; break; - case SQLITE_CONSTRAINT_CHECK: zName = "SQLITE_CONSTRAINT_CHECK"; break; - case SQLITE_CONSTRAINT_PRIMARYKEY: - zName = "SQLITE_CONSTRAINT_PRIMARYKEY"; break; - case SQLITE_CONSTRAINT_NOTNULL: zName = "SQLITE_CONSTRAINT_NOTNULL";break; - case SQLITE_CONSTRAINT_COMMITHOOK: - zName = "SQLITE_CONSTRAINT_COMMITHOOK"; break; - case SQLITE_CONSTRAINT_VTAB: zName = "SQLITE_CONSTRAINT_VTAB"; break; - case SQLITE_CONSTRAINT_FUNCTION: - zName = "SQLITE_CONSTRAINT_FUNCTION"; break; - case SQLITE_CONSTRAINT_ROWID: zName = "SQLITE_CONSTRAINT_ROWID"; break; - case SQLITE_MISMATCH: zName = "SQLITE_MISMATCH"; break; - case SQLITE_MISUSE: zName = "SQLITE_MISUSE"; break; - case SQLITE_NOLFS: zName = "SQLITE_NOLFS"; break; - case SQLITE_AUTH: zName = "SQLITE_AUTH"; break; - case SQLITE_FORMAT: zName = "SQLITE_FORMAT"; break; - case SQLITE_RANGE: zName = "SQLITE_RANGE"; break; - case SQLITE_NOTADB: zName = "SQLITE_NOTADB"; break; - case SQLITE_ROW: zName = "SQLITE_ROW"; break; - case SQLITE_NOTICE: zName = "SQLITE_NOTICE"; break; - case SQLITE_NOTICE_RECOVER_WAL: zName = "SQLITE_NOTICE_RECOVER_WAL";break; - case SQLITE_NOTICE_RECOVER_ROLLBACK: - zName = "SQLITE_NOTICE_RECOVER_ROLLBACK"; break; - case SQLITE_WARNING: zName = "SQLITE_WARNING"; break; - case SQLITE_WARNING_AUTOINDEX: zName = "SQLITE_WARNING_AUTOINDEX"; break; - case SQLITE_DONE: zName = "SQLITE_DONE"; break; - } - } - if( zName==0 ){ - static char zBuf[50]; - sqlite3_snprintf(sizeof(zBuf), zBuf, "SQLITE_UNKNOWN(%d)", origRc); - zName = zBuf; - } - return zName; -} diff --git a/src/database/common.h b/src/database/common.h index 57026dd4..c682ecf6 100644 --- a/src/database/common.h +++ b/src/database/common.h @@ -54,8 +54,8 @@ bool FTLDBerror(void) __attribute__ ((pure)); // Check SQLite3 non-success return codes for possible database corruption bool checkFTLDBrc(const int rc); -// Get human-readable *extended* error codes -const char *sqlite3ErrName(int rc); +// Get human-readable *extended* error codes (defined in sqlite3.c) +extern const char *sqlite3ErrName(int rc); // Database macros #define SQL_bool(db, ...) {\ From 167d1a6f93493c169aa36d9c973542a954da16cc Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 25 Nov 2022 11:51:53 +0100 Subject: [PATCH 190/218] Ensure compiled scripts are updated if the scripts themselves are updated Signed-off-by: DL6ER --- build.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/build.sh b/build.sh index e2fd1b56..169cbe04 100755 --- a/build.sh +++ b/build.sh @@ -31,6 +31,14 @@ if [[ -n "${clean}" ]]; then fi fi +# Remove compiled LUA scripts if older than the plain ones +for scriptname in src/lua/scripts/*.lua; do + if [ -f "${scriptname}.hex" ] && [ "${scriptname}.hex" -ot "${scriptname}" ]; then + echo "INFO: ${scriptname} is outdated and will be recompiled" + rm "${scriptname}.hex" + fi +done + # Configure build, pass CMake CACHE entries if present # Wrap multiple options in "" as first argument to ./build.sh: # ./build.sh "-DA=1 -DB=2" install From a1ba4bc97d91c25b5d443a37521e341ae917fbfd Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 25 Nov 2022 11:52:12 +0100 Subject: [PATCH 191/218] Update inspect.lua Signed-off-by: DL6ER --- src/lua/scripts/inspect.lua | 526 ++++++++++++++++++------------------ 1 file changed, 262 insertions(+), 264 deletions(-) diff --git a/src/lua/scripts/inspect.lua b/src/lua/scripts/inspect.lua index 8d33a981..ebe722d9 100644 --- a/src/lua/scripts/inspect.lua +++ b/src/lua/scripts/inspect.lua @@ -1,335 +1,333 @@ -local inspect ={ - _VERSION = 'inspect.lua 3.1.0', - _URL = 'http://github.com/kikito/inspect.lua', - _DESCRIPTION = 'human-readable representations of tables', - _LICENSE = [[ - MIT LICENSE +local _tl_compat; if (tonumber((_VERSION or ''):match('[%d.]*$')) or 0) < 5.3 then local p, m = pcall(require, 'compat53.module'); if p then _tl_compat = m end end; local math = _tl_compat and _tl_compat.math or math; local string = _tl_compat and _tl_compat.string or string; local table = _tl_compat and _tl_compat.table or table +local inspect = {Options = {}, } - Copyright (c) 2013 Enrique García Cota - Permission is hereby granted, free of charge, to any person obtaining a - copy of this software and associated documentation files (the - "Software"), to deal in the Software without restriction, including - without limitation the rights to use, copy, modify, merge, publish, - distribute, sublicense, and/or sell copies of the Software, and to - permit persons to whom the Software is furnished to do so, subject to - the following conditions: - The above copyright notice and this permission notice shall be included - in all copies or substantial portions of the Software. - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS - OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY - CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, - TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE - SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - ]] -} + + + + + + + + + + + + + +inspect._VERSION = 'inspect.lua 3.1.0' +inspect._URL = 'http://github.com/kikito/inspect.lua' +inspect._DESCRIPTION = 'human-readable representations of tables' +inspect._LICENSE = [[ + MIT LICENSE + Copyright (c) 2022 Enrique García Cota + Permission is hereby granted, free of charge, to any person obtaining a + copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to + permit persons to whom the Software is furnished to do so, subject to + the following conditions: + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY + CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, + TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +]] +inspect.KEY = setmetatable({}, { __tostring = function() return 'inspect.KEY' end }) +inspect.METATABLE = setmetatable({}, { __tostring = function() return 'inspect.METATABLE' end }) local tostring = tostring - -inspect.KEY = setmetatable({}, {__tostring = function() return 'inspect.KEY' end}) -inspect.METATABLE = setmetatable({}, {__tostring = function() return 'inspect.METATABLE' end}) +local rep = string.rep +local match = string.match +local char = string.char +local gsub = string.gsub +local fmt = string.format local function rawpairs(t) - return next, t, nil + return next, t, nil end --- Apostrophizes the string if it has quotes, but not aphostrophes --- Otherwise, it returns a regular quoted string + + local function smartQuote(str) - if str:match('"') and not str:match("'") then - return "'" .. str .. "'" - end - return '"' .. str:gsub('"', '\\"') .. '"' + if match(str, '"') and not match(str, "'") then + return "'" .. str .. "'" + end + return '"' .. gsub(str, '"', '\\"') .. '"' end --- \a => '\\a', \0 => '\\0', 31 => '\31' + local shortControlCharEscapes = { - ["\a"] = "\\a", ["\b"] = "\\b", ["\f"] = "\\f", ["\n"] = "\\n", - ["\r"] = "\\r", ["\t"] = "\\t", ["\v"] = "\\v" + ["\a"] = "\\a", ["\b"] = "\\b", ["\f"] = "\\f", ["\n"] = "\\n", + ["\r"] = "\\r", ["\t"] = "\\t", ["\v"] = "\\v", ["\127"] = "\\127", } -local longControlCharEscapes = {} -- \a => nil, \0 => \000, 31 => \031 -for i=0, 31 do - local ch = string.char(i) - if not shortControlCharEscapes[ch] then - shortControlCharEscapes[ch] = "\\"..i - longControlCharEscapes[ch] = string.format("\\%03d", i) - end +local longControlCharEscapes = { ["\127"] = "\127" } +for i = 0, 31 do + local ch = char(i) + if not shortControlCharEscapes[ch] then + shortControlCharEscapes[ch] = "\\" .. i + longControlCharEscapes[ch] = fmt("\\%03d", i) + end end local function escape(str) - return (str:gsub("\\", "\\\\") - :gsub("(%c)%f[0-9]", longControlCharEscapes) - :gsub("%c", shortControlCharEscapes)) + return (gsub(gsub(gsub(str, "\\", "\\\\"), + "(%c)%f[0-9]", longControlCharEscapes), + "%c", shortControlCharEscapes)) end local function isIdentifier(str) - return type(str) == 'string' and str:match( "^[_%a][_%a%d]*$" ) + return type(str) == "string" and not not str:match("^[_%a][_%a%d]*$") end +local flr = math.floor local function isSequenceKey(k, sequenceLength) - return type(k) == 'number' - and 1 <= k - and k <= sequenceLength - and math.floor(k) == k + return type(k) == "number" and + flr(k) == k and + 1 <= (k) and + k <= sequenceLength end local defaultTypeOrders = { - ['number'] = 1, ['boolean'] = 2, ['string'] = 3, ['table'] = 4, - ['function'] = 5, ['userdata'] = 6, ['thread'] = 7 + ['number'] = 1, ['boolean'] = 2, ['string'] = 3, ['table'] = 4, + ['function'] = 5, ['userdata'] = 6, ['thread'] = 7, } local function sortKeys(a, b) - local ta, tb = type(a), type(b) + local ta, tb = type(a), type(b) - -- strings and numbers are sorted numerically/alphabetically - if ta == tb and (ta == 'string' or ta == 'number') then return a < b end - local dta, dtb = defaultTypeOrders[ta], defaultTypeOrders[tb] - -- Two default types are compared according to the defaultTypeOrders table - if dta and dtb then return defaultTypeOrders[ta] < defaultTypeOrders[tb] - elseif dta then return true -- default types before custom ones - elseif dtb then return false -- custom types after default ones - end + if ta == tb and (ta == 'string' or ta == 'number') then + return (a) < (b) + end - -- custom types are sorted out alphabetically - return ta < tb + local dta = defaultTypeOrders[ta] or 100 + local dtb = defaultTypeOrders[tb] or 100 + + + return dta == dtb and ta < tb or dta < dtb end --- For implementation reasons, the behavior of rawlen & # is "undefined" when --- tables aren't pure sequences. So we implement our own # operator. -local function getSequenceLength(t) - local len = 1 - local v = rawget(t,len) - while v ~= nil do - len = len + 1 - v = rawget(t,len) - end - return len - 1 -end +local function getKeys(t) -local function getNonSequentialKeys(t) - local keys, keysLength = {}, 0 - local sequenceLength = getSequenceLength(t) - for k,_ in rawpairs(t) do - if not isSequenceKey(k, sequenceLength) then - keysLength = keysLength + 1 - keys[keysLength] = k - end - end - table.sort(keys, sortKeys) - return keys, keysLength, sequenceLength -end + local seqLen = 1 + while rawget(t, seqLen) ~= nil do + seqLen = seqLen + 1 + end + seqLen = seqLen - 1 -local function countTableAppearances(t, tableAppearances) - tableAppearances = tableAppearances or {} - - if type(t) == 'table' then - if not tableAppearances[t] then - tableAppearances[t] = 1 - for k,v in rawpairs(t) do - countTableAppearances(k, tableAppearances) - countTableAppearances(v, tableAppearances) + local keys, keysLen = {}, 0 + for k in rawpairs(t) do + if not isSequenceKey(k, seqLen) then + keysLen = keysLen + 1 + keys[keysLen] = k end - countTableAppearances(getmetatable(t), tableAppearances) - else - tableAppearances[t] = tableAppearances[t] + 1 - end - end - - return tableAppearances + end + table.sort(keys, sortKeys) + return keys, keysLen, seqLen end -local copySequence = function(s) - local copy, len = {}, #s - for i=1, len do copy[i] = s[i] end - return copy, len -end - -local function makePath(path, ...) - local keys = {...} - local newPath, len = copySequence(path) - for i=1, #keys do - newPath[len + i] = keys[i] - end - return newPath -end - -local function processRecursive(process, item, path, visited) - if item == nil then return nil end - if visited[item] then return visited[item] end - - local processed = process(item, path) - if type(processed) == 'table' then - local processedCopy = {} - visited[item] = processedCopy - local processedKey - - for k,v in rawpairs(processed) do - processedKey = processRecursive(process, k, makePath(path, k, inspect.KEY), visited) - if processedKey ~= nil then - processedCopy[processedKey] = processRecursive(process, v, makePath(path, processedKey), visited) +local function countCycles(x, cycles) + if type(x) == "table" then + if cycles[x] then + cycles[x] = cycles[x] + 1 + else + cycles[x] = 1 + for k, v in rawpairs(x) do + countCycles(k, cycles) + countCycles(v, cycles) + end + countCycles(getmetatable(x), cycles) end - end + end +end - local mt = processRecursive(process, getmetatable(processed), makePath(path, inspect.METATABLE), visited) - if type(mt) ~= 'table' then mt = nil end -- ignore not nil/table __metatable field - setmetatable(processedCopy, mt) - processed = processedCopy - end - return processed +local function makePath(path, a, b) + local newPath = {} + local len = #path + for i = 1, len do newPath[i] = path[i] end + + newPath[len + 1] = a + newPath[len + 2] = b + + return newPath end +local function processRecursive(process, + item, + path, + visited) + if item == nil then return nil end + if visited[item] then return visited[item] end + + local processed = process(item, path) + if type(processed) == "table" then + local processedCopy = {} + visited[item] = processedCopy + local processedKey + + for k, v in rawpairs(processed) do + processedKey = processRecursive(process, k, makePath(path, k, inspect.KEY), visited) + if processedKey ~= nil then + processedCopy[processedKey] = processRecursive(process, v, makePath(path, processedKey), visited) + end + end + + local mt = processRecursive(process, getmetatable(processed), makePath(path, inspect.METATABLE), visited) + if type(mt) ~= 'table' then mt = nil end + setmetatable(processedCopy, mt) + processed = processedCopy + end + return processed +end + +local function puts(buf, str) + buf.n = buf.n + 1 + buf[buf.n] = str +end + -------------------------------------------------------------------- local Inspector = {} -local Inspector_mt = {__index = Inspector} -function Inspector:puts(...) - local args = {...} - local buffer = self.buffer - local len = #buffer - for i=1, #args do - len = len + 1 - buffer[len] = args[i] - end -end -function Inspector:down(f) - self.level = self.level + 1 - f() - self.level = self.level - 1 -end -function Inspector:tabify() - self:puts(self.newline, string.rep(self.indent, self.level)) -end -function Inspector:alreadyVisited(v) - return self.ids[v] ~= nil + + + + + + +local Inspector_mt = { __index = Inspector } + +local function tabify(inspector) + puts(inspector.buf, inspector.newline .. rep(inspector.indent, inspector.level)) end function Inspector:getId(v) - local id = self.ids[v] - if not id then - local tv = type(v) - id = (self.maxIds[tv] or 0) + 1 - self.maxIds[tv] = id - self.ids[v] = id - end - return tostring(id) -end - -function Inspector:putKey(k) - if isIdentifier(k) then return self:puts(k) end - self:puts("[") - self:putValue(k) - self:puts("]") -end - -function Inspector:putTable(t) - if t == inspect.KEY or t == inspect.METATABLE then - self:puts(tostring(t)) - elseif self:alreadyVisited(t) then - self:puts('
      ') - elseif self.level >= self.depth then - self:puts('{...}') - else - if self.tableAppearances[t] > 1 then self:puts('<', self:getId(t), '>') end - - local nonSequentialKeys, nonSequentialKeysLength, sequenceLength = getNonSequentialKeys(t) - local mt = getmetatable(t) - - self:puts('{') - self:down(function() - local count = 0 - for i=1, sequenceLength do - if count > 0 then self:puts(',') end - self:puts(' ') - self:putValue(t[i]) - count = count + 1 - end - - for i=1, nonSequentialKeysLength do - local k = nonSequentialKeys[i] - if count > 0 then self:puts(',') end - self:tabify() - self:putKey(k) - self:puts(' = ') - self:putValue(t[k]) - count = count + 1 - end - - if type(mt) == 'table' then - if count > 0 then self:puts(',') end - self:tabify() - self:puts(' = ') - self:putValue(mt) - end - end) - - if nonSequentialKeysLength > 0 or type(mt) == 'table' then -- result is multi-lined. Justify closing } - self:tabify() - elseif sequenceLength > 0 then -- array tables have one extra space before closing } - self:puts(' ') - end - - self:puts('}') - end + local id = self.ids[v] + local ids = self.ids + if not id then + local tv = type(v) + id = (ids[tv] or 0) + 1 + ids[v], ids[tv] = id, id + end + return tostring(id) end function Inspector:putValue(v) - local tv = type(v) + local buf = self.buf + local tv = type(v) + if tv == 'string' then + puts(buf, smartQuote(escape(v))) + elseif tv == 'number' or tv == 'boolean' or tv == 'nil' or + tv == 'cdata' or tv == 'ctype' then + puts(buf, tostring(v)) + elseif tv == 'table' and not self.ids[v] then + local t = v - if tv == 'string' then - self:puts(smartQuote(escape(v))) - elseif tv == 'number' or tv == 'boolean' or tv == 'nil' or - tv == 'cdata' or tv == 'ctype' then - self:puts(tostring(v)) - elseif tv == 'table' then - self:putTable(v) - else - self:puts('<', tv, ' ', self:getId(v), '>') - end + if t == inspect.KEY or t == inspect.METATABLE then + puts(buf, tostring(t)) + elseif self.level >= self.depth then + puts(buf, '{...}') + else + if self.cycles[t] > 1 then puts(buf, fmt('<%d>', self:getId(t))) end + + local keys, keysLen, seqLen = getKeys(t) + + puts(buf, '{') + self.level = self.level + 1 + + for i = 1, seqLen + keysLen do + if i > 1 then puts(buf, ',') end + if i <= seqLen then + puts(buf, ' ') + self:putValue(t[i]) + else + local k = keys[i - seqLen] + tabify(self) + if isIdentifier(k) then + puts(buf, k) + else + puts(buf, "[") + self:putValue(k) + puts(buf, "]") + end + puts(buf, ' = ') + self:putValue(t[k]) + end + end + + local mt = getmetatable(t) + if type(mt) == 'table' then + if seqLen + keysLen > 0 then puts(buf, ',') end + tabify(self) + puts(buf, ' = ') + self:putValue(mt) + end + + self.level = self.level - 1 + + if keysLen > 0 or type(mt) == 'table' then + tabify(self) + elseif seqLen > 0 then + puts(buf, ' ') + end + + puts(buf, '}') + end + + else + puts(buf, fmt('<%s %d>', tv, self:getId(v))) + end end -------------------------------------------------------------------- + + function inspect.inspect(root, options) - options = options or {} + options = options or {} - local depth = options.depth or math.huge - local newline = options.newline or '\n' - local indent = options.indent or ' ' - local process = options.process + local depth = options.depth or (math.huge) + local newline = options.newline or '\n' + local indent = options.indent or ' ' + local process = options.process - if process then - root = processRecursive(process, root, {}, {}) - end + if process then + root = processRecursive(process, root, {}, {}) + end - local inspector = setmetatable({ - depth = depth, - level = 0, - buffer = {}, - ids = {}, - maxIds = {}, - newline = newline, - indent = indent, - tableAppearances = countTableAppearances(root) - }, Inspector_mt) + local cycles = {} + countCycles(root, cycles) - inspector:putValue(root) + local inspector = setmetatable({ + buf = { n = 0 }, + ids = {}, + cycles = cycles, + depth = depth, + level = 0, + newline = newline, + indent = indent, + }, Inspector_mt) - return table.concat(inspector.buffer) + inspector:putValue(root) + + return table.concat(inspector.buf) end -setmetatable(inspect, { __call = function(_, ...) return inspect.inspect(...) end }) - -return inspect - +setmetatable(inspect, { + __call = function(_, root, options) + return inspect.inspect(root, options) + end, +}) +return inspect \ No newline at end of file From d066946e3f146da1f96baa6c2abf7a3c6abf1f54 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 25 Nov 2022 12:04:26 +0100 Subject: [PATCH 192/218] Print embedded libraries in pihole-FTL -vv Signed-off-by: DL6ER --- src/args.c | 4 +++- src/lua/ftl_lua.c | 28 ++++++++++++++++++++++++++-- src/lua/ftl_lua.h | 1 + 3 files changed, 30 insertions(+), 3 deletions(-) diff --git a/src/args.c b/src/args.c index 439edd56..92965719 100644 --- a/src/args.c +++ b/src/args.c @@ -264,7 +264,9 @@ void parse_args(int argc, char* argv[]) printf("\n"); printf("******************************** LUA ********************************\n"); printf(LUA_COPYRIGHT"\n"); - printf("\n"); + printf("Embedded libraries: "); + print_embedded_scripts(); + printf("\n\n"); printf("***************************** LIBNETTLE *****************************\n"); printf("Version: %d.%d\n", NETTLE_VERSION_MAJOR, NETTLE_VERSION_MINOR); printf("GMP: %s\n", NETTLE_USE_MINI_GMP ? "Mini" : "Full"); diff --git a/src/lua/ftl_lua.c b/src/lua/ftl_lua.c index 72c371c3..2577b90b 100644 --- a/src/lua/ftl_lua.c +++ b/src/lua/ftl_lua.c @@ -120,8 +120,32 @@ static bool ftl_lua_load_embedded_script(lua_State *L, const char *name, const c return true; } -// Load bundled libraries and make the available globally +struct { + const char *name; + const char *content; + const size_t contentlen; + const bool global; +} scripts[] = +{ + {"inspect", inspect_lua, sizeof(inspect_lua), true}, +}; + +// Loop over bundled LUA libraries and print their names on the console +void print_embedded_scripts(void) +{ + for(unsigned int i = 0; i < sizeof(scripts)/sizeof(scripts[0]); i++) + { + char prefix[2] = { 0 }; + double formatted = 0.0; + format_memory_size(prefix, scripts[i].contentlen, &formatted); + + printf("%s.lua (%.2f %sB) ", scripts[i].name, formatted, prefix); + } +} + +// Loop over bundled LUA libraries and load them void ftl_lua_init(lua_State *L) { - ftl_lua_load_embedded_script(L, "inspect", inspect_lua, sizeof(inspect_lua), true); + for(unsigned int i = 0; i < sizeof(scripts)/sizeof(scripts[0]); i++) + ftl_lua_load_embedded_script(L, scripts[i].name, scripts[i].content, scripts[i].contentlen, scripts[i].global); } diff --git a/src/lua/ftl_lua.h b/src/lua/ftl_lua.h index c91ab5c3..6c51a871 100644 --- a/src/lua/ftl_lua.h +++ b/src/lua/ftl_lua.h @@ -21,6 +21,7 @@ int run_luac(const int argc, char **argv); int lua_main (int argc, char **argv); int luac_main (int argc, char **argv); +void print_embedded_scripts(void); void ftl_lua_init(lua_State *L); #endif //FTL_LUA_H \ No newline at end of file From 717181335be3e098548021caaa67737530fc9354 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 25 Nov 2022 18:49:59 +0100 Subject: [PATCH 193/218] Add LUA patches Signed-off-by: DL6ER --- patch/lua.sh | 4 ++ patch/lua/0001-add-pihole-library.patch | 80 +++++++++++++++++++++++++ patch/sqlite3.sh | 4 +- src/lua/linit.c | 2 - src/lua/lua.c | 7 --- src/lua/lualib.h | 2 - 6 files changed, 86 insertions(+), 13 deletions(-) create mode 100644 patch/lua.sh create mode 100644 patch/lua/0001-add-pihole-library.patch diff --git a/patch/lua.sh b/patch/lua.sh new file mode 100644 index 00000000..478d4581 --- /dev/null +++ b/patch/lua.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -e + +patch -p1 < patch/lua/0001-add-pihole-library.patch diff --git a/patch/lua/0001-add-pihole-library.patch b/patch/lua/0001-add-pihole-library.patch new file mode 100644 index 00000000..13ea290e --- /dev/null +++ b/patch/lua/0001-add-pihole-library.patch @@ -0,0 +1,80 @@ +diff --git a/src/lua/linit.c b/src/lua/linit.c +index 69808f84..83b89555 100644 +--- a/src/lua/linit.c ++++ b/src/lua/linit.c +@@ -50,6 +50,7 @@ static const luaL_Reg loadedlibs[] = { + {LUA_MATHLIBNAME, luaopen_math}, + {LUA_UTF8LIBNAME, luaopen_utf8}, + {LUA_DBLIBNAME, luaopen_debug}, ++ {LUA_PIHOLELIBNAME, luaopen_pihole}, + {NULL, NULL} + }; + +diff --git a/src/lua/lua.c b/src/lua/lua.c +index 454ce12f..a363925c 100644 +--- a/src/lua/lua.c ++++ b/src/lua/lua.c +@@ -20,6 +20,8 @@ + #include "lauxlib.h" + #include "lualib.h" + ++#include "ftl_lua.h" ++ + + #if !defined(LUA_PROGNAME) + #define LUA_PROGNAME "lua" +@@ -190,7 +192,7 @@ static int dostring (lua_State *L, const char *s, const char *name) { + ** Calls 'require(name)' and stores the result in a global variable + ** with the given name. + */ +-static int dolibrary (lua_State *L, const char *name) { ++int dolibrary (lua_State *L, const char *name) { + int status; + lua_getglobal(L, "require"); + lua_pushstring(L, name); +@@ -597,6 +599,10 @@ static int pmain (lua_State *L) { + if (handle_luainit(L) != LUA_OK) /* run LUA_INIT */ + return 0; /* error running LUA_INIT */ + } ++ ++ // Load and enable libraries bundled with Pi-hole ++ ftl_lua_init(L); ++ + if (!runargs(L, argv, script)) /* execute arguments -e and -l */ + return 0; /* something failed */ + if (script < argc && /* execute main script (if there is one) */ +@@ -616,7 +622,7 @@ static int pmain (lua_State *L) { + } + + +-int main (int argc, char **argv) { ++int lua_main (int argc, char **argv) { + int status, result; + lua_State *L = luaL_newstate(); /* create state */ + if (L == NULL) { +diff --git a/src/lua/luac.c b/src/lua/luac.c +index 56ddc414..d7d219e4 100644 +--- a/src/lua/luac.c ++++ b/src/lua/luac.c +@@ -194,7 +194,7 @@ static int pmain(lua_State* L) + return 0; + } + +-int main(int argc, char* argv[]) ++int luac_main(int argc, char* argv[]) + { + lua_State* L; + int i=doargs(argc,argv); +diff --git a/src/lua/lualib.h b/src/lua/lualib.h +index eb08b530..e6dfbf6c 100644 +--- a/src/lua/lualib.h ++++ b/src/lua/lualib.h +@@ -44,6 +44,8 @@ LUAMOD_API int (luaopen_debug) (lua_State *L); + #define LUA_LOADLIBNAME "package" + LUAMOD_API int (luaopen_package) (lua_State *L); + ++#define LUA_PIHOLELIBNAME "pihole" ++LUAMOD_API int (luaopen_pihole) (lua_State *L); + + /* open all previous libraries */ + LUALIB_API void (luaL_openlibs) (lua_State *L); diff --git a/patch/sqlite3.sh b/patch/sqlite3.sh index dd0c103a..9b447217 100644 --- a/patch/sqlite3.sh +++ b/patch/sqlite3.sh @@ -1,5 +1,5 @@ #!/bin/sh set -e -patch src/database/shell.c patches/sqlite3/0001-print-FTL-version-in-interactive-shell.patch -patch src/database/sqlite3.c patches/sqlite3/0002-make-sqlite3ErrName-public.patch +patch -p1 < patch/sqlite3/0001-print-FTL-version-in-interactive-shell.patch +patch -p1 < patch/sqlite3/0002-make-sqlite3ErrName-public.patch diff --git a/src/lua/linit.c b/src/lua/linit.c index 446982ef..83b89555 100644 --- a/src/lua/linit.c +++ b/src/lua/linit.c @@ -50,9 +50,7 @@ static const luaL_Reg loadedlibs[] = { {LUA_MATHLIBNAME, luaopen_math}, {LUA_UTF8LIBNAME, luaopen_utf8}, {LUA_DBLIBNAME, luaopen_debug}, - /************** Pi-hole modification ***************/ {LUA_PIHOLELIBNAME, luaopen_pihole}, - /***************************************************/ {NULL, NULL} }; diff --git a/src/lua/lua.c b/src/lua/lua.c index fa52d252..a363925c 100644 --- a/src/lua/lua.c +++ b/src/lua/lua.c @@ -20,9 +20,7 @@ #include "lauxlib.h" #include "lualib.h" -/************** Pi-hole modification ***************/ #include "ftl_lua.h" -/***************************************************/ #if !defined(LUA_PROGNAME) @@ -194,9 +192,7 @@ static int dostring (lua_State *L, const char *s, const char *name) { ** Calls 'require(name)' and stores the result in a global variable ** with the given name. */ -/************** Pi-hole modification ***************/ int dolibrary (lua_State *L, const char *name) { -/***************************************************/ int status; lua_getglobal(L, "require"); lua_pushstring(L, name); @@ -604,11 +600,8 @@ static int pmain (lua_State *L) { return 0; /* error running LUA_INIT */ } - - /************** Pi-hole modification ***************/ // Load and enable libraries bundled with Pi-hole ftl_lua_init(L); - /***************************************************/ if (!runargs(L, argv, script)) /* execute arguments -e and -l */ return 0; /* something failed */ diff --git a/src/lua/lualib.h b/src/lua/lualib.h index a78ecdb2..e6dfbf6c 100644 --- a/src/lua/lualib.h +++ b/src/lua/lualib.h @@ -44,10 +44,8 @@ LUAMOD_API int (luaopen_debug) (lua_State *L); #define LUA_LOADLIBNAME "package" LUAMOD_API int (luaopen_package) (lua_State *L); -/************** Pi-hole modification ***************/ #define LUA_PIHOLELIBNAME "pihole" LUAMOD_API int (luaopen_pihole) (lua_State *L); -/***************************************************/ /* open all previous libraries */ LUALIB_API void (luaL_openlibs) (lua_State *L); From 8957ee456b12cb86ad4abbaffa93be8c4307f627 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 25 Nov 2022 19:09:24 +0100 Subject: [PATCH 194/218] Update embedded LUA to 5.4.4 Signed-off-by: DL6ER --- patch/lua/0001-add-pihole-library.patch | 12 +- src/lua/ftl_lua.h | 2 +- src/lua/lapi.c | 126 +++-- src/lua/lapi.h | 2 + src/lua/lauxlib.c | 147 ++++-- src/lua/lauxlib.h | 31 +- src/lua/lbaselib.c | 40 +- src/lua/lcode.c | 78 +-- src/lua/lcorolib.c | 19 +- src/lua/ldblib.c | 40 +- src/lua/ldebug.c | 218 ++++++--- src/lua/ldebug.h | 11 + src/lua/ldo.c | 609 +++++++++++++++--------- src/lua/ldo.h | 8 +- src/lua/lfunc.c | 172 ++++--- src/lua/lfunc.h | 13 +- src/lua/lgc.c | 109 +++-- src/lua/lgc.h | 10 + src/lua/liolib.c | 35 +- src/lua/llex.c | 56 ++- src/lua/llimits.h | 48 +- src/lua/lmathlib.c | 9 +- src/lua/lmem.c | 33 +- src/lua/loadlib.c | 17 +- src/lua/lobject.c | 12 +- src/lua/lobject.h | 36 +- src/lua/lopcodes.h | 39 +- src/lua/loslib.c | 8 +- src/lua/lparser.c | 142 +++--- src/lua/lparser.h | 11 +- src/lua/lstate.c | 156 +++--- src/lua/lstate.h | 132 ++--- src/lua/lstring.c | 28 +- src/lua/lstring.h | 3 +- src/lua/lstrlib.c | 251 ++++++---- src/lua/ltable.c | 190 +++++--- src/lua/ltable.h | 8 +- src/lua/ltablib.c | 16 +- src/lua/ltm.c | 5 +- src/lua/lua.c | 79 ++- src/lua/lua.h | 11 +- src/lua/luac.c | 7 +- src/lua/luaconf.h | 132 ++--- src/lua/lualib.h | 6 - src/lua/lutf8lib.c | 11 +- src/lua/lvm.c | 184 ++++--- src/lua/lvm.h | 6 +- 47 files changed, 1968 insertions(+), 1350 deletions(-) diff --git a/patch/lua/0001-add-pihole-library.patch b/patch/lua/0001-add-pihole-library.patch index 13ea290e..9b3173c0 100644 --- a/patch/lua/0001-add-pihole-library.patch +++ b/patch/lua/0001-add-pihole-library.patch @@ -24,14 +24,14 @@ index 454ce12f..a363925c 100644 #if !defined(LUA_PROGNAME) #define LUA_PROGNAME "lua" @@ -190,7 +192,7 @@ static int dostring (lua_State *L, const char *s, const char *name) { - ** Calls 'require(name)' and stores the result in a global variable - ** with the given name. + /* + ** Receives 'globname[=modname]' and runs 'globname = require(modname)'. */ --static int dolibrary (lua_State *L, const char *name) { -+int dolibrary (lua_State *L, const char *name) { +-static int dolibrary (lua_State *L, char *globname) { ++int dolibrary (lua_State *L, char *globname) { int status; - lua_getglobal(L, "require"); - lua_pushstring(L, name); + char *modname = strchr(globname, '='); + if (modname == NULL) /* no explicit name? */ @@ -597,6 +599,10 @@ static int pmain (lua_State *L) { if (handle_luainit(L) != LUA_OK) /* run LUA_INIT */ return 0; /* error running LUA_INIT */ diff --git a/src/lua/ftl_lua.h b/src/lua/ftl_lua.h index c2ef12ab..f92c9ff2 100644 --- a/src/lua/ftl_lua.h +++ b/src/lua/ftl_lua.h @@ -21,7 +21,7 @@ int run_luac(const int argc, char **argv); int lua_main (int argc, char **argv); int luac_main (int argc, char **argv); -extern int dolibrary (lua_State *L, const char *name); +extern int dolibrary (lua_State *L, char *name); void ftl_lua_init(lua_State *L); diff --git a/src/lua/lapi.c b/src/lua/lapi.c index 9048245f..5ee65792 100644 --- a/src/lua/lapi.c +++ b/src/lua/lapi.c @@ -39,7 +39,7 @@ const char lua_ident[] = /* -** Test for a valid index. +** Test for a valid index (one that is not the 'nilvalue'). ** '!ttisnil(o)' implies 'o != &G(L)->nilvalue', so it is not needed. ** However, it covers the most common cases in a faster way. */ @@ -53,6 +53,10 @@ const char lua_ident[] = #define isupvalue(i) ((i) < LUA_REGISTRYINDEX) +/* +** Convert an acceptable index to a pointer to its respective value. +** Non-valid indices return the special nil value 'G(L)->nilvalue'. +*/ static TValue *index2value (lua_State *L, int idx) { CallInfo *ci = L->ci; if (idx > 0) { @@ -70,21 +74,28 @@ static TValue *index2value (lua_State *L, int idx) { else { /* upvalues */ idx = LUA_REGISTRYINDEX - idx; api_check(L, idx <= MAXUPVAL + 1, "upvalue index too large"); - if (ttislcf(s2v(ci->func))) /* light C function? */ - return &G(L)->nilvalue; /* it has no upvalues */ - else { + if (ttisCclosure(s2v(ci->func))) { /* C closure? */ CClosure *func = clCvalue(s2v(ci->func)); - return (idx <= func->nupvalues) ? &func->upvalue[idx-1] : &G(L)->nilvalue; + return (idx <= func->nupvalues) ? &func->upvalue[idx-1] + : &G(L)->nilvalue; + } + else { /* light C function or Lua function (through a hook)?) */ + api_check(L, ttislcf(s2v(ci->func)), "caller not a C function"); + return &G(L)->nilvalue; /* no upvalues */ } } } -static StkId index2stack (lua_State *L, int idx) { + +/* +** Convert a valid actual index (not a pseudo-index) to its address. +*/ +l_sinline StkId index2stack (lua_State *L, int idx) { CallInfo *ci = L->ci; if (idx > 0) { StkId o = ci->func + idx; - api_check(L, o < L->top, "unacceptable index"); + api_check(L, o < L->top, "invalid index"); return o; } else { /* non-positive index */ @@ -172,7 +183,7 @@ LUA_API int lua_gettop (lua_State *L) { LUA_API void lua_settop (lua_State *L, int idx) { CallInfo *ci; - StkId func; + StkId func, newtop; ptrdiff_t diff; /* difference for new top */ lua_lock(L); ci = L->ci; @@ -187,9 +198,26 @@ LUA_API void lua_settop (lua_State *L, int idx) { api_check(L, -(idx+1) <= (L->top - (func + 1)), "invalid new top"); diff = idx + 1; /* will "subtract" index (as it is negative) */ } - if (diff < 0 && hastocloseCfunc(ci->nresults)) - luaF_close(L, L->top + diff, LUA_OK); - L->top += diff; /* correct top only after closing any upvalue */ + api_check(L, L->tbclist < L->top, "previous pop of an unclosed slot"); + newtop = L->top + diff; + if (diff < 0 && L->tbclist >= newtop) { + lua_assert(hastocloseCfunc(ci->nresults)); + luaF_close(L, newtop, CLOSEKTOP, 0); + } + L->top = newtop; /* correct top only after closing any upvalue */ + lua_unlock(L); +} + + +LUA_API void lua_closeslot (lua_State *L, int idx) { + StkId level; + lua_lock(L); + level = index2stack(L, idx); + api_check(L, hastocloseCfunc(L->ci->nresults) && L->tbclist == level, + "no variable to close at given level"); + luaF_close(L, level, CLOSEKTOP, 0); + level = index2stack(L, idx); /* stack may be moved */ + setnilvalue(s2v(level)); lua_unlock(L); } @@ -200,7 +228,7 @@ LUA_API void lua_settop (lua_State *L, int idx) { ** Note that we move(copy) only the value inside the stack. ** (We do not move additional fields that may exist.) */ -static void reverse (lua_State *L, StkId from, StkId to) { +l_sinline void reverse (lua_State *L, StkId from, StkId to) { for (; from < to; from++, to--) { TValue temp; setobj(L, &temp, s2v(from)); @@ -420,7 +448,7 @@ LUA_API lua_CFunction lua_tocfunction (lua_State *L, int idx) { } -static void *touserdata (const TValue *o) { +l_sinline void *touserdata (const TValue *o) { switch (ttype(o)) { case LUA_TUSERDATA: return getudatamem(uvalue(o)); case LUA_TLIGHTUSERDATA: return pvalue(o); @@ -612,7 +640,7 @@ LUA_API int lua_pushthread (lua_State *L) { */ -static int auxgetstr (lua_State *L, const TValue *t, const char *k) { +l_sinline int auxgetstr (lua_State *L, const TValue *t, const char *k) { const TValue *slot; TString *str = luaS_new(L, k); if (luaV_fastget(L, t, str, slot, luaH_getstr)) { @@ -629,11 +657,21 @@ static int auxgetstr (lua_State *L, const TValue *t, const char *k) { } +/* +** Get the global table in the registry. Since all predefined +** indices in the registry were inserted right when the registry +** was created and never removed, they must always be in the array +** part of the registry. +*/ +#define getGtable(L) \ + (&hvalue(&G(L)->l_registry)->array[LUA_RIDX_GLOBALS - 1]) + + LUA_API int lua_getglobal (lua_State *L, const char *name) { - Table *reg; + const TValue *G; lua_lock(L); - reg = hvalue(&G(L)->l_registry); - return auxgetstr(L, luaH_getint(reg, LUA_RIDX_GLOBALS), name); + G = getGtable(L); + return auxgetstr(L, G, name); } @@ -677,7 +715,7 @@ LUA_API int lua_geti (lua_State *L, int idx, lua_Integer n) { } -static int finishrawget (lua_State *L, const TValue *val) { +l_sinline int finishrawget (lua_State *L, const TValue *val) { if (isempty(val)) /* avoid copying empty items to the stack */ setnilvalue(s2v(L->top)); else @@ -811,10 +849,10 @@ static void auxsetstr (lua_State *L, const TValue *t, const char *k) { LUA_API void lua_setglobal (lua_State *L, const char *name) { - Table *reg; + const TValue *G; lua_lock(L); /* unlock done in 'auxsetstr' */ - reg = hvalue(&G(L)->l_registry); - auxsetstr(L, luaH_getint(reg, LUA_RIDX_GLOBALS), name); + G = getGtable(L); + auxsetstr(L, G, name); } @@ -861,12 +899,10 @@ LUA_API void lua_seti (lua_State *L, int idx, lua_Integer n) { static void aux_rawset (lua_State *L, int idx, TValue *key, int n) { Table *t; - TValue *slot; lua_lock(L); api_checknelems(L, n); t = gettable(L, idx); - slot = luaH_set(L, t, key); - setobj2t(L, slot, s2v(L->top - 1)); + luaH_set(L, t, key, s2v(L->top - 1)); invalidateTMcache(t); luaC_barrierback(L, obj2gco(t), s2v(L->top - 1)); L->top -= n; @@ -1063,8 +1099,7 @@ LUA_API int lua_load (lua_State *L, lua_Reader reader, void *data, LClosure *f = clLvalue(s2v(L->top - 1)); /* get newly created function */ if (f->nupvalues >= 1) { /* does it have an upvalue? */ /* get global table from registry */ - Table *reg = hvalue(&G(L)->l_registry); - const TValue *gt = luaH_getint(reg, LUA_RIDX_GLOBALS); + const TValue *gt = getGtable(L); /* set global table as 1st upvalue of 'f' (may be LUA_ENV) */ setobj(L, f->upvals[0]->v, gt); luaC_barrier(L, f->upvals[0], gt); @@ -1101,18 +1136,19 @@ LUA_API int lua_status (lua_State *L) { LUA_API int lua_gc (lua_State *L, int what, ...) { va_list argp; int res = 0; - global_State *g; + global_State *g = G(L); + if (g->gcstp & GCSTPGC) /* internal stop? */ + return -1; /* all options are invalid when stopped */ lua_lock(L); - g = G(L); va_start(argp, what); switch (what) { case LUA_GCSTOP: { - g->gcrunning = 0; + g->gcstp = GCSTPUSR; /* stopped by the user */ break; } case LUA_GCRESTART: { luaE_setdebt(g, 0); - g->gcrunning = 1; + g->gcstp = 0; /* (GCSTPGC must be already zero here) */ break; } case LUA_GCCOLLECT: { @@ -1131,8 +1167,8 @@ LUA_API int lua_gc (lua_State *L, int what, ...) { case LUA_GCSTEP: { int data = va_arg(argp, int); l_mem debt = 1; /* =1 to signal that it did an actual step */ - lu_byte oldrunning = g->gcrunning; - g->gcrunning = 1; /* allow GC to run */ + lu_byte oldstp = g->gcstp; + g->gcstp = 0; /* allow GC to run (GCSTPGC must be zero here) */ if (data == 0) { luaE_setdebt(g, 0); /* do a basic step */ luaC_step(L); @@ -1142,7 +1178,7 @@ LUA_API int lua_gc (lua_State *L, int what, ...) { luaE_setdebt(g, debt); luaC_checkGC(L); } - g->gcrunning = oldrunning; /* restore previous state */ + g->gcstp = oldstp; /* restore previous state */ if (debt > 0 && g->gcstate == GCSpause) /* end of cycle? */ res = 1; /* signal it */ break; @@ -1160,7 +1196,7 @@ LUA_API int lua_gc (lua_State *L, int what, ...) { break; } case LUA_GCISRUNNING: { - res = g->gcrunning; + res = gcrunning(g); break; } case LUA_GCGEN: { @@ -1240,8 +1276,7 @@ LUA_API void lua_toclose (lua_State *L, int idx) { lua_lock(L); o = index2stack(L, idx); nresults = L->ci->nresults; - api_check(L, L->openupval == NULL || uplevel(L->openupval) <= o, - "marked index below or equal new one"); + api_check(L, L->tbclist < o, "given index below or equal a marked one"); luaF_newtbcupval(L, o); /* create new to-be-closed upvalue */ if (!hastocloseCfunc(nresults)) /* function not marked yet? */ L->ci->nresults = codeNresults(nresults); /* mark it */ @@ -1383,13 +1418,16 @@ LUA_API const char *lua_setupvalue (lua_State *L, int funcindex, int n) { static UpVal **getupvalref (lua_State *L, int fidx, int n, LClosure **pf) { + static const UpVal *const nullup = NULL; LClosure *f; TValue *fi = index2value(L, fidx); api_check(L, ttisLclosure(fi), "Lua function expected"); f = clLvalue(fi); - api_check(L, (1 <= n && n <= f->p->sizeupvalues), "invalid upvalue index"); if (pf) *pf = f; - return &f->upvals[n - 1]; /* get its upvalue pointer */ + if (1 <= n && n <= f->p->sizeupvalues) + return &f->upvals[n - 1]; /* get its upvalue pointer */ + else + return (UpVal**)&nullup; } @@ -1401,11 +1439,14 @@ LUA_API void *lua_upvalueid (lua_State *L, int fidx, int n) { } case LUA_VCCL: { /* C closure */ CClosure *f = clCvalue(fi); - api_check(L, 1 <= n && n <= f->nupvalues, "invalid upvalue index"); - return &f->upvalue[n - 1]; - } + if (1 <= n && n <= f->nupvalues) + return &f->upvalue[n - 1]; + /* else */ + } /* FALLTHROUGH */ + case LUA_VLCF: + return NULL; /* light C functions have no upvalues */ default: { - api_check(L, 0, "closure expected"); + api_check(L, 0, "function expected"); return NULL; } } @@ -1417,6 +1458,7 @@ LUA_API void lua_upvaluejoin (lua_State *L, int fidx1, int n1, LClosure *f1; UpVal **up1 = getupvalref(L, fidx1, n1, &f1); UpVal **up2 = getupvalref(L, fidx2, n2, NULL); + api_check(L, *up1 != NULL && *up2 != NULL, "invalid upvalue index"); *up1 = *up2; luaC_objbarrier(L, f1, *up1); } diff --git a/src/lua/lapi.h b/src/lua/lapi.h index 41216b27..9e99cc44 100644 --- a/src/lua/lapi.h +++ b/src/lua/lapi.h @@ -42,6 +42,8 @@ #define hastocloseCfunc(n) ((n) < LUA_MULTRET) +/* Map [-1, inf) (range of 'nresults') into (-inf, -2] */ #define codeNresults(n) (-(n) - 3) +#define decodeNresults(n) (-(n) - 3) #endif diff --git a/src/lua/lauxlib.c b/src/lua/lauxlib.c index cbe9ed31..8ed1da11 100644 --- a/src/lua/lauxlib.c +++ b/src/lua/lauxlib.c @@ -190,7 +190,7 @@ LUALIB_API int luaL_argerror (lua_State *L, int arg, const char *extramsg) { } -int luaL_typeerror (lua_State *L, int arg, const char *tname) { +LUALIB_API int luaL_typeerror (lua_State *L, int arg, const char *tname) { const char *msg; const char *typearg; /* name for the type of the actual argument */ if (luaL_getmetafield(L, arg, "__name") == LUA_TSTRING) @@ -283,10 +283,10 @@ LUALIB_API int luaL_fileresult (lua_State *L, int stat, const char *fname) { LUALIB_API int luaL_execresult (lua_State *L, int stat) { - const char *what = "exit"; /* type of termination */ if (stat != 0 && errno != 0) /* error with an 'errno'? */ return luaL_fileresult(L, 0, NULL); else { + const char *what = "exit"; /* type of termination */ l_inspectstat(stat, what); /* interpret result */ if (*what == 'e' && stat == 0) /* successful termination? */ lua_pushboolean(L, 1); @@ -378,7 +378,7 @@ LUALIB_API int luaL_checkoption (lua_State *L, int arg, const char *def, ** but without 'msg'.) */ LUALIB_API void luaL_checkstack (lua_State *L, int space, const char *msg) { - if (!lua_checkstack(L, space)) { + if (l_unlikely(!lua_checkstack(L, space))) { if (msg) luaL_error(L, "stack overflow (%s)", msg); else @@ -388,20 +388,20 @@ LUALIB_API void luaL_checkstack (lua_State *L, int space, const char *msg) { LUALIB_API void luaL_checktype (lua_State *L, int arg, int t) { - if (lua_type(L, arg) != t) + if (l_unlikely(lua_type(L, arg) != t)) tag_error(L, arg, t); } LUALIB_API void luaL_checkany (lua_State *L, int arg) { - if (lua_type(L, arg) == LUA_TNONE) + if (l_unlikely(lua_type(L, arg) == LUA_TNONE)) luaL_argerror(L, arg, "value expected"); } LUALIB_API const char *luaL_checklstring (lua_State *L, int arg, size_t *len) { const char *s = lua_tolstring(L, arg, len); - if (!s) tag_error(L, arg, LUA_TSTRING); + if (l_unlikely(!s)) tag_error(L, arg, LUA_TSTRING); return s; } @@ -420,7 +420,7 @@ LUALIB_API const char *luaL_optlstring (lua_State *L, int arg, LUALIB_API lua_Number luaL_checknumber (lua_State *L, int arg) { int isnum; lua_Number d = lua_tonumberx(L, arg, &isnum); - if (!isnum) + if (l_unlikely(!isnum)) tag_error(L, arg, LUA_TNUMBER); return d; } @@ -442,7 +442,7 @@ static void interror (lua_State *L, int arg) { LUALIB_API lua_Integer luaL_checkinteger (lua_State *L, int arg) { int isnum; lua_Integer d = lua_tointegerx(L, arg, &isnum); - if (!isnum) { + if (l_unlikely(!isnum)) { interror(L, arg); } return d; @@ -475,7 +475,7 @@ static void *resizebox (lua_State *L, int idx, size_t newsize) { lua_Alloc allocf = lua_getallocf(L, &ud); UBox *box = (UBox *)lua_touserdata(L, idx); void *temp = allocf(ud, box->box, box->bsize, newsize); - if (temp == NULL && newsize > 0) { /* allocation error? */ + if (l_unlikely(temp == NULL && newsize > 0)) { /* allocation error? */ lua_pushliteral(L, "not enough memory"); lua_error(L); /* raise a memory error */ } @@ -515,13 +515,22 @@ static void newbox (lua_State *L) { #define buffonstack(B) ((B)->b != (B)->init.b) +/* +** Whenever buffer is accessed, slot 'idx' must either be a box (which +** cannot be NULL) or it is a placeholder for the buffer. +*/ +#define checkbufferlevel(B,idx) \ + lua_assert(buffonstack(B) ? lua_touserdata(B->L, idx) != NULL \ + : lua_touserdata(B->L, idx) == (void*)B) + + /* ** Compute new size for buffer 'B', enough to accommodate extra 'sz' ** bytes. */ static size_t newbuffsize (luaL_Buffer *B, size_t sz) { size_t newsize = B->size * 2; /* double buffer size */ - if (MAX_SIZET - sz < B->n) /* overflow in (B->n + sz)? */ + if (l_unlikely(MAX_SIZET - sz < B->n)) /* overflow in (B->n + sz)? */ return luaL_error(B->L, "buffer too large"); if (newsize < B->n + sz) /* double is not big enough? */ newsize = B->n + sz; @@ -531,10 +540,11 @@ static size_t newbuffsize (luaL_Buffer *B, size_t sz) { /* ** Returns a pointer to a free area with at least 'sz' bytes in buffer -** 'B'. 'boxidx' is the relative position in the stack where the -** buffer's box is or should be. +** 'B'. 'boxidx' is the relative position in the stack where is the +** buffer's box or its placeholder. */ static char *prepbuffsize (luaL_Buffer *B, size_t sz, int boxidx) { + checkbufferlevel(B, boxidx); if (B->size - B->n >= sz) /* enough space? */ return B->b + B->n; else { @@ -545,10 +555,9 @@ static char *prepbuffsize (luaL_Buffer *B, size_t sz, int boxidx) { if (buffonstack(B)) /* buffer already has a box? */ newbuff = (char *)resizebox(L, boxidx, newsize); /* resize it */ else { /* no box yet */ - lua_pushnil(L); /* reserve slot for final result */ + lua_remove(L, boxidx); /* remove placeholder */ newbox(L); /* create a new box */ - /* move box (and slot) to its intended position */ - lua_rotate(L, boxidx - 1, 2); + lua_insert(L, boxidx); /* move box to its intended position */ lua_toclose(L, boxidx); newbuff = (char *)resizebox(L, boxidx, newsize); memcpy(newbuff, B->b, B->n * sizeof(char)); /* copy original content */ @@ -583,11 +592,11 @@ LUALIB_API void luaL_addstring (luaL_Buffer *B, const char *s) { LUALIB_API void luaL_pushresult (luaL_Buffer *B) { lua_State *L = B->L; + checkbufferlevel(B, -1); lua_pushlstring(L, B->b, B->n); - if (buffonstack(B)) { - lua_copy(L, -1, -3); /* move string to reserved slot */ - lua_pop(L, 2); /* pop string and box (closing the box) */ - } + if (buffonstack(B)) + lua_closeslot(L, -2); /* close the box */ + lua_remove(L, -2); /* remove box or placeholder from the stack */ } @@ -622,6 +631,7 @@ LUALIB_API void luaL_buffinit (lua_State *L, luaL_Buffer *B) { B->b = B->init.b; B->n = 0; B->size = LUAL_BUFFERSIZE; + lua_pushlightuserdata(L, (void*)B); /* push placeholder */ } @@ -639,10 +649,14 @@ LUALIB_API char *luaL_buffinitsize (lua_State *L, luaL_Buffer *B, size_t sz) { ** ======================================================= */ -/* index of free-list header */ -#define freelist 0 - +/* index of free-list header (after the predefined values) */ +#define freelist (LUA_RIDX_LAST + 1) +/* +** The previously freed references form a linked list: +** t[freelist] is the index of a first free index, or zero if list is +** empty; t[t[freelist]] is the index of the second element; etc. +*/ LUALIB_API int luaL_ref (lua_State *L, int t) { int ref; if (lua_isnil(L, -1)) { @@ -650,9 +664,16 @@ LUALIB_API int luaL_ref (lua_State *L, int t) { return LUA_REFNIL; /* 'nil' has a unique fixed reference */ } t = lua_absindex(L, t); - lua_rawgeti(L, t, freelist); /* get first free element */ - ref = (int)lua_tointeger(L, -1); /* ref = t[freelist] */ - lua_pop(L, 1); /* remove it from stack */ + if (lua_rawgeti(L, t, freelist) == LUA_TNIL) { /* first access? */ + ref = 0; /* list is empty */ + lua_pushinteger(L, 0); /* initialize as an empty list */ + lua_rawseti(L, t, freelist); /* ref = t[freelist] = 0 */ + } + else { /* already initialized */ + lua_assert(lua_isinteger(L, -1)); + ref = (int)lua_tointeger(L, -1); /* ref = t[freelist] */ + } + lua_pop(L, 1); /* remove element from stack */ if (ref != 0) { /* any free element? */ lua_rawgeti(L, t, ref); /* remove it from list */ lua_rawseti(L, t, freelist); /* (t[freelist] = t[ref]) */ @@ -668,6 +689,7 @@ LUALIB_API void luaL_unref (lua_State *L, int t, int ref) { if (ref >= 0) { t = lua_absindex(L, t); lua_rawgeti(L, t, freelist); + lua_assert(lua_isinteger(L, -1)); lua_rawseti(L, t, ref); /* t[ref] = t[freelist] */ lua_pushinteger(L, ref); lua_rawseti(L, t, freelist); /* t[freelist] = ref */ @@ -851,7 +873,7 @@ LUALIB_API lua_Integer luaL_len (lua_State *L, int idx) { int isnum; lua_len(L, idx); l = lua_tointegerx(L, -1, &isnum); - if (!isnum) + if (l_unlikely(!isnum)) luaL_error(L, "object length is not an integer"); lua_pop(L, 1); /* remove object */ return l; @@ -859,6 +881,7 @@ LUALIB_API lua_Integer luaL_len (lua_State *L, int idx) { LUALIB_API const char *luaL_tolstring (lua_State *L, int idx, size_t *len) { + idx = lua_absindex(L,idx); if (luaL_callmeta(L, idx, "__tostring")) { /* metafield? */ if (!lua_isstring(L, -1)) luaL_error(L, "'__tostring' must return a string"); @@ -1006,43 +1029,67 @@ static int panic (lua_State *L) { /* -** Emit a warning. '*warnstate' means: -** 0 - warning system is off; -** 1 - ready to start a new message; -** 2 - previous message is to be continued. +** Warning functions: +** warnfoff: warning system is off +** warnfon: ready to start a new message +** warnfcont: previous message is to be continued */ -static void warnf (void *ud, const char *message, int tocont) { - int *warnstate = (int *)ud; - if (*warnstate != 2 && !tocont && *message == '@') { /* control message? */ - if (strcmp(message, "@off") == 0) - *warnstate = 0; - else if (strcmp(message, "@on") == 0) - *warnstate = 1; - return; +static void warnfoff (void *ud, const char *message, int tocont); +static void warnfon (void *ud, const char *message, int tocont); +static void warnfcont (void *ud, const char *message, int tocont); + + +/* +** Check whether message is a control message. If so, execute the +** control or ignore it if unknown. +*/ +static int checkcontrol (lua_State *L, const char *message, int tocont) { + if (tocont || *(message++) != '@') /* not a control message? */ + return 0; + else { + if (strcmp(message, "off") == 0) + lua_setwarnf(L, warnfoff, L); /* turn warnings off */ + else if (strcmp(message, "on") == 0) + lua_setwarnf(L, warnfon, L); /* turn warnings on */ + return 1; /* it was a control message */ } - else if (*warnstate == 0) /* warnings off? */ - return; - if (*warnstate == 1) /* previous message was the last? */ - lua_writestringerror("%s", "Lua warning: "); /* start a new warning */ +} + + +static void warnfoff (void *ud, const char *message, int tocont) { + checkcontrol((lua_State *)ud, message, tocont); +} + + +/* +** Writes the message and handle 'tocont', finishing the message +** if needed and setting the next warn function. +*/ +static void warnfcont (void *ud, const char *message, int tocont) { + lua_State *L = (lua_State *)ud; lua_writestringerror("%s", message); /* write message */ if (tocont) /* not the last part? */ - *warnstate = 2; /* to be continued */ + lua_setwarnf(L, warnfcont, L); /* to be continued */ else { /* last part */ lua_writestringerror("%s", "\n"); /* finish message with end-of-line */ - *warnstate = 1; /* ready to start a new message */ + lua_setwarnf(L, warnfon, L); /* next call is a new message */ } } +static void warnfon (void *ud, const char *message, int tocont) { + if (checkcontrol((lua_State *)ud, message, tocont)) /* control message? */ + return; /* nothing else to be done */ + lua_writestringerror("%s", "Lua warning: "); /* start a new warning */ + warnfcont(ud, message, tocont); /* finish processing */ +} + + LUALIB_API lua_State *luaL_newstate (void) { lua_State *L = lua_newstate(l_alloc, NULL); - if (L) { - int *warnstate; /* space for warning state */ + if (l_likely(L)) { lua_atpanic(L, &panic); - warnstate = (int *)lua_newuserdatauv(L, sizeof(int), 0); - luaL_ref(L, LUA_REGISTRYINDEX); /* make sure it won't be collected */ - *warnstate = 0; /* default is warnings off */ - lua_setwarnf(L, warnf, warnstate); + lua_setwarnf(L, warnfoff, L); /* default is warnings off */ } return L; } diff --git a/src/lua/lauxlib.h b/src/lua/lauxlib.h index 59fef6af..5b977e2a 100644 --- a/src/lua/lauxlib.h +++ b/src/lua/lauxlib.h @@ -12,6 +12,7 @@ #include #include +#include "luaconf.h" #include "lua.h" @@ -101,7 +102,7 @@ LUALIB_API lua_State *(luaL_newstate) (void); LUALIB_API lua_Integer (luaL_len) (lua_State *L, int idx); -LUALIB_API void luaL_addgsub (luaL_Buffer *b, const char *s, +LUALIB_API void (luaL_addgsub) (luaL_Buffer *b, const char *s, const char *p, const char *r); LUALIB_API const char *(luaL_gsub) (lua_State *L, const char *s, const char *p, const char *r); @@ -130,10 +131,10 @@ LUALIB_API void (luaL_requiref) (lua_State *L, const char *modname, (luaL_checkversion(L), luaL_newlibtable(L,l), luaL_setfuncs(L,l,0)) #define luaL_argcheck(L, cond,arg,extramsg) \ - ((void)((cond) || luaL_argerror(L, (arg), (extramsg)))) + ((void)(luai_likely(cond) || luaL_argerror(L, (arg), (extramsg)))) #define luaL_argexpected(L,cond,arg,tname) \ - ((void)((cond) || luaL_typeerror(L, (arg), (tname)))) + ((void)(luai_likely(cond) || luaL_typeerror(L, (arg), (tname)))) #define luaL_checkstring(L,n) (luaL_checklstring(L, (n), NULL)) #define luaL_optstring(L,n,d) (luaL_optlstring(L, (n), (d), NULL)) @@ -153,10 +154,34 @@ LUALIB_API void (luaL_requiref) (lua_State *L, const char *modname, #define luaL_loadbuffer(L,s,sz,n) luaL_loadbufferx(L,s,sz,n,NULL) +/* +** Perform arithmetic operations on lua_Integer values with wrap-around +** semantics, as the Lua core does. +*/ +#define luaL_intop(op,v1,v2) \ + ((lua_Integer)((lua_Unsigned)(v1) op (lua_Unsigned)(v2))) + + /* push the value used to represent failure/error */ #define luaL_pushfail(L) lua_pushnil(L) +/* +** Internal assertions for in-house debugging +*/ +#if !defined(lua_assert) + +#if defined LUAI_ASSERT + #include + #define lua_assert(c) assert(c) +#else + #define lua_assert(c) ((void)0) +#endif + +#endif + + + /* ** {====================================================== ** Generic Buffer manipulation diff --git a/src/lua/lbaselib.c b/src/lua/lbaselib.c index 747fd45a..1d60c9de 100644 --- a/src/lua/lbaselib.c +++ b/src/lua/lbaselib.c @@ -138,7 +138,7 @@ static int luaB_setmetatable (lua_State *L) { int t = lua_type(L, 2); luaL_checktype(L, 1, LUA_TTABLE); luaL_argexpected(L, t == LUA_TNIL || t == LUA_TTABLE, 2, "nil or table"); - if (luaL_getmetafield(L, 1, "__metatable") != LUA_TNIL) + if (l_unlikely(luaL_getmetafield(L, 1, "__metatable") != LUA_TNIL)) return luaL_error(L, "cannot change a protected metatable"); lua_settop(L, 2); lua_setmetatable(L, 1); @@ -182,11 +182,20 @@ static int luaB_rawset (lua_State *L) { static int pushmode (lua_State *L, int oldmode) { - lua_pushstring(L, (oldmode == LUA_GCINC) ? "incremental" : "generational"); + if (oldmode == -1) + luaL_pushfail(L); /* invalid call to 'lua_gc' */ + else + lua_pushstring(L, (oldmode == LUA_GCINC) ? "incremental" + : "generational"); return 1; } +/* +** check whether call to 'lua_gc' was valid (not inside a finalizer) +*/ +#define checkvalres(res) { if (res == -1) break; } + static int luaB_collectgarbage (lua_State *L) { static const char *const opts[] = {"stop", "restart", "collect", "count", "step", "setpause", "setstepmul", @@ -199,12 +208,14 @@ static int luaB_collectgarbage (lua_State *L) { case LUA_GCCOUNT: { int k = lua_gc(L, o); int b = lua_gc(L, LUA_GCCOUNTB); + checkvalres(k); lua_pushnumber(L, (lua_Number)k + ((lua_Number)b/1024)); return 1; } case LUA_GCSTEP: { int step = (int)luaL_optinteger(L, 2, 0); int res = lua_gc(L, o, step); + checkvalres(res); lua_pushboolean(L, res); return 1; } @@ -212,11 +223,13 @@ static int luaB_collectgarbage (lua_State *L) { case LUA_GCSETSTEPMUL: { int p = (int)luaL_optinteger(L, 2, 0); int previous = lua_gc(L, o, p); + checkvalres(previous); lua_pushinteger(L, previous); return 1; } case LUA_GCISRUNNING: { int res = lua_gc(L, o); + checkvalres(res); lua_pushboolean(L, res); return 1; } @@ -233,10 +246,13 @@ static int luaB_collectgarbage (lua_State *L) { } default: { int res = lua_gc(L, o); + checkvalres(res); lua_pushinteger(L, res); return 1; } } + luaL_pushfail(L); /* invalid call (inside a finalizer) */ + return 1; } @@ -260,6 +276,11 @@ static int luaB_next (lua_State *L) { } +static int pairscont (lua_State *L, int status, lua_KContext k) { + (void)L; (void)status; (void)k; /* unused */ + return 3; +} + static int luaB_pairs (lua_State *L) { luaL_checkany(L, 1); if (luaL_getmetafield(L, 1, "__pairs") == LUA_TNIL) { /* no metamethod? */ @@ -269,7 +290,7 @@ static int luaB_pairs (lua_State *L) { } else { lua_pushvalue(L, 1); /* argument 'self' to metamethod */ - lua_call(L, 1, 3); /* get 3 values from metamethod */ + lua_callk(L, 1, 3, 0, pairscont); /* get 3 values from metamethod */ } return 3; } @@ -279,7 +300,8 @@ static int luaB_pairs (lua_State *L) { ** Traversal function for 'ipairs' */ static int ipairsaux (lua_State *L) { - lua_Integer i = luaL_checkinteger(L, 2) + 1; + lua_Integer i = luaL_checkinteger(L, 2); + i = luaL_intop(+, i, 1); lua_pushinteger(L, i); return (lua_geti(L, 1, i) == LUA_TNIL) ? 1 : 2; } @@ -299,7 +321,7 @@ static int luaB_ipairs (lua_State *L) { static int load_aux (lua_State *L, int status, int envidx) { - if (status == LUA_OK) { + if (l_likely(status == LUA_OK)) { if (envidx != 0) { /* 'env' parameter? */ lua_pushvalue(L, envidx); /* environment for loaded function */ if (!lua_setupvalue(L, -2, 1)) /* set it as 1st upvalue */ @@ -355,7 +377,7 @@ static const char *generic_reader (lua_State *L, void *ud, size_t *size) { *size = 0; return NULL; } - else if (!lua_isstring(L, -1)) + else if (l_unlikely(!lua_isstring(L, -1))) luaL_error(L, "reader function must return a string"); lua_replace(L, RESERVEDSLOT); /* save string in reserved slot */ return lua_tolstring(L, RESERVEDSLOT, size); @@ -393,7 +415,7 @@ static int dofilecont (lua_State *L, int d1, lua_KContext d2) { static int luaB_dofile (lua_State *L) { const char *fname = luaL_optstring(L, 1, NULL); lua_settop(L, 1); - if (luaL_loadfile(L, fname) != LUA_OK) + if (l_unlikely(luaL_loadfile(L, fname) != LUA_OK)) return lua_error(L); lua_callk(L, 0, LUA_MULTRET, 0, dofilecont); return dofilecont(L, 0, 0); @@ -401,7 +423,7 @@ static int luaB_dofile (lua_State *L) { static int luaB_assert (lua_State *L) { - if (lua_toboolean(L, 1)) /* condition is true? */ + if (l_likely(lua_toboolean(L, 1))) /* condition is true? */ return lua_gettop(L); /* return all arguments */ else { /* error */ luaL_checkany(L, 1); /* there must be a condition */ @@ -437,7 +459,7 @@ static int luaB_select (lua_State *L) { ** ignored). */ static int finishpcall (lua_State *L, int status, lua_KContext extra) { - if (status != LUA_OK && status != LUA_YIELD) { /* error? */ + if (l_unlikely(status != LUA_OK && status != LUA_YIELD)) { /* error? */ lua_pushboolean(L, 0); /* first result (false) */ lua_pushvalue(L, -2); /* error message */ return 2; /* return false, msg */ diff --git a/src/lua/lcode.c b/src/lua/lcode.c index 6f241c94..06425a1d 100644 --- a/src/lua/lcode.c +++ b/src/lua/lcode.c @@ -10,6 +10,7 @@ #include "lprefix.h" +#include #include #include #include @@ -314,15 +315,6 @@ void luaK_patchtohere (FuncState *fs, int list) { } -/* -** MAXimum number of successive Instructions WiTHout ABSolute line -** information. -*/ -#if !defined(MAXIWTHABS) -#define MAXIWTHABS 120 -#endif - - /* limit for difference between lines in relative line info. */ #define LIMLINEDIFF 0x80 @@ -337,13 +329,13 @@ void luaK_patchtohere (FuncState *fs, int list) { static void savelineinfo (FuncState *fs, Proto *f, int line) { int linedif = line - fs->previousline; int pc = fs->pc - 1; /* last instruction coded */ - if (abs(linedif) >= LIMLINEDIFF || fs->iwthabs++ > MAXIWTHABS) { + if (abs(linedif) >= LIMLINEDIFF || fs->iwthabs++ >= MAXIWTHABS) { luaM_growvector(fs->ls->L, f->abslineinfo, fs->nabslineinfo, f->sizeabslineinfo, AbsLineInfo, MAX_INT, "lines"); f->abslineinfo[fs->nabslineinfo].pc = pc; f->abslineinfo[fs->nabslineinfo++].line = line; linedif = ABSLINEINFO; /* signal that there is absolute information */ - fs->iwthabs = 0; /* restart counter */ + fs->iwthabs = 1; /* restart counter */ } luaM_growvector(fs->ls->L, f->lineinfo, pc, f->sizelineinfo, ls_byte, MAX_INT, "opcodes"); @@ -545,11 +537,14 @@ static void freeexps (FuncState *fs, expdesc *e1, expdesc *e2) { ** and try to reuse constants. Because some values should not be used ** as keys (nil cannot be a key, integer keys can collapse with float ** keys), the caller must provide a useful 'key' for indexing the cache. +** Note that all functions share the same table, so entering or exiting +** a function can make some indices wrong. */ static int addk (FuncState *fs, TValue *key, TValue *v) { + TValue val; lua_State *L = fs->ls->L; Proto *f = fs->f; - TValue *idx = luaH_set(L, fs->ls->h, key); /* index scanner table */ + const TValue *idx = luaH_get(fs->ls->h, key); /* query scanner table */ int k, oldsize; if (ttisinteger(idx)) { /* is there an index there? */ k = cast_int(ivalue(idx)); @@ -563,7 +558,8 @@ static int addk (FuncState *fs, TValue *key, TValue *v) { k = fs->nk; /* numerical value does not need GC barrier; table has no metatable, so it does not need to invalidate cache */ - setivalue(idx, k); + setivalue(&val, k); + luaH_finishset(L, fs->ls->h, key, idx, &val); luaM_growvector(L, f->k, k, f->sizek, TValue, MAXARG_Ax, "constants"); while (oldsize < f->sizek) setnilvalue(&f->k[oldsize++]); setobj(L, &f->k[k], v); @@ -585,24 +581,41 @@ static int stringK (FuncState *fs, TString *s) { /* ** Add an integer to list of constants and return its index. -** Integers use userdata as keys to avoid collision with floats with -** same value; conversion to 'void*' is used only for hashing, so there -** are no "precision" problems. */ static int luaK_intK (FuncState *fs, lua_Integer n) { - TValue k, o; - setpvalue(&k, cast_voidp(cast_sizet(n))); + TValue o; setivalue(&o, n); - return addk(fs, &k, &o); + return addk(fs, &o, &o); /* use integer itself as key */ } /* -** Add a float to list of constants and return its index. +** Add a float to list of constants and return its index. Floats +** with integral values need a different key, to avoid collision +** with actual integers. To that, we add to the number its smaller +** power-of-two fraction that is still significant in its scale. +** For doubles, that would be 1/2^52. +** (This method is not bulletproof: there may be another float +** with that value, and for floats larger than 2^53 the result is +** still an integer. At worst, this only wastes an entry with +** a duplicate.) */ static int luaK_numberK (FuncState *fs, lua_Number r) { TValue o; + lua_Integer ik; setfltvalue(&o, r); - return addk(fs, &o, &o); /* use number itself as key */ + if (!luaV_flttointeger(r, &ik, F2Ieq)) /* not an integral value? */ + return addk(fs, &o, &o); /* use number itself as key */ + else { /* must build an alternative key */ + const int nbm = l_floatatt(MANT_DIG); + const lua_Number q = l_mathop(ldexp)(l_mathop(1.0), -nbm + 1); + const lua_Number k = (ik == 0) ? q : r + r*q; /* new key */ + TValue kv; + setfltvalue(&kv, k); + /* result is not an integral value, unless value is too large */ + lua_assert(!luaV_flttointeger(k, &ik, F2Ieq) || + l_mathop(fabs)(r) >= l_mathop(1e6)); + return addk(fs, &kv, &o); + } } @@ -753,7 +766,7 @@ void luaK_setoneret (FuncState *fs, expdesc *e) { /* -** Ensure that expression 'e' is not a variable (nor a constant). +** Ensure that expression 'e' is not a variable (nor a ). ** (Expression still may have jump lists.) */ void luaK_dischargevars (FuncState *fs, expdesc *e) { @@ -763,7 +776,7 @@ void luaK_dischargevars (FuncState *fs, expdesc *e) { break; } case VLOCAL: { /* already in a register */ - e->u.info = e->u.var.sidx; + e->u.info = e->u.var.ridx; e->k = VNONRELOC; /* becomes a non-relocatable value */ break; } @@ -805,8 +818,8 @@ void luaK_dischargevars (FuncState *fs, expdesc *e) { /* -** Ensures expression value is in register 'reg' (and therefore -** 'e' will become a non-relocatable expression). +** Ensure expression value is in register 'reg', making 'e' a +** non-relocatable expression. ** (Expression still may have jump lists.) */ static void discharge2reg (FuncState *fs, expdesc *e, int reg) { @@ -860,7 +873,8 @@ static void discharge2reg (FuncState *fs, expdesc *e, int reg) { /* -** Ensures expression value is in any register. +** Ensure expression value is in a register, making 'e' a +** non-relocatable expression. ** (Expression still may have jump lists.) */ static void discharge2anyreg (FuncState *fs, expdesc *e) { @@ -946,8 +960,11 @@ int luaK_exp2anyreg (FuncState *fs, expdesc *e) { exp2reg(fs, e, e->u.info); /* put final result in it */ return e->u.info; } + /* else expression has jumps and cannot change its register + to hold the jump values, because it is a local variable. + Go through to the default case. */ } - luaK_exp2nextreg(fs, e); /* otherwise, use next available register */ + luaK_exp2nextreg(fs, e); /* default: use next available register */ return e->u.info; } @@ -1032,7 +1049,7 @@ void luaK_storevar (FuncState *fs, expdesc *var, expdesc *ex) { switch (var->k) { case VLOCAL: { freeexp(fs, ex); - exp2reg(fs, ex, var->u.var.sidx); /* compute 'ex' into proper place */ + exp2reg(fs, ex, var->u.var.ridx); /* compute 'ex' into proper place */ return; } case VUPVAL: { @@ -1272,7 +1289,7 @@ void luaK_indexed (FuncState *fs, expdesc *t, expdesc *k) { } else { /* register index of the table */ - t->u.ind.t = (t->k == VLOCAL) ? t->u.var.sidx: t->u.info; + t->u.ind.t = (t->k == VLOCAL) ? t->u.var.ridx: t->u.info; if (isKstr(fs, k)) { t->u.ind.idx = k->u.info; /* literal string */ t->k = VINDEXSTR; @@ -1299,7 +1316,8 @@ static int validop (int op, TValue *v1, TValue *v2) { case LUA_OPBAND: case LUA_OPBOR: case LUA_OPBXOR: case LUA_OPSHL: case LUA_OPSHR: case LUA_OPBNOT: { /* conversion errors */ lua_Integer i; - return (tointegerns(v1, &i) && tointegerns(v2, &i)); + return (luaV_tointegerns(v1, &i, LUA_FLOORN2I) && + luaV_tointegerns(v2, &i, LUA_FLOORN2I)); } case LUA_OPDIV: case LUA_OPIDIV: case LUA_OPMOD: /* division by 0 */ return (nvalue(v2) != 0); diff --git a/src/lua/lcorolib.c b/src/lua/lcorolib.c index c165031d..785a1e81 100644 --- a/src/lua/lcorolib.c +++ b/src/lua/lcorolib.c @@ -31,14 +31,14 @@ static lua_State *getco (lua_State *L) { */ static int auxresume (lua_State *L, lua_State *co, int narg) { int status, nres; - if (!lua_checkstack(co, narg)) { + if (l_unlikely(!lua_checkstack(co, narg))) { lua_pushliteral(L, "too many arguments to resume"); return -1; /* error flag */ } lua_xmove(L, co, narg); status = lua_resume(co, L, narg, &nres); - if (status == LUA_OK || status == LUA_YIELD) { - if (!lua_checkstack(L, nres + 1)) { + if (l_likely(status == LUA_OK || status == LUA_YIELD)) { + if (l_unlikely(!lua_checkstack(L, nres + 1))) { lua_pop(co, nres); /* remove results anyway */ lua_pushliteral(L, "too many results to resume"); return -1; /* error flag */ @@ -57,7 +57,7 @@ static int luaB_coresume (lua_State *L) { lua_State *co = getco(L); int r; r = auxresume(L, co, lua_gettop(L) - 1); - if (r < 0) { + if (l_unlikely(r < 0)) { lua_pushboolean(L, 0); lua_insert(L, -2); return 2; /* return false + error message */ @@ -73,10 +73,13 @@ static int luaB_coresume (lua_State *L) { static int luaB_auxwrap (lua_State *L) { lua_State *co = lua_tothread(L, lua_upvalueindex(1)); int r = auxresume(L, co, lua_gettop(L)); - if (r < 0) { /* error? */ + if (l_unlikely(r < 0)) { /* error? */ int stat = lua_status(co); - if (stat != LUA_OK && stat != LUA_YIELD) /* error in the coroutine? */ - lua_resetthread(co); /* close its tbc variables */ + if (stat != LUA_OK && stat != LUA_YIELD) { /* error in the coroutine? */ + stat = lua_resetthread(co); /* close its tbc variables */ + lua_assert(stat != LUA_OK); + lua_xmove(co, L, 1); /* move error message to the caller */ + } if (stat != LUA_ERRMEM && /* not a memory error and ... */ lua_type(L, -1) == LUA_TSTRING) { /* ... error object is a string? */ luaL_where(L, 1); /* add extra info, if available */ @@ -176,7 +179,7 @@ static int luaB_close (lua_State *L) { } else { lua_pushboolean(L, 0); - lua_xmove(co, L, 1); /* copy error message */ + lua_xmove(co, L, 1); /* move error message */ return 2; } } diff --git a/src/lua/ldblib.c b/src/lua/ldblib.c index 59eb8f0e..6dcbaa98 100644 --- a/src/lua/ldblib.c +++ b/src/lua/ldblib.c @@ -33,7 +33,7 @@ static const char *const HOOKKEY = "_HOOKKEY"; ** checked. */ static void checkstack (lua_State *L, lua_State *L1, int n) { - if (L != L1 && !lua_checkstack(L1, n)) + if (l_unlikely(L != L1 && !lua_checkstack(L1, n))) luaL_error(L, "stack overflow"); } @@ -152,6 +152,7 @@ static int db_getinfo (lua_State *L) { lua_State *L1 = getthread(L, &arg); const char *options = luaL_optstring(L, arg+2, "flnSrtu"); checkstack(L, L1, 3); + luaL_argcheck(L, options[0] != '>', arg + 2, "invalid option '>'"); if (lua_isfunction(L, arg + 1)) { /* info about a function? */ options = lua_pushfstring(L, ">%s", options); /* add '>' to 'options' */ lua_pushvalue(L, arg + 1); /* move function to 'L1' stack */ @@ -212,7 +213,7 @@ static int db_getlocal (lua_State *L) { lua_Debug ar; const char *name; int level = (int)luaL_checkinteger(L, arg + 1); - if (!lua_getstack(L1, level, &ar)) /* out of range? */ + if (l_unlikely(!lua_getstack(L1, level, &ar))) /* out of range? */ return luaL_argerror(L, arg+1, "level out of range"); checkstack(L, L1, 1); name = lua_getlocal(L1, &ar, nvar); @@ -237,7 +238,7 @@ static int db_setlocal (lua_State *L) { lua_Debug ar; int level = (int)luaL_checkinteger(L, arg + 1); int nvar = (int)luaL_checkinteger(L, arg + 2); - if (!lua_getstack(L1, level, &ar)) /* out of range? */ + if (l_unlikely(!lua_getstack(L1, level, &ar))) /* out of range? */ return luaL_argerror(L, arg+1, "level out of range"); luaL_checkany(L, arg+3); lua_settop(L, arg+3); @@ -281,25 +282,33 @@ static int db_setupvalue (lua_State *L) { ** Check whether a given upvalue from a given closure exists and ** returns its index */ -static int checkupval (lua_State *L, int argf, int argnup) { +static void *checkupval (lua_State *L, int argf, int argnup, int *pnup) { + void *id; int nup = (int)luaL_checkinteger(L, argnup); /* upvalue index */ luaL_checktype(L, argf, LUA_TFUNCTION); /* closure */ - luaL_argcheck(L, (lua_getupvalue(L, argf, nup) != NULL), argnup, - "invalid upvalue index"); - return nup; + id = lua_upvalueid(L, argf, nup); + if (pnup) { + luaL_argcheck(L, id != NULL, argnup, "invalid upvalue index"); + *pnup = nup; + } + return id; } static int db_upvalueid (lua_State *L) { - int n = checkupval(L, 1, 2); - lua_pushlightuserdata(L, lua_upvalueid(L, 1, n)); + void *id = checkupval(L, 1, 2, NULL); + if (id != NULL) + lua_pushlightuserdata(L, id); + else + luaL_pushfail(L); return 1; } static int db_upvaluejoin (lua_State *L) { - int n1 = checkupval(L, 1, 2); - int n2 = checkupval(L, 3, 4); + int n1, n2; + checkupval(L, 1, 2, &n1); + checkupval(L, 3, 4, &n2); luaL_argcheck(L, !lua_iscfunction(L, 1), 1, "Lua function expected"); luaL_argcheck(L, !lua_iscfunction(L, 3), 3, "Lua function expected"); lua_upvaluejoin(L, 1, n1, 3, n2); @@ -369,7 +378,7 @@ static int db_sethook (lua_State *L) { } if (!luaL_getsubtable(L, LUA_REGISTRYINDEX, HOOKKEY)) { /* table just created; initialize it */ - lua_pushstring(L, "k"); + lua_pushliteral(L, "k"); lua_setfield(L, -2, "__mode"); /** hooktable.__mode = "k" */ lua_pushvalue(L, -1); lua_setmetatable(L, -2); /* metatable(hooktable) = hooktable */ @@ -412,7 +421,7 @@ static int db_debug (lua_State *L) { for (;;) { char buffer[250]; lua_writestringerror("%s", "lua_debug> "); - if (fgets(buffer, sizeof(buffer), stdin) == 0 || + if (fgets(buffer, sizeof(buffer), stdin) == NULL || strcmp(buffer, "cont\n") == 0) return 0; if (luaL_loadbuffer(L, buffer, strlen(buffer), "=(debug command)") || @@ -440,10 +449,7 @@ static int db_traceback (lua_State *L) { static int db_setcstacklimit (lua_State *L) { int limit = (int)luaL_checkinteger(L, 1); int res = lua_setcstacklimit(L, limit); - if (res == 0) - lua_pushboolean(L, 0); - else - lua_pushinteger(L, res); + lua_pushinteger(L, res); return 1; } diff --git a/src/lua/ldebug.c b/src/lua/ldebug.c index 8cb00e51..a716d95e 100644 --- a/src/lua/ldebug.c +++ b/src/lua/ldebug.c @@ -33,11 +33,9 @@ #define noLuaClosure(f) ((f) == NULL || (f)->c.tt == LUA_VCCL) -/* inverse of 'pcRel' */ -#define invpcRel(pc, p) ((p)->code + (pc) + 1) -static const char *funcnamefromcode (lua_State *L, CallInfo *ci, - const char **name); +static const char *funcnamefromcall (lua_State *L, CallInfo *ci, + const char **name); static int currentpc (CallInfo *ci) { @@ -48,10 +46,16 @@ static int currentpc (CallInfo *ci) { /* ** Get a "base line" to find the line corresponding to an instruction. -** For that, search the array of absolute line info for the largest saved -** instruction smaller or equal to the wanted instruction. A special -** case is when there is no absolute info or the instruction is before -** the first absolute one. +** Base lines are regularly placed at MAXIWTHABS intervals, so usually +** an integer division gets the right place. When the source file has +** large sequences of empty/comment lines, it may need extra entries, +** so the original estimate needs a correction. +** If the original estimate is -1, the initial 'if' ensures that the +** 'while' will run at least once. +** The assertion that the estimate is a lower bound for the correct base +** is valid as long as the debug info has been generated with the same +** value for MAXIWTHABS or smaller. (Previous releases use a little +** smaller value.) */ static int getbaseline (const Proto *f, int pc, int *basepc) { if (f->sizeabslineinfo == 0 || pc < f->abslineinfo[0].pc) { @@ -59,20 +63,12 @@ static int getbaseline (const Proto *f, int pc, int *basepc) { return f->linedefined; } else { - unsigned int i; - if (pc >= f->abslineinfo[f->sizeabslineinfo - 1].pc) - i = f->sizeabslineinfo - 1; /* instruction is after last saved one */ - else { /* binary search */ - unsigned int j = f->sizeabslineinfo - 1; /* pc < anchorlines[j] */ - i = 0; /* abslineinfo[i] <= pc */ - while (i < j - 1) { - unsigned int m = (j + i) / 2; - if (pc >= f->abslineinfo[m].pc) - i = m; - else - j = m; - } - } + int i = cast_uint(pc) / MAXIWTHABS - 1; /* get an estimate */ + /* estimate must be a lower bound of the correct base */ + lua_assert(i < 0 || + (i < f->sizeabslineinfo && f->abslineinfo[i].pc <= pc)); + while (i + 1 < f->sizeabslineinfo && pc >= f->abslineinfo[i + 1].pc) + i++; /* low estimate; adjust it */ *basepc = f->abslineinfo[i].pc; return f->abslineinfo[i].line; } @@ -305,8 +301,15 @@ static void collectvalidlines (lua_State *L, Closure *f) { sethvalue2s(L, L->top, t); /* push it on stack */ api_incr_top(L); setbtvalue(&v); /* boolean 'true' to be the value of all indices */ - for (i = 0; i < p->sizelineinfo; i++) { /* for all lines with code */ - currentline = nextline(p, currentline, i); + if (!p->is_vararg) /* regular function? */ + i = 0; /* consider all instructions */ + else { /* vararg function */ + lua_assert(GET_OPCODE(p->code[0]) == OP_VARARGPREP); + currentline = nextline(p, currentline, 0); + i = 1; /* skip first instruction (OP_VARARGPREP) */ + } + for (; i < p->sizelineinfo; i++) { /* for each instruction */ + currentline = nextline(p, currentline, i); /* get its line */ luaH_setint(L, t, currentline, &v); /* table[line] = true */ } } @@ -314,15 +317,9 @@ static void collectvalidlines (lua_State *L, Closure *f) { static const char *getfuncname (lua_State *L, CallInfo *ci, const char **name) { - if (ci == NULL) /* no 'ci'? */ - return NULL; /* no info */ - else if (ci->callstatus & CIST_FIN) { /* is this a finalizer? */ - *name = "__gc"; - return "metamethod"; /* report it as such */ - } - /* calling function is a known Lua function? */ - else if (!(ci->callstatus & CIST_TAIL) && isLua(ci->previous)) - return funcnamefromcode(L, ci->previous, name); + /* calling function is a known function? */ + if (ci != NULL && !(ci->callstatus & CIST_TAIL)) + return funcnamefromcall(L, ci->previous, name); else return NULL; /* no way to find a name */ } @@ -594,16 +591,10 @@ static const char *getobjname (const Proto *p, int lastpc, int reg, ** Returns what the name is (e.g., "for iterator", "method", ** "metamethod") and sets '*name' to point to the name. */ -static const char *funcnamefromcode (lua_State *L, CallInfo *ci, - const char **name) { +static const char *funcnamefromcode (lua_State *L, const Proto *p, + int pc, const char **name) { TMS tm = (TMS)0; /* (initial value avoids warnings) */ - const Proto *p = ci_func(ci)->p; /* calling function */ - int pc = currentpc(ci); /* calling instruction index */ Instruction i = p->code[pc]; /* calling instruction */ - if (ci->callstatus & CIST_HOOKED) { /* was it called inside a hook? */ - *name = "?"; - return "hook"; - } switch (GET_OPCODE(i)) { case OP_CALL: case OP_TAILCALL: @@ -629,12 +620,10 @@ static const char *funcnamefromcode (lua_State *L, CallInfo *ci, case OP_LEN: tm = TM_LEN; break; case OP_CONCAT: tm = TM_CONCAT; break; case OP_EQ: tm = TM_EQ; break; - case OP_LT: case OP_LE: case OP_LTI: case OP_LEI: - *name = "order"; /* '<=' can call '__lt', etc. */ - return "metamethod"; - case OP_CLOSE: case OP_RETURN: - *name = "close"; - return "metamethod"; + /* no cases for OP_EQI and OP_EQK, as they don't call metamethods */ + case OP_LT: case OP_LTI: case OP_GTI: tm = TM_LT; break; + case OP_LE: case OP_LEI: case OP_GEI: tm = TM_LE; break; + case OP_CLOSE: case OP_RETURN: tm = TM_CLOSE; break; default: return NULL; /* cannot find a reasonable name */ } @@ -642,19 +631,43 @@ static const char *funcnamefromcode (lua_State *L, CallInfo *ci, return "metamethod"; } + +/* +** Try to find a name for a function based on how it was called. +*/ +static const char *funcnamefromcall (lua_State *L, CallInfo *ci, + const char **name) { + if (ci->callstatus & CIST_HOOKED) { /* was it called inside a hook? */ + *name = "?"; + return "hook"; + } + else if (ci->callstatus & CIST_FIN) { /* was it called as a finalizer? */ + *name = "__gc"; + return "metamethod"; /* report it as such */ + } + else if (isLua(ci)) + return funcnamefromcode(L, ci_func(ci)->p, currentpc(ci), name); + else + return NULL; +} + /* }====================================================== */ /* -** The subtraction of two potentially unrelated pointers is -** not ISO C, but it should not crash a program; the subsequent -** checks are ISO C and ensure a correct result. +** Check whether pointer 'o' points to some value in the stack +** frame of the current function. Because 'o' may not point to a +** value in this stack, we cannot compare it with the region +** boundaries (undefined behaviour in ISO C). */ static int isinstack (CallInfo *ci, const TValue *o) { - StkId base = ci->func + 1; - ptrdiff_t i = cast(StkId, o) - base; - return (0 <= i && i < (ci->top - base) && s2v(base + i) == o); + StkId pos; + for (pos = ci->func + 1; pos < ci->top; pos++) { + if (o == s2v(pos)) + return 1; + } + return 0; /* not found */ } @@ -677,9 +690,21 @@ static const char *getupvalname (CallInfo *ci, const TValue *o, } +static const char *formatvarinfo (lua_State *L, const char *kind, + const char *name) { + if (kind == NULL) + return ""; /* no information */ + else + return luaO_pushfstring(L, " (%s '%s')", kind, name); +} + +/* +** Build a string with a "description" for the value 'o', such as +** "variable 'x'" or "upvalue 'y'". +*/ static const char *varinfo (lua_State *L, const TValue *o) { - const char *name = NULL; /* to avoid warnings */ CallInfo *ci = L->ci; + const char *name = NULL; /* to avoid warnings */ const char *kind = NULL; if (isLua(ci)) { kind = getupvalname(ci, o, &name); /* check whether 'o' is an upvalue */ @@ -687,13 +712,40 @@ static const char *varinfo (lua_State *L, const TValue *o) { kind = getobjname(ci_func(ci)->p, currentpc(ci), cast_int(cast(StkId, o) - (ci->func + 1)), &name); } - return (kind) ? luaO_pushfstring(L, " (%s '%s')", kind, name) : ""; + return formatvarinfo(L, kind, name); } -l_noret luaG_typeerror (lua_State *L, const TValue *o, const char *op) { +/* +** Raise a type error +*/ +static l_noret typeerror (lua_State *L, const TValue *o, const char *op, + const char *extra) { const char *t = luaT_objtypename(L, o); - luaG_runerror(L, "attempt to %s a %s value%s", op, t, varinfo(L, o)); + luaG_runerror(L, "attempt to %s a %s value%s", op, t, extra); +} + + +/* +** Raise a type error with "standard" information about the faulty +** object 'o' (using 'varinfo'). +*/ +l_noret luaG_typeerror (lua_State *L, const TValue *o, const char *op) { + typeerror(L, o, op, varinfo(L, o)); +} + + +/* +** Raise an error for calling a non-callable object. Try to find a name +** for the object based on how it was called ('funcnamefromcall'); if it +** cannot get a name there, try 'varinfo'. +*/ +l_noret luaG_callerror (lua_State *L, const TValue *o) { + CallInfo *ci = L->ci; + const char *name = NULL; /* to avoid warnings */ + const char *kind = funcnamefromcall(L, ci, &name); + const char *extra = kind ? formatvarinfo(L, kind, name) : varinfo(L, o); + typeerror(L, o, "call", extra); } @@ -722,7 +774,7 @@ l_noret luaG_opinterror (lua_State *L, const TValue *p1, */ l_noret luaG_tointerror (lua_State *L, const TValue *p1, const TValue *p2) { lua_Integer temp; - if (!tointegerns(p1, &temp)) + if (!luaV_tointegerns(p1, &temp, LUA_FLOORN2I)) p2 = p1; luaG_runerror(L, "number%s has no integer representation", varinfo(L, p2)); } @@ -780,16 +832,30 @@ l_noret luaG_runerror (lua_State *L, const char *fmt, ...) { /* ** Check whether new instruction 'newpc' is in a different line from -** previous instruction 'oldpc'. +** previous instruction 'oldpc'. More often than not, 'newpc' is only +** one or a few instructions after 'oldpc' (it must be after, see +** caller), so try to avoid calling 'luaG_getfuncline'. If they are +** too far apart, there is a good chance of a ABSLINEINFO in the way, +** so it goes directly to 'luaG_getfuncline'. */ static int changedline (const Proto *p, int oldpc, int newpc) { if (p->lineinfo == NULL) /* no debug information? */ return 0; - while (oldpc++ < newpc) { - if (p->lineinfo[oldpc] != 0) - return (luaG_getfuncline(p, oldpc - 1) != luaG_getfuncline(p, newpc)); + if (newpc - oldpc < MAXIWTHABS / 2) { /* not too far apart? */ + int delta = 0; /* line diference */ + int pc = oldpc; + for (;;) { + int lineinfo = p->lineinfo[++pc]; + if (lineinfo == ABSLINEINFO) + break; /* cannot compute delta; fall through */ + delta += lineinfo; + if (pc == newpc) + return (delta != 0); /* delta computed successfully */ + } } - return 0; /* no line changes between positions */ + /* either instructions are too far apart or there is an absolute line + info in the way; compute line difference explicitly */ + return (luaG_getfuncline(p, oldpc) != luaG_getfuncline(p, newpc)); } @@ -797,20 +863,19 @@ static int changedline (const Proto *p, int oldpc, int newpc) { ** Traces the execution of a Lua function. Called before the execution ** of each opcode, when debug is on. 'L->oldpc' stores the last ** instruction traced, to detect line changes. When entering a new -** function, 'npci' will be zero and will test as a new line without -** the need for 'oldpc'; so, 'oldpc' does not need to be initialized -** before. Some exceptional conditions may return to a function without -** updating 'oldpc'. In that case, 'oldpc' may be invalid; if so, it is -** reset to zero. (A wrong but valid 'oldpc' at most causes an extra -** call to a line hook.) +** function, 'npci' will be zero and will test as a new line whatever +** the value of 'oldpc'. Some exceptional conditions may return to +** a function without setting 'oldpc'. In that case, 'oldpc' may be +** invalid; if so, use zero as a valid value. (A wrong but valid 'oldpc' +** at most causes an extra call to a line hook.) +** This function is not "Protected" when called, so it should correct +** 'L->top' before calling anything that can run the GC. */ int luaG_traceexec (lua_State *L, const Instruction *pc) { CallInfo *ci = L->ci; lu_byte mask = L->hookmask; const Proto *p = ci_func(ci)->p; int counthook; - /* 'L->oldpc' may be invalid; reset it in this case */ - int oldpc = (L->oldpc < p->sizecode) ? L->oldpc : 0; if (!(mask & (LUA_MASKLINE | LUA_MASKCOUNT))) { /* no hooks? */ ci->u.l.trap = 0; /* don't need to stop again */ return 0; /* turn off 'trap' */ @@ -826,15 +891,16 @@ int luaG_traceexec (lua_State *L, const Instruction *pc) { ci->callstatus &= ~CIST_HOOKYIELD; /* erase mark */ return 1; /* do not call hook again (VM yielded, so it did not move) */ } - if (!isIT(*(ci->u.l.savedpc - 1))) - L->top = ci->top; /* prepare top */ + if (!isIT(*(ci->u.l.savedpc - 1))) /* top not being used? */ + L->top = ci->top; /* correct top */ if (counthook) luaD_hook(L, LUA_HOOKCOUNT, -1, 0, 0); /* call count hook */ if (mask & LUA_MASKLINE) { + /* 'L->oldpc' may be invalid; use zero in this case */ + int oldpc = (L->oldpc < p->sizecode) ? L->oldpc : 0; int npci = pcRel(pc, p); - if (npci == 0 || /* call linehook when enter a new function, */ - pc <= invpcRel(oldpc, p) || /* when jump back (loop), or when */ - changedline(p, oldpc, npci)) { /* enter new line */ + if (npci <= oldpc || /* call hook when jump back (loop), */ + changedline(p, oldpc, npci)) { /* or when enter new line */ int newline = luaG_getfuncline(p, npci); luaD_hook(L, LUA_HOOKLINE, newline, 0, 0); /* call line hook */ } diff --git a/src/lua/ldebug.h b/src/lua/ldebug.h index a0a58486..974960e9 100644 --- a/src/lua/ldebug.h +++ b/src/lua/ldebug.h @@ -26,11 +26,22 @@ */ #define ABSLINEINFO (-0x80) + +/* +** MAXimum number of successive Instructions WiTHout ABSolute line +** information. (A power of two allows fast divisions.) +*/ +#if !defined(MAXIWTHABS) +#define MAXIWTHABS 128 +#endif + + LUAI_FUNC int luaG_getfuncline (const Proto *f, int pc); LUAI_FUNC const char *luaG_findlocal (lua_State *L, CallInfo *ci, int n, StkId *pos); LUAI_FUNC l_noret luaG_typeerror (lua_State *L, const TValue *o, const char *opname); +LUAI_FUNC l_noret luaG_callerror (lua_State *L, const TValue *o); LUAI_FUNC l_noret luaG_forerror (lua_State *L, const TValue *o, const char *what); LUAI_FUNC l_noret luaG_concaterror (lua_State *L, const TValue *p1, diff --git a/src/lua/ldo.c b/src/lua/ldo.c index 5473815a..a48e35f9 100644 --- a/src/lua/ldo.c +++ b/src/lua/ldo.c @@ -98,11 +98,12 @@ void luaD_seterrorobj (lua_State *L, int errcode, StkId oldtop) { setsvalue2s(L, oldtop, luaS_newliteral(L, "error in error handling")); break; } - case CLOSEPROTECT: { + case LUA_OK: { /* special case only for closing upvalues */ setnilvalue(s2v(oldtop)); /* no error message */ break; } default: { + lua_assert(errorstatus(errcode)); /* real error */ setobjs2s(L, oldtop, L->top - 1); /* error message on current top */ break; } @@ -118,17 +119,13 @@ l_noret luaD_throw (lua_State *L, int errcode) { } else { /* thread has no error handler */ global_State *g = G(L); - errcode = luaF_close(L, L->stack, errcode); /* close all upvalues */ - L->status = cast_byte(errcode); /* mark it as dead */ + errcode = luaE_resetthread(L, errcode); /* close all upvalues */ if (g->mainthread->errorJmp) { /* main thread has a handler? */ setobjs2s(L, g->mainthread->top++, L->top - 1); /* copy error obj. */ luaD_throw(g->mainthread, errcode); /* re-throw in main thread */ } else { /* no handler at all; abort */ if (g->panic) { /* panic function? */ - luaD_seterrorobj(L, errcode, L->top); /* assume EXTRA_STACK */ - if (L->ci->top < L->top) - L->ci->top = L->top; /* pushing msg. can break this invariant */ lua_unlock(L); g->panic(L); /* call panic function (last chance to jump out) */ } @@ -139,8 +136,7 @@ l_noret luaD_throw (lua_State *L, int errcode) { int luaD_rawrunprotected (lua_State *L, Pfunc f, void *ud) { - global_State *g = G(L); - l_uint32 oldnCcalls = g->Cstacklimit - (L->nCcalls + L->nci); + l_uint32 oldnCcalls = L->nCcalls; struct lua_longjmp lj; lj.status = LUA_OK; lj.previous = L->errorJmp; /* chain new error handler */ @@ -149,7 +145,7 @@ int luaD_rawrunprotected (lua_State *L, Pfunc f, void *ud) { (*f)(L, ud); ); L->errorJmp = lj.previous; /* restore old error handler */ - L->nCcalls = g->Cstacklimit - oldnCcalls - L->nci; + L->nCcalls = oldnCcalls; return lj.status; } @@ -164,9 +160,8 @@ int luaD_rawrunprotected (lua_State *L, Pfunc f, void *ud) { static void correctstack (lua_State *L, StkId oldstack, StkId newstack) { CallInfo *ci; UpVal *up; - if (oldstack == newstack) - return; /* stack address did not change */ L->top = (L->top - oldstack) + newstack; + L->tbclist = (L->tbclist - oldstack) + newstack; for (up = L->openupval; up != NULL; up = up->u.open.next) up->v = s2v((uplevel(up) - oldstack) + newstack); for (ci = L->ci; ci != NULL; ci = ci->previous) { @@ -182,22 +177,37 @@ static void correctstack (lua_State *L, StkId oldstack, StkId newstack) { #define ERRORSTACKSIZE (LUAI_MAXSTACK + 200) +/* +** Reallocate the stack to a new size, correcting all pointers into +** it. (There are pointers to a stack from its upvalues, from its list +** of call infos, plus a few individual pointers.) The reallocation is +** done in two steps (allocation + free) because the correction must be +** done while both addresses (the old stack and the new one) are valid. +** (In ISO C, any pointer use after the pointer has been deallocated is +** undefined behavior.) +** In case of allocation error, raise an error or return false according +** to 'raiseerror'. +*/ int luaD_reallocstack (lua_State *L, int newsize, int raiseerror) { - int lim = L->stacksize; - StkId newstack = luaM_reallocvector(L, L->stack, lim, newsize, StackValue); + int oldsize = stacksize(L); + int i; + StkId newstack = luaM_reallocvector(L, NULL, 0, + newsize + EXTRA_STACK, StackValue); lua_assert(newsize <= LUAI_MAXSTACK || newsize == ERRORSTACKSIZE); - lua_assert(L->stack_last - L->stack == L->stacksize - EXTRA_STACK); - if (unlikely(newstack == NULL)) { /* reallocation failed? */ + if (l_unlikely(newstack == NULL)) { /* reallocation failed? */ if (raiseerror) luaM_error(L); else return 0; /* do not raise an error */ } - for (; lim < newsize; lim++) - setnilvalue(s2v(newstack + lim)); /* erase new segment */ + /* number of elements to be copied to the new stack */ + i = ((oldsize <= newsize) ? oldsize : newsize) + EXTRA_STACK; + memcpy(newstack, L->stack, i * sizeof(StackValue)); + for (; i < newsize + EXTRA_STACK; i++) + setnilvalue(s2v(newstack + i)); /* erase new segment */ correctstack(L, L->stack, newstack); + luaM_freearray(L, L->stack, oldsize + EXTRA_STACK); L->stack = newstack; - L->stacksize = newsize; - L->stack_last = L->stack + newsize - EXTRA_STACK; + L->stack_last = L->stack + newsize; return 1; } @@ -207,51 +217,73 @@ int luaD_reallocstack (lua_State *L, int newsize, int raiseerror) { ** is true, raises any error; otherwise, return 0 in case of errors. */ int luaD_growstack (lua_State *L, int n, int raiseerror) { - int size = L->stacksize; - int newsize = 2 * size; /* tentative new size */ - if (unlikely(size > LUAI_MAXSTACK)) { /* need more space after extra size? */ + int size = stacksize(L); + if (l_unlikely(size > LUAI_MAXSTACK)) { + /* if stack is larger than maximum, thread is already using the + extra space reserved for errors, that is, thread is handling + a stack error; cannot grow further than that. */ + lua_assert(stacksize(L) == ERRORSTACKSIZE); if (raiseerror) luaD_throw(L, LUA_ERRERR); /* error inside message handler */ - else return 0; + return 0; /* if not 'raiseerror', just signal it */ } else { - int needed = cast_int(L->top - L->stack) + n + EXTRA_STACK; + int newsize = 2 * size; /* tentative new size */ + int needed = cast_int(L->top - L->stack) + n; if (newsize > LUAI_MAXSTACK) /* cannot cross the limit */ newsize = LUAI_MAXSTACK; if (newsize < needed) /* but must respect what was asked for */ newsize = needed; - if (unlikely(newsize > LUAI_MAXSTACK)) { /* stack overflow? */ + if (l_likely(newsize <= LUAI_MAXSTACK)) + return luaD_reallocstack(L, newsize, raiseerror); + else { /* stack overflow */ /* add extra size to be able to handle the error message */ luaD_reallocstack(L, ERRORSTACKSIZE, raiseerror); if (raiseerror) luaG_runerror(L, "stack overflow"); - else return 0; + return 0; } - } /* else no errors */ - return luaD_reallocstack(L, newsize, raiseerror); + } } static int stackinuse (lua_State *L) { CallInfo *ci; + int res; StkId lim = L->top; for (ci = L->ci; ci != NULL; ci = ci->previous) { if (lim < ci->top) lim = ci->top; } lua_assert(lim <= L->stack_last); - return cast_int(lim - L->stack) + 1; /* part of stack in use */ + res = cast_int(lim - L->stack) + 1; /* part of stack in use */ + if (res < LUA_MINSTACK) + res = LUA_MINSTACK; /* ensure a minimum size */ + return res; } +/* +** If stack size is more than 3 times the current use, reduce that size +** to twice the current use. (So, the final stack size is at most 2/3 the +** previous size, and half of its entries are empty.) +** As a particular case, if stack was handling a stack overflow and now +** it is not, 'max' (limited by LUAI_MAXSTACK) will be smaller than +** stacksize (equal to ERRORSTACKSIZE in this case), and so the stack +** will be reduced to a "regular" size. +*/ void luaD_shrinkstack (lua_State *L) { int inuse = stackinuse(L); - int goodsize = inuse + BASIC_STACK_SIZE; - if (goodsize > LUAI_MAXSTACK) - goodsize = LUAI_MAXSTACK; /* respect stack limit */ + int nsize = inuse * 2; /* proposed new size */ + int max = inuse * 3; /* maximum "reasonable" size */ + if (max > LUAI_MAXSTACK) { + max = LUAI_MAXSTACK; /* respect stack limit */ + if (nsize > LUAI_MAXSTACK) + nsize = LUAI_MAXSTACK; + } /* if thread is currently not handling a stack overflow and its - good size is smaller than current size, shrink its stack */ - if (inuse <= (LUAI_MAXSTACK - EXTRA_STACK) && goodsize < L->stacksize) - luaD_reallocstack(L, goodsize, 0); /* ok if that fails */ + size is larger than maximum "reasonable" size, shrink it */ + if (inuse <= LUAI_MAXSTACK && stacksize(L) > max) + luaD_reallocstack(L, nsize, 0); /* ok if that fails */ else /* don't change stack */ condmovestack(L,{},{}); /* (change only for debugging) */ luaE_shrinkCI(L); /* shrink CI list */ @@ -277,8 +309,8 @@ void luaD_hook (lua_State *L, int event, int line, if (hook && L->allowhook) { /* make sure there is a hook */ int mask = CIST_HOOKED; CallInfo *ci = L->ci; - ptrdiff_t top = savestack(L, L->top); - ptrdiff_t ci_top = savestack(L, ci->top); + ptrdiff_t top = savestack(L, L->top); /* preserve original 'top' */ + ptrdiff_t ci_top = savestack(L, ci->top); /* idem for 'ci->top' */ lua_Debug ar; ar.event = event; ar.currentline = line; @@ -288,8 +320,10 @@ void luaD_hook (lua_State *L, int event, int line, ci->u2.transferinfo.ftransfer = ftransfer; ci->u2.transferinfo.ntransfer = ntransfer; } + if (isLua(ci) && L->top < ci->top) + L->top = ci->top; /* protect entire activation register */ luaD_checkstack(L, LUA_MINSTACK); /* ensure minimum stack size */ - if (L->top + LUA_MINSTACK > ci->top) + if (ci->top < L->top + LUA_MINSTACK) ci->top = L->top + LUA_MINSTACK; L->allowhook = 0; /* cannot call hooks inside a hook */ ci->callstatus |= mask; @@ -311,55 +345,60 @@ void luaD_hook (lua_State *L, int event, int line, ** active. */ void luaD_hookcall (lua_State *L, CallInfo *ci) { - int hook = (ci->callstatus & CIST_TAIL) ? LUA_HOOKTAILCALL : LUA_HOOKCALL; - Proto *p; - if (!(L->hookmask & LUA_MASKCALL)) /* some other hook? */ - return; /* don't call hook */ - p = clLvalue(s2v(ci->func))->p; - L->top = ci->top; /* prepare top */ - ci->u.l.savedpc++; /* hooks assume 'pc' is already incremented */ - luaD_hook(L, hook, -1, 1, p->numparams); - ci->u.l.savedpc--; /* correct 'pc' */ + L->oldpc = 0; /* set 'oldpc' for new function */ + if (L->hookmask & LUA_MASKCALL) { /* is call hook on? */ + int event = (ci->callstatus & CIST_TAIL) ? LUA_HOOKTAILCALL + : LUA_HOOKCALL; + Proto *p = ci_func(ci)->p; + ci->u.l.savedpc++; /* hooks assume 'pc' is already incremented */ + luaD_hook(L, event, -1, 1, p->numparams); + ci->u.l.savedpc--; /* correct 'pc' */ + } } -static StkId rethook (lua_State *L, CallInfo *ci, StkId firstres, int nres) { - ptrdiff_t oldtop = savestack(L, L->top); /* hook may change top */ - int delta = 0; - if (isLuacode(ci)) { - Proto *p = ci_func(ci)->p; - if (p->is_vararg) - delta = ci->u.l.nextraargs + p->numparams + 1; - if (L->top < ci->top) - L->top = ci->top; /* correct top to run hook */ - } +/* +** Executes a return hook for Lua and C functions and sets/corrects +** 'oldpc'. (Note that this correction is needed by the line hook, so it +** is done even when return hooks are off.) +*/ +static void rethook (lua_State *L, CallInfo *ci, int nres) { if (L->hookmask & LUA_MASKRET) { /* is return hook on? */ + StkId firstres = L->top - nres; /* index of first result */ + int delta = 0; /* correction for vararg functions */ int ftransfer; + if (isLua(ci)) { + Proto *p = ci_func(ci)->p; + if (p->is_vararg) + delta = ci->u.l.nextraargs + p->numparams + 1; + } ci->func += delta; /* if vararg, back to virtual 'func' */ ftransfer = cast(unsigned short, firstres - ci->func); luaD_hook(L, LUA_HOOKRET, -1, ftransfer, nres); /* call it */ ci->func -= delta; } if (isLua(ci = ci->previous)) - L->oldpc = pcRel(ci->u.l.savedpc, ci_func(ci)->p); /* update 'oldpc' */ - return restorestack(L, oldtop); + L->oldpc = pcRel(ci->u.l.savedpc, ci_func(ci)->p); /* set 'oldpc' */ } /* ** Check whether 'func' has a '__call' metafield. If so, put it in the -** stack, below original 'func', so that 'luaD_call' can call it. Raise +** stack, below original 'func', so that 'luaD_precall' can call it. Raise ** an error if there is no '__call' metafield. */ -void luaD_tryfuncTM (lua_State *L, StkId func) { - const TValue *tm = luaT_gettmbyobj(L, s2v(func), TM_CALL); +StkId luaD_tryfuncTM (lua_State *L, StkId func) { + const TValue *tm; StkId p; - if (unlikely(ttisnil(tm))) - luaG_typeerror(L, s2v(func), "call"); /* nothing to call */ + checkstackGCp(L, 1, func); /* space for metamethod */ + tm = luaT_gettmbyobj(L, s2v(func), TM_CALL); /* (after previous GC) */ + if (l_unlikely(ttisnil(tm))) + luaG_callerror(L, s2v(func)); /* nothing to call */ for (p = L->top; p > func; p--) /* open space for metamethod */ setobjs2s(L, p, p-1); L->top++; /* stack space pre-allocated by the caller */ setobj2s(L, func, tm); /* metamethod is the new function to be called */ + return func; } @@ -369,7 +408,7 @@ void luaD_tryfuncTM (lua_State *L, StkId func) { ** expressions, multiple results for tail calls/single parameters) ** separated. */ -static void moveresults (lua_State *L, StkId res, int nres, int wanted) { +l_sinline void moveresults (lua_State *L, StkId res, int nres, int wanted) { StkId firstresult; int i; switch (wanted) { /* handle typical cases separately */ @@ -379,27 +418,34 @@ static void moveresults (lua_State *L, StkId res, int nres, int wanted) { case 1: /* one value needed */ if (nres == 0) /* no results? */ setnilvalue(s2v(res)); /* adjust with nil */ - else + else /* at least one result */ setobjs2s(L, res, L->top - nres); /* move it to proper place */ L->top = res + 1; return; case LUA_MULTRET: wanted = nres; /* we want all results */ break; - default: /* multiple results (or to-be-closed variables) */ + default: /* two/more results and/or to-be-closed variables */ if (hastocloseCfunc(wanted)) { /* to-be-closed variables? */ ptrdiff_t savedres = savestack(L, res); - luaF_close(L, res, LUA_OK); /* may change the stack */ - res = restorestack(L, savedres); - wanted = codeNresults(wanted); /* correct value */ + L->ci->callstatus |= CIST_CLSRET; /* in case of yields */ + L->ci->u2.nres = nres; + luaF_close(L, res, CLOSEKTOP, 1); + L->ci->callstatus &= ~CIST_CLSRET; + if (L->hookmask) /* if needed, call hook after '__close's */ + rethook(L, L->ci, nres); + res = restorestack(L, savedres); /* close and hook can move stack */ + wanted = decodeNresults(wanted); if (wanted == LUA_MULTRET) - wanted = nres; + wanted = nres; /* we want all results */ } break; } + /* generic case */ firstresult = L->top - nres; /* index of first result */ - /* move all results to correct place */ - for (i = 0; i < nres && i < wanted; i++) + if (nres > wanted) /* extra results? */ + nres = wanted; /* don't need them */ + for (i = 0; i < nres; i++) /* move all results to correct place */ setobjs2s(L, res + i, firstresult + i); for (; i < wanted; i++) /* complete wanted number of results */ setnilvalue(s2v(res + i)); @@ -408,15 +454,21 @@ static void moveresults (lua_State *L, StkId res, int nres, int wanted) { /* -** Finishes a function call: calls hook if necessary, removes CallInfo, -** moves current number of results to proper place. +** Finishes a function call: calls hook if necessary, moves current +** number of results to proper place, and returns to previous call +** info. If function has to close variables, hook must be called after +** that. */ void luaD_poscall (lua_State *L, CallInfo *ci, int nres) { - if (L->hookmask) - L->top = rethook(L, ci, L->top - nres, nres); - L->ci = ci->previous; /* back to caller */ + int wanted = ci->nresults; + if (l_unlikely(L->hookmask && !hastocloseCfunc(wanted))) + rethook(L, ci, nres); /* move results to proper place */ - moveresults(L, ci->func, nres, ci->nresults); + moveresults(L, ci->func, nres, wanted); + /* function cannot be in any of these cases when returning */ + lua_assert(!(ci->callstatus & + (CIST_HOOKED | CIST_YPCALL | CIST_FIN | CIST_TRAN | CIST_CLSRET))); + L->ci = ci->previous; /* back to caller (after closing variables) */ } @@ -424,66 +476,101 @@ void luaD_poscall (lua_State *L, CallInfo *ci, int nres) { #define next_ci(L) (L->ci->next ? L->ci->next : luaE_extendCI(L)) -/* -** Prepare a function for a tail call, building its call info on top -** of the current call info. 'narg1' is the number of arguments plus 1 -** (so that it includes the function itself). -*/ -void luaD_pretailcall (lua_State *L, CallInfo *ci, StkId func, int narg1) { - Proto *p = clLvalue(s2v(func))->p; - int fsize = p->maxstacksize; /* frame size */ - int nfixparams = p->numparams; - int i; - for (i = 0; i < narg1; i++) /* move down function and arguments */ - setobjs2s(L, ci->func + i, func + i); - checkstackGC(L, fsize); - func = ci->func; /* moved-down function */ - for (; narg1 <= nfixparams; narg1++) - setnilvalue(s2v(func + narg1)); /* complete missing arguments */ - ci->top = func + 1 + fsize; /* top for new function */ - lua_assert(ci->top <= L->stack_last); - ci->u.l.savedpc = p->code; /* starting point */ - ci->callstatus |= CIST_TAIL; - L->top = func + narg1; /* set top */ +l_sinline CallInfo *prepCallInfo (lua_State *L, StkId func, int nret, + int mask, StkId top) { + CallInfo *ci = L->ci = next_ci(L); /* new frame */ + ci->func = func; + ci->nresults = nret; + ci->callstatus = mask; + ci->top = top; + return ci; } /* -** Call a function (C or Lua). The function to be called is at *func. -** The arguments are on the stack, right after the function. -** When returns, all the results are on the stack, starting at the original -** function position. +** precall for C functions */ -void luaD_call (lua_State *L, StkId func, int nresults) { - lua_CFunction f; +l_sinline int precallC (lua_State *L, StkId func, int nresults, + lua_CFunction f) { + int n; /* number of returns */ + CallInfo *ci; + checkstackGCp(L, LUA_MINSTACK, func); /* ensure minimum stack size */ + L->ci = ci = prepCallInfo(L, func, nresults, CIST_C, + L->top + LUA_MINSTACK); + lua_assert(ci->top <= L->stack_last); + if (l_unlikely(L->hookmask & LUA_MASKCALL)) { + int narg = cast_int(L->top - func) - 1; + luaD_hook(L, LUA_HOOKCALL, -1, 1, narg); + } + lua_unlock(L); + n = (*f)(L); /* do the actual call */ + lua_lock(L); + api_checknelems(L, n); + luaD_poscall(L, ci, n); + return n; +} + + +/* +** Prepare a function for a tail call, building its call info on top +** of the current call info. 'narg1' is the number of arguments plus 1 +** (so that it includes the function itself). Return the number of +** results, if it was a C function, or -1 for a Lua function. +*/ +int luaD_pretailcall (lua_State *L, CallInfo *ci, StkId func, + int narg1, int delta) { retry: switch (ttypetag(s2v(func))) { case LUA_VCCL: /* C closure */ - f = clCvalue(s2v(func))->f; - goto Cfunc; + return precallC(L, func, LUA_MULTRET, clCvalue(s2v(func))->f); case LUA_VLCF: /* light C function */ - f = fvalue(s2v(func)); - Cfunc: { - int n; /* number of returns */ - CallInfo *ci; - checkstackGCp(L, LUA_MINSTACK, func); /* ensure minimum stack size */ - L->ci = ci = next_ci(L); - ci->nresults = nresults; - ci->callstatus = CIST_C; - ci->top = L->top + LUA_MINSTACK; - ci->func = func; + return precallC(L, func, LUA_MULTRET, fvalue(s2v(func))); + case LUA_VLCL: { /* Lua function */ + Proto *p = clLvalue(s2v(func))->p; + int fsize = p->maxstacksize; /* frame size */ + int nfixparams = p->numparams; + int i; + checkstackGCp(L, fsize - delta, func); + ci->func -= delta; /* restore 'func' (if vararg) */ + for (i = 0; i < narg1; i++) /* move down function and arguments */ + setobjs2s(L, ci->func + i, func + i); + func = ci->func; /* moved-down function */ + for (; narg1 <= nfixparams; narg1++) + setnilvalue(s2v(func + narg1)); /* complete missing arguments */ + ci->top = func + 1 + fsize; /* top for new function */ lua_assert(ci->top <= L->stack_last); - if (L->hookmask & LUA_MASKCALL) { - int narg = cast_int(L->top - func) - 1; - luaD_hook(L, LUA_HOOKCALL, -1, 1, narg); - } - lua_unlock(L); - n = (*f)(L); /* do the actual call */ - lua_lock(L); - api_checknelems(L, n); - luaD_poscall(L, ci, n); - break; + ci->u.l.savedpc = p->code; /* starting point */ + ci->callstatus |= CIST_TAIL; + L->top = func + narg1; /* set top */ + return -1; } + default: { /* not a function */ + func = luaD_tryfuncTM(L, func); /* try to get '__call' metamethod */ + /* return luaD_pretailcall(L, ci, func, narg1 + 1, delta); */ + narg1++; + goto retry; /* try again */ + } + } +} + + +/* +** Prepares the call to a function (C or Lua). For C functions, also do +** the call. The function to be called is at '*func'. The arguments +** are on the stack, right after the function. Returns the CallInfo +** to be executed, if it was a Lua function. Otherwise (a C function) +** returns NULL, with all the results on the stack, starting at the +** original function position. +*/ +CallInfo *luaD_precall (lua_State *L, StkId func, int nresults) { + retry: + switch (ttypetag(s2v(func))) { + case LUA_VCCL: /* C closure */ + precallC(L, func, nresults, clCvalue(s2v(func))->f); + return NULL; + case LUA_VLCF: /* light C function */ + precallC(L, func, nresults, fvalue(s2v(func))); + return NULL; case LUA_VLCL: { /* Lua function */ CallInfo *ci; Proto *p = clLvalue(s2v(func))->p; @@ -491,22 +578,16 @@ void luaD_call (lua_State *L, StkId func, int nresults) { int nfixparams = p->numparams; int fsize = p->maxstacksize; /* frame size */ checkstackGCp(L, fsize, func); - L->ci = ci = next_ci(L); - ci->nresults = nresults; + L->ci = ci = prepCallInfo(L, func, nresults, 0, func + 1 + fsize); ci->u.l.savedpc = p->code; /* starting point */ - ci->callstatus = 0; - ci->top = func + 1 + fsize; - ci->func = func; - L->ci = ci; for (; narg < nfixparams; narg++) setnilvalue(s2v(L->top++)); /* complete missing arguments */ lua_assert(ci->top <= L->stack_last); - luaV_execute(L, ci); /* run the function */ - break; + return ci; } default: { /* not a function */ - checkstackGCp(L, 1, func); /* space for metamethod */ - luaD_tryfuncTM(L, func); /* try to get '__call' metamethod */ + func = luaD_tryfuncTM(L, func); /* try to get '__call' metamethod */ + /* return luaD_precall(L, func, nresults); */ goto retry; /* try again with metamethod */ } } @@ -514,41 +595,108 @@ void luaD_call (lua_State *L, StkId func, int nresults) { /* -** Similar to 'luaD_call', but does not allow yields during the call. +** Call a function (C or Lua) through C. 'inc' can be 1 (increment +** number of recursive invocations in the C stack) or nyci (the same +** plus increment number of non-yieldable calls). */ -void luaD_callnoyield (lua_State *L, StkId func, int nResults) { - incXCcalls(L); - if (getCcalls(L) <= CSTACKERR) { /* possible C stack overflow? */ - luaE_exitCcall(L); /* to compensate decrement in next call */ - luaE_enterCcall(L); /* check properly */ +l_sinline void ccall (lua_State *L, StkId func, int nResults, int inc) { + CallInfo *ci; + L->nCcalls += inc; + if (l_unlikely(getCcalls(L) >= LUAI_MAXCCALLS)) + luaE_checkcstack(L); + if ((ci = luaD_precall(L, func, nResults)) != NULL) { /* Lua function? */ + ci->callstatus = CIST_FRESH; /* mark that it is a "fresh" execute */ + luaV_execute(L, ci); /* call it */ } - luaD_call(L, func, nResults); - decXCcalls(L); + L->nCcalls -= inc; } /* -** Completes the execution of an interrupted C function, calling its -** continuation function. +** External interface for 'ccall' */ -static void finishCcall (lua_State *L, int status) { - CallInfo *ci = L->ci; - int n; - /* must have a continuation and must be able to call it */ - lua_assert(ci->u.c.k != NULL && yieldable(L)); - /* error status can only happen in a protected call */ - lua_assert((ci->callstatus & CIST_YPCALL) || status == LUA_YIELD); - if (ci->callstatus & CIST_YPCALL) { /* was inside a pcall? */ - ci->callstatus &= ~CIST_YPCALL; /* continuation is also inside it */ - L->errfunc = ci->u.c.old_errfunc; /* with the same error function */ +void luaD_call (lua_State *L, StkId func, int nResults) { + ccall(L, func, nResults, 1); +} + + +/* +** Similar to 'luaD_call', but does not allow yields during the call. +*/ +void luaD_callnoyield (lua_State *L, StkId func, int nResults) { + ccall(L, func, nResults, nyci); +} + + +/* +** Finish the job of 'lua_pcallk' after it was interrupted by an yield. +** (The caller, 'finishCcall', does the final call to 'adjustresults'.) +** The main job is to complete the 'luaD_pcall' called by 'lua_pcallk'. +** If a '__close' method yields here, eventually control will be back +** to 'finishCcall' (when that '__close' method finally returns) and +** 'finishpcallk' will run again and close any still pending '__close' +** methods. Similarly, if a '__close' method errs, 'precover' calls +** 'unroll' which calls ''finishCcall' and we are back here again, to +** close any pending '__close' methods. +** Note that, up to the call to 'luaF_close', the corresponding +** 'CallInfo' is not modified, so that this repeated run works like the +** first one (except that it has at least one less '__close' to do). In +** particular, field CIST_RECST preserves the error status across these +** multiple runs, changing only if there is a new error. +*/ +static int finishpcallk (lua_State *L, CallInfo *ci) { + int status = getcistrecst(ci); /* get original status */ + if (l_likely(status == LUA_OK)) /* no error? */ + status = LUA_YIELD; /* was interrupted by an yield */ + else { /* error */ + StkId func = restorestack(L, ci->u2.funcidx); + L->allowhook = getoah(ci->callstatus); /* restore 'allowhook' */ + luaF_close(L, func, status, 1); /* can yield or raise an error */ + func = restorestack(L, ci->u2.funcidx); /* stack may be moved */ + luaD_seterrorobj(L, status, func); + luaD_shrinkstack(L); /* restore stack size in case of overflow */ + setcistrecst(ci, LUA_OK); /* clear original status */ + } + ci->callstatus &= ~CIST_YPCALL; + L->errfunc = ci->u.c.old_errfunc; + /* if it is here, there were errors or yields; unlike 'lua_pcallk', + do not change status */ + return status; +} + + +/* +** Completes the execution of a C function interrupted by an yield. +** The interruption must have happened while the function was either +** closing its tbc variables in 'moveresults' or executing +** 'lua_callk'/'lua_pcallk'. In the first case, it just redoes +** 'luaD_poscall'. In the second case, the call to 'finishpcallk' +** finishes the interrupted execution of 'lua_pcallk'. After that, it +** calls the continuation of the interrupted function and finally it +** completes the job of the 'luaD_call' that called the function. In +** the call to 'adjustresults', we do not know the number of results +** of the function called by 'lua_callk'/'lua_pcallk', so we are +** conservative and use LUA_MULTRET (always adjust). +*/ +static void finishCcall (lua_State *L, CallInfo *ci) { + int n; /* actual number of results from C function */ + if (ci->callstatus & CIST_CLSRET) { /* was returning? */ + lua_assert(hastocloseCfunc(ci->nresults)); + n = ci->u2.nres; /* just redo 'luaD_poscall' */ + /* don't need to reset CIST_CLSRET, as it will be set again anyway */ + } + else { + int status = LUA_YIELD; /* default if there were no errors */ + /* must have a continuation and must be able to call it */ + lua_assert(ci->u.c.k != NULL && yieldable(L)); + if (ci->callstatus & CIST_YPCALL) /* was inside a 'lua_pcallk'? */ + status = finishpcallk(L, ci); /* finish it */ + adjustresults(L, LUA_MULTRET); /* finish 'lua_callk' */ + lua_unlock(L); + n = (*ci->u.c.k)(L, status, ci->u.c.ctx); /* call continuation */ + lua_lock(L); + api_checknelems(L, n); } - /* finish 'lua_callk'/'lua_pcall'; CIST_YPCALL and 'errfunc' already - handled */ - adjustresults(L, ci->nresults); - lua_unlock(L); - n = (*ci->u.c.k)(L, status, ci->u.c.ctx); /* call continuation function */ - lua_lock(L); - api_checknelems(L, n); luaD_poscall(L, ci, n); /* finish 'luaD_call' */ } @@ -556,18 +704,14 @@ static void finishCcall (lua_State *L, int status) { /* ** Executes "full continuation" (everything in the stack) of a ** previously interrupted coroutine until the stack is empty (or another -** interruption long-jumps out of the loop). If the coroutine is -** recovering from an error, 'ud' points to the error status, which must -** be passed to the first continuation function (otherwise the default -** status is LUA_YIELD). +** interruption long-jumps out of the loop). */ static void unroll (lua_State *L, void *ud) { CallInfo *ci; - if (ud != NULL) /* error status? */ - finishCcall(L, *(int *)ud); /* finish 'lua_pcallk' callee */ + UNUSED(ud); while ((ci = L->ci) != &L->base_ci) { /* something in the stack */ if (!isLua(ci)) /* C function? */ - finishCcall(L, LUA_YIELD); /* complete its execution */ + finishCcall(L, ci); /* complete its execution */ else { /* Lua function */ luaV_finishOp(L); /* finish interrupted instruction */ luaV_execute(L, ci); /* execute down to higher C 'boundary' */ @@ -590,28 +734,6 @@ static CallInfo *findpcall (lua_State *L) { } -/* -** Recovers from an error in a coroutine. Finds a recover point (if -** there is one) and completes the execution of the interrupted -** 'luaD_pcall'. If there is no recover point, returns zero. -*/ -static int recover (lua_State *L, int status) { - StkId oldtop; - CallInfo *ci = findpcall(L); - if (ci == NULL) return 0; /* no recovery point */ - /* "finish" luaD_pcall */ - oldtop = restorestack(L, ci->u2.funcidx); - luaF_close(L, oldtop, status); /* may change the stack */ - oldtop = restorestack(L, ci->u2.funcidx); - luaD_seterrorobj(L, status, oldtop); - L->ci = ci; - L->allowhook = getoah(ci->callstatus); /* restore original 'allowhook' */ - luaD_shrinkstack(L); - L->errfunc = ci->u.c.old_errfunc; - return 1; /* continue running the coroutine */ -} - - /* ** Signal an error in the call to 'lua_resume', not in the execution ** of the coroutine itself. (Such errors should not be handled by any @@ -637,14 +759,15 @@ static void resume (lua_State *L, void *ud) { int n = *(cast(int*, ud)); /* number of arguments */ StkId firstArg = L->top - n; /* first argument */ CallInfo *ci = L->ci; - if (L->status == LUA_OK) { /* starting a coroutine? */ - luaD_call(L, firstArg - 1, LUA_MULTRET); - } + if (L->status == LUA_OK) /* starting a coroutine? */ + ccall(L, firstArg - 1, LUA_MULTRET, 0); /* just call its body */ else { /* resuming from previous yield */ lua_assert(L->status == LUA_YIELD); L->status = LUA_OK; /* mark that it is running (again) */ - if (isLua(ci)) /* yielded inside a hook? */ + if (isLua(ci)) { /* yielded inside a hook? */ + L->top = firstArg; /* discard arguments */ luaV_execute(L, ci); /* just continue running Lua code */ + } else { /* 'common' yield */ if (ci->u.c.k != NULL) { /* does it have a continuation function? */ lua_unlock(L); @@ -658,6 +781,26 @@ static void resume (lua_State *L, void *ud) { } } + +/* +** Unrolls a coroutine in protected mode while there are recoverable +** errors, that is, errors inside a protected call. (Any error +** interrupts 'unroll', and this loop protects it again so it can +** continue.) Stops with a normal end (status == LUA_OK), an yield +** (status == LUA_YIELD), or an unprotected error ('findpcall' doesn't +** find a recover point). +*/ +static int precover (lua_State *L, int status) { + CallInfo *ci; + while (errorstatus(status) && (ci = findpcall(L)) != NULL) { + L->ci = ci; /* go down to recovery functions */ + setcistrecst(ci, status); /* status to finish 'pcall' */ + status = luaD_rawrunprotected(L, unroll, NULL); + } + return status; +} + + LUA_API int lua_resume (lua_State *L, lua_State *from, int nargs, int *nresults) { int status; @@ -670,21 +813,16 @@ LUA_API int lua_resume (lua_State *L, lua_State *from, int nargs, } else if (L->status != LUA_YIELD) /* ended with errors? */ return resume_error(L, "cannot resume dead coroutine", nargs); - if (from == NULL) - L->nCcalls = CSTACKTHREAD; - else /* correct 'nCcalls' for this thread */ - L->nCcalls = getCcalls(from) - L->nci - CSTACKCF; - if (L->nCcalls <= CSTACKERR) + L->nCcalls = (from) ? getCcalls(from) : 0; + if (getCcalls(L) >= LUAI_MAXCCALLS) return resume_error(L, "C stack overflow", nargs); + L->nCcalls++; luai_userstateresume(L, nargs); api_checknelems(L, (L->status == LUA_OK) ? nargs + 1 : nargs); status = luaD_rawrunprotected(L, resume, &nargs); /* continue running after recoverable errors */ - while (errorstatus(status) && recover(L, status)) { - /* unroll continuation */ - status = luaD_rawrunprotected(L, unroll, &status); - } - if (likely(!errorstatus(status))) + status = precover(L, status); + if (l_likely(!errorstatus(status))) lua_assert(status == L->status); /* normal end or yield */ else { /* unrecoverable error */ L->status = cast_byte(status); /* mark thread as 'dead' */ @@ -710,22 +848,22 @@ LUA_API int lua_yieldk (lua_State *L, int nresults, lua_KContext ctx, lua_lock(L); ci = L->ci; api_checknelems(L, nresults); - if (unlikely(!yieldable(L))) { + if (l_unlikely(!yieldable(L))) { if (L != G(L)->mainthread) luaG_runerror(L, "attempt to yield across a C-call boundary"); else luaG_runerror(L, "attempt to yield from outside a coroutine"); } L->status = LUA_YIELD; + ci->u2.nyield = nresults; /* save number of results */ if (isLua(ci)) { /* inside a hook? */ lua_assert(!isLuacode(ci)); + api_check(L, nresults == 0, "hooks cannot yield values"); api_check(L, k == NULL, "hooks cannot continue after yielding"); - ci->u2.nyield = 0; /* no results */ } else { if ((ci->u.c.k = k) != NULL) /* is there a continuation? */ ci->u.c.ctx = ctx; /* save context */ - ci->u2.nyield = nresults; /* save number of results */ luaD_throw(L, LUA_YIELD); } lua_assert(ci->callstatus & CIST_HOOKED); /* must be inside a hook */ @@ -734,6 +872,45 @@ LUA_API int lua_yieldk (lua_State *L, int nresults, lua_KContext ctx, } +/* +** Auxiliary structure to call 'luaF_close' in protected mode. +*/ +struct CloseP { + StkId level; + int status; +}; + + +/* +** Auxiliary function to call 'luaF_close' in protected mode. +*/ +static void closepaux (lua_State *L, void *ud) { + struct CloseP *pcl = cast(struct CloseP *, ud); + luaF_close(L, pcl->level, pcl->status, 0); +} + + +/* +** Calls 'luaF_close' in protected mode. Return the original status +** or, in case of errors, the new status. +*/ +int luaD_closeprotected (lua_State *L, ptrdiff_t level, int status) { + CallInfo *old_ci = L->ci; + lu_byte old_allowhooks = L->allowhook; + for (;;) { /* keep closing upvalues until no more errors */ + struct CloseP pcl; + pcl.level = restorestack(L, level); pcl.status = status; + status = luaD_rawrunprotected(L, &closepaux, &pcl); + if (l_likely(status == LUA_OK)) /* no more errors? */ + return pcl.status; + else { /* an error occurred; restore saved state and repeat */ + L->ci = old_ci; + L->allowhook = old_allowhooks; + } + } +} + + /* ** Call the C function 'func' in protected mode, restoring basic ** thread information ('allowhook', etc.) and in particular @@ -747,14 +924,12 @@ int luaD_pcall (lua_State *L, Pfunc func, void *u, ptrdiff_t old_errfunc = L->errfunc; L->errfunc = ef; status = luaD_rawrunprotected(L, func, u); - if (unlikely(status != LUA_OK)) { /* an error occurred? */ - StkId oldtop = restorestack(L, old_top); + if (l_unlikely(status != LUA_OK)) { /* an error occurred? */ L->ci = old_ci; L->allowhook = old_allowhooks; - status = luaF_close(L, oldtop, status); - oldtop = restorestack(L, old_top); /* previous call may change stack */ - luaD_seterrorobj(L, status, oldtop); - luaD_shrinkstack(L); + status = luaD_closeprotected(L, old_top, status); + luaD_seterrorobj(L, status, restorestack(L, old_top)); + luaD_shrinkstack(L); /* restore stack size in case of overflow */ } L->errfunc = old_errfunc; return status; diff --git a/src/lua/ldo.h b/src/lua/ldo.h index 6c6cb285..911e67f6 100644 --- a/src/lua/ldo.h +++ b/src/lua/ldo.h @@ -23,7 +23,7 @@ ** at every check. */ #define luaD_checkstackaux(L,n,pre,pos) \ - if (L->stack_last - L->top <= (n)) \ + if (l_unlikely(L->stack_last - L->top <= (n))) \ { pre; luaD_growstack(L, n, 1); pos; } \ else { condmovestack(L,pre,pos); } @@ -58,10 +58,12 @@ LUAI_FUNC int luaD_protectedparser (lua_State *L, ZIO *z, const char *name, LUAI_FUNC void luaD_hook (lua_State *L, int event, int line, int fTransfer, int nTransfer); LUAI_FUNC void luaD_hookcall (lua_State *L, CallInfo *ci); -LUAI_FUNC void luaD_pretailcall (lua_State *L, CallInfo *ci, StkId func, int n); +LUAI_FUNC int luaD_pretailcall (lua_State *L, CallInfo *ci, StkId func, int narg1, int delta); +LUAI_FUNC CallInfo *luaD_precall (lua_State *L, StkId func, int nResults); LUAI_FUNC void luaD_call (lua_State *L, StkId func, int nResults); LUAI_FUNC void luaD_callnoyield (lua_State *L, StkId func, int nResults); -LUAI_FUNC void luaD_tryfuncTM (lua_State *L, StkId func); +LUAI_FUNC StkId luaD_tryfuncTM (lua_State *L, StkId func); +LUAI_FUNC int luaD_closeprotected (lua_State *L, ptrdiff_t level, int status); LUAI_FUNC int luaD_pcall (lua_State *L, Pfunc func, void *u, ptrdiff_t oldtop, ptrdiff_t ef); LUAI_FUNC void luaD_poscall (lua_State *L, CallInfo *ci, int nres); diff --git a/src/lua/lfunc.c b/src/lua/lfunc.c index 88d45328..f5889a21 100644 --- a/src/lua/lfunc.c +++ b/src/lua/lfunc.c @@ -53,7 +53,7 @@ void luaF_initupvals (lua_State *L, LClosure *cl) { uv->v = &uv->u.value; /* make it closed */ setnilvalue(uv->v); cl->upvals[i] = uv; - luaC_objbarrier(L, cl, o); + luaC_objbarrier(L, cl, uv); } } @@ -100,115 +100,83 @@ UpVal *luaF_findupval (lua_State *L, StkId level) { } -static void callclose (lua_State *L, void *ud) { - UNUSED(ud); - luaD_callnoyield(L, L->top - 3, 0); -} - - /* -** Prepare closing method plus its arguments for object 'obj' with -** error message 'err'. (This function assumes EXTRA_STACK.) +** Call closing method for object 'obj' with error message 'err'. The +** boolean 'yy' controls whether the call is yieldable. +** (This function assumes EXTRA_STACK.) */ -static int prepclosingmethod (lua_State *L, TValue *obj, TValue *err) { +static void callclosemethod (lua_State *L, TValue *obj, TValue *err, int yy) { StkId top = L->top; const TValue *tm = luaT_gettmbyobj(L, obj, TM_CLOSE); - if (ttisnil(tm)) /* no metamethod? */ - return 0; /* nothing to call */ setobj2s(L, top, tm); /* will call metamethod... */ setobj2s(L, top + 1, obj); /* with 'self' as the 1st argument */ setobj2s(L, top + 2, err); /* and error msg. as 2nd argument */ L->top = top + 3; /* add function and arguments */ - return 1; + if (yy) + luaD_call(L, top, 0); + else + luaD_callnoyield(L, top, 0); } /* -** Raise an error with message 'msg', inserting the name of the -** local variable at position 'level' in the stack. +** Check whether object at given level has a close metamethod and raise +** an error if not. */ -static void varerror (lua_State *L, StkId level, const char *msg) { - int idx = cast_int(level - L->ci->func); - const char *vname = luaG_findlocal(L, L->ci, idx, NULL); - if (vname == NULL) vname = "?"; - luaG_runerror(L, msg, vname); +static void checkclosemth (lua_State *L, StkId level) { + const TValue *tm = luaT_gettmbyobj(L, s2v(level), TM_CLOSE); + if (ttisnil(tm)) { /* no metamethod? */ + int idx = cast_int(level - L->ci->func); /* variable index */ + const char *vname = luaG_findlocal(L, L->ci, idx, NULL); + if (vname == NULL) vname = "?"; + luaG_runerror(L, "variable '%s' got a non-closable value", vname); + } } /* -** Prepare and call a closing method. If status is OK, code is still -** inside the original protected call, and so any error will be handled -** there. Otherwise, a previous error already activated the original -** protected call, and so the call to the closing method must be -** protected here. (A status == CLOSEPROTECT behaves like a previous -** error, to also run the closing method in protected mode). -** If status is OK, the call to the closing method will be pushed -** at the top of the stack. Otherwise, values are pushed after -** the 'level' of the upvalue being closed, as everything after -** that won't be used again. +** Prepare and call a closing method. +** If status is CLOSEKTOP, the call to the closing method will be pushed +** at the top of the stack. Otherwise, values can be pushed right after +** the 'level' of the upvalue being closed, as everything after that +** won't be used again. */ -static int callclosemth (lua_State *L, StkId level, int status) { +static void prepcallclosemth (lua_State *L, StkId level, int status, int yy) { TValue *uv = s2v(level); /* value being closed */ - if (likely(status == LUA_OK)) { - if (prepclosingmethod(L, uv, &G(L)->nilvalue)) /* something to call? */ - callclose(L, NULL); /* call closing method */ - else if (!l_isfalse(uv)) /* non-closable non-false value? */ - varerror(L, level, "attempt to close non-closable variable '%s'"); + TValue *errobj; + if (status == CLOSEKTOP) + errobj = &G(L)->nilvalue; /* error object is nil */ + else { /* 'luaD_seterrorobj' will set top to level + 2 */ + errobj = s2v(level + 1); /* error object goes after 'uv' */ + luaD_seterrorobj(L, status, level + 1); /* set error object */ } - else { /* must close the object in protected mode */ - ptrdiff_t oldtop; - level++; /* space for error message */ - oldtop = savestack(L, level + 1); /* top will be after that */ - luaD_seterrorobj(L, status, level); /* set error message */ - if (prepclosingmethod(L, uv, s2v(level))) { /* something to call? */ - int newstatus = luaD_pcall(L, callclose, NULL, oldtop, 0); - if (newstatus != LUA_OK && status == CLOSEPROTECT) /* first error? */ - status = newstatus; /* this will be the new error */ - else { - if (newstatus != LUA_OK) /* suppressed error? */ - luaE_warnerror(L, "__close metamethod"); - /* leave original error (or nil) on top */ - L->top = restorestack(L, oldtop); - } - } - /* else no metamethod; ignore this case and keep original error */ - } - return status; + callclosemethod(L, uv, errobj, yy); } /* -** Try to create a to-be-closed upvalue -** (can raise a memory-allocation error) +** Maximum value for deltas in 'tbclist', dependent on the type +** of delta. (This macro assumes that an 'L' is in scope where it +** is used.) */ -static void trynewtbcupval (lua_State *L, void *ud) { - newupval(L, 1, cast(StkId, ud), &L->openupval); -} +#define MAXDELTA \ + ((256ul << ((sizeof(L->stack->tbclist.delta) - 1) * 8)) - 1) /* -** Create a to-be-closed upvalue. If there is a memory error -** when creating the upvalue, the closing method must be called here, -** as there is no upvalue to call it later. +** Insert a variable in the list of to-be-closed variables. */ void luaF_newtbcupval (lua_State *L, StkId level) { - TValue *obj = s2v(level); - lua_assert(L->openupval == NULL || uplevel(L->openupval) < level); - if (!l_isfalse(obj)) { /* false doesn't need to be closed */ - int status; - const TValue *tm = luaT_gettmbyobj(L, obj, TM_CLOSE); - if (ttisnil(tm)) /* no metamethod? */ - varerror(L, level, "variable '%s' got a non-closable value"); - status = luaD_rawrunprotected(L, trynewtbcupval, level); - if (unlikely(status != LUA_OK)) { /* memory error creating upvalue? */ - lua_assert(status == LUA_ERRMEM); - luaD_seterrorobj(L, LUA_ERRMEM, level + 1); /* save error message */ - /* next call must succeed, as object is closable */ - prepclosingmethod(L, s2v(level), s2v(level + 1)); - callclose(L, NULL); /* call closing method */ - luaD_throw(L, LUA_ERRMEM); /* throw memory error */ - } + lua_assert(level > L->tbclist); + if (l_isfalse(s2v(level))) + return; /* false doesn't need to be closed */ + checkclosemth(L, level); /* value must have a close method */ + while (cast_uint(level - L->tbclist) > MAXDELTA) { + L->tbclist += MAXDELTA; /* create a dummy node at maximum delta */ + L->tbclist->tbclist.delta = 0; } + level->tbclist.delta = cast(unsigned short, level - L->tbclist); + L->tbclist = level; } @@ -220,18 +188,16 @@ void luaF_unlinkupval (UpVal *uv) { } -int luaF_close (lua_State *L, StkId level, int status) { +/* +** Close all upvalues up to the given stack level. +*/ +void luaF_closeupval (lua_State *L, StkId level) { UpVal *uv; - while ((uv = L->openupval) != NULL && uplevel(uv) >= level) { + StkId upl; /* stack index pointed by 'uv' */ + while ((uv = L->openupval) != NULL && (upl = uplevel(uv)) >= level) { TValue *slot = &uv->u.value; /* new position for value */ lua_assert(uplevel(uv) < L->top); - if (uv->tbc && status != NOCLOSINGMETH) { - /* must run closing method, which may change the stack */ - ptrdiff_t levelrel = savestack(L, level); - status = callclosemth(L, uplevel(uv), status); - level = restorestack(L, levelrel); - } - luaF_unlinkupval(uv); + luaF_unlinkupval(uv); /* remove upvalue from 'openupval' list */ setobj(L, slot, uv->v); /* move value to upvalue slot */ uv->v = slot; /* now current value lives here */ if (!iswhite(uv)) { /* neither white nor dead? */ @@ -239,7 +205,35 @@ int luaF_close (lua_State *L, StkId level, int status) { luaC_barrier(L, uv, slot); } } - return status; +} + + +/* +** Remove firt element from the tbclist plus its dummy nodes. +*/ +static void poptbclist (lua_State *L) { + StkId tbc = L->tbclist; + lua_assert(tbc->tbclist.delta > 0); /* first element cannot be dummy */ + tbc -= tbc->tbclist.delta; + while (tbc > L->stack && tbc->tbclist.delta == 0) + tbc -= MAXDELTA; /* remove dummy nodes */ + L->tbclist = tbc; +} + + +/* +** Close all upvalues and to-be-closed variables up to the given stack +** level. +*/ +void luaF_close (lua_State *L, StkId level, int status, int yy) { + ptrdiff_t levelrel = savestack(L, level); + luaF_closeupval(L, level); /* first, close the upvalues */ + while (L->tbclist >= level) { /* traverse tbc's down to that level */ + StkId tbc = L->tbclist; /* get variable index */ + poptbclist(L); /* remove it from list */ + prepcallclosemth(L, tbc, status, yy); /* close variable */ + level = restorestack(L, levelrel); + } } diff --git a/src/lua/lfunc.h b/src/lua/lfunc.h index 8d6f965c..dc1cebcc 100644 --- a/src/lua/lfunc.h +++ b/src/lua/lfunc.h @@ -42,15 +42,9 @@ #define MAXMISS 10 -/* -** Special "status" for 'luaF_close' -*/ -/* close upvalues without running their closing methods */ -#define NOCLOSINGMETH (-1) - -/* close upvalues running all closing methods in protected mode */ -#define CLOSEPROTECT (-2) +/* special status to close upvalues preserving the top of the stack */ +#define CLOSEKTOP (-1) LUAI_FUNC Proto *luaF_newproto (lua_State *L); @@ -59,7 +53,8 @@ LUAI_FUNC LClosure *luaF_newLclosure (lua_State *L, int nupvals); LUAI_FUNC void luaF_initupvals (lua_State *L, LClosure *cl); LUAI_FUNC UpVal *luaF_findupval (lua_State *L, StkId level); LUAI_FUNC void luaF_newtbcupval (lua_State *L, StkId level); -LUAI_FUNC int luaF_close (lua_State *L, StkId level, int status); +LUAI_FUNC void luaF_closeupval (lua_State *L, StkId level); +LUAI_FUNC void luaF_close (lua_State *L, StkId level, int status, int yy); LUAI_FUNC void luaF_unlinkupval (UpVal *uv); LUAI_FUNC void luaF_freeproto (lua_State *L, Proto *f); LUAI_FUNC const char *luaF_getlocalname (const Proto *func, int local_number, diff --git a/src/lua/lgc.c b/src/lua/lgc.c index 4a7bcaed..42a73d81 100644 --- a/src/lua/lgc.c +++ b/src/lua/lgc.c @@ -161,18 +161,17 @@ static void linkgclist_ (GCObject *o, GCObject **pnext, GCObject **list) { /* -** Clear keys for empty entries in tables. If entry is empty -** and its key is not marked, mark its entry as dead. This allows the -** collection of the key, but keeps its entry in the table (its removal -** could break a chain). The main feature of a dead key is that it must -** be different from any other value, to do not disturb searches. -** Other places never manipulate dead keys, because its associated empty -** value is enough to signal that the entry is logically empty. +** Clear keys for empty entries in tables. If entry is empty, mark its +** entry as dead. This allows the collection of the key, but keeps its +** entry in the table: its removal could break a chain and could break +** a table traversal. Other places never manipulate dead keys, because +** its associated empty value is enough to signal that the entry is +** logically empty. */ static void clearkey (Node *n) { lua_assert(isempty(gval(n))); - if (keyiswhite(n)) - setdeadkey(n); /* unused and unmarked key; remove it */ + if (keyiscollectable(n)) + setdeadkey(n); /* unused key; remove it */ } @@ -301,7 +300,7 @@ static void reallymarkobject (global_State *g, GCObject *o) { if (upisopen(uv)) set2gray(uv); /* open upvalues are kept gray */ else - set2black(o); /* closed upvalues are visited here */ + set2black(uv); /* closed upvalues are visited here */ markvalue(g, uv->v); /* mark its content */ break; } @@ -309,7 +308,7 @@ static void reallymarkobject (global_State *g, GCObject *o) { Udata *u = gco2u(o); if (u->nuvalue == 0) { /* no user values? */ markobjectN(g, u->metatable); /* mark its metatable */ - set2black(o); /* nothing else to mark */ + set2black(u); /* nothing else to mark */ break; } /* else... */ @@ -633,9 +632,8 @@ static int traversethread (global_State *g, lua_State *th) { for (uv = th->openupval; uv != NULL; uv = uv->u.open.next) markobject(g, uv); /* open upvalues cannot be collected */ if (g->gcstate == GCSatomic) { /* final traversal? */ - StkId lim = th->stack + th->stacksize; /* real end of stack */ - for (; o < lim; o++) /* clear not-marked stack slice */ - setnilvalue(s2v(o)); + for (; o < th->stack_last + EXTRA_STACK; o++) + setnilvalue(s2v(o)); /* clear dead stack slice */ /* 'remarkupvals' may have removed thread from 'twups' list */ if (!isintwups(th) && th->openupval != NULL) { th->twups = g->twups; /* link it back to the list */ @@ -644,7 +642,7 @@ static int traversethread (global_State *g, lua_State *th) { } else if (!g->gcemergency) luaD_shrinkstack(th); /* do not change stack in emergency cycle */ - return 1 + th->stacksize; + return 1 + stacksize(th); } @@ -771,12 +769,16 @@ static void freeobj (lua_State *L, GCObject *o) { case LUA_VUPVAL: freeupval(L, gco2upv(o)); break; - case LUA_VLCL: - luaM_freemem(L, o, sizeLclosure(gco2lcl(o)->nupvalues)); + case LUA_VLCL: { + LClosure *cl = gco2lcl(o); + luaM_freemem(L, cl, sizeLclosure(cl->nupvalues)); break; - case LUA_VCCL: - luaM_freemem(L, o, sizeCclosure(gco2ccl(o)->nupvalues)); + } + case LUA_VCCL: { + CClosure *cl = gco2ccl(o); + luaM_freemem(L, cl, sizeCclosure(cl->nupvalues)); break; + } case LUA_VTABLE: luaH_free(L, gco2t(o)); break; @@ -788,13 +790,17 @@ static void freeobj (lua_State *L, GCObject *o) { luaM_freemem(L, o, sizeudata(u->nuvalue, u->len)); break; } - case LUA_VSHRSTR: - luaS_remove(L, gco2ts(o)); /* remove it from hash table */ - luaM_freemem(L, o, sizelstring(gco2ts(o)->shrlen)); + case LUA_VSHRSTR: { + TString *ts = gco2ts(o); + luaS_remove(L, ts); /* remove it from hash table */ + luaM_freemem(L, ts, sizelstring(ts->shrlen)); break; - case LUA_VLNGSTR: - luaM_freemem(L, o, sizelstring(gco2ts(o)->u.lnglen)); + } + case LUA_VLNGSTR: { + TString *ts = gco2ts(o); + luaM_freemem(L, ts, sizelstring(ts->u.lnglen)); break; + } default: lua_assert(0); } } @@ -900,18 +906,18 @@ static void GCTM (lua_State *L) { if (!notm(tm)) { /* is there a finalizer? */ int status; lu_byte oldah = L->allowhook; - int running = g->gcrunning; + int oldgcstp = g->gcstp; + g->gcstp |= GCSTPGC; /* avoid GC steps */ L->allowhook = 0; /* stop debug hooks during GC metamethod */ - g->gcrunning = 0; /* avoid GC steps */ setobj2s(L, L->top++, tm); /* push finalizer... */ setobj2s(L, L->top++, &v); /* ... and its argument */ L->ci->callstatus |= CIST_FIN; /* will run a finalizer */ status = luaD_pcall(L, dothecall, NULL, savestack(L, L->top - 2), 0); L->ci->callstatus &= ~CIST_FIN; /* not running a finalizer anymore */ L->allowhook = oldah; /* restore hooks */ - g->gcrunning = running; /* restore state */ - if (unlikely(status != LUA_OK)) { /* error while running __gc? */ - luaE_warnerror(L, "__gc metamethod"); + g->gcstp = oldgcstp; /* restore state */ + if (l_unlikely(status != LUA_OK)) { /* error while running __gc? */ + luaE_warnerror(L, "__gc"); L->top--; /* pops error object */ } } @@ -1005,7 +1011,8 @@ static void correctpointers (global_State *g, GCObject *o) { void luaC_checkfinalizer (lua_State *L, GCObject *o, Table *mt) { global_State *g = G(L); if (tofinalize(o) || /* obj. is already marked... */ - gfasttm(g, mt, TM_GC) == NULL) /* or has no finalizer? */ + gfasttm(g, mt, TM_GC) == NULL || /* or has no finalizer... */ + (g->gcstp & GCSTPCLS)) /* or closing state? */ return; /* nothing to be done */ else { /* move 'o' to 'finobj' list */ GCObject **p; @@ -1496,12 +1503,13 @@ static void deletelist (lua_State *L, GCObject *p, GCObject *limit) { */ void luaC_freeallobjects (lua_State *L) { global_State *g = G(L); + g->gcstp = GCSTPCLS; /* no extra finalizers after here */ luaC_changemode(L, KGC_INC); separatetobefnz(g, 1); /* separate all objects with finalizers */ lua_assert(g->finobj == NULL); callallpendingfinalizers(L); deletelist(L, g->allgc, obj2gco(g->mainthread)); - deletelist(L, g->finobj, NULL); + lua_assert(g->finobj == NULL); /* no new finalizers */ deletelist(L, g->fixedgc, NULL); /* collect fixed objects */ lua_assert(g->strt.nuse == 0); } @@ -1569,52 +1577,64 @@ static int sweepstep (lua_State *L, global_State *g, static lu_mem singlestep (lua_State *L) { global_State *g = G(L); + lu_mem work; + lua_assert(!g->gcstopem); /* collector is not reentrant */ + g->gcstopem = 1; /* no emergency collections while collecting */ switch (g->gcstate) { case GCSpause: { restartcollection(g); g->gcstate = GCSpropagate; - return 1; + work = 1; + break; } case GCSpropagate: { if (g->gray == NULL) { /* no more gray objects? */ g->gcstate = GCSenteratomic; /* finish propagate phase */ - return 0; + work = 0; } else - return propagatemark(g); /* traverse one gray object */ + work = propagatemark(g); /* traverse one gray object */ + break; } case GCSenteratomic: { - lu_mem work = atomic(L); /* work is what was traversed by 'atomic' */ + work = atomic(L); /* work is what was traversed by 'atomic' */ entersweep(L); g->GCestimate = gettotalbytes(g); /* first estimate */; - return work; + break; } case GCSswpallgc: { /* sweep "regular" objects */ - return sweepstep(L, g, GCSswpfinobj, &g->finobj); + work = sweepstep(L, g, GCSswpfinobj, &g->finobj); + break; } case GCSswpfinobj: { /* sweep objects with finalizers */ - return sweepstep(L, g, GCSswptobefnz, &g->tobefnz); + work = sweepstep(L, g, GCSswptobefnz, &g->tobefnz); + break; } case GCSswptobefnz: { /* sweep objects to be finalized */ - return sweepstep(L, g, GCSswpend, NULL); + work = sweepstep(L, g, GCSswpend, NULL); + break; } case GCSswpend: { /* finish sweeps */ checkSizes(L, g); g->gcstate = GCScallfin; - return 0; + work = 0; + break; } case GCScallfin: { /* call remaining finalizers */ if (g->tobefnz && !g->gcemergency) { - int n = runafewfinalizers(L, GCFINMAX); - return n * GCFINALIZECOST; + g->gcstopem = 0; /* ok collections during finalizers */ + work = runafewfinalizers(L, GCFINMAX) * GCFINALIZECOST; } else { /* emergency mode or no more finalizers */ g->gcstate = GCSpause; /* finish collection */ - return 0; + work = 0; } + break; } default: lua_assert(0); return 0; } + g->gcstopem = 0; + return work; } @@ -1629,6 +1649,7 @@ void luaC_runtilstate (lua_State *L, int statesmask) { } + /* ** Performs a basic incremental step. The debt and step size are ** converted from bytes to "units of work"; then the function loops @@ -1660,7 +1681,7 @@ static void incstep (lua_State *L, global_State *g) { void luaC_step (lua_State *L) { global_State *g = G(L); lua_assert(!g->gcemergency); - if (g->gcrunning) { /* running? */ + if (gcrunning(g)) { /* running? */ if(isdecGCmodegen(g)) genstep(L, g); else diff --git a/src/lua/lgc.h b/src/lua/lgc.h index 073e2a40..4a125634 100644 --- a/src/lua/lgc.h +++ b/src/lua/lgc.h @@ -148,6 +148,16 @@ */ #define isdecGCmodegen(g) (g->gckind == KGC_GEN || g->lastatomic != 0) + +/* +** Control when GC is running: +*/ +#define GCSTPUSR 1 /* bit true when GC stopped by user */ +#define GCSTPGC 2 /* bit true when GC stopped by itself */ +#define GCSTPCLS 4 /* bit true when closing Lua state */ +#define gcrunning(g) ((g)->gcstp == 0) + + /* ** Does one step of collection when debt becomes positive. 'pre'/'pos' ** allows some adjustments to be done only when needed. macro diff --git a/src/lua/liolib.c b/src/lua/liolib.c index 60ab1bfa..b08397da 100644 --- a/src/lua/liolib.c +++ b/src/lua/liolib.c @@ -52,12 +52,6 @@ static int l_checkmode (const char *mode) { ** ======================================================= */ -#if !defined(l_checkmodep) -/* By default, Lua accepts only "r" or "w" as mode */ -#define l_checkmodep(m) ((m[0] == 'r' || m[0] == 'w') && m[1] == '\0') -#endif - - #if !defined(l_popen) /* { */ #if defined(LUA_USE_POSIX) /* { */ @@ -70,6 +64,12 @@ static int l_checkmode (const char *mode) { #define l_popen(L,c,m) (_popen(c,m)) #define l_pclose(L,file) (_pclose(file)) +#if !defined(l_checkmodep) +/* Windows accepts "[rw][bt]?" as valid modes */ +#define l_checkmodep(m) ((m[0] == 'r' || m[0] == 'w') && \ + (m[1] == '\0' || ((m[1] == 'b' || m[1] == 't') && m[2] == '\0'))) +#endif + #else /* }{ */ /* ISO C definitions */ @@ -83,6 +83,12 @@ static int l_checkmode (const char *mode) { #endif /* } */ + +#if !defined(l_checkmodep) +/* By default, Lua accepts only "r" or "w" as valid modes */ +#define l_checkmodep(m) ((m[0] == 'r' || m[0] == 'w') && m[1] == '\0') +#endif + /* }====================================================== */ @@ -180,7 +186,7 @@ static int f_tostring (lua_State *L) { static FILE *tofile (lua_State *L) { LStream *p = tolstream(L); - if (isclosed(p)) + if (l_unlikely(isclosed(p))) luaL_error(L, "attempt to use a closed file"); lua_assert(p->f); return p->f; @@ -255,7 +261,7 @@ static LStream *newfile (lua_State *L) { static void opencheck (lua_State *L, const char *fname, const char *mode) { LStream *p = newfile(L); p->f = fopen(fname, mode); - if (p->f == NULL) + if (l_unlikely(p->f == NULL)) luaL_error(L, "cannot open file '%s' (%s)", fname, strerror(errno)); } @@ -303,7 +309,7 @@ static FILE *getiofile (lua_State *L, const char *findex) { LStream *p; lua_getfield(L, LUA_REGISTRYINDEX, findex); p = (LStream *)lua_touserdata(L, -1); - if (isclosed(p)) + if (l_unlikely(isclosed(p))) luaL_error(L, "default %s file is closed", findex + IOPREF_LEN); return p->f; } @@ -430,7 +436,7 @@ typedef struct { ** Add current char to buffer (if not out of space) and read next one */ static int nextc (RN *rn) { - if (rn->n >= L_MAXLENNUM) { /* buffer overflow? */ + if (l_unlikely(rn->n >= L_MAXLENNUM)) { /* buffer overflow? */ rn->buff[0] = '\0'; /* invalidate result */ return 0; /* fail */ } @@ -493,8 +499,8 @@ static int read_number (lua_State *L, FILE *f) { ungetc(rn.c, rn.f); /* unread look-ahead char */ l_unlockfile(rn.f); rn.buff[rn.n] = '\0'; /* finish string */ - if (lua_stringtonumber(L, rn.buff)) /* is this a valid number? */ - return 1; /* ok */ + if (l_likely(lua_stringtonumber(L, rn.buff))) + return 1; /* ok, it is a valid number */ else { /* invalid format */ lua_pushnil(L); /* "result" to be removed */ return 0; /* read fails */ @@ -670,7 +676,8 @@ static int g_write (lua_State *L, FILE *f, int arg) { status = status && (fwrite(s, sizeof(char), l, f) == l); } } - if (status) return 1; /* file handle already on stack top */ + if (l_likely(status)) + return 1; /* file handle already on stack top */ else return luaL_fileresult(L, status, NULL); } @@ -697,7 +704,7 @@ static int f_seek (lua_State *L) { luaL_argcheck(L, (lua_Integer)offset == p3, 3, "not an integer in proper range"); op = l_fseek(f, offset, mode[op]); - if (op) + if (l_unlikely(op)) return luaL_fileresult(L, 0, NULL); /* error */ else { lua_pushinteger(L, (lua_Integer)l_ftell(f)); diff --git a/src/lua/llex.c b/src/lua/llex.c index 3d6b2b97..e9915178 100644 --- a/src/lua/llex.c +++ b/src/lua/llex.c @@ -122,26 +122,29 @@ l_noret luaX_syntaxerror (LexState *ls, const char *msg) { /* -** creates a new string and anchors it in scanner's table so that -** it will not be collected until the end of the compilation -** (by that time it should be anchored somewhere) +** Creates a new string and anchors it in scanner's table so that it +** will not be collected until the end of the compilation; by that time +** it should be anchored somewhere. It also internalizes long strings, +** ensuring there is only one copy of each unique string. The table +** here is used as a set: the string enters as the key, while its value +** is irrelevant. We use the string itself as the value only because it +** is a TValue readly available. Later, the code generation can change +** this value. */ TString *luaX_newstring (LexState *ls, const char *str, size_t l) { lua_State *L = ls->L; - TValue *o; /* entry for 'str' */ TString *ts = luaS_newlstr(L, str, l); /* create new string */ - setsvalue2s(L, L->top++, ts); /* temporarily anchor it in stack */ - o = luaH_set(L, ls->h, s2v(L->top - 1)); - if (isempty(o)) { /* not in use yet? */ - /* boolean value does not need GC barrier; - table is not a metatable, so it does not need to invalidate cache */ - setbtvalue(o); /* t[string] = true */ + const TValue *o = luaH_getstr(ls->h, ts); + if (!ttisnil(o)) /* string already present? */ + ts = keystrval(nodefromval(o)); /* get saved copy */ + else { /* not in use yet */ + TValue *stv = s2v(L->top++); /* reserve stack space for string */ + setsvalue(L, stv, ts); /* temporarily anchor the string */ + luaH_finishset(L, ls->h, stv, o, stv); /* t[string] = string */ + /* table is not a metatable, so it does not need to invalidate cache */ luaC_checkGC(L); + L->top--; /* remove string from stack */ } - else { /* string already present */ - ts = keystrval(nodefromval(o)); /* re-use value previously stored */ - } - L->top--; /* remove string from stack */ return ts; } @@ -254,9 +257,10 @@ static int read_numeral (LexState *ls, SemInfo *seminfo) { /* -** reads a sequence '[=*[' or ']=*]', leaving the last bracket. -** If sequence is well formed, return its number of '='s + 2; otherwise, -** return 1 if there is no '='s or 0 otherwise (an unfinished '[==...'). +** read a sequence '[=*[' or ']=*]', leaving the last bracket. If +** sequence is well formed, return its number of '='s + 2; otherwise, +** return 1 if it is a single bracket (no '='s and no 2nd bracket); +** otherwise (an unfinished '[==...') return 0. */ static size_t skip_sep (LexState *ls) { size_t count = 0; @@ -481,34 +485,34 @@ static int llex (LexState *ls, SemInfo *seminfo) { } case '=': { next(ls); - if (check_next1(ls, '=')) return TK_EQ; + if (check_next1(ls, '=')) return TK_EQ; /* '==' */ else return '='; } case '<': { next(ls); - if (check_next1(ls, '=')) return TK_LE; - else if (check_next1(ls, '<')) return TK_SHL; + if (check_next1(ls, '=')) return TK_LE; /* '<=' */ + else if (check_next1(ls, '<')) return TK_SHL; /* '<<' */ else return '<'; } case '>': { next(ls); - if (check_next1(ls, '=')) return TK_GE; - else if (check_next1(ls, '>')) return TK_SHR; + if (check_next1(ls, '=')) return TK_GE; /* '>=' */ + else if (check_next1(ls, '>')) return TK_SHR; /* '>>' */ else return '>'; } case '/': { next(ls); - if (check_next1(ls, '/')) return TK_IDIV; + if (check_next1(ls, '/')) return TK_IDIV; /* '//' */ else return '/'; } case '~': { next(ls); - if (check_next1(ls, '=')) return TK_NE; + if (check_next1(ls, '=')) return TK_NE; /* '~=' */ else return '~'; } case ':': { next(ls); - if (check_next1(ls, ':')) return TK_DBCOLON; + if (check_next1(ls, ':')) return TK_DBCOLON; /* '::' */ else return ':'; } case '"': case '\'': { /* short literal strings */ @@ -547,7 +551,7 @@ static int llex (LexState *ls, SemInfo *seminfo) { return TK_NAME; } } - else { /* single-char tokens (+ - / ...) */ + else { /* single-char tokens ('+', '*', '%', '{', '}', ...) */ int c = ls->current; next(ls); return c; diff --git a/src/lua/llimits.h b/src/lua/llimits.h index 48c97f95..52a32f92 100644 --- a/src/lua/llimits.h +++ b/src/lua/llimits.h @@ -149,22 +149,6 @@ typedef LUAI_UACINT l_uacInt; #endif -/* -** macros to improve jump prediction (used mainly for error handling) -*/ -#if !defined(likely) - -#if defined(__GNUC__) -#define likely(x) (__builtin_expect(((x) != 0), 1)) -#define unlikely(x) (__builtin_expect(((x) != 0), 0)) -#else -#define likely(x) (x) -#define unlikely(x) (x) -#endif - -#endif - - /* ** non-return type */ @@ -181,6 +165,20 @@ typedef LUAI_UACINT l_uacInt; #endif +/* +** Inline functions +*/ +#if !defined(LUA_USE_C89) +#define l_inline inline +#elif defined(__GNUC__) +#define l_inline __inline__ +#else +#define l_inline /* empty */ +#endif + +#define l_sinline static l_inline + + /* ** type for virtual-machine instructions; ** must be an unsigned with (at least) 4 bytes (see details in lopcodes.h) @@ -234,6 +232,17 @@ typedef l_uint32 Instruction; #endif +/* +** Maximum depth for nested C calls, syntactical nested non-terminals, +** and other features implemented through recursion in C. (Value must +** fit in a 16-bit unsigned integer. It must also be compatible with +** the size of the C stack.) +*/ +#if !defined(LUAI_MAXCCALLS) +#define LUAI_MAXCCALLS 200 +#endif + + /* ** macros that are executed whenever program enters the Lua core ** ('lua_lock') and leaves the core ('lua_unlock') @@ -315,7 +324,8 @@ typedef l_uint32 Instruction; /* exponentiation */ #if !defined(luai_numpow) -#define luai_numpow(L,a,b) ((void)L, l_mathop(pow)(a,b)) +#define luai_numpow(L,a,b) \ + ((void)L, (b == 2) ? (a)*(a) : l_mathop(pow)(a,b)) #endif /* the others are quite standard operations */ @@ -344,14 +354,14 @@ typedef l_uint32 Instruction; #else /* realloc stack keeping its size */ #define condmovestack(L,pre,pos) \ - { int sz_ = (L)->stacksize; pre; luaD_reallocstack((L), sz_, 0); pos; } + { int sz_ = stacksize(L); pre; luaD_reallocstack((L), sz_, 0); pos; } #endif #if !defined(HARDMEMTESTS) #define condchangemem(L,pre,pos) ((void)0) #else #define condchangemem(L,pre,pos) \ - { if (G(L)->gcrunning) { pre; luaC_fullgc(L, 0); pos; } } + { if (gcrunning(G(L))) { pre; luaC_fullgc(L, 0); pos; } } #endif #endif diff --git a/src/lua/lmathlib.c b/src/lua/lmathlib.c index 86def470..e0c61a16 100644 --- a/src/lua/lmathlib.c +++ b/src/lua/lmathlib.c @@ -73,7 +73,7 @@ static int math_atan (lua_State *L) { static int math_toint (lua_State *L) { int valid; lua_Integer n = lua_tointegerx(L, 1, &valid); - if (valid) + if (l_likely(valid)) lua_pushinteger(L, n); else { luaL_checkany(L, 1); @@ -175,7 +175,8 @@ static int math_log (lua_State *L) { lua_Number base = luaL_checknumber(L, 2); #if !defined(LUA_USE_C89) if (base == l_mathop(2.0)) - res = l_mathop(log2)(x); else + res = l_mathop(log2)(x); + else #endif if (base == l_mathop(10.0)) res = l_mathop(log10)(x); @@ -474,7 +475,7 @@ static lua_Number I2d (Rand64 x) { /* 2^(-FIGS) = 1.0 / 2^30 / 2^3 / 2^(FIGS-33) */ #define scaleFIG \ - ((lua_Number)1.0 / (UONE << 30) / 8.0 / (UONE << (FIGS - 33))) + (l_mathop(1.0) / (UONE << 30) / l_mathop(8.0) / (UONE << (FIGS - 33))) /* ** use FIGS - 32 bits from lower half, throwing out the other @@ -485,7 +486,7 @@ static lua_Number I2d (Rand64 x) { /* ** higher 32 bits go after those (FIGS - 32) bits: shiftHI = 2^(FIGS - 32) */ -#define shiftHI ((lua_Number)(UONE << (FIGS - 33)) * 2.0) +#define shiftHI ((lua_Number)(UONE << (FIGS - 33)) * l_mathop(2.0)) static lua_Number I2d (Rand64 x) { diff --git a/src/lua/lmem.c b/src/lua/lmem.c index 43739bff..9029d588 100644 --- a/src/lua/lmem.c +++ b/src/lua/lmem.c @@ -24,12 +24,12 @@ #if defined(EMERGENCYGCTESTS) /* -** First allocation will fail whenever not building initial state -** and not shrinking a block. (This fail will trigger 'tryagain' and -** a full GC cycle at every allocation.) +** First allocation will fail whenever not building initial state. +** (This fail will trigger 'tryagain' and a full GC cycle at every +** allocation.) */ static void *firsttry (global_State *g, void *block, size_t os, size_t ns) { - if (ttisnil(&g->nilvalue) && ns > os) + if (completestate(g) && ns > 0) /* frees never fail */ return NULL; /* fail */ else /* normal allocation */ return (*g->frealloc)(g->ud, block, os, ns); @@ -83,7 +83,7 @@ void *luaM_growaux_ (lua_State *L, void *block, int nelems, int *psize, if (nelems + 1 <= size) /* does one extra element still fit? */ return block; /* nothing to be done */ if (size >= limit / 2) { /* cannot double it? */ - if (unlikely(size >= limit)) /* cannot grow even a little? */ + if (l_unlikely(size >= limit)) /* cannot grow even a little? */ luaG_runerror(L, "too many %s (limit is %d)", what, limit); size = limit; /* still have at least one free place */ } @@ -138,15 +138,17 @@ void luaM_free_ (lua_State *L, void *block, size_t osize) { /* -** In case of allocation fail, this function will call the GC to try -** to free some memory and then try the allocation again. -** (It should not be called when shrinking a block, because then the -** interpreter may be in the middle of a collection step.) +** In case of allocation fail, this function will do an emergency +** collection to free some memory and then try the allocation again. +** The GC should not be called while state is not fully built, as the +** collector is not yet fully initialized. Also, it should not be called +** when 'gcstopem' is true, because then the interpreter is in the +** middle of a collection step. */ static void *tryagain (lua_State *L, void *block, size_t osize, size_t nsize) { global_State *g = G(L); - if (ttisnil(&g->nilvalue)) { /* is state fully build? */ + if (completestate(g) && !g->gcstopem) { luaC_fullgc(L, 1); /* try to free some memory... */ return (*g->frealloc)(g->ud, block, osize, nsize); /* try again */ } @@ -156,17 +158,14 @@ static void *tryagain (lua_State *L, void *block, /* ** Generic allocation routine. -** If allocation fails while shrinking a block, do not try again; the -** GC shrinks some blocks and it is not reentrant. */ void *luaM_realloc_ (lua_State *L, void *block, size_t osize, size_t nsize) { void *newblock; global_State *g = G(L); lua_assert((osize == 0) == (block == NULL)); newblock = firsttry(g, block, osize, nsize); - if (unlikely(newblock == NULL && nsize > 0)) { - if (nsize > osize) /* not shrinking a block? */ - newblock = tryagain(L, block, osize, nsize); + if (l_unlikely(newblock == NULL && nsize > 0)) { + newblock = tryagain(L, block, osize, nsize); if (newblock == NULL) /* still no memory? */ return NULL; /* do not update 'GCdebt' */ } @@ -179,7 +178,7 @@ void *luaM_realloc_ (lua_State *L, void *block, size_t osize, size_t nsize) { void *luaM_saferealloc_ (lua_State *L, void *block, size_t osize, size_t nsize) { void *newblock = luaM_realloc_(L, block, osize, nsize); - if (unlikely(newblock == NULL && nsize > 0)) /* allocation failed? */ + if (l_unlikely(newblock == NULL && nsize > 0)) /* allocation failed? */ luaM_error(L); return newblock; } @@ -191,7 +190,7 @@ void *luaM_malloc_ (lua_State *L, size_t size, int tag) { else { global_State *g = G(L); void *newblock = firsttry(g, NULL, tag, size); - if (unlikely(newblock == NULL)) { + if (l_unlikely(newblock == NULL)) { newblock = tryagain(L, NULL, tag, size); if (newblock == NULL) luaM_error(L); diff --git a/src/lua/loadlib.c b/src/lua/loadlib.c index c0ec9a13..6f9fa373 100644 --- a/src/lua/loadlib.c +++ b/src/lua/loadlib.c @@ -132,14 +132,16 @@ static void lsys_unloadlib (void *lib) { static void *lsys_load (lua_State *L, const char *path, int seeglb) { void *lib = dlopen(path, RTLD_NOW | (seeglb ? RTLD_GLOBAL : RTLD_LOCAL)); - if (lib == NULL) lua_pushstring(L, dlerror()); + if (l_unlikely(lib == NULL)) + lua_pushstring(L, dlerror()); return lib; } static lua_CFunction lsys_sym (lua_State *L, void *lib, const char *sym) { lua_CFunction f = cast_func(dlsym(lib, sym)); - if (f == NULL) lua_pushstring(L, dlerror()); + if (l_unlikely(f == NULL)) + lua_pushstring(L, dlerror()); return f; } @@ -410,7 +412,7 @@ static int ll_loadlib (lua_State *L) { const char *path = luaL_checkstring(L, 1); const char *init = luaL_checkstring(L, 2); int stat = lookforfunc(L, path, init); - if (stat == 0) /* no errors? */ + if (l_likely(stat == 0)) /* no errors? */ return 1; /* return the loaded function */ else { /* error; error message is on stack top */ luaL_pushfail(L); @@ -523,14 +525,14 @@ static const char *findfile (lua_State *L, const char *name, const char *path; lua_getfield(L, lua_upvalueindex(1), pname); path = lua_tostring(L, -1); - if (path == NULL) + if (l_unlikely(path == NULL)) luaL_error(L, "'package.%s' must be a string", pname); return searchpath(L, name, path, ".", dirsep); } static int checkload (lua_State *L, int stat, const char *filename) { - if (stat) { /* module loaded successfully? */ + if (l_likely(stat)) { /* module loaded successfully? */ lua_pushstring(L, filename); /* will be 2nd argument to module */ return 2; /* return open function and file name */ } @@ -623,13 +625,14 @@ static void findloader (lua_State *L, const char *name) { int i; luaL_Buffer msg; /* to build error message */ /* push 'package.searchers' to index 3 in the stack */ - if (lua_getfield(L, lua_upvalueindex(1), "searchers") != LUA_TTABLE) + if (l_unlikely(lua_getfield(L, lua_upvalueindex(1), "searchers") + != LUA_TTABLE)) luaL_error(L, "'package.searchers' must be a table"); luaL_buffinit(L, &msg); /* iterate over available searchers to find a loader */ for (i = 1; ; i++) { luaL_addstring(&msg, "\n\t"); /* error-message prefix */ - if (lua_rawgeti(L, 3, i) == LUA_TNIL) { /* no more searchers? */ + if (l_unlikely(lua_rawgeti(L, 3, i) == LUA_TNIL)) { /* no more searchers? */ lua_pop(L, 1); /* remove nil */ luaL_buffsub(&msg, 2); /* remove prefix */ luaL_pushresult(&msg); /* create error message */ diff --git a/src/lua/lobject.c b/src/lua/lobject.c index f8ea917a..301aa900 100644 --- a/src/lua/lobject.c +++ b/src/lua/lobject.c @@ -164,7 +164,7 @@ static int isneg (const char **s) { */ static lua_Number lua_strx2number (const char *s, char **endptr) { int dot = lua_getlocaledecpoint(); - lua_Number r = 0.0; /* result (accumulator) */ + lua_Number r = l_mathop(0.0); /* result (accumulator) */ int sigdig = 0; /* number of significant digits */ int nosigdig = 0; /* number of non-significant digits */ int e = 0; /* exponent correction */ @@ -174,7 +174,7 @@ static lua_Number lua_strx2number (const char *s, char **endptr) { while (lisspace(cast_uchar(*s))) s++; /* skip initial spaces */ neg = isneg(&s); /* check sign */ if (!(*s == '0' && (*(s + 1) == 'x' || *(s + 1) == 'X'))) /* check '0x' */ - return 0.0; /* invalid format (no '0x') */ + return l_mathop(0.0); /* invalid format (no '0x') */ for (s += 2; ; s++) { /* skip '0x' and read numeral */ if (*s == dot) { if (hasdot) break; /* second dot? stop loop */ @@ -184,14 +184,14 @@ static lua_Number lua_strx2number (const char *s, char **endptr) { if (sigdig == 0 && *s == '0') /* non-significant digit (zero)? */ nosigdig++; else if (++sigdig <= MAXSIGDIG) /* can read it without overflow? */ - r = (r * cast_num(16.0)) + luaO_hexavalue(*s); + r = (r * l_mathop(16.0)) + luaO_hexavalue(*s); else e++; /* too many digits; ignore, but still count for exponent */ if (hasdot) e--; /* decimal digit? correct exponent */ } else break; /* neither a dot nor a digit */ } if (nosigdig + sigdig == 0) /* no digits? */ - return 0.0; /* invalid format */ + return l_mathop(0.0); /* invalid format */ *endptr = cast_charp(s); /* valid up to here */ e *= 4; /* each digit multiplies/divides value by 2^4 */ if (*s == 'p' || *s == 'P') { /* exponent part? */ @@ -200,7 +200,7 @@ static lua_Number lua_strx2number (const char *s, char **endptr) { s++; /* skip 'p' */ neg1 = isneg(&s); /* sign */ if (!lisdigit(cast_uchar(*s))) - return 0.0; /* invalid; must have at least one digit */ + return l_mathop(0.0); /* invalid; must have at least one digit */ while (lisdigit(cast_uchar(*s))) /* read exponent */ exp1 = exp1 * 10 + *(s++) - '0'; if (neg1) exp1 = -exp1; @@ -258,7 +258,7 @@ static const char *l_str2d (const char *s, lua_Number *result) { if (endptr == NULL) { /* failed? may be a different locale */ char buff[L_MAXLENNUM + 1]; const char *pdot = strchr(s, '.'); - if (strlen(s) > L_MAXLENNUM || pdot == NULL) + if (pdot == NULL || strlen(s) > L_MAXLENNUM) return NULL; /* string too long or no dot; fail */ strcpy(buff, s); /* copy string to buffer */ buff[pdot - s] = lua_getlocaledecpoint(); /* correct decimal point */ diff --git a/src/lua/lobject.h b/src/lua/lobject.h index a9d45785..0e05b3e4 100644 --- a/src/lua/lobject.h +++ b/src/lua/lobject.h @@ -21,10 +21,12 @@ */ #define LUA_TUPVAL LUA_NUMTYPES /* upvalues */ #define LUA_TPROTO (LUA_NUMTYPES+1) /* function prototypes */ +#define LUA_TDEADKEY (LUA_NUMTYPES+2) /* removed keys in tables */ + /* -** number of all possible types (including LUA_TNONE) +** number of all possible types (including LUA_TNONE but excluding DEADKEY) */ #define LUA_TOTALTYPES (LUA_TPROTO + 2) @@ -66,7 +68,7 @@ typedef struct TValue { #define val_(o) ((o)->value_) -#define valraw(o) (&val_(o)) +#define valraw(o) (val_(o)) /* raw type tag of a TValue */ @@ -110,7 +112,7 @@ typedef struct TValue { #define settt_(o,t) ((o)->tt_=(t)) -/* main macro to copy values (from 'obj1' to 'obj2') */ +/* main macro to copy values (from 'obj2' to 'obj1') */ #define setobj(L,obj1,obj2) \ { TValue *io1=(obj1); const TValue *io2=(obj2); \ io1->value_ = io2->value_; settt_(io1, io2->tt_); \ @@ -134,10 +136,19 @@ typedef struct TValue { /* -** Entries in the Lua stack +** Entries in a Lua stack. Field 'tbclist' forms a list of all +** to-be-closed variables active in this stack. Dummy entries are +** used when the distance between two tbc variables does not fit +** in an unsigned short. They are represented by delta==0, and +** their real delta is always the maximum value that fits in +** that field. */ typedef union StackValue { TValue val; + struct { + TValuefields; + unsigned short delta; + } tbclist; } StackValue; @@ -555,7 +566,7 @@ typedef struct Proto { /* ** {================================================================== -** Closures +** Functions ** =================================================================== */ @@ -568,10 +579,11 @@ typedef struct Proto { #define LUA_VCCL makevariant(LUA_TFUNCTION, 2) /* C closure */ #define ttisfunction(o) checktype(o, LUA_TFUNCTION) -#define ttisclosure(o) ((rawtt(o) & 0x1F) == LUA_VLCL) #define ttisLclosure(o) checktag((o), ctb(LUA_VLCL)) #define ttislcf(o) checktag((o), LUA_VLCF) #define ttisCclosure(o) checktag((o), ctb(LUA_VCCL)) +#define ttisclosure(o) (ttisLclosure(o) || ttisCclosure(o)) + #define isLfunction(o) ttisLclosure(o) @@ -743,13 +755,13 @@ typedef struct Table { /* -** Use a "nil table" to mark dead keys in a table. Those keys serve -** to keep space for removed entries, which may still be part of -** chains. Note that the 'keytt' does not have the BIT_ISCOLLECTABLE -** set, so these values are considered not collectable and are different -** from any valid value. +** Dead keys in tables have the tag DEADKEY but keep their original +** gcvalue. This distinguishes them from regular keys but allows them to +** be found when searched in a special way. ('next' needs that to find +** keys removed from a table during a traversal.) */ -#define setdeadkey(n) (keytt(n) = LUA_TTABLE, gckey(n) = NULL) +#define setdeadkey(node) (keytt(node) = LUA_TDEADKEY) +#define keyisdead(node) (keytt(node) == LUA_TDEADKEY) /* }================================================================== */ diff --git a/src/lua/lopcodes.h b/src/lua/lopcodes.h index 122e5d21..7c274515 100644 --- a/src/lua/lopcodes.h +++ b/src/lua/lopcodes.h @@ -190,7 +190,8 @@ enum OpMode {iABC, iABx, iAsBx, iAx, isJ}; /* basic instruction formats */ /* -** grep "ORDER OP" if you change these enums +** Grep "ORDER OP" if you change these enums. Opcodes marked with a (*) +** has extra descriptions in the notes after the enumeration. */ typedef enum { @@ -203,7 +204,7 @@ OP_LOADF,/* A sBx R[A] := (lua_Number)sBx */ OP_LOADK,/* A Bx R[A] := K[Bx] */ OP_LOADKX,/* A R[A] := K[extra arg] */ OP_LOADFALSE,/* A R[A] := false */ -OP_LFALSESKIP,/*A R[A] := false; pc++ */ +OP_LFALSESKIP,/*A R[A] := false; pc++ (*) */ OP_LOADTRUE,/* A R[A] := true */ OP_LOADNIL,/* A B R[A], R[A+1], ..., R[A+B] := nil */ OP_GETUPVAL,/* A B R[A] := UpValue[B] */ @@ -225,13 +226,13 @@ OP_SELF,/* A B C R[A+1] := R[B]; R[A] := R[B][RK(C):string] */ OP_ADDI,/* A B sC R[A] := R[B] + sC */ -OP_ADDK,/* A B C R[A] := R[B] + K[C] */ -OP_SUBK,/* A B C R[A] := R[B] - K[C] */ -OP_MULK,/* A B C R[A] := R[B] * K[C] */ -OP_MODK,/* A B C R[A] := R[B] % K[C] */ -OP_POWK,/* A B C R[A] := R[B] ^ K[C] */ -OP_DIVK,/* A B C R[A] := R[B] / K[C] */ -OP_IDIVK,/* A B C R[A] := R[B] // K[C] */ +OP_ADDK,/* A B C R[A] := R[B] + K[C]:number */ +OP_SUBK,/* A B C R[A] := R[B] - K[C]:number */ +OP_MULK,/* A B C R[A] := R[B] * K[C]:number */ +OP_MODK,/* A B C R[A] := R[B] % K[C]:number */ +OP_POWK,/* A B C R[A] := R[B] ^ K[C]:number */ +OP_DIVK,/* A B C R[A] := R[B] / K[C]:number */ +OP_IDIVK,/* A B C R[A] := R[B] // K[C]:number */ OP_BANDK,/* A B C R[A] := R[B] & K[C]:integer */ OP_BORK,/* A B C R[A] := R[B] | K[C]:integer */ @@ -254,14 +255,14 @@ OP_BXOR,/* A B C R[A] := R[B] ~ R[C] */ OP_SHL,/* A B C R[A] := R[B] << R[C] */ OP_SHR,/* A B C R[A] := R[B] >> R[C] */ -OP_MMBIN,/* A B C call C metamethod over R[A] and R[B] */ +OP_MMBIN,/* A B C call C metamethod over R[A] and R[B] (*) */ OP_MMBINI,/* A sB C k call C metamethod over R[A] and sB */ OP_MMBINK,/* A B C k call C metamethod over R[A] and K[B] */ OP_UNM,/* A B R[A] := -R[B] */ OP_BNOT,/* A B R[A] := ~R[B] */ OP_NOT,/* A B R[A] := not R[B] */ -OP_LEN,/* A B R[A] := length of R[B] */ +OP_LEN,/* A B R[A] := #R[B] (length operator) */ OP_CONCAT,/* A B R[A] := R[A].. ... ..R[A + B - 1] */ @@ -280,7 +281,7 @@ OP_GTI,/* A sB k if ((R[A] > sB) ~= k) then pc++ */ OP_GEI,/* A sB k if ((R[A] >= sB) ~= k) then pc++ */ OP_TEST,/* A k if (not R[A] == k) then pc++ */ -OP_TESTSET,/* A B k if (not R[B] == k) then pc++ else R[A] := R[B] */ +OP_TESTSET,/* A B k if (not R[B] == k) then pc++ else R[A] := R[B] (*) */ OP_CALL,/* A B C R[A], ... ,R[A+C-2] := R[A](R[A+1], ... ,R[A+B-1]) */ OP_TAILCALL,/* A B C k return R[A](R[A+1], ... ,R[A+B-1]) */ @@ -297,7 +298,7 @@ OP_TFORPREP,/* A Bx create upvalue for R[A + 3]; pc+=Bx */ OP_TFORCALL,/* A C R[A+4], ... ,R[A+3+C] := R[A](R[A+1], R[A+2]); */ OP_TFORLOOP,/* A Bx if R[A+2] ~= nil then { R[A]=R[A+2]; pc -= Bx } */ -OP_SETLIST,/* A B C k R[A][(C-1)*FPF+i] := R[A+i], 1 <= i <= B */ +OP_SETLIST,/* A B C k R[A][C+i] := R[A+i], 1 <= i <= B */ OP_CLOSURE,/* A Bx R[A] := closure(KPROTO[Bx]) */ @@ -315,6 +316,18 @@ OP_EXTRAARG/* Ax extra (larger) argument for previous opcode */ /*=========================================================================== Notes: + + (*) Opcode OP_LFALSESKIP is used to convert a condition to a boolean + value, in a code equivalent to (not cond ? false : true). (It + produces false and skips the next instruction producing true.) + + (*) Opcodes OP_MMBIN and variants follow each arithmetic and + bitwise opcode. If the operation succeeds, it skips this next + opcode. Otherwise, this opcode calls the corresponding metamethod. + + (*) Opcode OP_TESTSET is used in short-circuit expressions that need + both to jump and to produce a value, such as (a = b or c). + (*) In OP_CALL, if (B == 0) then B = top - A. If (C == 0), then 'top' is set to last_result+1, so next open instruction (OP_CALL, OP_RETURN*, OP_SETLIST) may use 'top'. diff --git a/src/lua/loslib.c b/src/lua/loslib.c index e65e188b..3e20d622 100644 --- a/src/lua/loslib.c +++ b/src/lua/loslib.c @@ -170,7 +170,7 @@ static int os_tmpname (lua_State *L) { char buff[LUA_TMPNAMBUFSIZE]; int err; lua_tmpnam(buff, err); - if (err) + if (l_unlikely(err)) return luaL_error(L, "unable to generate a unique filename"); lua_pushstring(L, buff); return 1; @@ -208,7 +208,7 @@ static int os_clock (lua_State *L) { */ static void setfield (lua_State *L, const char *key, int value, int delta) { #if (defined(LUA_NUMTIME) && LUA_MAXINTEGER <= INT_MAX) - if (value > LUA_MAXINTEGER - delta) + if (l_unlikely(value > LUA_MAXINTEGER - delta)) luaL_error(L, "field '%s' is out-of-bound", key); #endif lua_pushinteger(L, (lua_Integer)value + delta); @@ -253,9 +253,9 @@ static int getfield (lua_State *L, const char *key, int d, int delta) { int t = lua_getfield(L, -1, key); /* get field and its type */ lua_Integer res = lua_tointegerx(L, -1, &isnum); if (!isnum) { /* field is not an integer? */ - if (t != LUA_TNIL) /* some other value? */ + if (l_unlikely(t != LUA_TNIL)) /* some other value? */ return luaL_error(L, "field '%s' is not an integer", key); - else if (d < 0) /* absent field; no default? */ + else if (l_unlikely(d < 0)) /* absent field; no default? */ return luaL_error(L, "field '%s' missing in date table", key); res = d; } diff --git a/src/lua/lparser.c b/src/lua/lparser.c index bc7d9a4f..3abe3d75 100644 --- a/src/lua/lparser.c +++ b/src/lua/lparser.c @@ -128,7 +128,7 @@ static void checknext (LexState *ls, int c) { ** in line 'where' (if that is not the current line). */ static void check_match (LexState *ls, int what, int who, int where) { - if (unlikely(!testnext(ls, what))) { + if (l_unlikely(!testnext(ls, what))) { if (where == ls->linenumber) /* all in the same line? */ error_expected(ls, what); /* do not need a complex message */ else { @@ -222,26 +222,26 @@ static Vardesc *getlocalvardesc (FuncState *fs, int vidx) { /* -** Convert 'nvar', a compiler index level, to it corresponding -** stack index level. For that, search for the highest variable -** below that level that is in the stack and uses its stack -** index ('sidx'). +** Convert 'nvar', a compiler index level, to its corresponding +** register. For that, search for the highest variable below that level +** that is in a register and uses its register index ('ridx') plus one. */ -static int stacklevel (FuncState *fs, int nvar) { +static int reglevel (FuncState *fs, int nvar) { while (nvar-- > 0) { - Vardesc *vd = getlocalvardesc(fs, nvar); /* get variable */ - if (vd->vd.kind != RDKCTC) /* is in the stack? */ - return vd->vd.sidx + 1; + Vardesc *vd = getlocalvardesc(fs, nvar); /* get previous variable */ + if (vd->vd.kind != RDKCTC) /* is in a register? */ + return vd->vd.ridx + 1; } - return 0; /* no variables in the stack */ + return 0; /* no variables in registers */ } /* -** Return the number of variables in the stack for function 'fs' +** Return the number of variables in the register stack for the given +** function. */ int luaY_nvarstack (FuncState *fs) { - return stacklevel(fs, fs->nactvar); + return reglevel(fs, fs->nactvar); } @@ -267,7 +267,7 @@ static void init_var (FuncState *fs, expdesc *e, int vidx) { e->f = e->t = NO_JUMP; e->k = VLOCAL; e->u.var.vidx = vidx; - e->u.var.sidx = getlocalvardesc(fs, vidx)->vd.sidx; + e->u.var.ridx = getlocalvardesc(fs, vidx)->vd.ridx; } @@ -310,12 +310,12 @@ static void check_readonly (LexState *ls, expdesc *e) { */ static void adjustlocalvars (LexState *ls, int nvars) { FuncState *fs = ls->fs; - int stklevel = luaY_nvarstack(fs); + int reglevel = luaY_nvarstack(fs); int i; for (i = 0; i < nvars; i++) { int vidx = fs->nactvar++; Vardesc *var = getlocalvardesc(fs, vidx); - var->vd.sidx = stklevel++; + var->vd.ridx = reglevel++; var->vd.pidx = registerlocalvar(ls, fs, var->vd.name); } } @@ -366,7 +366,7 @@ static int newupvalue (FuncState *fs, TString *name, expdesc *v) { FuncState *prev = fs->prev; if (v->k == VLOCAL) { up->instack = 1; - up->idx = v->u.var.sidx; + up->idx = v->u.var.ridx; up->kind = getlocalvardesc(prev, v->u.var.vidx)->vd.kind; lua_assert(eqstr(name, getlocalvardesc(prev, v->u.var.vidx)->vd.name)); } @@ -416,6 +416,17 @@ static void markupval (FuncState *fs, int level) { } +/* +** Mark that current block has a to-be-closed variable. +*/ +static void marktobeclosed (FuncState *fs) { + BlockCnt *bl = fs->bl; + bl->upval = 1; + bl->insidetbc = 1; + fs->needclose = 1; +} + + /* ** Find a variable with the given name 'n'. If it is an upvalue, add ** this upvalue into all intermediate functions. If it is a global, set @@ -489,12 +500,10 @@ static void adjust_assign (LexState *ls, int nvars, int nexps, expdesc *e) { } -/* -** Macros to limit the maximum recursion depth while parsing -*/ -#define enterlevel(ls) luaE_enterCcall((ls)->L) +#define enterlevel(ls) luaE_incCstack(ls->L) -#define leavelevel(ls) luaE_exitCcall((ls)->L) + +#define leavelevel(ls) ((ls)->L->nCcalls--) /* @@ -519,7 +528,7 @@ static void solvegoto (LexState *ls, int g, Labeldesc *label) { Labellist *gl = &ls->dyd->gt; /* list of goto's */ Labeldesc *gt = &gl->arr[g]; /* goto to be resolved */ lua_assert(eqstr(gt->name, label->name)); - if (unlikely(gt->nactvar < label->nactvar)) /* enter some scope? */ + if (l_unlikely(gt->nactvar < label->nactvar)) /* enter some scope? */ jumpscopeerror(ls, gt); luaK_patchlist(ls->fs, gt->pc, label->pc); for (i = g; i < gl->n - 1; i++) /* remove goto from pending list */ @@ -622,7 +631,7 @@ static void movegotosout (FuncState *fs, BlockCnt *bl) { for (i = bl->firstgoto; i < gl->n; i++) { /* for each pending goto */ Labeldesc *gt = &gl->arr[i]; /* leaving a variable scope? */ - if (stacklevel(fs, gt->nactvar) > stacklevel(fs, bl->nactvar)) + if (reglevel(fs, gt->nactvar) > reglevel(fs, bl->nactvar)) gt->close |= bl->upval; /* jump may need a close */ gt->nactvar = bl->nactvar; /* update goto level */ } @@ -663,7 +672,7 @@ static void leaveblock (FuncState *fs) { BlockCnt *bl = fs->bl; LexState *ls = fs->ls; int hasclose = 0; - int stklevel = stacklevel(fs, bl->nactvar); /* level outside the block */ + int stklevel = reglevel(fs, bl->nactvar); /* level outside the block */ if (bl->isloop) /* fix pending breaks? */ hasclose = createlabel(ls, luaS_newliteral(ls->L, "break"), 0, 0); if (!hasclose && bl->previous && bl->upval) @@ -947,7 +956,7 @@ static void setvararg (FuncState *fs, int nparams) { static void parlist (LexState *ls) { - /* parlist -> [ param { ',' param } ] */ + /* parlist -> [ {NAME ','} (NAME | '...') ] */ FuncState *fs = ls->fs; Proto *f = fs->f; int nparams = 0; @@ -955,12 +964,12 @@ static void parlist (LexState *ls) { if (ls->t.token != ')') { /* is 'parlist' not empty? */ do { switch (ls->t.token) { - case TK_NAME: { /* param -> NAME */ + case TK_NAME: { new_localvar(ls, str_checkname(ls)); nparams++; break; } - case TK_DOTS: { /* param -> '...' */ + case TK_DOTS: { luaX_next(ls); isvararg = 1; break; @@ -1332,13 +1341,13 @@ static void check_conflict (LexState *ls, struct LHS_assign *lh, expdesc *v) { } } else { /* table is a register */ - if (v->k == VLOCAL && lh->v.u.ind.t == v->u.var.sidx) { + if (v->k == VLOCAL && lh->v.u.ind.t == v->u.var.ridx) { conflict = 1; /* table is the local being assigned now */ lh->v.u.ind.t = extra; /* assignment will use safe copy */ } /* is index the local being assigned? */ if (lh->v.k == VINDEXED && v->k == VLOCAL && - lh->v.u.ind.idx == v->u.var.sidx) { + lh->v.u.ind.idx == v->u.var.ridx) { conflict = 1; lh->v.u.ind.idx = extra; /* previous assignment will use safe copy */ } @@ -1348,7 +1357,7 @@ static void check_conflict (LexState *ls, struct LHS_assign *lh, expdesc *v) { if (conflict) { /* copy upvalue/local value to a temporary (in position 'extra') */ if (v->k == VLOCAL) - luaK_codeABC(fs, OP_MOVE, extra, v->u.var.sidx, 0); + luaK_codeABC(fs, OP_MOVE, extra, v->u.var.ridx, 0); else luaK_codeABC(fs, OP_GETUPVAL, extra, v->u.info, 0); luaK_reserveregs(fs, 1); @@ -1413,7 +1422,7 @@ static void gotostat (LexState *ls) { newgotoentry(ls, name, line, luaK_jump(fs)); else { /* found a label */ /* backward jump; will be resolved here */ - int lblevel = stacklevel(fs, lb->nactvar); /* label level */ + int lblevel = reglevel(fs, lb->nactvar); /* label level */ if (luaY_nvarstack(fs) > lblevel) /* leaving the scope of a variable? */ luaK_codeABC(fs, OP_CLOSE, lblevel, 0, 0); /* create jump and link it to the label */ @@ -1437,7 +1446,7 @@ static void breakstat (LexState *ls) { */ static void checkrepeated (LexState *ls, TString *name) { Labeldesc *lb = findlabel(ls, name); - if (unlikely(lb != NULL)) { /* already defined? */ + if (l_unlikely(lb != NULL)) { /* already defined? */ const char *msg = "label '%s' already defined on line %d"; msg = luaO_pushfstring(ls->L, msg, getstr(name), lb->line); luaK_semerror(ls, msg); /* error */ @@ -1490,7 +1499,7 @@ static void repeatstat (LexState *ls, int line) { if (bl2.upval) { /* upvalues? */ int exit = luaK_jump(fs); /* normal exit must jump over fix */ luaK_patchtohere(fs, condexit); /* repetition must close upvalues */ - luaK_codeABC(fs, OP_CLOSE, stacklevel(fs, bl2.nactvar), 0, 0); + luaK_codeABC(fs, OP_CLOSE, reglevel(fs, bl2.nactvar), 0, 0); condexit = luaK_jump(fs); /* repeat after closing upvalues */ luaK_patchtohere(fs, exit); /* normal exit comes to here */ } @@ -1522,7 +1531,7 @@ static void fixforjump (FuncState *fs, int pc, int dest, int back) { int offset = dest - (pc + 1); if (back) offset = -offset; - if (unlikely(offset > MAXARG_Bx)) + if (l_unlikely(offset > MAXARG_Bx)) luaX_syntaxerror(fs->ls, "control structure too long"); SETARG_Bx(*jmp, offset); } @@ -1601,7 +1610,7 @@ static void forlist (LexState *ls, TString *indexname) { line = ls->linenumber; adjust_assign(ls, 4, explist(ls, &e), &e); adjustlocalvars(ls, 4); /* control variables */ - markupval(fs, fs->nactvar); /* last control var. must be closed */ + marktobeclosed(fs); /* last control var. must be closed */ luaK_checkstack(fs, 3); /* extra space to call generator */ forbody(ls, base, line, nvars - 4, 1); } @@ -1625,59 +1634,21 @@ static void forstat (LexState *ls, int line) { } -/* -** Check whether next instruction is a single jump (a 'break', a 'goto' -** to a forward label, or a 'goto' to a backward label with no variable -** to close). If so, set the name of the 'label' it is jumping to -** ("break" for a 'break') or to where it is jumping to ('target') and -** return true. If not a single jump, leave input unchanged, to be -** handled as a regular statement. -*/ -static int issinglejump (LexState *ls, TString **label, int *target) { - if (testnext(ls, TK_BREAK)) { /* a break? */ - *label = luaS_newliteral(ls->L, "break"); - return 1; - } - else if (ls->t.token != TK_GOTO || luaX_lookahead(ls) != TK_NAME) - return 0; /* not a valid goto */ - else { - TString *lname = ls->lookahead.seminfo.ts; /* label's id */ - Labeldesc *lb = findlabel(ls, lname); - if (lb) { /* a backward jump? */ - /* does it need to close variables? */ - if (luaY_nvarstack(ls->fs) > stacklevel(ls->fs, lb->nactvar)) - return 0; /* not a single jump; cannot optimize */ - *target = lb->pc; - } - else /* jump forward */ - *label = lname; - luaX_next(ls); /* skip goto */ - luaX_next(ls); /* skip name */ - return 1; - } -} - - static void test_then_block (LexState *ls, int *escapelist) { /* test_then_block -> [IF | ELSEIF] cond THEN block */ BlockCnt bl; - int line; FuncState *fs = ls->fs; - TString *jlb = NULL; - int target = NO_JUMP; expdesc v; int jf; /* instruction to skip 'then' code (if condition is false) */ luaX_next(ls); /* skip IF or ELSEIF */ expr(ls, &v); /* read condition */ checknext(ls, TK_THEN); - line = ls->linenumber; - if (issinglejump(ls, &jlb, &target)) { /* 'if x then goto' ? */ - luaK_goiffalse(ls->fs, &v); /* will jump to label if condition is true */ + if (ls->t.token == TK_BREAK) { /* 'if x then break' ? */ + int line = ls->linenumber; + luaK_goiffalse(ls->fs, &v); /* will jump if condition is true */ + luaX_next(ls); /* skip 'break' */ enterblock(fs, &bl, 0); /* must enter block before 'goto' */ - if (jlb != NULL) /* forward jump? */ - newgotoentry(ls, jlb, line, v.t); /* will be resolved later */ - else /* backward jump */ - luaK_patchlist(fs, v.t, target); /* jump directly to 'target' */ + newgotoentry(ls, luaS_newliteral(ls->L, "break"), line, v.t); while (testnext(ls, ';')) {} /* skip semicolons */ if (block_follow(ls, 0)) { /* jump is the entire block? */ leaveblock(fs); @@ -1686,7 +1657,7 @@ static void test_then_block (LexState *ls, int *escapelist) { else /* must skip over 'then' part if condition is false */ jf = luaK_jump(fs); } - else { /* regular case (not a jump) */ + else { /* regular case (not a break) */ luaK_goiftrue(ls->fs, &v); /* skip over block if condition is false */ enterblock(fs, &bl, 0); jf = v.f; @@ -1743,18 +1714,16 @@ static int getlocalattribute (LexState *ls) { } -static void checktoclose (LexState *ls, int level) { +static void checktoclose (FuncState *fs, int level) { if (level != -1) { /* is there a to-be-closed variable? */ - FuncState *fs = ls->fs; - markupval(fs, level + 1); - fs->bl->insidetbc = 1; /* in the scope of a to-be-closed variable */ - luaK_codeABC(fs, OP_TBC, stacklevel(fs, level), 0, 0); + marktobeclosed(fs); + luaK_codeABC(fs, OP_TBC, reglevel(fs, level), 0, 0); } } static void localstat (LexState *ls) { - /* stat -> LOCAL ATTRIB NAME {',' ATTRIB NAME} ['=' explist] */ + /* stat -> LOCAL NAME ATTRIB { ',' NAME ATTRIB } ['=' explist] */ FuncState *fs = ls->fs; int toclose = -1; /* index of to-be-closed variable (if any) */ Vardesc *var; /* last variable */ @@ -1791,7 +1760,7 @@ static void localstat (LexState *ls) { adjust_assign(ls, nvars, nexps, &e); adjustlocalvars(ls, nvars); } - checktoclose(ls, toclose); + checktoclose(fs, toclose); } @@ -1816,6 +1785,7 @@ static void funcstat (LexState *ls, int line) { luaX_next(ls); /* skip FUNCTION */ ismethod = funcname(ls, &v); body(ls, &b, ismethod, line); + check_readonly(ls, &v); luaK_storevar(ls->fs, &v, &b); luaK_fixline(ls->fs, line); /* definition "happens" in the first line */ } diff --git a/src/lua/lparser.h b/src/lua/lparser.h index 618cb010..5e4500f1 100644 --- a/src/lua/lparser.h +++ b/src/lua/lparser.h @@ -23,7 +23,7 @@ /* kinds of variables/expressions */ typedef enum { - VVOID, /* when 'expdesc' describes the last expression a list, + VVOID, /* when 'expdesc' describes the last expression of a list, this kind means an empty list (so, no expression) */ VNIL, /* constant nil */ VTRUE, /* constant true */ @@ -35,10 +35,11 @@ typedef enum { (string is fixed by the lexer) */ VNONRELOC, /* expression has its value in a fixed register; info = result register */ - VLOCAL, /* local variable; var.sidx = stack index (local register); + VLOCAL, /* local variable; var.ridx = register index; var.vidx = relative index in 'actvar.arr' */ VUPVAL, /* upvalue variable; info = index of upvalue in 'upvalues' */ - VCONST, /* compile-time constant; info = absolute index in 'actvar.arr' */ + VCONST, /* compile-time variable; + info = absolute index in 'actvar.arr' */ VINDEXED, /* indexed variable; ind.t = table register; ind.idx = key's R index */ @@ -76,7 +77,7 @@ typedef struct expdesc { lu_byte t; /* table (register or upvalue) */ } ind; struct { /* for local variables */ - lu_byte sidx; /* index in the stack */ + lu_byte ridx; /* register holding the variable */ unsigned short vidx; /* compiler index (in 'actvar.arr') */ } var; } u; @@ -96,7 +97,7 @@ typedef union Vardesc { struct { TValuefields; /* constant value (if it is a compile-time constant) */ lu_byte kind; - lu_byte sidx; /* index of the variable in the stack */ + lu_byte ridx; /* register holding the variable */ short pidx; /* index of the variable in the Proto's 'locvars' array */ TString *name; /* variable name */ } vd; diff --git a/src/lua/lstate.c b/src/lua/lstate.c index 86b3761f..1ffe1a0f 100644 --- a/src/lua/lstate.c +++ b/src/lua/lstate.c @@ -76,7 +76,7 @@ static unsigned int luai_makeseed (lua_State *L) { addbuff(buff, p, &h); /* local variable */ addbuff(buff, p, &lua_newstate); /* public function */ lua_assert(p == sizeof(buff)); - return luaS_hash(buff, p, h, 1); + return luaS_hash(buff, p, h); } #endif @@ -97,66 +97,14 @@ void luaE_setdebt (global_State *g, l_mem debt) { LUA_API int lua_setcstacklimit (lua_State *L, unsigned int limit) { - global_State *g = G(L); - int ccalls; - luaE_freeCI(L); /* release unused CIs */ - ccalls = getCcalls(L); - if (limit >= 40000) - return 0; /* out of bounds */ - limit += CSTACKERR; - if (L != g-> mainthread) - return 0; /* only main thread can change the C stack */ - else if (ccalls <= CSTACKERR) - return 0; /* handling overflow */ - else { - int diff = limit - g->Cstacklimit; - if (ccalls + diff <= CSTACKERR) - return 0; /* new limit would cause an overflow */ - g->Cstacklimit = limit; /* set new limit */ - L->nCcalls += diff; /* correct 'nCcalls' */ - return limit - diff - CSTACKERR; /* success; return previous limit */ - } -} - - -/* -** Decrement count of "C calls" and check for overflows. In case of -** a stack overflow, check appropriate error ("regular" overflow or -** overflow while handling stack overflow). If 'nCcalls' is smaller -** than CSTACKERR but larger than CSTACKMARK, it means it has just -** entered the "overflow zone", so the function raises an overflow -** error. If 'nCcalls' is smaller than CSTACKMARK (which means it is -** already handling an overflow) but larger than CSTACKERRMARK, does -** not report an error (to allow message handling to work). Otherwise, -** report a stack overflow while handling a stack overflow (probably -** caused by a repeating error in the message handling function). -*/ - -void luaE_enterCcall (lua_State *L) { - int ncalls = getCcalls(L); - L->nCcalls--; - if (ncalls <= CSTACKERR) { /* possible overflow? */ - luaE_freeCI(L); /* release unused CIs */ - ncalls = getCcalls(L); /* update call count */ - if (ncalls <= CSTACKERR) { /* still overflow? */ - if (ncalls <= CSTACKERRMARK) /* below error-handling zone? */ - luaD_throw(L, LUA_ERRERR); /* error while handling stack error */ - else if (ncalls >= CSTACKMARK) { - /* not in error-handling zone; raise the error now */ - L->nCcalls = (CSTACKMARK - 1); /* enter error-handling zone */ - luaG_runerror(L, "C stack overflow"); - } - /* else stack is in the error-handling zone; - allow message handler to work */ - } - } + UNUSED(L); UNUSED(limit); + return LUAI_MAXCCALLS; /* warning?? */ } CallInfo *luaE_extendCI (lua_State *L) { CallInfo *ci; lua_assert(L->ci->next == NULL); - luaE_enterCcall(L); ci = luaM_new(L, CallInfo); lua_assert(L->ci->next == NULL); L->ci->next = ci; @@ -175,13 +123,11 @@ void luaE_freeCI (lua_State *L) { CallInfo *ci = L->ci; CallInfo *next = ci->next; ci->next = NULL; - L->nCcalls += L->nci; /* add removed elements back to 'nCcalls' */ while ((ci = next) != NULL) { next = ci->next; luaM_free(L, ci); L->nci--; } - L->nCcalls -= L->nci; /* adjust result */ } @@ -194,7 +140,6 @@ void luaE_shrinkCI (lua_State *L) { CallInfo *next; if (ci == NULL) return; /* no extra elements */ - L->nCcalls += L->nci; /* add removed elements back to 'nCcalls' */ while ((next = ci->next) != NULL) { /* two extra elements? */ CallInfo *next2 = next->next; /* next's next */ ci->next = next2; /* remove next from the list */ @@ -207,19 +152,40 @@ void luaE_shrinkCI (lua_State *L) { ci = next2; /* continue */ } } - L->nCcalls -= L->nci; /* adjust result */ +} + + +/* +** Called when 'getCcalls(L)' larger or equal to LUAI_MAXCCALLS. +** If equal, raises an overflow error. If value is larger than +** LUAI_MAXCCALLS (which means it is handling an overflow) but +** not much larger, does not report an error (to allow overflow +** handling to work). +*/ +void luaE_checkcstack (lua_State *L) { + if (getCcalls(L) == LUAI_MAXCCALLS) + luaG_runerror(L, "C stack overflow"); + else if (getCcalls(L) >= (LUAI_MAXCCALLS / 10 * 11)) + luaD_throw(L, LUA_ERRERR); /* error while handling stack error */ +} + + +LUAI_FUNC void luaE_incCstack (lua_State *L) { + L->nCcalls++; + if (l_unlikely(getCcalls(L) >= LUAI_MAXCCALLS)) + luaE_checkcstack(L); } static void stack_init (lua_State *L1, lua_State *L) { int i; CallInfo *ci; /* initialize stack array */ - L1->stack = luaM_newvector(L, BASIC_STACK_SIZE, StackValue); - L1->stacksize = BASIC_STACK_SIZE; - for (i = 0; i < BASIC_STACK_SIZE; i++) + L1->stack = luaM_newvector(L, BASIC_STACK_SIZE + EXTRA_STACK, StackValue); + L1->tbclist = L1->stack; + for (i = 0; i < BASIC_STACK_SIZE + EXTRA_STACK; i++) setnilvalue(s2v(L1->stack + i)); /* erase new stack */ L1->top = L1->stack; - L1->stack_last = L1->stack + L1->stacksize - EXTRA_STACK; + L1->stack_last = L1->stack + BASIC_STACK_SIZE; /* initialize first ci */ ci = &L1->base_ci; ci->next = ci->previous = NULL; @@ -240,7 +206,7 @@ static void freestack (lua_State *L) { L->ci = &L->base_ci; /* free the entire 'ci' list */ luaE_freeCI(L); lua_assert(L->nci == 0); - luaM_freearray(L, L->stack, L->stacksize); /* free stack array */ + luaM_freearray(L, L->stack, stacksize(L) + EXTRA_STACK); /* free stack */ } @@ -248,24 +214,19 @@ static void freestack (lua_State *L) { ** Create registry table and its predefined values */ static void init_registry (lua_State *L, global_State *g) { - TValue temp; /* create registry */ Table *registry = luaH_new(L); sethvalue(L, &g->l_registry, registry); luaH_resize(L, registry, LUA_RIDX_LAST, 0); /* registry[LUA_RIDX_MAINTHREAD] = L */ - setthvalue(L, &temp, L); /* temp = L */ - luaH_setint(L, registry, LUA_RIDX_MAINTHREAD, &temp); - /* registry[LUA_RIDX_GLOBALS] = table of globals */ - sethvalue(L, &temp, luaH_new(L)); /* temp = new table (global table) */ - luaH_setint(L, registry, LUA_RIDX_GLOBALS, &temp); + setthvalue(L, ®istry->array[LUA_RIDX_MAINTHREAD - 1], L); + /* registry[LUA_RIDX_GLOBALS] = new table (table of globals) */ + sethvalue(L, ®istry->array[LUA_RIDX_GLOBALS - 1], luaH_new(L)); } /* ** open parts of the state that may cause memory-allocation errors. -** ('g->nilvalue' being a nil value flags that the state was completely -** build.) */ static void f_luaopen (lua_State *L, void *ud) { global_State *g = G(L); @@ -275,8 +236,8 @@ static void f_luaopen (lua_State *L, void *ud) { luaS_init(L); luaT_init(L); luaX_init(L); - g->gcrunning = 1; /* allow gc */ - setnilvalue(&g->nilvalue); + g->gcstp = 0; /* allow gc */ + setnilvalue(&g->nilvalue); /* now state is complete */ luai_userstateopen(L); } @@ -290,8 +251,8 @@ static void preinit_thread (lua_State *L, global_State *g) { L->stack = NULL; L->ci = NULL; L->nci = 0; - L->stacksize = 0; L->twups = L; /* thread has no upvalues */ + L->nCcalls = 0; L->errorJmp = NULL; L->hook = NULL; L->hookmask = 0; @@ -307,10 +268,14 @@ static void preinit_thread (lua_State *L, global_State *g) { static void close_state (lua_State *L) { global_State *g = G(L); - luaF_close(L, L->stack, CLOSEPROTECT); /* close all upvalues */ - luaC_freeallobjects(L); /* collect all objects */ - if (ttisnil(&g->nilvalue)) /* closing a fully built state? */ + if (!completestate(g)) /* closing a partially built state? */ + luaC_freeallobjects(L); /* just collect its objects */ + else { /* closing a fully built state */ + L->ci = &L->base_ci; /* unwind CallInfo list */ + luaD_closeprotected(L, 1, LUA_OK); /* close all upvalues */ + luaC_freeallobjects(L); /* collect all objects */ luai_userstateclose(L); + } luaM_freearray(L, G(L)->strt.hash, G(L)->strt.size); freestack(L); lua_assert(gettotalbytes(g) == sizeof(LG)); @@ -335,7 +300,6 @@ LUA_API lua_State *lua_newthread (lua_State *L) { setthvalue2s(L, L->top, L1); api_incr_top(L); preinit_thread(L1, g); - L1->nCcalls = getCcalls(L); L1->hookmask = L->hookmask; L1->basehookcount = L->basehookcount; L1->hook = L->hook; @@ -352,7 +316,7 @@ LUA_API lua_State *lua_newthread (lua_State *L) { void luaE_freethread (lua_State *L, lua_State *L1) { LX *l = fromstate(L1); - luaF_close(L1, L1->stack, NOCLOSINGMETH); /* close all upvalues */ + luaF_closeupval(L1, L1->stack); /* close all upvalues */ lua_assert(L1->openupval == NULL); luai_userstatefree(L, L1); freestack(L1); @@ -360,23 +324,29 @@ void luaE_freethread (lua_State *L, lua_State *L1) { } -int lua_resetthread (lua_State *L) { - CallInfo *ci; - int status; - lua_lock(L); - L->ci = ci = &L->base_ci; /* unwind CallInfo list */ +int luaE_resetthread (lua_State *L, int status) { + CallInfo *ci = L->ci = &L->base_ci; /* unwind CallInfo list */ setnilvalue(s2v(L->stack)); /* 'function' entry for basic 'ci' */ ci->func = L->stack; ci->callstatus = CIST_C; - status = luaF_close(L, L->stack, CLOSEPROTECT); - if (status != CLOSEPROTECT) /* real errors? */ - luaD_seterrorobj(L, status, L->stack + 1); - else { + if (status == LUA_YIELD) status = LUA_OK; + L->status = LUA_OK; /* so it can run __close metamethods */ + status = luaD_closeprotected(L, 1, status); + if (status != LUA_OK) /* errors? */ + luaD_seterrorobj(L, status, L->stack + 1); + else L->top = L->stack + 1; - } ci->top = L->top + LUA_MINSTACK; - L->status = status; + luaD_reallocstack(L, cast_int(ci->top - L->stack), 0); + return status; +} + + +LUA_API int lua_resetthread (lua_State *L) { + int status; + lua_lock(L); + status = luaE_resetthread(L, L->status); lua_unlock(L); return status; } @@ -396,7 +366,6 @@ LUA_API lua_State *lua_newstate (lua_Alloc f, void *ud) { preinit_thread(L, g); g->allgc = obj2gco(L); /* by now, only object is the main thread */ L->next = NULL; - g->Cstacklimit = L->nCcalls = LUAI_MAXCSTACK + CSTACKERR; incnny(L); /* main thread is always non yieldable */ g->frealloc = f; g->ud = ud; @@ -404,13 +373,14 @@ LUA_API lua_State *lua_newstate (lua_Alloc f, void *ud) { g->ud_warn = NULL; g->mainthread = L; g->seed = luai_makeseed(L); - g->gcrunning = 0; /* no GC while building state */ + g->gcstp = GCSTPGC; /* no GC while building state */ g->strt.size = g->strt.nuse = 0; g->strt.hash = NULL; setnilvalue(&g->l_registry); g->panic = NULL; g->gcstate = GCSpause; g->gckind = KGC_INC; + g->gcstopem = 0; g->gcemergency = 0; g->finobj = g->tobefnz = g->fixedgc = NULL; g->firstold1 = g->survival = g->old1 = g->reallyold = NULL; diff --git a/src/lua/lstate.h b/src/lua/lstate.h index c1c38204..61e82cde 100644 --- a/src/lua/lstate.h +++ b/src/lua/lstate.h @@ -87,49 +87,13 @@ /* -** About 'nCcalls': each thread in Lua (a lua_State) keeps a count of -** how many "C calls" it still can do in the C stack, to avoid C-stack -** overflow. This count is very rough approximation; it considers only -** recursive functions inside the interpreter, as non-recursive calls -** can be considered using a fixed (although unknown) amount of stack -** space. -** -** The count has two parts: the lower part is the count itself; the -** higher part counts the number of non-yieldable calls in the stack. -** (They are together so that we can change both with one instruction.) -** -** Because calls to external C functions can use an unknown amount -** of space (e.g., functions using an auxiliary buffer), calls -** to these functions add more than one to the count (see CSTACKCF). -** -** The proper count excludes the number of CallInfo structures allocated -** by Lua, as a kind of "potential" calls. So, when Lua calls a function -** (and "consumes" one CallInfo), it needs neither to decrement nor to -** check 'nCcalls', as its use of C stack is already accounted for. +** About 'nCcalls': This count has two parts: the lower 16 bits counts +** the number of recursive invocations in the C stack; the higher +** 16 bits counts the number of non-yieldable calls in the stack. +** (They are together so that we can change and save both with one +** instruction.) */ -/* number of "C stack slots" used by an external C function */ -#define CSTACKCF 10 - - -/* -** The C-stack size is sliced in the following zones: -** - larger than CSTACKERR: normal stack; -** - [CSTACKMARK, CSTACKERR]: buffer zone to signal a stack overflow; -** - [CSTACKCF, CSTACKERRMARK]: error-handling zone; -** - below CSTACKERRMARK: buffer zone to signal overflow during overflow; -** (Because the counter can be decremented CSTACKCF at once, we need -** the so called "buffer zones", with at least that size, to properly -** detect a change from one zone to the next.) -*/ -#define CSTACKERR (8 * CSTACKCF) -#define CSTACKMARK (CSTACKERR - (CSTACKCF + 2)) -#define CSTACKERRMARK (CSTACKCF + 2) - - -/* initial limit for the C-stack of threads */ -#define CSTACKTHREAD (2 * CSTACKERR) - /* true if this thread does not have non-yieldable calls in the stack */ #define yieldable(L) (((L)->nCcalls & 0xffff0000) == 0) @@ -144,13 +108,8 @@ /* Decrement the number of non-yieldable calls */ #define decnny(L) ((L)->nCcalls -= 0x10000) -/* Increment the number of non-yieldable calls and decrement nCcalls */ -#define incXCcalls(L) ((L)->nCcalls += 0x10000 - CSTACKCF) - -/* Decrement the number of non-yieldable calls and increment nCcalls */ -#define decXCcalls(L) ((L)->nCcalls -= 0x10000 - CSTACKCF) - - +/* Non-yieldable call increment */ +#define nyci (0x10000 | 1) @@ -168,12 +127,20 @@ struct lua_longjmp; /* defined in ldo.c */ #endif -/* extra stack space to handle TM calls and some other extras */ +/* +** Extra stack space to handle TM calls and some other extras. This +** space is not included in 'stack_last'. It is used only to avoid stack +** checks, either because the element will be promptly popped or because +** there will be a stack check soon after the push. Function frames +** never use this extra space, so it does not need to be kept clean. +*/ #define EXTRA_STACK 5 #define BASIC_STACK_SIZE (2*LUA_MINSTACK) +#define stacksize(th) cast_int((th)->stack_last - (th)->stack) + /* kinds of Garbage Collection */ #define KGC_INC 0 /* incremental gc */ @@ -189,6 +156,18 @@ typedef struct stringtable { /* ** Information about a call. +** About union 'u': +** - field 'l' is used only for Lua functions; +** - field 'c' is used only for C functions. +** About union 'u2': +** - field 'funcidx' is used only by C functions while doing a +** protected call; +** - field 'nyield' is used only while a function is "doing" an +** yield (from the yield until the next resume); +** - field 'nres' is used only while closing tbc variables when +** returning from a function; +** - field 'transferinfo' is used only during call/returnhooks, +** before the function starts or after it ends. */ typedef struct CallInfo { StkId func; /* function index in the stack */ @@ -209,6 +188,7 @@ typedef struct CallInfo { union { int funcidx; /* called-function index */ int nyield; /* number of values yielded */ + int nres; /* number of values returned */ struct { /* info about transferred values (for call/return hooks) */ unsigned short ftransfer; /* offset of first value transferred */ unsigned short ntransfer; /* number of values transferred */ @@ -224,16 +204,34 @@ typedef struct CallInfo { */ #define CIST_OAH (1<<0) /* original value of 'allowhook' */ #define CIST_C (1<<1) /* call is running a C function */ -#define CIST_HOOKED (1<<2) /* call is running a debug hook */ -#define CIST_YPCALL (1<<3) /* call is a yieldable protected call */ -#define CIST_TAIL (1<<4) /* call was tail called */ -#define CIST_HOOKYIELD (1<<5) /* last hook called yielded */ -#define CIST_FIN (1<<6) /* call is running a finalizer */ -#define CIST_TRAN (1<<7) /* 'ci' has transfer information */ +#define CIST_FRESH (1<<2) /* call is on a fresh "luaV_execute" frame */ +#define CIST_HOOKED (1<<3) /* call is running a debug hook */ +#define CIST_YPCALL (1<<4) /* doing a yieldable protected call */ +#define CIST_TAIL (1<<5) /* call was tail called */ +#define CIST_HOOKYIELD (1<<6) /* last hook called yielded */ +#define CIST_FIN (1<<7) /* function "called" a finalizer */ +#define CIST_TRAN (1<<8) /* 'ci' has transfer information */ +#define CIST_CLSRET (1<<9) /* function is closing tbc variables */ +/* Bits 10-12 are used for CIST_RECST (see below) */ +#define CIST_RECST 10 #if defined(LUA_COMPAT_LT_LE) -#define CIST_LEQ (1<<8) /* using __lt for __le */ +#define CIST_LEQ (1<<13) /* using __lt for __le */ #endif + +/* +** Field CIST_RECST stores the "recover status", used to keep the error +** status while closing to-be-closed variables in coroutines, so that +** Lua can correctly resume after an yield from a __close method called +** because of an error. (Three bits are enough for error status.) +*/ +#define getcistrecst(ci) (((ci)->callstatus >> CIST_RECST) & 7) +#define setcistrecst(ci,st) \ + check_exp(((st) & 7) == (st), /* status must fit in three bits */ \ + ((ci)->callstatus = ((ci)->callstatus & ~(7 << CIST_RECST)) \ + | ((st) << CIST_RECST))) + + /* active function is a Lua function */ #define isLua(ci) (!((ci)->callstatus & CIST_C)) @@ -262,9 +260,10 @@ typedef struct global_State { lu_byte currentwhite; lu_byte gcstate; /* state of garbage collector */ lu_byte gckind; /* kind of GC running */ + lu_byte gcstopem; /* stops emergency collections */ lu_byte genminormul; /* control for minor generational collections */ lu_byte genmajormul; /* control for major generational collections */ - lu_byte gcrunning; /* true if GC is running */ + lu_byte gcstp; /* control whether GC is running */ lu_byte gcemergency; /* true if this is an emergency collection */ lu_byte gcpause; /* size of pause between successive GCs */ lu_byte gcstepmul; /* GC "speed" */ @@ -296,7 +295,6 @@ typedef struct global_State { TString *strcache[STRCACHE_N][STRCACHE_M]; /* cache for strings in API */ lua_WarnFunction warnf; /* warning function */ void *ud_warn; /* auxiliary data to 'warnf' */ - unsigned int Cstacklimit; /* current limit for the C stack */ } global_State; @@ -311,18 +309,18 @@ struct lua_State { StkId top; /* first free slot in the stack */ global_State *l_G; CallInfo *ci; /* call info for current function */ - StkId stack_last; /* last free slot in the stack */ + StkId stack_last; /* end of stack (last element + 1) */ StkId stack; /* stack base */ UpVal *openupval; /* list of open upvalues in this stack */ + StkId tbclist; /* list of to-be-closed variables */ GCObject *gclist; struct lua_State *twups; /* list of threads with open upvalues */ struct lua_longjmp *errorJmp; /* current error recover point */ CallInfo base_ci; /* CallInfo for first level (C calling Lua) */ volatile lua_Hook hook; ptrdiff_t errfunc; /* current error handling function (stack index) */ - l_uint32 nCcalls; /* number of allowed nested C calls - 'nci' */ + l_uint32 nCcalls; /* number of nested (non-yieldable | C) calls */ int oldpc; /* last pc traced */ - int stacksize; int basehookcount; int hookcount; volatile l_signalT hookmask; @@ -331,6 +329,12 @@ struct lua_State { #define G(L) (L->l_G) +/* +** 'g->nilvalue' being a nil value flags that the state was completely +** build. +*/ +#define completestate(g) ttisnil(&g->nilvalue) + /* ** Union of all collectable objects (only for conversions) @@ -389,12 +393,12 @@ LUAI_FUNC void luaE_freethread (lua_State *L, lua_State *L1); LUAI_FUNC CallInfo *luaE_extendCI (lua_State *L); LUAI_FUNC void luaE_freeCI (lua_State *L); LUAI_FUNC void luaE_shrinkCI (lua_State *L); -LUAI_FUNC void luaE_enterCcall (lua_State *L); +LUAI_FUNC void luaE_checkcstack (lua_State *L); +LUAI_FUNC void luaE_incCstack (lua_State *L); LUAI_FUNC void luaE_warning (lua_State *L, const char *msg, int tocont); LUAI_FUNC void luaE_warnerror (lua_State *L, const char *where); +LUAI_FUNC int luaE_resetthread (lua_State *L, int status); -#define luaE_exitCcall(L) ((L)->nCcalls++) - #endif diff --git a/src/lua/lstring.c b/src/lua/lstring.c index 6f157473..13dcaf42 100644 --- a/src/lua/lstring.c +++ b/src/lua/lstring.c @@ -22,16 +22,6 @@ #include "lstring.h" -/* -** Lua will use at most ~(2^LUAI_HASHLIMIT) bytes from a long string to -** compute its hash -*/ -#if !defined(LUAI_HASHLIMIT) -#define LUAI_HASHLIMIT 5 -#endif - - - /* ** Maximum size for string table. */ @@ -50,10 +40,9 @@ int luaS_eqlngstr (TString *a, TString *b) { } -unsigned int luaS_hash (const char *str, size_t l, unsigned int seed, - size_t step) { +unsigned int luaS_hash (const char *str, size_t l, unsigned int seed) { unsigned int h = seed ^ cast_uint(l); - for (; l >= step; l -= step) + for (; l > 0; l--) h ^= ((h<<5) + (h>>2) + cast_byte(str[l - 1])); return h; } @@ -63,8 +52,7 @@ unsigned int luaS_hashlongstr (TString *ts) { lua_assert(ts->tt == LUA_VLNGSTR); if (ts->extra == 0) { /* no hash? */ size_t len = ts->u.lnglen; - size_t step = (len >> LUAI_HASHLIMIT) + 1; - ts->hash = luaS_hash(getstr(ts), len, ts->hash, step); + ts->hash = luaS_hash(getstr(ts), len, ts->hash); ts->extra = 1; /* now it has its hash */ } return ts->hash; @@ -101,7 +89,7 @@ void luaS_resize (lua_State *L, int nsize) { if (nsize < osize) /* shrinking table? */ tablerehash(tb->hash, osize, nsize); /* depopulate shrinking part */ newvect = luaM_reallocvector(L, tb->hash, osize, nsize, TString*); - if (unlikely(newvect == NULL)) { /* reallocation failed? */ + if (l_unlikely(newvect == NULL)) { /* reallocation failed? */ if (nsize < osize) /* was it shrinking table? */ tablerehash(tb->hash, nsize, osize); /* restore to original size */ /* leave table as it was */ @@ -184,7 +172,7 @@ void luaS_remove (lua_State *L, TString *ts) { static void growstrtab (lua_State *L, stringtable *tb) { - if (unlikely(tb->nuse == MAX_INT)) { /* too many strings? */ + if (l_unlikely(tb->nuse == MAX_INT)) { /* too many strings? */ luaC_fullgc(L, 1); /* try to free some... */ if (tb->nuse == MAX_INT) /* still too many? */ luaM_error(L); /* cannot even create a message... */ @@ -201,7 +189,7 @@ static TString *internshrstr (lua_State *L, const char *str, size_t l) { TString *ts; global_State *g = G(L); stringtable *tb = &g->strt; - unsigned int h = luaS_hash(str, l, g->seed, 1); + unsigned int h = luaS_hash(str, l, g->seed); TString **list = &tb->hash[lmod(h, tb->size)]; lua_assert(str != NULL); /* otherwise 'memcmp'/'memcpy' are undefined */ for (ts = *list; ts != NULL; ts = ts->u.hnext) { @@ -235,7 +223,7 @@ TString *luaS_newlstr (lua_State *L, const char *str, size_t l) { return internshrstr(L, str, l); else { TString *ts; - if (unlikely(l >= (MAX_SIZE - sizeof(TString))/sizeof(char))) + if (l_unlikely(l >= (MAX_SIZE - sizeof(TString))/sizeof(char))) luaM_toobig(L); ts = luaS_createlngstrobj(L, l); memcpy(getstr(ts), str, l * sizeof(char)); @@ -271,7 +259,7 @@ Udata *luaS_newudata (lua_State *L, size_t s, int nuvalue) { Udata *u; int i; GCObject *o; - if (unlikely(s > MAX_SIZE - udatamemoffset(nuvalue))) + if (l_unlikely(s > MAX_SIZE - udatamemoffset(nuvalue))) luaM_toobig(L); o = luaC_newobj(L, LUA_VUSERDATA, sizeudata(nuvalue, s)); u = gco2u(o); diff --git a/src/lua/lstring.h b/src/lua/lstring.h index a413a9d3..450c2390 100644 --- a/src/lua/lstring.h +++ b/src/lua/lstring.h @@ -41,8 +41,7 @@ #define eqshrstr(a,b) check_exp((a)->tt == LUA_VSHRSTR, (a) == (b)) -LUAI_FUNC unsigned int luaS_hash (const char *str, size_t l, - unsigned int seed, size_t step); +LUAI_FUNC unsigned int luaS_hash (const char *str, size_t l, unsigned int seed); LUAI_FUNC unsigned int luaS_hashlongstr (TString *ts); LUAI_FUNC int luaS_eqlngstr (TString *a, TString *b); LUAI_FUNC void luaS_resize (lua_State *L, int newsize); diff --git a/src/lua/lstrlib.c b/src/lua/lstrlib.c index 2ba8bde4..0b4fdbb7 100644 --- a/src/lua/lstrlib.c +++ b/src/lua/lstrlib.c @@ -152,8 +152,9 @@ static int str_rep (lua_State *L) { const char *s = luaL_checklstring(L, 1, &l); lua_Integer n = luaL_checkinteger(L, 2); const char *sep = luaL_optlstring(L, 3, "", &lsep); - if (n <= 0) lua_pushliteral(L, ""); - else if (l + lsep < l || l + lsep > MAXSIZE / n) /* may overflow? */ + if (n <= 0) + lua_pushliteral(L, ""); + else if (l_unlikely(l + lsep < l || l + lsep > MAXSIZE / n)) return luaL_error(L, "resulting string too large"); else { size_t totallen = (size_t)n * l + (size_t)(n - 1) * lsep; @@ -181,7 +182,7 @@ static int str_byte (lua_State *L) { size_t pose = getendpos(L, 3, pi, l); int n, i; if (posi > pose) return 0; /* empty interval; return no values */ - if (pose - posi >= (size_t)INT_MAX) /* arithmetic overflow? */ + if (l_unlikely(pose - posi >= (size_t)INT_MAX)) /* arithmetic overflow? */ return luaL_error(L, "string slice too long"); n = (int)(pose - posi) + 1; luaL_checkstack(L, n, "string slice too long"); @@ -235,7 +236,7 @@ static int str_dump (lua_State *L) { luaL_checktype(L, 1, LUA_TFUNCTION); lua_settop(L, 1); /* ensure function is on the top of the stack */ state.init = 0; - if (lua_dump(L, writer, &state, strip) != 0) + if (l_unlikely(lua_dump(L, writer, &state, strip) != 0)) return luaL_error(L, "unable to dump given function"); luaL_pushresult(&state.B); return 1; @@ -275,7 +276,8 @@ static int tonum (lua_State *L, int arg) { static void trymt (lua_State *L, const char *mtname) { lua_settop(L, 2); /* back to the original arguments */ - if (lua_type(L, 2) == LUA_TSTRING || !luaL_getmetafield(L, 2, mtname)) + if (l_unlikely(lua_type(L, 2) == LUA_TSTRING || + !luaL_getmetafield(L, 2, mtname))) luaL_error(L, "attempt to %s a '%s' with a '%s'", mtname + 2, luaL_typename(L, -2), luaL_typename(L, -1)); lua_insert(L, -3); /* put metamethod before arguments */ @@ -383,7 +385,8 @@ static const char *match (MatchState *ms, const char *s, const char *p); static int check_capture (MatchState *ms, int l) { l -= '1'; - if (l < 0 || l >= ms->level || ms->capture[l].len == CAP_UNFINISHED) + if (l_unlikely(l < 0 || l >= ms->level || + ms->capture[l].len == CAP_UNFINISHED)) return luaL_error(ms->L, "invalid capture index %%%d", l + 1); return l; } @@ -400,14 +403,14 @@ static int capture_to_close (MatchState *ms) { static const char *classend (MatchState *ms, const char *p) { switch (*p++) { case L_ESC: { - if (p == ms->p_end) + if (l_unlikely(p == ms->p_end)) luaL_error(ms->L, "malformed pattern (ends with '%%')"); return p+1; } case '[': { if (*p == '^') p++; do { /* look for a ']' */ - if (p == ms->p_end) + if (l_unlikely(p == ms->p_end)) luaL_error(ms->L, "malformed pattern (missing ']')"); if (*(p++) == L_ESC && p < ms->p_end) p++; /* skip escapes (e.g. '%]') */ @@ -482,7 +485,7 @@ static int singlematch (MatchState *ms, const char *s, const char *p, static const char *matchbalance (MatchState *ms, const char *s, const char *p) { - if (p >= ms->p_end - 1) + if (l_unlikely(p >= ms->p_end - 1)) luaL_error(ms->L, "malformed pattern (missing arguments to '%%b')"); if (*s != *p) return NULL; else { @@ -565,7 +568,7 @@ static const char *match_capture (MatchState *ms, const char *s, int l) { static const char *match (MatchState *ms, const char *s, const char *p) { - if (ms->matchdepth-- == 0) + if (l_unlikely(ms->matchdepth-- == 0)) luaL_error(ms->L, "pattern too complex"); init: /* using goto's to optimize tail recursion */ if (p != ms->p_end) { /* end of pattern? */ @@ -599,7 +602,7 @@ static const char *match (MatchState *ms, const char *s, const char *p) { case 'f': { /* frontier? */ const char *ep; char previous; p += 2; - if (*p != '[') + if (l_unlikely(*p != '[')) luaL_error(ms->L, "missing '[' after '%%f' in pattern"); ep = classend(ms, p); /* points to what is next */ previous = (s == ms->src_init) ? '\0' : *(s - 1); @@ -699,7 +702,7 @@ static const char *lmemfind (const char *s1, size_t l1, static size_t get_onecapture (MatchState *ms, int i, const char *s, const char *e, const char **cap) { if (i >= ms->level) { - if (i != 0) + if (l_unlikely(i != 0)) luaL_error(ms->L, "invalid capture index %%%d", i + 1); *cap = s; return e - s; @@ -707,7 +710,7 @@ static size_t get_onecapture (MatchState *ms, int i, const char *s, else { ptrdiff_t capl = ms->capture[i].len; *cap = ms->capture[i].init; - if (capl == CAP_UNFINISHED) + if (l_unlikely(capl == CAP_UNFINISHED)) luaL_error(ms->L, "unfinished capture"); else if (capl == CAP_POSITION) lua_pushinteger(ms->L, (ms->capture[i].init - ms->src_init) + 1); @@ -926,7 +929,7 @@ static int add_value (MatchState *ms, luaL_Buffer *b, const char *s, luaL_addlstring(b, s, e - s); /* keep original text */ return 0; /* no changes */ } - else if (!lua_isstring(L, -1)) + else if (l_unlikely(!lua_isstring(L, -1))) return luaL_error(L, "invalid replacement value (a %s)", luaL_typename(L, -1)); else { @@ -1058,7 +1061,7 @@ static int lua_number2strx (lua_State *L, char *buff, int sz, for (i = 0; i < n; i++) buff[i] = toupper(uchar(buff[i])); } - else if (fmt[SIZELENMOD] != 'a') + else if (l_unlikely(fmt[SIZELENMOD] != 'a')) return luaL_error(L, "modifiers for format '%%a'/'%%A' not implemented"); return n; } @@ -1087,13 +1090,31 @@ static int lua_number2strx (lua_State *L, char *buff, int sz, /* valid flags in a format specification */ -#if !defined(L_FMTFLAGS) -#define L_FMTFLAGS "-+ #0" +#if !defined(L_FMTFLAGSF) + +/* valid flags for a, A, e, E, f, F, g, and G conversions */ +#define L_FMTFLAGSF "-+#0 " + +/* valid flags for o, x, and X conversions */ +#define L_FMTFLAGSX "-#0" + +/* valid flags for d and i conversions */ +#define L_FMTFLAGSI "-+0 " + +/* valid flags for u conversions */ +#define L_FMTFLAGSU "-0" + +/* valid flags for c, p, and s conversions */ +#define L_FMTFLAGSC "-" + #endif /* -** maximum size of each format specification (such as "%-099.99d") +** Maximum size of each format specification (such as "%-099.99d"): +** Initial '%', flags (up to 5), width (2), period, precision (2), +** length modifier (8), conversion specifier, and final '\0', plus some +** extra. */ #define MAX_FORMAT 32 @@ -1186,25 +1207,53 @@ static void addliteral (lua_State *L, luaL_Buffer *b, int arg) { } -static const char *scanformat (lua_State *L, const char *strfrmt, char *form) { - const char *p = strfrmt; - while (*p != '\0' && strchr(L_FMTFLAGS, *p) != NULL) p++; /* skip flags */ - if ((size_t)(p - strfrmt) >= sizeof(L_FMTFLAGS)/sizeof(char)) - luaL_error(L, "invalid format (repeated flags)"); - if (isdigit(uchar(*p))) p++; /* skip width */ - if (isdigit(uchar(*p))) p++; /* (2 digits at most) */ - if (*p == '.') { - p++; - if (isdigit(uchar(*p))) p++; /* skip precision */ - if (isdigit(uchar(*p))) p++; /* (2 digits at most) */ +static const char *get2digits (const char *s) { + if (isdigit(uchar(*s))) { + s++; + if (isdigit(uchar(*s))) s++; /* (2 digits at most) */ } - if (isdigit(uchar(*p))) - luaL_error(L, "invalid format (width or precision too long)"); + return s; +} + + +/* +** Check whether a conversion specification is valid. When called, +** first character in 'form' must be '%' and last character must +** be a valid conversion specifier. 'flags' are the accepted flags; +** 'precision' signals whether to accept a precision. +*/ +static void checkformat (lua_State *L, const char *form, const char *flags, + int precision) { + const char *spec = form + 1; /* skip '%' */ + spec += strspn(spec, flags); /* skip flags */ + if (*spec != '0') { /* a width cannot start with '0' */ + spec = get2digits(spec); /* skip width */ + if (*spec == '.' && precision) { + spec++; + spec = get2digits(spec); /* skip precision */ + } + } + if (!isalpha(uchar(*spec))) /* did not go to the end? */ + luaL_error(L, "invalid conversion specification: '%s'", form); +} + + +/* +** Get a conversion specification and copy it to 'form'. +** Return the address of its last character. +*/ +static const char *getformat (lua_State *L, const char *strfrmt, + char *form) { + /* spans flags, width, and precision ('0' is included as a flag) */ + size_t len = strspn(strfrmt, L_FMTFLAGSF "123456789."); + len++; /* adds following character (should be the specifier) */ + /* still needs space for '%', '\0', plus a length modifier */ + if (len >= MAX_FORMAT - 10) + luaL_error(L, "invalid format (too long)"); *(form++) = '%'; - memcpy(form, strfrmt, ((p - strfrmt) + 1) * sizeof(char)); - form += (p - strfrmt) + 1; - *form = '\0'; - return p; + memcpy(form, strfrmt, len * sizeof(char)); + *(form + len) = '\0'; + return strfrmt + len - 1; } @@ -1227,6 +1276,7 @@ static int str_format (lua_State *L) { size_t sfl; const char *strfrmt = luaL_checklstring(L, arg, &sfl); const char *strfrmt_end = strfrmt+sfl; + const char *flags; luaL_Buffer b; luaL_buffinit(L, &b); while (strfrmt < strfrmt_end) { @@ -1236,25 +1286,35 @@ static int str_format (lua_State *L) { luaL_addchar(&b, *strfrmt++); /* %% */ else { /* format item */ char form[MAX_FORMAT]; /* to store the format ('%...') */ - int maxitem = MAX_ITEM; - char *buff = luaL_prepbuffsize(&b, maxitem); /* to put formatted item */ - int nb = 0; /* number of bytes in added item */ + int maxitem = MAX_ITEM; /* maximum length for the result */ + char *buff = luaL_prepbuffsize(&b, maxitem); /* to put result */ + int nb = 0; /* number of bytes in result */ if (++arg > top) return luaL_argerror(L, arg, "no value"); - strfrmt = scanformat(L, strfrmt, form); + strfrmt = getformat(L, strfrmt, form); switch (*strfrmt++) { case 'c': { + checkformat(L, form, L_FMTFLAGSC, 0); nb = l_sprintf(buff, maxitem, form, (int)luaL_checkinteger(L, arg)); break; } case 'd': case 'i': - case 'o': case 'u': case 'x': case 'X': { + flags = L_FMTFLAGSI; + goto intcase; + case 'u': + flags = L_FMTFLAGSU; + goto intcase; + case 'o': case 'x': case 'X': + flags = L_FMTFLAGSX; + intcase: { lua_Integer n = luaL_checkinteger(L, arg); + checkformat(L, form, flags, 1); addlenmod(form, LUA_INTEGER_FRMLEN); nb = l_sprintf(buff, maxitem, form, (LUAI_UACINT)n); break; } case 'a': case 'A': + checkformat(L, form, L_FMTFLAGSF, 1); addlenmod(form, LUA_NUMBER_FRMLEN); nb = lua_number2strx(L, buff, maxitem, form, luaL_checknumber(L, arg)); @@ -1265,12 +1325,14 @@ static int str_format (lua_State *L) { /* FALLTHROUGH */ case 'e': case 'E': case 'g': case 'G': { lua_Number n = luaL_checknumber(L, arg); + checkformat(L, form, L_FMTFLAGSF, 1); addlenmod(form, LUA_NUMBER_FRMLEN); nb = l_sprintf(buff, maxitem, form, (LUAI_UACNUMBER)n); break; } case 'p': { const void *p = lua_topointer(L, arg); + checkformat(L, form, L_FMTFLAGSC, 0); if (p == NULL) { /* avoid calling 'printf' with argument NULL */ p = "(null)"; /* result */ form[strlen(form) - 1] = 's'; /* format it as a string */ @@ -1291,7 +1353,8 @@ static int str_format (lua_State *L) { luaL_addvalue(&b); /* keep entire string */ else { luaL_argcheck(L, l == strlen(s), arg, "string contains zeros"); - if (!strchr(form, '.') && l >= 100) { + checkformat(L, form, L_FMTFLAGSC, 1); + if (strchr(form, '.') == NULL && l >= 100) { /* no precision and string is too long to be formatted */ luaL_addvalue(&b); /* keep entire string */ } @@ -1349,26 +1412,6 @@ static const union { } nativeendian = {1}; -/* dummy structure to get native alignment requirements */ -struct cD { - char c; - union { double d; void *p; lua_Integer i; lua_Number n; } u; -}; - -#define MAXALIGN (offsetof(struct cD, u)) - - -/* -** Union for serializing floats -*/ -typedef union Ftypes { - float f; - double d; - lua_Number n; - char buff[5 * sizeof(lua_Number)]; /* enough for any float type */ -} Ftypes; - - /* ** information to pack/unpack stuff */ @@ -1385,7 +1428,9 @@ typedef struct Header { typedef enum KOption { Kint, /* signed integers */ Kuint, /* unsigned integers */ - Kfloat, /* floating-point numbers */ + Kfloat, /* single-precision floating-point numbers */ + Knumber, /* Lua "native" floating-point numbers */ + Kdouble, /* double-precision floating-point numbers */ Kchar, /* fixed-length strings */ Kstring, /* strings with prefixed length */ Kzstr, /* zero-terminated strings */ @@ -1420,7 +1465,7 @@ static int getnum (const char **fmt, int df) { */ static int getnumlimit (Header *h, const char **fmt, int df) { int sz = getnum(fmt, df); - if (sz > MAXINTSIZE || sz <= 0) + if (l_unlikely(sz > MAXINTSIZE || sz <= 0)) return luaL_error(h->L, "integral size (%d) out of limits [1,%d]", sz, MAXINTSIZE); return sz; @@ -1441,6 +1486,8 @@ static void initheader (lua_State *L, Header *h) { ** Read and classify next option. 'size' is filled with option's size. */ static KOption getoption (Header *h, const char **fmt, int *size) { + /* dummy structure to get native alignment requirements */ + struct cD { char c; union { LUAI_MAXALIGN; } u; }; int opt = *((*fmt)++); *size = 0; /* default */ switch (opt) { @@ -1454,14 +1501,14 @@ static KOption getoption (Header *h, const char **fmt, int *size) { case 'J': *size = sizeof(lua_Integer); return Kuint; case 'T': *size = sizeof(size_t); return Kuint; case 'f': *size = sizeof(float); return Kfloat; - case 'd': *size = sizeof(double); return Kfloat; - case 'n': *size = sizeof(lua_Number); return Kfloat; + case 'n': *size = sizeof(lua_Number); return Knumber; + case 'd': *size = sizeof(double); return Kdouble; case 'i': *size = getnumlimit(h, fmt, sizeof(int)); return Kint; case 'I': *size = getnumlimit(h, fmt, sizeof(int)); return Kuint; case 's': *size = getnumlimit(h, fmt, sizeof(size_t)); return Kstring; case 'c': *size = getnum(fmt, -1); - if (*size == -1) + if (l_unlikely(*size == -1)) luaL_error(h->L, "missing size for format option 'c'"); return Kchar; case 'z': return Kzstr; @@ -1471,7 +1518,11 @@ static KOption getoption (Header *h, const char **fmt, int *size) { case '<': h->islittle = 1; break; case '>': h->islittle = 0; break; case '=': h->islittle = nativeendian.little; break; - case '!': h->maxalign = getnumlimit(h, fmt, MAXALIGN); break; + case '!': { + const int maxalign = offsetof(struct cD, u); + h->maxalign = getnumlimit(h, fmt, maxalign); + break; + } default: luaL_error(h->L, "invalid format option '%c'", opt); } return Knop; @@ -1500,7 +1551,7 @@ static KOption getdetails (Header *h, size_t totalsize, else { if (align > h->maxalign) /* enforce maximum alignment */ align = h->maxalign; - if ((align & (align - 1)) != 0) /* is 'align' not a power of 2? */ + if (l_unlikely((align & (align - 1)) != 0)) /* not a power of 2? */ luaL_argerror(h->L, 1, "format asks for alignment not power of 2"); *ntoalign = (align - (int)(totalsize & (align - 1))) & (align - 1); } @@ -1535,12 +1586,10 @@ static void packint (luaL_Buffer *b, lua_Unsigned n, ** Copy 'size' bytes from 'src' to 'dest', correcting endianness if ** given 'islittle' is different from native endianness. */ -static void copywithendian (volatile char *dest, volatile const char *src, +static void copywithendian (char *dest, const char *src, int size, int islittle) { - if (islittle == nativeendian.little) { - while (size-- != 0) - *(dest++) = *(src++); - } + if (islittle == nativeendian.little) + memcpy(dest, src, size); else { dest += size - 1; while (size-- != 0) @@ -1583,15 +1632,27 @@ static int str_pack (lua_State *L) { packint(&b, (lua_Unsigned)n, h.islittle, size, 0); break; } - case Kfloat: { /* floating-point options */ - volatile Ftypes u; - char *buff = luaL_prepbuffsize(&b, size); - lua_Number n = luaL_checknumber(L, arg); /* get argument */ - if (size == sizeof(u.f)) u.f = (float)n; /* copy it into 'u' */ - else if (size == sizeof(u.d)) u.d = (double)n; - else u.n = n; - /* move 'u' to final result, correcting endianness if needed */ - copywithendian(buff, u.buff, size, h.islittle); + case Kfloat: { /* C float */ + float f = (float)luaL_checknumber(L, arg); /* get argument */ + char *buff = luaL_prepbuffsize(&b, sizeof(f)); + /* move 'f' to final result, correcting endianness if needed */ + copywithendian(buff, (char *)&f, sizeof(f), h.islittle); + luaL_addsize(&b, size); + break; + } + case Knumber: { /* Lua float */ + lua_Number f = luaL_checknumber(L, arg); /* get argument */ + char *buff = luaL_prepbuffsize(&b, sizeof(f)); + /* move 'f' to final result, correcting endianness if needed */ + copywithendian(buff, (char *)&f, sizeof(f), h.islittle); + luaL_addsize(&b, size); + break; + } + case Kdouble: { /* C double */ + double f = (double)luaL_checknumber(L, arg); /* get argument */ + char *buff = luaL_prepbuffsize(&b, sizeof(f)); + /* move 'f' to final result, correcting endianness if needed */ + copywithendian(buff, (char *)&f, sizeof(f), h.islittle); luaL_addsize(&b, size); break; } @@ -1682,7 +1743,7 @@ static lua_Integer unpackint (lua_State *L, const char *str, else if (size > SZINT) { /* must check unread bytes */ int mask = (!issigned || (lua_Integer)res >= 0) ? 0 : MC; for (i = limit; i < size; i++) { - if ((unsigned char)str[islittle ? i : size - 1 - i] != mask) + if (l_unlikely((unsigned char)str[islittle ? i : size - 1 - i] != mask)) luaL_error(L, "%d-byte integer does not fit into Lua Integer", size); } } @@ -1717,13 +1778,21 @@ static int str_unpack (lua_State *L) { break; } case Kfloat: { - volatile Ftypes u; - lua_Number num; - copywithendian(u.buff, data + pos, size, h.islittle); - if (size == sizeof(u.f)) num = (lua_Number)u.f; - else if (size == sizeof(u.d)) num = (lua_Number)u.d; - else num = u.n; - lua_pushnumber(L, num); + float f; + copywithendian((char *)&f, data + pos, sizeof(f), h.islittle); + lua_pushnumber(L, (lua_Number)f); + break; + } + case Knumber: { + lua_Number f; + copywithendian((char *)&f, data + pos, sizeof(f), h.islittle); + lua_pushnumber(L, f); + break; + } + case Kdouble: { + double f; + copywithendian((char *)&f, data + pos, sizeof(f), h.islittle); + lua_pushnumber(L, (lua_Number)f); break; } case Kchar: { @@ -1738,7 +1807,7 @@ static int str_unpack (lua_State *L) { break; } case Kzstr: { - size_t len = (int)strlen(data + pos); + size_t len = strlen(data + pos); luaL_argcheck(L, pos + len < ld, 2, "unfinished string for format 'z'"); lua_pushlstring(L, data + pos, len); diff --git a/src/lua/ltable.c b/src/lua/ltable.c index 5a0d066f..1b1cd241 100644 --- a/src/lua/ltable.c +++ b/src/lua/ltable.c @@ -68,18 +68,21 @@ #define MAXHSIZE luaM_limitN(1u << MAXHBITS, Node) +/* +** When the original hash value is good, hashing by a power of 2 +** avoids the cost of '%'. +*/ #define hashpow2(t,n) (gnode(t, lmod((n), sizenode(t)))) +/* +** for other types, it is better to avoid modulo by power of 2, as +** they can have many 2 factors. +*/ +#define hashmod(t,n) (gnode(t, ((n) % ((sizenode(t)-1)|1)))) + + #define hashstr(t,str) hashpow2(t, (str)->hash) #define hashboolean(t,p) hashpow2(t, p) -#define hashint(t,i) hashpow2(t, i) - - -/* -** for some types, it is better to avoid modulus by power of 2, as -** they tend to have many 2 factors. -*/ -#define hashmod(t,n) (gnode(t, ((n) % ((sizenode(t)-1)|1)))) #define hashpointer(t,p) hashmod(t, point2uint(p)) @@ -96,6 +99,20 @@ static const Node dummynode_ = { static const TValue absentkey = {ABSTKEYCONSTANT}; +/* +** Hash for integers. To allow a good hash, use the remainder operator +** ('%'). If integer fits as a non-negative int, compute an int +** remainder, which is faster. Otherwise, use an unsigned-integer +** remainder, which uses all bits and ensures a non-negative result. +*/ +static Node *hashint (const Table *t, lua_Integer i) { + lua_Unsigned ui = l_castS2U(i); + if (ui <= (unsigned int)INT_MAX) + return hashmod(t, cast_int(ui)); + else + return hashmod(t, ui); +} + /* ** Hash for floating-point numbers. @@ -129,54 +146,78 @@ static int l_hashfloat (lua_Number n) { /* ** returns the 'main' position of an element in a table (that is, -** the index of its hash value). The key comes broken (tag in 'ktt' -** and value in 'vkl') so that we can call it on keys inserted into -** nodes. +** the index of its hash value). */ -static Node *mainposition (const Table *t, int ktt, const Value *kvl) { - switch (withvariant(ktt)) { - case LUA_VNUMINT: - return hashint(t, ivalueraw(*kvl)); - case LUA_VNUMFLT: - return hashmod(t, l_hashfloat(fltvalueraw(*kvl))); - case LUA_VSHRSTR: - return hashstr(t, tsvalueraw(*kvl)); - case LUA_VLNGSTR: - return hashpow2(t, luaS_hashlongstr(tsvalueraw(*kvl))); +static Node *mainpositionTV (const Table *t, const TValue *key) { + switch (ttypetag(key)) { + case LUA_VNUMINT: { + lua_Integer i = ivalue(key); + return hashint(t, i); + } + case LUA_VNUMFLT: { + lua_Number n = fltvalue(key); + return hashmod(t, l_hashfloat(n)); + } + case LUA_VSHRSTR: { + TString *ts = tsvalue(key); + return hashstr(t, ts); + } + case LUA_VLNGSTR: { + TString *ts = tsvalue(key); + return hashpow2(t, luaS_hashlongstr(ts)); + } case LUA_VFALSE: return hashboolean(t, 0); case LUA_VTRUE: return hashboolean(t, 1); - case LUA_VLIGHTUSERDATA: - return hashpointer(t, pvalueraw(*kvl)); - case LUA_VLCF: - return hashpointer(t, fvalueraw(*kvl)); - default: - return hashpointer(t, gcvalueraw(*kvl)); + case LUA_VLIGHTUSERDATA: { + void *p = pvalue(key); + return hashpointer(t, p); + } + case LUA_VLCF: { + lua_CFunction f = fvalue(key); + return hashpointer(t, f); + } + default: { + GCObject *o = gcvalue(key); + return hashpointer(t, o); + } } } -/* -** Returns the main position of an element given as a 'TValue' -*/ -static Node *mainpositionTV (const Table *t, const TValue *key) { - return mainposition(t, rawtt(key), valraw(key)); +l_sinline Node *mainpositionfromnode (const Table *t, Node *nd) { + TValue key; + getnodekey(cast(lua_State *, NULL), &key, nd); + return mainpositionTV(t, &key); } /* -** Check whether key 'k1' is equal to the key in node 'n2'. -** This equality is raw, so there are no metamethods. Floats -** with integer values have been normalized, so integers cannot -** be equal to floats. It is assumed that 'eqshrstr' is simply -** pointer equality, so that short strings are handled in the -** default case. +** Check whether key 'k1' is equal to the key in node 'n2'. This +** equality is raw, so there are no metamethods. Floats with integer +** values have been normalized, so integers cannot be equal to +** floats. It is assumed that 'eqshrstr' is simply pointer equality, so +** that short strings are handled in the default case. +** A true 'deadok' means to accept dead keys as equal to their original +** values. All dead keys are compared in the default case, by pointer +** identity. (Only collectable objects can produce dead keys.) Note that +** dead long strings are also compared by identity. +** Once a key is dead, its corresponding value may be collected, and +** then another value can be created with the same address. If this +** other value is given to 'next', 'equalkey' will signal a false +** positive. In a regular traversal, this situation should never happen, +** as all keys given to 'next' came from the table itself, and therefore +** could not have been collected. Outside a regular traversal, we +** have garbage in, garbage out. What is relevant is that this false +** positive does not break anything. (In particular, 'next' will return +** some other valid item on the table or nil.) */ -static int equalkey (const TValue *k1, const Node *n2) { - if (rawtt(k1) != keytt(n2)) /* not the same variants? */ +static int equalkey (const TValue *k1, const Node *n2, int deadok) { + if ((rawtt(k1) != keytt(n2)) && /* not the same variants? */ + !(deadok && keyisdead(n2) && iscollectable(k1))) return 0; /* cannot be same key */ - switch (ttypetag(k1)) { + switch (keytt(n2)) { case LUA_VNIL: case LUA_VFALSE: case LUA_VTRUE: return 1; case LUA_VNUMINT: @@ -187,7 +228,7 @@ static int equalkey (const TValue *k1, const Node *n2) { return pvalue(k1) == pvalueraw(keyval(n2)); case LUA_VLCF: return fvalue(k1) == fvalueraw(keyval(n2)); - case LUA_VLNGSTR: + case ctb(LUA_VLNGSTR): return luaS_eqlngstr(tsvalue(k1), keystrval(n2)); default: return gcvalue(k1) == gcvalueraw(keyval(n2)); @@ -251,11 +292,12 @@ static unsigned int setlimittosize (Table *t) { /* ** "Generic" get version. (Not that generic: not valid for integers, ** which may be in array part, nor for floats with integral values.) +** See explanation about 'deadok' in function 'equalkey'. */ -static const TValue *getgeneric (Table *t, const TValue *key) { +static const TValue *getgeneric (Table *t, const TValue *key, int deadok) { Node *n = mainpositionTV(t, key); for (;;) { /* check whether 'key' is somewhere in the chain */ - if (equalkey(key, n)) + if (equalkey(key, n, deadok)) return gval(n); /* that's it */ else { int nx = gnext(n); @@ -292,8 +334,8 @@ static unsigned int findindex (lua_State *L, Table *t, TValue *key, if (i - 1u < asize) /* is 'key' inside array part? */ return i; /* yes; that's the index */ else { - const TValue *n = getgeneric(t, key); - if (unlikely(isabstkey(n))) + const TValue *n = getgeneric(t, key, 1); + if (l_unlikely(isabstkey(n))) luaG_runerror(L, "invalid key to 'next'"); /* key not found */ i = cast_int(nodefromval(n) - gnode(t, 0)); /* key index in hash table */ /* hash elements are numbered after array ones */ @@ -471,7 +513,7 @@ static void reinsert (lua_State *L, Table *ot, Table *t) { already present in the table */ TValue k; getnodekey(L, &k, old); - setobjt2t(L, luaH_set(L, t, &k), gval(old)); + luaH_set(L, t, &k, gval(old)); } } } @@ -527,7 +569,7 @@ void luaH_resize (lua_State *L, Table *t, unsigned int newasize, } /* allocate new array */ newarray = luaM_reallocvector(L, t->array, oldasize, newasize, TValue); - if (unlikely(newarray == NULL && newasize > 0)) { /* allocation failed? */ + if (l_unlikely(newarray == NULL && newasize > 0)) { /* allocation failed? */ freehash(L, &newt); /* release new hash part */ luaM_error(L); /* raise error (with array unchanged) */ } @@ -618,10 +660,10 @@ static Node *getfreepos (Table *t) { ** put new key in its main position; otherwise (colliding node is in its main ** position), new key goes to an empty position. */ -TValue *luaH_newkey (lua_State *L, Table *t, const TValue *key) { +void luaH_newkey (lua_State *L, Table *t, const TValue *key, TValue *value) { Node *mp; TValue aux; - if (unlikely(ttisnil(key))) + if (l_unlikely(ttisnil(key))) luaG_runerror(L, "table index is nil"); else if (ttisfloat(key)) { lua_Number f = fltvalue(key); @@ -630,9 +672,11 @@ TValue *luaH_newkey (lua_State *L, Table *t, const TValue *key) { setivalue(&aux, k); key = &aux; /* insert it as an integer */ } - else if (unlikely(luai_numisnan(f))) + else if (l_unlikely(luai_numisnan(f))) luaG_runerror(L, "table index is NaN"); } + if (ttisnil(value)) + return; /* do not insert nil values */ mp = mainpositionTV(t, key); if (!isempty(gval(mp)) || isdummy(t)) { /* main position is taken? */ Node *othern; @@ -640,10 +684,11 @@ TValue *luaH_newkey (lua_State *L, Table *t, const TValue *key) { if (f == NULL) { /* cannot find a free place? */ rehash(L, t, key); /* grow table */ /* whatever called 'newkey' takes care of TM cache */ - return luaH_set(L, t, key); /* insert key into grown table */ + luaH_set(L, t, key, value); /* insert key into grown table */ + return; } lua_assert(!isdummy(t)); - othern = mainposition(t, keytt(mp), &keyval(mp)); + othern = mainpositionfromnode(t, mp); if (othern != mp) { /* is colliding node out of its main position? */ /* yes; move colliding node into free position */ while (othern + gnext(othern) != mp) /* find previous */ @@ -668,7 +713,7 @@ TValue *luaH_newkey (lua_State *L, Table *t, const TValue *key) { setnodekey(L, mp, key); luaC_barrierback(L, obj2gco(t), key); lua_assert(isempty(gval(mp))); - return gval(mp); + setobj2t(L, gval(mp), value); } @@ -730,7 +775,7 @@ const TValue *luaH_getstr (Table *t, TString *key) { else { /* for long strings, use generic case */ TValue ko; setsvalue(cast(lua_State *, NULL), &ko, key); - return getgeneric(t, &ko); + return getgeneric(t, &ko, 0); } } @@ -750,34 +795,45 @@ const TValue *luaH_get (Table *t, const TValue *key) { /* else... */ } /* FALLTHROUGH */ default: - return getgeneric(t, key); + return getgeneric(t, key, 0); } } +/* +** Finish a raw "set table" operation, where 'slot' is where the value +** should have been (the result of a previous "get table"). +** Beware: when using this function you probably need to check a GC +** barrier and invalidate the TM cache. +*/ +void luaH_finishset (lua_State *L, Table *t, const TValue *key, + const TValue *slot, TValue *value) { + if (isabstkey(slot)) + luaH_newkey(L, t, key, value); + else + setobj2t(L, cast(TValue *, slot), value); +} + + /* ** beware: when using this function you probably need to check a GC ** barrier and invalidate the TM cache. */ -TValue *luaH_set (lua_State *L, Table *t, const TValue *key) { - const TValue *p = luaH_get(t, key); - if (!isabstkey(p)) - return cast(TValue *, p); - else return luaH_newkey(L, t, key); +void luaH_set (lua_State *L, Table *t, const TValue *key, TValue *value) { + const TValue *slot = luaH_get(t, key); + luaH_finishset(L, t, key, slot, value); } void luaH_setint (lua_State *L, Table *t, lua_Integer key, TValue *value) { const TValue *p = luaH_getint(t, key); - TValue *cell; - if (!isabstkey(p)) - cell = cast(TValue *, p); - else { + if (isabstkey(p)) { TValue k; setivalue(&k, key); - cell = luaH_newkey(L, t, &k); + luaH_newkey(L, t, &k, value); } - setobj2t(L, cell, value); + else + setobj2t(L, cast(TValue *, p), value); } diff --git a/src/lua/ltable.h b/src/lua/ltable.h index c0060f4b..7bbbcb21 100644 --- a/src/lua/ltable.h +++ b/src/lua/ltable.h @@ -41,8 +41,12 @@ LUAI_FUNC void luaH_setint (lua_State *L, Table *t, lua_Integer key, LUAI_FUNC const TValue *luaH_getshortstr (Table *t, TString *key); LUAI_FUNC const TValue *luaH_getstr (Table *t, TString *key); LUAI_FUNC const TValue *luaH_get (Table *t, const TValue *key); -LUAI_FUNC TValue *luaH_newkey (lua_State *L, Table *t, const TValue *key); -LUAI_FUNC TValue *luaH_set (lua_State *L, Table *t, const TValue *key); +LUAI_FUNC void luaH_newkey (lua_State *L, Table *t, const TValue *key, + TValue *value); +LUAI_FUNC void luaH_set (lua_State *L, Table *t, const TValue *key, + TValue *value); +LUAI_FUNC void luaH_finishset (lua_State *L, Table *t, const TValue *key, + const TValue *slot, TValue *value); LUAI_FUNC Table *luaH_new (lua_State *L); LUAI_FUNC void luaH_resize (lua_State *L, Table *t, unsigned int nasize, unsigned int nhsize); diff --git a/src/lua/ltablib.c b/src/lua/ltablib.c index d344a47e..868d78fd 100644 --- a/src/lua/ltablib.c +++ b/src/lua/ltablib.c @@ -59,8 +59,9 @@ static void checktab (lua_State *L, int arg, int what) { static int tinsert (lua_State *L) { - lua_Integer e = aux_getn(L, 1, TAB_RW) + 1; /* first empty element */ lua_Integer pos; /* where to insert new element */ + lua_Integer e = aux_getn(L, 1, TAB_RW); + e = luaL_intop(+, e, 1); /* first empty element */ switch (lua_gettop(L)) { case 2: { /* called with only 2 arguments */ pos = e; /* insert new element at the end */ @@ -145,9 +146,9 @@ static int tmove (lua_State *L) { static void addfield (lua_State *L, luaL_Buffer *b, lua_Integer i) { lua_geti(L, 1, i); - if (!lua_isstring(L, -1)) - luaL_error(L, "invalid value (%s) at index %d in table for 'concat'", - luaL_typename(L, -1), i); + if (l_unlikely(!lua_isstring(L, -1))) + luaL_error(L, "invalid value (%s) at index %I in table for 'concat'", + luaL_typename(L, -1), (LUAI_UACINT)i); luaL_addvalue(b); } @@ -196,7 +197,8 @@ static int tunpack (lua_State *L) { lua_Integer e = luaL_opt(L, luaL_checkinteger, 3, luaL_len(L, 1)); if (i > e) return 0; /* empty range */ n = (lua_Unsigned)e - i; /* number of elements minus 1 (avoid overflows) */ - if (n >= (unsigned int)INT_MAX || !lua_checkstack(L, (int)(++n))) + if (l_unlikely(n >= (unsigned int)INT_MAX || + !lua_checkstack(L, (int)(++n)))) return luaL_error(L, "too many results to unpack"); for (; i < e; i++) { /* push arg[i..e - 1] (to avoid overflows) */ lua_geti(L, 1, i); @@ -300,14 +302,14 @@ static IdxT partition (lua_State *L, IdxT lo, IdxT up) { for (;;) { /* next loop: repeat ++i while a[i] < P */ while ((void)lua_geti(L, 1, ++i), sort_comp(L, -1, -2)) { - if (i == up - 1) /* a[i] < P but a[up - 1] == P ?? */ + if (l_unlikely(i == up - 1)) /* a[i] < P but a[up - 1] == P ?? */ luaL_error(L, "invalid order function for sorting"); lua_pop(L, 1); /* remove a[i] */ } /* after the loop, a[i] >= P and a[lo .. i - 1] < P */ /* next loop: repeat --j while P < a[j] */ while ((void)lua_geti(L, 1, --j), sort_comp(L, -3, -1)) { - if (j < i) /* j < i but a[j] > P ?? */ + if (l_unlikely(j < i)) /* j < i but a[j] > P ?? */ luaL_error(L, "invalid order function for sorting"); lua_pop(L, 1); /* remove a[j] */ } diff --git a/src/lua/ltm.c b/src/lua/ltm.c index 4770f96b..b657b783 100644 --- a/src/lua/ltm.c +++ b/src/lua/ltm.c @@ -147,7 +147,7 @@ static int callbinTM (lua_State *L, const TValue *p1, const TValue *p2, void luaT_trybinTM (lua_State *L, const TValue *p1, const TValue *p2, StkId res, TMS event) { - if (!callbinTM(L, p1, p2, res, event)) { + if (l_unlikely(!callbinTM(L, p1, p2, res, event))) { switch (event) { case TM_BAND: case TM_BOR: case TM_BXOR: case TM_SHL: case TM_SHR: case TM_BNOT: { @@ -166,7 +166,8 @@ void luaT_trybinTM (lua_State *L, const TValue *p1, const TValue *p2, void luaT_tryconcatTM (lua_State *L) { StkId top = L->top; - if (!callbinTM(L, s2v(top - 2), s2v(top - 1), top - 2, TM_CONCAT)) + if (l_unlikely(!callbinTM(L, s2v(top - 2), s2v(top - 1), top - 2, + TM_CONCAT))) luaG_concaterror(L, s2v(top - 2), s2v(top - 1)); } diff --git a/src/lua/lua.c b/src/lua/lua.c index a363925c..4d5274fa 100644 --- a/src/lua/lua.c +++ b/src/lua/lua.c @@ -39,6 +39,26 @@ static lua_State *globalL = NULL; static const char *progname = LUA_PROGNAME; +#if defined(LUA_USE_POSIX) /* { */ + +/* +** Use 'sigaction' when available. +*/ +static void setsignal (int sig, void (*handler)(int)) { + struct sigaction sa; + sa.sa_handler = handler; + sa.sa_flags = 0; + sigemptyset(&sa.sa_mask); /* do not mask any signal */ + sigaction(sig, &sa, NULL); +} + +#else /* }{ */ + +#define setsignal signal + +#endif /* } */ + + /* ** Hook set by signal function to stop the interpreter. */ @@ -57,7 +77,7 @@ static void lstop (lua_State *L, lua_Debug *ar) { */ static void laction (int i) { int flag = LUA_MASKCALL | LUA_MASKRET | LUA_MASKLINE | LUA_MASKCOUNT; - signal(i, SIG_DFL); /* if another SIGINT happens, terminate process */ + setsignal(i, SIG_DFL); /* if another SIGINT happens, terminate process */ lua_sethook(globalL, lstop, flag, 1); } @@ -71,14 +91,15 @@ static void print_usage (const char *badoption) { lua_writestringerror( "usage: %s [options] [script [args]]\n" "Available options are:\n" - " -e stat execute string 'stat'\n" - " -i enter interactive mode after executing 'script'\n" - " -l name require library 'name' into global 'name'\n" - " -v show version information\n" - " -E ignore environment variables\n" - " -W turn warnings on\n" - " -- stop handling options\n" - " - stop handling options and execute stdin\n" + " -e stat execute string 'stat'\n" + " -i enter interactive mode after executing 'script'\n" + " -l mod require library 'mod' into global 'mod'\n" + " -l g=mod require library 'mod' into global 'g'\n" + " -v show version information\n" + " -E ignore environment variables\n" + " -W turn warnings on\n" + " -- stop handling options\n" + " - stop handling options and execute stdin\n" , progname); } @@ -137,9 +158,9 @@ static int docall (lua_State *L, int narg, int nres) { lua_pushcfunction(L, msghandler); /* push message handler */ lua_insert(L, base); /* put it under function and args */ globalL = L; /* to be available to 'laction' */ - signal(SIGINT, laction); /* set C-signal handler */ + setsignal(SIGINT, laction); /* set C-signal handler */ status = lua_pcall(L, narg, nres, base); - signal(SIGINT, SIG_DFL); /* reset C-signal handler */ + setsignal(SIGINT, SIG_DFL); /* reset C-signal handler */ lua_remove(L, base); /* remove message handler from the stack */ return status; } @@ -189,16 +210,22 @@ static int dostring (lua_State *L, const char *s, const char *name) { /* -** Calls 'require(name)' and stores the result in a global variable -** with the given name. +** Receives 'globname[=modname]' and runs 'globname = require(modname)'. */ -int dolibrary (lua_State *L, const char *name) { +int dolibrary (lua_State *L, char *globname) { int status; + char *modname = strchr(globname, '='); + if (modname == NULL) /* no explicit name? */ + modname = globname; /* module name is equal to global name */ + else { + *modname = '\0'; /* global name ends here */ + modname++; /* module name starts after the '=' */ + } lua_getglobal(L, "require"); - lua_pushstring(L, name); - status = docall(L, 1, 1); /* call 'require(name)' */ + lua_pushstring(L, modname); + status = docall(L, 1, 1); /* call 'require(modname)' */ if (status == LUA_OK) - lua_setglobal(L, name); /* global[name] = require return */ + lua_setglobal(L, globname); /* globname = require(modname) */ return report(L, status); } @@ -309,7 +336,7 @@ static int runargs (lua_State *L, char **argv, int n) { switch (option) { case 'e': case 'l': { int status; - const char *extra = argv[i] + 2; /* both options need an argument */ + char *extra = argv[i] + 2; /* both options need an argument */ if (*extra == '\0') extra = argv[++i]; lua_assert(extra != NULL); status = (option == 'e') @@ -418,14 +445,18 @@ static int handle_luainit (lua_State *L) { /* -** Returns the string to be used as a prompt by the interpreter. +** Return the string to be used as a prompt by the interpreter. Leave +** the string (or nil, if using the default value) on the stack, to keep +** it anchored. */ static const char *get_prompt (lua_State *L, int firstline) { - const char *p; - lua_getglobal(L, firstline ? "_PROMPT" : "_PROMPT2"); - p = lua_tostring(L, -1); - if (p == NULL) p = (firstline ? LUA_PROMPT : LUA_PROMPT2); - return p; + if (lua_getglobal(L, firstline ? "_PROMPT" : "_PROMPT2") == LUA_TNIL) + return (firstline ? LUA_PROMPT : LUA_PROMPT2); /* use the default */ + else { /* apply 'tostring' over the value */ + const char *p = luaL_tolstring(L, -1, NULL); + lua_remove(L, -2); /* remove original value */ + return p; + } } /* mark in error messages for incomplete statements */ diff --git a/src/lua/lua.h b/src/lua/lua.h index 08c6a64a..e6618392 100644 --- a/src/lua/lua.h +++ b/src/lua/lua.h @@ -18,14 +18,14 @@ #define LUA_VERSION_MAJOR "5" #define LUA_VERSION_MINOR "4" -#define LUA_VERSION_RELEASE "1" +#define LUA_VERSION_RELEASE "4" #define LUA_VERSION_NUM 504 -#define LUA_VERSION_RELEASE_NUM (LUA_VERSION_NUM * 100 + 0) +#define LUA_VERSION_RELEASE_NUM (LUA_VERSION_NUM * 100 + 4) #define LUA_VERSION "Lua " LUA_VERSION_MAJOR "." LUA_VERSION_MINOR #define LUA_RELEASE LUA_VERSION "." LUA_VERSION_RELEASE -#define LUA_COPYRIGHT LUA_RELEASE " Copyright (C) 1994-2020 Lua.org, PUC-Rio" +#define LUA_COPYRIGHT LUA_RELEASE " Copyright (C) 1994-2022 Lua.org, PUC-Rio" #define LUA_AUTHORS "R. Ierusalimschy, L. H. de Figueiredo, W. Celes" @@ -347,7 +347,8 @@ LUA_API size_t (lua_stringtonumber) (lua_State *L, const char *s); LUA_API lua_Alloc (lua_getallocf) (lua_State *L, void **ud); LUA_API void (lua_setallocf) (lua_State *L, lua_Alloc f, void *ud); -LUA_API void (lua_toclose) (lua_State *L, int idx); +LUA_API void (lua_toclose) (lua_State *L, int idx); +LUA_API void (lua_closeslot) (lua_State *L, int idx); /* @@ -491,7 +492,7 @@ struct lua_Debug { /****************************************************************************** -* Copyright (C) 1994-2020 Lua.org, PUC-Rio. +* Copyright (C) 1994-2022 Lua.org, PUC-Rio. * * Permission is hereby granted, free of charge, to any person obtaining * a copy of this software and associated documentation files (the diff --git a/src/lua/luac.c b/src/lua/luac.c index d7d219e4..8b7e0c6e 100644 --- a/src/lua/luac.c +++ b/src/lua/luac.c @@ -155,6 +155,7 @@ static const Proto* combine(lua_State* L, int n) f->p[i]=toproto(L,i-n-1); if (f->p[i]->sizeupvalues>0) f->p[i]->upvalues[0].instack=0; } + luaM_freearray(L,f->lineinfo,f->sizelineinfo); f->sizelineinfo=0; return f; } @@ -600,11 +601,11 @@ static void PrintCode(const Proto* f) if (c==0) printf("all out"); else printf("%d out",c-1); break; case OP_TAILCALL: - printf("%d %d %d",a,b,c); + printf("%d %d %d%s",a,b,c,ISK); printf(COMMENT "%d in",b-1); break; case OP_RETURN: - printf("%d %d %d",a,b,c); + printf("%d %d %d%s",a,b,c,ISK); printf(COMMENT); if (b==0) printf("all out"); else printf("%d out",b-1); break; @@ -619,7 +620,7 @@ static void PrintCode(const Proto* f) break; case OP_FORPREP: printf("%d %d",a,bx); - printf(COMMENT "to %d",pc+bx+2); + printf(COMMENT "exit to %d",pc+bx+3); break; case OP_TFORPREP: printf("%d %d",a,bx); diff --git a/src/lua/luaconf.h b/src/lua/luaconf.h index bdf927e7..d42d14b7 100644 --- a/src/lua/luaconf.h +++ b/src/lua/luaconf.h @@ -16,13 +16,13 @@ ** =================================================================== ** General Configuration File for Lua ** -** Some definitions here can be changed externally, through the -** compiler (e.g., with '-D' options). Those are protected by -** '#if !defined' guards. However, several other definitions should -** be changed directly here, either because they affect the Lua -** ABI (by making the changes here, you ensure that all software -** connected to Lua, such as C libraries, will be compiled with the -** same configuration); or because they are seldom changed. +** Some definitions here can be changed externally, through the compiler +** (e.g., with '-D' options): They are commented out or protected +** by '#if !defined' guards. However, several other definitions +** should be changed directly here, either because they affect the +** Lua ABI (by making the changes here, you ensure that all software +** connected to Lua, such as C libraries, will be compiled with the same +** configuration); or because they are seldom changed. ** ** Search for "@@" to find all configurable definitions. ** =================================================================== @@ -36,21 +36,6 @@ ** ===================================================================== */ -/* -@@ LUAI_MAXCSTACK defines the maximum depth for nested calls and -** also limits the maximum depth of other recursive algorithms in -** the implementation, such as syntactic analysis. A value too -** large may allow the interpreter to crash (C-stack overflow). -** The default value seems ok for regular machines, but may be -** too high for restricted hardware. -** The test file 'cstack.lua' may help finding a good limit. -** (It will crash with a limit too high.) -*/ -#if !defined(LUAI_MAXCSTACK) -#define LUAI_MAXCSTACK 2000 -#endif - - /* @@ LUA_USE_C89 controls the use of non-ISO-C89 features. ** Define it if you want Lua to avoid the use of a few C99 features @@ -96,26 +81,12 @@ /* ** {================================================================== -** Configuration for Number types. +** Configuration for Number types. These options should not be +** set externally, because any other code connected to Lua must +** use the same configuration. ** =================================================================== */ -/* -@@ LUA_32BITS enables Lua with 32-bit integers and 32-bit floats. -*/ -/* #define LUA_32BITS */ - - -/* -@@ LUA_C89_NUMBERS ensures that Lua uses the largest types available for -** C89 ('long' and 'double'); Windows always has '__int64', so it does -** not need to use this case. -*/ -#if defined(LUA_USE_C89) && !defined(LUA_USE_WINDOWS) -#define LUA_C89_NUMBERS -#endif - - /* @@ LUA_INT_TYPE defines the type for Lua integers. @@ LUA_FLOAT_TYPE defines the type for Lua floats. @@ -136,7 +107,31 @@ #define LUA_FLOAT_DOUBLE 2 #define LUA_FLOAT_LONGDOUBLE 3 -#if defined(LUA_32BITS) /* { */ + +/* Default configuration ('long long' and 'double', for 64-bit Lua) */ +#define LUA_INT_DEFAULT LUA_INT_LONGLONG +#define LUA_FLOAT_DEFAULT LUA_FLOAT_DOUBLE + + +/* +@@ LUA_32BITS enables Lua with 32-bit integers and 32-bit floats. +*/ +#define LUA_32BITS 0 + + +/* +@@ LUA_C89_NUMBERS ensures that Lua uses the largest types available for +** C89 ('long' and 'double'); Windows always has '__int64', so it does +** not need to use this case. +*/ +#if defined(LUA_USE_C89) && !defined(LUA_USE_WINDOWS) +#define LUA_C89_NUMBERS 1 +#else +#define LUA_C89_NUMBERS 0 +#endif + + +#if LUA_32BITS /* { */ /* ** 32-bit integers and 'float' */ @@ -147,27 +142,22 @@ #endif #define LUA_FLOAT_TYPE LUA_FLOAT_FLOAT -#elif defined(LUA_C89_NUMBERS) /* }{ */ +#elif LUA_C89_NUMBERS /* }{ */ /* ** largest types available for C89 ('long' and 'double') */ #define LUA_INT_TYPE LUA_INT_LONG #define LUA_FLOAT_TYPE LUA_FLOAT_DOUBLE +#else /* }{ */ +/* use defaults */ + +#define LUA_INT_TYPE LUA_INT_DEFAULT +#define LUA_FLOAT_TYPE LUA_FLOAT_DEFAULT + #endif /* } */ -/* -** default configuration for 64-bit Lua ('long long' and 'double') -*/ -#if !defined(LUA_INT_TYPE) -#define LUA_INT_TYPE LUA_INT_LONGLONG -#endif - -#if !defined(LUA_FLOAT_TYPE) -#define LUA_FLOAT_TYPE LUA_FLOAT_DOUBLE -#endif - /* }================================================================== */ @@ -388,14 +378,13 @@ /* ** {================================================================== -** Configuration for Numbers. +** Configuration for Numbers (low-level part). ** Change these definitions if no predefined LUA_FLOAT_* / LUA_INT_* ** satisfy your needs. ** =================================================================== */ /* -@@ LUA_NUMBER is the floating-point type used by Lua. @@ LUAI_UACNUMBER is the result of a 'default argument promotion' @@ over a floating number. @@ l_floatatt(x) corrects float attribute 'x' to the proper float type @@ -488,10 +477,7 @@ /* -@@ LUA_INTEGER is the integer type used by Lua. -** @@ LUA_UNSIGNED is the unsigned version of LUA_INTEGER. -** @@ LUAI_UACINT is the result of a 'default argument promotion' @@ over a LUA_INTEGER. @@ LUA_INTEGER_FRMLEN is the length modifier for reading/writing integers. @@ -499,7 +485,6 @@ @@ LUA_MAXINTEGER is the maximum value for a LUA_INTEGER. @@ LUA_MININTEGER is the minimum value for a LUA_INTEGER. @@ LUA_MAXUNSIGNED is the maximum value for a LUA_UNSIGNED. -@@ LUA_UNSIGNEDBITS is the number of bits in a LUA_UNSIGNED. @@ lua_integer2str converts an integer to a string. */ @@ -520,9 +505,6 @@ #define LUA_UNSIGNED unsigned LUAI_UACINT -#define LUA_UNSIGNEDBITS (sizeof(LUA_UNSIGNED) * CHAR_BIT) - - /* now the variable definitions */ #if LUA_INT_TYPE == LUA_INT_INT /* { int */ @@ -674,6 +656,34 @@ #define lua_getlocaledecpoint() (localeconv()->decimal_point[0]) #endif + +/* +** macros to improve jump prediction, used mostly for error handling +** and debug facilities. (Some macros in the Lua API use these macros. +** Define LUA_NOBUILTIN if you do not want '__builtin_expect' in your +** code.) +*/ +#if !defined(luai_likely) + +#if defined(__GNUC__) && !defined(LUA_NOBUILTIN) +#define luai_likely(x) (__builtin_expect(((x) != 0), 1)) +#define luai_unlikely(x) (__builtin_expect(((x) != 0), 0)) +#else +#define luai_likely(x) (x) +#define luai_unlikely(x) (x) +#endif + +#endif + + +#if defined(LUA_CORE) || defined(LUA_LIB) +/* shorter names for Lua's own use */ +#define l_likely(x) luai_likely(x) +#define l_unlikely(x) luai_unlikely(x) +#endif + + + /* }================================================================== */ diff --git a/src/lua/lualib.h b/src/lua/lualib.h index e6dfbf6c..c74ae09c 100644 --- a/src/lua/lualib.h +++ b/src/lua/lualib.h @@ -51,10 +51,4 @@ LUAMOD_API int (luaopen_pihole) (lua_State *L); LUALIB_API void (luaL_openlibs) (lua_State *L); - -#if !defined(lua_assert) -#define lua_assert(x) ((void)0) -#endif - - #endif diff --git a/src/lua/lutf8lib.c b/src/lua/lutf8lib.c index 901d985f..e7bf098f 100644 --- a/src/lua/lutf8lib.c +++ b/src/lua/lutf8lib.c @@ -224,14 +224,11 @@ static int byteoffset (lua_State *L) { static int iter_aux (lua_State *L, int strict) { size_t len; const char *s = luaL_checklstring(L, 1, &len); - lua_Integer n = lua_tointeger(L, 2) - 1; - if (n < 0) /* first iteration? */ - n = 0; /* start from here */ - else if (n < (lua_Integer)len) { - n++; /* skip current byte */ - while (iscont(s + n)) n++; /* and its continuations */ + lua_Unsigned n = (lua_Unsigned)lua_tointeger(L, 2); + if (n < len) { + while (iscont(s + n)) n++; /* skip continuation bytes */ } - if (n >= (lua_Integer)len) + if (n >= len) /* (also handles original 'n' being negative) */ return 0; /* no more codepoints */ else { utfint code; diff --git a/src/lua/lvm.c b/src/lua/lvm.c index 08681af1..2ec34400 100644 --- a/src/lua/lvm.c +++ b/src/lua/lvm.c @@ -229,17 +229,17 @@ static int forprep (lua_State *L, StkId ra) { count /= l_castS2U(-(step + 1)) + 1u; } /* store the counter in place of the limit (which won't be - needed anymore */ + needed anymore) */ setivalue(plimit, l_castU2S(count)); } } else { /* try making all values floats */ lua_Number init; lua_Number limit; lua_Number step; - if (unlikely(!tonumber(plimit, &limit))) + if (l_unlikely(!tonumber(plimit, &limit))) luaG_forerror(L, plimit, "limit"); - if (unlikely(!tonumber(pstep, &step))) + if (l_unlikely(!tonumber(pstep, &step))) luaG_forerror(L, pstep, "step"); - if (unlikely(!tonumber(pinit, &init))) + if (l_unlikely(!tonumber(pinit, &init))) luaG_forerror(L, pinit, "initial value"); if (step == 0) luaG_runerror(L, "'for' step is zero"); @@ -292,7 +292,7 @@ void luaV_finishget (lua_State *L, const TValue *t, TValue *key, StkId val, if (slot == NULL) { /* 't' is not a table? */ lua_assert(!ttistable(t)); tm = luaT_gettmbyobj(L, t, TM_INDEX); - if (unlikely(notm(tm))) + if (l_unlikely(notm(tm))) luaG_typeerror(L, t, "index"); /* no metamethod */ /* else will try the metamethod */ } @@ -337,10 +337,7 @@ void luaV_finishset (lua_State *L, const TValue *t, TValue *key, lua_assert(isempty(slot)); /* slot must be empty */ tm = fasttm(L, h->metatable, TM_NEWINDEX); /* get metamethod */ if (tm == NULL) { /* no metamethod? */ - if (isabstkey(slot)) /* no previous entry? */ - slot = luaH_newkey(L, h, key); /* create one */ - /* no metamethod and (now) there is an entry with given key */ - setobj2t(L, cast(TValue *, slot), val); /* set its new value */ + luaH_finishset(L, h, key, slot, val); /* set new value */ invalidateTMcache(h); luaC_barrierback(L, obj2gco(h), val); return; @@ -349,7 +346,7 @@ void luaV_finishset (lua_State *L, const TValue *t, TValue *key, } else { /* not a table; check metamethod */ tm = luaT_gettmbyobj(L, t, TM_NEWINDEX); - if (unlikely(notm(tm))) + if (l_unlikely(notm(tm))) luaG_typeerror(L, t, "index"); } /* try the metamethod */ @@ -409,7 +406,7 @@ static int l_strcmp (const TString *ls, const TString *rs) { ** from float to int.) ** When 'f' is NaN, comparisons must result in false. */ -static int LTintfloat (lua_Integer i, lua_Number f) { +l_sinline int LTintfloat (lua_Integer i, lua_Number f) { if (l_intfitsf(i)) return luai_numlt(cast_num(i), f); /* compare them as floats */ else { /* i < f <=> i < ceil(f) */ @@ -426,7 +423,7 @@ static int LTintfloat (lua_Integer i, lua_Number f) { ** Check whether integer 'i' is less than or equal to float 'f'. ** See comments on previous function. */ -static int LEintfloat (lua_Integer i, lua_Number f) { +l_sinline int LEintfloat (lua_Integer i, lua_Number f) { if (l_intfitsf(i)) return luai_numle(cast_num(i), f); /* compare them as floats */ else { /* i <= f <=> i <= floor(f) */ @@ -443,7 +440,7 @@ static int LEintfloat (lua_Integer i, lua_Number f) { ** Check whether float 'f' is less than integer 'i'. ** See comments on previous function. */ -static int LTfloatint (lua_Number f, lua_Integer i) { +l_sinline int LTfloatint (lua_Number f, lua_Integer i) { if (l_intfitsf(i)) return luai_numlt(f, cast_num(i)); /* compare them as floats */ else { /* f < i <=> floor(f) < i */ @@ -460,7 +457,7 @@ static int LTfloatint (lua_Number f, lua_Integer i) { ** Check whether float 'f' is less than or equal to integer 'i'. ** See comments on previous function. */ -static int LEfloatint (lua_Number f, lua_Integer i) { +l_sinline int LEfloatint (lua_Number f, lua_Integer i) { if (l_intfitsf(i)) return luai_numle(f, cast_num(i)); /* compare them as floats */ else { /* f <= i <=> ceil(f) <= i */ @@ -476,7 +473,7 @@ static int LEfloatint (lua_Number f, lua_Integer i) { /* ** Return 'l < r', for numbers. */ -static int LTnum (const TValue *l, const TValue *r) { +l_sinline int LTnum (const TValue *l, const TValue *r) { lua_assert(ttisnumber(l) && ttisnumber(r)); if (ttisinteger(l)) { lua_Integer li = ivalue(l); @@ -498,7 +495,7 @@ static int LTnum (const TValue *l, const TValue *r) { /* ** Return 'l <= r', for numbers. */ -static int LEnum (const TValue *l, const TValue *r) { +l_sinline int LEnum (const TValue *l, const TValue *r) { lua_assert(ttisnumber(l) && ttisnumber(r)); if (ttisinteger(l)) { lua_Integer li = ivalue(l); @@ -571,8 +568,13 @@ int luaV_equalobj (lua_State *L, const TValue *t1, const TValue *t2) { if (ttype(t1) != ttype(t2) || ttype(t1) != LUA_TNUMBER) return 0; /* only numbers can be equal with different variants */ else { /* two numbers with different variants */ - lua_Integer i1, i2; /* compare them as integers */ - return (tointegerns(t1, &i1) && tointegerns(t2, &i2) && i1 == i2); + /* One of them is an integer. If the other does not have an + integer value, they cannot be equal; otherwise, compare their + integer values. */ + lua_Integer i1, i2; + return (luaV_tointegerns(t1, &i1, F2Ieq) && + luaV_tointegerns(t2, &i2, F2Ieq) && + i1 == i2); } } /* values have same type and same variant */ @@ -654,7 +656,7 @@ void luaV_concat (lua_State *L, int total) { /* collect total length and number of strings */ for (n = 1; n < total && tostring(L, s2v(top - n - 1)); n++) { size_t l = vslen(s2v(top - n - 1)); - if (unlikely(l >= (MAX_SIZE/sizeof(char)) - tl)) + if (l_unlikely(l >= (MAX_SIZE/sizeof(char)) - tl)) luaG_runerror(L, "string length overflow"); tl += l; } @@ -698,7 +700,7 @@ void luaV_objlen (lua_State *L, StkId ra, const TValue *rb) { } default: { /* try metamethod */ tm = luaT_gettmbyobj(L, rb, TM_LEN); - if (unlikely(notm(tm))) /* no metamethod? */ + if (l_unlikely(notm(tm))) /* no metamethod? */ luaG_typeerror(L, rb, "get length of"); break; } @@ -714,7 +716,7 @@ void luaV_objlen (lua_State *L, StkId ra, const TValue *rb) { ** otherwise 'floor(q) == trunc(q) - 1'. */ lua_Integer luaV_idiv (lua_State *L, lua_Integer m, lua_Integer n) { - if (unlikely(l_castS2U(n) + 1u <= 1u)) { /* special cases: -1 or 0 */ + if (l_unlikely(l_castS2U(n) + 1u <= 1u)) { /* special cases: -1 or 0 */ if (n == 0) luaG_runerror(L, "attempt to divide by zero"); return intop(-, 0, m); /* n==-1; avoid overflow with 0x80000...//-1 */ @@ -734,7 +736,7 @@ lua_Integer luaV_idiv (lua_State *L, lua_Integer m, lua_Integer n) { ** about luaV_idiv.) */ lua_Integer luaV_mod (lua_State *L, lua_Integer m, lua_Integer n) { - if (unlikely(l_castS2U(n) + 1u <= 1u)) { /* special cases: -1 or 0 */ + if (l_unlikely(l_castS2U(n) + 1u <= 1u)) { /* special cases: -1 or 0 */ if (n == 0) luaG_runerror(L, "attempt to perform 'n%%0'"); return 0; /* m % -1 == 0; avoid overflow with 0x80000...%-1 */ @@ -764,7 +766,8 @@ lua_Number luaV_modf (lua_State *L, lua_Number m, lua_Number n) { /* ** Shift left operation. (Shift right just negates 'y'.) */ -#define luaV_shiftr(x,y) luaV_shiftl(x,-(y)) +#define luaV_shiftr(x,y) luaV_shiftl(x,intop(-, 0, y)) + lua_Integer luaV_shiftl (lua_Integer x, lua_Integer y) { if (y < 0) { /* shift right? */ @@ -845,6 +848,19 @@ void luaV_finishOp (lua_State *L) { luaV_concat(L, total); /* concat them (may yield again) */ break; } + case OP_CLOSE: { /* yielded closing variables */ + ci->u.l.savedpc--; /* repeat instruction to close other vars. */ + break; + } + case OP_RETURN: { /* yielded closing variables */ + StkId ra = base + GETARG_A(inst); + /* adjust top to signal correct number of returns, in case the + return is "up to top" ('isIT') */ + L->top = ra + ci->u2.nres; + /* repeat instruction to close other vars. and complete the return */ + ci->u.l.savedpc--; + break; + } default: { /* only these other opcodes can yield */ lua_assert(op == OP_TFORCALL || op == OP_CALL || @@ -920,7 +936,7 @@ void luaV_finishOp (lua_State *L) { */ #define op_arithfK(L,fop) { \ TValue *v1 = vRB(i); \ - TValue *v2 = KC(i); \ + TValue *v2 = KC(i); lua_assert(ttisnumber(v2)); \ op_arithf_aux(L, v1, v2, fop); } @@ -949,7 +965,7 @@ void luaV_finishOp (lua_State *L) { */ #define op_arithK(L,iop,fop) { \ TValue *v1 = vRB(i); \ - TValue *v2 = KC(i); \ + TValue *v2 = KC(i); lua_assert(ttisnumber(v2)); \ op_arith_aux(L, v1, v2, iop, fop); } @@ -1048,7 +1064,8 @@ void luaV_finishOp (lua_State *L) { #define updatebase(ci) (base = ci->func + 1) -#define updatestack(ci) { if (trap) { updatebase(ci); ra = RA(i); } } +#define updatestack(ci) \ + { if (l_unlikely(trap)) { updatebase(ci); ra = RA(i); } } /* @@ -1092,15 +1109,11 @@ void luaV_finishOp (lua_State *L) { #define ProtectNT(exp) (savepc(L), (exp), updatetrap(ci)) /* -** Protect code that will finish the loop (returns) or can only raise -** errors. (That is, it will not return to the interpreter main loop -** after changing the stack or hooks.) +** Protect code that can only raise errors. (That is, it cannot change +** the stack or hooks.) */ #define halfProtect(exp) (savestate(L,ci), (exp)) -/* idem, but without changing the stack */ -#define halfProtectNT(exp) (savepc(L), (exp)) - /* 'c' is the limit of live values in the stack */ #define checkGC(L,c) \ { luaC_condGC(L, (savepc(L), L->top = (c)), \ @@ -1110,7 +1123,7 @@ void luaV_finishOp (lua_State *L) { /* fetch an instruction and prepare its execution */ #define vmfetch() { \ - if (trap) { /* stack reallocation or hooks? */ \ + if (l_unlikely(trap)) { /* stack reallocation or hooks? */ \ trap = luaG_traceexec(L, pc); /* handle hooks */ \ updatebase(ci); /* correct stack */ \ } \ @@ -1132,17 +1145,20 @@ void luaV_execute (lua_State *L, CallInfo *ci) { #if LUA_USE_JUMPTABLE #include "ljumptab.h" #endif - tailcall: + startfunc: trap = L->hookmask; + returning: /* trap already set */ cl = clLvalue(s2v(ci->func)); k = cl->p->k; pc = ci->u.l.savedpc; - if (trap) { - if (cl->p->is_vararg) - trap = 0; /* hooks will start after VARARGPREP instruction */ - else if (pc == cl->p->code) /* first instruction (not resuming)? */ - luaD_hookcall(L, ci); - ci->u.l.trap = 1; /* there may be other hooks */ + if (l_unlikely(trap)) { + if (pc == cl->p->code) { /* first instruction (not resuming)? */ + if (cl->p->is_vararg) + trap = 0; /* hooks will start after VARARGPREP instruction */ + else /* check 'call' hook */ + luaD_hookcall(L, ci); + } + ci->u.l.trap = 1; /* assume trap is on, for now */ } base = ci->func + 1; /* main loop of interpreter */ @@ -1150,8 +1166,12 @@ void luaV_execute (lua_State *L, CallInfo *ci) { Instruction i; /* instruction being executed */ StkId ra; /* instruction's A register */ vmfetch(); + #if 0 + /* low-level line tracing for debugging Lua */ + printf("line: %d\n", luaG_getfuncline(cl->p, pcRel(pc, cl->p))); + #endif lua_assert(base == ci->func + 1); - lua_assert(base <= L->top && L->top < L->stack + L->stacksize); + lua_assert(base <= L->top && L->top < L->stack_last); /* invalidate top for instructions not expecting it */ lua_assert(isIT(i) || (cast_void(L->top = base), 1)); vmdispatch (GET_OPCODE(i)) { @@ -1528,7 +1548,7 @@ void luaV_execute (lua_State *L, CallInfo *ci) { vmbreak; } vmcase(OP_CLOSE) { - Protect(luaF_close(L, ra, LUA_OK)); + Protect(luaF_close(L, ra, LUA_OK, 1)); vmbreak; } vmcase(OP_TBC) { @@ -1606,47 +1626,45 @@ void luaV_execute (lua_State *L, CallInfo *ci) { vmbreak; } vmcase(OP_CALL) { + CallInfo *newci; int b = GETARG_B(i); int nresults = GETARG_C(i) - 1; if (b != 0) /* fixed number of arguments? */ L->top = ra + b; /* top signals number of arguments */ /* else previous instruction set top */ - ProtectNT(luaD_call(L, ra, nresults)); + savepc(L); /* in case of errors */ + if ((newci = luaD_precall(L, ra, nresults)) == NULL) + updatetrap(ci); /* C call; nothing else to be done */ + else { /* Lua call: run function in this same C frame */ + ci = newci; + goto startfunc; + } vmbreak; } vmcase(OP_TAILCALL) { int b = GETARG_B(i); /* number of arguments + 1 (function) */ + int n; /* number of results when calling a C function */ int nparams1 = GETARG_C(i); - /* delat is virtual 'func' - real 'func' (vararg functions) */ + /* delta is virtual 'func' - real 'func' (vararg functions) */ int delta = (nparams1) ? ci->u.l.nextraargs + nparams1 : 0; if (b != 0) L->top = ra + b; else /* previous instruction set top */ b = cast_int(L->top - ra); - savepc(ci); /* some calls here can raise errors */ + savepc(ci); /* several calls here can raise errors */ if (TESTARG_k(i)) { - /* close upvalues from current call; the compiler ensures - that there are no to-be-closed variables here, so this - call cannot change the stack */ - luaF_close(L, base, NOCLOSINGMETH); + luaF_closeupval(L, base); /* close upvalues from current call */ + lua_assert(L->tbclist < base); /* no pending tbc variables */ lua_assert(base == ci->func + 1); } - while (!ttisfunction(s2v(ra))) { /* not a function? */ - luaD_tryfuncTM(L, ra); /* try '__call' metamethod */ - b++; /* there is now one extra argument */ - checkstackGCp(L, 1, ra); + if ((n = luaD_pretailcall(L, ci, ra, b, delta)) < 0) /* Lua function? */ + goto startfunc; /* execute the callee */ + else { /* C function? */ + ci->func -= delta; /* restore 'func' (if vararg) */ + luaD_poscall(L, ci, n); /* finish caller */ + updatetrap(ci); /* 'luaD_poscall' can change hooks */ + goto ret; /* caller returns after the tail call */ } - if (!ttisLclosure(s2v(ra))) { /* C function? */ - luaD_call(L, ra, LUA_MULTRET); /* call it */ - updatetrap(ci); - updatestack(ci); /* stack may have been relocated */ - ci->func -= delta; - luaD_poscall(L, ci, cast_int(L->top - ra)); - return; - } - ci->func -= delta; - luaD_pretailcall(L, ci, ra, b); /* prepare call frame */ - goto tailcall; } vmcase(OP_RETURN) { int n = GETARG_B(i) - 1; /* number of results */ @@ -1655,9 +1673,10 @@ void luaV_execute (lua_State *L, CallInfo *ci) { n = cast_int(L->top - ra); /* get what is available */ savepc(ci); if (TESTARG_k(i)) { /* may there be open upvalues? */ + ci->u2.nres = n; /* save number of returns */ if (L->top < ci->top) L->top = ci->top; - luaF_close(L, base, LUA_OK); + luaF_close(L, base, CLOSEKTOP, 1); updatetrap(ci); updatestack(ci); } @@ -1665,26 +1684,31 @@ void luaV_execute (lua_State *L, CallInfo *ci) { ci->func -= ci->u.l.nextraargs + nparams1; L->top = ra + n; /* set call for 'luaD_poscall' */ luaD_poscall(L, ci, n); - return; + updatetrap(ci); /* 'luaD_poscall' can change hooks */ + goto ret; } vmcase(OP_RETURN0) { - if (L->hookmask) { + if (l_unlikely(L->hookmask)) { L->top = ra; - halfProtectNT(luaD_poscall(L, ci, 0)); /* no hurry... */ + savepc(ci); + luaD_poscall(L, ci, 0); /* no hurry... */ + trap = 1; } else { /* do the 'poscall' here */ - int nres = ci->nresults; + int nres; L->ci = ci->previous; /* back to caller */ L->top = base - 1; - while (nres-- > 0) + for (nres = ci->nresults; l_unlikely(nres > 0); nres--) setnilvalue(s2v(L->top++)); /* all results are nil */ } - return; + goto ret; } vmcase(OP_RETURN1) { - if (L->hookmask) { + if (l_unlikely(L->hookmask)) { L->top = ra + 1; - halfProtectNT(luaD_poscall(L, ci, 1)); /* no hurry... */ + savepc(ci); + luaD_poscall(L, ci, 1); /* no hurry... */ + trap = 1; } else { /* do the 'poscall' here */ int nres = ci->nresults; @@ -1694,11 +1718,17 @@ void luaV_execute (lua_State *L, CallInfo *ci) { else { setobjs2s(L, base - 1, ra); /* at least this result */ L->top = base; - while (--nres > 0) /* complete missing results */ - setnilvalue(s2v(L->top++)); + for (; l_unlikely(nres > 1); nres--) + setnilvalue(s2v(L->top++)); /* complete missing results */ } } - return; + ret: /* return from a Lua function */ + if (ci->callstatus & CIST_FRESH) + return; /* end this frame */ + else { + ci = ci->previous; + goto returning; /* continue running caller in this frame */ + } } vmcase(OP_FORLOOP) { if (ttisinteger(s2v(ra + 2))) { /* integer loop? */ @@ -1792,7 +1822,7 @@ void luaV_execute (lua_State *L, CallInfo *ci) { } vmcase(OP_VARARGPREP) { ProtectNT(luaT_adjustvarargs(L, GETARG_A(i), ci, cl->p)); - if (trap) { + if (l_unlikely(trap)) { /* previous "Protect" updated trap */ luaD_hookcall(L, ci); L->oldpc = 1; /* next opcode will be seen as a "new" line */ } diff --git a/src/lua/lvm.h b/src/lua/lvm.h index 2d4ac160..1bc16f3a 100644 --- a/src/lua/lvm.h +++ b/src/lua/lvm.h @@ -60,12 +60,14 @@ typedef enum { /* convert an object to an integer (including string coercion) */ #define tointeger(o,i) \ - (ttisinteger(o) ? (*(i) = ivalue(o), 1) : luaV_tointeger(o,i,LUA_FLOORN2I)) + (l_likely(ttisinteger(o)) ? (*(i) = ivalue(o), 1) \ + : luaV_tointeger(o,i,LUA_FLOORN2I)) /* convert an object to an integer (without string coercion) */ #define tointegerns(o,i) \ - (ttisinteger(o) ? (*(i) = ivalue(o), 1) : luaV_tointegerns(o,i,LUA_FLOORN2I)) + (l_likely(ttisinteger(o)) ? (*(i) = ivalue(o), 1) \ + : luaV_tointegerns(o,i,LUA_FLOORN2I)) #define intop(op,v1,v2) l_castU2S(l_castS2U(v1) op l_castS2U(v2)) From d626c0d6c7e002532a95dfe6b031d4c2bf8cd74e Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 25 Nov 2022 19:18:47 +0100 Subject: [PATCH 195/218] Add missing newline in front of LUA details in pihole-FTL -vv Signed-off-by: DL6ER --- src/args.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/args.c b/src/args.c index 439edd56..877e064a 100644 --- a/src/args.c +++ b/src/args.c @@ -261,7 +261,7 @@ void parse_args(int argc, char* argv[]) printf(" "); printf("%s", opt); } - printf("\n"); + printf("\n\n"); printf("******************************** LUA ********************************\n"); printf(LUA_COPYRIGHT"\n"); printf("\n"); From 4fdf3d9ab437c3c1da22ff5c3306b91acdaf9cd8 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 26 Nov 2022 18:13:33 +0100 Subject: [PATCH 196/218] Re-add /*** Pi-hole modification ***/ comments Signed-off-by: DL6ER --- patch/lua/0001-add-pihole-library.patch | 28 ++++++++++++++++++------- src/lua/linit.c | 2 ++ src/lua/lua.c | 8 +++++++ src/lua/luac.c | 2 ++ src/lua/lualib.h | 2 ++ 5 files changed, 35 insertions(+), 7 deletions(-) diff --git a/patch/lua/0001-add-pihole-library.patch b/patch/lua/0001-add-pihole-library.patch index 9b3173c0..30a0f592 100644 --- a/patch/lua/0001-add-pihole-library.patch +++ b/patch/lua/0001-add-pihole-library.patch @@ -2,11 +2,13 @@ diff --git a/src/lua/linit.c b/src/lua/linit.c index 69808f84..83b89555 100644 --- a/src/lua/linit.c +++ b/src/lua/linit.c -@@ -50,6 +50,7 @@ static const luaL_Reg loadedlibs[] = { +@@ -50,6 +50,9 @@ static const luaL_Reg loadedlibs[] = { {LUA_MATHLIBNAME, luaopen_math}, {LUA_UTF8LIBNAME, luaopen_utf8}, {LUA_DBLIBNAME, luaopen_debug}, ++ /****** Pi-hole modification ******/ + {LUA_PIHOLELIBNAME, luaopen_pihole}, ++ /**********************************/ {NULL, NULL} }; @@ -14,41 +16,49 @@ diff --git a/src/lua/lua.c b/src/lua/lua.c index 454ce12f..a363925c 100644 --- a/src/lua/lua.c +++ b/src/lua/lua.c -@@ -20,6 +20,8 @@ +@@ -20,6 +20,10 @@ #include "lauxlib.h" #include "lualib.h" ++/** Pi-hole modification **/ +#include "ftl_lua.h" ++/**************************/ + #if !defined(LUA_PROGNAME) #define LUA_PROGNAME "lua" -@@ -190,7 +192,7 @@ static int dostring (lua_State *L, const char *s, const char *name) { +@@ -190,7 +194,9 @@ static int dostring (lua_State *L, const char *s, const char *name) { /* ** Receives 'globname[=modname]' and runs 'globname = require(modname)'. */ -static int dolibrary (lua_State *L, char *globname) { ++/************** Pi-hole modification ***************/ +int dolibrary (lua_State *L, char *globname) { ++/***************************************************/ int status; char *modname = strchr(globname, '='); if (modname == NULL) /* no explicit name? */ -@@ -597,6 +599,10 @@ static int pmain (lua_State *L) { +@@ -597,6 +599,12 @@ static int pmain (lua_State *L) { if (handle_luainit(L) != LUA_OK) /* run LUA_INIT */ return 0; /* error running LUA_INIT */ } + ++ /************** Pi-hole modification ***************/ + // Load and enable libraries bundled with Pi-hole + ftl_lua_init(L); ++ /***************************************************/ + if (!runargs(L, argv, script)) /* execute arguments -e and -l */ return 0; /* something failed */ if (script < argc && /* execute main script (if there is one) */ -@@ -616,7 +622,7 @@ static int pmain (lua_State *L) { +@@ -616,7 +622,9 @@ static int pmain (lua_State *L) { } -int main (int argc, char **argv) { ++/******* Pi-hole modification ********/ +int lua_main (int argc, char **argv) { ++/*************************************/ int status, result; lua_State *L = luaL_newstate(); /* create state */ if (L == NULL) { @@ -56,12 +66,14 @@ diff --git a/src/lua/luac.c b/src/lua/luac.c index 56ddc414..d7d219e4 100644 --- a/src/lua/luac.c +++ b/src/lua/luac.c -@@ -194,7 +194,7 @@ static int pmain(lua_State* L) +@@ -194,7 +194,9 @@ static int pmain(lua_State* L) return 0; } -int main(int argc, char* argv[]) ++/******* Pi-hole modification ********/ +int luac_main(int argc, char* argv[]) ++/*************************************/ { lua_State* L; int i=doargs(argc,argv); @@ -69,12 +81,14 @@ diff --git a/src/lua/lualib.h b/src/lua/lualib.h index eb08b530..e6dfbf6c 100644 --- a/src/lua/lualib.h +++ b/src/lua/lualib.h -@@ -44,6 +44,8 @@ LUAMOD_API int (luaopen_debug) (lua_State *L); +@@ -44,6 +44,10 @@ LUAMOD_API int (luaopen_debug) (lua_State *L); #define LUA_LOADLIBNAME "package" LUAMOD_API int (luaopen_package) (lua_State *L); ++/************ Pi-hole modification *************/ +#define LUA_PIHOLELIBNAME "pihole" +LUAMOD_API int (luaopen_pihole) (lua_State *L); ++/***********************************************/ /* open all previous libraries */ LUALIB_API void (luaL_openlibs) (lua_State *L); diff --git a/src/lua/linit.c b/src/lua/linit.c index 83b89555..9a5bcfdc 100644 --- a/src/lua/linit.c +++ b/src/lua/linit.c @@ -50,7 +50,9 @@ static const luaL_Reg loadedlibs[] = { {LUA_MATHLIBNAME, luaopen_math}, {LUA_UTF8LIBNAME, luaopen_utf8}, {LUA_DBLIBNAME, luaopen_debug}, + /****** Pi-hole modification ******/ {LUA_PIHOLELIBNAME, luaopen_pihole}, + /**********************************/ {NULL, NULL} }; diff --git a/src/lua/lua.c b/src/lua/lua.c index 4d5274fa..03815a05 100644 --- a/src/lua/lua.c +++ b/src/lua/lua.c @@ -20,7 +20,9 @@ #include "lauxlib.h" #include "lualib.h" +/** Pi-hole modification **/ #include "ftl_lua.h" +/**************************/ #if !defined(LUA_PROGNAME) @@ -212,7 +214,9 @@ static int dostring (lua_State *L, const char *s, const char *name) { /* ** Receives 'globname[=modname]' and runs 'globname = require(modname)'. */ +/************** Pi-hole modification ***************/ int dolibrary (lua_State *L, char *globname) { +/***************************************************/ int status; char *modname = strchr(globname, '='); if (modname == NULL) /* no explicit name? */ @@ -631,8 +635,10 @@ static int pmain (lua_State *L) { return 0; /* error running LUA_INIT */ } + /************** Pi-hole modification ***************/ // Load and enable libraries bundled with Pi-hole ftl_lua_init(L); + /***************************************************/ if (!runargs(L, argv, script)) /* execute arguments -e and -l */ return 0; /* something failed */ @@ -653,7 +659,9 @@ static int pmain (lua_State *L) { } +/******* Pi-hole modification ********/ int lua_main (int argc, char **argv) { +/*************************************/ int status, result; lua_State *L = luaL_newstate(); /* create state */ if (L == NULL) { diff --git a/src/lua/luac.c b/src/lua/luac.c index 8b7e0c6e..6b556b26 100644 --- a/src/lua/luac.c +++ b/src/lua/luac.c @@ -195,7 +195,9 @@ static int pmain(lua_State* L) return 0; } +/******* Pi-hole modification ********/ int luac_main(int argc, char* argv[]) +/*************************************/ { lua_State* L; int i=doargs(argc,argv); diff --git a/src/lua/lualib.h b/src/lua/lualib.h index c74ae09c..a7cab4eb 100644 --- a/src/lua/lualib.h +++ b/src/lua/lualib.h @@ -44,8 +44,10 @@ LUAMOD_API int (luaopen_debug) (lua_State *L); #define LUA_LOADLIBNAME "package" LUAMOD_API int (luaopen_package) (lua_State *L); +/************ Pi-hole modification *************/ #define LUA_PIHOLELIBNAME "pihole" LUAMOD_API int (luaopen_pihole) (lua_State *L); +/***********************************************/ /* open all previous libraries */ LUALIB_API void (luaL_openlibs) (lua_State *L); From 208a1824b13e3c8266b173f1782525b2728f4345 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Men=C5=A1=C3=ADk?= Date: Sat, 26 Nov 2022 18:49:21 +0000 Subject: [PATCH 197/218] Fix use-after-free in mark_servers() Signed-off-by: DL6ER --- src/dnsmasq/domain-match.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/dnsmasq/domain-match.c b/src/dnsmasq/domain-match.c index bef460a1..fe8e25a3 100644 --- a/src/dnsmasq/domain-match.c +++ b/src/dnsmasq/domain-match.c @@ -559,7 +559,7 @@ static int maybe_free_servers = 0; /* Must be called before add_update_server() to set daemon->servers_tail */ void mark_servers(int flag) { - struct server *serv, **up; + struct server *serv, *next, **up; maybe_free_servers = !!flag; @@ -580,11 +580,13 @@ void mark_servers(int flag) 1) numerous and 2) not reloaded often. We just delete and recreate. */ if (flag) - for (serv = daemon->local_domains, up = &daemon->local_domains; serv; serv = serv->next) + for (serv = daemon->local_domains, up = &daemon->local_domains; serv; serv = next) { + next = serv->next; + if (serv->flags & flag) { - *up = serv->next; + *up = next; free(serv->domain); free(serv); } From 1f2ef8c037bf1accd85e51953b8e00ec894f39a4 Mon Sep 17 00:00:00 2001 From: Dominik Derigs Date: Sat, 26 Nov 2022 21:18:34 +0000 Subject: [PATCH 198/218] Make max staleness of stale cache entries configurable and default to one day. Signed-off-by: DL6ER --- src/dnsmasq/cache.c | 16 ++++++++++++---- src/dnsmasq/config.h | 1 + src/dnsmasq/dnsmasq.h | 6 +++--- src/dnsmasq/option.c | 22 ++++++++++++++++++++-- 4 files changed, 36 insertions(+), 9 deletions(-) diff --git a/src/dnsmasq/cache.c b/src/dnsmasq/cache.c index 5b3ceb43..36cc15d8 100644 --- a/src/dnsmasq/cache.c +++ b/src/dnsmasq/cache.c @@ -381,9 +381,17 @@ static int is_outdated_cname_pointer(struct crec *crecp) static int is_expired(time_t now, struct crec *crecp) { - /* Don't dump expired entries if we're using them, cache becomes strictly LRU in that case. - Never use expired DS or DNSKEY entries. */ - if (option_bool(OPT_STALE_CACHE) && !(crecp->flags & (F_DS | F_DNSKEY))) + /* Don't dump expired entries if they are within the accepted timeout range. + The cache becomes approx. LRU. Never use expired DS or DNSKEY entries. + Possible values for daemon->cache_max_expiry: + -1 == serve cached content regardless how long ago it expired + 0 == the option is disabled, expired content isn't served + == serve cached content only if it expire less than seconds + ago (where n is a positive integer) */ + if (daemon->cache_max_expiry != 0 && + (daemon->cache_max_expiry == -1 || + difftime(now, crecp->ttd) < daemon->cache_max_expiry) && + !(crecp->flags & (F_DS | F_DNSKEY))) return 0; if (crecp->flags & F_IMMORTAL) @@ -1802,7 +1810,7 @@ void dump_cache(time_t now) daemon->cachesize, daemon->metrics[METRIC_DNS_CACHE_LIVE_FREED], daemon->metrics[METRIC_DNS_CACHE_INSERTED]); my_syslog(LOG_INFO, _("queries forwarded %u, queries answered locally %u"), daemon->metrics[METRIC_DNS_QUERIES_FORWARDED], daemon->metrics[METRIC_DNS_LOCAL_ANSWERED]); - if (option_bool(OPT_STALE_CACHE)) + if (daemon->cache_max_expiry != 0) my_syslog(LOG_INFO, _("queries answered from stale cache %u"), daemon->metrics[METRIC_DNS_STALE_ANSWERED]); #ifdef HAVE_AUTH my_syslog(LOG_INFO, _("queries for authoritative zones %u"), daemon->metrics[METRIC_DNS_AUTH_ANSWERED]); diff --git a/src/dnsmasq/config.h b/src/dnsmasq/config.h index 2bb709ad..3ee9be1f 100644 --- a/src/dnsmasq/config.h +++ b/src/dnsmasq/config.h @@ -62,6 +62,7 @@ #define LOOP_TEST_DOMAIN "test" /* domain for loop testing, "test" is reserved by RFC 2606 and won't therefore clash */ #define LOOP_TEST_TYPE T_TXT #define DEFAULT_FAST_RETRY 1000 /* ms, default delay before fast retry */ +#define STALE_CACHE_EXPIRY 86400 /* 1 day in secs, default maximum expiry time for stale cache data */ /* compile-time options: uncomment below to enable or do eg. make COPTS=-DHAVE_BROKEN_RTC diff --git a/src/dnsmasq/dnsmasq.h b/src/dnsmasq/dnsmasq.h index f91729cb..9c9e825f 100644 --- a/src/dnsmasq/dnsmasq.h +++ b/src/dnsmasq/dnsmasq.h @@ -280,9 +280,8 @@ struct event_desc { #define OPT_FILTER_AAAA 68 #define OPT_STRIP_ECS 69 #define OPT_STRIP_MAC 70 -#define OPT_STALE_CACHE 71 -#define OPT_NORR 72 -#define OPT_LAST 73 +#define OPT_NORR 71 +#define OPT_LAST 72 #define OPTION_BITS (sizeof(unsigned int)*8) #define OPTION_SIZE ( (OPT_LAST/OPTION_BITS)+((OPT_LAST%OPTION_BITS)!=0) ) @@ -1206,6 +1205,7 @@ extern struct daemon { unsigned long soa_sn, soa_refresh, soa_retry, soa_expiry; u32 metrics[__METRIC_MAX]; int fast_retry_time, fast_retry_timeout; + int cache_max_expiry; #ifdef HAVE_DNSSEC struct ds_config *ds; char *timestamp_file; diff --git a/src/dnsmasq/option.c b/src/dnsmasq/option.c index d35745e5..0c61a22f 100644 --- a/src/dnsmasq/option.c +++ b/src/dnsmasq/option.c @@ -377,7 +377,7 @@ static const struct myoption opts[] = { "quiet-tftp", 0, 0, LOPT_QUIET_TFTP }, { "port-limit", 1, 0, LOPT_RANDPORT_LIM }, { "fast-dns-retry", 2, 0, LOPT_FAST_RETRY }, - { "use-stale-cache", 0, 0 , LOPT_STALE_CACHE }, + { "use-stale-cache", 2, 0 , LOPT_STALE_CACHE }, { NULL, 0, 0, 0 } }; @@ -435,7 +435,7 @@ static struct { { 'M', ARG_DUP, "", gettext_noop("Specify BOOTP options to DHCP server."), NULL }, { 'n', OPT_NO_POLL, NULL, gettext_noop("Do NOT poll %s file, reload only on SIGHUP."), RESOLVFILE }, { 'N', OPT_NO_NEG, NULL, gettext_noop("Do NOT cache failed search results."), NULL }, - { LOPT_STALE_CACHE, OPT_STALE_CACHE, NULL, gettext_noop("Use expired cache data for faster reply."), NULL }, + { LOPT_STALE_CACHE, ARG_ONE, "[=]", gettext_noop("Use expired cache data for faster reply."), NULL }, { 'o', OPT_ORDER, NULL, gettext_noop("Use nameservers strictly in the order given in %s."), RESOLVFILE }, { 'O', ARG_DUP, "", gettext_noop("Specify options to be sent to DHCP clients."), NULL }, { LOPT_FORCE, ARG_DUP, "", gettext_noop("DHCP option sent even if the client does not request it."), NULL}, @@ -5155,6 +5155,24 @@ err: break; } + case LOPT_STALE_CACHE: + { + int max_expiry = STALE_CACHE_EXPIRY; + if (arg) + { + /* Don't accept negative TTLs here, they'd have the counter-intuitive + side-effect of evicting cache records before they expire */ + if (!atoi_check(arg, &max_expiry) || max_expiry < 0) + ret_err(gen_err); + /* Store "serve expired forever" as -1 internally, the option isn't + active for daemon->cache_max_expiry == 0 */ + if (max_expiry == 0) + max_expiry = -1; + } + daemon->cache_max_expiry = max_expiry; + break; + } + #ifdef HAVE_DNSSEC case LOPT_DNSSEC_STAMP: /* --dnssec-timestamp */ daemon->timestamp_file = opt_string_alloc(arg); From c81b15ee657fddb4f8fb11cb154e8e18092cbad0 Mon Sep 17 00:00:00 2001 From: Brad Smith Date: Sat, 26 Nov 2022 21:48:17 +0000 Subject: [PATCH 199/218] Fix warning in cache.c Signed-off-by: DL6ER --- src/dnsmasq/cache.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/dnsmasq/cache.c b/src/dnsmasq/cache.c index 36cc15d8..89c40f82 100644 --- a/src/dnsmasq/cache.c +++ b/src/dnsmasq/cache.c @@ -1937,8 +1937,10 @@ void dump_cache(time_t now) char *record_source(unsigned int index) { struct hostsfile *ah; +#ifdef HAVE_INOTIFY struct dyndir *dd; - +#endif + if (index == SRC_CONFIG) return "config"; else if (index == SRC_HOSTS) From 9bb96b5b9377a3ea25dcef80ce992e0a712190e8 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 27 Nov 2022 10:47:03 +0100 Subject: [PATCH 200/218] Undo Pi-hole modification in src/dnsmasq/forward.c Signed-off-by: DL6ER --- src/dnsmasq/forward.c | 22 +--------------------- src/dnsmasq/rfc1035.c | 1 - 2 files changed, 1 insertion(+), 22 deletions(-) diff --git a/src/dnsmasq/forward.c b/src/dnsmasq/forward.c index 986bdc58..58b0b433 100644 --- a/src/dnsmasq/forward.c +++ b/src/dnsmasq/forward.c @@ -836,27 +836,7 @@ static size_t process_reply(struct dns_header *header, time_t now, struct server n = rrfilter(header, n, RRFILTER_AAAA); } - /******************************** Pi-hole modification ********************************/ - int ret = extract_addresses(header, n, daemon->namebuff, now, ipsets, nftsets, is_sign, check_rebind, no_cache, cache_secure, &doctored); - if (ret == 2) - { - cache_secure = 0; - // Make a private copy of the pheader to ensure - // we are not accidentially rewriting what is in - // the pheader when we're creating a crafted reply - // further below (when a query is to be blocked) - if (pheader) - { - pheader_copy = calloc(1, plen); - memcpy(pheader_copy, pheader, plen); - } - - // Generate DNS packet for reply, a possibly existing pseudo header - // will be restored later inside resize_packet() - n = FTL_make_answer(header, ((char *) header) + 65536, n, &ede); - } - else if(ret) - /**************************************************************************************/ + if (extract_addresses(header, n, daemon->namebuff, now, ipsets, nftsets, is_sign, check_rebind, no_cache, cache_secure, &doctored)) { my_syslog(LOG_WARNING, _("possible DNS-rebind attack detected: %s"), daemon->namebuff); munged = 1; diff --git a/src/dnsmasq/rfc1035.c b/src/dnsmasq/rfc1035.c index 3a0c03af..0237d3d3 100644 --- a/src/dnsmasq/rfc1035.c +++ b/src/dnsmasq/rfc1035.c @@ -540,7 +540,6 @@ static int print_txt(struct dns_header *header, const size_t qlen, char *name, either because of lack of memory, or lack of SOA records. These are treated by the cache code as expired and cleaned out that way. Return 1 if we reject an address because it look like part of dns-rebinding attack. */ -// Pi-hole: Return 2 if we reject a part of a CNAME chain int extract_addresses(struct dns_header *header, size_t qlen, char *name, time_t now, struct ipsets *ipsets, struct ipsets *nftsets, int is_sign, int check_rebind, int no_cache_dnssec, int secure, int *doctored) From 4650c3d9f5a26f59b2368ea9b89155e6d45b680b Mon Sep 17 00:00:00 2001 From: Simon Kelley Date: Sat, 26 Nov 2022 22:19:29 +0000 Subject: [PATCH 201/218] Handle malformed DNS replies better. If we detect that that reply from usptream is malformed, transform it into a SERVFAIL reply before sending to the original requestor. Signed-off-by: DL6ER --- src/dnsmasq/forward.c | 12 +++++++++++- src/dnsmasq/rfc1035.c | 34 ++++++++++++++++++---------------- 2 files changed, 29 insertions(+), 17 deletions(-) diff --git a/src/dnsmasq/forward.c b/src/dnsmasq/forward.c index 58b0b433..76c7c1a1 100644 --- a/src/dnsmasq/forward.c +++ b/src/dnsmasq/forward.c @@ -836,12 +836,22 @@ static size_t process_reply(struct dns_header *header, time_t now, struct server n = rrfilter(header, n, RRFILTER_AAAA); } - if (extract_addresses(header, n, daemon->namebuff, now, ipsets, nftsets, is_sign, check_rebind, no_cache, cache_secure, &doctored)) + switch (extract_addresses(header, n, daemon->namebuff, now, ipsets, nftsets, is_sign, check_rebind, no_cache, cache_secure, &doctored)) { + case 1: my_syslog(LOG_WARNING, _("possible DNS-rebind attack detected: %s"), daemon->namebuff); munged = 1; cache_secure = 0; ede = EDE_BLOCKED; + break; + + /* extract_addresses() found a malformed answer. */ + case 2: + munged = 1; + SET_RCODE(header, SERVFAIL); + cache_secure = 0; + ede = EDE_OTHER; + break; } if (doctored) diff --git a/src/dnsmasq/rfc1035.c b/src/dnsmasq/rfc1035.c index 0237d3d3..80aca3fb 100644 --- a/src/dnsmasq/rfc1035.c +++ b/src/dnsmasq/rfc1035.c @@ -539,7 +539,9 @@ static int print_txt(struct dns_header *header, const size_t qlen, char *name, /* Note that the following code can create CNAME chains that don't point to a real record, either because of lack of memory, or lack of SOA records. These are treated by the cache code as expired and cleaned out that way. - Return 1 if we reject an address because it look like part of dns-rebinding attack. */ + Return 1 if we reject an address because it look like part of dns-rebinding attack. + Return 2 if the packet is malformed. +*/ int extract_addresses(struct dns_header *header, size_t qlen, char *name, time_t now, struct ipsets *ipsets, struct ipsets *nftsets, int is_sign, int check_rebind, int no_cache_dnssec, int secure, int *doctored) @@ -590,7 +592,7 @@ int extract_addresses(struct dns_header *header, size_t qlen, char *name, time_t namep = p = (unsigned char *)(header+1); if (ntohs(header->qdcount) != 1 || !extract_name(header, qlen, &p, name, 1, 4)) - return 0; /* bad packet */ + return 2; /* bad packet */ GETSHORT(qtype, p); GETSHORT(qclass, p); @@ -608,13 +610,13 @@ int extract_addresses(struct dns_header *header, size_t qlen, char *name, time_t { cname_loop: if (!(p1 = skip_questions(header, qlen))) - return 0; + return 2; for (j = 0; j < ntohs(header->ancount); j++) { int secflag = 0; if (!(res = extract_name(header, qlen, &p1, name, 0, 10))) - return 0; /* bad packet */ + return 2; /* bad packet */ GETSHORT(aqtype, p1); GETSHORT(aqclass, p1); @@ -651,7 +653,7 @@ int extract_addresses(struct dns_header *header, size_t qlen, char *name, time_t log_query(secflag | F_CNAME | F_FORWARD | F_UPSTREAM, name, NULL, NULL, 0); if (!extract_name(header, qlen, &p1, name, 1, 0)) - return 0; + return 2; if (aqtype == T_CNAME) { @@ -677,7 +679,7 @@ int extract_addresses(struct dns_header *header, size_t qlen, char *name, time_t p1 = endrr; if (!CHECK_LEN(header, p1, qlen, 0)) - return 0; /* bad packet */ + return 2; /* bad packet */ } } @@ -722,14 +724,14 @@ int extract_addresses(struct dns_header *header, size_t qlen, char *name, time_t cname_loop1: if (!(p1 = skip_questions(header, qlen))) - return 0; + return 2; for (j = 0; j < ntohs(header->ancount); j++) { int secflag = 0; if (!(res = extract_name(header, qlen, &p1, name, 0, 10))) - return 0; /* bad packet */ + return 2; /* bad packet */ GETSHORT(aqtype, p1); GETSHORT(aqclass, p1); @@ -747,7 +749,7 @@ int extract_addresses(struct dns_header *header, size_t qlen, char *name, time_t { p1 = endrr; if (!CHECK_LEN(header, p1, qlen, 0)) - return 0; /* bad packet */ + return 2; /* bad packet */ continue; } @@ -791,7 +793,7 @@ int extract_addresses(struct dns_header *header, size_t qlen, char *name, time_t namep = p1; if (!extract_name(header, qlen, &p1, name, 1, 0)) - return 0; + return 2; // ****************************** Pi-hole modification ****************************** const char *src = cpp != NULL ? cpp->flags & F_BIGNAME ? cpp->name.bname->name : cpp->name.sname : NULL; @@ -825,25 +827,25 @@ int extract_addresses(struct dns_header *header, size_t qlen, char *name, time_t unsigned char *tmp = namep; if (!CHECK_LEN(header, p1, qlen, 6)) - return 0; /* bad packet */ + return 2; /* bad packet */ GETSHORT(addr.srv.priority, p1); GETSHORT(addr.srv.weight, p1); GETSHORT(addr.srv.srvport, p1); if (!extract_name(header, qlen, &p1, name, 1, 0)) - return 0; + return 2; addr.srv.targetlen = strlen(name) + 1; /* include terminating zero */ if (!(addr.srv.target = blockdata_alloc(name, addr.srv.targetlen))) return 0; /* we overwrote the original name, so get it back here. */ if (!extract_name(header, qlen, &tmp, name, 1, 0)) - return 0; + return 2; } else if (flags & (F_IPV4 | F_IPV6)) { /* copy address into aligned storage */ if (!CHECK_LEN(header, p1, qlen, addrlen)) - return 0; /* bad packet */ + return 2; /* bad packet */ memcpy(&addr, p1, addrlen); /* check for returned address in private space */ @@ -887,7 +889,7 @@ int extract_addresses(struct dns_header *header, size_t qlen, char *name, time_t if (aqtype == T_TXT) { if (!print_txt(header, qlen, name, p1, ardlen, secflag)) - return 0; + return 2; } else log_query(flags | F_FORWARD | secflag | F_UPSTREAM, name, &addr, NULL, aqtype); @@ -895,7 +897,7 @@ int extract_addresses(struct dns_header *header, size_t qlen, char *name, time_t p1 = endrr; if (!CHECK_LEN(header, p1, qlen, 0)) - return 0; /* bad packet */ + return 2; /* bad packet */ } if (!found && (qtype != T_ANY || (flags & F_NXDOMAIN))) From f72bbc9e0ebd0794fcdfe3e7426de2bfe6c7b9dc Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 27 Nov 2022 11:03:28 +0100 Subject: [PATCH 202/218] Re-install Pi-hole modifications to src/dnsmasq/forward.c and src/dnsmasq/rfc1035.c Signed-off-by: DL6ER --- src/dnsmasq/forward.c | 18 ++++++++++++++++++ src/dnsmasq/rfc1035.c | 3 ++- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/src/dnsmasq/forward.c b/src/dnsmasq/forward.c index 76c7c1a1..c1070067 100644 --- a/src/dnsmasq/forward.c +++ b/src/dnsmasq/forward.c @@ -852,6 +852,24 @@ static size_t process_reply(struct dns_header *header, time_t now, struct server cache_secure = 0; ede = EDE_OTHER; break; + + /* Pi-hole modification */ + case 99: + cache_secure = 0; + // Make a private copy of the pheader to ensure + // we are not accidentially rewriting what is in + // the pheader when we're creating a crafted reply + // further below (when a query is to be blocked) + if (pheader) + { + pheader_copy = calloc(1, plen); + memcpy(pheader_copy, pheader, plen); + } + + // Generate DNS packet for reply, a possibly existing pseudo header + // will be restored later inside resize_packet() + n = FTL_make_answer(header, ((char *) header) + 65536, n, &ede); + break; } if (doctored) diff --git a/src/dnsmasq/rfc1035.c b/src/dnsmasq/rfc1035.c index 80aca3fb..93f6a530 100644 --- a/src/dnsmasq/rfc1035.c +++ b/src/dnsmasq/rfc1035.c @@ -541,6 +541,7 @@ static int print_txt(struct dns_header *header, const size_t qlen, char *name, expired and cleaned out that way. Return 1 if we reject an address because it look like part of dns-rebinding attack. Return 2 if the packet is malformed. + Return 3 if we reject parts of a CNAME chain (*** Pi-hole modification ***) */ int extract_addresses(struct dns_header *header, size_t qlen, char *name, time_t now, struct ipsets *ipsets, struct ipsets *nftsets, int is_sign, int check_rebind, @@ -803,7 +804,7 @@ int extract_addresses(struct dns_header *header, size_t qlen, char *name, time_t // This query is to be blocked as we found a blocked // domain while walking the CNAME path. Log to pihole.log here log_query(F_UPSTREAM, name, NULL, "blocked during CNAME inspection", 0); - return 2; + return 99; } // ********************************************************************************** if (qtype != T_CNAME) From 0766a9e4fc8ea701915669754da98a55879da1fa Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 27 Nov 2022 12:03:43 +0100 Subject: [PATCH 203/218] Update embedded dnsmasq to v2.88rc5 Signed-off-by: DL6ER --- CMakeLists.txt | 2 +- src/dnsmasq/rfc1035.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 27b799d9..7e780773 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -11,6 +11,6 @@ cmake_minimum_required(VERSION 2.8.12) project(PIHOLE_FTL C) -set(DNSMASQ_VERSION pi-hole-v2.88rc4) +set(DNSMASQ_VERSION pi-hole-v2.88rc5) add_subdirectory(src) diff --git a/src/dnsmasq/rfc1035.c b/src/dnsmasq/rfc1035.c index 93f6a530..9b5f2650 100644 --- a/src/dnsmasq/rfc1035.c +++ b/src/dnsmasq/rfc1035.c @@ -541,7 +541,7 @@ static int print_txt(struct dns_header *header, const size_t qlen, char *name, expired and cleaned out that way. Return 1 if we reject an address because it look like part of dns-rebinding attack. Return 2 if the packet is malformed. - Return 3 if we reject parts of a CNAME chain (*** Pi-hole modification ***) + Return 99 if we reject parts of a CNAME chain (*** Pi-hole modification ***) */ int extract_addresses(struct dns_header *header, size_t qlen, char *name, time_t now, struct ipsets *ipsets, struct ipsets *nftsets, int is_sign, int check_rebind, From 983544e6d390a9661592c3e294ba9620ed8dc762 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 3 Dec 2022 12:43:20 +0100 Subject: [PATCH 204/218] Fix incorrect DNSSEC-related warning during history import Signed-off-by: DL6ER --- src/database/query-table.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/database/query-table.c b/src/database/query-table.c index bf2bdaaf..7e0161c5 100644 --- a/src/database/query-table.c +++ b/src/database/query-table.c @@ -840,7 +840,7 @@ void DB_read_queries(void) { // The field has been added for database version 12 dnssec = sqlite3_column_int(stmt, 10); - if(dnssec < DNSSEC_UNSPECIFIED || dnssec >= DNSSEC_ABANDONED) + if(dnssec < DNSSEC_UNSPECIFIED || dnssec > DNSSEC_ABANDONED) { logg("DB warn: DNSSEC value %i is invalid, %lli", dnssec, (long long)queryTimeStamp); continue; From 8cacd8f5ec77bc986a8077b32f0fb43ab04f8679 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 9 Dec 2022 10:27:16 +0100 Subject: [PATCH 205/218] Adds colors (if available) to verbose version output and completely revamp the help function Signed-off-by: DL6ER --- src/args.c | 297 +++++++++++++++++++++++++----------------- src/dnsmasq/dnsmasq.c | 9 +- test/test_suite.bats | 7 +- 3 files changed, 189 insertions(+), 124 deletions(-) diff --git a/src/args.c b/src/args.c index 338dabad..10a72d43 100644 --- a/src/args.c +++ b/src/args.c @@ -35,7 +35,7 @@ // run_dhcp_discover() #include "dhcp-discover.h" // defined in dnsmasq.c -extern void print_dnsmasq_version(void); +extern void print_dnsmasq_version(const char *yellow, const char *green, const char *bold, const char *normal); // defined in database/shell.c extern int sqlite3_shell_main(int argc, char **argv); @@ -45,6 +45,86 @@ bool daemonmode = true, cli_mode = false; int argc_dnsmasq = 0; const char** argv_dnsmasq = NULL; +// Extended SGR sequence: +// +// "\x1b[%dm" +// +// where %d is one of the following values for commonly supported colors: +// +// 0: reset colors/style +// 1: bold +// 4: underline +// 30 - 37: black, red, green, yellow, blue, magenta, cyan, and white text +// 40 - 47: black, red, green, yellow, blue, magenta, cyan, and white background +// +// https://en.wikipedia.org/wiki/ANSI_escape_code#SGR +// +#define COL_NC "\x1b[0m" // normal font +#define COL_BOLD "\x1b[1m" // bold font +#define COL_ITALIC "\x1b[3m" // italic font +#define COL_ULINE "\x1b[4m" // underline font +#define COL_GREEN "\x1b[32m" // normal foreground color +#define COL_YELLOW "\x1b[33m" // normal foreground color +#define COL_GRAY "\x1b[90m" // bright foreground color +#define COL_RED "\x1b[91m" // bright foreground color +#define COL_BLUE "\x1b[94m" // bright foreground color +#define COL_PURPLE "\x1b[95m" // bright foreground color +#define COL_CYAN "\x1b[96m" // bright foreground color + +static inline bool __attribute__ ((pure)) is_term(void) +{ + // test whether STDOUT refers to a terminal + return isatty(fileno(stdout)) == 1; +} + +// Returns green [✓] +const char __attribute__ ((pure)) *cli_tick(void) +{ + return is_term() ? "["COL_GREEN"✓"COL_NC"]" : "[✓]"; +} + +// Returns red [✗] +const char __attribute__ ((pure)) *cli_cross(void) +{ + return is_term() ? "["COL_RED"✗"COL_NC"]" : "[✗]"; +} + +// Returns [i] +const char __attribute__ ((pure)) *cli_info(void) +{ + return is_term() ? COL_BOLD"[i]"COL_NC : "[i]"; +} + +// Returns [?] +const char __attribute__ ((const)) *cli_qst(void) +{ + return "[?]"; +} + +// Returns green "done!"" +const char __attribute__ ((pure)) *cli_done(void) +{ + return is_term() ? COL_GREEN"done!"COL_NC : "done!"; +} + +// Sets font to bold +const char __attribute__ ((pure)) *cli_bold(void) +{ + return is_term() ? COL_BOLD : ""; +} + +// Resets font to normal +const char __attribute__ ((pure)) *cli_normal(void) +{ + return is_term() ? COL_NC : ""; +} + +// Set color if STDOUT is a terminal +static const char __attribute__ ((pure)) *cli_color(const char *color) +{ + return is_term() ? color : ""; +} + static inline bool strEndsWith(const char *input, const char *end){ return strcmp(input + strlen(input) - strlen(end), end) == 0; } @@ -238,21 +318,30 @@ void parse_args(int argc, char* argv[]) // Extended version output if(strcmp(argv[i], "-vv") == 0) { + const char *bold = cli_bold(); + const char *normal = cli_normal(); + const char *green = cli_color(COL_GREEN); + const char *yellow = cli_color(COL_YELLOW); + // Print FTL version - printf("****************************** FTL **********************************\n"); - printf("Version: %s\n", get_FTL_version()); - printf("Branch: %s\n", GIT_BRANCH); - printf("Commit: %s (%s)\n", GIT_HASH, GIT_DATE); - printf("Architecture: %s\n", FTL_ARCH); - printf("Compiler: %s\n\n", FTL_CC); + printf("****************************** %s%sFTL%s **********************************\n", + yellow, bold, normal); + printf("Version: %s%s%s%s\n", + green, bold, get_FTL_version(), normal); + printf("Branch: " GIT_BRANCH "\n"); + printf("Commit: " GIT_HASH " (" GIT_DATE ")\n"); + printf("Architecture: " FTL_ARCH "\n"); + printf("Compiler: " FTL_CC "\n\n"); // Print dnsmasq version and compile time options - print_dnsmasq_version(); + print_dnsmasq_version(yellow, green, bold, normal); // Print SQLite3 version and compile time options - printf("****************************** SQLite3 ******************************\n"); - printf("Version: %s\n", sqlite3_libversion()); - printf("Compile options: "); + printf("****************************** %s%sSQLite3%s ******************************\n", + yellow, bold, normal); + printf("Version: %s%s%s%s\n", + green, bold, sqlite3_libversion(), normal); + printf("Features: "); unsigned int o = 0; const char *opt = NULL; while((opt = sqlite3_compileoption_get(o++)) != NULL) @@ -262,14 +351,18 @@ void parse_args(int argc, char* argv[]) printf("%s", opt); } printf("\n\n"); - printf("******************************** LUA ********************************\n"); - printf(LUA_COPYRIGHT"\n"); - printf("Embedded libraries: "); + printf("******************************** %s%sLUA%s ********************************\n", + yellow, bold, normal); + printf("Version: %s%s" LUA_VERSION_MAJOR "." LUA_VERSION_MINOR"%s\n", + green, bold, normal); + printf("Libraries: "); print_embedded_scripts(); printf("\n\n"); - printf("***************************** LIBNETTLE *****************************\n"); - printf("Version: %d.%d\n", NETTLE_VERSION_MAJOR, NETTLE_VERSION_MINOR); - printf("GMP: %s\n", NETTLE_USE_MINI_GMP ? "Mini" : "Full"); + printf("***************************** %s%sLIBNETTLE%s *****************************\n", + yellow, bold, normal); + printf("Version: %s%s" xstr(NETTLE_VERSION_MAJOR) "." xstr(NETTLE_VERSION_MINOR) "%s\n", + green, bold, normal); + printf("GMP: %s\n", NETTLE_USE_MINI_GMP ? "Mini" : "Full"); exit(EXIT_SUCCESS); } @@ -335,33 +428,79 @@ void parse_args(int argc, char* argv[]) // List of implemented arguments if(strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "help") == 0 || strcmp(argv[i], "--help") == 0) { - printf("pihole-FTL - The Pi-hole FTL engine\n\n"); - printf("Usage: sudo service pihole-FTL \n"); - printf("where '' is one of start / stop / restart\n\n"); - printf("Available arguments:\n"); - printf("\t debug More verbose logging,\n"); - printf("\t don't go into daemon mode\n"); - printf("\t test Don't start pihole-FTL but\n"); - printf("\t instead quit immediately\n"); - printf("\t-v, version Return FTL version\n"); - printf("\t-vv Return more version information\n"); - printf("\t-t, tag Return git tag\n"); - printf("\t-b, branch Return git branch\n"); - printf("\t-f, no-daemon Don't go into daemon mode\n"); - printf("\t-h, help Display this help and exit\n"); - printf("\tdnsmasq-test Test syntax of dnsmasq's\n"); - printf("\t config files and exit\n"); - printf("\tregex-test str Test str against all regular\n"); + const char *bold = cli_bold(); + const char *normal = cli_normal(); + const char *blue = cli_color(COL_BLUE); + const char *cyan = cli_color(COL_CYAN); + const char *gray = cli_color(COL_GRAY); + const char *green = cli_color(COL_GREEN); + const char *yellow = cli_color(COL_YELLOW); + const char *purple = cli_color(COL_PURPLE); + + printf("%sThe Pi-hole FTL engine - %s%s\n\n", bold, get_FTL_version(), normal); + printf("Typically, pihole-FTL runs as a system service and is controlled\n"); + printf("by %ssudo service pihole-FTL %s%s where %s%s is one out\n", green, purple, normal, purple, normal); + printf("of %sstart%s, %sstop%s, or %srestart%s.\n\n", green, normal, green, normal, green, normal); + printf("pihole-FTL exposes some features going beyond the standard\n"); + printf("%sservice pihole-FTL%s command. These are:\n\n", green, normal); + + printf("%sVersion information:%s\n", yellow, normal); + printf("\t%s-v%s, %sversion%s Return FTL version\n", green, normal, green, normal); + printf("\t%s-vv%s Return verbose version information\n", green, normal); + printf("\t%s-t%s, %stag%s Return git tag\n", green, normal, green, normal); + printf("\t%s-b%s, %sbranch%s Return git branch\n\n", green, normal, green, normal); + + printf("%sRegular expression testing:%s\n", yellow, normal); + printf("\t%sregex-test %sstr%s Test %sstr%s against all regular\n", green, blue, normal, blue, normal); printf("\t expressions in the database\n"); - printf("\tregex-test str rgx Test str against regular expression\n"); - printf("\t given by rgx\n"); - printf("\t--lua, lua FTL's lua interpreter\n"); - printf("\t--luac, luac FTL's lua compiler\n"); - printf("\tdhcp-discover Discover DHCP servers in the local\n"); + printf("\t%sregex-test %sstr %srgx%s Test %sstr%s against regular expression\n", green, blue, cyan, normal, blue, normal); + printf("\t given by regular expression %srgx%s\n\n", cyan, normal); + + printf(" Example: %spihole-FTL regex-test %ssomebad.domain %sbad%s\n", green, blue, cyan, normal); + printf(" to test %ssomebad.domain%s against %sbad%s\n\n", blue, normal, cyan, normal); + + printf("%sEmbedded Lua engine:%s\n", yellow, normal); + printf("\t%s--lua%s, %slua%s FTL's lua interpreter\n", green, normal, green, normal); + printf("\t%s--luac%s, %sluac%s FTL's lua compiler\n\n", green, normal, green, normal); + + printf(" Usage: %spihole-FTL lua %s[OPTIONS] [SCRIPT [ARGS]]%s\n\n", green, cyan, normal); + printf(" Options:\n\n"); + printf(" - %s[OPTIONS]%s is an optional set of options. All available\n", cyan, normal); + printf(" options can be seen by running %spihole-FTL lua --help%s\n", green, normal); + printf(" - %s[SCRIPT]%s is the optional name of a Lua script.\n", cyan, normal); + printf(" If this script does not exist, an interactive shell is\n"); + printf(" started instead.\n"); + printf(" - %s[SCRIPT [ARGS]]%s can be used to pass optional args to\n", cyan, normal); + printf(" the script.\n\n"); + + printf("%sEmbedded SQLite3 shell:%s\n", yellow, normal); + printf("\t%ssql %s[-h]%s, %ssqlite3 %s[-h]%s FTL's SQLite3 shell\n", green, gray, normal, green, gray, normal); + printf("\tthe optional %s-h%s starts a special %shuman-readable mode%s\n\n", gray, normal, bold, normal); + + printf(" Usage: %spihole-FTL sqlite3 %s[OPTIONS] [FILENAME] [SQL]%s\n\n", green, cyan, normal); + printf(" Options:\n\n"); + printf(" - %s[OPTIONS]%s is an optional set of options. All available\n", cyan, normal); + printf(" options can be found in %spihole-FTL sqlite3 --help%s\n", green, normal); + printf(" - %s[FILENAME]%s is the optional name of an SQLite database.\n", cyan, normal); + printf(" A new database is created if the file does not previously\n"); + printf(" exist. If this argument is omitted, SQLite3 will use a\n"); + printf(" transient in-memory database instead.\n"); + printf(" - %s[SQL]%s is an optional SQL statement to be executed. If\n", cyan, normal); + printf(" omitted, an interactive shell is started instead.\n\n"); + + printf("%sDebugging and special use:%s\n", yellow, normal); + printf("\t%sdebug%s More verbose logging,\n", green, normal); + printf("\t don't go into daemon mode\n"); + printf("\t%stest%s Don't start pihole-FTL but\n", green, normal); + printf("\t instead quit immediately\n"); + printf("\t%s-f%s, %sno-daemon%s Don't go into daemon mode\n\n", green, normal, green, normal); + + printf("%sOther:%s\n", yellow, normal); + printf("\t%sdnsmasq-test%s Test syntax of dnsmasq's\n", green, normal); + printf("\t config files and exit\n\n"); + printf("\t%sdhcp-discover%s Discover DHCP servers in the local\n", green, normal); printf("\t network\n"); - printf("\tsql, sqlite3 FTL's SQLite3 shell\n"); - printf("\tsql -h, sqlite3 -h FTL's SQLite3 shell (human-readable mode)\n"); - printf("\n\nOnline help: https://github.com/pi-hole/FTL\n"); + printf("\t%s-h%s, %shelp%s Display this help and exit\n\n", green, normal, green, normal); exit(EXIT_SUCCESS); } @@ -394,77 +533,3 @@ void parse_args(int argc, char* argv[]) } } } - -// Extended SGR sequence: -// -// "\x1b[%dm" -// -// where %d is one of the following values for commonly supported colors: -// -// 0: reset colors/style -// 1: bold -// 4: underline -// 30 - 37: black, red, green, yellow, blue, magenta, cyan, and white text -// 40 - 47: black, red, green, yellow, blue, magenta, cyan, and white background -// -// https://en.wikipedia.org/wiki/ANSI_escape_code#SGR -// -#define COL_NC "\x1b[0m" // normal font -#define COL_BOLD "\x1b[1m" // bold font -#define COL_ITALIC "\x1b[3m" // italic font -#define COL_ULINE "\x1b[4m" // underline font -#define COL_GREEN "\x1b[32m" // normal foreground color -#define COL_YELLOW "\x1b[33m" // normal foreground color -#define COL_GRAY "\x1b[90m" // bright foreground color -#define COL_RED "\x1b[91m" // bright foreground color -#define COL_BLUE "\x1b[94m" // bright foreground color -#define COL_PURPLE "\x1b[95m" // bright foreground color -#define COL_CYAN "\x1b[96m" // bright foreground color - -static inline bool __attribute__ ((pure)) is_term(void) -{ - // test whether STDOUT refers to a terminal - return isatty(fileno(stdout)) == 1; -} - -// Returns green [✓] -const char __attribute__ ((pure)) *cli_tick(void) -{ - return is_term() ? "["COL_GREEN"✓"COL_NC"]" : "[✓]"; -} - -// Returns red [✗] -const char __attribute__ ((pure)) *cli_cross(void) -{ - return is_term() ? "["COL_RED"✗"COL_NC"]" : "[✗]"; -} - -// Returns [i] -const char __attribute__ ((pure)) *cli_info(void) -{ - return is_term() ? COL_BOLD"[i]"COL_NC : "[i]"; -} - -// Returns [?] -const char __attribute__ ((const)) *cli_qst(void) -{ - return "[?]"; -} - -// Returns green "done!"" -const char __attribute__ ((pure)) *cli_done(void) -{ - return is_term() ? COL_GREEN"done!"COL_NC : "done!"; -} - -// Sets font to bold -const char __attribute__ ((pure)) *cli_bold(void) -{ - return is_term() ? COL_BOLD : ""; -} - -// Resets font to normal -const char __attribute__ ((pure)) *cli_normal(void) -{ - return is_term() ? COL_NC : ""; -} diff --git a/src/dnsmasq/dnsmasq.c b/src/dnsmasq/dnsmasq.c index 1d02462c..59bd4c15 100644 --- a/src/dnsmasq/dnsmasq.c +++ b/src/dnsmasq/dnsmasq.c @@ -2231,10 +2231,11 @@ int delay_dhcp(time_t start, int sec, int fd, uint32_t addr, unsigned short id) #endif /* HAVE_DHCP */ /******************************** Pi-hole modification ********************************/ -void print_dnsmasq_version(void) +void print_dnsmasq_version(const char *yellow, const char *green, const char *bold, const char *normal) { - printf("****************************** dnsmasq ******************************\n"); - printf(_("Version: %s\n"), VERSION); - printf(_("Compile options: %s\n\n"), compile_opts); + printf("****************************** %s%sdnsmasq%s ******************************\n", + bold, yellow, normal); + printf(_("Version: %s%s%s%s\n"), bold, green, VERSION, normal); + printf(_("Features: %s\n\n"), compile_opts); } /**************************************************************************************/ diff --git a/test/test_suite.bats b/test/test_suite.bats index c2c55608..d5e73b3f 100644 --- a/test/test_suite.bats +++ b/test/test_suite.bats @@ -15,9 +15,9 @@ run bash -c './pihole-FTL -vv | grep "cryptohash"' printf "%s\n" "${lines[@]}" if [[ "${CI_ARCH}" == "x86_64_full" ]]; then - [[ ${lines[0]} == "Compile options: IPv6 GNU-getopt DBus no-UBus no-i18n IDN DHCP DHCPv6 Lua TFTP conntrack ipset nftset auth cryptohash DNSSEC loop-detect inotify dumpfile" ]] + [[ ${lines[0]} == "Features: IPv6 GNU-getopt DBus no-UBus no-i18n IDN DHCP DHCPv6 Lua TFTP conntrack ipset nftset auth cryptohash DNSSEC loop-detect inotify dumpfile" ]] else - [[ ${lines[0]} == "Compile options: IPv6 GNU-getopt no-DBus no-UBus no-i18n IDN DHCP DHCPv6 Lua TFTP no-conntrack ipset no-nftset auth cryptohash DNSSEC loop-detect inotify dumpfile" ]] + [[ ${lines[0]} == "Features: IPv6 GNU-getopt no-DBus no-UBus no-i18n IDN DHCP DHCPv6 Lua TFTP no-conntrack ipset no-nftset auth cryptohash DNSSEC loop-detect inotify dumpfile" ]] fi [[ ${lines[1]} == "" ]] } @@ -691,8 +691,7 @@ @test "Help CLI argument return help text" { run bash -c '/home/pihole/pihole-FTL help' printf "%s\n" "${lines[@]}" - [[ ${lines[0]} == "pihole-FTL - The Pi-hole FTL engine" ]] - [[ ${lines[3]} == "Available arguments:" ]] + [[ ${lines[0]} == "The Pi-hole FTL engine - "* ]] } @test "No WARNING messages in FTL.log (besides known capability issues)" { From 4e392f0ad81f72302094b966e8c925c33d6d1577 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 9 Dec 2022 11:37:36 +0100 Subject: [PATCH 206/218] Minor improvements (show -h in the qlite3 usage and remove extra newline after dnsmasq-test) Signed-off-by: DL6ER --- src/args.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/args.c b/src/args.c index 10a72d43..c7666248 100644 --- a/src/args.c +++ b/src/args.c @@ -477,7 +477,7 @@ void parse_args(int argc, char* argv[]) printf("\t%ssql %s[-h]%s, %ssqlite3 %s[-h]%s FTL's SQLite3 shell\n", green, gray, normal, green, gray, normal); printf("\tthe optional %s-h%s starts a special %shuman-readable mode%s\n\n", gray, normal, bold, normal); - printf(" Usage: %spihole-FTL sqlite3 %s[OPTIONS] [FILENAME] [SQL]%s\n\n", green, cyan, normal); + printf(" Usage: %spihole-FTL sqlite3 %s[-h] %s[OPTIONS] [FILENAME] [SQL]%s\n\n", green, gray, cyan, normal); printf(" Options:\n\n"); printf(" - %s[OPTIONS]%s is an optional set of options. All available\n", cyan, normal); printf(" options can be found in %spihole-FTL sqlite3 --help%s\n", green, normal); @@ -497,7 +497,7 @@ void parse_args(int argc, char* argv[]) printf("%sOther:%s\n", yellow, normal); printf("\t%sdnsmasq-test%s Test syntax of dnsmasq's\n", green, normal); - printf("\t config files and exit\n\n"); + printf("\t config files and exit\n"); printf("\t%sdhcp-discover%s Discover DHCP servers in the local\n", green, normal); printf("\t network\n"); printf("\t%s-h%s, %shelp%s Display this help and exit\n\n", green, normal, green, normal); From 92d7d314f9a8a947f132c433ade180923988fe81 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 9 Dec 2022 11:49:49 +0100 Subject: [PATCH 207/218] Modify text slightly to prevent codespell from misinterpreting a control sequence as spellchecking error Signed-off-by: DL6ER --- src/args.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/args.c b/src/args.c index c7666248..2c4482aa 100644 --- a/src/args.c +++ b/src/args.c @@ -475,7 +475,7 @@ void parse_args(int argc, char* argv[]) printf("%sEmbedded SQLite3 shell:%s\n", yellow, normal); printf("\t%ssql %s[-h]%s, %ssqlite3 %s[-h]%s FTL's SQLite3 shell\n", green, gray, normal, green, gray, normal); - printf("\tthe optional %s-h%s starts a special %shuman-readable mode%s\n\n", gray, normal, bold, normal); + printf("\t%s-h%s starts a special %shuman-readable mode%s\n\n", gray, normal, bold, normal); printf(" Usage: %spihole-FTL sqlite3 %s[-h] %s[OPTIONS] [FILENAME] [SQL]%s\n\n", green, gray, cyan, normal); printf(" Options:\n\n"); From 3f1be068148d54a9d485ae5ccb2dddca85549664 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 9 Dec 2022 16:43:43 +0100 Subject: [PATCH 208/218] Fix logic for status code parsing Signed-off-by: DL6ER --- src/database/query-table.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/database/query-table.c b/src/database/query-table.c index 7e0161c5..961cb132 100644 --- a/src/database/query-table.c +++ b/src/database/query-table.c @@ -761,7 +761,7 @@ void DB_read_queries(void) } const int status_int = sqlite3_column_int(stmt, 3); - if(status_int < QUERY_UNKNOWN || status_int > QUERY_STATUS_MAX) + if(status_int < QUERY_UNKNOWN || status_int >= QUERY_STATUS_MAX) { logg("DB warn: STATUS should be within [%i,%i] but is %i", QUERY_UNKNOWN, QUERY_STATUS_MAX-1, status_int); continue; From f6e8d7fff65be93c2989cc7c4f8e80fa3a77e4a3 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 9 Dec 2022 16:45:50 +0100 Subject: [PATCH 209/218] Exit immediately after running dnsmasq-test Signed-off-by: DL6ER --- src/args.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/args.c b/src/args.c index 338dabad..59008d4a 100644 --- a/src/args.c +++ b/src/args.c @@ -137,8 +137,7 @@ void parse_args(int argc, char* argv[]) const char *arg[2]; arg[0] = ""; arg[1] = "--test"; - main_dnsmasq(2, arg); - ok = true; + exit(main_dnsmasq(2, arg)); } // If we find "--" we collect everything behind that for dnsmasq From bf0bbd9092b4fc17cbeff0e674159aa458c52051 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 10 Dec 2022 13:49:52 +0100 Subject: [PATCH 210/218] Review comments Signed-off-by: DL6ER --- src/args.c | 43 ++++++++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/src/args.c b/src/args.c index 2c4482aa..ad06c1c2 100644 --- a/src/args.c +++ b/src/args.c @@ -227,23 +227,23 @@ void parse_args(int argc, char* argv[]) // Remember that the rest is for dnsmasq ... consume_for_dnsmasq = true; - // Special command interpretation for "pihole-FTL -- --help dhcp" - if(argc > 1 && strcmp(argv[argc-2], "--help") == 0 && strcmp(argv[argc-1], "dhcp") == 0) - { - display_opts(); - exit(EXIT_SUCCESS); - } - // and "pihole-FTL -- --help dhcp6" - if(argc > 1 && strcmp(argv[argc-2], "--help") == 0 && strcmp(argv[argc-1], "dhcp6") == 0) - { - display_opts6(); - exit(EXIT_SUCCESS); - } - // ... and skip the current argument ("--") continue; } + // List available DHCPv4 config options + if(strcmp(argv[i], "--list-dhcp") == 0 || strcmp(argv[i], "--list-dhcp4") == 0) + { + display_opts(); + exit(EXIT_SUCCESS); + } + // List available DHCPv6 config options + if(strcmp(argv[i], "--list-dhcp6") == 0) + { + display_opts6(); + exit(EXIT_SUCCESS); + } + // If consume_for_dnsmasq is true, we collect all remaining options for // dnsmasq if(consume_for_dnsmasq) @@ -448,7 +448,8 @@ void parse_args(int argc, char* argv[]) printf("\t%s-v%s, %sversion%s Return FTL version\n", green, normal, green, normal); printf("\t%s-vv%s Return verbose version information\n", green, normal); printf("\t%s-t%s, %stag%s Return git tag\n", green, normal, green, normal); - printf("\t%s-b%s, %sbranch%s Return git branch\n\n", green, normal, green, normal); + printf("\t%s-b%s, %sbranch%s Return git branch\n", green, normal, green, normal); + printf("\t%s--hash%s Return git commit hash\n\n", green, normal); printf("%sRegular expression testing:%s\n", yellow, normal); printf("\t%sregex-test %sstr%s Test %sstr%s against all regular\n", green, blue, normal, blue, normal); @@ -458,6 +459,8 @@ void parse_args(int argc, char* argv[]) printf(" Example: %spihole-FTL regex-test %ssomebad.domain %sbad%s\n", green, blue, cyan, normal); printf(" to test %ssomebad.domain%s against %sbad%s\n\n", blue, normal, cyan, normal); + printf(" An optional %s-q%s prevents any output (exit code testing):\n", gray, normal); + printf(" %spihole-FTL %s-q%s regex-test %ssomebad.domain %sbad%s\n\n", green, gray, green, blue, cyan, normal); printf("%sEmbedded Lua engine:%s\n", yellow, normal); printf("\t%s--lua%s, %slua%s FTL's lua interpreter\n", green, normal, green, normal); @@ -477,7 +480,7 @@ void parse_args(int argc, char* argv[]) printf("\t%ssql %s[-h]%s, %ssqlite3 %s[-h]%s FTL's SQLite3 shell\n", green, gray, normal, green, gray, normal); printf("\t%s-h%s starts a special %shuman-readable mode%s\n\n", gray, normal, bold, normal); - printf(" Usage: %spihole-FTL sqlite3 %s[-h] %s[OPTIONS] [FILENAME] [SQL]%s\n\n", green, gray, cyan, normal); + printf(" Usage: %spihole-FTL sqlite3 %s[-h] %s[OPTIONS] [FILENAME] [SQL]%s\n\n", green, gray, cyan, normal); printf(" Options:\n\n"); printf(" - %s[OPTIONS]%s is an optional set of options. All available\n", cyan, normal); printf(" options can be found in %spihole-FTL sqlite3 --help%s\n", green, normal); @@ -488,16 +491,18 @@ void parse_args(int argc, char* argv[]) printf(" - %s[SQL]%s is an optional SQL statement to be executed. If\n", cyan, normal); printf(" omitted, an interactive shell is started instead.\n\n"); + printf("%sEmbedded dnsmasq options:%s\n", yellow, normal); + printf("\t%sdnsmasq-test%s Test syntax of dnsmasq's config\n", green, normal); + printf("\t%s--list-dhcp4%s List known DHCPv4 config options\n", green, normal); + printf("\t%s--list-dhcp6%s List known DHCPv6 config options\n\n", green, normal); + printf("%sDebugging and special use:%s\n", yellow, normal); - printf("\t%sdebug%s More verbose logging,\n", green, normal); - printf("\t don't go into daemon mode\n"); + printf("\t%sd%s, %sdebug%s Enter debugging mode\n", green, normal, green, normal); printf("\t%stest%s Don't start pihole-FTL but\n", green, normal); printf("\t instead quit immediately\n"); printf("\t%s-f%s, %sno-daemon%s Don't go into daemon mode\n\n", green, normal, green, normal); printf("%sOther:%s\n", yellow, normal); - printf("\t%sdnsmasq-test%s Test syntax of dnsmasq's\n", green, normal); - printf("\t config files and exit\n"); printf("\t%sdhcp-discover%s Discover DHCP servers in the local\n", green, normal); printf("\t network\n"); printf("\t%s-h%s, %shelp%s Display this help and exit\n\n", green, normal, green, normal); From b2dadba5af92983b34745b29b1f31be978178e81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Tue, 13 Dec 2022 22:41:29 +0100 Subject: [PATCH 211/218] Add merge conflict workflow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- .github/workflows/merge-conflict.yml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 .github/workflows/merge-conflict.yml diff --git a/.github/workflows/merge-conflict.yml b/.github/workflows/merge-conflict.yml new file mode 100644 index 00000000..971ecb88 --- /dev/null +++ b/.github/workflows/merge-conflict.yml @@ -0,0 +1,21 @@ +name: "Check for merge conflicts" +on: + # So that PRs touching the same files as the push are updated + push: + # So that the `dirtyLabel` is removed if conflicts are resolve + # We recommend `pull_request_target` so that github secrets are available. + # In `pull_request` we wouldn't be able to change labels of fork PRs + pull_request_target: + types: [synchronize] + +jobs: + main: + runs-on: ubuntu-latest + steps: + - name: Check if PRs are have merge conflicts + uses: eps1lon/actions-label-merge-conflict@v2.1.0 + with: + dirtyLabel: "Merge conflicts" + repoToken: "${{ secrets.GITHUB_TOKEN }}" + commentOnDirty: "This pull request has conflicts, please resolve those before we can evaluate the pull request." + commentOnClean: "Conflicts have been resolved. A maintainer will review the pull request shortly." From bda6721135cff50831ebf58e93dfdeffdc2fb820 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Wed, 14 Dec 2022 21:54:28 +0100 Subject: [PATCH 212/218] Use github cli to sync master back to development MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- .github/workflows/sync-back-to-dev.yml | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/.github/workflows/sync-back-to-dev.yml b/.github/workflows/sync-back-to-dev.yml index f689ae36..3a5133f9 100644 --- a/.github/workflows/sync-back-to-dev.yml +++ b/.github/workflows/sync-back-to-dev.yml @@ -13,15 +13,6 @@ jobs: - name: Checkout uses: actions/checkout@v3.1.0 - name: Opening pull request - id: pull - uses: tretuna/sync-branches@1.4.0 - with: + run: gh pr create -B development -H master --title 'Sync master back into development' --body 'Created by Github action' --label 'internal' + env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - FROM_BRANCH: 'master' - TO_BRANCH: 'development' - - name: Label the pull request to ignore for release note generation - uses: actions-ecosystem/action-add-labels@v1.1.3 - with: - labels: internal - repo: ${{ github.repository }} - number: ${{ steps.pull.outputs.PULL_REQUEST_NUMBER }} From 61047520d1262aa6edf1598ffa229f92df684013 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 17 Dec 2022 10:01:11 +0000 Subject: [PATCH 213/218] Bump actions/checkout from 3.1.0 to 3.2.0 Bumps [actions/checkout](https://github.com/actions/checkout) from 3.1.0 to 3.2.0. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v3.1.0...v3.2.0) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 6 +++--- .github/workflows/codespell.yml | 2 +- .github/workflows/sync-back-to-dev.yml | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 38c040e4..e860cd9f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -24,7 +24,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v3.1.0 + uses: actions/checkout@v3.2.0 - name: "Calculate required variables" id: variables @@ -88,7 +88,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v3.1.0 + uses: actions/checkout@v3.2.0 - name: "Fix ownership of repository" run: chown -R root . @@ -131,7 +131,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v3.1.0 + uses: actions/checkout@v3.2.0 - name: Get Binaries built in previous jobs uses: actions/download-artifact@v3.0.1 diff --git a/.github/workflows/codespell.yml b/.github/workflows/codespell.yml index 4930242c..49a30797 100644 --- a/.github/workflows/codespell.yml +++ b/.github/workflows/codespell.yml @@ -10,7 +10,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3.1.0 + uses: actions/checkout@v3.2.0 - name: Spell-Checking uses: codespell-project/actions-codespell@master diff --git a/.github/workflows/sync-back-to-dev.yml b/.github/workflows/sync-back-to-dev.yml index f689ae36..88f0f359 100644 --- a/.github/workflows/sync-back-to-dev.yml +++ b/.github/workflows/sync-back-to-dev.yml @@ -11,7 +11,7 @@ jobs: name: Syncing branches steps: - name: Checkout - uses: actions/checkout@v3.1.0 + uses: actions/checkout@v3.2.0 - name: Opening pull request id: pull uses: tretuna/sync-branches@1.4.0 From db98d0b4953f74af48a661fbc0b6cae7b5a260b7 Mon Sep 17 00:00:00 2001 From: Simon Kelley Date: Sun, 4 Dec 2022 22:00:54 +0000 Subject: [PATCH 214/218] Add posix-timezone and tzdb-timezone DHCPv6 options. They are already in place for DHCPv4. --- src/dnsmasq/dhcp-common.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/dnsmasq/dhcp-common.c b/src/dnsmasq/dhcp-common.c index 545311ed..1b6cc849 100644 --- a/src/dnsmasq/dhcp-common.c +++ b/src/dnsmasq/dhcp-common.c @@ -722,6 +722,8 @@ static const struct opttab_t opttab6[] = { { "sntp-server", 31, OT_ADDR_LIST }, { "information-refresh-time", 32, OT_TIME }, { "FQDN", 39, OT_INTERNAL | OT_RFC1035_NAME }, + { "posix-timezone", 41, OT_NAME }, /* RFC 4833, Sec. 3 */ + { "tzdb-timezone", 42, OT_NAME }, /* RFC 4833, Sec. 3 */ { "ntp-server", 56, 0 /* OT_ADDR_LIST | OT_RFC1035_NAME */ }, { "bootfile-url", 59, OT_NAME }, { "bootfile-param", 60, OT_CSTRING }, From e51864d47a1e8d92aa1252110b1a444848df6f7a Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 21 Dec 2022 20:38:57 +0100 Subject: [PATCH 215/218] Update embedded dnsmasq to v2.88 Signed-off-by: DL6ER --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7e780773..76f7c68f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -11,6 +11,6 @@ cmake_minimum_required(VERSION 2.8.12) project(PIHOLE_FTL C) -set(DNSMASQ_VERSION pi-hole-v2.88rc5) +set(DNSMASQ_VERSION pi-hole-v2.88) add_subdirectory(src) From df399e3014cea8271c3a55423f53c46b53cb87da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Thu, 22 Dec 2022 22:20:11 +0100 Subject: [PATCH 216/218] Be honest MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- .github/workflows/merge-conflict.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/merge-conflict.yml b/.github/workflows/merge-conflict.yml index 971ecb88..6722d665 100644 --- a/.github/workflows/merge-conflict.yml +++ b/.github/workflows/merge-conflict.yml @@ -18,4 +18,4 @@ jobs: dirtyLabel: "Merge conflicts" repoToken: "${{ secrets.GITHUB_TOKEN }}" commentOnDirty: "This pull request has conflicts, please resolve those before we can evaluate the pull request." - commentOnClean: "Conflicts have been resolved. A maintainer will review the pull request shortly." + commentOnClean: "Conflicts have been resolved." From fbdb2e65c9640a7ea5fad3780e48d9f531f972e5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 24 Dec 2022 10:01:21 +0000 Subject: [PATCH 217/218] Bump actions/stale from 6.0.1 to 7.0.0 Bumps [actions/stale](https://github.com/actions/stale) from 6.0.1 to 7.0.0. - [Release notes](https://github.com/actions/stale/releases) - [Changelog](https://github.com/actions/stale/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/stale/compare/v6.0.1...v7.0.0) --- updated-dependencies: - dependency-name: actions/stale dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/stale.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 27db3ded..9c3c9829 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -13,7 +13,7 @@ jobs: issues: write steps: - - uses: actions/stale@v6.0.1 + - uses: actions/stale@v7.0.0 with: repo-token: ${{ secrets.GITHUB_TOKEN }} days-before-stale: 30 From f0b133b28568288fdf2906ca4f748bb40c1213d6 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 27 Dec 2022 07:31:39 +0100 Subject: [PATCH 218/218] Always store time we start to save to the database (not only in debug mode). This avoids errorneous timing reports in case of errors. Signed-off-by: DL6ER --- src/database/query-table.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/database/query-table.c b/src/database/query-table.c index 961cb132..d4d57565 100644 --- a/src/database/query-table.c +++ b/src/database/query-table.c @@ -63,8 +63,7 @@ int DB_save_queries(sqlite3 *db) return DB_FAILED; // Start database timer - if(config.debug & DEBUG_DATABASE) - timer_start(DATABASE_WRITE_TIMER); + timer_start(DATABASE_WRITE_TIMER); // Open pihole-FTL.db database file if needed bool db_opened = false;