Merge branch 'development-v6' into new/queryLogRegex

Signed-off-by: DL6ER <dl6er@dl6er.de>

.devcontainer/devcontainer.json

@@ -1,6 +1,6 @@
 {
   "name": "FTL x86_64 Build Env",
-  "image": "ghcr.io/pi-hole/ftl-build:v2.4.1",
+  "image": "ghcr.io/pi-hole/ftl-build:v2.5",
   "runArgs": [ "--cap-add=SYS_PTRACE", "--security-opt", "seccomp=unconfined" ],
   "customizations": {
     "vscode": {

.github/Dockerfile

@@ -1,4 +1,4 @@
-FROM ghcr.io/pi-hole/ftl-build:v2.4.1 AS builder
+FROM ghcr.io/pi-hole/ftl-build:v2.5 AS builder
 
 WORKDIR /app
 

.github/workflows/build.yml

@@ -119,7 +119,7 @@ jobs:
       -
         name: Store binary artifacts for later deployoment
         if: github.event_name != 'pull_request'
-        uses: actions/upload-artifact@v4.0.0
+        uses: actions/upload-artifact@v4.1.0
         with:
           name: ${{ matrix.bin_name }}-binary
           path: '${{ matrix.bin_name }}*'
@@ -131,7 +131,7 @@ jobs:
       -
         name: Upload documentation artifacts for deployoment
         if: github.event_name != 'pull_request' && matrix.platform == 'linux/amd64'
-        uses: actions/upload-artifact@v4.0.0
+        uses: actions/upload-artifact@v4.1.0
         with:
           name: pihole-api-docs
           path: 'api-docs.tar.gz'
@@ -146,7 +146,7 @@ jobs:
         uses: actions/checkout@v4.1.1
       -
         name: Get Binaries and documentation built in previous jobs
-        uses: actions/download-artifact@v4.1.0
+        uses: actions/download-artifact@v4.1.1
         id: download
         with:
           path: ftl_builds/

src/api/api.h

@@ -24,6 +24,7 @@
 int api_handler(struct mg_connection *conn, void *ignored);
 
 // Statistic methods
+int __attribute__((pure)) cmpdesc(const void *a, const void *b);
 int api_stats_summary(struct ftl_conn *api);
 int api_stats_query_types(struct ftl_conn *api);
 int api_stats_upstreams(struct ftl_conn *api);

src/api/auth.c

@@ -339,14 +339,18 @@ static int get_session_object(struct ftl_conn *api, cJSON *json, const int user_
 	return 0;
 }
 
-static void delete_session(const int user_id)
+static bool delete_session(const int user_id)
 {
 	// Skip if nothing to be done here
 	if(user_id < 0 || user_id >= max_sessions)
-		return;
+		return false;
+
+	const bool was_valid = auth_data[user_id].used;
 
 	// Zero out this session (also sets valid to false == 0)
 	memset(&auth_data[user_id], 0, sizeof(auth_data[user_id]));
+
+	return was_valid;
 }
 
 void delete_all_sessions(void)
@@ -357,24 +361,6 @@ void delete_all_sessions(void)
 
 static int send_api_auth_status(struct ftl_conn *api, const int user_id, const time_t now)
 {
-	if(user_id == API_AUTH_LOCALHOST)
-	{
-		log_debug(DEBUG_API, "API Auth status: OK (localhost does not need auth)");
-
-		cJSON *json = JSON_NEW_OBJECT();
-		get_session_object(api, json, user_id, now);
-		JSON_SEND_OBJECT(json);
-	}
-
-	if(user_id == API_AUTH_EMPTYPASS)
-	{
-		log_debug(DEBUG_API, "API Auth status: OK (empty password)");
-
-		cJSON *json = JSON_NEW_OBJECT();
-		get_session_object(api, json, user_id, now);
-		JSON_SEND_OBJECT(json);
-	}
-
 	if(user_id > API_AUTH_UNAUTHORIZED && (api->method == HTTP_GET || api->method == HTTP_POST))
 	{
 		log_debug(DEBUG_API, "API Auth status: OK");
@@ -391,17 +377,45 @@ static int send_api_auth_status(struct ftl_conn *api, const int user_id, const t
 		get_session_object(api, json, user_id, now);
 		JSON_SEND_OBJECT(json);
 	}
-	else if(user_id > API_AUTH_UNAUTHORIZED && api->method == HTTP_DELETE)
+	else if(api->method == HTTP_DELETE)
 	{
-		log_debug(DEBUG_API, "API Auth status: Logout, asking to delete cookie");
+		if(user_id > API_AUTH_UNAUTHORIZED)
+		{
+			log_debug(DEBUG_API, "API Auth status: Logout, asking to delete cookie");
 
-		// Revoke client authentication. This slot can be used by a new client afterwards.
-		delete_session(user_id);
+			strncpy(pi_hole_extra_headers, FTL_DELETE_COOKIE, sizeof(pi_hole_extra_headers));
+
+			// Revoke client authentication. This slot can be used by a new client afterwards.
+			const int code = delete_session(user_id) ? 204 : 404;
+
+			// Send empty reply with appropriate HTTP status code
+			send_http_code(api, "application/json; charset=utf-8", code, "");
+			return code;
+		}
+		else
+		{
+			log_debug(DEBUG_API, "API Auth status: Logout, but not authenticated");
+
+			cJSON *json = JSON_NEW_OBJECT();
+			get_session_object(api, json, user_id, now);
+			JSON_SEND_OBJECT_CODE(json, 401); // 401 Unauthorized
+		}
+	}
+	else if(user_id == API_AUTH_LOCALHOST)
+	{
+		log_debug(DEBUG_API, "API Auth status: OK (localhost does not need auth)");
 
-		strncpy(pi_hole_extra_headers, FTL_DELETE_COOKIE, sizeof(pi_hole_extra_headers));
 		cJSON *json = JSON_NEW_OBJECT();
 		get_session_object(api, json, user_id, now);
-		JSON_SEND_OBJECT_CODE(json, 410); // 410 Gone
+		JSON_SEND_OBJECT(json);
+	}
+	else if(user_id == API_AUTH_EMPTYPASS)
+	{
+		log_debug(DEBUG_API, "API Auth status: OK (empty password)");
+
+		cJSON *json = JSON_NEW_OBJECT();
+		get_session_object(api, json, user_id, now);
+		JSON_SEND_OBJECT(json);
 	}
 	else
 	{
@@ -566,7 +580,7 @@ int api_auth(struct ftl_conn *api)
 		{
 			// Expired slow, mark as unused
 			if(auth_data[i].used &&
-				auth_data[i].valid_until < now)
+			   auth_data[i].valid_until < now)
 			{
 				log_debug(DEBUG_API, "API: Session of client %u (%s) expired, freeing...",
 						i, auth_data[i].remote_addr);
@@ -637,6 +651,11 @@ int api_auth(struct ftl_conn *api)
 		                       "Rate-limiting login attempts",
 		                       NULL);
 	}
+	else if(result == NO_PASSWORD_SET)
+	{
+		// No password set
+		log_debug(DEBUG_API, "API: Trying to auth with password but none set: '%s'", password);
+	}
 	else
 	{
 		log_debug(DEBUG_API, "API: Password incorrect: '%s'", password);
@@ -670,9 +689,9 @@ int api_auth_session_delete(struct ftl_conn *api)
 		return send_json_error(api, 400, "bad_request", "Session ID not in use", NULL);
 
 	// Delete session
-	delete_session(uid);
+	const int code = delete_session(uid) ? 204 : 404;
 
-	// Send empty reply with code 204 No Content
-	send_http_code(api, "application/json; charset=utf-8", 204, "");
-	return 204;
+	// Send empty reply with appropriate HTTP status code
+	send_http_code(api, "application/json; charset=utf-8", code, "");
+	return code;
 }

src/api/config.c

@@ -294,7 +294,7 @@ static const char *getJSONvalue(struct conf_item *conf_item, cJSON *elem, struct
 			}
 
 			if(!set_and_check_password(conf_item, elem->valuestring))
-				return "Failed to create password hash (verification failed), password remains unchanged";
+				return "password hash verification failed";
 
 			break;
 		}
@@ -904,7 +904,7 @@ static int api_config_put_delete(struct ftl_conn *api)
 			                            key, true);
 		}
 
-		// Check if this entry does already exist in the array
+		// Check if this entry exists in the array
 		int idx = 0;
 		for(; idx < cJSON_GetArraySize(new_item->v.json); idx++)
 		{
@@ -938,13 +938,12 @@ static int api_config_put_delete(struct ftl_conn *api)
 			if(found)
 			{
 				// Remove item from array
+				found = true;
 				cJSON_DeleteItemFromArray(new_item->v.json, idx);
 			}
 			else
 			{
 				// Item not found
-				message = "Item not found";
-				hint = "Can only delete existing items";
 				break;
 			}
 		}
@@ -964,13 +963,16 @@ static int api_config_put_delete(struct ftl_conn *api)
 	// Release allocated memory
 	free_config_path(requested_path);
 
-	// Error 404 if not found
-	if(!found || message != NULL)
+	// Error 404 if config element not found
+	if(!found)
+	{
+		cJSON *json = JSON_NEW_OBJECT();
+		JSON_SEND_OBJECT_CODE(json, 404);
+	}
+
+	// Error 400 if unique item already present
+	if(message != NULL)
 	{
-		// For any other error, a more specific message will have been added
-		// above
-		if(!message)
-			message = "No item specified";
 		return send_json_error(api, 400,
 		                       "bad_request",
 		                       message,

src/api/dhcp.c

@@ -73,7 +73,7 @@ int api_dhcp_leases_GET(struct ftl_conn *api)
 }
 
 // defined in dnsmasq_interface.c
-extern bool FTL_unlink_DHCP_lease(const char *ipaddr);
+extern bool FTL_unlink_DHCP_lease(const char *ipaddr, const char **hint);
 
 // Delete DHCP leases
 int api_dhcp_leases_DELETE(struct ftl_conn *api)
@@ -85,16 +85,29 @@ int api_dhcp_leases_DELETE(struct ftl_conn *api)
 		// Send empty reply with code 204 No Content
 		return send_json_error(api,
 		                       400,
-				       "bad_request",
+		                       "bad_request",
 		                       "The provided IPv4 address is invalid",
-				       api->item);
+		                       api->item);
 	}
 
 	// Delete lease
 	log_debug(DEBUG_API, "Deleting DHCP lease for address %s", api->item);
-	FTL_unlink_DHCP_lease(api->item);
 
-	// Send empty reply with code 204 No Content
+	const char *hint = NULL;
+	const bool found = FTL_unlink_DHCP_lease(api->item, &hint);
+	if(!found && hint != NULL)
+	{
+		// Send error when something went wrong (hint is not NULL)
+		return send_json_error(api,
+		                       400,
+		                       "bad_request",
+		                       "Failed to delete DHCP lease",
+		                       hint);
+	}
+
+	// Send empty reply with codes:
+	// - 204 No Content (if a lease was deleted)
+	// - 404 Not Found (if no lease was found)
 	cJSON *json = JSON_NEW_OBJECT();
-	JSON_SEND_OBJECT_CODE(json, 204);
+	JSON_SEND_OBJECT_CODE(json, found ? 204 : 404);
 }

src/api/docs/content/specs/auth.yaml

@@ -118,21 +118,27 @@ components:
           - Authentication
         operationId: "delete_groups"
         description: |
-          A logout attempt without a valid session will result in a `401 Unauthorized` error.
+          This endpoint can be used to delete the current session. It will
+          invalidate the session token and the CSRF token. The session can be
+          extended before its expiration by performing any authenticated action.
+          By default, the session lasts for 5 minutes. It can be invalidated by
+          either logging out or deleting the session. Additionally, the session
+          becomes invalid when the password is altered or a new application
+          password is created.
 
-          A session that was not created due to a login cannot be deleted (e.g., empty API password).
+          You can also delete a session by its ID using the `DELETE /auth/session/{id}` endpoint.
+
+          Note that you cannot delete the current session if you have not
+          authenticated (e.g., no password has been set on your Pi-hole).
         responses:
-          '200':
-            description: OK (session not deletable)
+          '204':
+            description: No Content (deleted)
+          '404':
+            description: Not Found (no session active)
             content:
               application/json:
                 schema:
-                  allOf:
-                    - $ref: 'auth.yaml#/components/schemas/session'
-                    - $ref: 'common.yaml#/components/schemas/took'
-                examples:
-                  no_login_required:
-                    $ref: 'auth.yaml#/components/examples/no_login_required'
+                  $ref: 'common.yaml#/components/schemas/took'
           '401':
             description: Unauthorized
             content:
@@ -141,17 +147,6 @@ components:
                   allOf:
                     - $ref: 'common.yaml#/components/errors/unauthorized'
                     - $ref: 'common.yaml#/components/schemas/took'
-          '410':
-            description: Gone
-            content:
-              application/json:
-                schema:
-                  allOf:
-                    - $ref: 'auth.yaml#/components/schemas/session'
-                    - $ref: 'common.yaml#/components/schemas/took'
-                examples:
-                  login_failed:
-                    $ref: 'auth.yaml#/components/examples/login_failed'
     session_list:
       get:
         summary: List of all current sessions
@@ -213,6 +208,12 @@ components:
         responses:
           '204':
             description: No Content (deleted)
+          '404':
+            description: Not Found (session not found)
+            content:
+              application/json:
+                schema:
+                  $ref: 'common.yaml#/components/schemas/took'
           '400':
             description: Bad Request
             content:

src/api/docs/content/specs/clients.yaml

@@ -95,6 +95,12 @@ components:
         responses:
           '204':
             description: Item deleted
+          '404':
+            description: Item not found
+            content:
+              application/json:
+                schema:
+                  $ref: 'common.yaml#/components/schemas/took'
           '400':
             description: Bad request
             content:
@@ -233,6 +239,12 @@ components:
         responses:
           '204':
             description: Items deleted
+          '404':
+            description: Item not found
+            content:
+              application/json:
+                schema:
+                  $ref: 'common.yaml#/components/schemas/took'
           '400':
             description: Bad request
             content:

src/api/docs/content/specs/config.yaml

@@ -121,7 +121,7 @@ components:
                 examples:
                   invalid_path_depth:
                     $ref: 'config.yaml#/components/examples/errors/bad_request/invalid_path_depth'
-                  item_not_found:
+                  item_already_present:
                     $ref: 'config.yaml#/components/examples/errors/bad_request/item_already_present'
           '401':
             description: Unauthorized
@@ -144,6 +144,12 @@ components:
         responses:
           '204':
             description: Item deleted
+          '404':
+            description: Item not found
+            content:
+              application/json:
+                schema:
+                  $ref: 'common.yaml#/components/schemas/took'
           '400':
             description: Bad request
             content:
@@ -155,8 +161,8 @@ components:
                 examples:
                   invalid_path_depth:
                     $ref: 'config.yaml#/components/examples/errors/bad_request/invalid_path_depth'
-                  item_not_found:
-                    $ref: 'config.yaml#/components/examples/errors/bad_request/item_not_found'
+                  item_already_present:
+                    $ref: 'config.yaml#/components/examples/errors/bad_request/item_already_present'
           '401':
             description: Unauthorized
             content:
@@ -420,6 +426,8 @@ components:
                         type: string
                     maxHistory:
                       type: integer
+                    maxClients:
+                      type: integer
                     allow_destructive:
                       type: boolean
                     temp:
@@ -694,6 +702,7 @@ components:
               excludeClients: [ '1\.2\.3\.4', 'localhost', 'fe80::345' ]
               excludeDomains: [ 'google\\.de', 'pi-hole\.net' ]
               maxHistory: 86400
+              maxClients: 10
               allow_destructive: true
               temp:
                 limit: 60.0
@@ -795,13 +804,6 @@ components:
               key: "bad_request"
               message: "Invalid path depth"
               hint: "Use, e.g., DELETE /config/dnsmasq/upstreams/127.0.0.1 to remove \"127.0.0.1\" from config.dns.upstreams"
-        item_not_found:
-          summary: Item to be deleted does not exist
-          value:
-            error:
-              key: "bad_request"
-              message: "Item not found"
-              hint: "Can only delete existing items"
         item_already_present:
           summary: Item to be added exists already
           value:

src/api/docs/content/specs/dhcp.yaml

@@ -36,10 +36,17 @@ components:
         operationId: "delete_dhcp"
         description: |
           This API hook removes a currently active DHCP lease.
+          Managing DHCP leases is only possible when the DHCP server is enabled.
           *Note:* There will be no content on success.
         responses:
           '204':
             description: Item deleted
+          '404':
+            description: Item not found
+            content:
+              application/json:
+                schema:
+                  $ref: 'common.yaml#/components/schemas/took'
           '400':
             description: Bad request
             content:

src/api/docs/content/specs/domains.yaml

@@ -128,6 +128,12 @@ components:
         responses:
           '204':
             description: Item deleted
+          '404':
+            description: Item not found
+            content:
+              application/json:
+                schema:
+                  $ref: 'common.yaml#/components/schemas/took'
           '400':
             description: Bad request
             content:
@@ -251,6 +257,12 @@ components:
         responses:
           '204':
             description: Items deleted
+          '404':
+            description: Item not found
+            content:
+              application/json:
+                schema:
+                  $ref: 'common.yaml#/components/schemas/took'
           '400':
             description: Bad request
             content:

src/api/docs/content/specs/groups.yaml

@@ -63,6 +63,9 @@ components:
                     - $ref: 'groups.yaml#/components/schemas/groups/get' # identical to GET
                     - $ref: 'groups.yaml#/components/schemas/lists_processed'
                     - $ref: 'common.yaml#/components/schemas/took'
+            headers:
+              Location:
+                $ref: 'common.yaml#/components/headers/Location'
           '400':
             description: Bad request
             content:
@@ -94,6 +97,12 @@ components:
         responses:
           '204':
             description: Item deleted
+          '404':
+            description: Item not found
+            content:
+              application/json:
+                schema:
+                  $ref: 'common.yaml#/components/schemas/took'
           '400':
             description: Bad request
             content:
@@ -193,18 +202,14 @@ components:
                   - "item": "test1"
                   - "item": "test2"
         responses:
-          '201':
-            description: Created item
+          '204':
+            description: Items deleted
+          '404':
+            description: Item not found
             content:
               application/json:
                 schema:
-                  allOf:
-                    - $ref: 'groups.yaml#/components/schemas/groups/get' # identical to GET
-                    - $ref: 'groups.yaml#/components/schemas/lists_processed'
-                    - $ref: 'common.yaml#/components/schemas/took'
-            headers:
-              Location:
-                $ref: 'common.yaml#/components/headers/Location'
+                  $ref: 'common.yaml#/components/schemas/took'
           '400':
             description: Bad request
             content:

src/api/docs/content/specs/history.yaml

@@ -62,7 +62,15 @@ components:
           - Metrics
         operationId: "get_client_metrics"
         description: |
-          Request data needed to generate the \"Client activity over last 24 hours\" graph
+          Request data needed to generate the \"Client activity over last 24 hours\" graph.
+          This endpoint returns the top N clients, sorted by total number of queries within 24 hours. If N is set to 0, all clients will be returned.
+          The client name is only available if the client's IP address can be resolved to a hostname.
+
+          The last client returned is a special client that contains the total number of queries that were not sent by any of the other shown clients , i.e. queries that were sent by clients that are not in the top N. This client is always present, even if it has 0 queries and can be identified by the special name "other clients" (mind the space in the hostname) and the IP address "0.0.0.0".
+
+          Note that, due to privacy settings, the returned data may also be empty.
+        parameters:
+          - $ref: 'history.yaml#/components/parameters/clients/N'
         responses:
           '200':
             description: OK
@@ -142,6 +150,38 @@ components:
     client_history:
       type: object
       properties:
+        clients:
+          type: array
+          description: Data array
+          items:
+            type: object
+            properties:
+              name:
+                type: string
+                nullable: true
+                description: Client name
+              ip:
+                type: string
+                description: Client IP address
+              total:
+                type: integer
+                description: Total number of queries
+          example:
+            - name: localhost
+              ip: "127.0.0.1"
+              total: 13428
+            - name: ip6-localnet
+              ip: "::1"
+              total: 2100
+            - name: null
+              ip: "192.168.1.1"
+              total: 254
+            - name: "pi.hole"
+              ip: "::"
+              total: 29
+            - name: "other clients"
+              ip: "0.0.0.0"
+              total: 14
         history:
           type: array
           description: Data array
@@ -162,28 +202,22 @@ components:
                 - 12
                 - 65
                 - 67
+                - 9
+                - 5
             - timestamp: 1511820500.583821
               data:
                 - 1
                 - 35
                 - 63
-        clients:
-          type: array
-          description: Data array
-          items:
-            type: object
-            properties:
-              name:
-                type: string
-                nullable: true
-                description: Client name
-              ip:
-                type: string
-                description: Client IP address
-          example:
-            - name: localhost
-              ip: "127.0.0.1"
-            - name: ip6-localnet
-              ip: "::1"
-            - name: null
-              ip: "192.168.1.1"
+                - 20
+                - 9
+  parameters:
+    clients:
+      N:
+        in: query
+        description: Maximum number of clients to return, setting this to 0 will return all clients
+        name: N
+        schema:
+          type: integer
+        required: false
+        example: 20

src/api/docs/content/specs/info.yaml

@@ -218,10 +218,16 @@ components:
         parameters:
           - $ref: 'info.yaml#/components/parameters/message_id'
         description: |
-          *Note:* There will be no content on success. You may specify multiple IDs to delete multiple messages at once (comma-separated in the path like `1,2,3`)
+          You may specify multiple IDs to delete multiple messages at once (comma-separated in the path like `1,2,3`)
         responses:
           '204':
             description: Item deleted
+          '404':
+            description: Not found
+            content:
+              application/json:
+                schema:
+                  $ref: 'common.yaml#/components/schemas/took'
           '400':
             description: Bad request
             content:
@@ -235,6 +241,14 @@ components:
                     $ref: 'info.yaml#/components/examples/errors/messages/uri_error'
                   bad_request:
                     $ref: 'info.yaml#/components/examples/errors/messages/bad_request'
+          '401':
+            description: Unauthorized
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'common.yaml#/components/errors/unauthorized'
+                    - $ref: 'common.yaml#/components/schemas/took'
     messages_count:
       get:
         summary: Get count of Pi-hole diagnosis messages
@@ -835,7 +849,7 @@ components:
                     version:
                       type: string
                       nullable: true
-                      description: Remote (Github) Pi-hole Core version
+                      description: Remote (Github) Pi-hole Core version (null if on custom branch)
                       example: "v6.1"
                     hash:
                       type: string
@@ -869,7 +883,7 @@ components:
                     version:
                       type: string
                       nullable: true
-                      description: Remote (Github) Pi-hole Web version
+                      description: Remote (Github) Pi-hole Web version (null if on custom branch)
                       example: "v6.1"
                     hash:
                       type: string
@@ -908,7 +922,7 @@ components:
                     version:
                       type: string
                       nullable: true
-                      description: Remote (Github) Pi-hole FTL version
+                      description: Remote (Github) Pi-hole FTL version (null if on custom branch)
                       example: "v6.1"
                     hash:
                       type: string

src/api/docs/content/specs/lists.yaml

@@ -93,6 +93,12 @@ components:
         responses:
           '204':
             description: Item deleted
+          '404':
+            description: Item not found
+            content:
+              application/json:
+                schema:
+                  $ref: 'common.yaml#/components/schemas/took'
           '400':
             description: Bad request
             content:
@@ -189,18 +195,14 @@ components:
               schema:
                 $ref: 'lists.yaml#/components/schemas/lists/post'
         responses:
-          '201':
-            description: Created item
+          '204':
+            description: Items deleted
+          '404':
+            description: Item not found
             content:
               application/json:
                 schema:
-                  allOf:
-                    - $ref: 'lists.yaml#/components/schemas/lists/get'
-                    - $ref: 'lists.yaml#/components/schemas/lists_processed'
-                    - $ref: 'common.yaml#/components/schemas/took'
-            headers:
-              Location:
-                $ref: 'common.yaml#/components/headers/Location'
+                  $ref: 'common.yaml#/components/schemas/took'
           '400':
             description: Bad request
             content:

src/api/docs/content/specs/network.yaml

@@ -93,6 +93,20 @@ components:
         responses:
           '204':
             description: No Content (deleted)
+          '404':
+            description: Not found
+            content:
+              application/json:
+                schema:
+                  $ref: 'common.yaml#/components/schemas/took'
+          '400':
+            description: Bad request
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'common.yaml#/components/errors/bad_request'
+                    - $ref: 'common.yaml#/components/schemas/took'
           '401':
             description: Unauthorized
             content:

src/api/history.c

@@ -66,13 +66,29 @@ int api_history_clients(struct ftl_conn *api)
 		JSON_SEND_OBJECT_UNLOCK(json);
 	}
 
+	// Get number of clients to return´
+	unsigned int Nc = min(counters->clients, config.webserver.api.maxClients.v.u16);
+	if(api->request->query_string != NULL)
+	{
+		// Does the user request a non-default number of clients
+		get_uint_var(api->request->query_string, "N", &Nc);
+
+		// Limit the number of clients to return to the number of
+		// clients to avoid possible overflows for very large N
+		// Also allow N=0 to return all clients
+		if((int)Nc > counters->clients || Nc == 0)
+			Nc = counters->clients;
+	}
+
+	// Lock shared memory
 	lock_shm();
 
 	// Get clients which the user doesn't want to see
 	// if skipclient[i] == true then this client should be hidden from
 	// returned data. We initialize it with false
 	bool *skipclient = calloc(counters->clients, sizeof(bool));
-	if(skipclient == NULL)
+	int *temparray = calloc(2*counters->clients, sizeof(int));
+	if(skipclient == NULL || temparray == NULL)
 	{
 		unlock_shm();
 		return send_json_error(api, 500,
@@ -81,19 +97,40 @@ int api_history_clients(struct ftl_conn *api)
 		                       NULL);
 	}
 
-	// Also skip clients included in others (in alias-clients)
+	// Skip clients included in others (in alias-clients)
 	for(int clientID = 0; clientID < counters->clients; clientID++)
 	{
 		// Get client pointer
 		const clientsData* client = getClient(clientID, true);
 		if(client == NULL)
 			continue;
+
+		// Check if this client should be skipped
 		if(!client->flags.aliasclient && client->aliasclient_id > -1)
 			skipclient[clientID] = true;
 	}
 
+	// Get MAX_CLIENTS clients with the highest number of queries
+	for(int clientID = 0; clientID < counters->clients; clientID++)
+	{
+		// Get client pointer
+		const clientsData* client = getClient(clientID, true);
+
+		// Skip invalid clients
+		if(client == NULL)
+			continue;
+
+		// Store clientID and number of queries in temporary array
+		temparray[2*clientID + 0] = clientID;
+		temparray[2*clientID + 1] = client->count;
+	}
+
+	// Sort temporary array
+	qsort(temparray, counters->clients, sizeof(int[2]), cmpdesc);
+
 	// Main return loop
 	cJSON *history = JSON_NEW_ARRAY();
+	int others_total = 0;
 	for(unsigned int slot = 0; slot < OVERTIME_SLOTS; slot++)
 	{
 		cJSON *item = JSON_NEW_OBJECT();
@@ -101,22 +138,31 @@ int api_history_clients(struct ftl_conn *api)
 
 		// Loop over clients to generate output to be sent to the client
 		cJSON *data = JSON_NEW_ARRAY();
-		for(int clientID = 0; clientID < counters->clients; clientID++)
+		int others = 0;
+		for(int id = 0; id < counters->clients; id++)
 		{
-			if(skipclient[clientID])
-				continue;
-
 			// Get client pointer
+			const int clientID = temparray[2*id + 0];
 			const clientsData* client = getClient(clientID, true);
 
-			// Skip invalid clients and also those managed by alias clients
-			if(client == NULL || client->aliasclient_id >= 0)
+			// Skip invalid (recycled) clients
+			if(client == NULL)
 				continue;
 
-			const int thisclient = client->overTime[slot];
+			// Skip clients which should be hidden and add them to the "others" counter.
+			// Also skip clients when we reached the maximum number of clients to return
+			if(skipclient[clientID] || id >= (int)Nc)
+			{
+				others += client->overTime[slot];
+				continue;
+			}
 
-			JSON_ADD_NUMBER_TO_ARRAY(data, thisclient);
+			JSON_ADD_NUMBER_TO_ARRAY(data, client->overTime[slot]);
 		}
+		// Add others as last element in the array
+		others_total += others;
+		JSON_ADD_NUMBER_TO_ARRAY(data, others);
+
 		JSON_ADD_ITEM_TO_OBJECT(item, "data", data);
 		JSON_ADD_ITEM_TO_ARRAY(history, item);
 	}
@@ -125,25 +171,40 @@ int api_history_clients(struct ftl_conn *api)
 
 	// Loop over clients to generate output to be sent to the client
 	cJSON *clients = JSON_NEW_ARRAY();
-	for(int clientID = 0; clientID < counters->clients; clientID++)
+	for(int id = 0; id < counters->clients; id++)
 	{
-		if(skipclient[clientID])
-			continue;
-
 		// Get client pointer
+		const int clientID = temparray[2*id + 0];
 		const clientsData* client = getClient(clientID, true);
+
+		// Skip invalid (recycled) clients
 		if(client == NULL)
 			continue;
 
+		// Skip clients which should be hidden. Also skip clients when
+		// we reached the maximum number of clients to return
+		if(skipclient[clientID] || id >= (int)Nc)
+			continue;
+
+		// Get client name and IP address
 		const char *client_ip = getstr(client->ippos);
 		const char *client_name = client->namepos != 0 ? getstr(client->namepos) : NULL;
 
+		// Create JSON object for this client
 		cJSON *item = JSON_NEW_OBJECT();
 		JSON_REF_STR_IN_OBJECT(item, "name", client_name);
 		JSON_REF_STR_IN_OBJECT(item, "ip", client_ip);
+		JSON_ADD_NUMBER_TO_OBJECT(item, "total", client->count);
 		JSON_ADD_ITEM_TO_ARRAY(clients, item);
 	}
 
+	// Add "others" client
+	cJSON *item = JSON_NEW_OBJECT();
+	JSON_REF_STR_IN_OBJECT(item, "name", "other clients");
+	JSON_REF_STR_IN_OBJECT(item, "ip", "0.0.0.0");
+	JSON_ADD_NUMBER_TO_OBJECT(item, "total", others_total);
+	JSON_ADD_ITEM_TO_ARRAY(clients, item);
+
 	// Unlock already here to avoid keeping the lock during JSON generation
 	// This is safe because we don't access any shared memory after this
 	// point and all strings in the JSON are references to idempotent shared
@@ -152,6 +213,7 @@ int api_history_clients(struct ftl_conn *api)
 
 	// Free memory
 	free(skipclient);
+	free(temparray);
 
 	JSON_ADD_ITEM_TO_OBJECT(json, "clients", clients);
 	JSON_SEND_OBJECT(json);

src/api/info.c

@@ -750,11 +750,26 @@ int api_info_version(struct ftl_conn *api)
 		//else if(strcmp(key, "FTL_VERSION") == 0)
 		//	JSON_COPY_STR_TO_OBJECT(ftl_local, "version", value);
 		else if(strcmp(key, "GITHUB_CORE_VERSION") == 0)
-			JSON_COPY_STR_TO_OBJECT(core_remote, "version", value);
+		{
+			if(strcmp(value, "null") == 0)
+				JSON_ADD_NULL_TO_OBJECT(core_remote, "version");
+			else
+				JSON_COPY_STR_TO_OBJECT(core_remote, "version", value);
+		}
 		else if(strcmp(key, "GITHUB_WEB_VERSION") == 0)
-			JSON_COPY_STR_TO_OBJECT(web_remote, "version", value);
+		{
+			if(strcmp(value, "null") == 0)
+				JSON_ADD_NULL_TO_OBJECT(web_remote, "version");
+			else
+				JSON_COPY_STR_TO_OBJECT(web_remote, "version", value);
+		}
 		else if(strcmp(key, "GITHUB_FTL_VERSION") == 0)
-			JSON_COPY_STR_TO_OBJECT(ftl_remote, "version", value);
+		{
+			if(strcmp(value, "null") == 0)
+				JSON_ADD_NULL_TO_OBJECT(ftl_remote, "version");
+			else
+				JSON_COPY_STR_TO_OBJECT(ftl_remote, "version", value);
+		}
 		else if(strcmp(key, "CORE_HASH") == 0)
 			JSON_COPY_STR_TO_OBJECT(core_local, "hash", value);
 		else if(strcmp(key, "WEB_HASH") == 0)
@@ -940,15 +955,18 @@ static int api_info_messages_DELETE(struct ftl_conn *api)
 	}
 
 	// Delete message with this ID from the database
-	delete_message(ids);
+	int deleted = 0;
+	delete_message(ids, &deleted);
 
 	// Free memory
 	free(id);
 	cJSON_free(ids);
 
-	// Send empty reply with code 204 No Content
+	// Send empty reply with codes:
+	// - 204 No Content (if any items were deleted)
+	// - 404 Not Found (if no items were deleted)
 	cJSON *json = JSON_NEW_OBJECT();
-	JSON_SEND_OBJECT_CODE(json, 204);
+	JSON_SEND_OBJECT_CODE(json, deleted > 0 ? 204 : 404);
 }
 
 int api_info_messages(struct ftl_conn *api)

src/api/list.c

@@ -705,7 +705,8 @@ static int api_list_remove(struct ftl_conn *api,
 	}
 
 	// From here on, we can assume the JSON payload is valid
-	if(gravityDB_delFromTable(listtype, array, &sql_msg))
+	unsigned int deleted = 0u;
+	if(gravityDB_delFromTable(listtype, array, &deleted, &sql_msg))
 	{
 		// Inform the resolver that it needs to reload gravity
 		set_event(RELOAD_GRAVITY);
@@ -714,9 +715,11 @@ static int api_list_remove(struct ftl_conn *api,
 		if(allocated_json)
 			cJSON_free(array);
 
-		// Send empty reply with code 204 No Content
+		// Send empty reply with codes:
+		// - 204 No Content (if any items were deleted)
+		// - 404 Not Found (if no items were deleted)
 		cJSON *json = JSON_NEW_OBJECT();
-		JSON_SEND_OBJECT_CODE(json, 204);
+		JSON_SEND_OBJECT_CODE(json, deleted > 0u ? 204 : 404);
 	}
 	else
 	{

src/api/network.c

@@ -440,7 +440,8 @@ static int api_network_devices_DELETE(struct ftl_conn *api)
 
 	// Delete row from network table by ID
 	const char *sql_msg = NULL;
-	if(!networkTable_deleteDevice(db, device_id, &sql_msg))
+	int deleted = 0;
+	if(!networkTable_deleteDevice(db, device_id, &deleted, &sql_msg))
 	{
 		// Add SQL message (may be NULL = not available)
 		return send_json_error(api, 500,
@@ -452,9 +453,11 @@ static int api_network_devices_DELETE(struct ftl_conn *api)
 	// Close database
 	dbclose(&db);
 
-	// Send empty reply with code 204 No Content
+	// Send empty reply with codes:
+	// - 204 No Content (if any items were deleted)
+	// - 404 Not Found (if no items were deleted)
 	cJSON *json = JSON_NEW_OBJECT();
-	JSON_SEND_OBJECT_CODE(json, 204);
+	JSON_SEND_OBJECT_CODE(json, deleted > 0 ? 204 : 404);
 }
 
 int api_network_devices(struct ftl_conn *api)

src/api/queries.c

@@ -475,14 +475,14 @@ int api_queries(struct ftl_conn *api)
 			if(rc != 0)
 			{
 				// Failed to compile regex
-				char errbuf[1024];
+				char errbuf[1024] = { 0 };
 				regerror(rc, &regex_domains[i], errbuf, sizeof(errbuf));
 				log_err("Failed to compile domain regex \"%s\": %s",
 					filter->valuestring, errbuf);
 				return send_json_error(api, 400,
-							"bad_request",
-							"Failed to compile domain regex",
-							filter->valuestring);
+				                       "bad_request",
+				                       "Failed to compile domain regex",
+				                       filter->valuestring);
 			}
 
 			i++;
@@ -524,14 +524,14 @@ int api_queries(struct ftl_conn *api)
 			if(rc != 0)
 			{
 				// Failed to compile regex
-				char errbuf[1024];
+				char errbuf[1024] = { 0 };
 				regerror(rc, &regex_clients[i], errbuf, sizeof(errbuf));
 				log_err("Failed to compile client regex \"%s\": %s",
 					filter->valuestring, errbuf);
 				return send_json_error(api, 400,
-							"bad_request",
-							"Failed to compile client regex",
-							filter->valuestring);
+				                       "bad_request",
+				                       "Failed to compile client regex",
+				                       filter->valuestring);
 			}
 
 			i++;

src/api/stats.c

@@ -42,7 +42,7 @@ static int __attribute__((pure)) cmpasc(const void *a, const void *b)
 } */
 
 // qsort subroutine, sort DESC
-static int __attribute__((pure)) cmpdesc(const void *a, const void *b)
+int __attribute__((pure)) cmpdesc(const void *a, const void *b)
 {
 	const int *elem1 = (int*)a;
 	const int *elem2 = (int*)b;
@@ -163,6 +163,7 @@ int api_stats_top_domains(struct ftl_conn *api)
 		return 0;
 	}
 
+
 	bool blocked = false; // Can be overwritten by query string
 	int count = 10;
 	// /api/stats/top_domains?blocked=true
@@ -246,7 +247,7 @@ int api_stats_top_domains(struct ftl_conn *api)
 			if(rc != 0)
 			{
 				// Failed to compile regex
-				char errbuf[1024];
+				char errbuf[1024] = { 0 };
 				regerror(rc, &regex_domains[i], errbuf, sizeof(errbuf));
 				log_err("Failed to compile domain regex \"%s\": %s",
 					filter->valuestring, errbuf);
@@ -433,7 +434,7 @@ int api_stats_top_clients(struct ftl_conn *api)
 			if(rc != 0)
 			{
 				// Failed to compile regex
-				char errbuf[1024];
+				char errbuf[1024] = { 0 };
 				regerror(rc, &regex_clients[i], errbuf, sizeof(errbuf));
 				log_err("Failed to compile client regex \"%s\": %s",
 					filter->valuestring, errbuf);

src/config/cli.c

@@ -160,8 +160,9 @@ static bool readStringValue(struct conf_item *conf_item, const char *value, stru
 			// Get password hash as allocated string (an empty string is hashed to an empty string)
 			char *pwhash = strlen(value) > 0 ? create_password(value) : strdup("");
 
-			// Verify that the password hash is valid
-			if(verify_password(value, pwhash, false) != PASSWORD_CORRECT)
+			// Verify that the password hash is either valid or empty
+			const enum password_result status = verify_password(value, pwhash, false);
+			if(status != PASSWORD_CORRECT && status != NO_PASSWORD_SET)
 			{
 				log_err("Failed to create password hash (verification failed), password remains unchanged");
 				free(pwhash);

src/config/config.c

@@ -972,6 +972,11 @@ void initConfig(struct config *conf)
 	conf->webserver.api.maxHistory.t = CONF_UINT;
 	conf->webserver.api.maxHistory.d.ui = MAXLOGAGE*3600;
 
+	conf->webserver.api.maxClients.k = "webserver.api.maxClients";
+	conf->webserver.api.maxClients.h = "Up to how many clients should be returned in the activity graph endpoint (/api/history/clients)?\n This setting can be overwritten at run-time using the parameter N";
+	conf->webserver.api.maxClients.t = CONF_UINT16;
+	conf->webserver.api.maxClients.d.u16 = 10;
+
 	conf->webserver.api.allow_destructive.k = "webserver.api.allow_destructive";
 	conf->webserver.api.allow_destructive.h = "Allow destructive API calls (e.g. deleting all queries, powering off the system, ...)";
 	conf->webserver.api.allow_destructive.t = CONF_BOOL;

src/config/config.h

@@ -245,6 +245,7 @@ struct config {
 			struct conf_item excludeClients;
 			struct conf_item excludeDomains;
 			struct conf_item maxHistory;
+			struct conf_item maxClients;
 			struct conf_item allow_destructive;
 			struct {
 				struct conf_item limit;

src/config/dnsmasq_config.c

@@ -307,7 +307,6 @@ bool __attribute__((const)) write_dnsmasq_config(struct config *conf, bool test_
 	}
 
 	write_config_header(pihole_conf, "Dnsmasq config for Pi-hole's FTLDNS");
-	fputs("addn-hosts=/etc/pihole/local.list\n", pihole_conf);
 	fputs("hostsdir="DNSMASQ_HOSTSDIR"\n", pihole_conf);
 	fputs("\n", pihole_conf);
 	fputs("# Don't read /etc/resolv.conf. Get upstream servers only from the configuration\n", pihole_conf);

src/config/password.c

@@ -328,6 +328,7 @@ enum password_result verify_login(const char *password)
 		log_debug(DEBUG_API, "App password correct");
 		return APPPASSWORD_CORRECT;
 	}
+
 	// Return result
 	return pw;
 }
@@ -336,7 +337,7 @@ enum password_result verify_password(const char *password, const char *pwhash, c
 {
 	// No password set
 	if(pwhash == NULL || pwhash[0] == '\0')
-		return PASSWORD_CORRECT;
+		return NO_PASSWORD_SET;
 
 	// No password supplied
 	if(password == NULL || password[0] == '\0')
@@ -606,8 +607,9 @@ bool set_and_check_password(struct conf_item *conf_item, const char *password)
 	// Get password hash as allocated string (an empty string is hashed to an empty string)
 	char *pwhash = strlen(password) > 0 ? create_password(password) : strdup("");
 
-	// Verify that the password hash is valid
-	if(verify_password(password, pwhash, false) != PASSWORD_CORRECT)
+	// Verify that the password hash is valid or that no password is set
+	const enum password_result status = verify_password(password, pwhash, false);
+	if(status != PASSWORD_CORRECT && status != NO_PASSWORD_SET)
 	{
 		free(pwhash);
 		log_warn("Failed to create password hash (verification failed), password remains unchanged");

src/config/password.h

@@ -26,6 +26,7 @@ enum password_result {
 	PASSWORD_INCORRECT = 0,
 	PASSWORD_CORRECT = 1,
 	APPPASSWORD_CORRECT = 2,
+	NO_PASSWORD_SET = 3,
 	PASSWORD_RATE_LIMITED = -1
 } __attribute__((packed));
 

src/database/gravity-db.c

@@ -1792,8 +1792,9 @@ bool gravityDB_addToTable(const enum gravity_list_type listtype, tablerow *row,
 	return okay;
 }
 
-bool gravityDB_delFromTable(const enum gravity_list_type listtype, const cJSON* array, const char **message)
+bool gravityDB_delFromTable(const enum gravity_list_type listtype, const cJSON* array, unsigned int *deleted, const char **message)
 {
+	// Return early if database is not available
 	if(gravity_db == NULL)
 	{
 		*message = "Database not available";
@@ -2004,6 +2005,9 @@ bool gravityDB_delFromTable(const enum gravity_list_type listtype, const cJSON*
 
 			break;
 		}
+
+		// Add number of deleted rows
+		*deleted += sqlite3_changes(gravity_db);
 	}
 
 	// Drop temporary table

src/database/gravity-db.h

@@ -69,7 +69,7 @@ bool gravityDB_readTableGetRow(const enum gravity_list_type listtype, tablerow *
 void gravityDB_readTableFinalize(void);
 bool gravityDB_addToTable(const enum gravity_list_type listtype, tablerow *row,
                           const char **message, const enum http_method method);
-bool gravityDB_delFromTable(const enum gravity_list_type listtype, const cJSON* array, const char **message);
+bool gravityDB_delFromTable(const enum gravity_list_type listtype, const cJSON* array, unsigned int *deleted, const char **message);
 bool gravityDB_edit_groups(const enum gravity_list_type listtype, cJSON *groups,
                            const tablerow *row, const char **message);
 

src/database/message-table.c

@@ -378,7 +378,7 @@ end_of_add_message: // Close database connection
 	return rowid;
 }
 
-bool delete_message(cJSON *ids)
+bool delete_message(cJSON *ids, int *deleted)
 {
 	// Return early if database is known to be broken
 	if(FTLDBerror())
@@ -413,6 +413,10 @@ bool delete_message(cJSON *ids)
 			log_err("SQL error (%i): %s", sqlite3_errcode(db), sqlite3_errmsg(db));
 			return false;
 		}
+
+		// Add to deleted count
+		*deleted += sqlite3_changes(db);
+
 		sqlite3_reset(res);
 		sqlite3_clear_bindings(res);
 	}

src/database/message-table.h

@@ -16,7 +16,7 @@
 int count_messages(const bool filter_dnsmasq_warnings);
 bool format_messages(cJSON *array);
 bool create_message_table(sqlite3 *db);
-bool delete_message(cJSON *ids);
+bool delete_message(cJSON *ids, int *deleted);
 bool flush_message_table(void);
 void logg_regex_warning(const char *type, const char *warning, const int dbindex, const char *regex);
 void logg_subnet_warning(const char *ip, const int matching_count, const char *matching_ids,

src/database/network-table.c

@@ -2425,7 +2425,7 @@ void networkTable_readIPsFinalize(sqlite3_stmt *read_stmt)
 	sqlite3_finalize(read_stmt);
 }
 
-bool networkTable_deleteDevice(sqlite3 *db, const int id, const char **message)
+bool networkTable_deleteDevice(sqlite3 *db, const int id, int *deleted, const char **message)
 {
 	// First step: Delete all associated IPs of this device
 	// Prepare SQLite statement
@@ -2462,6 +2462,9 @@ bool networkTable_deleteDevice(sqlite3 *db, const int id, const char **message)
 		return false;
 	}
 
+	// Check if we deleted any rows
+	*deleted += sqlite3_changes(db);
+
 	// Finalize statement
 	sqlite3_finalize(stmt);
 
@@ -2498,6 +2501,9 @@ bool networkTable_deleteDevice(sqlite3 *db, const int id, const char **message)
 		return false;
 	}
 
+	// Check if we deleted any rows
+	*deleted += sqlite3_changes(db);
+
 	// Finalize statement
 	sqlite3_finalize(stmt);
 

src/database/network-table.h

@@ -52,6 +52,6 @@ bool networkTable_readIPs(sqlite3 *db, sqlite3_stmt **read_stmt, const int id, c
 bool networkTable_readIPsGetRecord(sqlite3_stmt *read_stmt, network_addresses_record *network_addresses, const char **message);
 void networkTable_readIPsFinalize(sqlite3_stmt *read_stmt);
 
-bool networkTable_deleteDevice(sqlite3 *db, const int id, const char **message);
+bool networkTable_deleteDevice(sqlite3 *db, const int id, int *deleted, const char **message);
 
 #endif //NETWORKTABLE_H

src/database/shell.c

@@ -298,7 +298,7 @@ extern LPWSTR sqlite3_win32_utf8_to_unicode(const char *zText);
 ** CIO_WIN_WC_XLATE is defined as 0 or 1, reflecting whether console I/O
 ** translation for Windows is effected for the build.
 */
-
+#define HAVE_CONSOLE_IO_H 1
 #ifndef SQLITE_INTERNAL_LINKAGE
 # define SQLITE_INTERNAL_LINKAGE extern /* external to translation unit */
 # include <stdio.h>
@@ -436,8 +436,8 @@ SQLITE_INTERNAL_LINKAGE int ePutsUtf8(const char *z);
 #ifdef CONSIO_SPUTB
 SQLITE_INTERNAL_LINKAGE int
 fPutbUtf8(FILE *pfOut, const char *cBuf, int nAccept);
-#endif
 /* Like fPutbUtf8 except stream is always the designated output. */
+#endif
 SQLITE_INTERNAL_LINKAGE int
 oPutbUtf8(const char *cBuf, int nAccept);
 /* Like fPutbUtf8 except stream is always the designated error. */
@@ -577,9 +577,11 @@ zSkipValidUtf8(const char *z, int nAccept, long ccm);
 # include <stdlib.h>
 # include <limits.h>
 # include <assert.h>
-# include "console_io.h"
 /* # include "sqlite3.h" */
 #endif
+#ifndef HAVE_CONSOLE_IO_H
+# include "console_io.h"
+#endif
 
 #ifndef SQLITE_CIO_NO_TRANSLATE
 # if (defined(_WIN32) || defined(WIN32)) && !SQLITE_OS_WINRT
@@ -1102,12 +1104,11 @@ zSkipValidUtf8(const char *z, int nAccept, long ccm){
 #endif /*!(defined(SQLITE_CIO_NO_UTF8SCAN)&&defined(SQLITE_CIO_NO_TRANSLATE))*/
 
 #ifndef SQLITE_CIO_NO_TRANSLATE
-
-#ifdef CONSIO_SPUTB
+# ifdef CONSIO_SPUTB
 SQLITE_INTERNAL_LINKAGE int
 fPutbUtf8(FILE *pfO, const char *cBuf, int nAccept){
   assert(pfO!=0);
-# if CIO_WIN_WC_XLATE
+#  if CIO_WIN_WC_XLATE
   PerStreamTags pst = PST_INITIALIZER; /* for unknown streams */
   PerStreamTags *ppst = getEmitStreamInfo(0, &pst, &pfO);
   if( pstReachesConsole(ppst) ){
@@ -1117,13 +1118,13 @@ fPutbUtf8(FILE *pfO, const char *cBuf, int nAccept){
     if( 0 == isKnownWritable(ppst->pf) ) restoreConsoleArb(ppst);
     return rv;
   }else {
-# endif
+#  endif
     return (int)fwrite(cBuf, 1, nAccept, pfO);
-# if CIO_WIN_WC_XLATE
+#  if CIO_WIN_WC_XLATE
   }
-# endif
+#  endif
 }
-#endif /* defined(CONSIO_SPUTB) */
+# endif
 
 SQLITE_INTERNAL_LINKAGE int
 oPutbUtf8(const char *cBuf, int nAccept){
@@ -1234,6 +1235,7 @@ SQLITE_INTERNAL_LINKAGE char* fGetsUtf8(char *cBuf, int ncMax, FILE *pfIn){
 /************************* End ../ext/consio/console_io.c ********************/
 
 #ifndef SQLITE_SHELL_FIDDLE
+
 /* From here onward, fgets() is redirected to the console_io library. */
 # define fgets(b,n,f) fGetsUtf8(b,n,f)
 /*
@@ -1258,6 +1260,7 @@ SQLITE_INTERNAL_LINKAGE char* fGetsUtf8(char *cBuf, int ncMax, FILE *pfIn){
 # define eputz(z) ePutsUtf8(z)
 # define eputf ePrintfUtf8
 # define oputb(buf,na) oPutbUtf8(buf,na)
+
 #else
 /* For Fiddle, all console handling and emit redirection is omitted. */
 # define sputz(fp,z) fputs(z,fp)
@@ -1341,7 +1344,7 @@ static void endTimer(void){
     sqlite3_int64 iEnd = timeOfDay();
     struct rusage sEnd;
     getrusage(RUSAGE_SELF, &sEnd);
-    oputf("Run Time: real %.3f user %f sys %f\n",
+    sputf(stdout, "Run Time: real %.3f user %f sys %f\n",
           (iEnd - iBegin)*0.001,
           timeDiff(&sBegin.ru_utime, &sEnd.ru_utime),
           timeDiff(&sBegin.ru_stime, &sEnd.ru_stime));
@@ -1420,7 +1423,7 @@ static void endTimer(void){
     FILETIME ftCreation, ftExit, ftKernelEnd, ftUserEnd;
     sqlite3_int64 ftWallEnd = timeOfDay();
     getProcessTimesAddr(hProcess,&ftCreation,&ftExit,&ftKernelEnd,&ftUserEnd);
-    oputf("Run Time: real %.3f user %f sys %f\n",
+    sputf(stdout, "Run Time: real %.3f user %f sys %f\n",
           (ftWallEnd - ftWallBegin)*0.001,
           timeDiff(&ftUserBegin, &ftUserEnd),
           timeDiff(&ftKernelBegin, &ftKernelEnd));
@@ -1717,14 +1720,14 @@ static int strlenChar(const char *z){
 */
 static FILE * openChrSource(const char *zFile){
 #if defined(_WIN32) || defined(WIN32)
-  struct _stat x = {0};
+  struct __stat64 x = {0};
 # define STAT_CHR_SRC(mode) ((mode & (_S_IFCHR|_S_IFIFO|_S_IFREG))!=0)
   /* On Windows, open first, then check the stream nature. This order
   ** is necessary because _stat() and sibs, when checking a named pipe,
   ** effectively break the pipe as its supplier sees it. */
   FILE *rv = fopen(zFile, "rb");
   if( rv==0 ) return 0;
-  if( _fstat(_fileno(rv), &x) != 0
+  if( _fstat64(_fileno(rv), &x) != 0
       || !STAT_CHR_SRC(x.st_mode)){
     fclose(rv);
     rv = 0;
@@ -14829,6 +14832,7 @@ static int dbdataNext(sqlite3_vtab_cursor *pCursor){
             bNextPage = 1;
           }else{
             iOff += dbdataGetVarintU32(&pCsr->aPage[iOff], &nPayload);
+            if( nPayload>0x7fffff00 ) nPayload &= 0x3fff;
           }
     
           /* If this is a leaf intkey cell, load the rowid */
@@ -18145,6 +18149,7 @@ struct ShellState {
   u8 eTraceType;         /* SHELL_TRACE_* value for type of trace */
   u8 bSafeMode;          /* True to prohibit unsafe operations */
   u8 bSafeModePersist;   /* The long-term value of bSafeMode */
+  u8 eRestoreState;      /* See comments above doAutoDetectRestore() */
   ColModeOpts cmOpts;    /* Option values affecting columnar mode output */
   unsigned statsOn;      /* True to display memory stats before each finalize */
   unsigned mEqpLines;    /* Mask of vertical lines in the EQP output graph */
@@ -22172,7 +22177,6 @@ static void open_db(ShellState *p, int openFlags){
         break;
       }
     }
-    globalDb = p->db;
     if( p->db==0 || SQLITE_OK!=sqlite3_errcode(p->db) ){
       eputf("Error: unable to open database \"%s\": %s\n",
             zDbFilename, sqlite3_errmsg(p->db));
@@ -22189,6 +22193,7 @@ static void open_db(ShellState *p, int openFlags){
               zDbFilename);
       }
     }
+    globalDb = p->db;
     sqlite3_db_config(p->db, SQLITE_DBCONFIG_STMT_SCANSTATUS, (int)0, (int*)0);
 
     /* Reflect the use or absence of --unsafe-testing invocation. */
@@ -23584,7 +23589,6 @@ static int lintDotCommand(
   return SQLITE_ERROR;
 }
 
-#if !defined SQLITE_OMIT_VIRTUALTABLE
 static void shellPrepare(
   sqlite3 *db,
   int *pRc,
@@ -23603,12 +23607,8 @@ static void shellPrepare(
 
 /*
 ** Create a prepared statement using printf-style arguments for the SQL.
-**
-** This routine is could be marked "static".  But it is not always used,
-** depending on compile-time options.  By omitting the "static", we avoid
-** nuisance compiler warnings about "defined but not used".
 */
-void shellPreparePrintf(
+static void shellPreparePrintf(
   sqlite3 *db,
   int *pRc,
   sqlite3_stmt **ppStmt,
@@ -23631,13 +23631,10 @@ void shellPreparePrintf(
   }
 }
 
-/* Finalize the prepared statement created using shellPreparePrintf().
-**
-** This routine is could be marked "static".  But it is not always used,
-** depending on compile-time options.  By omitting the "static", we avoid
-** nuisance compiler warnings about "defined but not used".
+/* 
+** Finalize the prepared statement created using shellPreparePrintf().
 */
-void shellFinalize(
+static void shellFinalize(
   int *pRc,
   sqlite3_stmt *pStmt
 ){
@@ -23653,6 +23650,7 @@ void shellFinalize(
   }
 }
 
+#if !defined SQLITE_OMIT_VIRTUALTABLE
 /* Reset the prepared statement created using shellPreparePrintf().
 **
 ** This routine is could be marked "static".  But it is not always used,
@@ -24719,6 +24717,30 @@ FROM (\
   }
 }
 
+/*
+** Check if the sqlite_schema table contains one or more virtual tables. If
+** parameter zLike is not NULL, then it is an SQL expression that the
+** sqlite_schema row must also match. If one or more such rows are found,
+** print the following warning to the output:
+**
+** WARNING: Script requires that SQLITE_DBCONFIG_DEFENSIVE be disabled
+*/
+static int outputDumpWarning(ShellState *p, const char *zLike){
+  int rc = SQLITE_OK;
+  sqlite3_stmt *pStmt = 0;
+  shellPreparePrintf(p->db, &rc, &pStmt,
+    "SELECT 1 FROM sqlite_schema o WHERE "
+    "sql LIKE 'CREATE VIRTUAL TABLE%%' AND %s", zLike ? zLike : "true"
+  );
+  if( rc==SQLITE_OK && sqlite3_step(pStmt)==SQLITE_ROW ){
+    oputz("/* WARNING: "
+          "Script requires that SQLITE_DBCONFIG_DEFENSIVE be disabled */\n"
+    );
+  }
+  shellFinalize(&rc, pStmt);
+  return rc;
+}
+
 /*
 ** If an input line begins with "." then invoke this routine to
 ** process that line.
@@ -25181,6 +25203,7 @@ static int do_meta_command(char *zLine, ShellState *p){
 
     open_db(p, 0);
 
+    outputDumpWarning(p, zLike);
     if( (p->shellFlgs & SHFLG_DumpDataOnly)==0 ){
       /* When playing back a "dump", the content might appear in an order
       ** which causes immediate foreign key constraints to be violated.
@@ -27609,6 +27632,7 @@ static int do_meta_command(char *zLine, ShellState *p){
     {"fk_no_action",       SQLITE_TESTCTRL_FK_NO_ACTION, 0, "BOOLEAN"       },
     {"imposter",         SQLITE_TESTCTRL_IMPOSTER,1,"SCHEMA ON/OFF ROOTPAGE"},
     {"internal_functions", SQLITE_TESTCTRL_INTERNAL_FUNCTIONS,0,""          },
+    {"json_selfcheck",     SQLITE_TESTCTRL_JSON_SELFCHECK ,0,"BOOLEAN"      },
     {"localtime_fault",    SQLITE_TESTCTRL_LOCALTIME_FAULT,0,"BOOLEAN"      },
     {"never_corrupt",      SQLITE_TESTCTRL_NEVER_CORRUPT,1, "BOOLEAN"       },
     {"optimizations",      SQLITE_TESTCTRL_OPTIMIZATIONS,0,"DISABLE-MASK"   },
@@ -27827,6 +27851,16 @@ static int do_meta_command(char *zLine, ShellState *p){
             isOk = 3;
           }
           break;
+        case SQLITE_TESTCTRL_JSON_SELFCHECK:
+          if( nArg==2 ){
+            rc2 = -1;
+            isOk = 1;
+          }else{
+            rc2 = booleanValue(azArg[2]);
+            isOk = 3;
+          }
+          sqlite3_test_control(testctrl, &rc2);
+          break;
       }
     }
     if( isOk==0 && iCtrl>=0 ){
@@ -28233,6 +28267,88 @@ static int line_is_complete(char *zSql, int nSql){
   return rc;
 }
 
+/*
+** This function is called after processing each line of SQL in the
+** runOneSqlLine() function. Its purpose is to detect scenarios where
+** defensive mode should be automatically turned off. Specifically, when
+**
+**   1. The first line of input is "PRAGMA foreign_keys=OFF;",
+**   2. The second line of input is "BEGIN TRANSACTION;",
+**   3. The database is empty, and
+**   4. The shell is not running in --safe mode.
+** 
+** The implementation uses the ShellState.eRestoreState to maintain state:
+**
+**    0: Have not seen any SQL.
+**    1: Have seen "PRAGMA foreign_keys=OFF;".
+**    2-6: Currently running .dump transaction. If the "2" bit is set,
+**         disable DEFENSIVE when done. If "4" is set, disable DQS_DDL.
+**    7: Nothing left to do. This function becomes a no-op.
+*/
+static int doAutoDetectRestore(ShellState *p, const char *zSql){
+  int rc = SQLITE_OK;
+
+  if( p->eRestoreState<7 ){
+    switch( p->eRestoreState ){
+      case 0: {
+        const char *zExpect = "PRAGMA foreign_keys=OFF;";
+        assert( strlen(zExpect)==24 );
+        if( p->bSafeMode==0 && memcmp(zSql, zExpect, 25)==0 ){
+          p->eRestoreState = 1;
+        }else{
+          p->eRestoreState = 7;
+        }
+        break;
+      };
+  
+      case 1: {
+        int bIsDump = 0;
+        const char *zExpect = "BEGIN TRANSACTION;";
+        assert( strlen(zExpect)==18 );
+        if( memcmp(zSql, zExpect, 19)==0 ){
+          /* Now check if the database is empty. */
+          const char *zQuery = "SELECT 1 FROM sqlite_schema LIMIT 1";
+          sqlite3_stmt *pStmt = 0;
+  
+          bIsDump = 1;
+          shellPrepare(p->db, &rc, zQuery, &pStmt);
+          if( rc==SQLITE_OK && sqlite3_step(pStmt)==SQLITE_ROW ){
+            bIsDump = 0;
+          }
+          shellFinalize(&rc, pStmt);
+        }
+        if( bIsDump && rc==SQLITE_OK ){
+          int bDefense = 0;
+          int bDqsDdl = 0;
+          sqlite3_db_config(p->db, SQLITE_DBCONFIG_DEFENSIVE, -1, &bDefense);
+          sqlite3_db_config(p->db, SQLITE_DBCONFIG_DQS_DDL, -1, &bDqsDdl);
+          sqlite3_db_config(p->db, SQLITE_DBCONFIG_DEFENSIVE, 0, 0);
+          sqlite3_db_config(p->db, SQLITE_DBCONFIG_DQS_DDL, 1, 0);
+          p->eRestoreState = (bDefense ? 2 : 0) + (bDqsDdl ? 4 : 0);
+        }else{
+          p->eRestoreState = 7;
+        }
+        break;
+      }
+  
+      default: {
+        if( sqlite3_get_autocommit(p->db) ){
+          if( (p->eRestoreState & 2) ){
+            sqlite3_db_config(p->db, SQLITE_DBCONFIG_DEFENSIVE, 1, 0);
+          }
+          if( (p->eRestoreState & 4) ){
+            sqlite3_db_config(p->db, SQLITE_DBCONFIG_DQS_DDL, 0, 0);
+          }
+          p->eRestoreState = 7;
+        }
+        break;
+      }
+    }
+  }
+
+  return rc;
+}
+
 /*
 ** Run a single line of SQL.  Return the number of errors.
 */
@@ -28280,6 +28396,8 @@ static int runOneSqlLine(ShellState *p, char *zSql, FILE *in, int startline){
             sqlite3_changes64(p->db), sqlite3_total_changes64(p->db));
     oputf("%s\n", zLineBuf);
   }
+
+  if( doAutoDetectRestore(p, zSql) ) return 1;
   return 0;
 }
 
@@ -28713,14 +28831,14 @@ static void printBold(const char *zText){
          FOREGROUND_RED|FOREGROUND_INTENSITY
   );
 #endif
-  oputz(zText);
+  sputz(stdout, zText);
 #if !SQLITE_OS_WINRT
   SetConsoleTextAttribute(out, defaultScreenInfo.wAttributes);
 #endif
 }
 #else
 static void printBold(const char *zText){
-  oputf("\033[1m%s\033[0m", zText);
+  sputf(stdout, "\033[1m%s\033[0m", zText);
 }
 #endif
 
@@ -28914,10 +29032,6 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
     }else if( cli_strcmp(z,"-init")==0 ){
       zInitFile = cmdline_option_value(argc, argv, ++i);
     }else if( cli_strcmp(z,"-interactive")==0 ){
-      /* Need to check for interactive override here to so that it can
-      ** affect console setup (for Windows only) and testing thereof.
-      */
-      stdin_is_interactive = 1;
     }else if( cli_strcmp(z,"-batch")==0 ){
       /* Need to check for batch mode here to so we can avoid printing
       ** informational messages (like from process_sqliterc) before
@@ -29187,11 +29301,14 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
     }else if( cli_strcmp(z,"-bail")==0 ){
       /* No-op.  The bail_on_error flag should already be set. */
     }else if( cli_strcmp(z,"-version")==0 ){
-      oputf("%s %s (%d-bit)\n", sqlite3_libversion(), sqlite3_sourceid(),
-            8*(int)sizeof(char*));
+      sputf(stdout, "%s %s (%d-bit)\n",
+            sqlite3_libversion(), sqlite3_sourceid(), 8*(int)sizeof(char*));
       return 0;
     }else if( cli_strcmp(z,"-interactive")==0 ){
-      /* already handled */
+      /* Need to check for interactive override here to so that it can
+      ** affect console setup (for Windows only) and testing thereof.
+      */
+      stdin_is_interactive = 1;
     }else if( cli_strcmp(z,"-batch")==0 ){
       /* already handled */
     }else if( cli_strcmp(z,"-utf8")==0 ){
@@ -29321,13 +29438,13 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
 #else
 # define SHELL_CIO_CHAR_SET ""
 #endif
-      oputf("SQLite version %s %.19s%s\n" /*extra-version-info*/
+      sputf(stdout, "SQLite version %s %.19s%s\n" /*extra-version-info*/
             "Enter \".help\" for usage hints.\n",
             sqlite3_libversion(), sqlite3_sourceid(), SHELL_CIO_CHAR_SET);
       if( warnInmemoryDb ){
-        oputz("Connected to a ");
+        sputz(stdout, "Connected to a ");
         printBold("transient in-memory database");
-        oputz(".\nUse \".open FILENAME\" to reopen on a"
+        sputz(stdout, ".\nUse \".open FILENAME\" to reopen on a"
               " persistent database.\n");
       }
       zHistory = getenv("SQLITE_HISTORY");

src/database/sqlite3.h

@@ -146,9 +146,9 @@ extern "C" {
 ** [sqlite3_libversion_number()], [sqlite3_sourceid()],
 ** [sqlite_version()] and [sqlite_source_id()].
 */
-#define SQLITE_VERSION        "3.44.2"
-#define SQLITE_VERSION_NUMBER 3044002
-#define SQLITE_SOURCE_ID      "2023-11-24 11:41:44 ebead0e7230cd33bcec9f95d2183069565b9e709bf745c9b5db65cc0cbf92c0f"
+#define SQLITE_VERSION        "3.45.0"
+#define SQLITE_VERSION_NUMBER 3045000
+#define SQLITE_SOURCE_ID      "2024-01-15 17:01:13 1066602b2b1976fe58b5150777cced894af17c803e068f5918390d6915b46e1d"
 
 /*
 ** CAPI3REF: Run-Time Library Version Numbers
@@ -3954,15 +3954,17 @@ SQLITE_API void sqlite3_free_filename(sqlite3_filename);
 ** </ul>
 **
 ** ^The sqlite3_errmsg() and sqlite3_errmsg16() return English-language
-** text that describes the error, as either UTF-8 or UTF-16 respectively.
+** text that describes the error, as either UTF-8 or UTF-16 respectively,
+** or NULL if no error message is available.
 ** (See how SQLite handles [invalid UTF] for exceptions to this rule.)
 ** ^(Memory to hold the error message string is managed internally.
 ** The application does not need to worry about freeing the result.
 ** However, the error string might be overwritten or deallocated by
 ** subsequent calls to other SQLite interface functions.)^
 **
-** ^The sqlite3_errstr() interface returns the English-language text
-** that describes the [result code], as UTF-8.
+** ^The sqlite3_errstr(E) interface returns the English-language text
+** that describes the [result code] E, as UTF-8, or NULL if E is not an
+** result code for which a text error message is available.
 ** ^(Memory to hold the error message string is managed internally
 ** and must not be freed by the application)^.
 **
@@ -8037,9 +8039,11 @@ SQLITE_API int sqlite3_vfs_unregister(sqlite3_vfs*);
 **
 ** ^(Some systems (for example, Windows 95) do not support the operation
 ** implemented by sqlite3_mutex_try().  On those systems, sqlite3_mutex_try()
-** will always return SQLITE_BUSY. The SQLite core only ever uses
-** sqlite3_mutex_try() as an optimization so this is acceptable
-** behavior.)^
+** will always return SQLITE_BUSY. In most cases the SQLite core only uses
+** sqlite3_mutex_try() as an optimization, so this is acceptable
+** behavior. The exceptions are unix builds that set the
+** SQLITE_ENABLE_SETLK_TIMEOUT build option. In that case a working
+** sqlite3_mutex_try() is required.)^
 **
 ** ^The sqlite3_mutex_leave() routine exits a mutex that was
 ** previously entered by the same thread.   The behavior
@@ -8298,6 +8302,7 @@ SQLITE_API int sqlite3_test_control(int op, ...);
 #define SQLITE_TESTCTRL_ASSERT                  12
 #define SQLITE_TESTCTRL_ALWAYS                  13
 #define SQLITE_TESTCTRL_RESERVE                 14  /* NOT USED */
+#define SQLITE_TESTCTRL_JSON_SELFCHECK          14
 #define SQLITE_TESTCTRL_OPTIMIZATIONS           15
 #define SQLITE_TESTCTRL_ISKEYWORD               16  /* NOT USED */
 #define SQLITE_TESTCTRL_SCRATCHMALLOC           17  /* NOT USED */
@@ -12811,8 +12816,11 @@ struct Fts5PhraseIter {
 **   created with the "columnsize=0" option.
 **
 ** xColumnText:
-**   This function attempts to retrieve the text of column iCol of the
-**   current document. If successful, (*pz) is set to point to a buffer
+**   If parameter iCol is less than zero, or greater than or equal to the
+**   number of columns in the table, SQLITE_RANGE is returned.
+**
+**   Otherwise, this function attempts to retrieve the text of column iCol of
+**   the current document. If successful, (*pz) is set to point to a buffer
 **   containing the text in utf-8 encoding, (*pn) is set to the size in bytes
 **   (not characters) of the buffer and SQLITE_OK is returned. Otherwise,
 **   if an error occurs, an SQLite error code is returned and the final values
@@ -12822,8 +12830,10 @@ struct Fts5PhraseIter {
 **   Returns the number of phrases in the current query expression.
 **
 ** xPhraseSize:
-**   Returns the number of tokens in phrase iPhrase of the query. Phrases
-**   are numbered starting from zero.
+**   If parameter iCol is less than zero, or greater than or equal to the
+**   number of phrases in the current query, as returned by xPhraseCount,
+**   0 is returned. Otherwise, this function returns the number of tokens in
+**   phrase iPhrase of the query. Phrases are numbered starting from zero.
 **
 ** xInstCount:
 **   Set *pnInst to the total number of occurrences of all phrases within
@@ -12839,12 +12849,13 @@ struct Fts5PhraseIter {
 **   Query for the details of phrase match iIdx within the current row.
 **   Phrase matches are numbered starting from zero, so the iIdx argument
 **   should be greater than or equal to zero and smaller than the value
-**   output by xInstCount().
+**   output by xInstCount(). If iIdx is less than zero or greater than
+**   or equal to the value returned by xInstCount(), SQLITE_RANGE is returned.
 **
-**   Usually, output parameter *piPhrase is set to the phrase number, *piCol
+**   Otherwise, output parameter *piPhrase is set to the phrase number, *piCol
 **   to the column in which it occurs and *piOff the token offset of the
-**   first token of the phrase. Returns SQLITE_OK if successful, or an error
-**   code (i.e. SQLITE_NOMEM) if an error occurs.
+**   first token of the phrase. SQLITE_OK is returned if successful, or an
+**   error code (i.e. SQLITE_NOMEM) if an error occurs.
 **
 **   This API can be quite slow if used with an FTS5 table created with the
 **   "detail=none" or "detail=column" option.
@@ -12870,6 +12881,10 @@ struct Fts5PhraseIter {
 **   Invoking Api.xUserData() returns a copy of the pointer passed as
 **   the third argument to pUserData.
 **
+**   If parameter iPhrase is less than zero, or greater than or equal to
+**   the number of phrases in the query, as returned by xPhraseCount(),
+**   this function returns SQLITE_RANGE.
+**
 **   If the callback function returns any value other than SQLITE_OK, the
 **   query is abandoned and the xQueryPhrase function returns immediately.
 **   If the returned value is SQLITE_DONE, xQueryPhrase returns SQLITE_OK.
@@ -12984,9 +12999,42 @@ struct Fts5PhraseIter {
 **
 ** xPhraseNextColumn()
 **   See xPhraseFirstColumn above.
+**
+** xQueryToken(pFts5, iPhrase, iToken, ppToken, pnToken)
+**   This is used to access token iToken of phrase iPhrase of the current
+**   query. Before returning, output parameter *ppToken is set to point
+**   to a buffer containing the requested token, and *pnToken to the
+**   size of this buffer in bytes.
+**
+**   If iPhrase or iToken are less than zero, or if iPhrase is greater than
+**   or equal to the number of phrases in the query as reported by
+**   xPhraseCount(), or if iToken is equal to or greater than the number of
+**   tokens in the phrase, SQLITE_RANGE is returned and *ppToken and *pnToken
+     are both zeroed.
+**
+**   The output text is not a copy of the query text that specified the
+**   token. It is the output of the tokenizer module. For tokendata=1
+**   tables, this includes any embedded 0x00 and trailing data.
+**
+** xInstToken(pFts5, iIdx, iToken, ppToken, pnToken)
+**   This is used to access token iToken of phrase hit iIdx within the
+**   current row. If iIdx is less than zero or greater than or equal to the
+**   value returned by xInstCount(), SQLITE_RANGE is returned.  Otherwise,
+**   output variable (*ppToken) is set to point to a buffer containing the
+**   matching document token, and (*pnToken) to the size of that buffer in
+**   bytes. This API is not available if the specified token matches a
+**   prefix query term. In that case both output variables are always set
+**   to 0.
+**
+**   The output text is not a copy of the document text that was tokenized.
+**   It is the output of the tokenizer module. For tokendata=1 tables, this
+**   includes any embedded 0x00 and trailing data.
+**
+**   This API can be quite slow if used with an FTS5 table created with the
+**   "detail=none" or "detail=column" option.
 */
 struct Fts5ExtensionApi {
-  int iVersion;                   /* Currently always set to 2 */
+  int iVersion;                   /* Currently always set to 3 */
 
   void *(*xUserData)(Fts5Context*);
 
@@ -13021,6 +13069,13 @@ struct Fts5ExtensionApi {
 
   int (*xPhraseFirstColumn)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*);
   void (*xPhraseNextColumn)(Fts5Context*, Fts5PhraseIter*, int *piCol);
+
+  /* Below this point are iVersion>=3 only */
+  int (*xQueryToken)(Fts5Context*,
+      int iPhrase, int iToken,
+      const char **ppToken, int *pnToken
+  );
+  int (*xInstToken)(Fts5Context*, int iIdx, int iToken, const char**, int*);
 };
 
 /*

src/datastructure.c

@@ -314,7 +314,7 @@ int _findClientID(const char *clientIP, const bool count, const bool aliasclient
 	// Set all MAC address bytes to zero
 	client->hwlen = -1;
 	memset(client->hwaddr, 0, sizeof(client->hwaddr));
-	// This may be a alias-client, the ID is set elsewhere
+	// This may be an alias-client, the ID is set elsewhere
 	client->flags.aliasclient = aliasclient;
 	client->aliasclient_id = -1;
 

src/dnsmasq/dnsmasq.c

@@ -94,7 +94,7 @@ int main_dnsmasq (int argc, char **argv)
   sigaction(SIGUSR1, &sigact, NULL);
   sigaction(SIGUSR2, &sigact, NULL);
   sigaction(SIGHUP, &sigact, NULL);
-  sigaction(SIGTERM, &sigact, NULL);
+  sigaction(SIGUSR6, &sigact, NULL); // Pi-hole modification
   sigaction(SIGALRM, &sigact, NULL);
   sigaction(SIGCHLD, &sigact, NULL);
   sigaction(SIGINT, &sigact, NULL);
@@ -1330,7 +1330,7 @@ static void sig_handler(int sig)
 	event = EVENT_CHILD;
       else if (sig == SIGALRM)
 	event = EVENT_ALARM;
-      else if (sig == SIGTERM)
+      else if (sig == SIGUSR6) // Pi-hole modified
 	event = EVENT_TERM;
       else if (sig == SIGUSR1)
 	event = EVENT_DUMP;

src/dnsmasq_interface.c

@@ -1915,7 +1915,7 @@ static void FTL_reply(const unsigned int flags, const char *name, const union al
 	if(!(flags & F_UPSTREAM))
 	{
 		cached = true;
-		if((flags & F_HOSTS) || // local.list, hostname.list, /etc/hosts and others
+		if((flags & F_HOSTS) || // hostname.list, /etc/hosts and others
 		   ((flags & F_NAMEP) && (flags & F_DHCP)) || // DHCP server reply
 		   (flags & F_FORWARD) || // cached answer to previously forwarded request
 		   (flags & F_REVERSE) || // cached answer to reverse request (PTR)
@@ -3244,12 +3244,18 @@ void FTL_TCP_worker_created(const int confd)
 	gravityDB_forked();
 }
 
-bool FTL_unlink_DHCP_lease(const char *ipaddr)
+bool FTL_unlink_DHCP_lease(const char *ipaddr, const char **hint)
 {
 	struct dhcp_lease *lease;
 	union all_addr addr;
 	const time_t now = dnsmasq_time();
 
+	if(!daemon->dhcp)
+	{
+		*hint = "DHCP is not enabled";
+		return false;
+	}
+
 	// Try to extract IP address
 	if (inet_pton(AF_INET, ipaddr, &addr.addr4) > 0)
 	{
@@ -3263,6 +3269,8 @@ bool FTL_unlink_DHCP_lease(const char *ipaddr)
 #endif
 	else
 	{
+		// Invalid IP address
+		*hint = "invalid target address (neither IPv4 nor IPv6)";
 		return false;
 	}
 
@@ -3279,6 +3287,11 @@ bool FTL_unlink_DHCP_lease(const char *ipaddr)
 		// (variable lease.c:dns_dirty is used here)
 		lease_update_dns(0);
 	}
+	else
+	{
+		*hint = NULL;
+		return false;
+	}
 
 	// Return success
 	return true;

src/dnsmasq_interface.h

@@ -46,7 +46,7 @@ void FTL_dnsmasq_reload(void);
 void FTL_TCP_worker_created(const int confd);
 void FTL_TCP_worker_terminating(bool finished);
 
-bool FTL_unlink_DHCP_lease(const char *ipaddr);
+bool FTL_unlink_DHCP_lease(const char *ipaddr, const char **hint);
 
 // defined in src/dnsmasq/cache.c
 extern char *querystr(char *desc, unsigned short type);

src/signals.c

@@ -311,11 +311,94 @@ static void SIGRT_handler(int signum, siginfo_t *si, void *unused)
 		// Parse neighbor cache
 		set_event(PARSE_NEIGHBOR_CACHE);
 	}
+	// else if(rtsig == 6)
+	// {
+	// 	// Signal internally used to signal dnsmasq it has to stop
+	// }
 
 	// Restore errno before returning back to previous context
 	errno = _errno;
 }
 
+static void SIGTERM_handler(int signum, siginfo_t *si, void *unused)
+{
+	// Ignore SIGTERM outside of the main process (TCP forks)
+	if(mpid != getpid())
+		return;
+
+	// Get PID and UID of the process that sent the terminating signal
+	const pid_t kill_pid = si->si_pid;
+	const uid_t kill_uid = si->si_uid;
+
+	// Get name of the process that sent the terminating signal
+	char kill_name[256] = { 0 };
+	char kill_exe [256] = { 0 };
+	snprintf(kill_exe, sizeof(kill_exe), "/proc/%ld/cmdline", (long int)kill_pid);
+	FILE *fp = fopen(kill_exe, "r");
+	if(fp != NULL)
+	{
+		// Successfully opened file
+		size_t read = 0;
+		// Read line from file
+		if((read = fread(kill_name, sizeof(char), sizeof(kill_name), fp)) > 0)
+		{
+			// Successfully read line
+
+			// cmdline contains the command-line arguments as a set
+			// of strings separated by null bytes ('\0'), with a
+			// further null byte after the last string. Hence, we
+			// need to replace all null bytes with spaces for
+			// displaying it below
+			for(unsigned int i = 0; i < min((size_t)read, sizeof(kill_name)); i++)
+			{
+				if(kill_name[i] == '\0')
+					kill_name[i] = ' ';
+			}
+
+			// Remove any trailing spaces
+			for(unsigned int i = read - 1; i > 0; i--)
+			{
+				if(kill_name[i] == ' ')
+					kill_name[i] = '\0';
+				else
+					break;
+			}
+		}
+		else
+		{
+			// Failed to read line
+			strcpy(kill_name, "N/A");
+		}
+	}
+	else
+	{
+		// Failed to open file
+		strcpy(kill_name, "N/A");
+	}
+
+	// Get username of the process that sent the terminating signal
+	char kill_user[256] = { 0 };
+	struct passwd *pwd = getpwuid(kill_uid);
+	if(pwd != NULL)
+	{
+		// Successfully obtained username
+		strncpy(kill_user, pwd->pw_name, sizeof(kill_user));
+	}
+	else
+	{
+		// Failed to obtain username
+		strcpy(kill_user, "N/A");
+	}
+
+	// Log who sent the signal
+	log_info("Asked to terminate by \"%s\" (PID %ld, user %s UID %ld)",
+	         kill_name, (long int)kill_pid,
+	         kill_user, (long int)kill_uid);
+
+	// Terminate dnsmasq to stop DNS service
+	raise(SIGUSR6);
+}
+
 // Register ordinary signals handler
 void handle_signals(void)
 {
@@ -337,6 +420,13 @@ void handle_signals(void)
 		}
 	}
 
+	// Also catch SIGTERM
+	struct sigaction SIGaction = { 0 };
+	SIGaction.sa_flags = SA_SIGINFO;
+	sigemptyset(&SIGaction.sa_mask);
+	SIGaction.sa_sigaction = &SIGTERM_handler;
+	sigaction(SIGTERM, &SIGaction, NULL);
+
 	// Log start time of FTL
 	FTLstarttime = time(NULL);
 }
@@ -351,8 +441,12 @@ void handle_realtime_signals(void)
 	// Catch all real-time signals
 	for(int signum = SIGRTMIN; signum <= SIGRTMAX; signum++)
 	{
-		struct sigaction SIGACTION;
-		memset(&SIGACTION, 0, sizeof(struct sigaction));
+		if(signum == SIGUSR6)
+			// Skip SIGUSR6 as it is used internally to signify
+			// dnsmasq to stop
+			continue;
+
+		struct sigaction SIGACTION = { 0 };
 		SIGACTION.sa_flags = SA_SIGINFO;
 		sigemptyset(&SIGACTION.sa_mask);
 		SIGACTION.sa_sigaction = &SIGRT_handler;

src/signals.h

@@ -12,6 +12,8 @@
 
 #include "enums.h"
 
+#define SIGUSR6 (SIGRTMIN + 6)
+
 // defined in dnsmasq/dnsmasq.h
 extern volatile char FTL_terminate;
 

src/tools/gravity-parseList.c

@@ -542,8 +542,6 @@ end_of_parseList:
 			// Print newline
 			puts("");
 		}
-		// Print final newline
-		puts("");
 	}
 
 	// Free memory

src/webserver/json_macros.h

@@ -200,7 +200,10 @@
 })
 
 #define JSON_SEND_OBJECT_CODE(object, code)({ \
-	cJSON_AddNumberToObject(object, "took", double_time() - api->now);\
+	if((code) != 204) \
+	{ \
+		cJSON_AddNumberToObject(object, "took", double_time() - api->now); \
+	} \
 	char *json_string = json_formatter(object); \
 	if(json_string == NULL) \
 	{ \

test/pihole.toml

@@ -702,6 +702,11 @@
     # 86400)
     maxHistory = 86400
 
+    # Up to how many clients should be returned in the activity graph endpoint
+    # (/api/history/clients)?
+    # This setting can be overwritten at run-time using the parameter N
+    maxClients = 10
+
     # Allow destructive API calls (e.g. deleting all queries, powering off the system, ...)
     allow_destructive = true