pi-hole/FTL
1
0
Fork 0
mirror of https://github.com/pi-hole/FTL.git synced 2024-10-26 16:52:18 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge branch 'development-v6' into tweak/env_vars_list

Browse Source
This commit is contained in:
DL6ER
2024-01-13 10:46:51 +01:00
parent 799d22750c 02962d9244
commit b05fe82a3f
66 changed files with 2252 additions and 735 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.devcontainer/devcontainer.json
+12 -7
View File
@@ -2,14 +2,19 @@
"name": "FTL x86_64 Build Env",
"image": "ghcr.io/pi-hole/ftl-build:v2.4.1",
"runArgs": [ "--cap-add=SYS_PTRACE", "--security-opt", "seccomp=unconfined" ],
"extensions": [
"jetmartin.bats",
"ms-vscode.cpptools",
"ms-vscode.cmake-tools",
"eamodio.gitlens"
],
"customizations": {
"vscode": {
"extensions": [
"jetmartin.bats",
"ms-vscode.cpptools",
"ms-vscode.cmake-tools",
"eamodio.gitlens"
]
}
},
"mounts": [
"type=bind,source=/home/${localEnv:USER}/.ssh,target=/root/.ssh,readonly"
"type=bind,source=/home/${localEnv:USER}/.ssh,target=/root/.ssh,readonly",
"type=bind,source=/var/www/html,target=/var/www/html,readonly"
]
}
.github/workflows/build.yml
+13 -11
View File
@@ -119,9 +119,9 @@ jobs:
-
name: Store binary artifacts for later deployoment
if: github.event_name != 'pull_request'
uses: actions/upload-artifact@v3.1.3
uses: actions/upload-artifact@v4.0.0
with:
name: tmp-storage
name: ${{ matrix.bin_name }}-binary
path: '${{ matrix.bin_name }}*'
-
name: Extract documentation files from container
@@ -131,9 +131,9 @@ jobs:
-
name: Upload documentation artifacts for deployoment
if: github.event_name != 'pull_request' && matrix.platform == 'linux/amd64'
uses: actions/upload-artifact@v3.1.3
uses: actions/upload-artifact@v4.0.0
with:
name: tmp-storage
name: pihole-api-docs
path: 'api-docs.tar.gz'
deploy:
@@ -146,15 +146,17 @@ jobs:
uses: actions/checkout@v4.1.1
-
name: Get Binaries and documentation built in previous jobs
uses: actions/download-artifact@v3.0.2
uses: actions/download-artifact@v4.1.0
id: download
with:
name: tmp-storage
path: ftl-builds/
path: ftl_builds/
pattern: pihole-*
merge-multiple: true
-
name: Display structure of downloaded files
run: ls -R
working-directory: ${{steps.download.outputs.download-path}}
-
name: Install SSH Key
uses: benoitchantre/setup-ssh-authentication-action@1.0.1
@@ -163,14 +165,14 @@ jobs:
known-hosts: ${{ secrets.KNOWN_HOSTS }}
-
name: Untar documentation files
working-directory: ${{steps.download.outputs.download-path}}
working-directory: ftl_builds/
run: |
mkdir docs/
tar xzvf api-docs.tar.gz -C docs/
-
name: Display structure of files ready for upload
run: ls -R
working-directory: ${{steps.download.outputs.download-path}}
working-directory: ftl_builds/
-
name: Transfer Builds to Pi-hole server for pihole checkout
if: github.actor != 'dependabot[bot]'
@@ -178,7 +180,7 @@ jobs:
USER: ${{ secrets.SSH_USER }}
HOST: ${{ secrets.SSH_HOST }}
TARGET_DIR: ${{ needs.smoke-tests.outputs.OUTPUT_DIR }}
SOURCE_DIR: ${{ steps.download.outputs.download-path }}
SOURCE_DIR: ftl_builds/
run: |
bash ./deploy.sh
-
@@ -187,4 +189,4 @@ jobs:
uses: softprops/action-gh-release@v1
with:
files: |
${{ steps.download.outputs.download-path }}/*
ftl_builds/*
src/CMakeLists.txt
+8 -2
View File
@@ -24,14 +24,20 @@ set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR})
# SQLITE_DEFAULT_MEMSTATUS=0: This setting causes the sqlite3_status() interfaces that track memory usage to be disabled. This helps the sqlite3_malloc() routines run much faster, and since SQLite uses sqlite3_malloc() internally, this helps to make the entire library faster.
# SQLITE_OMIT_DEPRECATED: Omitting deprecated interfaces and features will not help SQLite to run any faster. It will reduce the library footprint, however. And it is the right thing to do.
# SQLITE_OMIT_PROGRESS_CALLBACK: The progress handler callback counter must be checked in the inner loop of the bytecode engine. By omitting this interface, a single conditional is removed from the inner loop of the bytecode engine, helping SQL statements to run slightly faster.
# SQLITE_OMIT_SHARED_CACHE: This option builds SQLite without support for shared cache mode. The sqlite3_enable_shared_cache() is omitted along with a fair amount of logic within the B-Tree subsystem associated with shared cache management. This compile-time option is recommended most applications as it results in improved performance and reduced library footprint.
# SQLITE_DEFAULT_FOREIGN_KEYS=1: This macro determines whether enforcement of foreign key constraints is enabled or disabled by default for new database connections.
# SQLITE_DQS=0: This setting disables the double-quoted string literal misfeature.
# SQLITE_ENABLE_DBPAGE_VTAB: Enables the SQLITE_DBPAGE virtual table. Warning: writing to the SQLITE_DBPAGE virtual table can very easily cause unrecoverably database corruption.
# SQLITE_TEMP_STORE=2: Store temporary tables in memory for reduced IO and higher performance (can be overwritten by the user at runtime).
# SQLITE_USE_URI=1: The advantage of using a URI filename is that query parameters on the URI can be used to control details of the newly created database connection.
# HAVE_READLINE: Enable readline support to allow easy editing, history and auto-completion
# SQLITE_DEFAULT_CACHE_SIZE=-16384: Allow up to 16 MiB of cache to be used by SQLite3 (default is 2000 kiB)
set(SQLITE_DEFINES "-DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_DEFAULT_MEMSTATUS=0 -DSQLITE_OMIT_DEPRECATED -DSQLITE_OMIT_PROGRESS_CALLBACK -DSQLITE_DEFAULT_FOREIGN_KEYS=1 -DSQLITE_DQS=0 -DSQLITE_ENABLE_DBPAGE_VTAB -DSQLITE_TEMP_STORE=2 -DSQLITE_USE_URI=1 -DHAVE_READLINE -DSQLITE_DEFAULT_CACHE_SIZE=16384")
# SQLITE_DEFAULT_SYNCHRONOUS=1: Use normal synchronous mode (default is 2)
# SQLITE_LIKE_DOESNT_MATCH_BLOBS: This option causes the LIKE operator to only match BLOB values against BLOB values and TEXT values against TEXT values. This compile-time option makes SQLite run more efficiently when processing queries that use the LIKE operator.
# HAVE_MALLOC_USABLE_SIZE: This option causes SQLite to try to use the malloc_usable_size() function to obtain the actual size of memory allocations from the underlying malloc() system interface. Applications are encouraged to use HAVE_MALLOC_USABLE_SIZE whenever possible.
# HAVE_FDATASYNC: This option causes SQLite to try to use the fdatasync() system call to sync the database file to disk when committing a transaction. Syncing using fdatasync() is faster than syncing using fsync() as fdatasync() does not wait for the file metadata to be written to disk.
# SQLITE_DEFAULT_WORKER_THREADS=4: This option sets the default number of worker threads to use when doing parallel sorting and indexing. The default is 0 which means to use a single thread. The default for SQLITE_MAX_WORKER_THREADS is 8.
# SQLITE_MAX_PREPARE_RETRY=200: This option sets the maximum number of automatic re-preparation attempts that can occur after encountering a schema change. This can be caused by running ANALYZE which is done periodically by FTL.
set(SQLITE_DEFINES "-DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_DEFAULT_MEMSTATUS=0 -DSQLITE_OMIT_DEPRECATED -DSQLITE_OMIT_PROGRESS_CALLBACK -DSQLITE_OMIT_SHARED_CACHE -DSQLITE_DEFAULT_FOREIGN_KEYS=1 -DSQLITE_DQS=0 -DSQLITE_ENABLE_DBPAGE_VTAB -DSQLITE_TEMP_STORE=2 -DHAVE_READLINE -DSQLITE_DEFAULT_CACHE_SIZE=16384 -DSQLITE_DEFAULT_SYNCHRONOUS=1 -DSQLITE_LIKE_DOESNT_MATCH_BLOBS -DHAVE_MALLOC_USABLE_SIZE -DHAVE_FDATASYNC -DSQLITE_DEFAULT_WORKER_THREADS=4 -DSQLITE_MAX_PREPARE_RETRY=200")
# Code hardening and debugging improvements
# -fstack-protector-strong: The program will be resistant to having its stack overflowed
src/FTL.h
+9
View File
@@ -50,6 +50,7 @@
// Number of elements in an array
#define ArraySize(X) (sizeof(X)/sizeof(X[0]))
// Constant socket buffer length
#define SOCKETBUFFERLEN 1024
// How often do we garbage collect (to ensure we only have data fitting to the MAXLOGAGE defined above)? [seconds]
@@ -133,6 +134,14 @@
// Special exit code used to signal that FTL wants to restart
#define RESTART_FTL_CODE 22
// How often should the database be analyzed?
// Default: 604800 (once per week)
#define DATABASE_ANALYZE_INTERVAL 604800
// How often should we update client vendor's from the MAC vendor database?
// Default: 2592000 (once per month)
#define DATABASE_MACVENDOR_INTERVAL 2592000
// Use out own syscalls handling functions that will detect possible errors
// and report accordingly in the log. This will make debugging FTL crash
// caused by insufficient memory or by code bugs (not properly dealing
src/api/api.c
+72 -68
View File
@@ -30,74 +30,78 @@ static struct {
bool require_auth;
enum http_method methods;
} api_request[] = {
// URI ARGUMENTS FUNCTION OPTIONS AUTH ALLOWED METHODS
// domains json fifo
// URI ARGUMENTS FUNCTION OPTIONS AUTH ALLOWED METHODS
// flags fifo ID
// Note: The order of appearance matters here, more specific URIs have to
// appear *before* less specific URIs: 1. "/a/b/c", 2. "/a/b", 3. "/a"
{ "/api/auth/sessions", "", api_auth_sessions, { false, true, 0 }, true, HTTP_GET },
{ "/api/auth/session", "/{id}", api_auth_session_delete, { false, true, 0 }, true, HTTP_DELETE },
{ "/api/auth/app", "", generateAppPw, { false, true, 0 }, true, HTTP_GET },
{ "/api/auth/totp", "", generateTOTP, { false, true, 0 }, true, HTTP_GET },
{ "/api/auth", "", api_auth, { false, true, 0 }, false, HTTP_GET | HTTP_POST | HTTP_DELETE },
{ "/api/dns/blocking", "", api_dns_blocking, { false, true, 0 }, true, HTTP_GET | HTTP_POST },
{ "/api/clients/_suggestions", "", api_client_suggestions, { false, true, 0 }, true, HTTP_GET },
{ "/api/clients", "/{client}", api_list, { false, true, 0 }, true, HTTP_GET | HTTP_PUT | HTTP_DELETE },
{ "/api/clients", "", api_list, { false, true, 0 }, true, HTTP_POST },
{ "/api/domains", "/{type}/{kind}/{domain}", api_list, { false, true, 0 }, true, HTTP_GET | HTTP_PUT | HTTP_DELETE },
{ "/api/domains", "/{type}/{kind}", api_list, { false, true, 0 }, true, HTTP_POST },
{ "/api/search", "/{domain}", api_search, { false, true, 0 }, true, HTTP_GET },
{ "/api/groups", "/{name}", api_list, { false, true, 0 }, true, HTTP_GET | HTTP_PUT | HTTP_DELETE },
{ "/api/groups", "", api_list, { false, true, 0 }, true, HTTP_POST },
{ "/api/lists", "/{list}", api_list, { false, true, 0 }, true, HTTP_GET | HTTP_PUT | HTTP_DELETE },
{ "/api/lists", "", api_list, { false, true, 0 }, true, HTTP_POST },
{ "/api/info/client", "", api_info_client, { false, true, 0 }, false, HTTP_GET },
{ "/api/info/login", "", api_info_login, { false, true, 0 }, false, HTTP_GET },
{ "/api/info/system", "", api_info_system, { false, true, 0 }, true, HTTP_GET },
{ "/api/info/database", "", api_info_database, { false, true, 0 }, true, HTTP_GET },
{ "/api/info/sensors", "", api_info_sensors, { false, true, 0 }, true, HTTP_GET },
{ "/api/info/host", "", api_info_host, { false, true, 0 }, true, HTTP_GET },
{ "/api/info/ftl", "", api_info_ftl, { false, true, 0 }, true, HTTP_GET },
{ "/api/info/version", "", api_info_version, { false, true, 0 }, true, HTTP_GET },
{ "/api/info/messages/count", "", api_info_messages_count, { false, true, 0 }, true, HTTP_GET },
{ "/api/info/messages", "/{message_id}", api_info_messages, { false, true, 0 }, true, HTTP_DELETE },
{ "/api/info/messages", "", api_info_messages, { false, true, 0 }, true, HTTP_GET },
{ "/api/info/metrics", "", api_info_metrics, { false, true, 0 }, true, HTTP_GET },
{ "/api/logs/dnsmasq", "", api_logs, { false, true, FIFO_DNSMASQ }, true, HTTP_GET },
{ "/api/logs/ftl", "", api_logs, { false, true, FIFO_FTL }, true, HTTP_GET },
{ "/api/logs/webserver", "", api_logs, { false, true, FIFO_WEBSERVER }, true, HTTP_GET },
{ "/api/history/clients", "", api_history_clients, { false, true, 0 }, true, HTTP_GET },
{ "/api/history/database/clients", "", api_history_database_clients, { false, true, 0 }, true, HTTP_GET },
{ "/api/history/database", "", api_history_database, { false, true, 0 }, true, HTTP_GET },
{ "/api/history", "", api_history, { false, true, 0 }, true, HTTP_GET },
{ "/api/queries/suggestions", "", api_queries_suggestions, { false, true, 0 }, true, HTTP_GET },
{ "/api/queries", "", api_queries, { false, true, 0 }, true, HTTP_GET },
{ "/api/stats/summary", "", api_stats_summary, { false, true, 0 }, true, HTTP_GET },
{ "/api/stats/query_types", "", api_stats_query_types, { false, true, 0 }, true, HTTP_GET },
{ "/api/stats/upstreams", "", api_stats_upstreams, { false, true, 0 }, true, HTTP_GET },
{ "/api/stats/top_domains", "", api_stats_top_domains, { false, true, 0 }, true, HTTP_GET },
{ "/api/stats/top_clients", "", api_stats_top_clients, { false, true, 0 }, true, HTTP_GET },
{ "/api/stats/recent_blocked", "", api_stats_recentblocked, { false, true, 0 }, true, HTTP_GET },
{ "/api/stats/database/top_domains", "", api_stats_database_top_items, { true, true, 0 }, true, HTTP_GET },
{ "/api/stats/database/top_clients", "", api_stats_database_top_items, { false, true, 0 }, true, HTTP_GET },
{ "/api/stats/database/summary", "", api_stats_database_summary, { false, true, 0 }, true, HTTP_GET },
{ "/api/stats/database/query_types", "", api_stats_database_query_types, { false, true, 0 }, true, HTTP_GET },
{ "/api/stats/database/upstreams", "", api_stats_database_upstreams, { false, true, 0 }, true, HTTP_GET },
{ "/api/config", "", api_config, { false, true, 0 }, true, HTTP_GET | HTTP_PATCH },
{ "/api/config", "/{element}", api_config, { false, true, 0 }, true, HTTP_GET },
{ "/api/config", "/{element}/{value}", api_config, { false, true, 0 }, true, HTTP_DELETE | HTTP_PUT },
{ "/api/network/gateway", "", api_network_gateway, { false, true, 0 }, true, HTTP_GET },
{ "/api/network/interfaces", "", api_network_interfaces, { false, true, 0 }, true, HTTP_GET },
{ "/api/network/devices", "", api_network_devices, { false, true, 0 }, true, HTTP_GET },
{ "/api/network/devices", "/{device_id}", api_network_devices, { false, true, 0 }, true, HTTP_DELETE },
{ "/api/endpoints", "", api_endpoints, { false, true, 0 }, true, HTTP_GET },
{ "/api/teleporter", "", api_teleporter, { false, false, 0 }, true, HTTP_GET | HTTP_POST },
{ "/api/dhcp/leases", "", api_dhcp_leases_GET, { false, true, 0 }, true, HTTP_GET },
{ "/api/dhcp/leases", "/{ip}", api_dhcp_leases_DELETE, { false, true, 0 }, true, HTTP_DELETE },
{ "/api/action/gravity", "", api_action_gravity, { false, true, 0 }, true, HTTP_POST },
{ "/api/action/restartdns", "", api_action_restartDNS, { false, true, 0 }, true, HTTP_POST },
{ "/api/action/flush/logs", "", api_action_flush_logs, { false, true, 0 }, true, HTTP_POST },
{ "/api/action/flush/arp", "", api_action_flush_arp, { false, true, 0 }, true, HTTP_POST },
{ "/api/docs", "", api_docs, { false, true, 0 }, false, HTTP_GET },
{ "/api/auth/sessions", "", api_auth_sessions, { API_PARSE_JSON, 0 }, true, HTTP_GET },
{ "/api/auth/session", "/{id}", api_auth_session_delete, { API_PARSE_JSON, 0 }, true, HTTP_DELETE },
{ "/api/auth/app", "", generateAppPw, { API_PARSE_JSON, 0 }, true, HTTP_GET },
{ "/api/auth/totp", "", generateTOTP, { API_PARSE_JSON, 0 }, true, HTTP_GET },
{ "/api/auth", "", api_auth, { API_PARSE_JSON, 0 }, false, HTTP_GET | HTTP_POST | HTTP_DELETE },
{ "/api/dns/blocking", "", api_dns_blocking, { API_PARSE_JSON, 0 }, true, HTTP_GET | HTTP_POST },
{ "/api/clients/_suggestions", "", api_client_suggestions, { API_PARSE_JSON, 0 }, true, HTTP_GET },
{ "/api/clients", "/{client}", api_list, { API_PARSE_JSON, 0 }, true, HTTP_GET | HTTP_PUT | HTTP_DELETE },
{ "/api/clients", "", api_list, { API_PARSE_JSON, 0 }, true, HTTP_POST },
{ "/api/clients:batchDelete", "", api_list, { API_PARSE_JSON | API_BATCHDELETE, 0 }, true, HTTP_POST },
{ "/api/domains", "/{type}/{kind}/{domain}", api_list, { API_PARSE_JSON, 0 }, true, HTTP_GET | HTTP_PUT | HTTP_DELETE },
{ "/api/domains", "/{type}/{kind}", api_list, { API_PARSE_JSON, 0 }, true, HTTP_POST },
{ "/api/domains:batchDelete", "", api_list, { API_PARSE_JSON | API_BATCHDELETE, 0 }, true, HTTP_POST },
{ "/api/search", "/{domain}", api_search, { API_PARSE_JSON, 0 }, true, HTTP_GET },
{ "/api/groups", "/{name}", api_list, { API_PARSE_JSON, 0 }, true, HTTP_GET | HTTP_PUT | HTTP_DELETE },
{ "/api/groups", "", api_list, { API_PARSE_JSON, 0 }, true, HTTP_POST },
{ "/api/groups:batchDelete", "", api_list, { API_PARSE_JSON | API_BATCHDELETE, 0 }, true, HTTP_POST },
{ "/api/lists", "/{list}", api_list, { API_PARSE_JSON, 0 }, true, HTTP_GET | HTTP_PUT | HTTP_DELETE },
{ "/api/lists", "", api_list, { API_PARSE_JSON, 0 }, true, HTTP_POST },
{ "/api/lists:batchDelete", "", api_list, { API_PARSE_JSON | API_BATCHDELETE, 0 }, true, HTTP_POST },
{ "/api/info/client", "", api_info_client, { API_PARSE_JSON, 0 }, false, HTTP_GET },
{ "/api/info/login", "", api_info_login, { API_PARSE_JSON, 0 }, false, HTTP_GET },
{ "/api/info/system", "", api_info_system, { API_PARSE_JSON, 0 }, true, HTTP_GET },
{ "/api/info/database", "", api_info_database, { API_PARSE_JSON, 0 }, true, HTTP_GET },
{ "/api/info/sensors", "", api_info_sensors, { API_PARSE_JSON, 0 }, true, HTTP_GET },
{ "/api/info/host", "", api_info_host, { API_PARSE_JSON, 0 }, true, HTTP_GET },
{ "/api/info/ftl", "", api_info_ftl, { API_PARSE_JSON, 0 }, true, HTTP_GET },
{ "/api/info/version", "", api_info_version, { API_PARSE_JSON, 0 }, true, HTTP_GET },
{ "/api/info/messages/count", "", api_info_messages_count, { API_PARSE_JSON, 0 }, true, HTTP_GET },
{ "/api/info/messages", "/{message_id}", api_info_messages, { API_PARSE_JSON, 0 }, true, HTTP_DELETE },
{ "/api/info/messages", "", api_info_messages, { API_PARSE_JSON, 0 }, true, HTTP_GET },
{ "/api/info/metrics", "", api_info_metrics, { API_PARSE_JSON, 0 }, true, HTTP_GET },
{ "/api/logs/dnsmasq", "", api_logs, { API_PARSE_JSON, FIFO_DNSMASQ }, true, HTTP_GET },
{ "/api/logs/ftl", "", api_logs, { API_PARSE_JSON, FIFO_FTL }, true, HTTP_GET },
{ "/api/logs/webserver", "", api_logs, { API_PARSE_JSON, FIFO_WEBSERVER }, true, HTTP_GET },
{ "/api/history/clients", "", api_history_clients, { API_PARSE_JSON, 0 }, true, HTTP_GET },
{ "/api/history/database/clients", "", api_history_database_clients, { API_PARSE_JSON, 0 }, true, HTTP_GET },
{ "/api/history/database", "", api_history_database, { API_PARSE_JSON, 0 }, true, HTTP_GET },
{ "/api/history", "", api_history, { API_PARSE_JSON, 0 }, true, HTTP_GET },
{ "/api/queries/suggestions", "", api_queries_suggestions, { API_PARSE_JSON, 0 }, true, HTTP_GET },
{ "/api/queries", "", api_queries, { API_PARSE_JSON, 0 }, true, HTTP_GET },
{ "/api/stats/summary", "", api_stats_summary, { API_PARSE_JSON, 0 }, true, HTTP_GET },
{ "/api/stats/query_types", "", api_stats_query_types, { API_PARSE_JSON, 0 }, true, HTTP_GET },
{ "/api/stats/upstreams", "", api_stats_upstreams, { API_PARSE_JSON, 0 }, true, HTTP_GET },
{ "/api/stats/top_domains", "", api_stats_top_domains, { API_PARSE_JSON, 0 }, true, HTTP_GET },
{ "/api/stats/top_clients", "", api_stats_top_clients, { API_PARSE_JSON, 0 }, true, HTTP_GET },
{ "/api/stats/recent_blocked", "", api_stats_recentblocked, { API_PARSE_JSON, 0 }, true, HTTP_GET },
{ "/api/stats/database/top_domains", "", api_stats_database_top_items, { API_DOMAINS | API_PARSE_JSON, 0 }, true, HTTP_GET },
{ "/api/stats/database/top_clients", "", api_stats_database_top_items, { API_PARSE_JSON, 0 }, true, HTTP_GET },
{ "/api/stats/database/summary", "", api_stats_database_summary, { API_PARSE_JSON, 0 }, true, HTTP_GET },
{ "/api/stats/database/query_types", "", api_stats_database_query_types, { API_PARSE_JSON, 0 }, true, HTTP_GET },
{ "/api/stats/database/upstreams", "", api_stats_database_upstreams, { API_PARSE_JSON, 0 }, true, HTTP_GET },
{ "/api/config", "", api_config, { API_PARSE_JSON, 0 }, true, HTTP_GET | HTTP_PATCH },
{ "/api/config", "/{element}", api_config, { API_PARSE_JSON, 0 }, true, HTTP_GET },
{ "/api/config", "/{element}/{value}", api_config, { API_PARSE_JSON, 0 }, true, HTTP_DELETE | HTTP_PUT },
{ "/api/network/gateway", "", api_network_gateway, { API_PARSE_JSON, 0 }, true, HTTP_GET },
{ "/api/network/interfaces", "", api_network_interfaces, { API_PARSE_JSON, 0 }, true, HTTP_GET },
{ "/api/network/devices", "", api_network_devices, { API_PARSE_JSON, 0 }, true, HTTP_GET },
{ "/api/network/devices", "/{device_id}", api_network_devices, { API_PARSE_JSON, 0 }, true, HTTP_DELETE },
{ "/api/endpoints", "", api_endpoints, { API_PARSE_JSON, 0 }, true, HTTP_GET },
{ "/api/teleporter", "", api_teleporter, { API_FLAG_NONE, 0 }, true, HTTP_GET | HTTP_POST },
{ "/api/dhcp/leases", "", api_dhcp_leases_GET, { API_PARSE_JSON, 0 }, true, HTTP_GET },
{ "/api/dhcp/leases", "/{ip}", api_dhcp_leases_DELETE, { API_PARSE_JSON, 0 }, true, HTTP_DELETE },
{ "/api/action/gravity", "", api_action_gravity, { API_PARSE_JSON, 0 }, true, HTTP_POST },
{ "/api/action/restartdns", "", api_action_restartDNS, { API_PARSE_JSON, 0 }, true, HTTP_POST },
{ "/api/action/flush/logs", "", api_action_flush_logs, { API_PARSE_JSON, 0 }, true, HTTP_POST },
{ "/api/action/flush/arp", "", api_action_flush_arp, { API_PARSE_JSON, 0 }, true, HTTP_POST },
{ "/api/docs", "", api_docs, { API_PARSE_JSON, 0 }, false, HTTP_GET },
};
int api_handler(struct mg_connection *conn, void *ignored)
@@ -113,7 +117,7 @@ int api_handler(struct mg_connection *conn, void *ignored)
double_time(),
{ false, NULL, NULL, NULL, 0u },
{ false },
{ false, false, 0 }
{ API_FLAG_NONE, 0 }
};
log_debug(DEBUG_API, "Requested API URI: %s -> %s %s ? %s (Content-Type %s)",
@@ -149,7 +153,7 @@ int api_handler(struct mg_connection *conn, void *ignored)
continue;
}
if(api_request[i].opts.parse_json)
if(api_request[i].opts.flags & API_PARSE_JSON)
{
// Allocate memory for the payload
api.payload.raw = calloc(MAX_PAYLOAD_BYTES, sizeof(char));
src/api/auth.c
+66 -31
View File
@@ -151,6 +151,7 @@ int check_client_auth(struct ftl_conn *api, const bool is_api)
}
}
// If not, does the client provide a session ID via COOKIE?
bool cookie_auth = false;
if(!sid_avail)
{
@@ -162,7 +163,22 @@ int check_client_auth(struct ftl_conn *api, const bool is_api)
// Mark SID as available
sid_avail = true;
}
}
// If not, does the client provide a session ID via URI?
if(!sid_avail && api->request->query_string && GET_VAR("sid", sid, api->request->query_string) > 0)
{
// "+" may have been replaced by " ", undo this here
for(unsigned int i = 0; i < SID_SIZE; i++)
if(sid[i] == ' ')
sid[i] = '+';
// Zero terminate SID string
sid[SID_SIZE-1] = '\0';
// Mention source of SID
sid_source = "URI";
// Mark SID as available
sid_avail = true;
}
if(!sid_avail)
@@ -320,14 +336,18 @@ static int get_session_object(struct ftl_conn *api, cJSON *json, const int user_
return 0;
}
static void delete_session(const int user_id)
static bool delete_session(const int user_id)
{
// Skip if nothing to be done here
if(user_id < 0 || user_id >= max_sessions)
return;
return false;
const bool was_valid = auth_data[user_id].used;
// Zero out this session (also sets valid to false == 0)
memset(&auth_data[user_id], 0, sizeof(auth_data[user_id]));
return was_valid;
}
void delete_all_sessions(void)
@@ -338,24 +358,6 @@ void delete_all_sessions(void)
static int send_api_auth_status(struct ftl_conn *api, const int user_id, const time_t now)
{
if(user_id == API_AUTH_LOCALHOST)
{
log_debug(DEBUG_API, "API Auth status: OK (localhost does not need auth)");
cJSON *json = JSON_NEW_OBJECT();
get_session_object(api, json, user_id, now);
JSON_SEND_OBJECT(json);
}
if(user_id == API_AUTH_EMPTYPASS)
{
log_debug(DEBUG_API, "API Auth status: OK (empty password)");
cJSON *json = JSON_NEW_OBJECT();
get_session_object(api, json, user_id, now);
JSON_SEND_OBJECT(json);
}
if(user_id > API_AUTH_UNAUTHORIZED && (api->method == HTTP_GET || api->method == HTTP_POST))
{
log_debug(DEBUG_API, "API Auth status: OK");
@@ -372,17 +374,45 @@ static int send_api_auth_status(struct ftl_conn *api, const int user_id, const t
get_session_object(api, json, user_id, now);
JSON_SEND_OBJECT(json);
}
else if(user_id > API_AUTH_UNAUTHORIZED && api->method == HTTP_DELETE)
else if(api->method == HTTP_DELETE)
{
log_debug(DEBUG_API, "API Auth status: Logout, asking to delete cookie");
if(user_id > API_AUTH_UNAUTHORIZED)
{
log_debug(DEBUG_API, "API Auth status: Logout, asking to delete cookie");
// Revoke client authentication. This slot can be used by a new client afterwards.
delete_session(user_id);
strncpy(pi_hole_extra_headers, FTL_DELETE_COOKIE, sizeof(pi_hole_extra_headers));
// Revoke client authentication. This slot can be used by a new client afterwards.
const int code = delete_session(user_id) ? 204 : 404;
// Send empty reply with appropriate HTTP status code
send_http_code(api, "application/json; charset=utf-8", code, "");
return code;
}
else
{
log_debug(DEBUG_API, "API Auth status: Logout, but not authenticated");
cJSON *json = JSON_NEW_OBJECT();
get_session_object(api, json, user_id, now);
JSON_SEND_OBJECT_CODE(json, 401); // 401 Unauthorized
}
}
else if(user_id == API_AUTH_LOCALHOST)
{
log_debug(DEBUG_API, "API Auth status: OK (localhost does not need auth)");
strncpy(pi_hole_extra_headers, FTL_DELETE_COOKIE, sizeof(pi_hole_extra_headers));
cJSON *json = JSON_NEW_OBJECT();
get_session_object(api, json, user_id, now);
JSON_SEND_OBJECT_CODE(json, 410); // 410 Gone
JSON_SEND_OBJECT(json);
}
else if(user_id == API_AUTH_EMPTYPASS)
{
log_debug(DEBUG_API, "API Auth status: OK (empty password)");
cJSON *json = JSON_NEW_OBJECT();
get_session_object(api, json, user_id, now);
JSON_SEND_OBJECT(json);
}
else
{
@@ -547,7 +577,7 @@ int api_auth(struct ftl_conn *api)
{
// Expired slow, mark as unused
if(auth_data[i].used &&
auth_data[i].valid_until < now)
auth_data[i].valid_until < now)
{
log_debug(DEBUG_API, "API: Session of client %u (%s) expired, freeing...",
i, auth_data[i].remote_addr);
@@ -618,6 +648,11 @@ int api_auth(struct ftl_conn *api)
"Rate-limiting login attempts",
NULL);
}
else if(result == NO_PASSWORD_SET)
{
// No password set
log_debug(DEBUG_API, "API: Trying to auth with password but none set: '%s'", password);
}
else
{
log_debug(DEBUG_API, "API: Password incorrect: '%s'", password);
@@ -651,9 +686,9 @@ int api_auth_session_delete(struct ftl_conn *api)
return send_json_error(api, 400, "bad_request", "Session ID not in use", NULL);
// Delete session
delete_session(uid);
const int code = delete_session(uid) ? 204 : 404;
// Send empty reply with code 204 No Content
send_http_code(api, "application/json; charset=utf-8", 204, "");
return 204;
// Send empty reply with appropriate HTTP status code
send_http_code(api, "application/json; charset=utf-8", code, "");
return code;
}
src/api/config.c
+12 -10
View File
@@ -294,7 +294,7 @@ static const char *getJSONvalue(struct conf_item *conf_item, cJSON *elem, struct
}
if(!set_and_check_password(conf_item, elem->valuestring))
return "Failed to create password hash (verification failed), password remains unchanged";
return "password hash verification failed";
break;
}
@@ -904,7 +904,7 @@ static int api_config_put_delete(struct ftl_conn *api)
key, true);
}
// Check if this entry does already exist in the array
// Check if this entry exists in the array
int idx = 0;
for(; idx < cJSON_GetArraySize(new_item->v.json); idx++)
{
@@ -938,13 +938,12 @@ static int api_config_put_delete(struct ftl_conn *api)
if(found)
{
// Remove item from array
found = true;
cJSON_DeleteItemFromArray(new_item->v.json, idx);
}
else
{
// Item not found
message = "Item not found";
hint = "Can only delete existing items";
break;
}
}
@@ -964,13 +963,16 @@ static int api_config_put_delete(struct ftl_conn *api)
// Release allocated memory
free_config_path(requested_path);
// Error 404 if not found
if(!found || message != NULL)
// Error 404 if config element not found
if(!found)
{
cJSON *json = JSON_NEW_OBJECT();
JSON_SEND_OBJECT_CODE(json, 404);
}
// Error 400 if unique item already present
if(message != NULL)
{
// For any other error, a more specific message will have been added
// above
if(!message)
message = "No item specified";
return send_json_error(api, 400,
"bad_request",
message,
src/api/dhcp.c
+7 -5
View File
@@ -85,16 +85,18 @@ int api_dhcp_leases_DELETE(struct ftl_conn *api)
// Send empty reply with code 204 No Content
return send_json_error(api,
400,
"bad_request",
"bad_request",
"The provided IPv4 address is invalid",
api->item);
api->item);
}
// Delete lease
log_debug(DEBUG_API, "Deleting DHCP lease for address %s", api->item);
FTL_unlink_DHCP_lease(api->item);
const bool found = FTL_unlink_DHCP_lease(api->item);
// Send empty reply with code 204 No Content
// Send empty reply with codes:
// - 204 No Content (if a lease was deleted)
// - 404 Not Found (if no lease was found)
cJSON *json = JSON_NEW_OBJECT();
JSON_SEND_OBJECT_CODE(json, 204);
JSON_SEND_OBJECT_CODE(json, found ? 204 : 404);
}
src/api/docs/content/specs/auth.yaml
+22 -21
View File
@@ -118,21 +118,27 @@ components:
- Authentication
operationId: "delete_groups"
description: |
A logout attempt without a valid session will result in a `401 Unauthorized` error.
This endpoint can be used to delete the current session. It will
invalidate the session token and the CSRF token. The session can be
extended before its expiration by performing any authenticated action.
By default, the session lasts for 5 minutes. It can be invalidated by
either logging out or deleting the session. Additionally, the session
becomes invalid when the password is altered or a new application
password is created.
A session that was not created due to a login cannot be deleted (e.g., empty API password).
You can also delete a session by its ID using the `DELETE /auth/session/{id}` endpoint.
Note that you cannot delete the current session if you have not
authenticated (e.g., no password has been set on your Pi-hole).
responses:
'200':
description: OK (session not deletable)
'204':
description: No Content (deleted)
'404':
description: Not Found (no session active)
content:
application/json:
schema:
allOf:
- $ref: 'auth.yaml#/components/schemas/session'
- $ref: 'common.yaml#/components/schemas/took'
examples:
no_login_required:
$ref: 'auth.yaml#/components/examples/no_login_required'
$ref: 'common.yaml#/components/schemas/took'
'401':
description: Unauthorized
content:
@@ -141,17 +147,6 @@ components:
allOf:
- $ref: 'common.yaml#/components/errors/unauthorized'
- $ref: 'common.yaml#/components/schemas/took'
'410':
description: Gone
content:
application/json:
schema:
allOf:
- $ref: 'auth.yaml#/components/schemas/session'
- $ref: 'common.yaml#/components/schemas/took'
examples:
login_failed:
$ref: 'auth.yaml#/components/examples/login_failed'
session_list:
get:
summary: List of all current sessions
@@ -213,6 +208,12 @@ components:
responses:
'204':
description: No Content (deleted)
'404':
description: Not Found (session not found)
content:
application/json:
schema:
$ref: 'common.yaml#/components/schemas/took'
'400':
description: Bad Request
content:
src/api/docs/content/specs/clients.yaml
+84 -17
View File
@@ -95,6 +95,12 @@ components:
responses:
'204':
description: Item deleted
'404':
description: Item not found
content:
application/json:
schema:
$ref: 'common.yaml#/components/schemas/took'
'400':
description: Bad request
content:
@@ -149,7 +155,7 @@ components:
Creates a new client in the `clients` object. The `{client}` itself is specified in the request body (POST JSON).
Clients may be described either by their IP addresses (IPv4 and IPv6 are supported),
IP subnets (CIDR notation, like `192.168.2.0/24`), their MAC addresses (like `12:34:56:78:9A:BC`), by their hostnames (like `localhost`), or by the interface they are connected to (prefaced with a colon, like `:eth0`).</p>
IP subnets (CIDR notation, like `192.168.2.0/24`), their MAC addresses (like `12:34:56:78:9A:BC`), by their hostnames (like `localhost`), or by the interface they are connected to (prefaced with a colon, like `:eth0`).
Note that client recognition by IP addresses (incl. subnet ranges) is preferred over MAC address, host name or interface recognition as the two latter will only be available after some time.
Furthermore, MAC address recognition only works for devices at most one networking hop away from your Pi-hole.
@@ -199,6 +205,65 @@ components:
allOf:
- $ref: 'common.yaml#/components/schemas/took'
- $ref: 'common.yaml#/components/errors/unauthorized'
batchDelete:
post:
summary: Delete multiple clients
tags:
- "Client management"
operationId: "batchDelete_clients"
description: |
Deletes multiple clients in the `clients` object. The `{client}`s themselves are specified in the request body (POST JSON).
Clients may be described either by their IP addresses (IPv4 and IPv6 are supported),
IP subnets (CIDR notation, like `192.168.2.0/24`), their MAC addresses (like `12:34:56:78:9A:BC`), by their hostnames (like `localhost`), or by the interface they are connected to (prefaced with a colon, like `:eth0`).</p>
*Note:* There will be no content on success.
requestBody:
description: Callback payload
content:
application/json:
schema:
type: array
items:
type: object
properties:
item:
type: string
description: client IP / MAC / hostname / interface
example:
- "item": "192.168.2.5"
- "item": "::1"
- "item": "12:34:56:78:9A:BC"
- "item": "localhost"
- "item": ":eth0"
responses:
'204':
description: Items deleted
'404':
description: Item not found
content:
application/json:
schema:
$ref: 'common.yaml#/components/schemas/took'
'400':
description: Bad request
content:
application/json:
schema:
allOf:
- $ref: 'common.yaml#/components/errors/bad_request'
- $ref: 'common.yaml#/components/schemas/took'
examples:
no_payload:
$ref: 'clients.yaml#/components/examples/errors/bad_request/no_payload'
'401':
description: Unauthorized
content:
application/json:
schema:
allOf:
- $ref: 'common.yaml#/components/errors/unauthorized'
- $ref: 'common.yaml#/components/schemas/took'
schemas:
clients:
get:
@@ -209,7 +274,7 @@ components:
description: Array of clients
items:
allOf:
- $ref: 'clients.yaml#/components/schemas/client'
- $ref: 'clients.yaml#/components/schemas/client_object'
- $ref: 'clients.yaml#/components/schemas/comment'
- $ref: 'clients.yaml#/components/schemas/groups'
- $ref: 'clients.yaml#/components/schemas/readonly'
@@ -252,25 +317,27 @@ components:
description: Comma-separated list of hostnames (if available)
example: "localhost,ip6-localhost"
client:
type: object
properties:
client:
description: client IP / MAC / hostname / interface
type: string
example: 127.0.0.1
description: client IP / MAC / hostname / interface
type: string
example: 127.0.0.1
client_array:
description: array of client IPs / MACs / hostnames / interfaces
type: array
items:
type: string
example: ["127.0.0.1", "192.168.2.12"]
client_maybe_array:
type: object
properties:
client:
description: array of client IPs / MACs / hostnames / interfaces
type: array
items:
type: string
example: ["127.0.0.1", "192.168.2.12"]
client_maybe_array:
oneOf:
- $ref: 'clients.yaml#/components/schemas/client'
- $ref: 'clients.yaml#/components/schemas/client_array'
oneOf:
- $ref: 'clients.yaml#/components/schemas/client'
- $ref: 'clients.yaml#/components/schemas/client_array'
client_object:
type: object
properties:
client:
$ref: 'clients.yaml#/components/schemas/client'
comment:
type: object
properties:
src/api/docs/content/specs/config.yaml
+9 -10
View File
@@ -121,7 +121,7 @@ components:
examples:
invalid_path_depth:
$ref: 'config.yaml#/components/examples/errors/bad_request/invalid_path_depth'
item_not_found:
item_already_present:
$ref: 'config.yaml#/components/examples/errors/bad_request/item_already_present'
'401':
description: Unauthorized
@@ -144,6 +144,12 @@ components:
responses:
'204':
description: Item deleted
'404':
description: Item not found
content:
application/json:
schema:
$ref: 'common.yaml#/components/schemas/took'
'400':
description: Bad request
content:
@@ -155,8 +161,8 @@ components:
examples:
invalid_path_depth:
$ref: 'config.yaml#/components/examples/errors/bad_request/invalid_path_depth'
item_not_found:
$ref: 'config.yaml#/components/examples/errors/bad_request/item_not_found'
item_already_present:
$ref: 'config.yaml#/components/examples/errors/bad_request/item_already_present'
'401':
description: Unauthorized
content:
@@ -795,13 +801,6 @@ components:
key: "bad_request"
message: "Invalid path depth"
hint: "Use, e.g., DELETE /config/dnsmasq/upstreams/127.0.0.1 to remove \"127.0.0.1\" from config.dns.upstreams"
item_not_found:
summary: Item to be deleted does not exist
value:
error:
key: "bad_request"
message: "Item not found"
hint: "Can only delete existing items"
item_already_present:
summary: Item to be added exists already
value:
src/api/docs/content/specs/dhcp.yaml
+6
View File
@@ -40,6 +40,12 @@ components:
responses:
'204':
description: Item deleted
'404':
description: Item not found
content:
application/json:
schema:
$ref: 'common.yaml#/components/schemas/took'
'400':
description: Bad request
content:
src/api/docs/content/specs/dns.yaml
+10 -3
View File
@@ -29,7 +29,7 @@ components:
description: |
Change the current blocking mode by setting `blocking` to the desired value.
The optional `timer` object may used to set a timer. Once this timer elapsed, the opposite blocking mode is automatically set.
For instance, you can request `{blocking: true, timer: 60}` to disable Pi-hole for one minute.
For instance, you can request `{blocking: false, timer: 60}` to disable Pi-hole for one minute.
Blocking will be automatically resumed afterwards.
You can terminate a possibly running timer by setting `timer` to `null` (the set mode becomes permanent).
@@ -39,9 +39,8 @@ components:
'application/json':
schema:
allOf:
- $ref: 'dns.yaml#/components/schemas/blocking'
- $ref: 'dns.yaml#/components/schemas/blocking_bool'
- $ref: 'dns.yaml#/components/schemas/timer'
- $ref: 'common.yaml#/components/schemas/took'
responses:
'200':
description: OK
@@ -83,6 +82,14 @@ components:
- "failed"
- "unknown"
example: "enabled"
blocking_bool:
type: object
properties:
blocking:
type: boolean
description: Blocking status
default: true
example: true
timer:
type: object
properties:
src/api/docs/content/specs/domains.yaml
+88 -16
View File
@@ -128,6 +128,12 @@ components:
responses:
'204':
description: Item deleted
'404':
description: Item not found
content:
application/json:
schema:
$ref: 'common.yaml#/components/schemas/took'
'400':
description: Bad request
content:
@@ -212,6 +218,70 @@ components:
allOf:
- $ref: 'common.yaml#/components/errors/unauthorized'
- $ref: 'common.yaml#/components/schemas/took'
batchDelete:
summary: Delete multiple domains
post:
summary: Delete multiple domains
tags:
- "Domain management"
operationId: "batchDelete_domains"
description: |
*Note:* There will be no content on success.
requestBody:
description: Callback payload
content:
application/json:
schema:
type: array
items:
type: object
properties:
item:
type: string
description: Domain to delete
example: "example.com"
type:
type: string
description: Type of domain to delete
enum:
- "allow"
- "deny"
example: "allow"
kind:
type: string
description: Kind of domain to delete
enum:
- "exact"
- "regex"
example: "exact"
responses:
'204':
description: Items deleted
'404':
description: Item not found
content:
application/json:
schema:
$ref: 'common.yaml#/components/schemas/took'
'400':
description: Bad request
content:
application/json:
schema:
allOf:
- $ref: 'common.yaml#/components/errors/bad_request'
- $ref: 'common.yaml#/components/schemas/took'
examples:
no_payload:
$ref: 'domains.yaml#/components/examples/errors/bad_request/no_payload'
'401':
description: Unauthorized
content:
application/json:
schema:
allOf:
- $ref: 'common.yaml#/components/errors/unauthorized'
- $ref: 'common.yaml#/components/schemas/took'
schemas:
domains:
get:
@@ -222,7 +292,7 @@ components:
description: Array of domains
items:
allOf:
- $ref: 'domains.yaml#/components/schemas/domain'
- $ref: 'domains.yaml#/components/schemas/domain_object'
- $ref: 'domains.yaml#/components/schemas/unicode'
- $ref: 'domains.yaml#/components/schemas/type'
- $ref: 'domains.yaml#/components/schemas/kind'
@@ -244,12 +314,9 @@ components:
- $ref: 'domains.yaml#/components/schemas/groups'
- $ref: 'domains.yaml#/components/schemas/enabled'
domain:
type: object
properties:
domain:
description: Domain
type: string
example: testdomain.com
description: Domain
type: string
example: testdomain.com
unicode:
type: object
properties:
@@ -258,18 +325,23 @@ components:
type: string
example: "äbc.com"
domain_array:
description: array of domains
type: array
items:
type: string
example: ["testdomain.com", "otherdomain.de"]
domain_maybe_array:
type: object
properties:
domain:
description: array of domains
type: array
items:
type: string
example: ["testdomain.com", "otherdomain.de"]
domain_maybe_array:
oneOf:
- $ref: 'domains.yaml#/components/schemas/domain'
- $ref: 'domains.yaml#/components/schemas/domain_array'
oneOf:
- $ref: 'domains.yaml#/components/schemas/domain'
- $ref: 'domains.yaml#/components/schemas/domain_array'
domain_object:
type: object
properties:
domain:
$ref: 'domains.yaml#/components/schemas/domain'
type:
type: object
properties:
src/api/docs/content/specs/groups.yaml
+86 -17
View File
@@ -63,6 +63,9 @@ components:
- $ref: 'groups.yaml#/components/schemas/groups/get' # identical to GET
- $ref: 'groups.yaml#/components/schemas/lists_processed'
- $ref: 'common.yaml#/components/schemas/took'
headers:
Location:
$ref: 'common.yaml#/components/headers/Location'
'400':
description: Bad request
content:
@@ -94,6 +97,12 @@ components:
responses:
'204':
description: Item deleted
'404':
description: Item not found
content:
application/json:
schema:
$ref: 'common.yaml#/components/schemas/took'
'400':
description: Bad request
content:
@@ -165,6 +174,63 @@ components:
allOf:
- $ref: 'common.yaml#/components/errors/unauthorized'
- $ref: 'common.yaml#/components/schemas/took'
batchDelete:
post:
summary: Delete multiple groups
tags:
- "Group management"
operationId: "batchDelete_groups"
description: |
Deletes multiple groups in the `groups` object. The `{groups}` themselves are specified in the request body (POST JSON).
On success, a new resource is created at `/groups/{name}`.
The `database_error` with message `UNIQUE constraint failed` error indicates that a group with the same name already exists.
requestBody:
description: Callback payload
content:
application/json:
schema:
type: array
items:
type: object
properties:
item:
type: string
description: group name
example:
- "item": "test1"
- "item": "test2"
responses:
'204':
description: Items deleted
'404':
description: Item not found
content:
application/json:
schema:
$ref: 'common.yaml#/components/schemas/took'
'400':
description: Bad request
content:
application/json:
schema:
allOf:
- $ref: 'common.yaml#/components/errors/bad_request'
- $ref: 'common.yaml#/components/schemas/took'
examples:
no_payload:
$ref: 'groups.yaml#/components/examples/errors/bad_request/no_payload'
duplicate:
$ref: 'groups.yaml#/components/examples/errors/database_error/duplicate'
'401':
description: Unauthorized
content:
application/json:
schema:
allOf:
- $ref: 'common.yaml#/components/errors/unauthorized'
- $ref: 'common.yaml#/components/schemas/took'
schemas:
groups:
get:
@@ -174,13 +240,14 @@ components:
type: array
items:
allOf:
- $ref: 'groups.yaml#/components/schemas/name'
- $ref: 'groups.yaml#/components/schemas/name_object'
- $ref: 'groups.yaml#/components/schemas/comment'
- $ref: 'groups.yaml#/components/schemas/enabled'
- $ref: 'groups.yaml#/components/schemas/readonly'
put:
allOf:
- $ref: 'groups.yaml#/components/schemas/name'
# Can rename group
- $ref: 'groups.yaml#/components/schemas/name_object'
- $ref: 'groups.yaml#/components/schemas/comment'
- $ref: 'groups.yaml#/components/schemas/enabled'
post:
@@ -189,25 +256,27 @@ components:
- $ref: 'groups.yaml#/components/schemas/comment'
- $ref: 'groups.yaml#/components/schemas/enabled'
name:
type: object
properties:
name:
description: Group name
type: string
example: test_group
description: Group name
type: string
example: test_group
name_array:
description: array of group names
type: array
items:
type: string
example: ["test1", "test2", "test3"]
name_maybe_array:
type: object
properties:
name:
description: array of group names
type: array
items:
type: string
example: ["test1", "test2", "test3"]
name_maybe_array:
oneOf:
- $ref: 'groups.yaml#/components/schemas/name'
- $ref: 'groups.yaml#/components/schemas/name_array'
oneOf:
- $ref: 'groups.yaml#/components/schemas/name'
- $ref: 'groups.yaml#/components/schemas/name_array'
name_object:
type: object
properties:
name:
$ref: 'groups.yaml#/components/schemas/name'
comment:
type: object
properties:
src/api/docs/content/specs/info.yaml
+15 -1
View File
@@ -218,10 +218,16 @@ components:
parameters:
- $ref: 'info.yaml#/components/parameters/message_id'
description: |
*Note:* There will be no content on success. You may specify multiple IDs to delete multiple messages at once (comma-separated in the path like `1,2,3`)
You may specify multiple IDs to delete multiple messages at once (comma-separated in the path like `1,2,3`)
responses:
'204':
description: Item deleted
'404':
description: Not found
content:
application/json:
schema:
$ref: 'common.yaml#/components/schemas/took'
'400':
description: Bad request
content:
@@ -235,6 +241,14 @@ components:
$ref: 'info.yaml#/components/examples/errors/messages/uri_error'
bad_request:
$ref: 'info.yaml#/components/examples/errors/messages/bad_request'
'401':
description: Unauthorized
content:
application/json:
schema:
allOf:
- $ref: 'common.yaml#/components/errors/unauthorized'
- $ref: 'common.yaml#/components/schemas/took'
messages_count:
get:
summary: Get count of Pi-hole diagnosis messages
src/api/docs/content/specs/lists.yaml
+71 -17
View File
@@ -93,6 +93,12 @@ components:
responses:
'204':
description: Item deleted
'404':
description: Item not found
content:
application/json:
schema:
$ref: 'common.yaml#/components/schemas/took'
'400':
description: Bad request
content:
@@ -170,6 +176,52 @@ components:
allOf:
- $ref: 'common.yaml#/components/errors/unauthorized'
- $ref: 'common.yaml#/components/schemas/took'
batchDelete:
post:
summary: Delete lists
tags:
- "List management"
operationId: "batchDelete_lists"
description: |
Deletes multiple lists in the `lists` object. The `{list}`s themselves are specified in the request body (POST JSON).
On success, a new resource is created at `/lists/{list}`.
The `database_error` with message `UNIQUE constraint failed` error indicates that this list already exists.
requestBody:
description: Callback payload
content:
application/json:
schema:
$ref: 'lists.yaml#/components/schemas/lists/post'
responses:
'204':
description: Items deleted
'404':
description: Item not found
content:
application/json:
schema:
$ref: 'common.yaml#/components/schemas/took'
'400':
description: Bad request
content:
application/json:
schema:
allOf:
- $ref: 'common.yaml#/components/errors/bad_request'
- $ref: 'common.yaml#/components/schemas/took'
examples:
no_payload:
$ref: 'lists.yaml#/components/examples/errors/bad_request/no_payload'
'401':
description: Unauthorized
content:
application/json:
schema:
allOf:
- $ref: 'common.yaml#/components/errors/unauthorized'
- $ref: 'common.yaml#/components/schemas/took'
schemas:
lists:
get:
@@ -180,7 +232,7 @@ components:
description: Array of lists
items:
allOf:
- $ref: 'lists.yaml#/components/schemas/list'
- $ref: 'lists.yaml#/components/schemas/address_object'
- $ref: 'lists.yaml#/components/schemas/type'
- $ref: 'lists.yaml#/components/schemas/comment'
- $ref: 'lists.yaml#/components/schemas/groups'
@@ -194,31 +246,33 @@ components:
- $ref: 'lists.yaml#/components/schemas/enabled'
post:
allOf:
- $ref: 'lists.yaml#/components/schemas/list_maybe_array'
- $ref: 'lists.yaml#/components/schemas/address_maybe_array'
- $ref: 'lists.yaml#/components/schemas/type'
- $ref: 'lists.yaml#/components/schemas/comment'
- $ref: 'lists.yaml#/components/schemas/groups'
- $ref: 'lists.yaml#/components/schemas/enabled'
list:
address:
description: Address of the list
type: string
example: https://hosts-file.net/ad_servers.txt
address_array:
description: array of list addresses
type: array
items:
type: string
example: ["https://hosts-file.net/ad_servers.txt"]
address_maybe_array:
type: object
properties:
address:
description: Address of the list
type: string
example: https://hosts-file.net/ad_servers.txt
list_array:
oneOf:
- $ref: 'lists.yaml#/components/schemas/address'
- $ref: 'lists.yaml#/components/schemas/address_array'
address_object:
type: object
properties:
list:
description: array of list addresses
type: array
items:
type: string
example: ["https://hosts-file.net/ad_servers.txt"]
list_maybe_array:
oneOf:
- $ref: 'lists.yaml#/components/schemas/list'
- $ref: 'lists.yaml#/components/schemas/list_array'
address:
$ref: 'lists.yaml#/components/schemas/address'
type:
type: object
properties:
src/api/docs/content/specs/main.yaml
+12
View File
@@ -142,18 +142,27 @@ paths:
/domains/{type}/{kind}:
$ref: 'domains.yaml#/components/paths/type_kind'
/domains:batchDelete:
$ref: 'domains.yaml#/components/paths/batchDelete'
/groups/{name}:
$ref: 'groups.yaml#/components/paths/name'
/groups:
$ref: 'groups.yaml#/components/paths/direct'
/groups:batchDelete:
$ref: 'groups.yaml#/components/paths/batchDelete'
/clients/{client}:
$ref: 'clients.yaml#/components/paths/client'
/clients:
$ref: 'clients.yaml#/components/paths/direct'
/clients:batchDelete:
$ref: 'clients.yaml#/components/paths/batchDelete'
/clients/_suggestions:
$ref: 'clients.yaml#/components/paths/suggestions'
@@ -163,6 +172,9 @@ paths:
/lists:
$ref: 'lists.yaml#/components/paths/direct'
/lists:batchDelete:
$ref: 'lists.yaml#/components/paths/batchDelete'
/info/client:
$ref: 'info.yaml#/components/paths/client'
src/api/docs/content/specs/network.yaml
+14
View File
@@ -93,6 +93,20 @@ components:
responses:
'204':
description: No Content (deleted)
'404':
description: Not found
content:
application/json:
schema:
$ref: 'common.yaml#/components/schemas/took'
'400':
description: Bad request
content:
application/json:
schema:
allOf:
- $ref: 'common.yaml#/components/errors/bad_request'
- $ref: 'common.yaml#/components/schemas/took'
'401':
description: Unauthorized
content:
src/api/info.c
+7 -4
View File
@@ -147,7 +147,7 @@ int api_info_database(struct ftl_conn *api)
JSON_ADD_ITEM_TO_OBJECT(json, "owner", owner);
// Add number of queries in on-disk database
const int queries_in_database = get_number_of_queries_in_DB(NULL, "query_storage", true);
const int queries_in_database = get_number_of_queries_in_DB(NULL, "query_storage");
JSON_ADD_NUMBER_TO_OBJECT(json, "queries", queries_in_database);
// Add SQLite library version
@@ -940,15 +940,18 @@ static int api_info_messages_DELETE(struct ftl_conn *api)
}
// Delete message with this ID from the database
delete_message(ids);
int deleted = 0;
delete_message(ids, &deleted);
// Free memory
free(id);
cJSON_free(ids);
// Send empty reply with code 204 No Content
// Send empty reply with codes:
// - 204 No Content (if any items were deleted)
// - 404 Not Found (if no items were deleted)
cJSON *json = JSON_NEW_OBJECT();
JSON_SEND_OBJECT_CODE(json, 204);
JSON_SEND_OBJECT_CODE(json, deleted > 0 ? 204 : 404);
}
int api_info_messages(struct ftl_conn *api)
src/api/list.c
+226 -10
View File
@@ -503,13 +503,27 @@ static int api_list_write(struct ftl_conn *api,
cJSON_AddItemToArray(okay ? success : errors, details);
}
// Inform the resolver that it needs to reload the domainlists
// Inform the resolver that it needs to reload gravity
set_event(RELOAD_GRAVITY);
int response_code = 201; // 201 - Created
if(api->method == HTTP_PUT)
response_code = 200; // 200 - OK
// Add "Location" header to response
if(snprintf(pi_hole_extra_headers, sizeof(pi_hole_extra_headers), "Location: %s/%s", api->action_path, row.item) >= (int)sizeof(pi_hole_extra_headers))
{
// This may happen for *extremely* long URLs but is not issue in
// itself. Merely add a warning to the log file
log_warn("Could not add Location header to response: URL too long");
// Truncate location by replacing the last characters with "...\0"
pi_hole_extra_headers[sizeof(pi_hole_extra_headers)-4] = '.';
pi_hole_extra_headers[sizeof(pi_hole_extra_headers)-3] = '.';
pi_hole_extra_headers[sizeof(pi_hole_extra_headers)-2] = '.';
pi_hole_extra_headers[sizeof(pi_hole_extra_headers)-1] = '\0';
}
// Send GET style reply
const int ret = api_list_read(api, response_code, listtype, row.item, processed);
@@ -525,21 +539,198 @@ static int api_list_remove(struct ftl_conn *api,
const char *item)
{
const char *sql_msg = NULL;
if(gravityDB_delFromTable(listtype, item, &sql_msg))
cJSON *array = api->payload.json;
bool allocated_json = false;
// If this is not a :batchDelete call, then the item is specified in the
// URI, not in the payload. Create a JSON array with the item and use
// that instead
const bool isBatchDelete = api->opts.flags & API_BATCHDELETE;
// If this is a domain callback, we need to translate type/kind into an
// integer for use in the database
if(listtype == GRAVITY_DOMAINLIST_ALLOW_EXACT ||
listtype == GRAVITY_DOMAINLIST_DENY_EXACT ||
listtype == GRAVITY_DOMAINLIST_ALLOW_REGEX ||
listtype == GRAVITY_DOMAINLIST_DENY_REGEX)
{
// Inform the resolver that it needs to reload the domainlists
int type = -1;
switch (listtype)
{
case GRAVITY_DOMAINLIST_ALLOW_EXACT:
type = 0;
break;
case GRAVITY_DOMAINLIST_DENY_EXACT:
type = 1;
break;
case GRAVITY_DOMAINLIST_ALLOW_REGEX:
type = 2;
break;
case GRAVITY_DOMAINLIST_DENY_REGEX:
type = 3;
case GRAVITY_GROUPS:
case GRAVITY_ADLISTS:
case GRAVITY_CLIENTS:
// No type required for these tables
break;
// Aggregate types cannot be handled by this routine
case GRAVITY_GRAVITY:
case GRAVITY_ANTIGRAVITY:
case GRAVITY_DOMAINLIST_ALLOW_ALL:
case GRAVITY_DOMAINLIST_DENY_ALL:
case GRAVITY_DOMAINLIST_ALL_EXACT:
case GRAVITY_DOMAINLIST_ALL_REGEX:
case GRAVITY_DOMAINLIST_ALL_ALL:
default:
return false;
}
// Create new JSON array with the item and type:
// array = [{"item": "example.com", "type": 0}]
array = cJSON_CreateArray();
cJSON *obj = cJSON_CreateObject();
cJSON_AddItemToObject(obj, "item", cJSON_CreateStringReference(item));
cJSON_AddItemToObject(obj, "type", cJSON_CreateNumber(type));
cJSON_AddItemToArray(array, obj);
allocated_json = true;
}
else if(isBatchDelete && listtype == GRAVITY_DOMAINLIST_ALL_ALL)
{
// Loop over all items and parse type/kind for each item
cJSON *it = NULL;
cJSON_ArrayForEach(it, array)
{
if(!cJSON_IsObject(it))
{
return send_json_error(api, 400,
"bad_request",
"Invalid request: Batch delete requires an array of objects",
NULL);
}
// Check if item is a string
cJSON *json_item = cJSON_GetObjectItemCaseSensitive(it, "item");
if(!cJSON_IsString(json_item))
{
return send_json_error(api, 400,
"bad_request",
"Invalid request: Batch delete requires an array of objects with \"item\" as string",
NULL);
}
// Check if type and kind are both present and strings
cJSON *json_type = cJSON_GetObjectItemCaseSensitive(it, "type");
cJSON *json_kind = cJSON_GetObjectItemCaseSensitive(it, "kind");
if(!cJSON_IsString(json_type) || !cJSON_IsString(json_kind))
{
return send_json_error(api, 400,
"bad_request",
"Invalid request: Batch delete requires an array of objects with \"type\" and \"kind\" as string",
NULL);
}
// Parse type and kind
// 0 = allow exact
// 1 = deny exact
// 2 = allow regex
// 3 = deny regex
int type = -1;
if(strcasecmp(json_type->valuestring, "allow") == 0)
{
if(strcasecmp(json_kind->valuestring, "exact") == 0)
type = 0;
else if(strcasecmp(json_kind->valuestring, "regex") == 0)
type = 2;
}
else if(strcasecmp(json_type->valuestring, "deny") == 0)
{
if(strcasecmp(json_kind->valuestring, "exact") == 0)
type = 1;
else if(strcasecmp(json_kind->valuestring, "regex") == 0)
type = 3;
}
// Check if type/kind combination is valid
if(type == -1)
{
return send_json_error(api, 400,
"bad_request",
"Invalid request: Batch delete requires an valid combination of \"type\" and \"kind\" for each object",
NULL);
}
// Replace type/kind with integer type
// array = [{"item": "example.com", "type": 0}]
cJSON_DeleteItemFromObject(it, "type");
cJSON_DeleteItemFromObject(it, "kind");
cJSON_AddNumberToObject(it, "type", type);
}
}
else if(!isBatchDelete)
{
// Create array with object (used for clients, groups, lists)
// array = [{"item": <item>}]
array = cJSON_CreateArray();
cJSON *obj = cJSON_CreateObject();
cJSON_AddItemToObject(obj, "item", cJSON_CreateStringReference(item));
cJSON_AddItemToArray(array, obj);
allocated_json = true;
}
// Verify that the payload is an array of objects each containing an
// item
if(isBatchDelete)
{
cJSON *it = NULL;
cJSON_ArrayForEach(it, array)
{
if(!cJSON_IsObject(it))
{
return send_json_error(api, 400,
"bad_request",
"Invalid request: Batch delete requires an array of objects",
NULL);
}
// Check if item is a string
cJSON *json_item = cJSON_GetObjectItemCaseSensitive(it, "item");
if(!cJSON_IsString(json_item))
{
return send_json_error(api, 400,
"bad_request",
"Invalid request: Batch delete requires an array of objects with \"item\" as string",
NULL);
}
}
}
// From here on, we can assume the JSON payload is valid
unsigned int deleted = 0u;
if(gravityDB_delFromTable(listtype, array, &deleted, &sql_msg))
{
// Inform the resolver that it needs to reload gravity
set_event(RELOAD_GRAVITY);
// Send empty reply with code 204 No Content
// Free memory allocated above
if(allocated_json)
cJSON_free(array);
// Send empty reply with codes:
// - 204 No Content (if any items were deleted)
// - 404 Not Found (if no items were deleted)
cJSON *json = JSON_NEW_OBJECT();
JSON_SEND_OBJECT_CODE(json, 204);
JSON_SEND_OBJECT_CODE(json, deleted > 0u ? 204 : 404);
}
else
{
// Free memory allocated above
if(allocated_json)
cJSON_free(array);
// Send error reply
return send_json_error(api, 400,
"database_error",
"Could not remove domain from database table",
"Could not remove entries from table",
sql_msg);
}
}
@@ -548,21 +739,40 @@ int api_list(struct ftl_conn *api)
{
enum gravity_list_type listtype;
bool can_modify = false;
bool batchDelete = false;
if((api->item = startsWith("/api/groups", api)) != NULL)
{
listtype = GRAVITY_GROUPS;
can_modify = true;
}
else if((api->item = startsWith("/api/groups:batchDelete", api)) != NULL)
{
listtype = GRAVITY_GROUPS;
can_modify = true;
batchDelete = true;
}
else if((api->item = startsWith("/api/lists", api)) != NULL)
{
listtype = GRAVITY_ADLISTS;
can_modify = true;
}
else if((api->item = startsWith("/api/lists:batchDelete", api)) != NULL)
{
listtype = GRAVITY_ADLISTS;
can_modify = true;
batchDelete = true;
}
else if((api->item = startsWith("/api/clients", api)) != NULL)
{
listtype = GRAVITY_CLIENTS;
can_modify = true;
}
else if((api->item = startsWith("/api/clients:batchDelete", api)) != NULL)
{
listtype = GRAVITY_CLIENTS;
can_modify = true;
batchDelete = true;
}
else if((api->item = startsWith("/api/domains/allow/exact", api)) != NULL)
{
listtype = GRAVITY_DOMAINLIST_ALLOW_EXACT;
@@ -575,7 +785,7 @@ int api_list(struct ftl_conn *api)
}
else if((api->item = startsWith("/api/domains/allow", api)) != NULL)
{
listtype = GRAVITY_DOMAINLIST_ALLOW_ALL;
listtype = GRAVITY_DOMAINLIST_ALLOW_ALL;
}
else if((api->item = startsWith("/api/domains/deny/exact", api)) != NULL)
{
@@ -603,6 +813,12 @@ int api_list(struct ftl_conn *api)
{
listtype = GRAVITY_DOMAINLIST_ALL_ALL;
}
else if((api->item = startsWith("/api/domains:batchDelete", api)) != NULL)
{
listtype = GRAVITY_DOMAINLIST_ALL_ALL;
can_modify = true;
batchDelete = true;
}
else
{
return send_json_error(api, 400,
@@ -643,7 +859,7 @@ int api_list(struct ftl_conn *api)
return ret;
}
}
else if(can_modify && api->method == HTTP_POST)
else if(can_modify && api->method == HTTP_POST && !batchDelete)
{
// Add item to list identified by payload
if(api->item != NULL && strlen(api->item) != 0)
@@ -651,7 +867,7 @@ int api_list(struct ftl_conn *api)
return send_json_error(api, 400,
"uri_error",
"Invalid request: Specify item in payload, not as URI parameter",
NULL);
api->item);
}
else
{
@@ -664,7 +880,7 @@ int api_list(struct ftl_conn *api)
return ret;
}
}
else if(can_modify && api->method == HTTP_DELETE)
else if(can_modify && (api->method == HTTP_DELETE || (api->method == HTTP_POST && batchDelete)))
{
// Delete item from list
// We would not actually need the SHM lock here, however, we do
src/api/network.c
+6 -3
View File
@@ -440,7 +440,8 @@ static int api_network_devices_DELETE(struct ftl_conn *api)
// Delete row from network table by ID
const char *sql_msg = NULL;
if(!networkTable_deleteDevice(db, device_id, &sql_msg))
int deleted = 0;
if(!networkTable_deleteDevice(db, device_id, &deleted, &sql_msg))
{
// Add SQL message (may be NULL = not available)
return send_json_error(api, 500,
@@ -452,9 +453,11 @@ static int api_network_devices_DELETE(struct ftl_conn *api)
// Close database
dbclose(&db);
// Send empty reply with code 204 No Content
// Send empty reply with codes:
// - 204 No Content (if any items were deleted)
// - 404 Not Found (if no items were deleted)
cJSON *json = JSON_NEW_OBJECT();
JSON_SEND_OBJECT_CODE(json, 204);
JSON_SEND_OBJECT_CODE(json, deleted > 0 ? 204 : 404);
}
int api_network_devices(struct ftl_conn *api)
src/api/queries.c
+9 -52
View File
@@ -34,8 +34,8 @@ static int add_strings_to_array(struct ftl_conn *api, cJSON *array, const char *
"Could not read from in-memory database",
NULL);
}
sqlite3_stmt *stmt;
sqlite3_stmt *stmt = NULL;
int rc = sqlite3_prepare_v2(memdb, querystr, -1, &stmt, NULL);
if( rc != SQLITE_OK )
{
@@ -438,29 +438,24 @@ int api_queries(struct ftl_conn *api)
}
}
// Get connection to in-memory database
sqlite3 *db = get_memdb();
// Finish preparing query string
querystr_finish(querystr, sort_col, sort_dir);
// Attach disk database if necessary
const char *message = "";
if(disk && !attach_disk_database(&message))
// Get connection to in-memory database
sqlite3 *memdb = get_memdb();
if(memdb == NULL)
{
return send_json_error(api, 500,
"internal_error",
"Internal server error, cannot attach disk database",
message);
return send_json_error(api, 500, // 500 Internal error
"database_error",
"Could not read from in-memory database",
NULL);
}
// Prepare SQLite3 statement
sqlite3_stmt *read_stmt = NULL;
int rc = sqlite3_prepare_v2(db, querystr, -1, &read_stmt, NULL);
int rc = sqlite3_prepare_v2(memdb, querystr, -1, &read_stmt, NULL);
if( rc != SQLITE_OK )
{
if(disk)
detach_disk_database(NULL);
return send_json_error(api, 500,
"internal_error",
"Internal server error, failed to prepare read SQL query",
@@ -484,8 +479,6 @@ int api_queries(struct ftl_conn *api)
{
sqlite3_reset(read_stmt);
sqlite3_finalize(read_stmt);
if(disk)
detach_disk_database(NULL);
return send_json_error(api, 500,
"internal_error",
"Internal server error, failed to bind timestamp:from to SQL query",
@@ -501,8 +494,6 @@ int api_queries(struct ftl_conn *api)
{
sqlite3_reset(read_stmt);
sqlite3_finalize(read_stmt);
if(disk)
detach_disk_database(NULL);
return send_json_error(api, 500,
"internal_error",
"Internal server error, failed to bind timestamp:until to SQL query",
@@ -518,8 +509,6 @@ int api_queries(struct ftl_conn *api)
{
sqlite3_reset(read_stmt);
sqlite3_finalize(read_stmt);
if(disk)
detach_disk_database(NULL);
return send_json_error(api, 500,
"internal_error",
"Internal server error, failed to bind domain to SQL query",
@@ -535,8 +524,6 @@ int api_queries(struct ftl_conn *api)
{
sqlite3_reset(read_stmt);
sqlite3_finalize(read_stmt);
if(disk)
detach_disk_database(NULL);
return send_json_error(api, 500,
"internal_error",
"Internal server error, failed to bind cip to SQL query",
@@ -552,8 +539,6 @@ int api_queries(struct ftl_conn *api)
{
sqlite3_reset(read_stmt);
sqlite3_finalize(read_stmt);
if(disk)
detach_disk_database(NULL);
return send_json_error(api, 500,
"internal_error",
"Internal server error, failed to bind client to SQL query",
@@ -569,8 +554,6 @@ int api_queries(struct ftl_conn *api)
{
sqlite3_reset(read_stmt);
sqlite3_finalize(read_stmt);
if(disk)
detach_disk_database(NULL);
return send_json_error(api, 500,
"internal_error",
"Internal server error, failed to bind upstream to SQL query",
@@ -595,8 +578,6 @@ int api_queries(struct ftl_conn *api)
{
sqlite3_reset(read_stmt);
sqlite3_finalize(read_stmt);
if(disk)
detach_disk_database(NULL);
return send_json_error(api, 500,
"internal_error",
"Internal server error, failed to bind type to SQL query",
@@ -605,8 +586,6 @@ int api_queries(struct ftl_conn *api)
}
else
{
if(disk)
detach_disk_database(NULL);
return send_json_error(api, 400,
"bad_request",
"Requested type is invalid",
@@ -631,8 +610,6 @@ int api_queries(struct ftl_conn *api)
{
sqlite3_reset(read_stmt);
sqlite3_finalize(read_stmt);
if(disk)
detach_disk_database(NULL);
return send_json_error(api, 500,
"internal_error",
"Internal server error, failed to bind status to SQL query",
@@ -641,8 +618,6 @@ int api_queries(struct ftl_conn *api)
}
else
{
if(disk)
detach_disk_database(NULL);
return send_json_error(api, 400,
"bad_request",
"Requested status is invalid",
@@ -667,8 +642,6 @@ int api_queries(struct ftl_conn *api)
{
sqlite3_reset(read_stmt);
sqlite3_finalize(read_stmt);
if(disk)
detach_disk_database(NULL);
return send_json_error(api, 500,
"internal_error",
"Internal server error, failed to bind reply to SQL query",
@@ -677,8 +650,6 @@ int api_queries(struct ftl_conn *api)
}
else
{
if(disk)
detach_disk_database(NULL);
return send_json_error(api, 400,
"bad_request",
"Requested reply is invalid",
@@ -703,8 +674,6 @@ int api_queries(struct ftl_conn *api)
{
sqlite3_reset(read_stmt);
sqlite3_finalize(read_stmt);
if(disk)
detach_disk_database(NULL);
return send_json_error(api, 500,
"internal_error",
"Internal server error, failed to bind dnssec to SQL query",
@@ -713,8 +682,6 @@ int api_queries(struct ftl_conn *api)
}
else
{
if(disk)
detach_disk_database(NULL);
return send_json_error(api, 400,
"bad_request",
"Requested dnssec is invalid",
@@ -731,8 +698,6 @@ int api_queries(struct ftl_conn *api)
{
sqlite3_reset(read_stmt);
sqlite3_finalize(read_stmt);
if(disk)
detach_disk_database(NULL);
return send_json_error(api, 500,
"internal_error",
"Internal server error, failed to bind count to SQL query",
@@ -901,13 +866,5 @@ int api_queries(struct ftl_conn *api)
// Finalize statements
sqlite3_finalize(read_stmt);
if(disk && !detach_disk_database(&message))
{
return send_json_error(api, 500,
"internal_error",
"Internal server error, cannot detach disk database",
message);
}
JSON_SEND_OBJECT(json);
}
src/api/stats.c
+32 -29
View File
@@ -8,22 +8,22 @@
* This file is copyright under the latest version of the EUPL.
* Please see LICENSE file for your rights under this license. */
#include "../FTL.h"
#include "../webserver/http-common.h"
#include "../webserver/json_macros.h"
#include "api.h"
#include "../shmem.h"
#include "../datastructure.h"
#include "FTL.h"
#include "webserver/http-common.h"
#include "webserver/json_macros.h"
#include "api/api.h"
#include "shmem.h"
#include "datastructure.h"
// read_setupVarsconf()
#include "../config/setupVars.h"
#include "config/setupVars.h"
// logging routines
#include "../log.h"
#include "log.h"
// config struct
#include "../config/config.h"
#include "config/config.h"
// overTime data
#include "../overTime.h"
#include "overTime.h"
// enum REGEX
#include "../regex_r.h"
#include "regex_r.h"
// sqrt()
#include <math.h>
@@ -137,15 +137,6 @@ int api_stats_summary(struct ftl_conn *api)
int api_stats_top_domains(struct ftl_conn *api)
{
int count = 10;
const int domains = counters->domains;
int *temparray = calloc(2*domains, sizeof(int*));
if(temparray == NULL)
{
log_err("Memory allocation failed in %s()", __FUNCTION__);
return 0;
}
// Exit before processing any data if requested via config setting
if(config.misc.privacylevel.v.privacy_level >= PRIVACY_HIDE_DOMAINS)
{
@@ -157,11 +148,23 @@ int api_stats_top_domains(struct ftl_conn *api)
cJSON *json = JSON_NEW_OBJECT();
cJSON *top_domains = JSON_NEW_ARRAY();
JSON_ADD_ITEM_TO_OBJECT(json, "top_domains", top_domains);
free(temparray);
JSON_SEND_OBJECT(json);
}
// Lock shared memory
lock_shm();
// Allocate memory
const int domains = counters->domains;
int *temparray = calloc(2*domains, sizeof(int));
if(temparray == NULL)
{
log_err("Memory allocation failed in %s()", __FUNCTION__);
return 0;
}
bool blocked = false; // Can be overwritten by query string
int count = 10;
// /api/stats/top_domains?blocked=true
if(api->request->query_string != NULL)
{
@@ -173,10 +176,8 @@ int api_stats_top_domains(struct ftl_conn *api)
get_int_var(api->request->query_string, "count", &count);
}
// Lock shared memory
lock_shm();
for(int domainID=0; domainID < domains; domainID++)
unsigned int added_domains = 0u;
for(int domainID = 0; domainID < domains; domainID++)
{
// Get domain pointer
const domainsData* domain = getDomain(domainID, true);
@@ -189,10 +190,12 @@ int api_stats_top_domains(struct ftl_conn *api)
else
// Count only permitted queries
temparray[2*domainID + 1] = (domain->count - domain->blockedcount);
added_domains++;
}
// Sort temporary array
qsort(temparray, domains, sizeof(int[2]), cmpdesc);
qsort(temparray, added_domains, sizeof(int[2]), cmpdesc);
// Get filter
const char* filter = read_setupVarsconf("API_QUERY_LOG_SHOW");
@@ -216,7 +219,7 @@ int api_stats_top_domains(struct ftl_conn *api)
int n = 0;
cJSON *top_domains = JSON_NEW_ARRAY();
for(int i = 0; i < domains; i++)
for(unsigned int i = 0; i < added_domains; i++)
{
// Get sorted index
const int domainID = temparray[2*i + 0];
@@ -282,7 +285,7 @@ int api_stats_top_clients(struct ftl_conn *api)
{
int count = 10;
const int clients = counters->clients;
int *temparray = calloc(2*clients, sizeof(int*));
int *temparray = calloc(2*clients, sizeof(int));
if(temparray == NULL)
{
log_err("Memory allocation failed in api_stats_top_clients()");
@@ -405,7 +408,7 @@ int api_stats_upstreams(struct ftl_conn *api)
{
unsigned int totalcount = 0;
const int upstreams = counters->upstreams;
int *temparray = calloc(2*upstreams, sizeof(int*));
int *temparray = calloc(2*upstreams, sizeof(int));
if(temparray == NULL)
{
log_err("Memory allocation failed in api_stats_upstreams()");
src/api/stats_database.c
+1 -1
View File
@@ -181,7 +181,7 @@ int api_stats_database_top_items(struct ftl_conn *api)
// Get options from API struct
bool blocked = false; // Can be overwritten by query string
const bool domains = api->opts.domains;
const bool domains = api->opts.flags & API_DOMAINS;
// Get parameters from query string
if(api->request->query_string != NULL)
src/api/teleporter.c
+513 -42
View File
@@ -15,8 +15,18 @@
#include "api/api.h"
// ERRBUF_SIZE
#include "config/dnsmasq_config.h"
// inflate_buffer()
#include "zip/gzip.h"
// find_file_in_tar()
#include "zip/tar.h"
// sqlite3_open_v2()
#include "database/sqlite3.h"
// dbquery()
#include "database/common.h"
// MAX_ROTATIONS
#include "files.h"
#define MAXZIPSIZE (50u*1024*1024)
#define MAXFILESIZE (50u*1024*1024)
static int api_teleporter_GET(struct ftl_conn *api)
{
@@ -58,9 +68,9 @@ static int api_teleporter_GET(struct ftl_conn *api)
struct upload_data {
bool too_large;
char *sid;
char *zip_data;
char *zip_filename;
size_t zip_size;
uint8_t *data;
char *filename;
size_t filesize;
};
// Callback function for CivetWeb to determine which fields we want to receive
@@ -79,7 +89,7 @@ static int field_found(const char *key,
is_sid = false;
if(strcasecmp(key, "file") == 0 && filename && *filename)
{
data->zip_filename = strdup(filename);
data->filename = strdup(filename);
is_file = true;
return MG_FORM_FIELD_STORAGE_GET;
}
@@ -103,21 +113,21 @@ static int field_get(const char *key, const char *value, size_t valuelen, void *
if(is_file)
{
if(data->zip_size + valuelen > MAXZIPSIZE)
if(data->filesize + valuelen > MAXFILESIZE)
{
log_warn("Uploaded Teleporter ZIP archive is too large (limit is %u bytes)",
MAXZIPSIZE);
log_warn("Uploaded Teleporter file is too large (limit is %u bytes)",
MAXFILESIZE);
data->too_large = true;
return MG_FORM_FIELD_HANDLE_ABORT;
}
// Allocate memory for the raw ZIP archive data
data->zip_data = realloc(data->zip_data, data->zip_size + valuelen);
// Copy the raw ZIP archive data
memcpy(data->zip_data + data->zip_size, value, valuelen);
// Store the size of the ZIP archive raw data
data->zip_size += valuelen;
log_debug(DEBUG_API, "Received ZIP archive (%zu bytes, buffer is now %zu bytes)",
valuelen, data->zip_size);
// Allocate memory for the raw file data
data->data = realloc(data->data, data->filesize + valuelen);
// Copy the raw file data
memcpy(data->data + data->filesize, value, valuelen);
// Store the size of the file raw data
data->filesize += valuelen;
log_debug(DEBUG_API, "Received file (%zu bytes, buffer is now %zu bytes)",
valuelen, data->filesize);
}
else if(is_sid)
{
@@ -143,24 +153,28 @@ static int field_stored(const char *path, long long file_size, void *user_data)
static int free_upload_data(struct upload_data *data)
{
// Free allocated memory
if(data->zip_filename)
if(data->filename)
{
free(data->zip_filename);
data->zip_filename = NULL;
free(data->filename);
data->filename = NULL;
}
if(data->sid)
{
free(data->sid);
data->sid = NULL;
}
if(data->zip_data)
if(data->data)
{
free(data->zip_data);
data->zip_data = NULL;
free(data->data);
data->data = NULL;
}
return 0;
}
// Private function prototypes
static int process_received_zip(struct ftl_conn *api, struct upload_data *data);
static int process_received_tar_gz(struct ftl_conn *api, struct upload_data *data);
static int api_teleporter_POST(struct ftl_conn *api)
{
struct upload_data data;
@@ -170,7 +184,7 @@ static int api_teleporter_POST(struct ftl_conn *api)
// Disallow large ZIP archives (> 50 MB) to prevent DoS attacks.
// Typically, the ZIP archive size should be around 30-100 kB.
if(req_info->content_length > MAXZIPSIZE)
if(req_info->content_length > MAXFILESIZE)
{
free_upload_data(&data);
return send_json_error(api, 400,
@@ -191,7 +205,7 @@ static int api_teleporter_POST(struct ftl_conn *api)
}
// Check if we received something we consider being a file
if(data.zip_data == NULL || data.zip_size == 0)
if(data.data == NULL || data.filesize == 0)
{
free_upload_data(&data);
return send_json_error(api, 400,
@@ -209,28 +223,46 @@ static int api_teleporter_POST(struct ftl_conn *api)
"ZIP archive too large",
NULL);
}
/*
// Set the payload to the SID we received (if available)
if(data.sid != NULL)
// Check if we received something that claims to be a ZIP archive
// - filename should end in ".zip"
// - the data itself
// - should be at least 40 bytes long
// - start with 0x04034b50 (local file header signature, see https://pkware.cachefly.net/webdocs/APPNOTE/APPNOTE-6.3.9.TXT)
if(strlen(data.filename) > 4 &&
strcmp(data.filename + strlen(data.filename) - 4, ".zip") == 0 &&
data.filesize >= 40 &&
memcmp(data.data, "\x50\x4b\x03\x04", 4) == 0)
{
const size_t bufsize = strlen(data.sid) + 5;
api->payload.raw = calloc(bufsize, sizeof(char));
strncpy(api->payload.raw, "sid=", 5);
strncat(api->payload.raw, data.sid, bufsize - 4);
return process_received_zip(api, &data);
}
// Check if we received something that claims to be a TAR.GZ archive
// - filename should end in ".tar.gz"
// - the data itself
// - should be at least 40 bytes long
// - start with 0x8b1f (local file header signature, see https://www.ietf.org/rfc/rfc1952.txt)
else if(strlen(data.filename) > 7 &&
strcmp(data.filename + strlen(data.filename) - 7, ".tar.gz") == 0 &&
data.filesize >= 40 &&
memcmp(data.data, "\x1f\x8b", 2) == 0)
{
return process_received_tar_gz(api, &data);
}
// Check if the client is authorized to use this API endpoint
if(check_client_auth(api) == API_AUTH_UNAUTHORIZED)
{
free_upload_data(&data);
return send_json_unauthorized(api);
}
*/
// Process what we received
// else: invalid file
free_upload_data(&data);
return send_json_error(api, 400,
"bad_request",
"Invalid file",
"The uploaded file does not appear to be a valid Pi-hole Teleporter archive");
}
static int process_received_zip(struct ftl_conn *api, struct upload_data *data)
{
char hint[ERRBUF_SIZE];
memset(hint, 0, sizeof(hint));
cJSON *json_files = JSON_NEW_ARRAY();
const char *error = read_teleporter_zip(data.zip_data, data.zip_size, hint, json_files);
const char *error = read_teleporter_zip(data->data, data->filesize, hint, json_files);
if(error != NULL)
{
const size_t msglen = strlen(error) + strlen(hint) + 4;
@@ -242,7 +274,7 @@ static int api_teleporter_POST(struct ftl_conn *api)
strcat(msg, ": ");
strcat(msg, hint);
}
free_upload_data(&data);
free_upload_data(data);
return send_json_error_free(api, 400,
"bad_request",
"Invalid ZIP archive",
@@ -250,7 +282,7 @@ static int api_teleporter_POST(struct ftl_conn *api)
}
// Free allocated memory
free_upload_data(&data);
free_upload_data(data);
// Send response
cJSON *json = JSON_NEW_OBJECT();
@@ -258,6 +290,445 @@ static int api_teleporter_POST(struct ftl_conn *api)
JSON_SEND_OBJECT(json);
}
static struct teleporter_files {
const char *filename; // Filename of the file in the archive
const char *table_name; // Name of the table in the database
const int listtype; // Type of list (only used for domainlist table)
const size_t num_columns; // Number of columns in the table
const char *columns[10]; // List of columns in the table
} teleporter_v5_files[] = {
{
.filename = "adlist.json",
.table_name = "adlist",
.listtype = -1,
.num_columns = 10,
.columns = { "id", "address", "enabled", "date_added", "date_modified", "comment", "date_updated", "number", "invalid_domains", "status" } // abp_entries and type are not defined in Pi-hole v5.x
},{
.filename = "adlist_by_group.json",
.table_name = "adlist_by_group",
.listtype = -1,
.num_columns = 2,
.columns = { "group_id", "adlist_id" }
},{
.filename = "blacklist.exact.json",
.table_name = "domainlist",
.listtype = 1, // GRAVITY_DOMAINLIST_DENY_EXACT
.num_columns = 7,
.columns = { "id", "domain", "enabled", "date_added", "date_modified", "comment", "type" }
},{
.filename = "blacklist.regex.json",
.table_name = "domainlist",
.listtype = 3, // GRAVITY_DOMAINLIST_DENY_REGEX
.num_columns = 7,
.columns = { "id", "domain", "enabled", "date_added", "date_modified", "comment", "type" }
},{
.filename = "client.json",
.table_name = "client",
.listtype = -1,
.num_columns = 5,
.columns = { "id", "ip", "date_added", "date_modified", "comment" }
},{
.filename = "client_by_group.json",
.table_name = "client_by_group",
.listtype = -1,
.num_columns = 2,
.columns = { "group_id", "client_id" }
},{
.filename = "domainlist_by_group.json",
.table_name = "domainlist_by_group",
.listtype = -1,
.num_columns = 2,
.columns = { "group_id", "domainlist_id" }
},{
.filename = "group.json",
.table_name = "group",
.listtype = -1,
.num_columns = 6,
.columns = { "id", "enabled", "name", "date_added", "date_modified", "description" }
},{
.filename = "whitelist.exact.json",
.table_name = "domainlist",
.listtype = 0, // GRAVITY_DOMAINLIST_ALLOW_EXACT
.num_columns = 7,
.columns = { "id", "domain", "enabled", "date_added", "date_modified", "comment", "type" }
},{
.filename = "whitelist.regex.json",
.table_name = "domainlist",
.listtype = 2, // GRAVITY_DOMAINLIST_ALLOW_REGEX
.num_columns = 7,
.columns = { "id", "domain", "enabled", "date_added", "date_modified", "comment", "type" }
}
};
static bool import_json_table(cJSON *json, struct teleporter_files *file)
{
// Check if the JSON object is an array
if(!cJSON_IsArray(json))
{
log_err("import_json_table(%s): JSON object is not an array", file->filename);
return false;
}
// Check if the JSON array is empty, if so, we can return early
const int num_entries = cJSON_GetArraySize(json);
// Check if all the JSON entries contain all the expected columns
cJSON *json_object = NULL;
cJSON_ArrayForEach(json_object, json)
{
if(!cJSON_IsObject(json_object))
{
log_err("import_json_table(%s): JSON array does not contain objects", file->filename);
return false;
}
// If this is a record for the domainlist table, add type/kind
if(strcmp(file->table_name, "domainlist") == 0)
{
// Add type/kind to the JSON object
cJSON_AddNumberToObject(json_object, "type", file->listtype);
}
// Check if the JSON object contains the expected columns
for(size_t i = 0; i < file->num_columns; i++)
{
if(cJSON_GetObjectItemCaseSensitive(json_object, file->columns[i]) == NULL)
{
log_err("import_json_table(%s): JSON object does not contain column \"%s\"", file->filename, file->columns[i]);
return false;
}
}
}
log_info("import_json_table(%s): JSON array contains %d entr%s", file->filename, num_entries, num_entries == 1 ? "y" : "ies");
// Open database connection
sqlite3 *db = NULL;
if(sqlite3_open_v2(config.files.gravity.v.s, &db, SQLITE_OPEN_READWRITE, NULL) != SQLITE_OK)
{
log_err("import_json_table(%s): Unable to open database file \"%s\": %s",
file->filename, config.files.database.v.s, sqlite3_errmsg(db));
sqlite3_close(db);
return false;
}
// Disable foreign key constraints
if(sqlite3_exec(db, "PRAGMA foreign_keys = OFF;", NULL, NULL, NULL) != SQLITE_OK)
{
log_err("import_json_table(%s): Unable to disable foreign key constraints: %s", file->filename, sqlite3_errmsg(db));
sqlite3_close(db);
return false;
}
// Start transaction
if(sqlite3_exec(db, "BEGIN TRANSACTION;", NULL, NULL, NULL) != SQLITE_OK)
{
log_err("import_json_table(%s): Unable to start transaction: %s", file->filename, sqlite3_errmsg(db));
sqlite3_close(db);
return false;
}
// Clear existing table entries
if(file->listtype < 0)
{
// Delete all entries in the table
log_debug(DEBUG_API, "import_json_table(%s): Deleting all entries from table \"%s\"", file->filename, file->table_name);
if(dbquery(db, "DELETE FROM \"%s\";", file->table_name) != SQLITE_OK)
{
log_err("import_json_table(%s): Unable to delete entries from table \"%s\": %s",
file->filename, file->table_name, sqlite3_errmsg(db));
sqlite3_close(db);
return false;
}
}
else
{
// Delete all entries in the table of the same type
log_debug(DEBUG_API, "import_json_table(%s): Deleting all entries from table \"%s\" of type %d", file->filename, file->table_name, file->listtype);
if(dbquery(db, "DELETE FROM \"%s\" WHERE type = %d;", file->table_name, file->listtype) != SQLITE_OK)
{
log_err("import_json_table(%s): Unable to delete entries from table \"%s\": %s",
file->filename, file->table_name, sqlite3_errmsg(db));
sqlite3_close(db);
return false;
}
}
// Build dynamic SQL insertion statement
// "INSERT OR IGNORE INTO table (column1, column2, ...) VALUES (?, ?, ...);"
char *sql = sqlite3_mprintf("INSERT OR IGNORE INTO \"%s\" (", file->table_name);
for(size_t i = 0; i < file->num_columns; i++)
{
char *sql2 = sqlite3_mprintf("%s%s", sql, file->columns[i]);
sqlite3_free(sql);
sql = NULL;
if(i < file->num_columns - 1)
{
sql = sqlite3_mprintf("%s, ", sql2);
sqlite3_free(sql2);
sql2 = NULL;
}
else
{
sql = sqlite3_mprintf("%s) VALUES (", sql2);
sqlite3_free(sql2);
sql2 = NULL;
}
}
for(size_t i = 0; i < file->num_columns; i++)
{
char *sql2 = sqlite3_mprintf("%s?", sql);
sqlite3_free(sql);
sql = NULL;
if(i < file->num_columns - 1)
{
sql = sqlite3_mprintf("%s, ", sql2);
sqlite3_free(sql2);
sql2 = NULL;
}
else
{
sql = sqlite3_mprintf("%s);", sql2);
sqlite3_free(sql2);
sql2 = NULL;
}
}
// Prepare SQL statement
sqlite3_stmt *stmt = NULL;
if(sqlite3_prepare_v2(db, sql, -1, &stmt, NULL) != SQLITE_OK)
{
log_err("Unable to prepare SQL statement: %s", sqlite3_errmsg(db));
sqlite3_free(sql);
sqlite3_close(db);
return false;
}
// Free allocated memory
sqlite3_free(sql);
sql = NULL;
// Iterate over all JSON objects
cJSON_ArrayForEach(json_object, json)
{
// Bind values to SQL statement
for(size_t i = 0; i < file->num_columns; i++)
{
cJSON *json_value = cJSON_GetObjectItemCaseSensitive(json_object, file->columns[i]);
if(cJSON_IsString(json_value))
{
// Bind string value
if(sqlite3_bind_text(stmt, i + 1, json_value->valuestring, -1, SQLITE_STATIC) != SQLITE_OK)
{
log_err("Unable to bind text value to SQL statement: %s", sqlite3_errmsg(db));
sqlite3_finalize(stmt);
sqlite3_close(db);
return false;
}
}
else if(cJSON_IsNumber(json_value))
{
// Bind integer value
if(sqlite3_bind_int(stmt, i + 1, json_value->valueint) != SQLITE_OK)
{
log_err("Unable to bind integer value to SQL statement: %s", sqlite3_errmsg(db));
sqlite3_finalize(stmt);
sqlite3_close(db);
return false;
}
}
else if(cJSON_IsNull(json_value))
{
// Bind NULL value
if(sqlite3_bind_null(stmt, i + 1) != SQLITE_OK)
{
log_err("Unable to bind NULL value to SQL statement: %s", sqlite3_errmsg(db));
sqlite3_finalize(stmt);
sqlite3_close(db);
return false;
}
}
else
{
log_err("Unable to bind value to SQL statement: type = %X", (unsigned int)json_value->type & 0xFF);
sqlite3_finalize(stmt);
sqlite3_close(db);
return false;
}
}
// Execute SQL statement
if(sqlite3_step(stmt) != SQLITE_DONE)
{
log_err("Unable to execute SQL statement: %s", sqlite3_errmsg(db));
sqlite3_finalize(stmt);
sqlite3_close(db);
return false;
}
// Reset SQL statement
if(sqlite3_reset(stmt) != SQLITE_OK)
{
log_err("Unable to reset SQL statement: %s", sqlite3_errmsg(db));
sqlite3_finalize(stmt);
sqlite3_close(db);
return false;
}
}
// Finalize SQL statement
if(sqlite3_finalize(stmt) != SQLITE_OK)
{
log_err("Unable to finalize SQL statement: %s", sqlite3_errmsg(db));
sqlite3_close(db);
return false;
}
// Commit transaction
if(sqlite3_exec(db, "COMMIT;", NULL, NULL, NULL) != SQLITE_OK)
{
log_err("Unable to commit transaction: %s", sqlite3_errmsg(db));
sqlite3_close(db);
return false;
}
// Close database connection
sqlite3_close(db);
return true;
}
static int process_received_tar_gz(struct ftl_conn *api, struct upload_data *data)
{
// Try to decompress the received data
uint8_t *archive = NULL;
mz_ulong archive_size = 0u;
if(!inflate_buffer(data->data, data->filesize, &archive, &archive_size))
{
free_upload_data(data);
return send_json_error(api, 400,
"bad_request",
"Invalid GZIP archive",
"The uploaded file does not appear to be a valid gzip archive - decompression failed");
}
// Print all files in the TAR archive if in debug mode
if(config.debug.api.v.b)
{
cJSON *json_files = list_files_in_tar(archive, archive_size);
cJSON *file = NULL;
cJSON_ArrayForEach(file, json_files)
{
const cJSON *name = cJSON_GetObjectItemCaseSensitive(file, "name");
const cJSON *size = cJSON_GetObjectItemCaseSensitive(file, "size");
if(name == NULL || size == NULL)
continue;
log_debug(DEBUG_API, "Found file in TAR archive: \"%s\" (%d bytes)",
name->valuestring, size->valueint);
}
}
// Parse JSON files in the TAR archive
cJSON *imported_files = JSON_NEW_ARRAY();
for(size_t i = 0; i < sizeof(teleporter_v5_files) / sizeof(struct teleporter_files); i++)
{
size_t fileSize = 0u;
cJSON *json = NULL;
const char *file = find_file_in_tar(archive, archive_size, teleporter_v5_files[i].filename, &fileSize);
if(file != NULL && fileSize > 0u && (json = cJSON_ParseWithLength(file, fileSize)) != NULL)
if(import_json_table(json, &teleporter_v5_files[i]))
JSON_COPY_STR_TO_ARRAY(imported_files, teleporter_v5_files[i].filename);
}
// Temporarily write further files to to disk so we can import them on restart
struct {
const char *archive_name;
const char *destination;
} extract_files[] = {
{
.archive_name = "custom.list",
.destination = DNSMASQ_CUSTOM_LIST_LEGACY
},{
.archive_name = "dhcp.leases",
.destination = DHCPLEASESFILE
},{
.archive_name = "pihole-FTL.conf",
.destination = GLOBALCONFFILE_LEGACY
},{
.archive_name = "setupVars.conf",
.destination = config.files.setupVars.v.s
}
};
for(size_t i = 0; i < sizeof(extract_files) / sizeof(*extract_files); i++)
{
size_t fileSize = 0u;
const char *file = find_file_in_tar(archive, archive_size, extract_files[i].archive_name, &fileSize);
if(file != NULL && fileSize > 0u)
{
// Write file to disk
log_info("Writing file \"%s\" (%zu bytes) to \"%s\"",
extract_files[i].archive_name, fileSize, extract_files[i].destination);
FILE *fp = fopen(extract_files[i].destination, "wb");
if(fp == NULL)
{
log_err("Unable to open file \"%s\" for writing: %s", extract_files[i].destination, strerror(errno));
continue;
}
if(fwrite(file, fileSize, 1, fp) != 1)
{
log_err("Unable to write file \"%s\": %s", extract_files[i].destination, strerror(errno));
fclose(fp);
continue;
}
fclose(fp);
JSON_COPY_STR_TO_ARRAY(imported_files, extract_files[i].destination);
}
}
// Append WEB_PORTS to setupVars.conf
FILE *fp = fopen(config.files.setupVars.v.s, "a");
if(fp == NULL)
log_err("Unable to open file \"%s\" for appending: %s", config.files.setupVars.v.s, strerror(errno));
else
{
fprintf(fp, "WEB_PORTS=%s\n", config.webserver.port.v.s);
fclose(fp);
}
// Remove pihole.toml to prevent it from being imported on restart
if(remove(GLOBALTOMLPATH) != 0)
log_err("Unable to remove file \"%s\": %s", GLOBALTOMLPATH, strerror(errno));
// Remove all rotated pihole.toml files to avoid automatic config
// restore on restart
for(unsigned int i = MAX_ROTATIONS; i > 0; i--)
{
const char *fname = GLOBALTOMLPATH;
const char *filename = basename(fname);
// extra 6 bytes is enough space for up to 999 rotations ("/", ".", "\0", "999")
const size_t buflen = strlen(filename) + strlen(BACKUP_DIR) + 6;
char *path = calloc(buflen, sizeof(char));
snprintf(path, buflen, BACKUP_DIR"/%s.%u", filename, i);
// Remove file (if it exists)
if(remove(path) != 0 && errno != ENOENT)
log_err("Unable to remove file \"%s\": %s", path, strerror(errno));
}
// Free allocated memory
free_upload_data(data);
// Signal FTL we want to restart for re-import
api->ftl.restart = true;
// Send response
cJSON *json = JSON_NEW_OBJECT();
JSON_ADD_ITEM_TO_OBJECT(json, "files", imported_files);
JSON_SEND_OBJECT(json);
}
int api_teleporter(struct ftl_conn *api)
{
if(api->method == HTTP_GET)
src/config/cli.c
+3 -2
View File
@@ -171,8 +171,9 @@ static bool readStringValue(struct conf_item *conf_item, const char *value, stru
// Get password hash as allocated string (an empty string is hashed to an empty string)
char *pwhash = strlen(value) > 0 ? create_password(value) : strdup("");
// Verify that the password hash is valid
if(verify_password(value, pwhash, false) != PASSWORD_CORRECT)
// Verify that the password hash is either valid or empty
const enum password_result status = verify_password(value, pwhash, false);
if(status != PASSWORD_CORRECT && status != NO_PASSWORD_SET)
{
log_err("Failed to create password hash (verification failed), password remains unchanged");
free(pwhash);
src/config/config.c
+31 -28
View File
@@ -1447,36 +1447,39 @@ bool readFTLconf(struct config *conf, const bool rewrite)
rename(GLOBALTOMLPATH, new_name);
}
// Determine default webserver ports
// Check if ports 80/TCP and 443/TCP are already in use
const in_port_t http_port = port_in_use(80) ? 8080 : 80;
const in_port_t https_port = port_in_use(443) ? 8443 : 443;
// Create a string with the default ports
// Allocate memory for the string
char *ports = calloc(32, sizeof(char));
if(ports == NULL)
// Determine default webserver ports if not imported from setupVars.conf
if(!(config.webserver.port.f & FLAG_CONF_IMPORTED))
{
log_err("Unable to allocate memory for default ports string");
return false;
// Check if ports 80/TCP and 443/TCP are already in use
const in_port_t http_port = port_in_use(80) ? 8080 : 80;
const in_port_t https_port = port_in_use(443) ? 8443 : 443;
// Create a string with the default ports
// Allocate memory for the string
char *ports = calloc(32, sizeof(char));
if(ports == NULL)
{
log_err("Unable to allocate memory for default ports string");
return false;
}
// Create the string
snprintf(ports, 32, "%d,%ds", http_port, https_port);
// Append IPv6 ports if IPv6 is enabled
const bool have_ipv6 = ipv6_enabled();
if(have_ipv6)
snprintf(ports + strlen(ports), 32 - strlen(ports),
",[::]:%d,[::]:%ds", http_port, https_port);
// Set default values for webserver ports
if(conf->webserver.port.t == CONF_STRING_ALLOCATED)
free(conf->webserver.port.v.s);
conf->webserver.port.v.s = ports;
conf->webserver.port.t = CONF_STRING_ALLOCATED;
log_info("Initialised webserver ports at %d (HTTP) and %d (HTTPS), IPv6 support is %s",
http_port, https_port, have_ipv6 ? "enabled" : "disabled");
}
// Create the string
snprintf(ports, 32, "%d,%ds", http_port, https_port);
// Append IPv6 ports if IPv6 is enabled
const bool have_ipv6 = ipv6_enabled();
if(have_ipv6)
snprintf(ports + strlen(ports), 32 - strlen(ports),
",[::]:%d,[::]:%ds", http_port, https_port);
// Set default values for webserver ports
if(conf->webserver.port.t == CONF_STRING_ALLOCATED)
free(conf->webserver.port.v.s);
conf->webserver.port.v.s = ports;
conf->webserver.port.t = CONF_STRING_ALLOCATED;
log_info("Initialised webserver ports at %d (HTTP) and %d (HTTPS), IPv6 support is %s",
http_port, https_port, have_ipv6 ? "enabled" : "disabled");
// Initialize the TOML config file
writeFTLtoml(true);
src/config/config.h
+4
View File
@@ -38,6 +38,9 @@
// characters will be replaced by their UTF-8 escape sequences (UCS-2)
#define TOML_UTF8
// Location of the legacy (pre-v6.0) config file
#define GLOBALCONFFILE_LEGACY "/etc/pihole/pihole-FTL.conf"
union conf_value {
bool b; // boolean value
int i; // integer value
@@ -94,6 +97,7 @@ enum conf_type {
#define FLAG_INVALIDATE_SESSIONS (1 << 3)
#define FLAG_WRITE_ONLY (1 << 4)
#define FLAG_ENV_VAR (1 << 5)
#define FLAG_CONF_IMPORTED (1 << 6)
struct conf_item {
const char *k; // item Key
src/config/legacy_reader.c
+5 -2
View File
@@ -43,7 +43,7 @@ static FILE * __attribute__((nonnull(1), malloc, warn_unused_result)) openFTLcon
return fp;
// Local file not present, try system file
*path = "/etc/pihole/pihole-FTL.conf";
*path = GLOBALCONFFILE_LEGACY;
fp = fopen(*path, "r");
return fp;
@@ -113,9 +113,12 @@ const char *readFTLlegacy(struct config *conf)
const char *path = NULL;
FILE *fp = openFTLconf(&path);
if(fp == NULL)
{
log_warn("No readable FTL config file found, using default settings");
return NULL;
}
log_notice("Reading legacy config file");
log_info("Reading legacy config files from %s", path);
// MAXDBDAYS
// defaults to: 365 days
src/config/password.c
+5 -3
View File
@@ -328,6 +328,7 @@ enum password_result verify_login(const char *password)
log_debug(DEBUG_API, "App password correct");
return APPPASSWORD_CORRECT;
}
// Return result
return pw;
}
@@ -336,7 +337,7 @@ enum password_result verify_password(const char *password, const char *pwhash, c
{
// No password set
if(pwhash == NULL || pwhash[0] == '\0')
return PASSWORD_CORRECT;
return NO_PASSWORD_SET;
// No password supplied
if(password == NULL || password[0] == '\0')
@@ -606,8 +607,9 @@ bool set_and_check_password(struct conf_item *conf_item, const char *password)
// Get password hash as allocated string (an empty string is hashed to an empty string)
char *pwhash = strlen(password) > 0 ? create_password(password) : strdup("");
// Verify that the password hash is valid
if(verify_password(password, pwhash, false) != PASSWORD_CORRECT)
// Verify that the password hash is valid or that no password is set
const enum password_result status = verify_password(password, pwhash, false);
if(status != PASSWORD_CORRECT && status != NO_PASSWORD_SET)
{
free(pwhash);
log_warn("Failed to create password hash (verification failed), password remains unchanged");
src/config/password.h
+1
View File
@@ -26,6 +26,7 @@ enum password_result {
PASSWORD_INCORRECT = 0,
PASSWORD_CORRECT = 1,
APPPASSWORD_CORRECT = 2,
NO_PASSWORD_SET = 3,
PASSWORD_RATE_LIMITED = -1
} __attribute__((packed));
src/config/setupVars.c
+22
View File
@@ -42,6 +42,7 @@ static void get_conf_string_from_setupVars(const char *key, struct conf_item *co
free(conf_item->v.s);
conf_item->v.s = strdup(setupVarsValue);
conf_item->t = CONF_STRING_ALLOCATED;
conf_item->f |= FLAG_CONF_IMPORTED;
// Free memory, harmless to call if read_setupVarsconf() didn't return a result
clearSetupVarsArray();
@@ -374,6 +375,8 @@ static void get_conf_listeningMode_from_setupVars(void)
void importsetupVarsConf(void)
{
log_info("Migrating config from %s", config.files.setupVars.v.s);
// Try to obtain password hash from setupVars.conf
get_conf_string_from_setupVars("WEBPASSWORD", &config.webserver.api.pwhash);
@@ -443,7 +446,26 @@ void importsetupVarsConf(void)
get_conf_bool_from_setupVars("DHCP_RAPID_COMMIT", &config.dhcp.rapidCommit);
get_conf_bool_from_setupVars("queryLogging", &config.dns.queryLogging);
get_conf_string_from_setupVars("GRAVITY_TMPDIR", &config.files.gravity_tmp);
// Ports may be temporarily stored when importing a legacy Teleporter v5 file
get_conf_string_from_setupVars("WEB_PORTS", &config.webserver.port);
// Move the setupVars.conf file to setupVars.conf.old
char *old_setupVars = calloc(strlen(config.files.setupVars.v.s) + 5, sizeof(char));
if(old_setupVars == NULL)
{
log_warn("Could not allocate memory for old_setupVars");
return;
}
strcpy(old_setupVars, config.files.setupVars.v.s);
strcat(old_setupVars, ".old");
if(rename(config.files.setupVars.v.s, old_setupVars) != 0)
log_warn("Could not move %s to %s", config.files.setupVars.v.s, old_setupVars);
else
log_info("Moved %s to %s", config.files.setupVars.v.s, old_setupVars);
free(old_setupVars);
}
char* __attribute__((pure)) find_equals(char *s)
src/database/common.c
+8 -2
View File
@@ -251,9 +251,15 @@ void SQLite3LogCallback(void *pArg, int iErrCode, const char *zMsg)
generate_backtrace();
if(iErrCode == SQLITE_WARNING)
log_warn("SQLite3 message: %s (%d)", zMsg, iErrCode);
log_warn("SQLite3: %s (%d)", zMsg, iErrCode);
else if(iErrCode == SQLITE_NOTICE || iErrCode == SQLITE_SCHEMA)
// SQLITE_SCHEMA is returned when the database schema has changed
// This is not necessarily an error, as sqlite3_step() will re-prepare
// the statement and try again. If it cannot, it will return an error
// and this will be handled over there.
log_debug(DEBUG_ANY, "SQLite3: %s (%d)", zMsg, iErrCode);
else
log_err("SQLite3 message: %s (%d)", zMsg, iErrCode);
log_err("SQLite3: %s (%d)", zMsg, iErrCode);
}
void db_init(void)
src/database/database-thread.c
+47 -1
View File
@@ -56,6 +56,27 @@ static bool delete_old_queries_in_DB(sqlite3 *db)
return true;
}
static bool analyze_database(sqlite3 *db)
{
// Optimize the database by running ANALYZE
// The ANALYZE command gathers statistics about tables and indices and
// stores the collected information in internal tables of the database
// where the query optimizer can access the information and use it to
// help make better query planning choices.
// Measure time
struct timespec start, end;
clock_gettime(CLOCK_MONOTONIC, &start);
SQL_bool(db, "ANALYZE;");
clock_gettime(CLOCK_MONOTONIC, &end);
// Print final message
log_info("Optimized database in %.3f seconds",
(double)(end.tv_sec - start.tv_sec) + 1e-9*(end.tv_nsec - start.tv_nsec));
return true;
}
#define DBOPEN_OR_AGAIN() { if(!db) db = dbopen(false, false); if(!db) { thread_sleepms(DB, 5000); continue; } }
#define BREAK_IF_KILLED() { if(killed) break; }
#define DBCLOSE_OR_BREAK() { dbclose(&db); BREAK_IF_KILLED(); }
@@ -72,6 +93,17 @@ void *DB_thread(void *val)
time_t before = time(NULL);
time_t lastDBsave = before - before%config.database.DBinterval.v.ui;
// Other timestamps, made independent from the exact time FTL was
// started
time_t lastAnalyze = before - before % DATABASE_ANALYZE_INTERVAL;
time_t lastMACVendor = before - before % DATABASE_MACVENDOR_INTERVAL;
// Add some randomness (up to ome hour) to these timestamps to avoid
// them running at the same time. This is not a security feature, so
// using rand() is fine.
lastAnalyze += rand() % 3600;
lastMACVendor += rand() % 3600;
// This thread runs until shutdown of the process. We keep this thread
// running when pihole-FTL.db is corrupted because reloading of privacy
// level, and the gravity database (initially and after gravity)
@@ -135,16 +167,30 @@ void *DB_thread(void *val)
set_event(PARSE_NEIGHBOR_CACHE);
}
// Intermediate cancellation-point
if(killed)
break;
// Optimize database once per week
if(now - lastAnalyze >= DATABASE_ANALYZE_INTERVAL)
{
DBOPEN_OR_AGAIN();
analyze_database(db);
lastAnalyze = now;
DBCLOSE_OR_BREAK();
}
// Intermediate cancellation-point
if(killed)
break;
// Update MAC vendor strings once a month (the MAC vendor
// database is not updated very often)
if(now % 2592000L == 0)
if(now - lastMACVendor >= DATABASE_MACVENDOR_INTERVAL)
{
DBOPEN_OR_AGAIN();
updateMACVendorRecords(db);
lastMACVendor = now;
DBCLOSE_OR_BREAK();
}
src/database/gravity-db.c
+214 -93
View File
@@ -1700,15 +1700,15 @@ bool gravityDB_addToTable(const enum gravity_list_type listtype, tablerow *row,
{
if(strcasecmp("allow", row->type) == 0 &&
strcasecmp("exact", row->kind) == 0)
oldtype = 0;
oldtype = 0;
else if(strcasecmp("deny", row->type) == 0 &&
strcasecmp("exact", row->kind) == 0)
oldtype = 1;
oldtype = 1;
else if(strcasecmp("allow", row->type) == 0 &&
strcasecmp("regex", row->kind) == 0)
oldtype = 2;
oldtype = 2;
else if(strcasecmp("deny", row->type) == 0 &&
strcasecmp("regex", row->kind) == 0)
strcasecmp("regex", row->kind) == 0)
oldtype = 3;
else
{
@@ -1792,131 +1792,252 @@ bool gravityDB_addToTable(const enum gravity_list_type listtype, tablerow *row,
return okay;
}
bool gravityDB_delFromTable(const enum gravity_list_type listtype, const char* argument, const char **message)
bool gravityDB_delFromTable(const enum gravity_list_type listtype, const cJSON* array, unsigned int *deleted, const char **message)
{
// Return early if database is not available
if(gravity_db == NULL)
{
*message = "Database not available";
return false;
}
int type = -1;
switch (listtype)
// Return early if passed JSON argument is not an array
if(!cJSON_IsArray(array))
{
case GRAVITY_DOMAINLIST_ALLOW_EXACT:
type = 0;
break;
case GRAVITY_DOMAINLIST_DENY_EXACT:
type = 1;
break;
case GRAVITY_DOMAINLIST_ALLOW_REGEX:
type = 2;
break;
case GRAVITY_DOMAINLIST_DENY_REGEX:
type = 3;
break;
case GRAVITY_GROUPS:
case GRAVITY_ADLISTS:
case GRAVITY_CLIENTS:
// No type required for these tables
break;
// Aggregate types cannot be handled by this routine
case GRAVITY_GRAVITY:
case GRAVITY_ANTIGRAVITY:
case GRAVITY_DOMAINLIST_ALLOW_ALL:
case GRAVITY_DOMAINLIST_DENY_ALL:
case GRAVITY_DOMAINLIST_ALL_EXACT:
case GRAVITY_DOMAINLIST_ALL_REGEX:
case GRAVITY_DOMAINLIST_ALL_ALL:
default:
return false;
*message = "Argument is not an array";
log_err("gravityDB_delFromTable(%d): %s",
listtype, *message);
return false;
}
// Prepare SQLite statement
const bool isDomain = listtype == GRAVITY_DOMAINLIST_ALLOW_EXACT ||
listtype == GRAVITY_DOMAINLIST_DENY_EXACT ||
listtype == GRAVITY_DOMAINLIST_ALLOW_REGEX ||
listtype == GRAVITY_DOMAINLIST_DENY_REGEX ||
listtype == GRAVITY_DOMAINLIST_ALL_ALL; // batch delete
// Begin transaction
const char *querystr = "BEGIN TRANSACTION;";
int rc = sqlite3_exec(gravity_db, querystr, NULL, NULL, NULL);
if(rc != SQLITE_OK)
{
*message = sqlite3_errmsg(gravity_db);
log_err("gravityDB_delFromTable(%d): SQL error exec(\"%s\"): %s",
listtype, querystr, *message);
return false;
}
// Create temporary table for JSON argument
if(isDomain)
// Create temporary table for domains to be deleted
querystr = "CREATE TEMPORARY TABLE deltable (type INT, item TEXT);";
else
querystr = "CREATE TEMPORARY TABLE deltable (item TEXT);";
sqlite3_stmt* stmt = NULL;
const char *querystr[3] = {NULL, NULL, NULL};
if(listtype == GRAVITY_GROUPS)
querystr[0] = "DELETE FROM \"group\" WHERE name = :argument;";
else if(listtype == GRAVITY_ADLISTS)
rc = sqlite3_prepare_v2(gravity_db, querystr, -1, &stmt, NULL);
if( rc != SQLITE_OK )
{
// This is actually a three-step deletion to satisfy foreign-key constraints
querystr[0] = "DELETE FROM gravity WHERE adlist_id = (SELECT id FROM adlist WHERE address = :argument);";
querystr[1] = "DELETE FROM antigravity WHERE adlist_id = (SELECT id FROM adlist WHERE address = :argument);";
querystr[2] = "DELETE FROM adlist WHERE address = :argument;";
*message = sqlite3_errmsg(gravity_db);
log_err("gravityDB_delFromTable(%d) - SQL error prepare(\"%s\"): %s",
listtype, querystr, *message);
// Rollback transaction
querystr = "ROLLBACK TRANSACTION;";
sqlite3_exec(gravity_db, querystr, NULL, NULL, NULL);
return false;
}
else if(listtype == GRAVITY_CLIENTS)
querystr[0] = "DELETE FROM client WHERE ip = :argument;";
else // domainlist
querystr[0] = "DELETE FROM domainlist WHERE domain = :argument AND type = :type;";
bool okay = true;
for(unsigned int i = 0; i < ArraySize(querystr); i++)
// Execute statement
if((rc = sqlite3_step(stmt)) != SQLITE_DONE)
{
// Finish if no more queries
if(querystr[i] == NULL)
break;
*message = sqlite3_errmsg(gravity_db);
log_err("gravityDB_delFromTable(%d) - SQL error step(\"%s\"): %s",
listtype, querystr, *message);
sqlite3_reset(stmt);
sqlite3_finalize(stmt);
// We need to perform a second SQL request
int rc = sqlite3_prepare_v2(gravity_db, querystr[i], -1, &stmt, NULL);
if( rc != SQLITE_OK )
{
*message = sqlite3_errmsg(gravity_db);
log_err("gravityDB_delFromTable(%d, %s) - SQL error prepare %u (%i): %s",
type, argument, i, rc, *message);
return false;
}
// Rollback transaction
querystr = "ROLLBACK TRANSACTION;";
sqlite3_exec(gravity_db, querystr, NULL, NULL, NULL);
// Bind domain to prepared statement (if requested)
const int arg_idx = sqlite3_bind_parameter_index(stmt, ":argument");
if(arg_idx > 0 && (rc = sqlite3_bind_text(stmt, arg_idx, argument, -1, SQLITE_STATIC)) != SQLITE_OK)
{
*message = sqlite3_errmsg(gravity_db);
log_err("gravityDB_delFromTable(%d, %s): Failed to bind argument %u (error %d) - %s",
type, argument, i, rc, *message);
sqlite3_reset(stmt);
sqlite3_finalize(stmt);
return false;
}
return false;
}
// Bind type to prepared statement (if requested)
// Finalize statement
sqlite3_reset(stmt);
sqlite3_finalize(stmt);
// Prepare statement for inserting items into virtual table
if(isDomain)
querystr = "INSERT INTO deltable (type, item) VALUES (:type, :item);";
else
querystr = "INSERT INTO deltable (item) VALUES (:item);";
rc = sqlite3_prepare_v2(gravity_db, querystr, -1, &stmt, NULL);
if( rc != SQLITE_OK )
{
*message = sqlite3_errmsg(gravity_db);
log_err("gravityDB_delFromTable(%d) - SQL error prepare(\"%s\"): %s",
listtype, querystr, *message);
// Rollback transaction
querystr = "ROLLBACK TRANSACTION;";
sqlite3_exec(gravity_db, querystr, NULL, NULL, NULL);
return false;
}
// Loop over all domains in the JSON array
cJSON *it = NULL;
cJSON_ArrayForEach(it, array)
{
// Bind type to prepared statement
cJSON *type = cJSON_GetObjectItemCaseSensitive(it, "type");
const int type_idx = sqlite3_bind_parameter_index(stmt, ":type");
if(type_idx > 0 && (rc = sqlite3_bind_int(stmt, type_idx, type)) != SQLITE_OK)
if(type_idx > 0 && (!cJSON_IsNumber(type) || (rc = sqlite3_bind_int(stmt, type_idx, type->valueint)) != SQLITE_OK))
{
*message = sqlite3_errmsg(gravity_db);
log_err("gravityDB_delFromTable(%d, %s): Failed to bind type (2) (error %d) - %s",
type, argument, rc, *message);
log_err("gravityDB_delFromTable(%d): Failed to bind type (error %d) - %s",
type->valueint, rc, *message);
sqlite3_reset(stmt);
sqlite3_finalize(stmt);
// Rollback transaction
querystr = "ROLLBACK TRANSACTION;";
sqlite3_exec(gravity_db, querystr, NULL, NULL, NULL);
return false;
}
// Bind item to prepared statement
cJSON *item = cJSON_GetObjectItemCaseSensitive(it, "item");
const int item_idx = sqlite3_bind_parameter_index(stmt, ":item");
if(item_idx > 0 && (!cJSON_IsString(item) || (rc = sqlite3_bind_text(stmt, item_idx, item->valuestring, -1, SQLITE_STATIC)) != SQLITE_OK))
{
*message = sqlite3_errmsg(gravity_db);
log_err("gravityDB_delFromTable(%d): Failed to bind item (error %d) - %s",
listtype, rc, *message);
sqlite3_reset(stmt);
sqlite3_finalize(stmt);
// Rollback transaction
querystr = "ROLLBACK TRANSACTION;";
sqlite3_exec(gravity_db, querystr, NULL, NULL, NULL);
return false;
}
// Execute statement
if((rc = sqlite3_step(stmt)) != SQLITE_DONE)
{
*message = sqlite3_errmsg(gravity_db);
log_err("gravityDB_delFromTable(%d) - SQL error step(\"%s\"): %s",
listtype, querystr, *message);
sqlite3_reset(stmt);
sqlite3_finalize(stmt);
// Rollback transaction
querystr = "ROLLBACK TRANSACTION;";
sqlite3_exec(gravity_db, querystr, NULL, NULL, NULL);
return false;
}
// Reset statement
sqlite3_reset(stmt);
// Debug output
if(config.debug.api.v.b)
{
log_debug(DEBUG_API, "SQL: %s", querystr[i]);
if(arg_idx > 0)
log_debug(DEBUG_API, " :argument = \"%s\"", argument);
log_debug(DEBUG_API, "SQL: %s", querystr);
if(item_idx > 0)
log_debug(DEBUG_API, " :item = \"%s\"", item->valuestring);
if(type_idx > 0)
log_debug(DEBUG_API, " :type = \"%i\"", type);
log_debug(DEBUG_API, " :type = %i", cJSON_IsNumber(type) ? type->valueint : -1);
}
}
// Perform step
okay = false;
if((rc = sqlite3_step(stmt)) == SQLITE_DONE)
{
// Item removed
okay = true;
}
else
// Finalize statement
sqlite3_finalize(stmt);
// Prepare SQL for deleting items from the requested table
const char *querystrs[4] = {NULL, NULL, NULL, NULL};
if(listtype == GRAVITY_GROUPS)
querystrs[0] = "DELETE FROM \"group\" WHERE name IN (SELECT item FROM deltable);";
else if(listtype == GRAVITY_ADLISTS)
{
// This is actually a three-step deletion to satisfy foreign-key constraints
querystrs[0] = "DELETE FROM gravity WHERE adlist_id = (SELECT id FROM adlist WHERE address IN (SELECT item FROM deltable));";
querystrs[1] = "DELETE FROM antigravity WHERE adlist_id = (SELECT id FROM adlist WHERE address IN (SELECT item FROM deltable));";
querystrs[2] = "DELETE FROM adlist WHERE address IN (SELECT item FROM deltable);";
}
else if(listtype == GRAVITY_CLIENTS)
querystrs[0] = "DELETE FROM client WHERE ip IN (SELECT item FROM deltable);";
else // domainlist
{
querystrs[0] = "DELETE FROM domainlist WHERE domain IN (SELECT item FROM deltable WHERE type = 0) AND type = 0;";
querystrs[1] = "DELETE FROM domainlist WHERE domain IN (SELECT item FROM deltable WHERE type = 1) AND type = 1;";
querystrs[2] = "DELETE FROM domainlist WHERE domain IN (SELECT item FROM deltable WHERE type = 2) AND type = 2;";
querystrs[3] = "DELETE FROM domainlist WHERE domain IN (SELECT item FROM deltable WHERE type = 3) AND type = 3;";
}
bool okay = true;
for(unsigned int i = 0; i < ArraySize(querystrs); i++)
{
// Finish if no more queries
if(querystrs[i] == NULL)
break;
// Execute statement
rc = sqlite3_exec(gravity_db, querystrs[i], NULL, NULL, NULL);
if(rc != SQLITE_OK)
{
*message = sqlite3_errmsg(gravity_db);
log_err("gravityDB_delFromTable(%d): SQL error exec(\"%s\"): %s",
listtype, querystrs[i], *message);
okay = false;
// Rollback transaction
querystr = "ROLLBACK TRANSACTION;";
sqlite3_exec(gravity_db, querystr, NULL, NULL, NULL);
break;
}
// Finalize statement
sqlite3_reset(stmt);
sqlite3_finalize(stmt);
// Add number of deleted rows
*deleted += sqlite3_changes(gravity_db);
}
// Drop temporary table
querystr = "DROP TABLE deltable;";
rc = sqlite3_exec(gravity_db, querystr, NULL, NULL, NULL);
if(rc != SQLITE_OK)
{
*message = sqlite3_errmsg(gravity_db);
log_err("gravityDB_delFromTable(%d): SQL error exec(\"%s\"): %s",
listtype, querystr, *message);
okay = false;
// Rollback transaction
querystr = "ROLLBACK TRANSACTION;";
sqlite3_exec(gravity_db, querystr, NULL, NULL, NULL);
}
// Commit transaction
querystr = "COMMIT TRANSACTION;";
rc = sqlite3_exec(gravity_db, querystr, NULL, NULL, NULL);
if(rc != SQLITE_OK)
{
*message = sqlite3_errmsg(gravity_db);
log_err("gravityDB_delFromTable(%d): SQL error exec(\"%s\"): %s",
listtype, querystr, *message);
okay = false;
// Rollback transaction
querystr = "ROLLBACK TRANSACTION;";
sqlite3_exec(gravity_db, querystr, NULL, NULL, NULL);
}
return okay;
src/database/gravity-db.h
+1 -1
View File
@@ -69,7 +69,7 @@ bool gravityDB_readTableGetRow(const enum gravity_list_type listtype, tablerow *
void gravityDB_readTableFinalize(void);
bool gravityDB_addToTable(const enum gravity_list_type listtype, tablerow *row,
const char **message, const enum http_method method);
bool gravityDB_delFromTable(const enum gravity_list_type listtype, const char* domain_name, const char **message);
bool gravityDB_delFromTable(const enum gravity_list_type listtype, const cJSON* array, unsigned int *deleted, const char **message);
bool gravityDB_edit_groups(const enum gravity_list_type listtype, cJSON *groups,
const tablerow *row, const char **message);
src/database/message-table.c
+9 -16
View File
@@ -27,6 +27,8 @@
#include "gc.h"
// get_filesystem_details()
#include "files.h"
// get_memdb()
#include "database/query-table.h"
static const char *get_message_type_str(const enum message_type type)
{
@@ -214,23 +216,10 @@ bool create_message_table(sqlite3 *db)
// Flush message table
bool flush_message_table(void)
{
// Return early if database is known to be broken
if(FTLDBerror())
return false;
sqlite3 *db;
// Open database connection
if((db = dbopen(false, false)) == NULL)
{
log_err("flush_message_table() - Failed to open DB");
return false;
}
sqlite3 *memdb = get_memdb();
// Flush message table
SQL_bool(db, "DELETE FROM message;");
// Close database connection
dbclose(&db);
SQL_bool(memdb, "DELETE FROM disk.message;");
return true;
}
@@ -389,7 +378,7 @@ end_of_add_message: // Close database connection
return rowid;
}
bool delete_message(cJSON *ids)
bool delete_message(cJSON *ids, int *deleted)
{
// Return early if database is known to be broken
if(FTLDBerror())
@@ -424,6 +413,10 @@ bool delete_message(cJSON *ids)
log_err("SQL error (%i): %s", sqlite3_errcode(db), sqlite3_errmsg(db));
return false;
}
// Add to deleted count
*deleted += sqlite3_changes(db);
sqlite3_reset(res);
sqlite3_clear_bindings(res);
}
src/database/message-table.h
+1 -1
View File
@@ -16,7 +16,7 @@
int count_messages(const bool filter_dnsmasq_warnings);
bool format_messages(cJSON *array);
bool create_message_table(sqlite3 *db);
bool delete_message(cJSON *ids);
bool delete_message(cJSON *ids, int *deleted);
bool flush_message_table(void);
void logg_regex_warning(const char *type, const char *warning, const int dbindex, const char *regex);
void logg_subnet_warning(const char *ip, const int matching_count, const char *matching_ids,
src/database/network-table.c
+7 -1
View File
@@ -2425,7 +2425,7 @@ void networkTable_readIPsFinalize(sqlite3_stmt *read_stmt)
sqlite3_finalize(read_stmt);
}
bool networkTable_deleteDevice(sqlite3 *db, const int id, const char **message)
bool networkTable_deleteDevice(sqlite3 *db, const int id, int *deleted, const char **message)
{
// First step: Delete all associated IPs of this device
// Prepare SQLite statement
@@ -2462,6 +2462,9 @@ bool networkTable_deleteDevice(sqlite3 *db, const int id, const char **message)
return false;
}
// Check if we deleted any rows
*deleted += sqlite3_changes(db);
// Finalize statement
sqlite3_finalize(stmt);
@@ -2498,6 +2501,9 @@ bool networkTable_deleteDevice(sqlite3 *db, const int id, const char **message)
return false;
}
// Check if we deleted any rows
*deleted += sqlite3_changes(db);
// Finalize statement
sqlite3_finalize(stmt);
src/database/network-table.h
+1 -1
View File
@@ -52,6 +52,6 @@ bool networkTable_readIPs(sqlite3 *db, sqlite3_stmt **read_stmt, const int id, c
bool networkTable_readIPsGetRecord(sqlite3_stmt *read_stmt, network_addresses_record *network_addresses, const char **message);
void networkTable_readIPsFinalize(sqlite3_stmt *read_stmt);
bool networkTable_deleteDevice(sqlite3 *db, const int id, const char **message);
bool networkTable_deleteDevice(sqlite3 *db, const int id, int *deleted, const char **message);
#endif //NETWORKTABLE_H
src/database/query-table.c
+63 -81
View File
@@ -22,7 +22,7 @@
#include "database/common.h"
#include "timers.h"
static sqlite3 *memdb = NULL;
static sqlite3 *_memdb = NULL;
static double new_last_timestamp = 0;
static unsigned int new_total = 0, new_blocked = 0;
static unsigned long last_mem_db_idx = 0, last_disk_db_idx = 0;
@@ -60,10 +60,10 @@ void db_counts(unsigned long *last_idx, unsigned long *mem_num, unsigned long *d
bool init_memory_database(void)
{
int rc;
const char *uri = "file:memdb?mode=memory&cache=shared";
// Try to open in-memory database
rc = sqlite3_open_v2(uri, &memdb, SQLITE_OPEN_READWRITE, NULL);
// The :memory: database always has synchronous=OFF since the content of
// it is ephemeral and is not expected to survive a power outage.
rc = sqlite3_open_v2(":memory:", &_memdb, SQLITE_OPEN_READWRITE, NULL);
if( rc != SQLITE_OK )
{
log_err("init_memory_database(): Step error while trying to open database: %s",
@@ -72,12 +72,12 @@ bool init_memory_database(void)
}
// Explicitly set busy handler to value defined in FTL.h
rc = sqlite3_busy_timeout(memdb, DATABASE_BUSY_TIMEOUT);
rc = sqlite3_busy_timeout(_memdb, DATABASE_BUSY_TIMEOUT);
if( rc != SQLITE_OK )
{
log_err("init_memory_database(): Step error while trying to set busy timeout (%d ms): %s",
DATABASE_BUSY_TIMEOUT, sqlite3_errstr(rc));
sqlite3_close(memdb);
sqlite3_close(_memdb);
return false;
}
@@ -85,11 +85,11 @@ bool init_memory_database(void)
for(unsigned int i = 0; i < ArraySize(table_creation); i++)
{
log_debug(DEBUG_DATABASE, "init_memory_database(): Executing %s", table_creation[i]);
rc = sqlite3_exec(memdb, table_creation[i], NULL, NULL, NULL);
rc = sqlite3_exec(_memdb, table_creation[i], NULL, NULL, NULL);
if( rc != SQLITE_OK ){
log_err("init_memory_database(\"%s\") failed: %s",
table_creation[i], sqlite3_errstr(rc));
sqlite3_close(memdb);
sqlite3_close(_memdb);
return false;
}
}
@@ -99,15 +99,34 @@ bool init_memory_database(void)
for(unsigned int i = 0; i < ArraySize(index_creation); i++)
{
log_debug(DEBUG_DATABASE, "init_memory_database(): Executing %s", index_creation[i]);
rc = sqlite3_exec(memdb, index_creation[i], NULL, NULL, NULL);
rc = sqlite3_exec(_memdb, index_creation[i], NULL, NULL, NULL);
if( rc != SQLITE_OK ){
log_err("init_memory_database(\"%s\") failed: %s",
index_creation[i], sqlite3_errstr(rc));
sqlite3_close(memdb);
sqlite3_close(_memdb);
return false;
}
}
// Attach disk database
if(!attach_database(_memdb, NULL, config.files.database.v.s, "disk"))
return false;
// Change journal mode to WAL
// - WAL is significantly faster in most scenarios.
// - WAL provides more concurrency as readers do not block writers and a
// writer does not block readers. Reading and writing can proceed
// concurrently.
// - Disk I/O operations tends to be more sequential using WAL.
rc = sqlite3_exec(_memdb, "PRAGMA disk.journal_mode=WAL", NULL, NULL, NULL);
if( rc != SQLITE_OK )
{
log_err("init_memory_database(): Step error while trying to set journal mode: %s",
sqlite3_errstr(rc));
sqlite3_close(_memdb);
return false;
}
// Everything went well
return true;
}
@@ -116,11 +135,15 @@ bool init_memory_database(void)
void close_memory_database(void)
{
// Return early if there is no memory database to be closed
if(memdb == NULL)
if(_memdb == NULL)
return;
// Detach disk database
if(!detach_database(_memdb, NULL, "disk"))
log_err("close_memory_database(): Failed to detach disk database");
// Close SQLite3 memory database
int ret = sqlite3_close(memdb);
int ret = sqlite3_close(_memdb);
if(ret != SQLITE_OK)
log_err("Finalizing memory database failed: %s",
sqlite3_errstr(ret));
@@ -128,12 +151,13 @@ void close_memory_database(void)
log_debug(DEBUG_DATABASE, "Closed memory database");
// Set global pointer to NULL
memdb = NULL;
_memdb = NULL;
}
sqlite3 *__attribute__((pure)) get_memdb(void)
{
return memdb;
log_debug(DEBUG_DATABASE, "Accessing in-memory database");
return _memdb;
}
// Get memory usage and size of in-memory tables
@@ -188,7 +212,7 @@ static bool get_memdb_size(sqlite3 *db, size_t *memsize, int *queries)
*memsize = page_count * page_size;
// Get number of queries in the memory table
if((*queries = get_number_of_queries_in_DB(db, "query_storage", false)) == DB_FAILED)
if((*queries = get_number_of_queries_in_DB(db, "query_storage")) == DB_FAILED)
return false;
return true;
@@ -202,6 +226,7 @@ static void log_in_memory_usage(void)
size_t memsize = 0;
int queries = 0;
sqlite3 *memdb = get_memdb();
if(get_memdb_size(memdb, &memsize, &queries))
{
char prefix[2] = { 0 };
@@ -212,11 +237,6 @@ static void log_in_memory_usage(void)
}
}
// Attach disk database to in-memory database
bool attach_disk_database(const char **message)
{
return attach_database(memdb, message, config.files.database.v.s, "disk");
}
// Attach database using specified path and alias
bool attach_database(sqlite3* db, const char **message, const char *path, const char *alias)
@@ -277,12 +297,6 @@ bool attach_database(sqlite3* db, const char **message, const char *path, const
return okay;
}
// Detach disk database to in-memory database
bool detach_disk_database(const char **message)
{
return detach_database(memdb, message, "disk");
}
// Detach a previously attached database by its alias
bool detach_database(sqlite3* db, const char **message, const char *alias)
{
@@ -333,13 +347,10 @@ bool detach_database(sqlite3* db, const char **message, const char *alias)
// Get number of queries either in the temp or in the on-diks database
// This routine is used by the API routines.
int get_number_of_queries_in_DB(sqlite3 *db, const char *tablename, const bool do_attach)
int get_number_of_queries_in_DB(sqlite3 *db, const char *tablename)
{
int rc = 0, num = 0;
sqlite3_stmt *stmt = NULL;
// Attach disk database if required
if(do_attach && !attach_disk_database(NULL))
return DB_FAILED;
// Count number of rows
const size_t buflen = 42 + strlen(tablename);
@@ -348,7 +359,7 @@ int get_number_of_queries_in_DB(sqlite3 *db, const char *tablename, const bool d
// The database pointer may be NULL, meaning we want the memdb
if(db == NULL)
db = memdb;
db = get_memdb();
// PRAGMA page_size
rc = sqlite3_prepare_v2(db, querystr, -1, &stmt, NULL);
@@ -358,8 +369,6 @@ int get_number_of_queries_in_DB(sqlite3 *db, const char *tablename, const bool d
log_err("get_number_of_queries_in_DB(%s): Prepare error: %s",
tablename, sqlite3_errstr(rc));
free(querystr);
if(do_attach)
detach_disk_database(NULL);
return false;
}
rc = sqlite3_step(stmt);
@@ -371,17 +380,11 @@ int get_number_of_queries_in_DB(sqlite3 *db, const char *tablename, const bool d
tablename, sqlite3_errstr(rc));
free(querystr);
sqlite3_finalize(stmt);
if(do_attach)
detach_disk_database(NULL);
return false;
}
sqlite3_finalize(stmt);
free(querystr);
// Detach only if attached herein
if(do_attach && !detach_disk_database(NULL))
return DB_FAILED;
return num;
}
@@ -395,24 +398,20 @@ bool import_queries_from_disk(void)
const double mintime = now - config.webserver.api.maxHistory.v.ui;
const char *querystr = "INSERT INTO query_storage SELECT * FROM disk.query_storage WHERE timestamp >= ?";
// Attach disk database
if(!attach_disk_database(NULL))
return false;
// Begin transaction
int rc;
sqlite3 *memdb = get_memdb();
if((rc = sqlite3_exec(memdb, "BEGIN TRANSACTION", NULL, NULL, NULL)) != SQLITE_OK)
{
log_err("import_queries_from_disk(): Cannot start transaction: %s", sqlite3_errstr(rc));
detach_disk_database(NULL);
return false;
}
// Prepare SQLite3 statement
sqlite3_stmt *stmt = NULL;
log_debug(DEBUG_DATABASE, "Accessing in-memory database");
if((rc = sqlite3_prepare_v2(memdb, querystr, -1, &stmt, NULL)) != SQLITE_OK){
log_err("import_queries_from_disk(): SQL error prepare: %s", sqlite3_errstr(rc));
detach_disk_database(NULL);
return false;
}
@@ -421,7 +420,6 @@ bool import_queries_from_disk(void)
{
log_err("import_queries_from_disk(): Failed to bind type mintime: %s", sqlite3_errstr(rc));
sqlite3_finalize(stmt);
detach_disk_database(NULL);
return false;
}
@@ -464,16 +462,12 @@ bool import_queries_from_disk(void)
if((rc = sqlite3_exec(memdb, "END TRANSACTION", NULL, NULL, NULL)) != SQLITE_OK)
{
log_err("import_queries_from_disk(): Cannot end transaction: %s", sqlite3_errstr(rc));
detach_disk_database(NULL);
return false;
}
// Get number of queries on disk before detaching
disk_db_num = get_number_of_queries_in_DB(memdb, "disk.query_storage", false);
mem_db_num = get_number_of_queries_in_DB(memdb, "query_storage", false);
if(!detach_disk_database(NULL))
return false;
disk_db_num = get_number_of_queries_in_DB(memdb, "disk.query_storage");
mem_db_num = get_number_of_queries_in_DB(memdb, "query_storage");
log_info("Imported %u queries from the on-disk database (it has %u rows)", mem_db_num, disk_db_num);
@@ -494,19 +488,16 @@ bool export_queries_to_disk(bool final)
// Start database timer
timer_start(DATABASE_WRITE_TIMER);
// Attach disk database
if(!attach_disk_database(NULL))
return false;
// Start transaction
sqlite3 *memdb = get_memdb();
SQL_bool(memdb, "BEGIN TRANSACTION");
// Prepare SQLite3 statement
sqlite3_stmt *stmt = NULL;
log_debug(DEBUG_DATABASE, "Accessing in-memory database");
int rc = sqlite3_prepare_v2(memdb, querystr, -1, &stmt, NULL);
if( rc != SQLITE_OK ){
log_err("export_queries_to_disk(): SQL error prepare: %s", sqlite3_errstr(rc));
detach_disk_database(NULL);
return false;
}
@@ -514,7 +505,6 @@ bool export_queries_to_disk(bool final)
if((rc = sqlite3_bind_int64(stmt, 1, last_disk_db_idx)) != SQLITE_OK)
{
log_err("export_queries_to_disk(): Failed to bind id: %s", sqlite3_errstr(rc));
detach_disk_database(NULL);
return false;
}
@@ -525,7 +515,6 @@ bool export_queries_to_disk(bool final)
if((rc = sqlite3_bind_double(stmt, 2, time)) != SQLITE_OK)
{
log_err("export_queries_to_disk(): Failed to bind time: %s", sqlite3_errstr(rc));
detach_disk_database(NULL);
return false;
}
@@ -548,6 +537,7 @@ bool export_queries_to_disk(bool final)
// Update last_disk_db_idx
// Prepare SQLite3 statement
log_debug(DEBUG_DATABASE, "Accessing in-memory database");
rc = sqlite3_prepare_v2(memdb, "SELECT MAX(id) FROM disk.query_storage;", -1, &stmt, NULL);
// Perform step
@@ -589,16 +579,11 @@ bool export_queries_to_disk(bool final)
if((rc = sqlite3_exec(memdb, "END TRANSACTION", NULL, NULL, NULL)) != SQLITE_OK)
{
log_err("export_queries_to_disk(): Cannot end transaction: %s", sqlite3_errstr(rc));
detach_disk_database(NULL);
return false;
}
// Update number of queries in the disk database
disk_db_num = get_number_of_queries_in_DB(memdb, "disk.query_storage", false);
// Detach disk database
if(!detach_disk_database(NULL))
return false;
disk_db_num = get_number_of_queries_in_DB(memdb, "disk.query_storage");
// All temp queries were stored to disk, update the IDs
last_disk_db_idx += insertions;
@@ -629,7 +614,7 @@ bool delete_old_queries_from_db(const bool use_memdb, const double mintime)
sqlite3 *db = NULL;
if(use_memdb)
db = memdb;
db = get_memdb();
else
db = dbopen(false, false);
@@ -657,7 +642,8 @@ bool delete_old_queries_from_db(const bool use_memdb, const double mintime)
mintime, sqlite3_errstr(rc));
// Update number of queries in in-memory database
const int new_num = get_number_of_queries_in_DB(memdb, "query_storage", false);
sqlite3 *memdb = get_memdb();
const int new_num = get_number_of_queries_in_DB(memdb, "query_storage");
log_debug(DEBUG_GC, "delete_old_queries_from_db(): Deleted %i (%u) queries, new number of queries in memory: %i",
sqlite3_changes(db), (mem_db_num - new_num), new_num);
mem_db_num = new_num;
@@ -903,6 +889,7 @@ void DB_read_queries(void)
// Prepare SQLite3 statement
sqlite3_stmt *stmt = NULL;
sqlite3 *memdb = get_memdb();
int rc = sqlite3_prepare_v2(memdb, querystr, -1, &stmt, NULL);
if( rc != SQLITE_OK )
{
@@ -1065,19 +1052,20 @@ void DB_read_queries(void)
query->qtype = type - 100;
}
counters->querytype[query->type]++;
log_debug(DEBUG_GC, "query type %d set (database), ID = %d, new count = %d", query->type, counters->queries, counters->querytype[query->type]);
log_debug(DEBUG_STATUS, "query type %d set (database), ID = %d, new count = %d", query->type, counters->queries, counters->querytype[query->type]);
// Status is set below
query->domainID = domainID;
query->clientID = clientID;
query->upstreamID = upstreamID;
query->cacheID = findCacheID(domainID, clientID, query->type, true);
query->id = counters->queries;
query->response = 0;
query->flags.response_calculated = reply_time_avail;
query->dnssec = dnssec;
query->reply = reply;
counters->reply[query->reply]++;
log_debug(DEBUG_GC, "reply type %d set (database), ID = %d, new count = %d", query->reply, counters->queries, counters->reply[query->reply]);
log_debug(DEBUG_STATUS, "reply type %d set (database), ID = %d, new count = %d", query->reply, counters->queries, counters->reply[query->reply]);
query->response = reply_time;
query->CNAME_domainID = -1;
// Initialize flags
@@ -1125,8 +1113,7 @@ void DB_read_queries(void)
// Set ID of the domainlist entry that was the reason for permitting/blocking this query
// We assume the value in this field is said ID when it is not a CNAME-related domain
// (checked above) and the value of additional_info is not NULL (0 bytes storage size)
const int cacheID = findCacheID(query->domainID, query->clientID, query->type, true);
DNSCacheData *cache = getDNSCache(cacheID, true);
DNSCacheData *cache = getDNSCache(query->cacheID, true);
// Only load if
// a) we have a cache entry
// b) the value of additional_info is not NULL (0 bytes storage size)
@@ -1216,12 +1203,9 @@ void update_disk_db_idx(void)
// starting counting from zero (would result in a UNIQUE constraint violation)
const char *querystr = "SELECT MAX(id) FROM disk.query_storage";
// Attach disk database
if(!attach_disk_database(NULL))
return;
// Prepare SQLite3 statement
sqlite3_stmt *stmt = NULL;
sqlite3 *memdb = get_memdb();
int rc = sqlite3_prepare_v2(memdb, querystr, -1, &stmt, NULL);
// Perform step
@@ -1236,9 +1220,6 @@ void update_disk_db_idx(void)
log_debug(DEBUG_DATABASE, "Last long-term idx is %lu", last_disk_db_idx);
if(!detach_disk_database(NULL))
return;
// Update indices so that the next call to DB_save_queries() skips the
// queries that we just imported from the database
last_mem_db_idx = last_disk_db_idx;
@@ -1274,6 +1255,7 @@ bool queries_to_database(void)
}
// Start preparing query
sqlite3 *memdb = get_memdb();
rc = sqlite3_prepare_v3(memdb, "REPLACE INTO query_storage VALUES "\
"(?1," \
"?2," \
@@ -1459,8 +1441,8 @@ bool queries_to_database(void)
}
// Get cache entry for this query
const int cacheID = findCacheID(query->domainID, query->clientID, query->type, false);
DNSCacheData *cache = cacheID < 0 ? NULL : getDNSCache(cacheID, true);
const int cacheID = query->cacheID >= 0 ? query->cacheID : findCacheID(query->domainID, query->clientID, query->type, false);
DNSCacheData *cache = getDNSCache(cacheID, true);
// ADDITIONAL_INFO
if(query->status == QUERY_GRAVITY_CNAME ||
@@ -1576,7 +1558,7 @@ bool queries_to_database(void)
}
// Update number of queries in in-memory database
mem_db_num = get_number_of_queries_in_DB(memdb, "query_storage", false);
mem_db_num = get_number_of_queries_in_DB(memdb, "query_storage");
if(config.debug.database.v.b && updated + added > 0)
{
src/database/query-table.h
+2 -4
View File
@@ -11,7 +11,7 @@
#define QUERY_TABLE_PRIVATE_H
// struct queriesData
#include "../datastructure.h"
#include "datastructure.h"
#define CREATE_FTL_TABLE "CREATE TABLE ftl ( id INTEGER PRIMARY KEY NOT NULL, value BLOB NOT NULL );"
@@ -111,11 +111,9 @@ bool init_memory_database(void);
sqlite3 *get_memdb(void) __attribute__((pure));
void close_memory_database(void);
bool import_queries_from_disk(void);
bool attach_disk_database(const char **msg);
bool attach_database(sqlite3* db, const char **message, const char *path, const char *alias);
bool detach_disk_database(const char **msg);
bool detach_database(sqlite3* db, const char **message, const char *alias);
int get_number_of_queries_in_DB(sqlite3 *db, const char *tablename, const bool do_attach);
int get_number_of_queries_in_DB(sqlite3 *db, const char *tablename);
bool export_queries_to_disk(bool final);
bool delete_old_queries_from_db(const bool use_memdb, const double mintime);
bool add_additional_info_column(sqlite3 *db);
src/database/session-table.c
+10 -15
View File
@@ -12,6 +12,8 @@
#include "database/session-table.h"
#include "database/common.h"
#include "config/config.h"
// get_memdb()
#include "database/query-table.h"
bool create_session_table(sqlite3 *db)
{
@@ -216,22 +218,17 @@ bool restore_db_sessions(struct session *sessions, const uint16_t max_sessions)
return true;
}
sqlite3 *db = dbopen(false, false);
if(db == NULL)
{
log_warn("Failed to open database in restore_db_sessions()");
return false;
}
sqlite3 *memdb = get_memdb();
// Remove expired sessions from database
SQL_bool(db, "DELETE FROM session WHERE valid_until < strftime('%%s', 'now');");
SQL_bool(memdb, "DELETE FROM disk.session WHERE valid_until < strftime('%%s', 'now');");
// Get all sessions from database
sqlite3_stmt *stmt = NULL;
if(sqlite3_prepare_v2(db, "SELECT login_at, valid_until, remote_addr, user_agent, sid, csrf, tls_login, tls_mixed, app FROM session;", -1, &stmt, 0) != SQLITE_OK)
if(sqlite3_prepare_v2(memdb, "SELECT login_at, valid_until, remote_addr, user_agent, sid, csrf, tls_login, tls_mixed, app FROM disk.session;", -1, &stmt, 0) != SQLITE_OK)
{
log_err("SQL error in restore_db_sessions(): %s (%d)",
sqlite3_errmsg(db), sqlite3_errcode(db));
sqlite3_errmsg(memdb), sqlite3_errcode(memdb));
return false;
}
@@ -303,7 +300,7 @@ bool restore_db_sessions(struct session *sessions, const uint16_t max_sessions)
if(sqlite3_finalize(stmt) != SQLITE_OK)
{
log_err("SQL error in restore_db_sessions(): %s (%d)",
sqlite3_errmsg(db), sqlite3_errcode(db));
sqlite3_errmsg(memdb), sqlite3_errcode(memdb));
return false;
}
@@ -311,11 +308,9 @@ bool restore_db_sessions(struct session *sessions, const uint16_t max_sessions)
// We use secure_delete to make sure the sessions are really gone
// In this mode, SQLite overwrites the deleted content with zeros
// (https://www.sqlite.org/pragma.html#pragma_secure_delete)
SQL_bool(db, "PRAGMA secure_delete = ON;");
SQL_bool(db, "DELETE FROM session;");
// Close database connection
dbclose(&db);
SQL_bool(memdb, "PRAGMA secure_delete = ON;");
SQL_bool(memdb, "DELETE FROM disk.session;");
SQL_bool(memdb, "PRAGMA secure_delete = OFF;");
return true;
}
src/datastructure.c
+25 -17
View File
@@ -137,10 +137,10 @@ int _findUpstreamID(const char *upstreamString, const in_port_t port, int line,
return upstreamID;
}
static int get_next_domainID(void)
static int get_next_free_domainID(void)
{
// Compare content of domain against known domain IP addresses
for(int domainID=0; domainID < counters->domains; domainID++)
for(int domainID = 0; domainID < counters->domains; domainID++)
{
// Get domain pointer
domainsData* domain = getDomain(domainID, false);
@@ -188,7 +188,7 @@ int _findDomainID(const char *domainString, const bool count, int line, const ch
// If we did not return until here, then this domain is not known
// Store ID
const int domainID = get_next_domainID();
const int domainID = get_next_free_domainID();
// Get domain pointer
domainsData* domain = _getDomain(domainID, false, line, func, file);
@@ -218,10 +218,10 @@ int _findDomainID(const char *domainString, const bool count, int line, const ch
return domainID;
}
static int get_next_clientID(void)
static int get_next_free_clientID(void)
{
// Compare content of client against known client IP addresses
for(int clientID=0; clientID < counters->clients; clientID++)
for(int clientID = 0; clientID < counters->clients; clientID++)
{
// Get client pointer
clientsData* client = getClient(clientID, false);
@@ -271,7 +271,7 @@ int _findClientID(const char *clientIP, const bool count, const bool aliasclient
// If we did not return until here, then this client is definitely new
// Store ID
const int clientID = get_next_clientID();
const int clientID = get_next_free_clientID();
// Get client pointer
clientsData* client = _getClient(clientID, false, line, func, file);
@@ -369,10 +369,10 @@ void change_clientcount(clientsData *client, int total, int blocked, int overTim
}
}
static int get_next_cacheID(void)
static int get_next_free_cacheID(void)
{
// Compare content of cache against known cache IP addresses
for(int cacheID=0; cacheID < counters->dns_cache_size; cacheID++)
for(int cacheID = 0; cacheID < counters->dns_cache_size; cacheID++)
{
// Get cache pointer
DNSCacheData* cache = getDNSCache(cacheID, false);
@@ -415,7 +415,7 @@ int _findCacheID(const int domainID, const int clientID, const enum query_type q
return -1;
// Get ID of new cache entry
const int cacheID = get_next_cacheID();
const int cacheID = get_next_free_cacheID();
// Get client pointer
DNSCacheData* dns_cache = _getDNSCache(cacheID, false, line, func, file);
@@ -426,6 +426,9 @@ int _findCacheID(const int domainID, const int clientID, const enum query_type q
return -1;
}
log_debug(DEBUG_GC, "New cache entry: domainID %d, clientID %d, query_type %d (ID %d)",
domainID, clientID, query_type, cacheID);
// Initialize cache entry
dns_cache->magic = MAGICBYTE;
dns_cache->blocking_status = UNKNOWN_BLOCKED;
@@ -555,12 +558,17 @@ void FTL_reset_per_client_domain_data(void)
for(int cacheID = 0; cacheID < counters->dns_cache_size; cacheID++)
{
// Reset all blocking yes/no fields for all domains and clients
// This forces a reprocessing of all available filters for any
// given domain and client the next time they are seen
DNSCacheData *dns_cache = getDNSCache(cacheID, true);
if(dns_cache != NULL)
dns_cache->blocking_status = UNKNOWN_BLOCKED;
// Get cache pointer
DNSCacheData* dns_cache = getDNSCache(cacheID, true);
// Check if the returned pointer is valid before trying to access it
if(dns_cache == NULL)
continue;
// Reset blocking status
dns_cache->blocking_status = UNKNOWN_BLOCKED;
// Reset domainlist ID
dns_cache->domainlist_id = -1;
}
}
@@ -1039,10 +1047,10 @@ void _query_set_status(queriesData *query, const enum query_status new_status, c
if(!init)
{
counters->status[old_status]--;
log_debug(DEBUG_GC, "status %d removed (!init), ID = %d, new count = %d", QUERY_UNKNOWN, query->id, counters->status[QUERY_UNKNOWN]);
log_debug(DEBUG_STATUS, "status %d removed (!init), ID = %d, new count = %d", QUERY_UNKNOWN, query->id, counters->status[QUERY_UNKNOWN]);
}
counters->status[new_status]++;
log_debug(DEBUG_GC, "status %d set, ID = %d, new count = %d", new_status, query->id, counters->status[new_status]);
log_debug(DEBUG_STATUS, "status %d set, ID = %d, new count = %d", new_status, query->id, counters->status[new_status]);
// ... update overTime counters, ...
const int timeidx = getOverTimeID(query->timestamp);
src/datastructure.h
+1
View File
@@ -30,6 +30,7 @@ typedef struct {
int domainID;
int clientID;
int upstreamID;
int cacheID;
int id; // the ID is a (signed) int in dnsmasq, so no need for a long int here
int CNAME_domainID; // only valid if query has a CNAME blocking status
int ede;
src/dnsmasq/dnsmasq.c
+2 -2
View File
@@ -94,7 +94,7 @@ int main_dnsmasq (int argc, char **argv)
sigaction(SIGUSR1, &sigact, NULL);
sigaction(SIGUSR2, &sigact, NULL);
sigaction(SIGHUP, &sigact, NULL);
sigaction(SIGTERM, &sigact, NULL);
sigaction(SIGUSR6, &sigact, NULL); // Pi-hole modification
sigaction(SIGALRM, &sigact, NULL);
sigaction(SIGCHLD, &sigact, NULL);
sigaction(SIGINT, &sigact, NULL);
@@ -1330,7 +1330,7 @@ static void sig_handler(int sig)
event = EVENT_CHILD;
else if (sig == SIGALRM)
event = EVENT_ALARM;
else if (sig == SIGTERM)
else if (sig == SIGUSR6) // Pi-hole modified
event = EVENT_TERM;
else if (sig == SIGUSR1)
event = EVENT_DUMP;
src/dnsmasq_interface.c
+41 -8
View File
@@ -747,7 +747,7 @@ bool _FTL_new_query(const unsigned int flags, const char *name,
query->timestamp = querytimestamp;
query->type = querytype;
counters->querytype[querytype]++;
log_debug(DEBUG_GC, "query type %d set (new query), ID = %d, new count = %d", query->type, id, counters->querytype[query->type]);
log_debug(DEBUG_STATUS, "query type %d set (new query), ID = %d, new count = %d", query->type, id, counters->querytype[query->type]);
query->qtype = qtype;
query->id = id; // Has to be set before calling query_set_status()
@@ -764,7 +764,7 @@ bool _FTL_new_query(const unsigned int flags, const char *name,
// Initialize reply type
query->reply = REPLY_UNKNOWN;
counters->reply[REPLY_UNKNOWN]++;
log_debug(DEBUG_GC, "reply type %d set (new query), ID = %d, new count = %d", query->reply, query->id, counters->reply[query->reply]);
log_debug(DEBUG_STATUS, "reply type %d set (new query), ID = %d, new count = %d", query->reply, query->id, counters->reply[query->reply]);
// Store DNSSEC result for this domain
query->dnssec = DNSSEC_UNKNOWN;
query->CNAME_domainID = -1;
@@ -783,6 +783,10 @@ bool _FTL_new_query(const unsigned int flags, const char *name,
// Query extended DNS error
query->ede = EDE_UNSET;
// Initialize cache ID, may be reusing an existing one if this
// (domain,client,type) tuple was already seen before
query->cacheID = findCacheID(domainID, clientID, querytype, true);
// This query is new and not yet known to the database
query->db = -1;
@@ -1295,7 +1299,12 @@ static bool _FTL_check_blocking(int queryID, int domainID, int clientID, const c
}
// Get cache pointer
unsigned int cacheID = findCacheID(domainID, clientID, query->type, true);
// When this function is called with a different domain than the one
// already stored in the query, we have to re-lookup the cache ID.
// This can happen when a CNAME chain is followed and analyzed
const int cacheID = query->domainID == domainID && query->clientID == clientID ?
query->cacheID :
findCacheID(domainID, clientID, query->type, true);
DNSCacheData *dns_cache = getDNSCache(cacheID, true);
if(dns_cache == NULL)
{
@@ -1588,7 +1597,7 @@ bool _FTL_CNAME(const char *dst, const char *src, const int id, const char* file
else if(query->status == QUERY_REGEX)
{
// Get parent and child DNS cache entries
const int parent_cacheID = findCacheID(parent_domainID, clientID, query->type, false);
const int parent_cacheID = query->cacheID;
const int child_cacheID = findCacheID(child_domainID, clientID, query->type, false);
// Get cache pointers
@@ -2733,12 +2742,12 @@ static void _query_set_reply(const unsigned int flags, const enum reply_type rep
// Subtract from old reply counter
counters->reply[query->reply]--;
log_debug(DEBUG_GC, "reply type %d removed (set_reply), ID = %d, new count = %d", query->reply, query->id, counters->reply[query->reply]);
log_debug(DEBUG_STATUS, "reply type %d removed (set_reply), ID = %d, new count = %d", query->reply, query->id, counters->reply[query->reply]);
// Add to new reply counter
counters->reply[new_reply]++;
// Store reply type
query->reply = new_reply;
log_debug(DEBUG_GC, "reply type %d added (set_reply), ID = %d, new count = %d", query->reply, query->id, counters->reply[query->reply]);
log_debug(DEBUG_STATUS, "reply type %d added (set_reply), ID = %d, new count = %d", query->reply, query->id, counters->reply[query->reply]);
// Save response time
// Skipped internally if already computed
@@ -2823,6 +2832,29 @@ void FTL_fork_and_bind_sockets(struct passwd *ent_pw, bool dnsmasq_start)
else
savepid();
// Initialize query database (pihole-FTL.db)
db_init();
// Initialize in-memory databases
if(!init_memory_database())
log_crit("Cannot initialize in-memory database.");
// Flush messages stored in the long-term database
flush_message_table();
// Try to import queries from long-term database if available
if(config.database.DBimport.v.b)
{
import_queries_from_disk();
DB_read_queries();
}
// Initialize in-memory database starting index
update_disk_db_idx();
// Log some information about the imported queries (if any)
log_counter_info();
// Handle real-time signals in this process (and its children)
// Helper processes are already split from the main instance
// so they will not listen to real-time signals
@@ -3231,6 +3263,7 @@ bool FTL_unlink_DHCP_lease(const char *ipaddr)
#endif
else
{
// Invalid IP address or no lease found
return false;
}
@@ -3355,10 +3388,10 @@ void FTL_multiple_replies(const int id, int *firstID)
// Copy relevant information over
counters->reply[duplicated_query->reply]--;
log_debug(DEBUG_GC, "duplicated_query reply type %d removed, ID = %d, new count = %d", duplicated_query->reply, duplicated_query->id, counters->reply[duplicated_query->reply]);
log_debug(DEBUG_STATUS, "duplicated_query reply type %d removed, ID = %d, new count = %d", duplicated_query->reply, duplicated_query->id, counters->reply[duplicated_query->reply]);
duplicated_query->reply = source_query->reply;
counters->reply[duplicated_query->reply]++;
log_debug(DEBUG_GC, "duplicated_query reply type %d set, ID = %d, new count = %d", duplicated_query->reply, duplicated_query->id, counters->reply[duplicated_query->reply]);
log_debug(DEBUG_STATUS, "duplicated_query reply type %d set, ID = %d, new count = %d", duplicated_query->reply, duplicated_query->id, counters->reply[duplicated_query->reply]);
duplicated_query->dnssec = source_query->dnssec;
duplicated_query->flags.complete = true;
src/gc.c
+18 -19
View File
@@ -51,8 +51,8 @@ static void recycle(void)
{
bool *client_used = calloc(counters->clients, sizeof(bool));
bool *domain_used = calloc(counters->domains, sizeof(bool));
bool *upstreams_used = calloc(counters->upstreams, sizeof(bool));
if(client_used == NULL || domain_used == NULL || upstreams_used == NULL)
bool *cache_used = calloc(counters->dns_cache_size, sizeof(bool));
if(client_used == NULL || domain_used == NULL || cache_used == NULL)
{
log_err("Cannot allocate memory for recycling");
return;
@@ -70,13 +70,13 @@ static void recycle(void)
client_used[query->clientID] = true;
domain_used[query->domainID] = true;
// Mark upstream as used (if any)
if(query->upstreamID > -1)
upstreams_used[query->upstreamID] = true;
// Mark CNAME domain as used (if any)
if(query->CNAME_domainID >= 0)
if(query->CNAME_domainID > -1)
domain_used[query->CNAME_domainID] = true;
// Mark cache entry as used (if any)
if(query->cacheID > -1)
cache_used[query->cacheID] = true;
}
// Recycle clients
@@ -128,12 +128,11 @@ static void recycle(void)
unsigned int cache_recycled = 0;
for(int cacheID = 0; cacheID < counters->dns_cache_size; cacheID++)
{
DNSCacheData *cache = getDNSCache(cacheID, true);
if(cache == NULL)
if(cache_used[cacheID])
continue;
// Skip cache entries that are still in use
if(cache->magic != 0x00)
DNSCacheData *cache = getDNSCache(cacheID, true);
if(cache == NULL)
continue;
log_debug(DEBUG_GC, "Recycling cache entry with ID %d", cacheID);
@@ -147,7 +146,7 @@ static void recycle(void)
// Free memory
free(client_used);
free(domain_used);
free(upstreams_used);
free(cache_used);
// Scan number of recycled clients and domains if in debug mode
if(config.debug.gc.v.b)
@@ -181,10 +180,10 @@ static void recycle(void)
free_cache++;
}
log_debug(DEBUG_GC, "Recycler summary: %u/%d (max %d) clients, %u/%d (max %d) domains and %u/%d (max %d) cache records are free",
free_clients, counters->clients, counters->clients_MAX,
free_domains, counters->domains, counters->domains_MAX,
free_cache, counters->dns_cache_size, counters->dns_cache_MAX);
log_debug(DEBUG_GC, "%d/%d clients, %d/%d domains and %d/%d cache records are free",
counters->clients_MAX + (int)free_clients - counters->clients, counters->clients_MAX,
counters->domains_MAX + (int)free_domains - counters->domains_MAX, counters->domains_MAX,
counters->dns_cache_MAX + (int)free_cache - counters->dns_cache_MAX, counters->dns_cache_MAX);
log_debug(DEBUG_GC, "Recycled additional %u clients, %u domains, and %u cache records (scanned %d queries)",
clients_recycled, domains_recycled, cache_recycled, counters->queries);
@@ -378,17 +377,17 @@ void runGC(const time_t now, time_t *lastGCrun, const bool flush)
// Update reply counters
counters->reply[query->reply]--;
log_debug(DEBUG_GC, "reply type %d removed (GC), ID = %d, new count = %d", query->reply, query->id, counters->reply[query->reply]);
log_debug(DEBUG_STATUS, "reply type %d removed (GC), ID = %d, new count = %d", query->reply, query->id, counters->reply[query->reply]);
// Update type counters
counters->querytype[query->type]--;
log_debug(DEBUG_GC, "query type %d removed (GC), ID = %d, new count = %d", query->type, query->id, counters->querytype[query->type]);
log_debug(DEBUG_STATUS, "query type %d removed (GC), ID = %d, new count = %d", query->type, query->id, counters->querytype[query->type]);
// Subtract UNKNOWN from the counters before
// setting the status if different.
// Minus one here and plus one below = net zero
counters->status[QUERY_UNKNOWN]--;
log_debug(DEBUG_GC, "status %d removed (GC), ID = %d, new count = %d", QUERY_UNKNOWN, query->id, counters->status[QUERY_UNKNOWN]);
log_debug(DEBUG_STATUS, "status %d removed (GC), ID = %d, new count = %d", QUERY_UNKNOWN, query->id, counters->status[QUERY_UNKNOWN]);
// Set query again to UNKNOWN to reset the counters
query_set_status(query, QUERY_UNKNOWN);
src/log.c
+1
View File
@@ -482,6 +482,7 @@ void log_counter_info(void)
log_info(" -> Unknown DNS queries: %i", counters->status[QUERY_UNKNOWN]);
log_info(" -> Unique domains: %i", counters->domains);
log_info(" -> Unique clients: %i", counters->clients);
log_info(" -> DNS cache records: %i", counters->dns_cache_size);
log_info(" -> Known forward destinations: %i", counters->upstreams);
}
src/main.c
+3 -32
View File
@@ -14,7 +14,6 @@
#include "config/setupVars.h"
#include "args.h"
#include "config/config.h"
#include "database/common.h"
#include "main.h"
// exit_code
#include "signals.h"
@@ -24,12 +23,10 @@
#include "capabilities.h"
#include "timers.h"
#include "procps.h"
// init_memory_database(), import_queries_from_disk()
#include "database/query-table.h"
// init_overtime()
#include "overTime.h"
// flush_message_table()
#include "database/message-table.h"
// export_queries_to_disk()
#include "database/query-table.h"
#if defined(__GLIBC__) && defined(__GLIBC_MINOR__)
#pragma message "Minimum GLIBC version: " xstr(__GLIBC__) "." xstr(__GLIBC_MINOR__)
@@ -110,32 +107,6 @@ int main (int argc, char *argv[])
// Initialize overTime datastructure
initOverTime();
// Initialize query database (pihole-FTL.db)
db_init();
// Initialize in-memory databases
if(!init_memory_database())
{
log_crit("FATAL: Cannot initialize in-memory database.");
return EXIT_FAILURE;
}
// Flush messages stored in the long-term database
flush_message_table();
// Try to import queries from long-term database if available
if(config.database.DBimport.v.b)
{
import_queries_from_disk();
DB_read_queries();
}
// Initialize in-memory database starting index
update_disk_db_idx();
// Log some information about the imported queries (if any)
log_counter_info();
// Check for availability of capabilities in debug mode
if(config.debug.caps.v.b)
check_capabilities();
@@ -186,7 +157,7 @@ int main (int argc, char *argv[])
cleanup(exit_code);
if(exit_code == RESTART_FTL_CODE)
execv(argv[0], argv);
execvp(argv[0], argv);
return exit_code;
}
src/signals.c
+96 -2
View File
@@ -311,11 +311,94 @@ static void SIGRT_handler(int signum, siginfo_t *si, void *unused)
// Parse neighbor cache
set_event(PARSE_NEIGHBOR_CACHE);
}
// else if(rtsig == 6)
// {
// // Signal internally used to signal dnsmasq it has to stop
// }
// Restore errno before returning back to previous context
errno = _errno;
}
static void SIGTERM_handler(int signum, siginfo_t *si, void *unused)
{
// Ignore SIGTERM outside of the main process (TCP forks)
if(mpid != getpid())
return;
// Get PID and UID of the process that sent the terminating signal
const pid_t kill_pid = si->si_pid;
const uid_t kill_uid = si->si_uid;
// Get name of the process that sent the terminating signal
char kill_name[256] = { 0 };
char kill_exe [256] = { 0 };
snprintf(kill_exe, sizeof(kill_exe), "/proc/%ld/cmdline", (long int)kill_pid);
FILE *fp = fopen(kill_exe, "r");
if(fp != NULL)
{
// Successfully opened file
size_t read = 0;
// Read line from file
if((read = fread(kill_name, sizeof(char), sizeof(kill_name), fp)) > 0)
{
// Successfully read line
// cmdline contains the command-line arguments as a set
// of strings separated by null bytes ('\0'), with a
// further null byte after the last string. Hence, we
// need to replace all null bytes with spaces for
// displaying it below
for(unsigned int i = 0; i < min((size_t)read, sizeof(kill_name)); i++)
{
if(kill_name[i] == '\0')
kill_name[i] = ' ';
}
// Remove any trailing spaces
for(unsigned int i = read - 1; i > 0; i--)
{
if(kill_name[i] == ' ')
kill_name[i] = '\0';
else
break;
}
}
else
{
// Failed to read line
strcpy(kill_name, "N/A");
}
}
else
{
// Failed to open file
strcpy(kill_name, "N/A");
}
// Get username of the process that sent the terminating signal
char kill_user[256] = { 0 };
struct passwd *pwd = getpwuid(kill_uid);
if(pwd != NULL)
{
// Successfully obtained username
strncpy(kill_user, pwd->pw_name, sizeof(kill_user));
}
else
{
// Failed to obtain username
strcpy(kill_user, "N/A");
}
// Log who sent the signal
log_info("Asked to terminate by \"%s\" (PID %ld, user %s UID %ld)",
kill_name, (long int)kill_pid,
kill_user, (long int)kill_uid);
// Terminate dnsmasq to stop DNS service
raise(SIGUSR6);
}
// Register ordinary signals handler
void handle_signals(void)
{
@@ -337,6 +420,13 @@ void handle_signals(void)
}
}
// Also catch SIGTERM
struct sigaction SIGaction = { 0 };
SIGaction.sa_flags = SA_SIGINFO;
sigemptyset(&SIGaction.sa_mask);
SIGaction.sa_sigaction = &SIGTERM_handler;
sigaction(SIGTERM, &SIGaction, NULL);
// Log start time of FTL
FTLstarttime = time(NULL);
}
@@ -351,8 +441,12 @@ void handle_realtime_signals(void)
// Catch all real-time signals
for(int signum = SIGRTMIN; signum <= SIGRTMAX; signum++)
{
struct sigaction SIGACTION;
memset(&SIGACTION, 0, sizeof(struct sigaction));
if(signum == SIGUSR6)
// Skip SIGUSR6 as it is used internally to signify
// dnsmasq to stop
continue;
struct sigaction SIGACTION = { 0 };
SIGACTION.sa_flags = SA_SIGINFO;
sigemptyset(&SIGACTION.sa_mask);
SIGACTION.sa_sigaction = &SIGRT_handler;
src/signals.h
+2
View File
@@ -12,6 +12,8 @@
#include "enums.h"
#define SIGUSR6 (SIGRTMIN + 6)
// defined in dnsmasq/dnsmasq.h
extern volatile char FTL_terminate;
src/tools/gravity-parseList.c
-2
View File
@@ -542,8 +542,6 @@ end_of_parseList:
// Print newline
puts("");
}
// Print final newline
puts("");
}
// Free memory
src/webserver/cJSON/cJSON.c
+31 -12
View File
@@ -96,9 +96,9 @@ CJSON_PUBLIC(const char *) cJSON_GetErrorPtr(void)
return (const char*) (global_error.json + global_error.position);
}
CJSON_PUBLIC(char *) cJSON_GetStringValue(const cJSON * const item)
CJSON_PUBLIC(char *) cJSON_GetStringValue(const cJSON * const item)
{
if (!cJSON_IsString(item))
if (!cJSON_IsString(item))
{
return NULL;
}
@@ -106,9 +106,9 @@ CJSON_PUBLIC(char *) cJSON_GetStringValue(const cJSON * const item)
return item->valuestring;
}
CJSON_PUBLIC(double) cJSON_GetNumberValue(const cJSON * const item)
CJSON_PUBLIC(double) cJSON_GetNumberValue(const cJSON * const item)
{
if (!cJSON_IsNumber(item))
if (!cJSON_IsNumber(item))
{
return (double) NAN;
}
@@ -117,7 +117,7 @@ CJSON_PUBLIC(double) cJSON_GetNumberValue(const cJSON * const item)
}
/* This is a safeguard to prevent copy-pasters from using incompatible C and header files */
#if (CJSON_VERSION_MAJOR != 1) || (CJSON_VERSION_MINOR != 7) || (CJSON_VERSION_PATCH != 15)
#if (CJSON_VERSION_MAJOR != 1) || (CJSON_VERSION_MINOR != 7) || (CJSON_VERSION_PATCH != 17)
#error cJSON.h and cJSON.c have different versions. Make sure that both have the same.
#endif
@@ -401,7 +401,12 @@ CJSON_PUBLIC(char*) cJSON_SetValuestring(cJSON *object, const char *valuestring)
{
char *copy = NULL;
/* if object's type is not cJSON_String or is cJSON_IsReference, it should not set valuestring */
if (!(object->type & cJSON_String) || (object->type & cJSON_IsReference))
if ((object == NULL) || !(object->type & cJSON_String) || (object->type & cJSON_IsReference))
{
return NULL;
}
/* return NULL if the object is corrupted */
if (object->valuestring == NULL)
{
return NULL;
}
@@ -511,7 +516,7 @@ static unsigned char* ensure(printbuffer * const p, size_t needed)
return NULL;
}
memcpy(newbuffer, p->buffer, p->offset + 1);
p->hooks.deallocate(p->buffer);
}
@@ -562,6 +567,10 @@ static cJSON_bool print_number(const cJSON * const item, printbuffer * const out
{
length = sprintf((char*)number_buffer, "null");
}
else if(d == (double)item->valueint)
{
length = sprintf((char*)number_buffer, "%d", item->valueint);
}
else
{
/* Try 15 decimal places of precision to avoid nonsignificant nonzero digits */
@@ -1103,7 +1112,7 @@ CJSON_PUBLIC(cJSON *) cJSON_ParseWithLengthOpts(const char *value, size_t buffer
}
buffer.content = (const unsigned char*)value;
buffer.length = buffer_length;
buffer.length = buffer_length;
buffer.offset = 0;
buffer.hooks = global_hooks;
@@ -2260,7 +2269,7 @@ CJSON_PUBLIC(cJSON_bool) cJSON_InsertItemInArray(cJSON *array, int which, cJSON
{
cJSON *after_inserted = NULL;
if (which < 0)
if (which < 0 || newitem == NULL)
{
return false;
}
@@ -2271,6 +2280,11 @@ CJSON_PUBLIC(cJSON_bool) cJSON_InsertItemInArray(cJSON *array, int which, cJSON
return add_item_to_array(array, newitem);
}
if (after_inserted != array->child && after_inserted->prev == NULL) {
/* return false if after_inserted is a corrupted array item */
return false;
}
newitem->next = after_inserted;
newitem->prev = after_inserted->prev;
after_inserted->prev = newitem;
@@ -2287,7 +2301,7 @@ CJSON_PUBLIC(cJSON_bool) cJSON_InsertItemInArray(cJSON *array, int which, cJSON
CJSON_PUBLIC(cJSON_bool) cJSON_ReplaceItemViaPointer(cJSON * const parent, cJSON * const item, cJSON * replacement)
{
if ((parent == NULL) || (replacement == NULL) || (item == NULL))
if ((parent == NULL) || (parent->child == NULL) || (replacement == NULL) || (item == NULL))
{
return false;
}
@@ -2357,6 +2371,11 @@ static cJSON_bool replace_item_in_object(cJSON *object, const char *string, cJSO
cJSON_free(replacement->string);
}
replacement->string = (char*)cJSON_strdup((const unsigned char*)string, &global_hooks);
if (replacement->string == NULL)
{
return false;
}
replacement->type &= ~cJSON_StringIsConst;
return cJSON_ReplaceItemViaPointer(object, get_object_item(object, string, case_sensitive), replacement);
@@ -2689,7 +2708,7 @@ CJSON_PUBLIC(cJSON *) cJSON_CreateStringArray(const char *const *strings, int co
if (a && a->child) {
a->child->prev = n;
}
return a;
}
@@ -3107,4 +3126,4 @@ CJSON_PUBLIC(void *) cJSON_malloc(size_t size)
CJSON_PUBLIC(void) cJSON_free(void *object)
{
global_hooks.deallocate(object);
}
}
src/webserver/cJSON/cJSON.h
+9 -2
View File
@@ -81,7 +81,7 @@ then using the CJSON_API_VISIBILITY flag to "export" the same symbols the way CJ
/* project version */
#define CJSON_VERSION_MAJOR 1
#define CJSON_VERSION_MINOR 7
#define CJSON_VERSION_PATCH 15
#define CJSON_VERSION_PATCH 17
#include <stddef.h>
@@ -279,6 +279,13 @@ CJSON_PUBLIC(double) cJSON_SetNumberHelper(cJSON *object, double number);
/* Change the valuestring of a cJSON_String object, only takes effect when type of object is cJSON_String */
CJSON_PUBLIC(char*) cJSON_SetValuestring(cJSON *object, const char *valuestring);
/* If the object is not a boolean type this does nothing and returns cJSON_Invalid else it returns the new type*/
#define cJSON_SetBoolValue(object, boolValue) ( \
(object != NULL && ((object)->type & (cJSON_False|cJSON_True))) ? \
(object)->type=((object)->type &(~(cJSON_False|cJSON_True)))|((boolValue)?cJSON_True:cJSON_False) : \
cJSON_Invalid\
)
/* Macro for iterating over an array or object */
#define cJSON_ArrayForEach(element, array) for(element = (array != NULL) ? (array)->child : NULL; element != NULL; element = element->next)
@@ -290,4 +297,4 @@ CJSON_PUBLIC(void) cJSON_free(void *object);
}
#endif
#endif
#endif
src/webserver/http-common.h
+8 -2
View File
@@ -33,9 +33,15 @@ enum http_method {
HTTP_OPTIONS = 1 << 5,
};
enum api_flags {
API_FLAG_NONE = 0,
API_DOMAINS = 1 << 0,
API_PARSE_JSON = 1 << 1,
API_BATCHDELETE = 1 << 2,
};
struct api_options {
bool domains :1;
bool parse_json :1;
enum api_flags flags;
enum fifo_logs which;
};
src/webserver/json_macros.h
+4 -1
View File
@@ -200,7 +200,10 @@
})
#define JSON_SEND_OBJECT_CODE(object, code)({ \
cJSON_AddNumberToObject(object, "took", double_time() - api->now);\
if((code) != 204) \
{ \
cJSON_AddNumberToObject(object, "took", double_time() - api->now); \
} \
char *json_string = json_formatter(object); \
if(json_string == NULL) \
{ \
src/zip/CMakeLists.txt
+2
View File
@@ -11,6 +11,8 @@
set(sources
gzip.c
gzip.h
tar.c
tar.h
teleporter.c
teleporter.h
)
src/zip/gzip.c
+2 -3
View File
@@ -14,7 +14,6 @@
#include <string.h>
// le32toh and friends
#include <endian.h>
#include "miniz/miniz.h"
#include "gzip.h"
#include "log.h"
@@ -103,8 +102,8 @@ static bool deflate_buffer(const unsigned char *buffer_uncompressed, const mz_ul
return true;
}
static bool inflate_buffer(unsigned char *buffer_compressed, mz_ulong size_compressed,
unsigned char **buffer_uncompressed, mz_ulong *size_uncompressed)
bool inflate_buffer(unsigned char *buffer_compressed, mz_ulong size_compressed,
unsigned char **buffer_uncompressed, mz_ulong *size_uncompressed)
{
// Check GZIP header (magic byte 1F 8B and compression algorithm deflate 08)
if(buffer_compressed[0] != 0x1F || buffer_compressed[1] != 0x8B)
src/zip/gzip.h
+4
View File
@@ -11,6 +11,10 @@
#define GZIP_H
#include <stdbool.h>
#include "miniz/miniz.h"
bool inflate_buffer(unsigned char *buffer_compressed, mz_ulong size_compressed,
unsigned char **buffer_uncompressed, mz_ulong *size_uncompressed);
bool deflate_file(const char *in, const char *out, bool verbose);
bool inflate_file(const char *infile, const char *outfile, bool verbose);
src/zip/tar.c
+128
View File
@@ -0,0 +1,128 @@
/* Pi-hole: A black hole for Internet advertisements
* (c) 2023 Pi-hole, LLC (https://pi-hole.net)
* Network-wide ad blocking via your own hardware.
*
* FTL Engine
* In-memory tar reading routines
*
* This file is copyright under the latest version of the EUPL.
* Please see LICENSE file for your rights under this license. */
#include "zip/tar.h"
#include "log.h"
// TAR offsets
#define TAR_NAME_OFFSET 0
#define TAR_SIZE_OFFSET 124
#define TAR_MAGIC_OFFSET 257
// TAR constants
#define TAR_BLOCK_SIZE 512
#define TAR_NAME_SIZE 100
#define TAR_SIZE_SIZE 12
#define TAR_MAGIC_SIZE 5
static const char MAGIC_CONST[] = "ustar"; // Modern GNU tar's magic const */
/**
* Find a file in a TAR archive
* @param tarData Pointer to the TAR archive in memory
* @param tarSize Size of the TAR archive in memory in bytes
* @param fileName Name of the file to find
* @param fileSize Pointer to a size_t variable to store the file size in
* @return Pointer to the file data or NULL if not found
*/
const char * __attribute__((nonnull (1,3,4))) find_file_in_tar(const uint8_t *tarData, const size_t tarSize,
const char *fileName, size_t *fileSize)
{
bool found = false;
size_t size, p = 0, newOffset = 0;
// Convert to char * to be able to do pointer arithmetic more easily
const char *tar = (const char *)tarData;
// Initialize fileSize to 0
*fileSize = 0;
// Loop through TAR file
do
{
// "Load" data from tar - just point to passed memory
const char *name = tar + TAR_NAME_OFFSET + p + newOffset;
const char *sz = tar + TAR_SIZE_OFFSET + p + newOffset; // size str
p += newOffset; // pointer to current file's data in TAR
// Check for supported TAR version or end of TAR
for (size_t i = 0; i < TAR_MAGIC_SIZE; i++)
if (tar[i + TAR_MAGIC_OFFSET + p] != MAGIC_CONST[i])
return NULL;
// Convert file size from string into integer
size = 0;
for (ssize_t i = TAR_SIZE_SIZE - 2, mul = 1; i >= 0; mul *= 8, i--) // Octal str to int
if ((sz[i] >= '1') && (sz[i] <= '9'))
size += (sz[i] - '0') * mul;
//Offset size in bytes. Depends on file size and TAR block size
newOffset = (1 + size / TAR_BLOCK_SIZE) * TAR_BLOCK_SIZE; //trim by block
if ((size % TAR_BLOCK_SIZE) > 0)
newOffset += TAR_BLOCK_SIZE;
found = strncmp(name, fileName, TAR_NAME_SIZE) == 0;
} while (!found && (p + newOffset + TAR_BLOCK_SIZE <= tarSize));
if (!found)
return NULL; // No file found in TAR - return NULL
// File found in TAR - return pointer to file data and set fileSize
*fileSize = size;
return tar + p + TAR_BLOCK_SIZE;
}
/**
* List all files in a TAR archive
* @param tarData Pointer to the TAR archive in memory
* @param tarSize Size of the TAR archive in memory in bytes
* @return Pointer to a cJSON array containing all file names with file size
*/
cJSON * __attribute__((nonnull (1))) list_files_in_tar(const uint8_t *tarData, const size_t tarSize)
{
cJSON *files = cJSON_CreateArray();
size_t size, p = 0, newOffset = 0;
// Convert to char * to be able to do pointer arithmetic more easily
const char *tar = (const char *)tarData;
// Loop through TAR file
do
{
// "Load" data from tar - just point to passed memory
const char *name = tar + TAR_NAME_OFFSET + p + newOffset;
const char *sz = tar + TAR_SIZE_OFFSET + p + newOffset; // size str
p += newOffset; // pointer to current file's data in TAR
// Check for supported TAR version or end of TAR
for (size_t i = 0; i < TAR_MAGIC_SIZE; i++)
if (tar[i + TAR_MAGIC_OFFSET + p] != MAGIC_CONST[i])
return files;
// Convert file size from string into integer
size = 0;
for (ssize_t i = TAR_SIZE_SIZE - 2, mul = 1; i >= 0; mul *= 8, i--) // Octal str to int
if ((sz[i] >= '1') && (sz[i] <= '9'))
size += (sz[i] - '0') * mul;
//Offset size in bytes. Depends on file size and TAR block size
newOffset = (1 + size / TAR_BLOCK_SIZE) * TAR_BLOCK_SIZE; //trim by block
if ((size % TAR_BLOCK_SIZE) > 0)
newOffset += TAR_BLOCK_SIZE;
// Add file name to cJSON array
cJSON *file = cJSON_CreateObject();
cJSON_AddItemToObject(file, "name", cJSON_CreateString(name));
cJSON_AddItemToObject(file, "size", cJSON_CreateNumber(size));
cJSON_AddItemToArray(files, file);
} while (p + newOffset + TAR_BLOCK_SIZE <= tarSize);
return files;
}
src/zip/tar.h
+19
View File
@@ -0,0 +1,19 @@
/* Pi-hole: A black hole for Internet advertisements
* (c) 2023 Pi-hole, LLC (https://pi-hole.net)
* Network-wide ad blocking via your own hardware.
*
* FTL Engine
* TAR reading routines
*
* This file is copyright under the latest version of the EUPL.
* Please see LICENSE file for your rights under this license. */
#ifndef TAR_H
#define TAR_H
#include "FTL.h"
#include "webserver/cJSON/cJSON.h"
const char *find_file_in_tar(const uint8_t *tar, const size_t tarSize, const char *fileName, size_t *fileSize) __attribute__((nonnull (1,3,4)));
cJSON *list_files_in_tar(const uint8_t *tarData, const size_t tarSize) __attribute__((nonnull (1)));
#endif // TAR_H
src/zip/teleporter.c
+1 -1
View File
@@ -523,7 +523,7 @@ static const char *test_and_import_database(void *ptr, size_t size, const char *
return NULL;
}
const char *read_teleporter_zip(char *buffer, const size_t buflen, char * const hint, cJSON *imported_files)
const char *read_teleporter_zip(uint8_t *buffer, const size_t buflen, char * const hint, cJSON *imported_files)
{
// Initialize ZIP archive
mz_zip_archive zip = { 0 };
src/zip/teleporter.h
+1 -1
View File
@@ -15,7 +15,7 @@
const char *generate_teleporter_zip(mz_zip_archive *zip, char filename[128], void **ptr, size_t *size);
bool free_teleporter_zip(mz_zip_archive *zip);
const char *read_teleporter_zip(char *buffer, const size_t buflen, char *hint, cJSON *json_files);
const char *read_teleporter_zip(uint8_t *buffer, const size_t buflen, char *hint, cJSON *json_files);
bool write_teleporter_zip_to_disk(void);
bool read_teleporter_zip_from_disk(const char *filename);
test/api/libs/FTLAPI.py
+10 -3
View File
@@ -15,6 +15,7 @@ import requests
from typing import List
import json
from hashlib import sha256
import urllib.parse
url = "http://pi.hole/api/auth"
@@ -23,6 +24,7 @@ class AuthenticationMethods(Enum):
HEADER = 1
BODY = 2
COOKIE = 3
QUERY_STR = 4
# Class to query the FTL API
class FTLAPI():
@@ -103,13 +105,18 @@ class FTLAPI():
def GET(self, uri: str, params: List[str] = [], expected_mimetype: str = "application/json", authenticate: AuthenticationMethods = AuthenticationMethods.BODY):
self.errors = []
try:
# Get json_data, headers and cookies
json_data, headers, cookies = self.get_jsondata_headers_cookies(authenticate)
# Add session ID to the request if authenticating via query string
if self.auth_method == AuthenticationMethods.QUERY_STR.name:
encoded_sid = urllib.parse.quote(self.session['sid'], safe='')
params.append("sid=" + encoded_sid)
# Add parameters to the URI (if any)
if len(params) > 0:
uri = uri + "?" + "&".join(params)
# Get json_data, headers and cookies
json_data, headers, cookies = self.get_jsondata_headers_cookies(authenticate)
if self.verbose:
print("GET " + self.api_url + uri + " with json_data: " + json.dumps(json_data))