From 3dff85623c289d47348102961a34132aaf386798 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 29 Jan 2024 16:02:01 +0100 Subject: [PATCH 1/2] Add new config option database.useWAL defaulting to true making the use of the WAL journal optional for the on-disk query database Signed-off-by: DL6ER --- src/api/docs/content/specs/config.yaml | 3 ++ src/config/config.c | 17 +++++++++ src/config/config.h | 1 + src/database/query-table.c | 52 ++++++++++++++++++++------ test/pihole.toml | 9 +++++ 5 files changed, 70 insertions(+), 12 deletions(-) diff --git a/src/api/docs/content/specs/config.yaml b/src/api/docs/content/specs/config.yaml index 5a3d9a04..fd033ec1 100644 --- a/src/api/docs/content/specs/config.yaml +++ b/src/api/docs/content/specs/config.yaml @@ -351,6 +351,8 @@ components: type: integer DBinterval: type: integer + useWAL: + type: boolean network: type: object properties: @@ -671,6 +673,7 @@ components: DBexport: true maxDBdays: 365 DBinterval: 60 + useWAL: true network: parseARPcache: true expire: 365 diff --git a/src/config/config.c b/src/config/config.c index 5b01ef81..a0c4e003 100644 --- a/src/config/config.c +++ b/src/config/config.c @@ -810,6 +810,23 @@ void initConfig(struct config *conf) conf->database.DBinterval.t = CONF_UINT; conf->database.DBinterval.d.ui = 60; + conf->database.useWAL.k = "database.useWAL"; + conf->database.useWAL.h = "Should FTL enable Write-Ahead Log (WAL) mode for the on-disk query database (configured via files.database)?\n It is recommended to leave this setting enabled for performance reasons. About the only reason to disable WAL mode is if you are experiencing specific issues with it, e.g., when using a database that is accessed from multiple hosts via a network share. When this setting is disabled, FTL will use SQLite3's default journal mode (rollback journal in DELETE mode)."; + conf->database.useWAL.t = CONF_BOOL; + // Note: We would not necessarily need to restart FTL when this setting + // is changed, but we do it anyway as this ensures the database is + // properly re-initialized and the new journal mode is used. As this is + // a setting that will be changed very rarely, this semms the better + // compromise than adding special code that can transform the database + // while being active. + // The in-memory database is not affected by this setting as it uses a + // MEMORY journal mode anyway (there is nothing to be restored after power + // loss). The gravity database is also not affected as it is only written + // to on an individual basis (explicit API calls) and not continuously + // (like the query database). + conf->database.useWAL.f = FLAG_ADVANCED_SETTING | FLAG_RESTART_FTL; + conf->database.useWAL.d.b = true; + // sub-struct database.network conf->database.network.parseARPcache.k = "database.network.parseARPcache"; conf->database.network.parseARPcache.h = "Should FTL analyze the local ARP cache? When disabled, client identification and the network table will stop working reliably."; diff --git a/src/config/config.h b/src/config/config.h index 13bce41a..4cd1b768 100644 --- a/src/config/config.h +++ b/src/config/config.h @@ -207,6 +207,7 @@ struct config { struct conf_item DBexport; struct conf_item maxDBdays; struct conf_item DBinterval; + struct conf_item useWAL; struct { struct conf_item parseARPcache; struct conf_item expire; diff --git a/src/database/query-table.c b/src/database/query-table.c index d5e79378..279cc253 100644 --- a/src/database/query-table.c +++ b/src/database/query-table.c @@ -112,19 +112,47 @@ bool init_memory_database(void) if(!attach_database(_memdb, NULL, config.files.database.v.s, "disk")) return false; - // Change journal mode to WAL - // - WAL is significantly faster in most scenarios. - // - WAL provides more concurrency as readers do not block writers and a - // writer does not block readers. Reading and writing can proceed - // concurrently. - // - Disk I/O operations tends to be more sequential using WAL. - rc = sqlite3_exec(_memdb, "PRAGMA disk.journal_mode=WAL", NULL, NULL, NULL); - if( rc != SQLITE_OK ) + // Enable WAL mode for the on-disk database (pihole-FTL.db) if + // configured (default is yes). User may not want to enable WAL + // mode if the database is on a network share as all processes + // accessing the database must be on the same host in WAL mode. + if(config.database.useWAL.v.b) { - log_err("init_memory_database(): Step error while trying to set journal mode: %s", - sqlite3_errstr(rc)); - sqlite3_close(_memdb); - return false; + // Change journal mode to WAL + // - WAL is significantly faster in most scenarios. + // - WAL provides more concurrency as readers do not block writers and a + // writer does not block readers. Reading and writing can proceed + // concurrently. + // - Disk I/O operations tends to be more sequential using WAL. + rc = sqlite3_exec(_memdb, "PRAGMA disk.journal_mode=WAL", NULL, NULL, NULL); + if( rc != SQLITE_OK ) + { + log_err("init_memory_database(): Step error while trying to set journal mode: %s", + sqlite3_errstr(rc)); + sqlite3_close(_memdb); + return false; + } + } + else + { + // Unlike the other journaling modes, PRAGMA journal_mode=WAL is + // persistent. If a process sets WAL mode, then closes and + // reopens the database, the database will come back in WAL + // mode. In contrast, if a process sets (for example) PRAGMA + // journal_mode=TRUNCATE and then closes and reopens the + // database will come back up in the default rollback mode of + // DELETE rather than the previous TRUNCATE setting. + + // Change journal mode back to DELETE due to user configuration + // (might have been changed to WAL before) + rc = sqlite3_exec(_memdb, "PRAGMA disk.journal_mode=DELETE", NULL, NULL, NULL); + if( rc != SQLITE_OK ) + { + log_err("init_memory_database(): Step error while trying to set journal mode: %s", + sqlite3_errstr(rc)); + sqlite3_close(_memdb); + return false; + } } // Everything went well diff --git a/test/pihole.toml b/test/pihole.toml index 9a3939b1..ddba68cf 100644 --- a/test/pihole.toml +++ b/test/pihole.toml @@ -490,6 +490,15 @@ # How often do we store queries in FTL's database [seconds]? DBinterval = 60 + # Should FTL enable Write-Ahead Log (WAL) mode for the on-disk query database + # (configured via files.database)? + # It is recommended to leave this setting enabled for performance reasons. About the + # only reason to disable WAL mode is if you are experiencing specific issues with it, + # e.g., when using a database that is accessed from multiple hosts via a network + # share. When this setting is disabled, FTL will use SQLite3's default journal mode + # (rollback journal in DELETE mode). + useWAL = true + [database.network] # Should FTL analyze the local ARP cache? When disabled, client identification and the # network table will stop working reliably. From a72033023903ccb42622b28b6032b12590041b7a Mon Sep 17 00:00:00 2001 From: Dominik Date: Tue, 30 Jan 2024 19:14:50 +0100 Subject: [PATCH 2/2] Apply suggestions from code review Co-authored-by: RD WebDesign Signed-off-by: Dominik --- src/config/config.c | 2 +- src/database/query-table.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/config/config.c b/src/config/config.c index a0c4e003..90f13e5a 100644 --- a/src/config/config.c +++ b/src/config/config.c @@ -816,7 +816,7 @@ void initConfig(struct config *conf) // Note: We would not necessarily need to restart FTL when this setting // is changed, but we do it anyway as this ensures the database is // properly re-initialized and the new journal mode is used. As this is - // a setting that will be changed very rarely, this semms the better + // a setting that will be changed very rarely, this seems the better // compromise than adding special code that can transform the database // while being active. // The in-memory database is not affected by this setting as it uses a diff --git a/src/database/query-table.c b/src/database/query-table.c index 279cc253..1d19250b 100644 --- a/src/database/query-table.c +++ b/src/database/query-table.c @@ -123,7 +123,7 @@ bool init_memory_database(void) // - WAL provides more concurrency as readers do not block writers and a // writer does not block readers. Reading and writing can proceed // concurrently. - // - Disk I/O operations tends to be more sequential using WAL. + // - Disk I/O operations tend to be more sequential using WAL. rc = sqlite3_exec(_memdb, "PRAGMA disk.journal_mode=WAL", NULL, NULL, NULL); if( rc != SQLITE_OK ) {