Merge pull request #1868 from pi-hole/new/useWAL

Make WAL mode for pihole-FTL.db optional

src/api/docs/content/specs/config.yaml

@@ -344,6 +344,8 @@ components:
                   type: integer
                 DBinterval:
                   type: integer
+                useWAL:
+                  type: boolean
                 network:
                   type: object
                   properties:
@@ -661,6 +663,7 @@ components:
             DBexport: true
             maxDBdays: 365
             DBinterval: 60
+            useWAL: true
             network:
               parseARPcache: true
               expire: 365

src/config/config.c

@@ -791,6 +791,23 @@ void initConfig(struct config *conf)
 	conf->database.DBinterval.t = CONF_UINT;
 	conf->database.DBinterval.d.ui = 60;
 
+	conf->database.useWAL.k = "database.useWAL";
+	conf->database.useWAL.h = "Should FTL enable Write-Ahead Log (WAL) mode for the on-disk query database (configured via files.database)?\n It is recommended to leave this setting enabled for performance reasons. About the only reason to disable WAL mode is if you are experiencing specific issues with it, e.g., when using a database that is accessed from multiple hosts via a network share. When this setting is disabled, FTL will use SQLite3's default journal mode (rollback journal in DELETE mode).";
+	conf->database.useWAL.t = CONF_BOOL;
+	// Note: We would not necessarily need to restart FTL when this setting
+	// is changed, but we do it anyway as this ensures the database is
+	// properly re-initialized and the new journal mode is used. As this is
+	// a setting that will be changed very rarely, this seems the better
+	// compromise than adding special code that can transform the database
+	// while being active.
+	// The in-memory database is not affected by this setting as it uses a
+	// MEMORY journal mode anyway (there is nothing to be restored after power
+	// loss). The gravity database is also not affected as it is only written
+	// to on an individual basis (explicit API calls) and not continuously
+	// (like the query database).
+	conf->database.useWAL.f = FLAG_ADVANCED_SETTING | FLAG_RESTART_FTL;
+	conf->database.useWAL.d.b = true;
+
 	// sub-struct database.network
 	conf->database.network.parseARPcache.k = "database.network.parseARPcache";
 	conf->database.network.parseARPcache.h = "Should FTL analyze the local ARP cache? When disabled, client identification and the network table will stop working reliably.";

src/config/config.h

@@ -202,6 +202,7 @@ struct config {
 		struct conf_item DBexport;
 		struct conf_item maxDBdays;
 		struct conf_item DBinterval;
+		struct conf_item useWAL;
 		struct {
 			struct conf_item parseARPcache;
 			struct conf_item expire;

src/database/query-table.c

@@ -112,19 +112,47 @@ bool init_memory_database(void)
 	if(!attach_database(_memdb, NULL, config.files.database.v.s, "disk"))
 		return false;
 
-	// Change journal mode to WAL
-	// - WAL is significantly faster in most scenarios.
-	// - WAL provides more concurrency as readers do not block writers and a
-	//   writer does not block readers. Reading and writing can proceed
-	//   concurrently.
-	// - Disk I/O operations tends to be more sequential using WAL.
-	rc = sqlite3_exec(_memdb, "PRAGMA disk.journal_mode=WAL", NULL, NULL, NULL);
-	if( rc != SQLITE_OK )
+	// Enable WAL mode for the on-disk database (pihole-FTL.db) if
+	// configured (default is yes). User may not want to enable WAL
+	// mode if the database is on a network share as all processes
+	// accessing the database must be on the same host in WAL mode.
+	if(config.database.useWAL.v.b)
 	{
-		log_err("init_memory_database(): Step error while trying to set journal mode: %s",
-		        sqlite3_errstr(rc));
-		sqlite3_close(_memdb);
-		return false;
+		// Change journal mode to WAL
+		// - WAL is significantly faster in most scenarios.
+		// - WAL provides more concurrency as readers do not block writers and a
+		//   writer does not block readers. Reading and writing can proceed
+		//   concurrently.
+		// - Disk I/O operations tend to be more sequential using WAL.
+		rc = sqlite3_exec(_memdb, "PRAGMA disk.journal_mode=WAL", NULL, NULL, NULL);
+		if( rc != SQLITE_OK )
+		{
+			log_err("init_memory_database(): Step error while trying to set journal mode: %s",
+			        sqlite3_errstr(rc));
+			sqlite3_close(_memdb);
+			return false;
+		}
+	}
+	else
+	{
+		// Unlike the other journaling modes, PRAGMA journal_mode=WAL is
+		// persistent. If a process sets WAL mode, then closes and
+		// reopens the database, the database will come back in WAL
+		// mode. In contrast, if a process sets (for example) PRAGMA
+		// journal_mode=TRUNCATE and then closes and reopens the
+		// database will come back up in the default rollback mode of
+		// DELETE rather than the previous TRUNCATE setting.
+
+		// Change journal mode back to DELETE due to user configuration
+		// (might have been changed to WAL before)
+		rc = sqlite3_exec(_memdb, "PRAGMA disk.journal_mode=DELETE", NULL, NULL, NULL);
+		if( rc != SQLITE_OK )
+		{
+			log_err("init_memory_database(): Step error while trying to set journal mode: %s",
+			        sqlite3_errstr(rc));
+			sqlite3_close(_memdb);
+			return false;
+		}
 	}
 
 	// Everything went well

test/pihole.toml

@@ -487,6 +487,15 @@
   # How often do we store queries in FTL's database [seconds]?
   DBinterval = 60
 
+  # Should FTL enable Write-Ahead Log (WAL) mode for the on-disk query database
+  # (configured via files.database)?
+  # It is recommended to leave this setting enabled for performance reasons. About the
+  # only reason to disable WAL mode is if you are experiencing specific issues with it,
+  # e.g., when using a database that is accessed from multiple hosts via a network
+  # share. When this setting is disabled, FTL will use SQLite3's default journal mode
+  # (rollback journal in DELETE mode).
+  useWAL = true
+
   [database.network]
     # Should FTL analyze the local ARP cache? When disabled, client identification and the
     # network table will stop working reliably.