checkpoint on stop and on encryption change, switch journal_mode to DELETE on export and back to WAL after

cabal.project

@@ -9,7 +9,7 @@ constraints: zip +disable-bzip2 +disable-zstd
 source-repository-package
     type: git
     location: https://github.com/simplex-chat/simplexmq.git
-    tag: fda1284ae4b7c33cae2eb8ed0376a511aecc1d51
+    tag: 053c0b513abd9021879471fea4d2c27e61db4b12
 
 source-repository-package
     type: git

scripts/nix/sha256map.nix

@@ -1,5 +1,5 @@
 {
-  "https://github.com/simplex-chat/simplexmq.git"."fda1284ae4b7c33cae2eb8ed0376a511aecc1d51" = "1gq7scv9z8x3xhzl914xr46na0kkrqd1i743xbw69lyx33kj9xb5";
+  "https://github.com/simplex-chat/simplexmq.git"."053c0b513abd9021879471fea4d2c27e61db4b12" = "1pcv855qm639barrkc0b94m1zbq40pm20izvrzfrlynclw0csf8i";
   "https://github.com/simplex-chat/hs-socks.git"."a30cc7a79a08d8108316094f8f2f82a0c5e1ac51" = "0yasvnr7g91k76mjkamvzab2kvlb1g5pspjyjn2fr6v83swjhj38";
   "https://github.com/kazu-yamamoto/http2.git"."b5a1b7200cf5bc7044af34ba325284271f6dff25" = "0dqb50j57an64nf4qcf5vcz4xkd1vzvghvf8bk529c1k30r9nfzb";
   "https://github.com/simplex-chat/direct-sqlcipher.git"."f814ee68b16a9447fbb467ccc8f29bdd3546bfd9" = "0kiwhvml42g9anw4d2v0zd1fpc790pj9syg5x3ik4l97fnkbbwpp";

src/Simplex/Chat.hs

@@ -84,7 +84,7 @@ import Simplex.Messaging.Agent.Client (AgentStatsKey (..), SubInfo (..), agentCl
 import Simplex.Messaging.Agent.Env.SQLite (AgentConfig (..), InitialAgentServers (..), createAgentStore, defaultAgentConfig)
 import Simplex.Messaging.Agent.Lock
 import Simplex.Messaging.Agent.Protocol
-import Simplex.Messaging.Agent.Store.SQLite (MigrationConfirmation (..), MigrationError, SQLiteStore (dbNew), execSQL, upMigration, withConnection, closeSQLiteStore, openSQLiteStore)
+import Simplex.Messaging.Agent.Store.SQLite (MigrationConfirmation (..), MigrationError, SQLiteStore (dbNew), execSQL, upMigration, withConnection, checkpointSQLiteStore, closeSQLiteStore, openSQLiteStore)
 import Simplex.Messaging.Agent.Store.SQLite.DB (SlowQueryStats (..))
 import qualified Simplex.Messaging.Agent.Store.SQLite.DB as DB
 import qualified Simplex.Messaging.Agent.Store.SQLite.Migrations as Migrations
@@ -334,9 +334,13 @@ stopChatController ChatController {chatStore, smpAgent, agentAsync = s, sndFiles
     keys <- M.keys <$> readTVar expireCIFlags
     forM_ keys $ \k -> TM.insert k False expireCIFlags
     writeTVar s Nothing
-  when closeStore $ liftIO $ do
-    closeSQLiteStore chatStore
-    closeSQLiteStore $ agentClientStore smpAgent
+  liftIO $ do
+    let agentStore = agentClientStore smpAgent
+    checkpointSQLiteStore chatStore
+    checkpointSQLiteStore agentStore
+    when closeStore $ do
+      closeSQLiteStore chatStore
+      closeSQLiteStore agentStore
   where
     closeFiles :: TVar (Map Int64 Handle) -> m ()
     closeFiles files = do

src/Simplex/Chat/Archive.hs

@@ -22,7 +22,7 @@ import qualified Data.Text as T
 import qualified Database.SQLite3 as SQL
 import Simplex.Chat.Controller
 import Simplex.Messaging.Agent.Client (agentClientStore)
-import Simplex.Messaging.Agent.Store.SQLite (SQLiteStore (..), sqlString)
+import Simplex.Messaging.Agent.Store.SQLite (SQLiteStore (..), setSQLiteModeWAL, sqlString)
 import Simplex.Messaging.Util
 import System.FilePath
 import UnliftIO.Directory
@@ -43,13 +43,21 @@ archiveFilesFolder = "simplex_v1_files"
 exportArchive :: ChatMonad m => ArchiveConfig -> m ()
 exportArchive cfg@ArchiveConfig {archivePath, disableCompression} =
   withTempDir cfg "simplex-chat." $ \dir -> do
+    setMode False
     StorageFiles {chatDb, agentDb, filesPath} <- storageFiles
     copyFile chatDb $ dir </> archiveChatDbFile
     copyFile agentDb $ dir </> archiveAgentDbFile
+    setMode True
     forM_ filesPath $ \fp ->
       copyDirectoryFiles fp $ dir </> archiveFilesFolder
     let method = if disableCompression == Just True then Z.Store else Z.Deflate
     Z.createArchive archivePath $ Z.packDirRecur method Z.mkEntrySelector dir
+  where
+    -- TODO this won't work on Windows as we close connection there when stopping chat
+    setMode walMode = do
+      ChatController {chatStore, smpAgent} <- ask
+      liftIO $ setSQLiteModeWAL chatStore walMode
+      liftIO $ setSQLiteModeWAL (agentClientStore smpAgent) walMode
 
 importArchive :: ChatMonad m => ArchiveConfig -> m [ArchiveError]
 importArchive cfg@ArchiveConfig {archivePath} =
@@ -145,7 +153,9 @@ sqlCipherExport DBEncryptionConfig {currentKey = DBEncryptionKey key, newKey = D
           T.unlines $
             keySQL key
               <> [ "ATTACH DATABASE " <> sqlString (f <> ".exported") <> " AS exported KEY " <> sqlString key' <> ";",
+                   "PRAGMA wal_checkpoint(TRUNCATE);",
                    "SELECT sqlcipher_export('exported');",
+                   "PRAGMA exported.wal_checkpoint(TRUNCATE);",
                    "DETACH DATABASE exported;"
                  ]
         testSQL =

stack.yaml

@@ -49,7 +49,7 @@ extra-deps:
   # - simplexmq-1.0.0@sha256:34b2004728ae396e3ae449cd090ba7410781e2b3cefc59259915f4ca5daa9ea8,8561
   # - ../simplexmq
   - github: simplex-chat/simplexmq
-    commit: fda1284ae4b7c33cae2eb8ed0376a511aecc1d51
+    commit: 053c0b513abd9021879471fea4d2c27e61db4b12
   - github: kazu-yamamoto/http2
     commit: b5a1b7200cf5bc7044af34ba325284271f6dff25
   # - ../direct-sqlcipher