From 497ef087c534f88a7e44905c190cf20b618243cb Mon Sep 17 00:00:00 2001 From: Evgeny Poberezkin <2769109+epoberezkin@users.noreply.github.com> Date: Thu, 28 Sep 2023 17:13:06 +0100 Subject: [PATCH] checkpoint on stop and on encryption change, switch journal_mode to DELETE on export and back to WAL after --- cabal.project | 2 +- scripts/nix/sha256map.nix | 2 +- src/Simplex/Chat.hs | 12 ++++++++---- src/Simplex/Chat/Archive.hs | 12 +++++++++++- stack.yaml | 2 +- 5 files changed, 22 insertions(+), 8 deletions(-) diff --git a/cabal.project b/cabal.project index b9753c9c00..6327767f8f 100644 --- a/cabal.project +++ b/cabal.project @@ -9,7 +9,7 @@ constraints: zip +disable-bzip2 +disable-zstd source-repository-package type: git location: https://github.com/simplex-chat/simplexmq.git - tag: fda1284ae4b7c33cae2eb8ed0376a511aecc1d51 + tag: 053c0b513abd9021879471fea4d2c27e61db4b12 source-repository-package type: git diff --git a/scripts/nix/sha256map.nix b/scripts/nix/sha256map.nix index faa2401b17..0aa307bdc2 100644 --- a/scripts/nix/sha256map.nix +++ b/scripts/nix/sha256map.nix @@ -1,5 +1,5 @@ { - "https://github.com/simplex-chat/simplexmq.git"."fda1284ae4b7c33cae2eb8ed0376a511aecc1d51" = "1gq7scv9z8x3xhzl914xr46na0kkrqd1i743xbw69lyx33kj9xb5"; + "https://github.com/simplex-chat/simplexmq.git"."053c0b513abd9021879471fea4d2c27e61db4b12" = "1pcv855qm639barrkc0b94m1zbq40pm20izvrzfrlynclw0csf8i"; "https://github.com/simplex-chat/hs-socks.git"."a30cc7a79a08d8108316094f8f2f82a0c5e1ac51" = "0yasvnr7g91k76mjkamvzab2kvlb1g5pspjyjn2fr6v83swjhj38"; "https://github.com/kazu-yamamoto/http2.git"."b5a1b7200cf5bc7044af34ba325284271f6dff25" = "0dqb50j57an64nf4qcf5vcz4xkd1vzvghvf8bk529c1k30r9nfzb"; "https://github.com/simplex-chat/direct-sqlcipher.git"."f814ee68b16a9447fbb467ccc8f29bdd3546bfd9" = "0kiwhvml42g9anw4d2v0zd1fpc790pj9syg5x3ik4l97fnkbbwpp"; diff --git a/src/Simplex/Chat.hs b/src/Simplex/Chat.hs index 822d6cd4be..597f127a19 100644 --- a/src/Simplex/Chat.hs +++ b/src/Simplex/Chat.hs @@ -84,7 +84,7 @@ import Simplex.Messaging.Agent.Client (AgentStatsKey (..), SubInfo (..), agentCl import Simplex.Messaging.Agent.Env.SQLite (AgentConfig (..), InitialAgentServers (..), createAgentStore, defaultAgentConfig) import Simplex.Messaging.Agent.Lock import Simplex.Messaging.Agent.Protocol -import Simplex.Messaging.Agent.Store.SQLite (MigrationConfirmation (..), MigrationError, SQLiteStore (dbNew), execSQL, upMigration, withConnection, closeSQLiteStore, openSQLiteStore) +import Simplex.Messaging.Agent.Store.SQLite (MigrationConfirmation (..), MigrationError, SQLiteStore (dbNew), execSQL, upMigration, withConnection, checkpointSQLiteStore, closeSQLiteStore, openSQLiteStore) import Simplex.Messaging.Agent.Store.SQLite.DB (SlowQueryStats (..)) import qualified Simplex.Messaging.Agent.Store.SQLite.DB as DB import qualified Simplex.Messaging.Agent.Store.SQLite.Migrations as Migrations @@ -334,9 +334,13 @@ stopChatController ChatController {chatStore, smpAgent, agentAsync = s, sndFiles keys <- M.keys <$> readTVar expireCIFlags forM_ keys $ \k -> TM.insert k False expireCIFlags writeTVar s Nothing - when closeStore $ liftIO $ do - closeSQLiteStore chatStore - closeSQLiteStore $ agentClientStore smpAgent + liftIO $ do + let agentStore = agentClientStore smpAgent + checkpointSQLiteStore chatStore + checkpointSQLiteStore agentStore + when closeStore $ do + closeSQLiteStore chatStore + closeSQLiteStore agentStore where closeFiles :: TVar (Map Int64 Handle) -> m () closeFiles files = do diff --git a/src/Simplex/Chat/Archive.hs b/src/Simplex/Chat/Archive.hs index 8d1b99328c..d40d3a515e 100644 --- a/src/Simplex/Chat/Archive.hs +++ b/src/Simplex/Chat/Archive.hs @@ -22,7 +22,7 @@ import qualified Data.Text as T import qualified Database.SQLite3 as SQL import Simplex.Chat.Controller import Simplex.Messaging.Agent.Client (agentClientStore) -import Simplex.Messaging.Agent.Store.SQLite (SQLiteStore (..), sqlString) +import Simplex.Messaging.Agent.Store.SQLite (SQLiteStore (..), setSQLiteModeWAL, sqlString) import Simplex.Messaging.Util import System.FilePath import UnliftIO.Directory @@ -43,13 +43,21 @@ archiveFilesFolder = "simplex_v1_files" exportArchive :: ChatMonad m => ArchiveConfig -> m () exportArchive cfg@ArchiveConfig {archivePath, disableCompression} = withTempDir cfg "simplex-chat." $ \dir -> do + setMode False StorageFiles {chatDb, agentDb, filesPath} <- storageFiles copyFile chatDb $ dir archiveChatDbFile copyFile agentDb $ dir archiveAgentDbFile + setMode True forM_ filesPath $ \fp -> copyDirectoryFiles fp $ dir archiveFilesFolder let method = if disableCompression == Just True then Z.Store else Z.Deflate Z.createArchive archivePath $ Z.packDirRecur method Z.mkEntrySelector dir + where + -- TODO this won't work on Windows as we close connection there when stopping chat + setMode walMode = do + ChatController {chatStore, smpAgent} <- ask + liftIO $ setSQLiteModeWAL chatStore walMode + liftIO $ setSQLiteModeWAL (agentClientStore smpAgent) walMode importArchive :: ChatMonad m => ArchiveConfig -> m [ArchiveError] importArchive cfg@ArchiveConfig {archivePath} = @@ -145,7 +153,9 @@ sqlCipherExport DBEncryptionConfig {currentKey = DBEncryptionKey key, newKey = D T.unlines $ keySQL key <> [ "ATTACH DATABASE " <> sqlString (f <> ".exported") <> " AS exported KEY " <> sqlString key' <> ";", + "PRAGMA wal_checkpoint(TRUNCATE);", "SELECT sqlcipher_export('exported');", + "PRAGMA exported.wal_checkpoint(TRUNCATE);", "DETACH DATABASE exported;" ] testSQL = diff --git a/stack.yaml b/stack.yaml index a466178ced..b9b24ab252 100644 --- a/stack.yaml +++ b/stack.yaml @@ -49,7 +49,7 @@ extra-deps: # - simplexmq-1.0.0@sha256:34b2004728ae396e3ae449cd090ba7410781e2b3cefc59259915f4ca5daa9ea8,8561 # - ../simplexmq - github: simplex-chat/simplexmq - commit: fda1284ae4b7c33cae2eb8ed0376a511aecc1d51 + commit: 053c0b513abd9021879471fea4d2c27e61db4b12 - github: kazu-yamamoto/http2 commit: b5a1b7200cf5bc7044af34ba325284271f6dff25 # - ../direct-sqlcipher