Merge remote-tracking branch 'origin/ab/zstd' into ab/zstd

apps/ios/Shared/Views/Database/DatabaseView.swift

@@ -19,7 +19,6 @@ enum DatabaseAlert: Identifiable {
     case chatDeleted
     case deleteLegacyDatabase
     case deleteFilesAndMedia
-    case deleteTempFiles
     case setChatItemTTL(ttl: ChatItemTTL)
     case error(title: LocalizedStringKey, error: String = "")
 
@@ -34,7 +33,6 @@ enum DatabaseAlert: Identifiable {
         case .chatDeleted: return "chatDeleted"
         case .deleteLegacyDatabase: return "deleteLegacyDatabase"
         case .deleteFilesAndMedia: return "deleteFilesAndMedia"
-        case .deleteTempFiles: return "deleteTempFiles"
         case .setChatItemTTL: return "setChatItemTTL"
         case let .error(title, _): return "error \(title)"
         }
@@ -51,7 +49,6 @@ struct DatabaseView: View {
     @State private var progressIndicator = false
     @AppStorage(DEFAULT_CHAT_ARCHIVE_NAME) private var chatArchiveName: String?
     @AppStorage(DEFAULT_CHAT_ARCHIVE_TIME) private var chatArchiveTime: Double = 0
-    @AppStorage(DEFAULT_DEVELOPER_TOOLS) private var developerTools = false
     @State private var dbContainer = dbContainerGroupDefault.get()
     @State private var legacyDatabase = hasLegacyDatabase()
     @State private var useKeychain = storeDBPassphraseGroupDefault.get()
@@ -194,16 +191,6 @@ struct DatabaseView: View {
                     }
                 }
             }
-
-            if developerTools {
-                Section {
-                    Button("Delete temporary files", role: .destructive) {
-                        alert = .deleteTempFiles
-                    }
-                } header: {
-                    Text("Temp files")
-                }
-            }
         }
         .onAppear {
             runChat = m.chatRunning ?? true
@@ -303,15 +290,6 @@ struct DatabaseView: View {
                 },
                 secondaryButton: .cancel()
             )
-        case .deleteTempFiles:
-            return Alert(
-                title: Text("Delete temporary files?"),
-                message: Text("This action cannot be undone - all temporary files will be deleted."),
-                primaryButton: .destructive(Text("Delete")) {
-                    deleteAppTempFiles()
-                },
-                secondaryButton: .cancel()
-            )
         case let .setChatItemTTL(ttl):
             return Alert(
                 title: Text("Enable automatic message deletion?"),

apps/ios/SimpleXChat/FileUtils.swift

@@ -82,7 +82,9 @@ public func deleteAppDatabaseAndFiles() {
     }
     try? fm.removeItem(atPath: dbPath + CHAT_DB_BAK)
     try? fm.removeItem(atPath: dbPath + AGENT_DB_BAK)
-    deleteAppTempFiles()
+    try? fm.removeItem(at: getTempFilesDirectory())
+    try? fm.removeItem(at: getMigrationTempFilesDirectory())
+    try? fm.createDirectory(at: getTempFilesDirectory(), withIntermediateDirectories: true)
     deleteAppFiles()
     _ = kcDatabasePassword.remove()
     storeDBPassphraseGroupDefault.set(true)
@@ -98,21 +100,6 @@ public func deleteAppFiles() {
     }
 }
 
-public func deleteAppTempFiles() {
-    let fm = FileManager.default
-    do {
-        try fm.removeItem(at: getTempFilesDirectory())
-        try fm.createDirectory(at: getTempFilesDirectory(), withIntermediateDirectories: true)
-    } catch {
-        logger.error("FileUtils deleteAppTempFiles error: \(error.localizedDescription)")
-    }
-    do {
-        try fm.removeItem(at: getMigrationTempFilesDirectory())
-    } catch {
-        logger.error("FileUtils deleteAppTempFiles fm.removeItem(at: getMigrationTempFilesDirectory()) error: \(error.localizedDescription)")
-    }
-}
-
 public func fileSize(_ url: URL) -> Int? { // in bytes
     do {
         let val = try url.resourceValues(forKeys: [.totalFileAllocatedSizeKey, .fileAllocatedSizeKey])

cabal.project

@@ -12,7 +12,7 @@ constraints: zip +disable-bzip2 +disable-zstd
 source-repository-package
     type: git
     location: https://github.com/simplex-chat/simplexmq.git
-    tag: 293a2ca3f10232fa8a5221388344acf68643ad92
+    tag: ca68eca86ef92ae266a4005ab1ad57b589f83933
 
 source-repository-package
     type: git

package.yaml

@@ -49,6 +49,7 @@ dependencies:
   - unliftio == 0.2.*
   - unliftio-core == 0.2.*
   - zip == 2.0.*
+  - zstd
 
 flags:
   swift:

scripts/nix/sha256map.nix

@@ -1,5 +1,5 @@
 {
-  "https://github.com/simplex-chat/simplexmq.git"."293a2ca3f10232fa8a5221388344acf68643ad92" = "13khaadp6w66rn9pfifd779amcyj6lap2f2c0kns4ijjqqb2c5j5";
+  "https://github.com/simplex-chat/simplexmq.git"."ca68eca86ef92ae266a4005ab1ad57b589f83933" = "10p1bn42hbmisdjk272q6jshrcx1vq1072r50n80hj6n6z1a0szf";
   "https://github.com/simplex-chat/hs-socks.git"."a30cc7a79a08d8108316094f8f2f82a0c5e1ac51" = "0yasvnr7g91k76mjkamvzab2kvlb1g5pspjyjn2fr6v83swjhj38";
   "https://github.com/simplex-chat/direct-sqlcipher.git"."f814ee68b16a9447fbb467ccc8f29bdd3546bfd9" = "1ql13f4kfwkbaq7nygkxgw84213i0zm7c1a8hwvramayxl38dq5d";
   "https://github.com/simplex-chat/sqlcipher-simple.git"."a46bd361a19376c5211f1058908fc0ae6bf42446" = "1z0r78d8f0812kxbgsm735qf6xx8lvaz27k1a0b4a2m0sshpd5gl";

simplex-chat.cabal

@@ -216,6 +216,7 @@ library
     , unliftio ==0.2.*
     , unliftio-core ==0.2.*
     , zip ==2.0.*
+    , zstd
   default-language: Haskell2010
   if flag(swift)
     cpp-options: -DswiftJSON
@@ -277,6 +278,7 @@ executable simplex-bot
     , unliftio ==0.2.*
     , unliftio-core ==0.2.*
     , zip ==2.0.*
+    , zstd
   default-language: Haskell2010
   if flag(swift)
     cpp-options: -DswiftJSON
@@ -338,6 +340,7 @@ executable simplex-bot-advanced
     , unliftio ==0.2.*
     , unliftio-core ==0.2.*
     , zip ==2.0.*
+    , zstd
   default-language: Haskell2010
   if flag(swift)
     cpp-options: -DswiftJSON
@@ -401,6 +404,7 @@ executable simplex-broadcast-bot
     , unliftio ==0.2.*
     , unliftio-core ==0.2.*
     , zip ==2.0.*
+    , zstd
   default-language: Haskell2010
   if flag(swift)
     cpp-options: -DswiftJSON
@@ -464,6 +468,7 @@ executable simplex-chat
     , unliftio-core ==0.2.*
     , websockets ==0.12.*
     , zip ==2.0.*
+    , zstd
   default-language: Haskell2010
   if flag(swift)
     cpp-options: -DswiftJSON
@@ -530,6 +535,7 @@ executable simplex-directory-service
     , unliftio ==0.2.*
     , unliftio-core ==0.2.*
     , zip ==2.0.*
+    , zstd
   default-language: Haskell2010
   if flag(swift)
     cpp-options: -DswiftJSON
@@ -626,6 +632,7 @@ test-suite simplex-chat-test
     , unliftio ==0.2.*
     , unliftio-core ==0.2.*
     , zip ==2.0.*
+    , zstd
   default-language: Haskell2010
   if flag(swift)
     cpp-options: -DswiftJSON

src/Simplex/Chat.hs

@@ -455,6 +455,14 @@ processChatCommand cmd =
 
 processChatCommand' :: forall m. ChatMonad m => (PQSupport -> VersionRangeChat) -> ChatCommand -> m ChatResponse
 processChatCommand' vr = \case
+  TestZstd outfile_ -> do
+    rows <- withStore' testZstd
+    case outfile_ of
+      Nothing -> pure $ CRZstdTest rows
+      Just path -> do
+        liftIO $ LB.writeFile path $ LB.unlines $
+          map (\ZstdRow {raw, z1, z3, z6, z9, z} -> LB.fromStrict . B.unwords $ map bshow [raw, z1, z3, z6, z9, z]) rows
+        ok_
   ShowActiveUser -> withUser' $ pure . CRActiveUser
   CreateActiveUser NewUser {profile, sameServers, pastTimestamp} -> do
     forM_ profile $ \Profile {displayName} -> checkValidName displayName
@@ -6737,7 +6745,8 @@ chatVersionRange = do
 chatCommandP :: Parser ChatCommand
 chatCommandP =
   choice
-    [ "/mute " *> ((`SetShowMessages` MFNone) <$> chatNameP),
+    [ "/zstd" *> (TestZstd <$> optional (A.space *> filePath)),
+      "/mute " *> ((`SetShowMessages` MFNone) <$> chatNameP),
       "/unmute " *> ((`SetShowMessages` MFAll) <$> chatNameP),
       "/unmute mentions " *> ((`SetShowMessages` MFMentions) <$> chatNameP),
       "/receipts " *> (SetSendReceipts <$> chatNameP <* " " <*> ((Just <$> onOffP) <|> ("default" $> Nothing))),

src/Simplex/Chat/Controller.hs

@@ -216,7 +216,8 @@ data HelpSection = HSMain | HSFiles | HSGroups | HSContacts | HSMyAddress | HSIn
   deriving (Show)
 
 data ChatCommand
-  = ShowActiveUser
+  = TestZstd (Maybe FilePath)
+  | ShowActiveUser
   | CreateActiveUser NewUser
   | ListUsers
   | APISetActiveUser UserId (Maybe UserPwd)
@@ -726,6 +727,12 @@ data ChatResponse
   | CRArchiveImported {archiveErrors :: [ArchiveError]}
   | CRAppSettings {appSettings :: AppSettings}
   | CRTimedAction {action :: String, durationMilliseconds :: Int64}
+  | CRZstdTest {zstdRows :: [ZstdRow]}
+  deriving (Show)
+
+data ZstdRow = ZstdRow
+  { raw, z1, z3, z6, z9, z :: !Int
+  }
   deriving (Show)
 
 -- some of these can only be used as command responses
@@ -1403,6 +1410,8 @@ $(JQ.deriveJSON (sumTypeJSON $ dropPrefix "RCSR") ''RemoteCtrlStopReason)
 
 $(JQ.deriveJSON (sumTypeJSON $ dropPrefix "RHSR") ''RemoteHostStopReason)
 
+$(JQ.deriveJSON defaultJSON ''ZstdRow)
+
 $(JQ.deriveJSON (sumTypeJSON $ dropPrefix "CR") ''ChatResponse)
 
 $(JQ.deriveFromJSON defaultJSON ''ArchiveConfig)

src/Simplex/Chat/Store/Messages.hs

@@ -111,18 +111,21 @@ module Simplex.Chat.Store.Messages
     getGroupSndStatuses,
     getGroupSndStatusCounts,
     getGroupHistoryItems,
+    testZstd,
   )
 where
 
+import qualified Codec.Compression.Zstd as Zstd
 import Control.Monad
 import Control.Monad.Except
 import Control.Monad.IO.Class
 import Crypto.Random (ChaChaDRG)
 import Data.Bifunctor (first)
 import Data.ByteString.Char8 (ByteString)
+import qualified Data.ByteString.Char8 as B
 import Data.Either (fromRight, rights)
 import Data.Int (Int64)
-import Data.List (sortBy)
+import Data.List (sortBy, sortOn)
 import Data.Maybe (fromMaybe, isJust, mapMaybe)
 import Data.Ord (Down (..), comparing)
 import Data.Text (Text)
@@ -131,7 +134,7 @@ import Data.Time (addUTCTime)
 import Data.Time.Clock (UTCTime (..), getCurrentTime)
 import Database.SQLite.Simple (NamedParam (..), Only (..), Query, (:.) (..))
 import Database.SQLite.Simple.QQ (sql)
-import Simplex.Chat.Controller (ChatListQuery (..), ChatPagination (..), PaginationByTime (..))
+import Simplex.Chat.Controller (ChatListQuery (..), ChatPagination (..), PaginationByTime (..), ZstdRow (..))
 import Simplex.Chat.Markdown
 import Simplex.Chat.Messages
 import Simplex.Chat.Messages.CIContent
@@ -147,8 +150,10 @@ import qualified Simplex.Messaging.Agent.Store.SQLite.DB as DB
 import qualified Simplex.Messaging.Crypto as C
 import Simplex.Messaging.Crypto.Ratchet (PQSupport)
 import Simplex.Messaging.Crypto.File (CryptoFile (..), CryptoFileArgs (..))
-import Simplex.Messaging.Util (eitherToMaybe)
+import Simplex.Messaging.Util (eitherToMaybe, (<$$>))
+import Simplex.Messaging.Version (VersionRange)
 import UnliftIO.STM
+import Debug.Trace
 
 deleteContactCIs :: DB.Connection -> User -> Contact -> IO ()
 deleteContactCIs db user@User {userId} ct@Contact {contactId} = do
@@ -2544,3 +2549,24 @@ getGroupHistoryItems db user@User {userId} GroupInfo {groupId} count = do
             LIMIT ?
           |]
           (userId, groupId, rcvMsgContentTag, sndMsgContentTag, count)
+
+testZstd :: DB.Connection -> IO [ZstdRow]
+testZstd db = do
+  samples <- fromOnly <$$> DB.query_ db "SELECT msg_body FROM messages"
+  let totalSize = sum $ map B.length samples
+      maxDictSize = totalSize `div` 100
+  dict16k <- either fail pure $ Zstd.trainFromSamples (1024 * 16) samples
+  dict100k <- either fail pure $ Zstd.trainFromSamples (1024 * 100) samples
+  dictMax <- either fail pure $ Zstd.trainFromSamples maxDictSize samples
+  traceShowM ('D', Zstd.getDictID dict100k, (totalSize, maxDictSize))
+  pure $ sortOn raw $ map (process dict16k dict100k dictMax) samples
+  where
+    process dict1 dict2 dict3 msg_body =
+      ZstdRow
+        { raw = B.length msg_body,
+          z1 = B.length $ Zstd.compress 1 msg_body,
+          z3 = B.length $ Zstd.compressUsingDict dict1 1 msg_body,
+          z6 = B.length $ Zstd.compressUsingDict dict2 1 msg_body,
+          z9 = B.length $ Zstd.compressUsingDict dict3 1 msg_body,
+          z = B.length $ Zstd.compress Zstd.maxCLevel msg_body
+        }

src/Simplex/Chat/View.hs

@@ -391,6 +391,7 @@ responseToView hu@(currentRH, user_) ChatConfig {logLevel, showReactions, showRe
   CRArchiveImported archiveErrs -> if null archiveErrs then ["ok"] else ["archive import errors: " <> plain (show archiveErrs)]
   CRAppSettings as -> ["app settings: " <> plain (LB.unpack $ J.encode as)]
   CRTimedAction _ _ -> []
+  CRZstdTest {zstdRows} -> map (\ZstdRow {raw, z1, z3, z6, z9, z} -> plain . T.unwords $ map tshow [raw, z1, z3, z6, z9, z]) zstdRows
   where
     ttyUser :: User -> [StyledString] -> [StyledString]
     ttyUser user@User {showNtfs, activeUser} ss