core: support deleting the last profile (always create user record in agent when user is created) (#3654)

* core: only skip creating agent user when app is first started

* firstTime

* prompt

* fix test

* if

* simplexmq

* ci timeout

* skip test

* add test timeout

* log test times

* simplexmq

* timeout

* simplexmq without new batching

* Revert "simplexmq without new batching"

This reverts commit 9879bcb57c.

* without new batching again

* with builder, without sized builder

* lazy bytestring, same batching logic

* fewer chunks

* remove lazy

* optimize batching in simplexmq
This commit is contained in:
Evgeny Poberezkin
2024-01-17 15:20:13 +00:00
committed by GitHub
parent 8c1114648a
commit 868acd18d6
20 changed files with 148 additions and 110 deletions
+1 -1
View File
@@ -12,7 +12,7 @@ constraints: zip +disable-bzip2 +disable-zstd
source-repository-package
type: git
location: https://github.com/simplex-chat/simplexmq.git
tag: f6ed4640d407f8879273d104a3e69069806dcb7c
tag: eb41abfb8fe84c7212abca8d1179b5308d937274
source-repository-package
type: git
+1 -1
View File
@@ -1,5 +1,5 @@
{
"https://github.com/simplex-chat/simplexmq.git"."f6ed4640d407f8879273d104a3e69069806dcb7c" = "072rakv697f85i8ldjl7bj7jc7vfmzphasx2i4ynwgz3kksydfp5";
"https://github.com/simplex-chat/simplexmq.git"."eb41abfb8fe84c7212abca8d1179b5308d937274" = "1hxrwib82gzx4j251dz88ivhi288sajg7fccdxbig28dj1gfz4s7";
"https://github.com/simplex-chat/hs-socks.git"."a30cc7a79a08d8108316094f8f2f82a0c5e1ac51" = "0yasvnr7g91k76mjkamvzab2kvlb1g5pspjyjn2fr6v83swjhj38";
"https://github.com/simplex-chat/direct-sqlcipher.git"."f814ee68b16a9447fbb467ccc8f29bdd3546bfd9" = "1ql13f4kfwkbaq7nygkxgw84213i0zm7c1a8hwvramayxl38dq5d";
"https://github.com/simplex-chat/sqlcipher-simple.git"."a46bd361a19376c5211f1058908fc0ae6bf42446" = "1z0r78d8f0812kxbgsm735qf6xx8lvaz27k1a0b4a2m0sshpd5gl";
+1
View File
@@ -0,0 +1 @@
--times
+9 -73
View File
@@ -108,9 +108,9 @@ import Simplex.Messaging.Util
import Simplex.Messaging.Version
import Simplex.RemoteControl.Invitation (RCInvitation (..), RCSignedInvitation (..))
import Simplex.RemoteControl.Types (RCCtrlAddress (..))
import System.Exit (ExitCode, exitFailure, exitSuccess)
import System.Exit (ExitCode, exitSuccess)
import System.FilePath (takeFileName, (</>))
import System.IO (Handle, IOMode (..), SeekMode (..), hFlush, stdout)
import System.IO (Handle, IOMode (..), SeekMode (..), hFlush)
import System.Random (randomRIO)
import Text.Read (readMaybe)
import UnliftIO.Async
@@ -454,17 +454,14 @@ processChatCommand' vr = \case
u <- asks currentUser
(smp, smpServers) <- chooseServers SPSMP
(xftp, xftpServers) <- chooseServers SPXFTP
auId <-
withStore' getUsers >>= \case
[] -> pure 1
users -> do
forM_ users $ \User {localDisplayName = n, activeUser, viewPwdHash} ->
when (n == displayName) . throwChatError $
if activeUser || isNothing viewPwdHash then CEUserExists displayName else CEInvalidDisplayName {displayName, validName = ""}
withAgent (\a -> createUser a smp xftp)
users <- withStore' getUsers
forM_ users $ \User {localDisplayName = n, activeUser, viewPwdHash} ->
when (n == displayName) . throwChatError $
if activeUser || isNothing viewPwdHash then CEUserExists displayName else CEInvalidDisplayName {displayName, validName = ""}
auId <- withAgent (\a -> createUser a smp xftp)
ts <- liftIO $ getCurrentTime >>= if pastTimestamp then coupleDaysAgo else pure
user <- withStore $ \db -> createUserRecordAt db (AgentUserId auId) p True ts
when (auId == 1) $ withStore (\db -> createContact db user simplexContactProfile) `catchChatError` \_ -> pure ()
when (null users) $ withStore (\db -> createContact db user simplexContactProfile) `catchChatError` \_ -> pure ()
withStore $ \db -> createNoteFolder db user
storeServers user smpServers
storeServers user xftpServers
@@ -2277,6 +2274,7 @@ processChatCommand' vr = \case
pure $ CRGroupUpdated user g g' Nothing
checkValidName :: GroupName -> m ()
checkValidName displayName = do
when (T.null displayName) $ throwChatError CEInvalidDisplayName {displayName, validName = ""}
let validName = T.pack $ mkValidName $ T.unpack displayName
when (displayName /= validName) $ throwChatError CEInvalidDisplayName {displayName, validName}
assertUserGroupRole :: GroupInfo -> GroupMemberRole -> m ()
@@ -6290,68 +6288,6 @@ createLocalChatItem user cd content createdAt = do
let smi_ = Just (SharedMsgId sharedMsgId)
in createNewChatItem_ db user cd Nothing smi_ content (Nothing, Nothing, Nothing, Nothing, Nothing) Nothing False createdAt Nothing createdAt
getCreateActiveUser :: SQLiteStore -> Bool -> IO User
getCreateActiveUser st testView = do
user <-
withTransaction st getUsers >>= \case
[] -> newUser
users -> maybe (selectUser users) pure (find activeUser users)
unless testView $ putStrLn $ "Current user: " <> userStr user
pure user
where
newUser :: IO User
newUser = do
putStrLn
"No user profiles found, it will be created now.\n\
\Please choose your display name and your full name.\n\
\They will be sent to your contacts when you connect.\n\
\They are only stored on your device and you can change them later."
loop
where
loop = do
displayName <- getContactName
withTransaction st (\db -> runExceptT $ createUserRecord db (AgentUserId 1) (profileFromName displayName) True) >>= \case
Left SEDuplicateName -> do
putStrLn "chosen display name is already used by another profile on this device, choose another one"
loop
Left e -> putStrLn ("database error " <> show e) >> exitFailure
Right user -> do
void . withTransaction st $ \db -> runExceptT $ createNoteFolder db user
pure user
selectUser :: [User] -> IO User
selectUser [user@User {userId}] = do
withTransaction st (`setActiveUser` userId)
pure user
selectUser users = do
putStrLn "Select user profile:"
forM_ (zip [1 ..] users) $ \(n :: Int, user) -> putStrLn $ show n <> " - " <> userStr user
loop
where
loop = do
nStr <- getWithPrompt $ "user profile number (1 .. " <> show (length users) <> ")"
case readMaybe nStr :: Maybe Int of
Nothing -> putStrLn "invalid user number" >> loop
Just n
| n <= 0 || n > length users -> putStrLn "invalid user number" >> loop
| otherwise -> do
let user@User {userId} = users !! (n - 1)
withTransaction st (`setActiveUser` userId)
pure user
userStr :: User -> String
userStr User {localDisplayName, profile = LocalProfile {fullName}} =
T.unpack $ localDisplayName <> if T.null fullName || localDisplayName == fullName then "" else " (" <> fullName <> ")"
getContactName :: IO ContactName
getContactName = do
displayName <- getWithPrompt "display name"
let validName = mkValidName displayName
if
| null displayName -> putStrLn "display name can't be empty" >> getContactName
| null validName -> putStrLn "display name is invalid, please choose another" >> getContactName
| displayName /= validName -> putStrLn ("display name is invalid, you could use this one: " <> validName) >> getContactName
| otherwise -> pure $ T.pack displayName
getWithPrompt :: String -> IO String
getWithPrompt s = putStr (s <> ": ") >> hFlush stdout >> getLine
withUser' :: ChatMonad m => (User -> m ChatResponse) -> m ChatResponse
withUser' action =
asks currentUser
+78 -3
View File
@@ -1,17 +1,34 @@
{-# LANGUAGE DuplicateRecordFields #-}
{-# LANGUAGE LambdaCase #-}
{-# LANGUAGE NamedFieldPuns #-}
{-# LANGUAGE OverloadedStrings #-}
module Simplex.Chat.Core where
module Simplex.Chat.Core
( simplexChatCore,
runSimplexChat,
sendChatCmdStr,
sendChatCmd,
)
where
import Control.Logger.Simple
import Control.Monad
import Control.Monad.Reader
import Data.List (find)
import qualified Data.Text as T
import Data.Text.Encoding (encodeUtf8)
import Data.Time.Clock (getCurrentTime)
import Data.Time.LocalTime (getCurrentTimeZone)
import Simplex.Chat
import Simplex.Chat.Controller
import Simplex.Chat.Options (ChatOpts (..), CoreChatOpts (..))
import Simplex.Chat.Store.Profiles
import Simplex.Chat.Types
import Simplex.Chat.View (serializeChatResponse)
import Simplex.Messaging.Agent.Store.SQLite (SQLiteStore, withTransaction)
import System.Exit (exitFailure)
import System.IO (hFlush, stdout)
import Text.Read (readMaybe)
import UnliftIO.Async
simplexChatCore :: ChatConfig -> ChatOpts -> (User -> ChatController -> IO ()) -> IO ()
@@ -27,8 +44,10 @@ simplexChatCore cfg@ChatConfig {confirmMigrations, testView} opts@ChatOpts {core
putStrLn $ "Error opening database: " <> show e
exitFailure
run db@ChatDatabase {chatStore} = do
u <- getCreateActiveUser chatStore testView
cc <- newChatController db (Just u) cfg opts False
u_ <- getSelectActiveUser chatStore
cc <- newChatController db u_ cfg opts False
u <- maybe (createActiveUser cc) pure u_
unless testView $ putStrLn $ "Current user: " <> userStr u
runSimplexChat opts u cc chat
runSimplexChat :: ChatOpts -> User -> ChatController -> (User -> ChatController -> IO ()) -> IO ()
@@ -44,3 +63,59 @@ sendChatCmdStr cc s = runReaderT (execChatCommand Nothing . encodeUtf8 $ T.pack
sendChatCmd :: ChatController -> ChatCommand -> IO ChatResponse
sendChatCmd cc cmd = runReaderT (execChatCommand' cmd) cc
getSelectActiveUser :: SQLiteStore -> IO (Maybe User)
getSelectActiveUser st = do
users <- withTransaction st getUsers
case find activeUser users of
Just u -> pure $ Just u
Nothing -> selectUser users
where
selectUser :: [User] -> IO (Maybe User)
selectUser = \case
[] -> pure Nothing
[user@User {userId}] -> do
withTransaction st (`setActiveUser` userId)
pure $ Just user
users -> do
putStrLn "Select user profile:"
forM_ (zip [1 :: Int ..] users) $ \(n, user) -> putStrLn $ show n <> ": " <> userStr user
loop
where
loop = do
nStr <- getWithPrompt $ "user number (1 .. " <> show (length users) <> ")"
case readMaybe nStr :: Maybe Int of
Nothing -> putStrLn "not a number" >> loop
Just n
| n <= 0 || n > length users -> putStrLn "invalid user number" >> loop
| otherwise -> do
let user@User {userId} = users !! (n - 1)
withTransaction st (`setActiveUser` userId)
pure $ Just user
createActiveUser :: ChatController -> IO User
createActiveUser cc = do
putStrLn
"No user profiles found, it will be created now.\n\
\Please choose your display name.\n\
\It will be sent to your contacts when you connect.\n\
\It is only stored on your device and you can change it later."
loop
where
loop = do
displayName <- T.pack <$> getWithPrompt "display name"
let profile = Just Profile {displayName, fullName = "", image = Nothing, contactLink = Nothing, preferences = Nothing}
execChatCommand' (CreateActiveUser NewUser {profile, sameServers = False, pastTimestamp = False}) `runReaderT` cc >>= \case
CRActiveUser user -> pure user
r -> do
ts <- getCurrentTime
tz <- getCurrentTimeZone
putStrLn $ serializeChatResponse (Nothing, Nothing) ts tz Nothing r
loop
getWithPrompt :: String -> IO String
getWithPrompt s = putStr (s <> ": ") >> hFlush stdout >> getLine
userStr :: User -> String
userStr User {localDisplayName, profile = LocalProfile {fullName}} =
T.unpack $ localDisplayName <> if T.null fullName || localDisplayName == fullName then "" else " (" <> fullName <> ")"
+3 -1
View File
@@ -1812,7 +1812,7 @@ viewChatError logLevel testView = \case
CEUserNotHidden _ -> ["user is not hidden"]
CEInvalidDisplayName {displayName, validName} ->
map plain $
["invalid display name: " <> viewName displayName]
[if T.null displayName then "display name can't be empty" else "invalid display name: " <> viewName displayName]
<> ["you could use this one: " <> viewName validName | not (T.null validName)]
CEChatNotStarted -> ["error: chat not started"]
CEChatNotStopped -> ["error: chat not stopped"]
@@ -1925,6 +1925,8 @@ viewChatError logLevel testView = \case
AGENT A_DUPLICATE -> [withConnEntity <> "error: AGENT A_DUPLICATE" | logLevel == CLLDebug]
AGENT A_PROHIBITED -> [withConnEntity <> "error: AGENT A_PROHIBITED" | logLevel <= CLLWarning]
CONN NOT_FOUND -> [withConnEntity <> "error: CONN NOT_FOUND" | logLevel <= CLLWarning]
CRITICAL restart e -> [plain $ "critical error: " <> e] <> ["please restart the app" | restart]
INTERNAL e -> [plain $ "internal error: " <> e]
e -> [withConnEntity <> "smp agent error: " <> sShow e | logLevel <= CLLWarning]
where
withConnEntity = case entity_ of
+1 -1
View File
@@ -15,7 +15,7 @@ import Simplex.Chat.Core
import Simplex.Chat.Options (CoreChatOpts (..))
import Simplex.Chat.Types (Profile (..))
import System.FilePath ((</>))
import Test.Hspec
import Test.Hspec hiding (it)
broadcastBotTests :: SpecWith FilePath
broadcastBotTests = do
+1 -1
View File
@@ -21,7 +21,7 @@ import Simplex.Chat.Core
import Simplex.Chat.Options (CoreChatOpts (..))
import Simplex.Chat.Types (GroupMemberRole (..), Profile (..))
import System.FilePath ((</>))
import Test.Hspec
import Test.Hspec hiding (it)
directoryServiceTests :: SpecWith FilePath
directoryServiceTests = do
+19 -16
View File
@@ -36,6 +36,7 @@ import Simplex.FileTransfer.Server.Env (XFTPServerConfig (..), defaultFileExpira
import Simplex.Messaging.Agent.Env.SQLite
import Simplex.Messaging.Agent.RetryInterval
import Simplex.Messaging.Agent.Store.SQLite (MigrationConfirmation (..))
import qualified Simplex.Messaging.Agent.Store.SQLite.DB as DB
import Simplex.Messaging.Client (ProtocolClientConfig (..), defaultNetworkConfig)
import Simplex.Messaging.Server (runSMPServerBlocking)
import Simplex.Messaging.Server.Env.STM
@@ -73,21 +74,22 @@ testOpts =
}
testCoreOpts :: CoreChatOpts
testCoreOpts = CoreChatOpts
{ dbFilePrefix = undefined,
dbKey = "",
-- dbKey = "this is a pass-phrase to encrypt the database",
smpServers = ["smp://LcJUMfVhwD8yxjAiSaDzzGF3-kLG4Uh0Fl_ZIjrRwjI=:server_password@localhost:7001"],
xftpServers = ["xftp://LcJUMfVhwD8yxjAiSaDzzGF3-kLG4Uh0Fl_ZIjrRwjI=:server_password@localhost:7002"],
networkConfig = defaultNetworkConfig,
logLevel = CLLImportant,
logConnections = False,
logServerHosts = False,
logAgent = Nothing,
logFile = Nothing,
tbqSize = 16,
highlyAvailable = False
}
testCoreOpts =
CoreChatOpts
{ dbFilePrefix = undefined,
dbKey = "",
-- dbKey = "this is a pass-phrase to encrypt the database",
smpServers = ["smp://LcJUMfVhwD8yxjAiSaDzzGF3-kLG4Uh0Fl_ZIjrRwjI=:server_password@localhost:7001"],
xftpServers = ["xftp://LcJUMfVhwD8yxjAiSaDzzGF3-kLG4Uh0Fl_ZIjrRwjI=:server_password@localhost:7002"],
networkConfig = defaultNetworkConfig,
logLevel = CLLImportant,
logConnections = False,
logServerHosts = False,
logAgent = Nothing,
logFile = Nothing,
tbqSize = 16,
highlyAvailable = False
}
getTestOpts :: Bool -> ScrubbedBytes -> ChatOpts
getTestOpts maintenance dbKey = testOpts {maintenance, coreOptions = testCoreOpts {dbKey}}
@@ -190,7 +192,8 @@ groupLinkViaContactVRange = mkVersionRange 1 2
createTestChat :: FilePath -> ChatConfig -> ChatOpts -> String -> Profile -> IO TestCC
createTestChat tmp cfg opts@ChatOpts {coreOptions = CoreChatOpts {dbKey}} dbPrefix profile = do
Right db@ChatDatabase {chatStore} <- createChatDatabase (tmp </> dbPrefix) dbKey False MCError
Right db@ChatDatabase {chatStore, agentStore} <- createChatDatabase (tmp </> dbPrefix) dbKey False MCError
withTransaction agentStore (`DB.execute_` "INSERT INTO users (user_id) VALUES (1);")
Right user <- withTransaction chatStore $ \db' -> runExceptT $ createUserRecord db' (AgentUserId 1) profile True
startTestChat_ db cfg opts user
+1 -1
View File
@@ -6,7 +6,7 @@ import ChatTests.Files
import ChatTests.Groups
import ChatTests.Local
import ChatTests.Profiles
import Test.Hspec
import Test.Hspec hiding (it)
chatTests :: SpecWith FilePath
chatTests = do
+1 -1
View File
@@ -4,7 +4,7 @@ import ChatClient
import ChatTests.Utils
import Data.Time.Clock (getCurrentTime)
import Data.Time.Format.ISO8601 (iso8601Show)
import Test.Hspec
import Test.Hspec hiding (it)
chatListTests :: SpecWith FilePath
chatListTests = do
+14 -1
View File
@@ -24,7 +24,7 @@ import qualified Simplex.Messaging.Crypto as C
import Simplex.Messaging.Version
import System.Directory (copyFile, doesDirectoryExist, doesFileExist)
import System.FilePath ((</>))
import Test.Hspec
import Test.Hspec hiding (it)
chatDirectTests :: SpecWith FilePath
chatDirectTests = do
@@ -1559,6 +1559,19 @@ testDeleteUser =
alice ##> "/create user alisa3"
showActiveUser alice "alisa3"
alice ##> "/delete user alisa3 del_smp=on"
alice <### ["ok", "completed deleting user"]
alice ##> "/users"
alice <## "no users"
alice ##> "/create user alisa4"
showActiveUser alice "alisa4"
connectUsers alice bob
alice <##> bob
alice ##> "/delete user alisa4 del_smp=on"
alice <### ["ok", "completed deleting user"]
alice ##> "/users"
alice <## "no users"
testUsersDifferentCIExpirationTTL :: HasCallStack => FilePath -> IO ()
testUsersDifferentCIExpirationTTL tmp = do
+1 -1
View File
@@ -21,7 +21,7 @@ import Simplex.Messaging.Crypto.File (CryptoFile (..), CryptoFileArgs (..))
import Simplex.Messaging.Encoding.String
import Simplex.Messaging.Util (unlessM)
import System.Directory (copyFile, createDirectoryIfMissing, doesFileExist, getFileSize)
import Test.Hspec
import Test.Hspec hiding (it)
chatFileTests :: SpecWith FilePath
chatFileTests = do
+1 -1
View File
@@ -19,7 +19,7 @@ import qualified Simplex.Messaging.Agent.Store.SQLite.DB as DB
import Simplex.Messaging.Version
import System.Directory (copyFile)
import System.FilePath ((</>))
import Test.Hspec
import Test.Hspec hiding (it)
chatGroupTests :: SpecWith FilePath
chatGroupTests = do
+1 -1
View File
@@ -11,7 +11,7 @@ import Data.Time.Format.ISO8601 (iso8601Show)
import Simplex.Chat.Controller (ChatConfig (..), InlineFilesConfig (..), defaultInlineFilesConfig)
import System.Directory (copyFile, doesFileExist)
import System.FilePath ((</>))
import Test.Hspec
import Test.Hspec hiding (it)
import UnliftIO.Async (concurrently_)
chatLocalChatsTests :: SpecWith FilePath
+1 -1
View File
@@ -16,7 +16,7 @@ import Simplex.Chat.Store.Shared (createContact)
import Simplex.Chat.Types (ConnStatus (..), GroupMemberRole (..), Profile (..))
import Simplex.Messaging.Encoding.String (StrEncoding (..))
import System.Directory (copyFile, createDirectoryIfMissing)
import Test.Hspec
import Test.Hspec hiding (it)
chatProfileTests :: SpecWith FilePath
chatProfileTests = do
+11 -3
View File
@@ -35,7 +35,9 @@ import System.Environment (lookupEnv, withArgs)
import System.FilePath ((</>))
import System.IO.Silently (capture_)
import System.Info (os)
import Test.Hspec
import Test.Hspec hiding (it)
import qualified Test.Hspec as Hspec
import UnliftIO (timeout)
defaultPrefs :: Maybe Preferences
defaultPrefs = Just $ toChatPrefs defaultChatPrefs
@@ -55,11 +57,17 @@ cathProfile = Profile {displayName = "cath", fullName = "Catherine", image = Not
danProfile :: Profile
danProfile = Profile {displayName = "dan", fullName = "Daniel", image = Nothing, contactLink = Nothing, preferences = defaultPrefs}
xit' :: (HasCallStack, Example a) => String -> a -> SpecWith (Arg a)
it :: HasCallStack => String -> (FilePath -> Expectation) -> SpecWith (Arg (FilePath -> Expectation))
it name test =
Hspec.it name $ \tmp -> timeout t (test tmp) >>= maybe (error "test timed out") pure
where
t = 90 * 1000000
xit' :: HasCallStack => String -> (FilePath -> Expectation) -> SpecWith (Arg (FilePath -> Expectation))
xit' = if os == "linux" then xit else it
xit'' :: (HasCallStack, Example a) => String -> a -> SpecWith (Arg a)
xit'' = ifCI xit it
xit'' = ifCI xit Hspec.it
xdescribe'' :: HasCallStack => String -> SpecWith a -> SpecWith a
xdescribe'' = ifCI xdescribe describe
+1 -1
View File
@@ -43,7 +43,7 @@ import Simplex.Messaging.Parsers (dropPrefix, sumTypeJSON)
import System.Directory (copyFile)
import System.FilePath ((</>))
import System.IO (utf8)
import Test.Hspec
import Test.Hspec hiding (it)
mobileTests :: HasCallStack => SpecWith FilePath
mobileTests = do
+1 -1
View File
@@ -21,7 +21,7 @@ import Simplex.Messaging.Crypto.File (CryptoFileArgs (..))
import Simplex.Messaging.Encoding.String (strEncode)
import Simplex.Messaging.Util
import System.FilePath ((</>))
import Test.Hspec
import Test.Hspec hiding (it)
import UnliftIO
import UnliftIO.Concurrent
import UnliftIO.Directory
+1 -1
View File
@@ -12,7 +12,7 @@ import MobileTests
import ProtocolTests
import RemoteTests
import SchemaDump
import Test.Hspec
import Test.Hspec hiding (it)
import UnliftIO.Temporary (withTempDirectory)
import ValidNames
import ViewTests