use chat protocol and contacts in chat commands/messages (#66)

* chat types, chat protocol syntax idea

* chat message syntax, raw message type

* chat message format and parsing

* raw chat message parsing test

* add message parsing tests

* interpret RawChatMessage

* use chat message format when sending messages

* save contacts and related connections to DB (WIP)

* use contacts in all chat commands (add, connect, send, delete)

* use contacts when receiving messages and notifications

* handle contact not found error

* automatically accept connection when CONF is received from the agent
This commit is contained in:
Evgeny Poberezkin
2021-07-04 18:42:24 +01:00
committed by GitHub
parent c3d5797a0b
commit 2f604d91ba
18 changed files with 903 additions and 179 deletions
+251 -28
View File
@@ -1,44 +1,267 @@
{-# LANGUAGE DataKinds #-}
{-# LANGUAGE DuplicateRecordFields #-}
{-# LANGUAGE GADTs #-}
{-# LANGUAGE KindSignatures #-}
{-# LANGUAGE LambdaCase #-}
{-# LANGUAGE NamedFieldPuns #-}
{-# LANGUAGE OverloadedStrings #-}
{-# LANGUAGE StandaloneDeriving #-}
module Simplex.Chat.Protocol where
import Data.ByteString (ByteString)
import Control.Applicative (optional, (<|>))
import Control.Monad.Except (throwError)
import Data.Attoparsec.ByteString.Char8 (Parser)
import qualified Data.Attoparsec.ByteString.Char8 as A
import Data.ByteString.Char8 (ByteString)
import qualified Data.ByteString.Char8 as B
import Data.Int (Int64)
import Data.List (findIndex)
import Data.Text (Text)
import Simplex.Messaging.Agent.Protocol (ConnId)
import Simplex.Chat.Types
import Simplex.Messaging.Agent.Protocol
import Simplex.Messaging.Parsers (base64P)
import Simplex.Messaging.Protocol (MsgBody)
import Simplex.Messaging.Util (bshow)
data ChatEvent = GroupEvent | MessageEvent | InfoEvent
data ChatTransmission
= ChatTransmission
{ agentMsgMeta :: MsgMeta,
chatDirection :: ChatDirection 'Agent,
chatMessage :: ChatMessage
}
| ChatTransmissionError
{ agentMsgMeta :: MsgMeta,
chatDirection :: ChatDirection 'Agent,
msgBody :: MsgBody,
msgError :: ByteString
}
| AgentTransmission
{ agentConnId :: ConnId,
chatDirection :: ChatDirection 'Agent,
agentMessage :: ACommand 'Agent
}
deriving (Eq, Show)
data Profile = Profile
{ profileId :: ByteString,
displayName :: Text
data ChatDirection (p :: AParty) where
ReceivedDirectMessage :: Contact -> ChatDirection 'Agent
SentDirectMessage :: Contact -> ChatDirection 'Client
ReceivedGroupMessage :: Group -> Contact -> ChatDirection 'Agent
SentGroupMessage :: Group -> ChatDirection 'Client
deriving instance Eq (ChatDirection p)
deriving instance Show (ChatDirection p)
newtype ChatMsgEvent = XMsgNew MessageType
deriving (Eq, Show)
data MessageType = MTText | MTImage deriving (Eq, Show)
toMsgType :: ByteString -> Either ByteString MessageType
toMsgType = \case
"c.text" -> Right MTText
"c.image" -> Right MTImage
t -> Left $ "invalid message type " <> t
rawMsgType :: MessageType -> ByteString
rawMsgType = \case
MTText -> "c.text"
MTImage -> "c.image"
data ChatMessage = ChatMessage
{ chatMsgId :: Maybe Int64,
chatMsgEvent :: ChatMsgEvent,
chatMsgBody :: [MsgBodyContent],
chatDAGIdx :: Maybe Int
}
deriving (Eq, Show)
data Contact = Contact
{ contactId :: ByteString,
profile :: Profile,
connections :: [Connection]
toChatMessage :: RawChatMessage -> Either ByteString ChatMessage
toChatMessage RawChatMessage {chatMsgId, chatMsgEvent, chatMsgParams, chatMsgBody} = do
body <- mapM toMsgBodyContent chatMsgBody
case chatMsgEvent of
"x.msg.new" -> case chatMsgParams of
[mt] -> do
t <- toMsgType mt
pure ChatMessage {chatMsgId, chatMsgEvent = XMsgNew t, chatMsgBody = body, chatDAGIdx = findDAG body}
_ -> throwError "x.msg.new expects one parameter"
_ -> throwError $ "unsupported event " <> chatMsgEvent
toChatMessage _ = Left "message continuation"
findDAG :: [MsgBodyContent] -> Maybe Int
findDAG = findIndex $ isContentType SimplexDAG
isContentType :: ContentType -> MsgBodyContent -> Bool
isContentType t MsgBodyContent {contentType = t'} = t == t'
isSimplexContentType :: XContentType -> MsgBodyContent -> Bool
isSimplexContentType = isContentType . SimplexContentType
rawChatMessage :: ChatMessage -> RawChatMessage
rawChatMessage ChatMessage {chatMsgId, chatMsgEvent = event, chatMsgBody = body} =
case event of
XMsgNew t ->
let chatMsgBody = map rawMsgBodyContent body
in RawChatMessage {chatMsgId, chatMsgEvent = "x.msg.new", chatMsgParams = [rawMsgType t], chatMsgBody}
toMsgBodyContent :: RawMsgBodyContent -> Either ByteString MsgBodyContent
toMsgBodyContent RawMsgBodyContent {contentType, contentHash, contentData} = do
cType <- toContentType contentType
pure MsgBodyContent {contentType = cType, contentHash, contentData}
rawMsgBodyContent :: MsgBodyContent -> RawMsgBodyContent
rawMsgBodyContent MsgBodyContent {contentType = t, contentHash, contentData} =
RawMsgBodyContent {contentType = rawContentType t, contentHash, contentData}
data MsgBodyContent = MsgBodyContent
{ contentType :: ContentType,
contentHash :: Maybe ByteString,
contentData :: MsgBodyPartData
}
deriving (Eq, Show)
data Connection = Connection
{ connId :: ConnId,
connLevel :: Int,
viaConn :: ConnId
data ContentType
= SimplexContentType XContentType
| MimeContentType MContentType
| SimplexDAG
deriving (Eq, Show)
data XContentType = XCText | XCImage deriving (Eq, Show)
data MContentType = MCImageJPG | MCImagePNG deriving (Eq, Show)
toContentType :: RawContentType -> Either ByteString ContentType
toContentType (RawContentType ns cType) = case ns of
"x" -> case cType of
"text" -> Right $ SimplexContentType XCText
"image" -> Right $ SimplexContentType XCImage
"dag" -> Right SimplexDAG
_ -> err
"m" -> case cType of
"image/jpg" -> Right $ MimeContentType MCImageJPG
"image/png" -> Right $ MimeContentType MCImagePNG
_ -> err
_ -> err
where
err = Left $ "invalid content type " <> ns <> "." <> cType
rawContentType :: ContentType -> RawContentType
rawContentType t = case t of
SimplexContentType t' -> RawContentType "x" $ case t' of
XCText -> "text"
XCImage -> "image"
MimeContentType t' -> RawContentType "m" $ case t' of
MCImageJPG -> "image/jpg"
MCImagePNG -> "image/png"
SimplexDAG -> RawContentType "x" "dag"
newtype ContentMsg = NewContentMsg ContentData
newtype ContentData = ContentText Text
data RawChatMessage
= RawChatMessage
{ chatMsgId :: Maybe Int64,
chatMsgEvent :: ByteString,
chatMsgParams :: [ByteString],
chatMsgBody :: [RawMsgBodyContent]
}
| RawChatMsgContinuation
{ prevChatMsgId :: Int64,
continuationId :: Int,
continuationData :: ByteString
}
deriving (Eq, Show)
data RawMsgBodyContent = RawMsgBodyContent
{ contentType :: RawContentType,
contentHash :: Maybe ByteString,
contentData :: MsgBodyPartData
}
deriving (Eq, Show)
data GroupMember = GroupMember
{ groupId :: ByteString,
sharedMemberId :: ByteString,
contact :: Contact,
memberRole :: GroupMemberRole,
memberStatus :: GroupMemberStatus
}
data RawContentType = RawContentType NameSpace ByteString
deriving (Eq, Show)
data GroupMemberRole = GROwner | GRAdmin | GRStandard
type NameSpace = ByteString
data GroupMemberStatus = GSInvited | GSConnected | GSConnectedAll
data MsgBodyPartData
= -- | fully loaded
MBFull MsgData
| -- | partially loaded
MBPartial Int MsgData
| -- | not loaded yet
MBEmpty Int
deriving (Eq, Show)
data Group = Group
{ groupId :: ByteString,
displayName :: Text,
members :: [GroupMember]
}
data MsgData
= MsgData ByteString
| MsgDataRec {dataId :: Int64, dataSize :: Int}
deriving (Eq, Show)
class DataLength a where
dataLength :: a -> Int
instance DataLength MsgBodyPartData where
dataLength (MBFull d) = dataLength d
dataLength (MBPartial l _) = l
dataLength (MBEmpty l) = l
instance DataLength MsgData where
dataLength (MsgData s) = B.length s
dataLength MsgDataRec {dataSize} = dataSize
rawChatMessageP :: Parser RawChatMessage
rawChatMessageP = A.char '#' *> chatMsgContP <|> chatMsgP
where
chatMsgContP :: Parser RawChatMessage
chatMsgContP = do
prevChatMsgId <- A.decimal <* A.char '.'
continuationId <- A.decimal <* A.space
continuationData <- A.takeByteString
pure RawChatMsgContinuation {prevChatMsgId, continuationId, continuationData}
chatMsgP :: Parser RawChatMessage
chatMsgP = do
chatMsgId <- optional A.decimal <* A.space
chatMsgEvent <- B.intercalate "." <$> identifier `A.sepBy1'` A.char '.' <* A.space
chatMsgParams <- A.takeWhile1 (not . A.inClass ", ") `A.sepBy'` A.char ',' <* A.space
chatMsgBody <- msgBodyContent =<< contentInfo `A.sepBy'` A.char ',' <* A.space
pure RawChatMessage {chatMsgId, chatMsgEvent, chatMsgParams, chatMsgBody}
identifier :: Parser ByteString
identifier = B.cons <$> A.letter_ascii <*> A.takeWhile (\c -> A.isAlpha_ascii c || A.isDigit c)
contentInfo :: Parser RawMsgBodyContent
contentInfo = do
contentType <- RawContentType <$> identifier <* A.char '.' <*> A.takeTill (A.inClass ":, ")
contentSize <- A.char ':' *> A.decimal
contentHash <- optional (A.char ':' *> base64P)
pure RawMsgBodyContent {contentType, contentHash, contentData = MBEmpty contentSize}
msgBodyContent :: [RawMsgBodyContent] -> Parser [RawMsgBodyContent]
msgBodyContent [] = pure []
msgBodyContent (p@RawMsgBodyContent {contentData = MBEmpty size} : ps) = do
s <- A.take size <* A.space <|> A.takeByteString
if B.length s == size
then ((p {contentData = MBFull $ MsgData s} :: RawMsgBodyContent) :) <$> msgBodyContent ps
else pure $ (if B.null s then p else p {contentData = MBPartial size $ MsgData s} :: RawMsgBodyContent) : ps
msgBodyContent _ = fail "expected contentData = MBEmpty"
serializeRawChatMessage :: RawChatMessage -> ByteString
serializeRawChatMessage = \case
RawChatMessage {chatMsgId, chatMsgEvent, chatMsgParams, chatMsgBody} ->
B.unwords
[ maybe "" bshow chatMsgId,
chatMsgEvent,
B.intercalate "," chatMsgParams,
B.unwords $ map serializeContentInfo chatMsgBody,
B.unwords $ map serializeContentData chatMsgBody
]
RawChatMsgContinuation {prevChatMsgId, continuationId, continuationData} ->
bshow prevChatMsgId <> "." <> bshow continuationId <> " " <> continuationData
serializeContentInfo :: RawMsgBodyContent -> ByteString
serializeContentInfo RawMsgBodyContent {contentType = RawContentType ns cType, contentHash, contentData} =
ns <> "." <> cType <> ":" <> bshow (dataLength contentData) <> maybe "" (":" <>) contentHash
serializeContentData :: RawMsgBodyContent -> ByteString
serializeContentData RawMsgBodyContent {contentData = MBFull (MsgData s)} = s
serializeContentData _ = ""
+44
View File
@@ -0,0 +1,44 @@
{-# LANGUAGE DuplicateRecordFields #-}
module Simplex.Chat.Protocol_ where
import Data.ByteString (ByteString)
import Data.Text (Text)
import Simplex.Messaging.Agent.Protocol (ConnId)
data ChatEvent = GroupEvent | MessageEvent | InfoEvent
data Profile = Profile
{ profileId :: ByteString,
displayName :: Text
}
data Contact = Contact
{ contactId :: ByteString,
profile :: Profile,
connections :: [Connection]
}
data Connection = Connection
{ connId :: ConnId,
connLevel :: Int,
viaConn :: ConnId
}
data GroupMember = GroupMember
{ groupId :: ByteString,
sharedMemberId :: ByteString,
contact :: Contact,
memberRole :: GroupMemberRole,
memberStatus :: GroupMemberStatus
}
data GroupMemberRole = GROwner | GRAdmin | GRStandard
data GroupMemberStatus = GSInvited | GSConnected | GSConnectedAll
data Group = Group
{ groupId :: ByteString,
displayName :: Text,
members :: [GroupMember]
}
+80
View File
@@ -0,0 +1,80 @@
{-# LANGUAGE DuplicateRecordFields #-}
{-# LANGUAGE LambdaCase #-}
{-# LANGUAGE OverloadedStrings #-}
module Simplex.Chat.Types where
import Data.ByteString.Char8 (ByteString)
import Data.Int (Int64)
import Data.Text (Text)
import Database.SQLite.Simple.FromField (FromField (..))
import Database.SQLite.Simple.ToField (ToField (..))
import Simplex.Messaging.Agent.Protocol (ConnId)
import Simplex.Messaging.Agent.Store.SQLite (fromTextField_)
data User = User
{ userId :: UserId,
profile :: Profile
}
type UserId = Int64
data Contact = Contact
{ contactId :: Int64,
localContactRef :: ContactRef,
profile :: Maybe Profile,
activeConn :: Connection
}
deriving (Eq, Show)
type ContactRef = Text
data Group = Group
{ groupId :: Int64,
localGroupRef :: Text
}
deriving (Eq, Show)
data Profile = Profile
{ profileId :: Int64,
contactRef :: ContactRef,
displayName :: Text
}
deriving (Eq, Show)
data Connection = Connection
{ connId :: Int64,
agentConnId :: ConnId,
connLevel :: Int,
viaContact :: Maybe Int64,
connStatus :: ConnStatus
}
deriving (Eq, Show)
data ConnStatus = ConnNew | ConnConfirmed | ConnAccepted | ConnReady
deriving (Eq, Show)
instance FromField ConnStatus where fromField = fromTextField_ connStatusT
instance ToField ConnStatus where toField = toField . serializeConnStatus
connStatusT :: Text -> Maybe ConnStatus
connStatusT = \case
"NEW" -> Just ConnNew
"CONF" -> Just ConnConfirmed
"ACPT" -> Just ConnAccepted
"READY" -> Just ConnReady
_ -> Nothing
serializeConnStatus :: ConnStatus -> Text
serializeConnStatus = \case
ConnNew -> "NEW"
ConnConfirmed -> "CONF"
ConnAccepted -> "ACPT"
ConnReady -> "READY"
data NewConnection = NewConnection
{ agentConnId :: ByteString,
connLevel :: Int,
viaConn :: Maybe Int64
}
+70
View File
@@ -0,0 +1,70 @@
# Chat protocol
## Design constraints
- the transport message has a fixed size (8 or 16kb)
- the chat message can have multiple parts/attachments
- the chat message including attachments can be of any size
- if the message is partially received, it should be possible to parse and display the received parts
## Questions
- should content types be:
- limited to MIME-types
- separate content types vocabulary
- both MIME types and extensions
- allow additional content types namespaces
## Message syntax
The syntax of the message inside agent MSG:
```abnf
agentMessageBody = message / msgContinuation
message = [chatMsgId] SP msgEvent SP [parameters] SP [contentParts [SP msgBodyParts]]
chatMsgId = 1*DIGIT ; used to refer to previous message;
; in the group should only be used in messages sent to all members,
; which is the main reason not to use external agent ID -
; some messages are sent only to one member
msgEvent = protocolNamespace 1*("." msgTypeName)
protocolNamespace = 1*ALPHA ; "x" for all events defined in the protocol
msgTypeName = 1*ALPHA
parameters = parameter *("," parameter)
parameter = 1*(%x21-2B / %x2D-7E) ; exclude control characters, space, comma (%x2C)
contentParts = contentPart *("," contentPart)
contentPart = contentTypeNamespace "." contentType ":" contentSize [":" contentHash]
contentType = "i." <mime-type> / contentTypeNamespace "." 1*("." contentTypeName)
contentTypeNamespace = 1*ALPHA
contentTypeName = 1*ALPHA
contentHash = <base64>
msgBodyParts = msgBodyPart *(SP msgBodyPart)
msgEventParents = msgEventParent *msgEventParent ; binary body part for content type "x.dag"
msgEventParent = memberId refMsgId refMsgHash
memberId = 8*8(OCTET) ; shared member ID
refMsgId = 8*8(OCTET) ; sequential message number - external agent message ID
refMsgHash = 16*16(OCTET) ; SHA256 of agent message body
msgContinuation = "#" prevMsgId "." continuationId continuationData
```
### Example: messages, updates, groups
```
"3 x.msg.new c.text c.text:5 hello "
"4 x.msg.new c.image i.image/jpg:256,i.image/png:4096 abcd abcd "
"4 x.msg.new c.image x.dag:32,i.image/jpg:8000:hash1,i.image/png:16000:hash2 binary1"
"#4.1 binary1end binary2"
"#4.2 binary2continued"
"#4.3 binary2end "
"5 x.msg.new c.image i.image/jpg:256,i.image/url:160 abcd https://media.example.com/asdf#abcd "
'6 x.msg.update 3 c.text:11,x.dag:16 hello there abcd '
'7 x.msg.delete 3'
'8 x.msg.new app/v1 i.text/html:NNN,i.text/css:NNN,c.js:NNN,c.json:NNN ... ... ... {...} '
'9 x.msg.eval 8 c.json:NNN {...} '
'10 x.msg.new c.text 2 c.text:16,x.dag:32 hello there @123 abcd '
' x.grp.mem.inv 23456,123 1 c.json NNN {...} '
' x.grp.mem.acpt 23456 1 c.text NNN <invitation> '
' x.grp.mem.intro 23456,234 1 c.json NNN {...} '
' x.grp.mem.inv 23456,234 1 c.text NNN <invitation> '
' x.grp.mem.req 23456,123 1 c.json NNN {...} '
' x.grp.mem.direct.inv 23456,234 1 text NNN <invitation> '
```