wip putIntoGroups refactor

* remove gaps

* consistent pagination size

* proper fix and around fix too

* optimize

* refactor

Dockerfile

@@ -29,7 +29,7 @@ RUN cp ./scripts/cabal.project.local.linux ./cabal.project.local
 
 # Compile simplex-chat
 RUN cabal update
-RUN cabal build exe:simplex-chat
+RUN cabal build exe:simplex-chat --constraint 'simplexmq +client_library'
 
 # Strip the binary from debug symbols to reduce size
 RUN bin=$(find /project/dist-newstyle -name "simplex-chat" -type f -executable) && \

apps/ios/Shared/Model/ChatModel.swift

@@ -58,6 +58,7 @@ class ItemsModel: ObservableObject {
     // this will cause reversedChatItems to be rendered without throttling
     @Published var isLoading = false
     @Published var showLoadingProgress = false
+    @State var anchors: [ChatItem.ID] = []
 
     init() {
         publisher

apps/ios/Shared/Model/SimpleXAPI.swift

@@ -318,10 +318,12 @@ private func apiChatsResponse(_ r: ChatResponse) throws -> [ChatData] {
     throw r
 }
 
-let loadItemsPerPage = 50
+let loadItemsPerPage = 100
+let preloadItem = 25
+let idealChatListSize = 300
 
 func apiGetChat(type: ChatType, id: Int64, search: String = "") async throws -> Chat {
-    let r = await chatSendCmd(.apiGetChat(type: type, id: id, pagination: .last(count: loadItemsPerPage), search: search))
+    let r = await chatSendCmd(.apiGetChat(type: type, id: id, pagination: .initial(count: loadItemsPerPage), search: search))
     if case let .apiChat(_, chat) = r { return Chat.init(chat) }
     throw r
 }
@@ -329,6 +331,15 @@ func apiGetChat(type: ChatType, id: Int64, search: String = "") async throws ->
 func apiGetChatItems(type: ChatType, id: Int64, pagination: ChatPagination, search: String = "") async throws -> [ChatItem] {
     let r = await chatSendCmd(.apiGetChat(type: type, id: id, pagination: pagination, search: search))
     if case let .apiChat(_, chat) = r { return chat.chatItems }
+    if case .chatCmdError(_, _) = r {
+        if case .chatError(_, let chatError) = r {
+            if case .errorStore(let storeError) = chatError {
+                if case .chatItemNotFound(_) = storeError {
+                    itemNotFoundAlert()
+                }
+            }
+        }
+    }
     throw r
 }
 
@@ -339,7 +350,9 @@ func loadChat(chat: Chat, search: String = "", clearItems: Bool = true) async {
         let im = ItemsModel.shared
         m.chatItemStatuses = [:]
         if clearItems {
-            await MainActor.run { im.reversedChatItems = [] }
+            await MainActor.run {
+                im.reversedChatItems = []
+            }
         }
         let chat = try await apiGetChat(type: cInfo.chatType, id: cInfo.apiId, search: search)
         await MainActor.run {
@@ -418,6 +431,13 @@ func apiCreateChatItems(noteFolderId: Int64, composedMessages: [ComposedMessage]
     return nil
 }
 
+func itemNotFoundAlert() {
+    AlertManager.shared.showAlertMsg(
+        title: "Message no longer available",
+        message: "The quoted message you are trying to view has been deleted."
+    )
+}
+
 private func sendMessageErrorAlert(_ r: ChatResponse) {
     logger.error("send message error: \(String(describing: r))")
     AlertManager.shared.showAlertMsg(

apps/ios/Shared/Views/Chat/ChatItem/FramedItemView.swift

@@ -48,10 +48,18 @@ struct FramedItemView: View {
                 if let qi = chatItem.quotedItem {
                     ciQuoteView(qi)
                         .onTapGesture {
-                            if let ci = ItemsModel.shared.reversedChatItems.first(where: { $0.id == qi.itemId }) {
-                                withAnimation {
-                                    scrollModel.scrollToItem(id: ci.id)
+                            if let itemId = qi.itemId {
+                                if !scrollToItem(itemId) {
+                                    Task {
+                                        //if await loadItemsAround(chat.chatInfo, itemId) != nil {
+                                          //  await MainActor.run {
+                                            //    let _ = scrollToItem(itemId)
+                                            //}
+                                        //}
+                                    }
                                 }
+                            } else {
+                                itemNotFoundAlert()
                             }
                         }
                 } else if let itemForwarded = chatItem.meta.itemForwarded {
@@ -323,6 +331,18 @@ struct FramedItemView: View {
             return videoWidth
         }
     }
+    
+    // Scroll to an item, if success returns true otherwise false
+    private func scrollToItem(_ itemId: Int64) -> Bool {
+        if let ci = ItemsModel.shared.reversedChatItems.first(where: { $0.id == itemId }) {
+            withAnimation {
+                scrollModel.scrollToItem(id: ci.id)
+            }
+            return true
+        } else {
+            return false
+        }
+    }    
 }
 
 @ViewBuilder func toggleSecrets<V: View>(_ ft: [FormattedText]?, _ showSecrets: Binding<Bool>, _ v: V) -> some View {

apps/ios/Shared/Views/Chat/ChatItemGroups.swift

@@ -0,0 +1,219 @@
+//
+//  ChatItemGroups.swift
+//  SimpleX (iOS)
+//
+//  Created by Diogo Cunha on 11/11/2024.
+//  Copyright © 2024 SimpleX Chat. All rights reserved.
+//
+
+import Foundation
+import SwiftUI
+import SimpleXChat
+
+/// Represents an anchor  in a list of chat items, indicating where data is missing and should be loaded.
+///
+/// - Parameters:
+///   - itemId: The unique identifier of the last item in the loaded list before the anchor.
+///     This ID corresponds to an item in the chat history, ordered from older to newer items.
+///     It is typically used when loading items via .around or .initial pagination when loading items
+///   - indexRange: The range of indexes within `reversedChatItems` array that
+///     represents the anchor. The first index in this range is the position of the anchor itself.
+///     For instance, if the array `[0, 1, 2, -100-, 101]` has an anchor at index 3, `indexRange`
+///     would be `3..<5`, indicating the anchor starts at index 3.
+///   - indexRangeInParentItems: The range of indexes in the `ReverseList` or parent UI component
+///     that considers revealed or hidden items, showing where the anchor appears in the visible list.
+///     The first index in this range points to where the anchor starts in the UI.
+struct AnchoredRange {
+    let itemId: Int64
+    let indexRange: Range<Int>
+    let indexRangeInParentItems: Range<Int>
+}
+struct SectionGroups {
+    let sections: [SectionItems]
+    let anchoredRanges: [AnchoredRange]
+}
+
+struct ListItem: Hashable, Equatable {
+    let item: ChatItem
+    let separation: ItemSeparation
+    let prevItemSeparationLargeGap: Bool
+}
+
+class SectionItems: ObservableObject {    
+    var mergeCategory: CIMergeCategory?
+    var items: [ListItem]
+    @Published var revealed: Bool
+    var showAvatar: Set<Int64>
+    var startIndexInParentItems: Int
+
+    init(mergeCategory: CIMergeCategory?, items: [ListItem], revealed: Bool, showAvatar: Set<Int64>, startIndexInParentItems: Int) {
+        self.mergeCategory = mergeCategory
+        self.items = items
+        self.revealed = revealed
+        self.showAvatar = showAvatar
+        self.startIndexInParentItems = startIndexInParentItems
+    }
+
+    func reveal(_ reveal: Bool, revealedItems: inout Set<Int64>) {
+        if reveal {
+            for item in items {
+                revealedItems.insert(item.item.id)
+            }
+        } else {
+            for item in items {
+                revealedItems.remove(item.item.id)
+            }
+        }
+        self.revealed = reveal
+    }
+}
+
+func putIntoGroups(chatItems: [ChatItem], revealedItems: Set<ChatItem.ID>, itemAnchors: Array<ChatItem.ID>) -> SectionGroups {
+    guard !chatItems.isEmpty else { return SectionGroups(sections: [], anchoredRanges: []) }
+
+    var groups: [SectionItems] = []
+    var anchoredRanges: [AnchoredRange] = []
+    var index = 0
+    var unclosedAnchorIndex: Int?
+    var unclosedAnchorIndexInParent: Int?
+    var unclosedAnchorItemId: Int64?
+    var visibleItemIndexInParent = -1
+    var recent: SectionItems?
+
+    while index < chatItems.count {
+        let item = chatItems[index]
+        let next = index + 1 < chatItems.count ? chatItems[index + 1] : nil
+        let category = item.mergeCategory
+        let itemIsAnchor = itemAnchors.contains(item.id)
+
+        let itemSeparation: ItemSeparation
+        let prevItemSeparationLargeGap: Bool
+
+        if let recentSection = recent, index > 0, recentSection.mergeCategory == category, !itemIsAnchor {
+            if recentSection.revealed {
+                let prev = index > 0 ? chatItems[index - 1] : nil
+                itemSeparation = getItemSeparation(item, at: index)
+                let nextForGap = (category != nil && category == prev?.mergeCategory) || index + 1 == chatItems.count ? nil : next
+                prevItemSeparationLargeGap = nextForGap == nil ? false : getItemSeparationLargeGap(item, at: index)
+                visibleItemIndexInParent += 1
+            } else {
+                itemSeparation = getItemSeparation(item, at: index)
+                prevItemSeparationLargeGap = false
+            }
+
+            let listItem = ListItem(item: item, separation: itemSeparation, prevItemSeparationLargeGap: prevItemSeparationLargeGap)
+            recentSection.items.append(listItem)
+            if shouldShowAvatar(current: item, older: next) {
+                recentSection.showAvatar.insert(item.id)
+            }
+        } else {
+            let revealed = item.mergeCategory == nil || revealedItems.contains(item.id)
+            visibleItemIndexInParent += 1
+
+            if revealed {
+                let prev = index > 0 ? chatItems[index - 1] : nil
+                itemSeparation = getItemSeparation(item, at: index)
+                let nextForGap = (category != nil && category == prev?.mergeCategory) || index + 1 == chatItems.count ? nil : next
+                prevItemSeparationLargeGap = nextForGap == nil ? false : getItemSeparationLargeGap(item, at: index)
+            } else {
+                itemSeparation = getItemSeparation(item, at: index)
+                prevItemSeparationLargeGap = false
+            }
+
+            let listItem = ListItem(item: item, separation: itemSeparation, prevItemSeparationLargeGap: prevItemSeparationLargeGap)
+            let newSection = SectionItems(
+                mergeCategory: item.mergeCategory,
+                items: [listItem],
+                revealed: revealed,
+                showAvatar: shouldShowAvatar(current: item, older: next) ? [item.id] : [],
+                startIndexInParentItems: visibleItemIndexInParent
+            )
+            groups.append(newSection)
+            recent = newSection
+        }
+
+        if itemIsAnchor {
+            if let unclosedIndex = unclosedAnchorIndex, let unclosedId = unclosedAnchorItemId, let unclosedIndexInParent = unclosedAnchorIndexInParent {
+                anchoredRanges.append(
+                    AnchoredRange(
+                        itemId: unclosedId,
+                        indexRange: unclosedIndex..<index,
+                        indexRangeInParentItems: unclosedIndexInParent..<visibleItemIndexInParent
+                    )
+                )
+            }
+            unclosedAnchorIndex = index
+            unclosedAnchorIndexInParent = visibleItemIndexInParent
+            unclosedAnchorItemId = item.id
+        } else if index + 1 == chatItems.count, let unclosedIndex = unclosedAnchorIndex, let unclosedId = unclosedAnchorItemId, let unclosedIndexInParent = unclosedAnchorIndexInParent {
+            anchoredRanges.append(
+                AnchoredRange(
+                    itemId: unclosedId,
+                    indexRange: unclosedIndex..<index + 1,
+                    indexRangeInParentItems: unclosedIndexInParent..<visibleItemIndexInParent + 1
+                )
+            )
+        }
+        index += 1
+    }
+    
+    return SectionGroups(sections: groups, anchoredRanges: anchoredRanges)
+}
+
+func getItemSectionItems(sections: Array<SectionItems>, itemId: ChatItem.ID) -> SectionItems? {
+    for sec in sections {
+        if sec.items.firstIndex(where: { $0.item.id == itemId }) != nil {
+            return sec
+        }
+    }
+    return nil
+}
+
+func getIndexInParentItems(sections: Array<SectionItems>, itemId: ChatItem.ID) -> Int {
+    for sec in sections {
+        if let index = sec.items.firstIndex(where: { $0.item.id == itemId }) {
+            return sec.startIndexInParentItems + (sec.revealed ? index : 0)
+        }
+    }
+    return -1
+}
+
+func getNewestItemAtParentIndexOrNull(sections: [SectionItems], parentIndex: Int) -> ChatItem? {
+    for group in sections {
+        let range = group.startIndexInParentItems...(group.startIndexInParentItems + group.items.count - 1)
+        if range.contains(parentIndex) {
+            if group.revealed {
+                return group.items[parentIndex - group.startIndexInParentItems].item
+            } else {
+                return group.items.first?.item
+            }
+        }
+    }
+    return nil
+}
+
+
+private func shouldShowAvatar(current: ChatItem, older: ChatItem?) -> Bool {
+    if case let .groupRcv(currentMember) = current.chatDir {
+        if let older = older, case let .groupRcv(olderMember) = older.chatDir {
+            return olderMember.memberId != currentMember.memberId
+        }
+        return true // Show avatar if there is no older item or if older is not a GroupRcv
+    }
+    return false
+}
+
+private func getItemSeparationLargeGap(_ chatItem: ChatItem, at index: Int?) -> Bool {
+    let im = ItemsModel.shared
+
+    if let index = index, index > 0, index < im.reversedChatItems.count {
+        let nextItem = im.reversedChatItems[index - 1]
+        let sameMemberAndDirection = nextItem.chatDir.sameDirection(chatItem.chatDir)
+        
+        // Return true if they are not the same direction or the time interval is more than 60 seconds.
+        return !sameMemberAndDirection || nextItem.meta.itemTs.timeIntervalSince(chatItem.meta.itemTs) > 60
+    } else {
+        // If there is no next item or it's out of bounds, consider it a large gap.
+        return true
+    }
+}

apps/ios/Shared/Views/Chat/ChatView.swift

@@ -13,6 +13,27 @@ import Combine
 
 private let memberImageSize: CGFloat = 34
 
+struct ItemSeparation: Equatable, Hashable {
+    let timestamp: Bool;
+    let largeGap: Bool;
+    let date: Date?
+}
+
+func getItemSeparation(_ chatItem: ChatItem, at i: Int?) -> ItemSeparation {
+    let im = ItemsModel.shared
+    if let i, i > 0 && im.reversedChatItems.count >= i {
+        let nextItem = im.reversedChatItems[i - 1]
+        let largeGap = !nextItem.chatDir.sameDirection(chatItem.chatDir) || nextItem.meta.itemTs.timeIntervalSince(chatItem.meta.itemTs) > 60
+        return ItemSeparation(
+            timestamp: largeGap || formatTimestampMeta(chatItem.meta.itemTs) != formatTimestampMeta(nextItem.meta.itemTs),
+            largeGap: largeGap,
+            date: Calendar.current.isDate(chatItem.meta.itemTs, inSameDayAs: nextItem.meta.itemTs) ? nil : nextItem.meta.itemTs
+        )
+    } else {
+        return ItemSeparation(timestamp: true, largeGap: true, date: nil)
+    }
+}
+
 struct ChatView: View {
     @EnvironmentObject var chatModel: ChatModel
     @ObservedObject var im = ItemsModel.shared
@@ -32,7 +53,6 @@ struct ChatView: View {
     @State private var connectionCode: String?
     @State private var loadingItems = false
     @State private var firstPage = false
-    @State private var revealedChatItem: ChatItem?
     @State private var searchMode = false
     @State private var searchText: String = ""
     @FocusState private var searchFocussed
@@ -46,6 +66,9 @@ struct ChatView: View {
     @State private var selectedChatItems: Set<Int64>? = nil
     @State private var showDeleteSelectedMessages: Bool = false
     @State private var allowToDeleteSelectedMessagesForAll: Bool = false
+    @State private var initialChatItem: ChatItem? = nil
+    @State private var revealedItems: Set<ChatItem.ID> = []
+    @State private var anchors: Array<ChatItem.ID> = []
 
     @AppStorage(DEFAULT_TOOLBAR_MATERIAL) private var toolbarMaterial = ToolbarMaterial.defaultMaterial
 
@@ -167,6 +190,7 @@ struct ChatView: View {
         }
         .onChange(of: chatModel.chatId) { cId in
             showChatInfoSheet = false
+            firstPage = false
             selectedChatItems = nil
             scrollModel.scrollToBottom()
             stopAudioPlayer()
@@ -180,7 +204,7 @@ struct ChatView: View {
                 dismiss()
             }
         }
-        .onChange(of: revealedChatItem) { _ in
+        .onChange(of: revealedItems.count) { _ in
             NotificationCenter.postReverseListNeedsLayout()
         }
         .onChange(of: im.isLoading) { isLoading in
@@ -362,7 +386,18 @@ struct ChatView: View {
                 await markChatUnread(chat, unreadChat: false)
             }
         }
+        if im.reversedChatItems.count == loadItemsPerPage {
+            loadChatItems(chat.chatInfo, .last(count: loadItemsPerPage))
+        }
+
         ChatView.FloatingButtonModel.shared.totalUnread = chat.chatStats.unreadCount
+        Task {
+            DispatchQueue.main.async {
+                if let firstunreadItem = self.getFirstUnreadItem() {
+                    initialChatItem = firstunreadItem
+                }
+            }
+        }
     }
 
     private func searchToolbar() -> some View {
@@ -416,9 +451,10 @@ struct ChatView: View {
 
     private func chatItemsList() -> some View {
         let cInfo = chat.chatInfo
-        let mergedItems = filtered(im.reversedChatItems)
+        let groups = putIntoGroups(chatItems: im.reversedChatItems, revealedItems: self.revealedItems, itemAnchors: self.anchors)
         return GeometryReader { g in
-            ReverseList(items: mergedItems, scrollState: $scrollModel.state) { ci in
+            ReverseList(groups: groups, scrollState: $scrollModel.state, initialChatItem: $initialChatItem) { li in
+                let ci = li.item
                 let voiceNoFrame = voiceWithoutFrame(ci)
                 let maxWidth = cInfo.chatType == .group
                                 ? voiceNoFrame
@@ -433,13 +469,19 @@ struct ChatView: View {
                     maxWidth: maxWidth,
                     composeState: $composeState,
                     selectedMember: $selectedMember,
-                    revealedChatItem: $revealedChatItem,
+                    revealedChatItems: $revealedItems,
                     selectedChatItems: $selectedChatItems,
-                    forwardedChatItems: $forwardedChatItems
+                    forwardedChatItems: $forwardedChatItems,
+                    onReveal: { revealState in
+                        if let sec = getItemSectionItems(sections: groups.sections, itemId: ci.id) {
+                            sec.reveal(revealState, revealedItems: &self.revealedItems)
+                        }
+                    }
                 )
                 .id(ci.id) // Required to trigger `onAppear` on iOS15
-            } loadPage: {
-                loadChatItems(cInfo)
+                
+            } loadPage: { pagination in
+                loadChatItems(cInfo, pagination)
             }
             .opacity(ItemsModel.shared.isLoading ? 0 : 1)
             .padding(.vertical, -InvertedTableView.inset)
@@ -471,6 +513,23 @@ struct ChatView: View {
             EmptyView()
         }
     }
+    
+    private func getFirstUnreadItem() -> ChatItem? {
+        var maybeItem: ChatItem? = nil
+        for i in stride(from: im.reversedChatItems.count - 1, through: 0, by: -1) {
+            let item = im.reversedChatItems[i]
+            if item.isRcvNew {
+                if item.mergeCategory == nil {
+                    return maybeItem ?? item
+                } else {
+                    maybeItem = item
+                }
+            } else if maybeItem != nil {
+                return maybeItem
+            }
+        }
+        return nil
+    }
 
     class FloatingButtonModel: ObservableObject {
         static let shared = FloatingButtonModel()
@@ -478,17 +537,24 @@ struct ChatView: View {
         @Published var isNearBottom: Bool = true
         @Published var date: Date?
         @Published var isDateVisible: Bool = false
+        @Published var bottomItemIndex: Int = 0
         var totalUnread: Int = 0
         var isReallyNearBottom: Bool = true
         var hideDateWorkItem: DispatchWorkItem?
 
         func updateOnListChange(_ listState: ListState) {
             let im = ItemsModel.shared
-            let unreadBelow =
+            let bottomItemIndex =
                 if let id = listState.bottomItemId,
-                   let index = im.reversedChatItems.firstIndex(where: { $0.id == id })
-                {
-                 im.reversedChatItems[..<index].reduce(into: 0) { unread, chatItem in
+                   let index = im.reversedChatItems.firstIndex(where: { $0.id == id }) {
+                    index
+                } else {
+                    -1
+                }
+            
+            var unreadBelow =
+                if bottomItemIndex != -1 {
+                 im.reversedChatItems[..<bottomItemIndex].reduce(into: 0) { unread, chatItem in
                      if chatItem.isRcvNew { unread += 1 }
                  }
                 } else {
@@ -508,6 +574,7 @@ struct ChatView: View {
                 it.unreadBelow = unreadBelow
                 it.date = date
                 it.isReallyNearBottom = listState.scrollOffset > 0 && listState.scrollOffset < 500
+                it.bottomItemIndex = bottomItemIndex
             }
             
             // set floating button indication mode
@@ -839,38 +906,133 @@ struct ChatView: View {
         }
     }
 
-    private func loadChatItems(_ cInfo: ChatInfo) {
+    private func loadChatItems(_ cInfo: ChatInfo, _ pagination: ChatPagination = .initial(count: loadItemsPerPage)) {
         Task {
-            if loadingItems || firstPage { return }
+            if loadingItems { return }
             loadingItems = true
+            
             do {
-                var reversedPage = Array<ChatItem>()
-                var chatItemsAvailable = true
-                // Load additional items until the page is +50 large after merging
-                while chatItemsAvailable && filtered(reversedPage).count < loadItemsPerPage {
-                    let pagination: ChatPagination =
-                        if let lastItem = reversedPage.last ?? im.reversedChatItems.last {
-                            .before(chatItemId: lastItem.id, count: loadItemsPerPage)
-                        } else {
-                            .last(count: loadItemsPerPage)
+                let chatItems = try await apiGetChatItems(
+                    type: cInfo.chatType,
+                    id: cInfo.apiId,
+                    pagination: pagination,
+                    search: searchText
+                )
+                
+                if (cInfo.id != chatModel.chatId) {
+                    await MainActor.run { loadingItems = false }
+                    return
+                }
+                
+                let im = ItemsModel.shared
+                var newItems = im.reversedChatItems
+                
+                switch pagination {
+                case .last:
+                    await MainActor.run {
+                        let newItemIds = Set(chatItems.map { $0.id })
+                        var duplicateFound = false
+                        newItems.removeAll {
+                            let isDuplicate = newItemIds.contains($0.id)
+                            duplicateFound = duplicateFound || isDuplicate
+                            return isDuplicate
                         }
-                    let chatItems = try await apiGetChatItems(
-                        type: cInfo.chatType,
-                        id: cInfo.apiId,
-                        pagination: pagination,
-                        search: searchText
-                    )
-                    chatItemsAvailable = !chatItems.isEmpty
-                    reversedPage.append(contentsOf: chatItems.reversed())
-                }
-                await MainActor.run {
-                    if reversedPage.count == 0 {
-                        firstPage = true
-                    } else {
-                        im.reversedChatItems.append(contentsOf: reversedPage)
+                        
+                        if !duplicateFound {
+                            if let existingItem = im.reversedChatItems.first {
+                                anchors = [existingItem.id]
+                            }
+                        }
+                        
+                        newItems.insert(contentsOf: chatItems.reversed(), at: 0)
+                        im.reversedChatItems = newItems
+                        loadingItems = false
+                    }
+                case .initial:
+                    await MainActor.run {
+                        im.reversedChatItems = chatItems.reversed()
+                        anchors = []
+                        loadingItems = false
+                    }
+                case let .after(chatItemId, _):
+                    guard let indexInCurrentItems = im.reversedChatItems.firstIndex(where: { $0.id == chatItemId }) else {
+                        return
+                    }
+                    
+                    let wasSize = newItems.count
+                    let newItemIds = Set(chatItems.map { $0.id })
+                    let indexInAnchors = anchors.firstIndex { $0 == chatItemId }
+                    var anchorAfterChatItem: [Int64] = []
+                    if let indexInAnchors = indexInAnchors, indexInAnchors + 1 <= anchors.count {
+                        anchorAfterChatItem = Array(anchors[indexInAnchors + 1..<anchors.count])
+                    }
+                    var anchorsToRemove = Set<Int64>()
+                    var reachedBottom: Bool = false
+                    
+                    newItems.removeAll { item in
+                        let isDuplicate = newItemIds.contains(item.id)
+                        if indexInAnchors != nil && newItemIds.contains(item.id) {
+                            if anchorAfterChatItem.contains(item.id) {
+                                anchorAfterChatItem.removeAll { $0 == item.id }
+                                anchorsToRemove.insert(item.id)
+                            } else if reachedBottom == false && anchorAfterChatItem.isEmpty {
+                                // We passed all anchors and found a duplicated item below all of them, indicating no more anchors below the loaded items.
+                                reachedBottom = true
+                            }
+                        }
+                        return isDuplicate
+                    }
+                    
+                    let insertAt = indexInCurrentItems - (wasSize - newItems.count)
+                    newItems.insert(contentsOf: chatItems.reversed(), at: insertAt)
+                    
+                    await MainActor.run {
+                        im.reversedChatItems = newItems
+                        var newAnchors = anchors.filter { !anchorsToRemove.contains($0) }
+                        
+                        if reachedBottom {
+                            newAnchors = []
+                        } else {
+                            if let enlargedAnchorIndex = anchors.firstIndex(where: { $0 == chatItemId }) {
+                                // Move the anchor to the end of the loaded items.
+                                newAnchors[enlargedAnchorIndex] = chatItems.last?.id ?? newAnchors[enlargedAnchorIndex]
+                            }
+                        }
+                        
+                        anchors = newAnchors
+                        loadingItems = false
+                    }
+                case let .before(chatItemId, _):
+                    guard let indexInCurrentItems = im.reversedChatItems.firstIndex(where: { $0.id == chatItemId }) else {
+                        return
+                    }
+                    let newItemIds = Set(chatItems.map { $0.id })
+                    newItems.removeAll { newItemIds.contains($0.id) }
+                    newItems.insert(contentsOf: chatItems.reversed(), at: min(indexInCurrentItems + 1, newItems.count))
+                                        
+                    await MainActor.run {
+                        if chatItems.count == 0 || newItems.count == im.reversedChatItems.count  {
+                            firstPage = true
+                        } else {
+                            im.reversedChatItems = newItems
+                        }
+                        anchors = anchors.filter { !newItemIds.contains($0) }
+                        loadingItems = false
+                    }
+                case .around(_, _):
+                    let newItemIds = Set(chatItems.map { $0.id })
+                    newItems.removeAll { newItemIds.contains($0.id) }
+                    newItems.insert(contentsOf: chatItems, at: 0)
+                    
+                    await MainActor.run {
+                        im.reversedChatItems = newItems
+                        if let lastItemId = chatItems.last?.id {
+                            anchors.insert(lastItemId, at: 0)
+                        }
+                        loadingItems = false
                     }
-                    loadingItems = false
                 }
+                
             } catch let error {
                 logger.error("apiGetChat error: \(responseError(error))")
                 await MainActor.run { loadingItems = false }
@@ -892,7 +1054,7 @@ struct ChatView: View {
         let maxWidth: CGFloat
         @Binding var composeState: ComposeState
         @Binding var selectedMember: GMember?
-        @Binding var revealedChatItem: ChatItem?
+        @Binding var revealedChatItems: Set<ChatItem.ID>
 
         @State private var deletingItem: ChatItem? = nil
         @State private var showDeleteMessage = false
@@ -907,25 +1069,8 @@ struct ChatView: View {
 
         @State private var allowMenu: Bool = true
         @State private var markedRead = false
-
-        var revealed: Bool { chatItem == revealedChatItem }
-
-        typealias ItemSeparation = (timestamp: Bool, largeGap: Bool, date: Date?)
-
-        func getItemSeparation(_ chatItem: ChatItem, at i: Int?) -> ItemSeparation {
-            let im = ItemsModel.shared
-            if let i, i > 0 && im.reversedChatItems.count >= i {
-                let nextItem = im.reversedChatItems[i - 1]
-                let largeGap = !nextItem.chatDir.sameDirection(chatItem.chatDir) || nextItem.meta.itemTs.timeIntervalSince(chatItem.meta.itemTs) > 60
-                return (
-                    timestamp: largeGap || formatTimestampMeta(chatItem.meta.itemTs) != formatTimestampMeta(nextItem.meta.itemTs),
-                    largeGap: largeGap,
-                    date: Calendar.current.isDate(chatItem.meta.itemTs, inSameDayAs: nextItem.meta.itemTs) ? nil : nextItem.meta.itemTs
-                )
-            } else {
-                return (timestamp: true, largeGap: true, date: nil)
-            }
-        }
+        let onReveal: (Bool) -> Void
+        var revealed: Bool { revealedChatItems.contains(chatItem.id) }
 
         var body: some View {
             let currIndex = m.getChatItemIndex(chatItem)
@@ -933,46 +1078,7 @@ struct ChatView: View {
             let (prevHidden, prevItem) = m.getPrevShownChatItem(currIndex, ciCategory)
             let range = itemsRange(currIndex, prevHidden)
             let timeSeparation = getItemSeparation(chatItem, at: currIndex)
-            let im = ItemsModel.shared
-            Group {
-                if revealed, let range = range {
-                    let items = Array(zip(Array(range), im.reversedChatItems[range]))
-                    VStack(spacing: 0) {
-                        ForEach(items.reversed(), id: \.1.viewId) { (i: Int, ci: ChatItem) in
-                            let prev = i == prevHidden ? prevItem : im.reversedChatItems[i + 1]
-                            chatItemView(ci, nil, prev, getItemSeparation(ci, at: i))
-                                .overlay {
-                                    if let selected = selectedChatItems, ci.canBeDeletedForSelf {
-                                        Color.clear
-                                            .contentShape(Rectangle())
-                                            .onTapGesture {
-                                                let checked = selected.contains(ci.id)
-                                                selectUnselectChatItem(select: !checked, ci)
-                                            }
-                                    }
-                                }
-                        }
-                    }
-                } else {
-                    VStack(spacing: 0) {
-                        chatItemView(chatItem, range, prevItem, timeSeparation)
-                        if let date = timeSeparation.date {
-                            DateSeparator(date: date).padding(8)
-                        }
-                    }
-                    .overlay {
-                        if let selected = selectedChatItems, chatItem.canBeDeletedForSelf {
-                            Color.clear
-                                .contentShape(Rectangle())
-                                .onTapGesture {
-                                    let checked = selected.contains(chatItem.id)
-                                    selectUnselectChatItem(select: !checked, chatItem)
-                                }
-                        }
-                    }
-                }
-            }
-            .onAppear {
+            func markAsRead() {
                 if markedRead {
                     return
                 } else {
@@ -991,6 +1097,30 @@ struct ChatView: View {
                     }
                 }
             }
+            return Group {
+                VStack(spacing: 0) {
+                    chatItemView(chatItem, range, prevItem, timeSeparation)
+                    if let date = timeSeparation.date {
+                        DateSeparator(date: date).padding(8)
+                    }
+                }
+                .overlay {
+                    if let selected = selectedChatItems, chatItem.canBeDeletedForSelf {
+                        Color.clear
+                            .contentShape(Rectangle())
+                            .onTapGesture {
+                                let checked = selected.contains(chatItem.id)
+                                selectUnselectChatItem(select: !checked, chatItem)
+                            }
+                    }
+                }
+            }
+            .onAppear {
+                markAsRead()
+            }
+            .onChange(of: ChatView.FloatingButtonModel.shared.bottomItemIndex) { _ in
+                markAsRead()
+            }
         }
 
         private func unreadItemIds(_ range: ClosedRange<Int>) -> [ChatItem.ID] {
@@ -1008,8 +1138,13 @@ struct ChatView: View {
         private func waitToMarkRead(_ op: @Sendable @escaping () async -> Void) {
             Task {
                 _ = try? await Task.sleep(nanoseconds: 600_000000)
-                if m.chatId == chat.chatInfo.id {
-                    await op()
+                let currIndex = m.getChatItemIndex(chatItem)                
+                if let currIndex = currIndex, currIndex >= ChatView.FloatingButtonModel.shared.bottomItemIndex - 3 {
+                    if m.chatId == chat.chatInfo.id {
+                        await op()
+                    }
+                } else {
+                    markedRead = false
                 }
             }
         }
@@ -1573,7 +1708,7 @@ struct ChatView: View {
         private func hideButton() -> Button<some View> {
             Button {
                 withConditionalAnimation {
-                    revealedChatItem = nil
+                    onReveal(false)
                 }
             } label: {
                 Label(
@@ -1648,7 +1783,7 @@ struct ChatView: View {
         private func revealButton(_ ci: ChatItem) -> Button<some View> {
             Button {
                 withConditionalAnimation {
-                    revealedChatItem = ci
+                    onReveal(true)
                 }
             } label: {
                 Label(
@@ -1661,7 +1796,7 @@ struct ChatView: View {
         private func expandButton() -> Button<some View> {
             Button {
                 withConditionalAnimation {
-                    revealedChatItem = chatItem
+                    onReveal(true)
                 }
             } label: {
                 Label(
@@ -1674,7 +1809,7 @@ struct ChatView: View {
         private func shrinkButton() -> Button<some View> {
             Button {
                 withConditionalAnimation {
-                    revealedChatItem = nil
+                    onReveal(false)
                 }
             } label: {
                 Label (

apps/ios/Shared/Views/Chat/ReverseList.swift

@@ -12,14 +12,15 @@ import SimpleXChat
 
 /// A List, which displays it's items in reverse order - from bottom to top
 struct ReverseList<Content: View>: UIViewControllerRepresentable {
-    let items: Array<ChatItem>
+    let groups: SectionGroups
 
     @Binding var scrollState: ReverseListScrollModel.State
+    @Binding var initialChatItem: ChatItem?
 
     /// Closure, that returns user interface for a given item
-    let content: (ChatItem) -> Content
+    let content: (ListItem) -> Content
 
-    let loadPage: () -> Void
+    let loadPage: (_ pagination: ChatPagination) -> Void
 
     func makeUIViewController(context: Context) -> Controller {
         Controller(representer: self)
@@ -27,18 +28,18 @@ struct ReverseList<Content: View>: UIViewControllerRepresentable {
 
     func updateUIViewController(_ controller: Controller, context: Context) {
         controller.representer = self
-        if case let .scrollingTo(destination) = scrollState, !items.isEmpty {
+        if case let .scrollingTo(destination) = scrollState, !groups.sections.isEmpty {
             controller.view.layer.removeAllAnimations()
             switch destination {
             case .nextPage:
                 controller.scrollToNextPage()
             case let .item(id):
-                controller.scroll(to: items.firstIndex(where: { $0.id == id }), position: .bottom)
+                controller.scroll(to: getIndexInParentItems(sections: groups.sections, itemId: id), position: .bottom)
             case .bottom:
                 controller.scroll(to: 0, position: .top)
             }
         } else {
-            controller.update(items: items)
+            controller.update(groups: groups)
         }
     }
 
@@ -46,10 +47,11 @@ struct ReverseList<Content: View>: UIViewControllerRepresentable {
     class Controller: UITableViewController {
         private enum Section { case main }
         var representer: ReverseList
-        private var dataSource: UITableViewDiffableDataSource<Section, ChatItem>!
+        private var dataSource: UITableViewDiffableDataSource<Section, ListItem>!
         private var itemCount: Int = 0
         private let updateFloatingButtons = PassthroughSubject<Void, Never>()
         private var bag = Set<AnyCancellable>()
+        private var revealedItems: Array<ListItem> = []
 
         init(representer: ReverseList) {
             self.representer = representer
@@ -75,11 +77,16 @@ struct ReverseList<Content: View>: UIViewControllerRepresentable {
             }
 
             // 3. Configure data source
-            self.dataSource = UITableViewDiffableDataSource<Section, ChatItem>(
+            self.dataSource = UITableViewDiffableDataSource<Section, ListItem>(
                 tableView: tableView
             ) { (tableView, indexPath, item) -> UITableViewCell? in
-                if indexPath.item > self.itemCount - 8 {
-                    self.representer.loadPage()
+                if self.representer.scrollState == .atDestination, self.representer.initialChatItem == nil {
+                    if indexPath.item > self.itemCount - preloadItem,
+                       let item = getNewestItemAtParentIndexOrNull(sections: self.representer.groups.sections, parentIndex: self.itemCount - 1) {
+                        self.representer.loadPage(.before(chatItemId: item.id, count: loadItemsPerPage))
+                    } else if let item = self.getFirstItemAfterPlacholder(indexPath) {
+                        self.representer.loadPage(.after(chatItemId: item.id, count: loadItemsPerPage))
+                    }
                 }
                 let cell = tableView.dequeueReusableCell(withIdentifier: cellReuseId, for: indexPath)
                 if #available(iOS 16.0, *) {
@@ -149,6 +156,29 @@ struct ReverseList<Content: View>: UIViewControllerRepresentable {
             tableView.clipsToBounds = false
             parent?.viewIfLoaded?.clipsToBounds = false
         }
+        
+        override func viewDidLayoutSubviews() {
+            super.viewDidLayoutSubviews()
+            if let cItem = self.representer.initialChatItem {
+                let index = getIndexInParentItems(sections: self.representer.groups.sections, itemId: cItem.id)
+                
+                if index == -1 {
+                    return
+                }
+                let indexPath = IndexPath(row: index, section: 0)
+                if !isVisible(indexPath: indexPath) {
+                    if tableView.numberOfRows(inSection: indexPath.section) > indexPath.row {
+                        let cellRect = tableView.rectForRow(at: indexPath)
+                        tableView.setContentOffset(CGPoint(x: 0, y: cellRect.maxY - tableView.bounds.height), animated: false)
+                    }
+                }
+                Task {
+                    DispatchQueue.main.async {
+                        self.representer.initialChatItem = nil
+                    }
+                }
+            }
+        }
 
         /// Scrolls up
         func scrollToNextPage() {
@@ -169,6 +199,7 @@ struct ReverseList<Content: View>: UIViewControllerRepresentable {
             if #available(iOS 16.0, *) {
                 animated = true
             }
+
             if let index, tableView.numberOfRows(inSection: 0) != 0 {
                 tableView.scrollToRow(
                     at: IndexPath(row: index, section: 0),
@@ -181,18 +212,49 @@ struct ReverseList<Content: View>: UIViewControllerRepresentable {
                     animated: animated
                 )
             }
-            Task { representer.scrollState = .atDestination }
+            
+            DispatchQueue.main.asyncAfter(deadline: .now() + 0.5) {
+                Task {
+                    self.representer.scrollState = .atDestination
+                }
+            }
         }
 
-        func update(items: [ChatItem]) {
-            var snapshot = NSDiffableDataSourceSnapshot<Section, ChatItem>()
+        func update(groups: SectionGroups) {
+            var snapshot = NSDiffableDataSourceSnapshot<Section, ListItem>()
+            revealedItems.removeAll()
+            groups.sections.forEach { sc in
+                if sc.revealed {
+                    revealedItems.append(contentsOf: sc.items)
+                } else if let item = sc.items.first {
+                    revealedItems.append(item)
+                }
+            }
             snapshot.appendSections([.main])
-            snapshot.appendItems(items)
+            snapshot.appendItems(revealedItems, toSection: .main)
             dataSource.defaultRowAnimation = .none
-            dataSource.apply(
-                snapshot,
-                animatingDifferences: itemCount != 0 && abs(items.count - itemCount) == 1
-            )
+            
+            let countDiff = max(0, revealedItems.count - itemCount)
+            if tableView.contentOffset.y == 100, itemCount < revealedItems.count, itemCount > 0 {
+                dataSource.apply(
+                    snapshot,
+                    animatingDifferences: false
+                )
+                
+                tableView.scrollToRow(
+                    at: IndexPath(row: countDiff, section: 0),
+                    at: .top,
+                    animated: false
+                )
+            } else {
+                tableView.beginUpdates()
+                dataSource.apply(
+                    snapshot,
+                    animatingDifferences: false
+                )
+                tableView.endUpdates()
+            }
+
             // Sets content offset on initial load
             if itemCount == 0 {
                 tableView.setContentOffset(
@@ -200,7 +262,7 @@ struct ReverseList<Content: View>: UIViewControllerRepresentable {
                     animated: false
                 )
             }
-            itemCount = items.count
+            itemCount = revealedItems.count
             updateFloatingButtons.send()
         }
 
@@ -210,17 +272,18 @@ struct ReverseList<Content: View>: UIViewControllerRepresentable {
 
         func getListState() -> ListState? {
             if let visibleRows = tableView.indexPathsForVisibleRows,
-                visibleRows.last?.item ?? 0 < representer.items.count {
+               visibleRows.last?.item ?? 0 < revealedItems.count {
                 let scrollOffset: Double = tableView.contentOffset.y + InvertedTableView.inset
+                
                 let topItemDate: Date? =
                     if let lastVisible = visibleRows.last(where: { isVisible(indexPath: $0) }) {
-                        representer.items[lastVisible.item].meta.itemTs
+                        revealedItems[lastVisible.item].item.meta.itemTs
                     } else {
                         nil
                     }
                 let bottomItemId: ChatItem.ID? =
                     if let firstVisible = visibleRows.first(where: { isVisible(indexPath: $0) }) {
-                        representer.items[firstVisible.item].id
+                        revealedItems[firstVisible.item].item.id
                     } else {
                         nil
                     }
@@ -238,6 +301,10 @@ struct ReverseList<Content: View>: UIViewControllerRepresentable {
                 relativeFrame.minY < tableView.frame.height - InvertedTableView.inset
             } else { false }
         }
+        
+        private func getFirstItemAfterPlacholder(_ indexPath: IndexPath) -> ChatItem? {
+            return nil
+        }
     }
 
     /// `UIHostingConfiguration` back-port for iOS14 and iOS15

apps/ios/SimpleX.xcodeproj/project.pbxproj

@@ -149,9 +149,9 @@
 		6419EC582AB97507004A607A /* CIMemberCreatedContactView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 6419EC572AB97507004A607A /* CIMemberCreatedContactView.swift */; };
 		6432857C2925443C00FBE5C8 /* GroupPreferencesView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 6432857B2925443C00FBE5C8 /* GroupPreferencesView.swift */; };
 		643B3B452CCBEB080083A2CF /* libgmpxx.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 643B3B402CCBEB080083A2CF /* libgmpxx.a */; };
-		643B3B462CCBEB080083A2CF /* libHSsimplex-chat-6.1.1.0-KwHIA7FZqPI5ZTCAoi00n9-ghc9.6.3.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 643B3B412CCBEB080083A2CF /* libHSsimplex-chat-6.1.1.0-KwHIA7FZqPI5ZTCAoi00n9-ghc9.6.3.a */; };
+		643B3B462CCBEB080083A2CF /* libHSsimplex-chat-6.1.1.0-CTfGB7l09cqEHVIdvhrnH5-ghc9.6.3.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 643B3B412CCBEB080083A2CF /* libHSsimplex-chat-6.1.1.0-CTfGB7l09cqEHVIdvhrnH5-ghc9.6.3.a */; };
 		643B3B472CCBEB080083A2CF /* libffi.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 643B3B422CCBEB080083A2CF /* libffi.a */; };
-		643B3B482CCBEB080083A2CF /* libHSsimplex-chat-6.1.1.0-KwHIA7FZqPI5ZTCAoi00n9.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 643B3B432CCBEB080083A2CF /* libHSsimplex-chat-6.1.1.0-KwHIA7FZqPI5ZTCAoi00n9.a */; };
+		643B3B482CCBEB080083A2CF /* libHSsimplex-chat-6.1.1.0-CTfGB7l09cqEHVIdvhrnH5.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 643B3B432CCBEB080083A2CF /* libHSsimplex-chat-6.1.1.0-CTfGB7l09cqEHVIdvhrnH5.a */; };
 		643B3B492CCBEB080083A2CF /* libgmp.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 643B3B442CCBEB080083A2CF /* libgmp.a */; };
 		6440CA00288857A10062C672 /* CIEventView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 6440C9FF288857A10062C672 /* CIEventView.swift */; };
 		6440CA03288AECA70062C672 /* AddGroupMembersView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 6440CA02288AECA70062C672 /* AddGroupMembersView.swift */; };
@@ -200,6 +200,7 @@
 		8CC4ED902BD7B8530078AEE8 /* CallAudioDeviceManager.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8CC4ED8F2BD7B8530078AEE8 /* CallAudioDeviceManager.swift */; };
 		8CC956EE2BC0041000412A11 /* NetworkObserver.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8CC956ED2BC0041000412A11 /* NetworkObserver.swift */; };
 		8CE848A32C5A0FA000D5C7C8 /* SelectableChatItemToolbars.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8CE848A22C5A0FA000D5C7C8 /* SelectableChatItemToolbars.swift */; };
+		B72540EB2CE277AC0041D1B4 /* ChatItemGroups.swift in Sources */ = {isa = PBXBuildFile; fileRef = B72540EA2CE277AC0041D1B4 /* ChatItemGroups.swift */; };
 		B76E6C312C5C41D900EC11AA /* ContactListNavLink.swift in Sources */ = {isa = PBXBuildFile; fileRef = B76E6C302C5C41D900EC11AA /* ContactListNavLink.swift */; };
 		CE176F202C87014C00145DBC /* InvertedForegroundStyle.swift in Sources */ = {isa = PBXBuildFile; fileRef = CE176F1F2C87014C00145DBC /* InvertedForegroundStyle.swift */; };
 		CE1EB0E42C459A660099D896 /* ShareAPI.swift in Sources */ = {isa = PBXBuildFile; fileRef = CE1EB0E32C459A660099D896 /* ShareAPI.swift */; };
@@ -492,9 +493,9 @@
 		6419EC572AB97507004A607A /* CIMemberCreatedContactView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CIMemberCreatedContactView.swift; sourceTree = "<group>"; };
 		6432857B2925443C00FBE5C8 /* GroupPreferencesView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = GroupPreferencesView.swift; sourceTree = "<group>"; };
 		643B3B402CCBEB080083A2CF /* libgmpxx.a */ = {isa = PBXFileReference; lastKnownFileType = archive.ar; name = libgmpxx.a; path = Libraries/libgmpxx.a; sourceTree = "<group>"; };
-		643B3B412CCBEB080083A2CF /* libHSsimplex-chat-6.1.1.0-KwHIA7FZqPI5ZTCAoi00n9-ghc9.6.3.a */ = {isa = PBXFileReference; lastKnownFileType = archive.ar; name = "libHSsimplex-chat-6.1.1.0-KwHIA7FZqPI5ZTCAoi00n9-ghc9.6.3.a"; path = "Libraries/libHSsimplex-chat-6.1.1.0-KwHIA7FZqPI5ZTCAoi00n9-ghc9.6.3.a"; sourceTree = "<group>"; };
+		643B3B412CCBEB080083A2CF /* libHSsimplex-chat-6.1.1.0-CTfGB7l09cqEHVIdvhrnH5-ghc9.6.3.a */ = {isa = PBXFileReference; lastKnownFileType = archive.ar; name = "libHSsimplex-chat-6.1.1.0-CTfGB7l09cqEHVIdvhrnH5-ghc9.6.3.a"; path = "Libraries/libHSsimplex-chat-6.1.1.0-CTfGB7l09cqEHVIdvhrnH5-ghc9.6.3.a"; sourceTree = "<group>"; };
 		643B3B422CCBEB080083A2CF /* libffi.a */ = {isa = PBXFileReference; lastKnownFileType = archive.ar; name = libffi.a; path = Libraries/libffi.a; sourceTree = "<group>"; };
-		643B3B432CCBEB080083A2CF /* libHSsimplex-chat-6.1.1.0-KwHIA7FZqPI5ZTCAoi00n9.a */ = {isa = PBXFileReference; lastKnownFileType = archive.ar; name = "libHSsimplex-chat-6.1.1.0-KwHIA7FZqPI5ZTCAoi00n9.a"; path = "Libraries/libHSsimplex-chat-6.1.1.0-KwHIA7FZqPI5ZTCAoi00n9.a"; sourceTree = "<group>"; };
+		643B3B432CCBEB080083A2CF /* libHSsimplex-chat-6.1.1.0-CTfGB7l09cqEHVIdvhrnH5.a */ = {isa = PBXFileReference; lastKnownFileType = archive.ar; name = "libHSsimplex-chat-6.1.1.0-CTfGB7l09cqEHVIdvhrnH5.a"; path = "Libraries/libHSsimplex-chat-6.1.1.0-CTfGB7l09cqEHVIdvhrnH5.a"; sourceTree = "<group>"; };
 		643B3B442CCBEB080083A2CF /* libgmp.a */ = {isa = PBXFileReference; lastKnownFileType = archive.ar; name = libgmp.a; path = Libraries/libgmp.a; sourceTree = "<group>"; };
 		6440C9FF288857A10062C672 /* CIEventView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CIEventView.swift; sourceTree = "<group>"; };
 		6440CA02288AECA70062C672 /* AddGroupMembersView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AddGroupMembersView.swift; sourceTree = "<group>"; };
@@ -544,6 +545,7 @@
 		8CC4ED8F2BD7B8530078AEE8 /* CallAudioDeviceManager.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CallAudioDeviceManager.swift; sourceTree = "<group>"; };
 		8CC956ED2BC0041000412A11 /* NetworkObserver.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = NetworkObserver.swift; sourceTree = "<group>"; };
 		8CE848A22C5A0FA000D5C7C8 /* SelectableChatItemToolbars.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SelectableChatItemToolbars.swift; sourceTree = "<group>"; };
+		B72540EA2CE277AC0041D1B4 /* ChatItemGroups.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ChatItemGroups.swift; sourceTree = "<group>"; };
 		B76E6C302C5C41D900EC11AA /* ContactListNavLink.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ContactListNavLink.swift; sourceTree = "<group>"; };
 		CE176F1F2C87014C00145DBC /* InvertedForegroundStyle.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = InvertedForegroundStyle.swift; sourceTree = "<group>"; };
 		CE1EB0E32C459A660099D896 /* ShareAPI.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ShareAPI.swift; sourceTree = "<group>"; };
@@ -663,8 +665,8 @@
 				5CE2BA93284534B000EC33A6 /* libiconv.tbd in Frameworks */,
 				5CE2BA94284534BB00EC33A6 /* libz.tbd in Frameworks */,
 				CE38A29C2C3FCD72005ED185 /* SwiftyGif in Frameworks */,
-				643B3B482CCBEB080083A2CF /* libHSsimplex-chat-6.1.1.0-KwHIA7FZqPI5ZTCAoi00n9.a in Frameworks */,
-				643B3B462CCBEB080083A2CF /* libHSsimplex-chat-6.1.1.0-KwHIA7FZqPI5ZTCAoi00n9-ghc9.6.3.a in Frameworks */,
+				643B3B482CCBEB080083A2CF /* libHSsimplex-chat-6.1.1.0-CTfGB7l09cqEHVIdvhrnH5.a in Frameworks */,
+				643B3B462CCBEB080083A2CF /* libHSsimplex-chat-6.1.1.0-CTfGB7l09cqEHVIdvhrnH5-ghc9.6.3.a in Frameworks */,
 			);
 			runOnlyForDeploymentPostprocessing = 0;
 		};
@@ -734,6 +736,7 @@
 				64C06EB42A0A4A7C00792D4D /* ChatItemInfoView.swift */,
 				648679AA2BC96A74006456E7 /* ChatItemForwardingView.swift */,
 				8CE848A22C5A0FA000D5C7C8 /* SelectableChatItemToolbars.swift */,
+				B72540EA2CE277AC0041D1B4 /* ChatItemGroups.swift */,
 			);
 			path = Chat;
 			sourceTree = "<group>";
@@ -815,8 +818,8 @@
 				643B3B422CCBEB080083A2CF /* libffi.a */,
 				643B3B442CCBEB080083A2CF /* libgmp.a */,
 				643B3B402CCBEB080083A2CF /* libgmpxx.a */,
-				643B3B412CCBEB080083A2CF /* libHSsimplex-chat-6.1.1.0-KwHIA7FZqPI5ZTCAoi00n9-ghc9.6.3.a */,
-				643B3B432CCBEB080083A2CF /* libHSsimplex-chat-6.1.1.0-KwHIA7FZqPI5ZTCAoi00n9.a */,
+				643B3B412CCBEB080083A2CF /* libHSsimplex-chat-6.1.1.0-CTfGB7l09cqEHVIdvhrnH5-ghc9.6.3.a */,
+				643B3B432CCBEB080083A2CF /* libHSsimplex-chat-6.1.1.0-CTfGB7l09cqEHVIdvhrnH5.a */,
 				5CA059C2279559F40002BEB4 /* Shared */,
 				5CDCAD462818589900503DA2 /* SimpleX NSE */,
 				CEE723A82C3BD3D70009AE93 /* SimpleX SE */,
@@ -1483,6 +1486,7 @@
 				5CB346E92869E8BA001FD2EF /* PushEnvironment.swift in Sources */,
 				5C55A91F283AD0E400C4E99E /* CallManager.swift in Sources */,
 				5CFA59D12864782E00863A68 /* ChatArchiveView.swift in Sources */,
+				B72540EB2CE277AC0041D1B4 /* ChatItemGroups.swift in Sources */,
 				649BCDA22805D6EF00C3A862 /* CIImageView.swift in Sources */,
 				5CADE79C292131E900072E13 /* ContactPreferencesView.swift in Sources */,
 				CEA6E91C2CBD21B0002B5DB4 /* UserDefault.swift in Sources */,

apps/ios/SimpleXChat/APITypes.swift

@@ -1133,12 +1133,16 @@ public enum ChatPagination {
     case last(count: Int)
     case after(chatItemId: Int64, count: Int)
     case before(chatItemId: Int64, count: Int)
-
+    case around(chatItemId: Int64, count: Int)
+    case initial(count: Int)
+    
     var cmdString: String {
         switch self {
         case let .last(count): return "count=\(count)"
         case let .after(chatItemId, count): return "after=\(chatItemId) count=\(count)"
         case let .before(chatItemId, count): return "before=\(chatItemId) count=\(count)"
+        case let .around(chatItemId, count): return "around=\(chatItemId) count=\(count)"
+        case let .initial(count): return "initial=\(count)"
         }
     }
 }

apps/ios/SimpleXChat/ChatTypes.swift

@@ -2663,7 +2663,7 @@ public struct ChatItem: Identifiable, Decodable, Hashable {
         item.isLiveDummy = true
         return item
     }
-
+    
     public static func invalidJSON(chatDir: CIDirection?, meta: CIMeta?, json: String) -> ChatItem {
         ChatItem(
             chatDir: chatDir ?? .directSnd,

apps/multiplatform/common/src/commonMain/kotlin/chat/simplex/common/views/chat/ComposeView.kt

@@ -381,6 +381,14 @@ fun ComposeView(
 
   suspend fun send(chat: Chat, mc: MsgContent, quoted: Long?, file: CryptoFile? = null, live: Boolean = false, ttl: Int?): ChatItem? {
     val cInfo = chat.chatInfo
+
+//    val composedMessages = Array(300) { index ->
+//      ComposedMessage(
+//        file,
+//        quoted,
+//        MsgContent.MCText("$index")
+//      )
+//    }.toList()
     val chatItems = if (chat.chatInfo.chatType == ChatType.Local)
       chatModel.controller.apiCreateChatItems(
         rh = chat.remoteHostId,

apps/simplex-directory-service/src/Directory/Service.hs

@@ -605,7 +605,7 @@ directoryService st DirectoryOpts {superUsers, serviceName, searchResults, testi
         listGroups count pending =
           readTVarIO (groupRegs st) >>= \groups -> do
             grs <-
-              if pending 
+              if pending
                 then filterM (fmap pendingApproval . readTVarIO . groupRegStatus) groups
                 else pure groups
             sendReply $ show (length grs) <> " registered group(s)" <> (if length grs > count then ", showing the last " <> show count else "")

cabal.project

@@ -12,7 +12,7 @@ constraints: zip +disable-bzip2 +disable-zstd
 source-repository-package
     type: git
     location: https://github.com/simplex-chat/simplexmq.git
-    tag: a8471eed5be93e7c3741aa4742b24193c9a2d6f5
+    tag: ffecf200d4874dfa34f6d15b269964c0115a54ca
 
 source-repository-package
     type: git

docs/JOIN_TEAM.md

@@ -8,25 +8,23 @@ layout: layouts/jobs.html
 
 SimpleX Chat Ltd is a seed stage startup with a lot of user growth in 2022-2023, and a lot of exciting technical and product problems to solve to grow faster.
 
-We currently have 4 full-time people in the team - all engineers, including the founder.
-
-We want to add up to 3 people to the team. 
+We currently have 6 full-time people in the team.
 
+We want to add 2 people to the team.
 
 ## Who we are looking for
 
-### Product/UI designer
+### Web designer & developer for a website contract
 
-You will be designing the user experience and the interface of both the app and the website in collaboration with the team.
+You will work with the founder and a product marketing expert to convert the stories we want to tell our current and prospective users into interactive experiences.
 
-The current focus of the app is privacy and security, but we hope to have the design that would support the feeling of psychological safety, enabling people to achieve the results in the smallest amount of time.
+You are an expert in creating interactive web experiences:
+- 15+ years of web development and design experience.
+- Passionate about communications, privacy and data ownership.
+- Competent using PhotoShop, 3D modelling, etc.
+- Competent in Web tech, including JavaScript, animations, etc.
 
-You are an experienced and innovative product designer with:
-- 8+ years of user experience and visual design.
-- Expertise in typography and high sensitivity to colors.
-- Exceptional precision and attention to details.
-- Strong opinions (weakly held).
-- A strong empathy.
+We will NOT consider agencies or groups – it must be one person working on the project.
 
 ### Application Haskell engineer
 
@@ -34,13 +32,12 @@ You will work with the Haskell core of the client applications and with the netw
 
 You are an expert in language models, databases and Haskell:
 - expert knowledge of SQL.
-- Haskell exception handling, concurrency, STM, type systems.
-- 8y+ of software engineering experience in complex projects, 
+- Haskell strictness, exceptions, [concurrency](https://simonmar.github.io/pages/pcph.html), STM, [type systems](https://thinkingwithtypes.com).
+- 15y+ of software engineering experience in complex projects.
 - deep understanding of the common programming principles:
   - data structures, bits and bytes, text encoding.
-  - software design and algorithms.
-  - concurrency.
-  - networking.
+  - [functional software design](https://mitp-content-server.mit.edu/books/content/sectbyfn/books_pres_0/6515/sicp.zip/index.html) and algorithms.
+  - protocols and networking.
 
 ## About you
 
@@ -48,6 +45,7 @@ You are an expert in language models, databases and Haskell:
   - already use SimpleX Chat to communicate with friends/family or participate in public SimpleX Chat groups.
   - passionate about privacy, security and communications.
   - interested to make contributions to SimpleX Chat open-source project in your free time before we hire you, as an extended test.
+  - you founded (and probably failed) at least one startup, or spent more time working for yourself than being employed.
 
 - **Exceptionally pragmatic, very fast and customer-focussed**:
   - care about the customers (aka users) and about the product we build much more than about the code quality, technology stack, etc.

flake.lock

@@ -156,11 +156,11 @@
     "ghc98X": {
       "flake": false,
       "locked": {
-        "lastModified": 1696643148,
-        "narHash": "sha256-E02DfgISH7EvvNAu0BHiPvl1E5FGMDi0pWdNZtIBC9I=",
+        "lastModified": 1715066704,
+        "narHash": "sha256-F0EVR8x/fcpj1st+hz96Wdsz5uwVIOziGKAwRxLOYJw=",
         "ref": "ghc-9.8",
-        "rev": "443e870d977b1ab6fc05f47a9a17bc49296adbd6",
-        "revCount": 61642,
+        "rev": "78a253543d466ac511a1664a3e6aff032ca684d5",
+        "revCount": 61757,
         "submodules": true,
         "type": "git",
         "url": "https://gitlab.haskell.org/ghc/ghc"
@@ -175,11 +175,11 @@
     "ghc99": {
       "flake": false,
       "locked": {
-        "lastModified": 1697054644,
-        "narHash": "sha256-kKarOuXUaAH3QWv7ASx+gGFMHaHKe0pK5Zu37ky2AL4=",
+        "lastModified": 1726585445,
+        "narHash": "sha256-IdwQBex4boY6s0Plj5+ixf36rfYSUyMdTWrztKvZH30=",
         "ref": "refs/heads/master",
-        "rev": "f383a242c76f90bcca8a4d7ee001dcb49c172a9a",
-        "revCount": 62040,
+        "rev": "7fd9e5e29ab54eb406880077463e8552e2ddd39a",
+        "revCount": 67238,
         "submodules": true,
         "type": "git",
         "url": "https://gitlab.haskell.org/ghc/ghc"
@@ -225,6 +225,8 @@
         "hls-2.2": "hls-2.2",
         "hls-2.3": "hls-2.3",
         "hls-2.4": "hls-2.4",
+        "hls-2.5": "hls-2.5",
+        "hls-2.6": "hls-2.6",
         "hpc-coveralls": "hpc-coveralls",
         "hydra": "hydra",
         "iserv-proxy": "iserv-proxy",
@@ -238,16 +240,17 @@
         "nixpkgs-2205": "nixpkgs-2205",
         "nixpkgs-2211": "nixpkgs-2211",
         "nixpkgs-2305": "nixpkgs-2305",
+        "nixpkgs-2311": "nixpkgs-2311",
         "nixpkgs-unstable": "nixpkgs-unstable",
         "old-ghc-nix": "old-ghc-nix",
         "stackage": "stackage"
       },
       "locked": {
-        "lastModified": 1701163700,
-        "narHash": "sha256-sOrewUS3LnzV09nGr7+3R6Q6zsgU4smJc61QsHq+4DE=",
+        "lastModified": 1705833500,
+        "narHash": "sha256-rUIr6JNbCedt1g4gVYVvE9t0oFU6FUspCA0DS5cA8Bg=",
         "owner": "input-output-hk",
         "repo": "haskell.nix",
-        "rev": "2808bfe3e62e9eb4ee8974cd623a00e1611f302b",
+        "rev": "d0c35e75cbbc6858770af42ac32b0b85495fbd71",
         "type": "github"
       },
       "original": {
@@ -328,16 +331,50 @@
     "hls-2.4": {
       "flake": false,
       "locked": {
-        "lastModified": 1696939266,
-        "narHash": "sha256-VOMf5+kyOeOmfXTHlv4LNFJuDGa7G3pDnOxtzYR40IU=",
+        "lastModified": 1699862708,
+        "narHash": "sha256-YHXSkdz53zd0fYGIYOgLt6HrA0eaRJi9mXVqDgmvrjk=",
         "owner": "haskell",
         "repo": "haskell-language-server",
-        "rev": "362fdd1293efb4b82410b676ab1273479f6d17ee",
+        "rev": "54507ef7e85fa8e9d0eb9a669832a3287ffccd57",
         "type": "github"
       },
       "original": {
         "owner": "haskell",
-        "ref": "2.4.0.0",
+        "ref": "2.4.0.1",
+        "repo": "haskell-language-server",
+        "type": "github"
+      }
+    },
+    "hls-2.5": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1701080174,
+        "narHash": "sha256-fyiR9TaHGJIIR0UmcCb73Xv9TJq3ht2ioxQ2mT7kVdc=",
+        "owner": "haskell",
+        "repo": "haskell-language-server",
+        "rev": "27f8c3d3892e38edaef5bea3870161815c4d014c",
+        "type": "github"
+      },
+      "original": {
+        "owner": "haskell",
+        "ref": "2.5.0.0",
+        "repo": "haskell-language-server",
+        "type": "github"
+      }
+    },
+    "hls-2.6": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1705325287,
+        "narHash": "sha256-+P87oLdlPyMw8Mgoul7HMWdEvWP/fNlo8jyNtwME8E8=",
+        "owner": "haskell",
+        "repo": "haskell-language-server",
+        "rev": "6e0b342fa0327e628610f2711f8c3e4eaaa08b1e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "haskell",
+        "ref": "2.6.0.0",
         "repo": "haskell-language-server",
         "type": "github"
       }
@@ -384,11 +421,11 @@
     "iserv-proxy": {
       "flake": false,
       "locked": {
-        "lastModified": 1691634696,
-        "narHash": "sha256-MZH2NznKC/gbgBu8NgIibtSUZeJ00HTLJ0PlWKCBHb0=",
+        "lastModified": 1707968597,
+        "narHash": "sha256-C53NqToxl+n9s1pQ0iLtiH6P5vX3rM+NW/mFt4Ykpsk=",
         "ref": "hkm/remote-iserv",
-        "rev": "43a979272d9addc29fbffc2e8542c5d96e993d73",
-        "revCount": 14,
+        "rev": "1b7f8aeb37bbc7c00f04e44d9379aa15a4409e8b",
+        "revCount": 18,
         "type": "git",
         "url": "https://gitlab.haskell.org/hamishmack/iserv-proxy.git"
       },
@@ -552,11 +589,11 @@
     },
     "nixpkgs-2305": {
       "locked": {
-        "lastModified": 1695416179,
-        "narHash": "sha256-610o1+pwbSu+QuF3GE0NU5xQdTHM3t9wyYhB9l94Cd8=",
+        "lastModified": 1705033721,
+        "narHash": "sha256-K5eJHmL1/kev6WuqyqqbS1cdNnSidIZ3jeqJ7GbrYnQ=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "715d72e967ec1dd5ecc71290ee072bcaf5181ed6",
+        "rev": "a1982c92d8980a0114372973cbdfe0a307f1bdea",
         "type": "github"
       },
       "original": {
@@ -566,6 +603,22 @@
         "type": "github"
       }
     },
+    "nixpkgs-2311": {
+      "locked": {
+        "lastModified": 1719957072,
+        "narHash": "sha256-gvFhEf5nszouwLAkT9nWsDzocUTqLWHuL++dvNjMp9I=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "7144d6241f02d171d25fba3edeaf15e0f2592105",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixpkgs-23.11-darwin",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
     "nixpkgs-lib": {
       "locked": {
         "dir": "lib",
@@ -602,17 +655,17 @@
     },
     "nixpkgs-unstable": {
       "locked": {
-        "lastModified": 1695318763,
-        "narHash": "sha256-FHVPDRP2AfvsxAdc+AsgFJevMz5VBmnZglFUMlxBkcY=",
+        "lastModified": 1694822471,
+        "narHash": "sha256-6fSDCj++lZVMZlyqOe9SIOL8tYSBz1bI8acwovRwoX8=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "e12483116b3b51a185a33a272bf351e357ba9a99",
+        "rev": "47585496bcb13fb72e4a90daeea2f434e2501998",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "nixpkgs-unstable",
         "repo": "nixpkgs",
+        "rev": "47585496bcb13fb72e4a90daeea2f434e2501998",
         "type": "github"
       }
     },
@@ -664,11 +717,11 @@
     "stackage": {
       "flake": false,
       "locked": {
-        "lastModified": 1699834215,
-        "narHash": "sha256-g/JKy0BCvJaxPuYDl3QVc4OY8cFEomgG+hW/eEV470M=",
+        "lastModified": 1726532152,
+        "narHash": "sha256-LRXbVY3M2S8uQWdwd2zZrsnVPEvt2GxaHGoy8EFFdJA=",
         "owner": "input-output-hk",
         "repo": "stackage.nix",
-        "rev": "47aacd04abcce6bad57f43cbbbd133538380248e",
+        "rev": "c77b3530cebad603812cb111c6f64968c2d2337d",
         "type": "github"
       },
       "original": {

flake.nix

@@ -198,6 +198,7 @@
                   packages.direct-sqlcipher.components.library.libs = pkgs.lib.mkForce [
                     pkgs.pkgsCross.mingwW64.openssl
                   ];
+                  packages.simplexmq.flags.client_library = true;
                   packages.simplexmq.components.library.libs = pkgs.lib.mkForce [
                     pkgs.pkgsCross.mingwW64.openssl
                   ];
@@ -335,6 +336,7 @@
                   packages.direct-sqlcipher.patches = [
                     ./scripts/nix/direct-sqlcipher-android-log.patch
                   ];
+                  packages.simplexmq.flags.client_library = true;
                   packages.simplexmq.components.library.libs = pkgs.lib.mkForce [
                     (android32Pkgs.openssl.override { static = true; enableKTLS = false; })
                   ];
@@ -443,6 +445,7 @@
                   packages.direct-sqlcipher.patches = [
                     ./scripts/nix/direct-sqlcipher-android-log.patch
                   ];
+                  packages.simplexmq.flags.client_library = true;
                   packages.simplexmq.components.library.libs = pkgs.lib.mkForce [
                     (androidPkgs.openssl.override { static = true; })
                   ];
@@ -547,6 +550,7 @@
                   packages.simplexmq.flags.swift = true;
                   packages.direct-sqlcipher.flags.commoncrypto = true;
                   packages.entropy.flags.DoNotGetEntropy = true;
+                  packages.simplexmq.flags.client_library = true;
                   packages.simplexmq.components.library.libs = pkgs.lib.mkForce [
                     # TODO: have a cross override for iOS, that sets this.
                     ((pkgs.openssl.override { static = true; }).overrideDerivation (old: { CFLAGS = "-mcpu=apple-a7 -march=armv8-a+norcpc" ;}))
@@ -561,6 +565,7 @@
                 extra-modules = [{
                   packages.direct-sqlcipher.flags.commoncrypto = true;
                   packages.entropy.flags.DoNotGetEntropy = true;
+                  packages.simplexmq.flags.client_library = true;
                   packages.simplexmq.components.library.libs = pkgs.lib.mkForce [
                     ((pkgs.openssl.override { static = true; }).overrideDerivation (old: { CFLAGS = "-mcpu=apple-a7 -march=armv8-a+norcpc" ;}))
                   ];
@@ -578,6 +583,7 @@
                   packages.simplexmq.flags.swift = true;
                   packages.direct-sqlcipher.flags.commoncrypto = true;
                   packages.entropy.flags.DoNotGetEntropy = true;
+                  packages.simplexmq.flags.client_library = true;
                   packages.simplexmq.components.library.libs = pkgs.lib.mkForce [
                     (pkgs.openssl.override { static = true; })
                   ];
@@ -591,6 +597,7 @@
                 extra-modules = [{
                   packages.direct-sqlcipher.flags.commoncrypto = true;
                   packages.entropy.flags.DoNotGetEntropy = true;
+                  packages.simplexmq.flags.client_library = true;
                   packages.simplexmq.components.library.libs = pkgs.lib.mkForce [
                     (pkgs.openssl.override { static = true; })
                   ];

scripts/desktop/build-lib-linux.sh

@@ -25,7 +25,7 @@ for elem in "${exports[@]}"; do count=$(grep -R "$elem$" libsimplex.dll.def | wc
 for elem in "${exports[@]}"; do count=$(grep -R "\"$elem\"" flake.nix | wc -l); if [ $count -ne 2 ]; then echo Wrong exports in flake.nix. Add \"$elem\" in two places of the file; exit 1; fi ; done
 
 rm -rf $BUILD_DIR
-cabal build lib:simplex-chat --ghc-options='-optl-Wl,-rpath,$ORIGIN -flink-rts -threaded'
+cabal build lib:simplex-chat --ghc-options='-optl-Wl,-rpath,$ORIGIN -flink-rts -threaded' --constraint 'simplexmq +client_library'
 cd $BUILD_DIR/build
 #patchelf --add-needed libHSrts_thr-ghc${GHC_VERSION}.so libHSsimplex-chat-*-inplace-ghc${GHC_VERSION}.so
 #patchelf --add-rpath '$ORIGIN' libHSsimplex-chat-*-inplace-ghc${GHC_VERSION}.so

scripts/desktop/build-lib-mac.sh

@@ -24,7 +24,7 @@ for elem in "${exports[@]}"; do count=$(grep -R "$elem$" libsimplex.dll.def | wc
 for elem in "${exports[@]}"; do count=$(grep -R "\"$elem\"" flake.nix | wc -l); if [ $count -ne 2 ]; then echo Wrong exports in flake.nix. Add \"$elem\" in two places of the file; exit 1; fi ; done
 
 rm -rf $BUILD_DIR
-cabal build lib:simplex-chat lib:simplex-chat --ghc-options="-optl-Wl,-rpath,@loader_path -optl-Wl,-L$GHC_LIBS_DIR/$ARCH-osx-ghc-$GHC_VERSION -optl-lHSrts_thr-ghc$GHC_VERSION -optl-lffi"
+cabal build lib:simplex-chat lib:simplex-chat --ghc-options="-optl-Wl,-rpath,@loader_path -optl-Wl,-L$GHC_LIBS_DIR/$ARCH-osx-ghc-$GHC_VERSION -optl-lHSrts_thr-ghc$GHC_VERSION -optl-lffi" --constraint 'simplexmq +client_library'
 
 cd $BUILD_DIR/build
 mkdir deps 2> /dev/null || true

scripts/desktop/build-lib-windows.sh

@@ -51,7 +51,7 @@ echo "    ghc-options: -shared -threaded -optl-L$openssl_windows_style_path -opt
 # Very important! Without it the build fails on linking step since the linker can't find exported symbols.
 # It looks like GHC bug because with such random path the build ends successfully
 sed -i "s/ld.lld.exe/abracadabra.exe/" `ghc --print-libdir`/settings
-cabal build lib:simplex-chat
+cabal build lib:simplex-chat --constraint 'simplexmq +client_library'
 
 rm -rf apps/multiplatform/common/src/commonMain/cpp/desktop/libs/$OS-$ARCH/
 rm -rf apps/multiplatform/desktop/build/cmake

scripts/nix/sha256map.nix

@@ -1,5 +1,5 @@
 {
-  "https://github.com/simplex-chat/simplexmq.git"."a8471eed5be93e7c3741aa4742b24193c9a2d6f5" = "093i40api0dp7rvw6f1f3pww3q5iv6mvbj577nlxp3qqcbvyh6fs";
+  "https://github.com/simplex-chat/simplexmq.git"."ffecf200d4874dfa34f6d15b269964c0115a54ca" = "0kb8hq37fc5g198wq7dswnlwjzk67q8rrzil2dii5lc6xfr47jbs";
   "https://github.com/simplex-chat/hs-socks.git"."a30cc7a79a08d8108316094f8f2f82a0c5e1ac51" = "0yasvnr7g91k76mjkamvzab2kvlb1g5pspjyjn2fr6v83swjhj38";
   "https://github.com/simplex-chat/direct-sqlcipher.git"."f814ee68b16a9447fbb467ccc8f29bdd3546bfd9" = "1ql13f4kfwkbaq7nygkxgw84213i0zm7c1a8hwvramayxl38dq5d";
   "https://github.com/simplex-chat/sqlcipher-simple.git"."a46bd361a19376c5211f1058908fc0ae6bf42446" = "1z0r78d8f0812kxbgsm735qf6xx8lvaz27k1a0b4a2m0sshpd5gl";

simplex-chat.cabal

@@ -150,6 +150,7 @@ library
       Simplex.Chat.Migrations.M20240920_user_order
       Simplex.Chat.Migrations.M20241008_indexes
       Simplex.Chat.Migrations.M20241010_contact_requests_contact_id
+      Simplex.Chat.Migrations.M20241023_chat_item_autoincrement_id
       Simplex.Chat.Mobile
       Simplex.Chat.Mobile.File
       Simplex.Chat.Mobile.Shared

src/Simplex/Chat.hs

@@ -8301,6 +8301,8 @@ chatCommandP =
       (CPLast <$ "count=" <*> A.decimal)
         <|> (CPAfter <$ "after=" <*> A.decimal <* A.space <* "count=" <*> A.decimal)
         <|> (CPBefore <$ "before=" <*> A.decimal <* A.space <* "count=" <*> A.decimal)
+        <|> (CPAround <$ "around=" <*> A.decimal <* A.space <* "count=" <*> A.decimal)
+        <|> (CPInitial <$ "initial=" <*> A.decimal)
     paginationByTimeP =
       (PTLast <$ "count=" <*> A.decimal)
         <|> (PTAfter <$ "after=" <*> strP <* A.space <* "count=" <*> A.decimal)

src/Simplex/Chat/Controller.hs

@@ -839,6 +839,8 @@ data ChatPagination
   = CPLast Int
   | CPAfter ChatItemId Int
   | CPBefore ChatItemId Int
+  | CPAround ChatItemId Int
+  | CPInitial Int
   deriving (Show)
 
 data PaginationByTime

src/Simplex/Chat/Messages.hs

@@ -239,6 +239,12 @@ chatItemTs (CChatItem _ ci) = chatItemTs' ci
 chatItemTs' :: ChatItem c d -> UTCTime
 chatItemTs' ChatItem {meta = CIMeta {itemTs}} = itemTs
 
+chatItemCreatedAt :: CChatItem c -> UTCTime
+chatItemCreatedAt (CChatItem _ ci) = chatItemCreatedAt' ci
+
+chatItemCreatedAt' :: ChatItem c d -> UTCTime
+chatItemCreatedAt' ChatItem {meta = CIMeta {createdAt}} = createdAt
+
 chatItemTimed :: ChatItem c d -> Maybe CITimed
 chatItemTimed ChatItem {meta = CIMeta {itemTimed}} = itemTimed
 

src/Simplex/Chat/Migrations/M20241023_chat_item_autoincrement_id.hs

@@ -0,0 +1,34 @@
+{-# LANGUAGE QuasiQuotes #-}
+
+module Simplex.Chat.Migrations.M20241023_chat_item_autoincrement_id where
+
+import Database.SQLite.Simple (Query)
+import Database.SQLite.Simple.QQ (sql)
+
+m20241023_chat_item_autoincrement_id :: Query
+m20241023_chat_item_autoincrement_id =
+    [sql|
+INSERT INTO sqlite_sequence (name, seq)
+SELECT 'chat_items', MAX(ROWID) FROM chat_items;
+
+PRAGMA writable_schema=1;
+
+UPDATE sqlite_master SET sql = replace(sql, 'INTEGER PRIMARY KEY', 'INTEGER PRIMARY KEY AUTOINCREMENT')
+WHERE name = 'chat_items' AND type = 'table';
+
+PRAGMA writable_schema=0;
+|]
+
+down_m20241023_chat_item_autoincrement_id :: Query
+down_m20241023_chat_item_autoincrement_id =
+    [sql|
+DELETE FROM sqlite_sequence WHERE name = 'chat_items';
+
+PRAGMA writable_schema=1;
+
+UPDATE sqlite_master
+SET sql = replace(sql, 'INTEGER PRIMARY KEY AUTOINCREMENT', 'INTEGER PRIMARY KEY')
+WHERE name = 'chat_items' AND type = 'table';
+
+PRAGMA writable_schema=0;
+|]

src/Simplex/Chat/Migrations/chat_schema.sql

@@ -360,7 +360,7 @@ CREATE TABLE pending_group_messages(
   updated_at TEXT NOT NULL DEFAULT(datetime('now'))
 );
 CREATE TABLE chat_items(
-  chat_item_id INTEGER PRIMARY KEY,
+  chat_item_id INTEGER PRIMARY KEY AUTOINCREMENT,
   user_id INTEGER NOT NULL REFERENCES users ON DELETE CASCADE,
   contact_id INTEGER REFERENCES contacts ON DELETE CASCADE,
   group_id INTEGER REFERENCES groups ON DELETE CASCADE,
@@ -399,6 +399,7 @@ CREATE TABLE chat_items(
   fwd_from_chat_item_id INTEGER REFERENCES chat_items ON DELETE SET NULL,
   via_proxy INTEGER
 );
+CREATE TABLE sqlite_sequence(name,seq);
 CREATE TABLE chat_item_messages(
   chat_item_id INTEGER NOT NULL REFERENCES chat_items ON DELETE CASCADE,
   message_id INTEGER NOT NULL UNIQUE REFERENCES messages ON DELETE CASCADE,
@@ -429,7 +430,6 @@ CREATE TABLE commands(
   created_at TEXT NOT NULL DEFAULT(datetime('now')),
   updated_at TEXT NOT NULL DEFAULT(datetime('now'))
 );
-CREATE TABLE sqlite_sequence(name,seq);
 CREATE TABLE settings(
   settings_id INTEGER PRIMARY KEY,
   chat_item_ttl INTEGER,

src/Simplex/Chat/Store/Messages.hs

@@ -3,6 +3,7 @@
 {-# LANGUAGE GADTs #-}
 {-# LANGUAGE KindSignatures #-}
 {-# LANGUAGE LambdaCase #-}
+{-# LANGUAGE MultiWayIf #-}
 {-# LANGUAGE NamedFieldPuns #-}
 {-# LANGUAGE OverloadedStrings #-}
 {-# LANGUAGE PatternSynonyms #-}
@@ -951,33 +952,37 @@ getDirectChat :: DB.Connection -> VersionRangeChat -> User -> Int64 -> ChatPagin
 getDirectChat db vr user contactId pagination search_ = do
   let search = fromMaybe "" search_
   ct <- getContact db vr user contactId
-  liftIO $ case pagination of
-    CPLast count -> getDirectChatLast_ db user ct count search
+  case pagination of
+    CPLast count -> liftIO $ getDirectChatLast_ db user ct count search
     CPAfter afterId count -> getDirectChatAfter_ db user ct afterId count search
     CPBefore beforeId count -> getDirectChatBefore_ db user ct beforeId count search
+    CPAround aroundId count -> getDirectChatAround_ db user ct aroundId count search
+    CPInitial count -> do
+      unless (null search) $ throwError $ SEInternalError "initial chat pagination doesn't support search"
+      getDirectChatInitial_ db user ct count
 
 -- the last items in reverse order (the last item in the conversation is the first in the returned list)
 getDirectChatLast_ :: DB.Connection -> User -> Contact -> Int -> String -> IO (Chat 'CTDirect)
-getDirectChatLast_ db user@User {userId} ct@Contact {contactId} count search = do
+getDirectChatLast_ db user ct count search = do
   let stats = ChatStats {unreadCount = 0, minUnreadItemId = 0, unreadChat = False}
-  chatItemIds <- getDirectChatItemIdsLast_
+  chatItemIds <- getDirectChatItemIdsLast_ db user ct count search
   currentTs <- getCurrentTime
   chatItems <- mapM (safeGetDirectItem db user ct currentTs) chatItemIds
   pure $ Chat (DirectChat ct) (reverse chatItems) stats
-  where
-    getDirectChatItemIdsLast_ :: IO [ChatItemId]
-    getDirectChatItemIdsLast_ =
-      map fromOnly
-        <$> DB.query
-          db
-          [sql|
-            SELECT chat_item_id
-            FROM chat_items
-            WHERE user_id = ? AND contact_id = ? AND item_text LIKE '%' || ? || '%'
-            ORDER BY created_at DESC, chat_item_id DESC
-            LIMIT ?
-          |]
-          (userId, contactId, search, count)
+
+getDirectChatItemIdsLast_ :: DB.Connection -> User -> Contact -> Int -> String -> IO [ChatItemId]
+getDirectChatItemIdsLast_ db User {userId} Contact {contactId} count search =
+  map fromOnly
+    <$> DB.query
+      db
+      [sql|
+        SELECT chat_item_id
+        FROM chat_items
+        WHERE user_id = ? AND contact_id = ? AND item_text LIKE '%' || ? || '%'
+        ORDER BY created_at DESC, chat_item_id DESC
+        LIMIT ?
+      |]
+      (userId, contactId, search, count)
 
 safeGetDirectItem :: DB.Connection -> User -> Contact -> UTCTime -> ChatItemId -> IO (CChatItem 'CTDirect)
 safeGetDirectItem db user ct currentTs itemId =
@@ -1021,51 +1026,96 @@ getDirectChatItemLast db user@User {userId} contactId = do
         (userId, contactId)
   getDirectChatItem db user contactId chatItemId
 
-getDirectChatAfter_ :: DB.Connection -> User -> Contact -> ChatItemId -> Int -> String -> IO (Chat 'CTDirect)
-getDirectChatAfter_ db user@User {userId} ct@Contact {contactId} afterChatItemId count search = do
+getDirectChatAfter_ :: DB.Connection -> User -> Contact -> ChatItemId -> Int -> String -> ExceptT StoreError IO (Chat 'CTDirect)
+getDirectChatAfter_ db user ct@Contact {contactId} afterChatItemId count search = do
   let stats = ChatStats {unreadCount = 0, minUnreadItemId = 0, unreadChat = False}
-  chatItemIds <- getDirectChatItemIdsAfter_
-  currentTs <- getCurrentTime
-  chatItems <- mapM (safeGetDirectItem db user ct currentTs) chatItemIds
+  afterChatItem <- getDirectChatItem db user contactId afterChatItemId
+  chatItemIds <- liftIO $ getDirectChatItemIdsAfter_ db user ct afterChatItemId count search (chatItemCreatedAt afterChatItem)
+  currentTs <- liftIO getCurrentTime
+  chatItems <- liftIO $ mapM (safeGetDirectItem db user ct currentTs) chatItemIds
   pure $ Chat (DirectChat ct) chatItems stats
-  where
-    getDirectChatItemIdsAfter_ :: IO [ChatItemId]
-    getDirectChatItemIdsAfter_ =
-      map fromOnly
-        <$> DB.query
-          db
-          [sql|
-            SELECT chat_item_id
-            FROM chat_items
-            WHERE user_id = ? AND contact_id = ? AND item_text LIKE '%' || ? || '%'
-              AND chat_item_id > ?
-            ORDER BY created_at ASC, chat_item_id ASC
-            LIMIT ?
-          |]
-          (userId, contactId, search, afterChatItemId, count)
 
-getDirectChatBefore_ :: DB.Connection -> User -> Contact -> ChatItemId -> Int -> String -> IO (Chat 'CTDirect)
-getDirectChatBefore_ db user@User {userId} ct@Contact {contactId} beforeChatItemId count search = do
+getDirectChatItemIdsAfter_ :: DB.Connection -> User -> Contact -> ChatItemId -> Int -> String -> UTCTime -> IO [ChatItemId]
+getDirectChatItemIdsAfter_ db User {userId} Contact {contactId} afterChatItemId count search afterChatItemCreatedAt =
+  map fromOnly
+    <$> DB.query
+      db
+      [sql|
+        SELECT chat_item_id
+        FROM chat_items
+        WHERE user_id = ? AND contact_id = ? AND item_text LIKE '%' || ? || '%'
+          AND (created_at > ? OR (created_at = ? AND chat_item_id > ?))
+        ORDER BY created_at ASC, chat_item_id ASC
+        LIMIT ?
+      |]
+      (userId, contactId, search, afterChatItemCreatedAt, afterChatItemCreatedAt, afterChatItemId, count)
+
+getDirectChatBefore_ :: DB.Connection -> User -> Contact -> ChatItemId -> Int -> String -> ExceptT StoreError IO (Chat 'CTDirect)
+getDirectChatBefore_ db user ct@Contact {contactId} beforeChatItemId count search = do
   let stats = ChatStats {unreadCount = 0, minUnreadItemId = 0, unreadChat = False}
-  chatItemIds <- getDirectChatItemsIdsBefore_
-  currentTs <- getCurrentTime
-  chatItems <- mapM (safeGetDirectItem db user ct currentTs) chatItemIds
+  beforeChatItem <- getDirectChatItem db user contactId beforeChatItemId
+  chatItemIds <- liftIO $ getDirectChatItemsIdsBefore_ db user ct beforeChatItemId count search (chatItemCreatedAt beforeChatItem)
+  currentTs <- liftIO getCurrentTime
+  chatItems <- liftIO $ mapM (safeGetDirectItem db user ct currentTs) chatItemIds
   pure $ Chat (DirectChat ct) (reverse chatItems) stats
+
+getDirectChatItemsIdsBefore_ :: DB.Connection -> User -> Contact -> ChatItemId -> Int -> String -> UTCTime -> IO [ChatItemId]
+getDirectChatItemsIdsBefore_ db User {userId} Contact {contactId} beforeChatItemId count search beforeChatItemCreatedAt =
+  map fromOnly
+    <$> DB.query
+      db
+      [sql|
+        SELECT chat_item_id
+        FROM chat_items
+        WHERE user_id = ? AND contact_id = ? AND item_text LIKE '%' || ? || '%'
+          AND (created_at < ? OR (created_at = ? AND chat_item_id < ?))
+        ORDER BY created_at DESC, chat_item_id DESC
+        LIMIT ?
+      |]
+      (userId, contactId, search, beforeChatItemCreatedAt, beforeChatItemCreatedAt, beforeChatItemId, count)
+
+getDirectChatAround_ :: DB.Connection -> User -> Contact -> ChatItemId -> Int -> String -> ExceptT StoreError IO (Chat 'CTDirect)
+getDirectChatAround_ db user ct@Contact {contactId} aroundItemId count search = do
+  let stats = ChatStats {unreadCount = 0, minUnreadItemId = 0, unreadChat = False}
+  let (fetchCountBefore, fetchCountAfter) = divideFetchCountAround_ (count - 1)
+  middleChatItem <- getDirectChatItem db user contactId aroundItemId
+  beforeIds <- liftIO $ getDirectChatItemsIdsBefore_ db user ct aroundItemId fetchCountBefore search (chatItemCreatedAt middleChatItem)
+  afterIds <- liftIO $ getDirectChatItemIdsAfter_ db user ct aroundItemId fetchCountAfter search (chatItemCreatedAt middleChatItem)
+  currentTs <- liftIO getCurrentTime
+  beforeChatItems <- liftIO $ mapM (safeGetDirectItem db user ct currentTs) beforeIds
+  afterChatItems <- liftIO $ mapM (safeGetDirectItem db user ct currentTs) afterIds
+  let remainingAfter = fetchCountAfter - length afterIds
+  let remainingBefore = fetchCountBefore - length beforeIds
+  if
+    | remainingBefore > 0 && remainingAfter <= 0 -> do
+        extraAfterIds <- liftIO $ getDirectChatItemIdsAfter_ db user ct (last afterIds) remainingBefore search (chatItemCreatedAt (last afterChatItems))
+        extraAfterItems <- liftIO $ mapM (safeGetDirectItem db user ct currentTs) extraAfterIds
+        pure $ Chat (DirectChat ct) (reverse beforeChatItems <> [middleChatItem] <> afterChatItems <> extraAfterItems) stats
+    | remainingAfter > 0 && remainingBefore <= 0 -> do
+        extraBeforeIds <- liftIO $ getDirectChatItemsIdsBefore_ db user ct (last beforeIds) remainingAfter search (chatItemCreatedAt (last beforeChatItems))
+        extraBeforeItems <- liftIO $ mapM (safeGetDirectItem db user ct currentTs) extraBeforeIds
+        pure $ Chat (DirectChat ct) (reverse (beforeChatItems <> extraBeforeItems) <> [middleChatItem] <> afterChatItems) stats
+    | otherwise ->
+        pure $ Chat (DirectChat ct) (reverse beforeChatItems <> [middleChatItem] <> afterChatItems) stats
+
+getDirectChatInitial_ :: DB.Connection -> User -> Contact -> Int -> ExceptT StoreError IO (Chat 'CTDirect)
+getDirectChatInitial_ db user@User {userId} ct@Contact {contactId} count = do
+  firstUnreadItemId_ <- liftIO getDirectChatMinUnreadItemId_
+  case firstUnreadItemId_ of
+    Just firstUnreadItemId -> getDirectChatAround_ db user ct firstUnreadItemId count ""
+    Nothing -> liftIO $ getDirectChatLast_ db user ct count ""
   where
-    getDirectChatItemsIdsBefore_ :: IO [ChatItemId]
-    getDirectChatItemsIdsBefore_ =
-      map fromOnly
-        <$> DB.query
+    getDirectChatMinUnreadItemId_ :: IO (Maybe ChatItemId)
+    getDirectChatMinUnreadItemId_ =
+      fmap join . maybeFirstRow fromOnly $
+        DB.query
           db
           [sql|
-            SELECT chat_item_id
+            SELECT MIN(chat_item_id)
             FROM chat_items
-            WHERE user_id = ? AND contact_id = ? AND item_text LIKE '%' || ? || '%'
-              AND chat_item_id < ?
-            ORDER BY created_at DESC, chat_item_id DESC
-            LIMIT ?
+            WHERE user_id = ? AND contact_id = ? AND item_status = ?
           |]
-          (userId, contactId, search, beforeChatItemId, count)
+          (userId, contactId, CISRcvNew)
 
 getGroupChat :: DB.Connection -> VersionRangeChat -> User -> Int64 -> ChatPagination -> Maybe String -> ExceptT StoreError IO (Chat 'CTGroup)
 getGroupChat db vr user groupId pagination search_ = do
@@ -1075,28 +1125,32 @@ getGroupChat db vr user groupId pagination search_ = do
     CPLast count -> liftIO $ getGroupChatLast_ db user g count search
     CPAfter afterId count -> getGroupChatAfter_ db user g afterId count search
     CPBefore beforeId count -> getGroupChatBefore_ db user g beforeId count search
+    CPAround aroundId count -> getGroupChatAround_ db user g aroundId count search
+    CPInitial count -> do
+      unless (null search) $ throwError $ SEInternalError "initial chat pagination doesn't support search"
+      getGroupChatInitial_ db user g count
 
 getGroupChatLast_ :: DB.Connection -> User -> GroupInfo -> Int -> String -> IO (Chat 'CTGroup)
-getGroupChatLast_ db user@User {userId} g@GroupInfo {groupId} count search = do
+getGroupChatLast_ db user g count search = do
   let stats = ChatStats {unreadCount = 0, minUnreadItemId = 0, unreadChat = False}
-  chatItemIds <- getGroupChatItemIdsLast_
+  chatItemIds <- getGroupChatItemIdsLast_ db user g count search
   currentTs <- getCurrentTime
   chatItems <- mapM (safeGetGroupItem db user g currentTs) chatItemIds
   pure $ Chat (GroupChat g) (reverse chatItems) stats
-  where
-    getGroupChatItemIdsLast_ :: IO [ChatItemId]
-    getGroupChatItemIdsLast_ =
-      map fromOnly
-        <$> DB.query
-          db
-          [sql|
-            SELECT chat_item_id
-            FROM chat_items
-            WHERE user_id = ? AND group_id = ? AND item_text LIKE '%' || ? || '%'
-            ORDER BY item_ts DESC, chat_item_id DESC
-            LIMIT ?
-          |]
-          (userId, groupId, search, count)
+
+getGroupChatItemIdsLast_ :: DB.Connection -> User -> GroupInfo -> Int -> String -> IO [ChatItemId]
+getGroupChatItemIdsLast_ db User {userId} GroupInfo {groupId} count search =
+  map fromOnly
+    <$> DB.query
+      db
+      [sql|
+        SELECT chat_item_id
+        FROM chat_items
+        WHERE user_id = ? AND group_id = ? AND item_text LIKE '%' || ? || '%'
+        ORDER BY item_ts DESC, chat_item_id DESC
+        LIMIT ?
+      |]
+      (userId, groupId, search, count)
 
 safeGetGroupItem :: DB.Connection -> User -> GroupInfo -> UTCTime -> ChatItemId -> IO (CChatItem 'CTGroup)
 safeGetGroupItem db user g currentTs itemId =
@@ -1141,83 +1195,130 @@ getGroupMemberChatItemLast db user@User {userId} groupId groupMemberId = do
   getGroupChatItem db user groupId chatItemId
 
 getGroupChatAfter_ :: DB.Connection -> User -> GroupInfo -> ChatItemId -> Int -> String -> ExceptT StoreError IO (Chat 'CTGroup)
-getGroupChatAfter_ db user@User {userId} g@GroupInfo {groupId} afterChatItemId count search = do
+getGroupChatAfter_ db user g@GroupInfo {groupId} afterChatItemId count search = do
   let stats = ChatStats {unreadCount = 0, minUnreadItemId = 0, unreadChat = False}
   afterChatItem <- getGroupChatItem db user groupId afterChatItemId
-  chatItemIds <- liftIO $ getGroupChatItemIdsAfter_ (chatItemTs afterChatItem)
+  chatItemIds <- liftIO $ getGroupChatItemIdsAfter_ db user g afterChatItemId count search (chatItemTs afterChatItem)
   currentTs <- liftIO getCurrentTime
   chatItems <- liftIO $ mapM (safeGetGroupItem db user g currentTs) chatItemIds
   pure $ Chat (GroupChat g) chatItems stats
-  where
-    getGroupChatItemIdsAfter_ :: UTCTime -> IO [ChatItemId]
-    getGroupChatItemIdsAfter_ afterChatItemTs =
-      map fromOnly
-        <$> DB.query
-          db
-          [sql|
-            SELECT chat_item_id
-            FROM chat_items
-            WHERE user_id = ? AND group_id = ? AND item_text LIKE '%' || ? || '%'
-              AND (item_ts > ? OR (item_ts = ? AND chat_item_id > ?))
-            ORDER BY item_ts ASC, chat_item_id ASC
-            LIMIT ?
-          |]
-          (userId, groupId, search, afterChatItemTs, afterChatItemTs, afterChatItemId, count)
+
+getGroupChatItemIdsAfter_ :: DB.Connection -> User -> GroupInfo -> ChatItemId -> Int -> String -> UTCTime -> IO [ChatItemId]
+getGroupChatItemIdsAfter_ db User {userId} GroupInfo {groupId} afterChatItemId count search afterChatItemTs =
+  map fromOnly
+    <$> DB.query
+      db
+      [sql|
+        SELECT chat_item_id
+        FROM chat_items
+        WHERE user_id = ? AND group_id = ? AND item_text LIKE '%' || ? || '%'
+          AND (item_ts > ? OR (item_ts = ? AND chat_item_id > ?))
+        ORDER BY item_ts ASC, chat_item_id ASC
+        LIMIT ?
+      |]
+      (userId, groupId, search, afterChatItemTs, afterChatItemTs, afterChatItemId, count)
 
 getGroupChatBefore_ :: DB.Connection -> User -> GroupInfo -> ChatItemId -> Int -> String -> ExceptT StoreError IO (Chat 'CTGroup)
-getGroupChatBefore_ db user@User {userId} g@GroupInfo {groupId} beforeChatItemId count search = do
+getGroupChatBefore_ db user g@GroupInfo {groupId} beforeChatItemId count search = do
   let stats = ChatStats {unreadCount = 0, minUnreadItemId = 0, unreadChat = False}
   beforeChatItem <- getGroupChatItem db user groupId beforeChatItemId
-  chatItemIds <- liftIO $ getGroupChatItemIdsBefore_ (chatItemTs beforeChatItem)
+  chatItemIds <- liftIO $ getGroupChatItemIdsBefore_ db user g beforeChatItemId count search (chatItemTs beforeChatItem)
   currentTs <- liftIO getCurrentTime
   chatItems <- liftIO $ mapM (safeGetGroupItem db user g currentTs) chatItemIds
   pure $ Chat (GroupChat g) (reverse chatItems) stats
+
+getGroupChatItemIdsBefore_ :: DB.Connection -> User -> GroupInfo -> ChatItemId -> Int -> String -> UTCTime -> IO [ChatItemId]
+getGroupChatItemIdsBefore_ db User {userId} GroupInfo {groupId} beforeChatItemId count search beforeChatItemTs =
+  map fromOnly
+    <$> DB.query
+      db
+      [sql|
+        SELECT chat_item_id
+        FROM chat_items
+        WHERE user_id = ? AND group_id = ? AND item_text LIKE '%' || ? || '%'
+          AND (item_ts < ? OR (item_ts = ? AND chat_item_id < ?))
+        ORDER BY item_ts DESC, chat_item_id DESC
+        LIMIT ?
+      |]
+      (userId, groupId, search, beforeChatItemTs, beforeChatItemTs, beforeChatItemId, count)
+
+getGroupChatAround_ :: DB.Connection -> User -> GroupInfo -> ChatItemId -> Int -> String -> ExceptT StoreError IO (Chat 'CTGroup)
+getGroupChatAround_ db user g@GroupInfo {groupId} aroundItemId count search = do
+  let stats = ChatStats {unreadCount = 0, minUnreadItemId = 0, unreadChat = False}
+  let (fetchCountBefore, fetchCountAfter) = divideFetchCountAround_ (count - 1)
+  middleChatItem <- getGroupChatItem db user groupId aroundItemId
+  beforeIds <- liftIO $ getGroupChatItemIdsBefore_ db user g aroundItemId fetchCountBefore search (chatItemTs middleChatItem)
+  afterIds <- liftIO $ getGroupChatItemIdsAfter_ db user g aroundItemId fetchCountAfter search (chatItemTs middleChatItem)
+  currentTs <- liftIO getCurrentTime
+  beforeChatItems <- liftIO $ mapM (safeGetGroupItem db user g currentTs) beforeIds
+  afterChatItems <- liftIO $ mapM (safeGetGroupItem db user g currentTs) afterIds
+  let remainingAfter = fetchCountAfter - length afterIds
+  let remainingBefore = fetchCountBefore - length beforeIds
+  if
+    | remainingBefore > 0 && remainingAfter <= 0 -> do
+        extraAfterIds <- liftIO $ getGroupChatItemIdsAfter_ db user g (last afterIds) remainingBefore search (chatItemTs (last afterChatItems))
+        extraAfterItems <- liftIO $ mapM (safeGetGroupItem db user g currentTs) extraAfterIds
+        pure $ Chat (GroupChat g) (reverse beforeChatItems <> [middleChatItem] <> afterChatItems <> extraAfterItems) stats
+    | remainingAfter > 0 && remainingBefore <= 0 -> do
+        extraBeforeIds <- liftIO $ getGroupChatItemIdsBefore_ db user g (last beforeIds) remainingAfter search (chatItemTs (last beforeChatItems))
+        extraBeforeItems <- liftIO $ mapM (safeGetGroupItem db user g currentTs) extraBeforeIds
+        pure $ Chat (GroupChat g) (reverse (beforeChatItems <> extraBeforeItems) <> [middleChatItem] <> afterChatItems) stats
+    | otherwise ->
+        pure $ Chat (GroupChat g) (reverse beforeChatItems <> [middleChatItem] <> afterChatItems) stats
+
+getGroupChatInitial_ :: DB.Connection -> User -> GroupInfo -> Int -> ExceptT StoreError IO (Chat 'CTGroup)
+getGroupChatInitial_ db user@User {userId} g@GroupInfo {groupId} count = do
+  firstUnreadItemId_ <- liftIO getGroupChatMinUnreadItemId_
+  case firstUnreadItemId_ of
+    Just firstUnreadItemId -> getGroupChatAround_ db user g firstUnreadItemId count ""
+    Nothing -> liftIO $ getGroupChatLast_ db user g count ""
   where
-    getGroupChatItemIdsBefore_ :: UTCTime -> IO [ChatItemId]
-    getGroupChatItemIdsBefore_ beforeChatItemTs =
-      map fromOnly
-        <$> DB.query
+    getGroupChatMinUnreadItemId_ :: IO (Maybe ChatItemId)
+    getGroupChatMinUnreadItemId_ =
+      fmap join . maybeFirstRow fromOnly $
+        DB.query
           db
           [sql|
-            SELECT chat_item_id
+            SELECT MIN(chat_item_id)
             FROM chat_items
-            WHERE user_id = ? AND group_id = ? AND item_text LIKE '%' || ? || '%'
-              AND (item_ts < ? OR (item_ts = ? AND chat_item_id < ?))
-            ORDER BY item_ts DESC, chat_item_id DESC
-            LIMIT ?
+            WHERE user_id = ? AND group_id = ? AND item_status = ?
           |]
-          (userId, groupId, search, beforeChatItemTs, beforeChatItemTs, beforeChatItemId, count)
+          (userId, groupId, CISRcvNew)
 
 getLocalChat :: DB.Connection -> User -> Int64 -> ChatPagination -> Maybe String -> ExceptT StoreError IO (Chat 'CTLocal)
 getLocalChat db user folderId pagination search_ = do
   let search = fromMaybe "" search_
   nf <- getNoteFolder db user folderId
-  liftIO $ case pagination of
-    CPLast count -> getLocalChatLast_ db user nf count search
+  case pagination of
+    CPLast count -> liftIO $ getLocalChatLast_ db user nf count search
     CPAfter afterId count -> getLocalChatAfter_ db user nf afterId count search
     CPBefore beforeId count -> getLocalChatBefore_ db user nf beforeId count search
+    CPAround aroundId count -> getLocalChatAround_ db user nf aroundId count search
+    CPInitial count -> do
+      unless (null search) $ throwError $ SEInternalError "initial chat pagination doesn't support search"
+      getLocalChatInitial_ db user nf count
 
 getLocalChatLast_ :: DB.Connection -> User -> NoteFolder -> Int -> String -> IO (Chat 'CTLocal)
-getLocalChatLast_ db user@User {userId} nf@NoteFolder {noteFolderId} count search = do
+getLocalChatLast_ db user nf count search = do
   let stats = ChatStats {unreadCount = 0, minUnreadItemId = 0, unreadChat = False}
-  chatItemIds <- getLocalChatItemIdsLast_
+  chatItemIds <- getLocalChatItemIdsLast_ db user nf count search
   currentTs <- getCurrentTime
   chatItems <- mapM (safeGetLocalItem db user nf currentTs) chatItemIds
   pure $ Chat (LocalChat nf) (reverse chatItems) stats
-  where
-    getLocalChatItemIdsLast_ :: IO [ChatItemId]
-    getLocalChatItemIdsLast_ =
-      map fromOnly
-        <$> DB.query
-          db
-          [sql|
-            SELECT chat_item_id
-            FROM chat_items
-            WHERE user_id = ? AND note_folder_id = ? AND item_text LIKE '%' || ? || '%'
-            ORDER BY created_at DESC, chat_item_id DESC
-            LIMIT ?
-          |]
-          (userId, noteFolderId, search, count)
+
+getLocalChatItemIdsLast_ :: DB.Connection -> User -> NoteFolder -> Int -> String -> IO [ChatItemId]
+getLocalChatItemIdsLast_ db User {userId} NoteFolder {noteFolderId} count search =
+  map fromOnly
+    <$> DB.query
+      db
+      [sql|
+        SELECT chat_item_id
+        FROM chat_items
+        WHERE user_id = ? AND note_folder_id = ? AND item_text LIKE '%' || ? || '%'
+        ORDER BY created_at DESC, chat_item_id DESC
+        LIMIT ?
+      |]
+      (userId, noteFolderId, search, count)
 
 safeGetLocalItem :: DB.Connection -> User -> NoteFolder -> UTCTime -> ChatItemId -> IO (CChatItem 'CTLocal)
 safeGetLocalItem db user NoteFolder {noteFolderId} currentTs itemId =
@@ -1245,51 +1346,105 @@ safeToLocalItem currentTs itemId = \case
                 file = Nothing
               }
 
-getLocalChatAfter_ :: DB.Connection -> User -> NoteFolder -> ChatItemId -> Int -> String -> IO (Chat 'CTLocal)
-getLocalChatAfter_ db user@User {userId} nf@NoteFolder {noteFolderId} afterChatItemId count search = do
+getLocalChatAfter_ :: DB.Connection -> User -> NoteFolder -> ChatItemId -> Int -> String -> ExceptT StoreError IO (Chat 'CTLocal)
+getLocalChatAfter_ db user nf@NoteFolder {noteFolderId} afterChatItemId count search = do
   let stats = ChatStats {unreadCount = 0, minUnreadItemId = 0, unreadChat = False}
-  chatItemIds <- getLocalChatItemIdsAfter_
-  currentTs <- getCurrentTime
-  chatItems <- mapM (safeGetLocalItem db user nf currentTs) chatItemIds
+  afterChatItem <- getLocalChatItem db user noteFolderId afterChatItemId
+  chatItemIds <- liftIO $ getLocalChatItemIdsAfter_ db user nf afterChatItemId count search (chatItemCreatedAt afterChatItem)
+  currentTs <- liftIO getCurrentTime
+  chatItems <- liftIO $ mapM (safeGetLocalItem db user nf currentTs) chatItemIds
   pure $ Chat (LocalChat nf) chatItems stats
-  where
-    getLocalChatItemIdsAfter_ :: IO [ChatItemId]
-    getLocalChatItemIdsAfter_ =
-      map fromOnly
-        <$> DB.query
-          db
-          [sql|
-            SELECT chat_item_id
-            FROM chat_items
-            WHERE user_id = ? AND note_folder_id = ? AND item_text LIKE '%' || ? || '%'
-              AND chat_item_id > ?
-            ORDER BY created_at ASC, chat_item_id ASC
-            LIMIT ?
-          |]
-          (userId, noteFolderId, search, afterChatItemId, count)
 
-getLocalChatBefore_ :: DB.Connection -> User -> NoteFolder -> ChatItemId -> Int -> String -> IO (Chat 'CTLocal)
-getLocalChatBefore_ db user@User {userId} nf@NoteFolder {noteFolderId} beforeChatItemId count search = do
+getLocalChatItemIdsAfter_ :: DB.Connection -> User -> NoteFolder -> ChatItemId -> Int -> String -> UTCTime -> IO [ChatItemId]
+getLocalChatItemIdsAfter_ db User {userId} NoteFolder {noteFolderId} afterChatItemId count search afterChatItemCreatedAt =
+  map fromOnly
+    <$> DB.query
+      db
+      [sql|
+        SELECT chat_item_id
+        FROM chat_items
+        WHERE user_id = ? AND note_folder_id = ? AND item_text LIKE '%' || ? || '%'
+          AND (created_at > ? OR (created_at = ? AND chat_item_id > ?))
+        ORDER BY created_at ASC, chat_item_id ASC
+        LIMIT ?
+      |]
+      (userId, noteFolderId, search, afterChatItemCreatedAt, afterChatItemCreatedAt, afterChatItemId, count)
+
+getLocalChatBefore_ :: DB.Connection -> User -> NoteFolder -> ChatItemId -> Int -> String -> ExceptT StoreError IO (Chat 'CTLocal)
+getLocalChatBefore_ db user nf@NoteFolder {noteFolderId} beforeChatItemId count search = do
   let stats = ChatStats {unreadCount = 0, minUnreadItemId = 0, unreadChat = False}
-  chatItemIds <- getLocalChatItemIdsBefore_
-  currentTs <- getCurrentTime
-  chatItems <- mapM (safeGetLocalItem db user nf currentTs) chatItemIds
+  beforeChatItem <- getLocalChatItem db user noteFolderId beforeChatItemId
+  chatItemIds <- liftIO $ getLocalChatItemIdsBefore_ db user nf beforeChatItemId count search (chatItemCreatedAt beforeChatItem)
+  currentTs <- liftIO getCurrentTime
+  chatItems <- liftIO $ mapM (safeGetLocalItem db user nf currentTs) chatItemIds
   pure $ Chat (LocalChat nf) (reverse chatItems) stats
+
+getLocalChatItemIdsBefore_ :: DB.Connection -> User -> NoteFolder -> ChatItemId -> Int -> String -> UTCTime -> IO [ChatItemId]
+getLocalChatItemIdsBefore_ db User {userId} NoteFolder {noteFolderId} beforeChatItemId count search beforeChatItemCreatedAt =
+  map fromOnly
+    <$> DB.query
+      db
+      [sql|
+        SELECT chat_item_id
+        FROM chat_items
+        WHERE user_id = ? AND note_folder_id = ? AND item_text LIKE '%' || ? || '%'
+          AND (created_at < ? OR (created_at = ? AND chat_item_id < ?))
+        ORDER BY created_at DESC, chat_item_id DESC
+        LIMIT ?
+      |]
+      (userId, noteFolderId, search, beforeChatItemCreatedAt, beforeChatItemCreatedAt, beforeChatItemId, count)
+
+getLocalChatAround_ :: DB.Connection -> User -> NoteFolder -> ChatItemId -> Int -> String -> ExceptT StoreError IO (Chat 'CTLocal)
+getLocalChatAround_ db user nf@NoteFolder {noteFolderId} aroundItemId count search = do
+  let stats = ChatStats {unreadCount = 0, minUnreadItemId = 0, unreadChat = False}
+  let (fetchCountBefore, fetchCountAfter) = divideFetchCountAround_ (count - 1)
+  middleChatItem <- getLocalChatItem db user noteFolderId aroundItemId
+  beforeIds <- liftIO $ getLocalChatItemIdsBefore_ db user nf aroundItemId fetchCountBefore search (chatItemCreatedAt middleChatItem)
+  afterIds <- liftIO $ getLocalChatItemIdsAfter_ db user nf aroundItemId fetchCountAfter search (chatItemCreatedAt middleChatItem)
+  currentTs <- liftIO getCurrentTime
+  beforeChatItems <- liftIO $ mapM (safeGetLocalItem db user nf currentTs) beforeIds
+  afterChatItems <- liftIO $ mapM (safeGetLocalItem db user nf currentTs) afterIds
+  let remainingAfter = fetchCountAfter - length afterIds
+  let remainingBefore = fetchCountBefore - length beforeIds
+  if
+    | remainingBefore > 0 && remainingAfter <= 0 -> do
+        extraAfterIds <- liftIO $ getLocalChatItemIdsAfter_ db user nf (last afterIds) remainingBefore search (chatItemCreatedAt (last afterChatItems))
+        extraAfterItems <- liftIO $ mapM (safeGetLocalItem db user nf currentTs) extraAfterIds
+        pure $ Chat (LocalChat nf) (reverse beforeChatItems <> [middleChatItem] <> afterChatItems <> extraAfterItems) stats
+    | remainingAfter > 0 && remainingBefore <= 0 -> do
+        extraBeforeIds <- liftIO $ getLocalChatItemIdsBefore_ db user nf (last beforeIds) remainingAfter search (chatItemCreatedAt (last beforeChatItems))
+        extraBeforeItems <- liftIO $ mapM (safeGetLocalItem db user nf currentTs) extraBeforeIds
+        pure $ Chat (LocalChat nf) (reverse (beforeChatItems <> extraBeforeItems) <> [middleChatItem] <> afterChatItems) stats
+    | otherwise ->
+        pure $ Chat (LocalChat nf) (reverse beforeChatItems <> [middleChatItem] <> afterChatItems) stats
+
+getLocalChatInitial_ :: DB.Connection -> User -> NoteFolder -> Int -> ExceptT StoreError IO (Chat 'CTLocal)
+getLocalChatInitial_ db user@User {userId} nf@NoteFolder {noteFolderId} count = do
+  firstUnreadItemId_ <- liftIO getLocalChatMinUnreadItemId_
+  case firstUnreadItemId_ of
+    Just firstUnreadItemId -> do
+      chat <- getLocalChatAround_ db user nf firstUnreadItemId count ""
+      let items = chatItems chat
+      if null items || length items == count
+        then pure chat
+        else do
+          let remainingCount = count - length items
+          let afterId = cchatItemId $ last items
+          after <- getLocalChatAfter_ db user nf afterId remainingCount ""
+          pure $ chat {chatItems = chatItems chat <> chatItems after}
+    Nothing -> liftIO $ getLocalChatLast_ db user nf count ""
   where
-    getLocalChatItemIdsBefore_ :: IO [ChatItemId]
-    getLocalChatItemIdsBefore_ =
-      map fromOnly
-        <$> DB.query
+    getLocalChatMinUnreadItemId_ :: IO (Maybe ChatItemId)
+    getLocalChatMinUnreadItemId_ =
+      fmap join . maybeFirstRow fromOnly $
+        DB.query
           db
           [sql|
-            SELECT chat_item_id
+            SELECT MIN(chat_item_id)
             FROM chat_items
-            WHERE user_id = ? AND note_folder_id = ? AND item_text LIKE '%' || ? || '%'
-              AND chat_item_id < ?
-            ORDER BY created_at DESC, chat_item_id DESC
-            LIMIT ?
+            WHERE user_id = ? AND note_folder_id = ? AND item_status = ?
           |]
-          (userId, noteFolderId, search, beforeChatItemId, count)
+          (userId, noteFolderId, CISRcvNew)
 
 toChatItemRef :: (ChatItemId, Maybe Int64, Maybe Int64, Maybe Int64) -> Either StoreError (ChatRef, ChatItemId)
 toChatItemRef = \case
@@ -1581,6 +1736,13 @@ getAllChatItems db vr user@User {userId} pagination search_ = do
       CPLast count -> liftIO $ getAllChatItemsLast_ count
       CPAfter afterId count -> liftIO . getAllChatItemsAfter_ afterId count . aChatItemTs =<< getAChatItem_ afterId
       CPBefore beforeId count -> liftIO . getAllChatItemsBefore_ beforeId count . aChatItemTs =<< getAChatItem_ beforeId
+      CPAround aroundId count -> liftIO . getAllChatItemsAround_ aroundId count . aChatItemTs =<< getAChatItem_ aroundId
+      CPInitial count -> do
+        unless (null search) $ throwError $ SEInternalError "initial chat pagination doesn't support search"
+        firstUnreadItemId <- liftIO getFirstUnreadItemId_
+        case firstUnreadItemId of
+          Just itemId -> liftIO . getAllChatItemsAround_ itemId count . aChatItemTs =<< getAChatItem_ itemId
+          Nothing -> liftIO $ getAllChatItemsLast_ count
   mapM (uncurry (getAChatItem db vr user)) itemRefs
   where
     search = fromMaybe "" search_
@@ -1624,6 +1786,31 @@ getAllChatItems db vr user@User {userId} pagination search_ = do
             LIMIT ?
           |]
           (userId, search, beforeTs, beforeTs, beforeId, count)
+    getChatItem chatId =
+      DB.query
+        db
+        [sql|
+          SELECT chat_item_id, contact_id, group_id, note_folder_id
+          FROM chat_items
+          WHERE chat_item_id = ?
+        |]
+        (Only chatId)
+    getAllChatItemsAround_ aroundId count aroundTs = do
+      let (fetchCountBefore, fetchCountAfter) = divideFetchCountAround_ (count - 1)
+      itemsBefore <- getAllChatItemsBefore_ aroundId fetchCountBefore aroundTs
+      item <- getChatItem aroundId
+      itemsAfter <- getAllChatItemsAfter_ aroundId fetchCountAfter aroundTs
+      pure $ itemsBefore <> item <> itemsAfter
+    getFirstUnreadItemId_ =
+      fmap join . maybeFirstRow fromOnly $
+        DB.query
+          db
+          [sql|
+            SELECT MIN(chat_item_id)
+            FROM chat_items
+            WHERE user_id = ? AND item_status = ?
+          |]
+          (userId, CISRcvNew)
 
 getChatItemIdsByAgentMsgId :: DB.Connection -> Int64 -> AgentMsgId -> IO [ChatItemId]
 getChatItemIdsByAgentMsgId db connId msgId =
@@ -2650,3 +2837,10 @@ getGroupHistoryItems db user@User {userId} GroupInfo {groupId} count = do
             LIMIT ?
           |]
           (userId, groupId, rcvMsgContentTag, sndMsgContentTag, count)
+
+divideFetchCountAround_ :: Int -> (Int, Int)
+divideFetchCountAround_ count =
+  let fetchCountEachSide = count `div` 2
+      fetchCountBefore = fetchCountEachSide + if count `mod` 2 /= 0 then 1 else 0
+      fetchCountAfter = fetchCountEachSide
+   in (fetchCountBefore, fetchCountAfter)

src/Simplex/Chat/Store/Migrations.hs

@@ -114,6 +114,7 @@ import Simplex.Chat.Migrations.M20240827_calls_uuid
 import Simplex.Chat.Migrations.M20240920_user_order
 import Simplex.Chat.Migrations.M20241008_indexes
 import Simplex.Chat.Migrations.M20241010_contact_requests_contact_id
+import Simplex.Chat.Migrations.M20241023_chat_item_autoincrement_id
 import Simplex.Messaging.Agent.Store.SQLite.Migrations (Migration (..))
 
 schemaMigrations :: [(String, Query, Maybe Query)]
@@ -227,7 +228,8 @@ schemaMigrations =
     ("20240827_calls_uuid", m20240827_calls_uuid, Just down_m20240827_calls_uuid),
     ("20240920_user_order", m20240920_user_order, Just down_m20240920_user_order),
     ("20241008_indexes", m20241008_indexes, Just down_m20241008_indexes),
-    ("20241010_contact_requests_contact_id", m20241010_contact_requests_contact_id, Just down_m20241010_contact_requests_contact_id)
+    ("20241010_contact_requests_contact_id", m20241010_contact_requests_contact_id, Just down_m20241010_contact_requests_contact_id),
+    ("20241023_chat_item_autoincrement_id", m20241023_chat_item_autoincrement_id, Just down_m20241023_chat_item_autoincrement_id)
   ]
 
 -- | The list of migrations in ascending order by date

tests/ChatClient.hs

@@ -51,6 +51,7 @@ import qualified Simplex.Messaging.Crypto.Ratchet as CR
 import Simplex.Messaging.Protocol (srvHostnamesSMPClientVersion)
 import Simplex.Messaging.Server (runSMPServerBlocking)
 import Simplex.Messaging.Server.Env.STM
+import Simplex.Messaging.Server.MsgStore.Types (AMSType (..), SMSType (..))
 import Simplex.Messaging.Transport
 import Simplex.Messaging.Transport.Server (ServerCredentials (..), defaultTransportServerConfig)
 import Simplex.Messaging.Version
@@ -424,6 +425,9 @@ smpServerCfg =
       tbqSize = 1,
       -- serverTbqSize = 1,
       msgQueueQuota = 16,
+      msgStoreType = AMSType SMSMemory,
+      maxJournalMsgCount = 1000,
+      maxJournalStateLines = 1000,
       queueIdBytes = 12,
       msgIdBytes = 6,
       storeLogFile = Nothing,

tests/ChatTests/Direct.hs

@@ -66,6 +66,7 @@ chatDirectTests = do
     it "repeat AUTH errors disable contact" testRepeatAuthErrorsDisableContact
     it "should send multiline message" testMultilineMessage
     it "send large message" testLargeMessage
+    it "initial chat pagination" testChatPaginationInitial
   describe "batch send messages" $ do
     it "send multiple messages api" testSendMulti
     it "send multiple timed messages" testSendMultiTimed
@@ -210,6 +211,7 @@ testAddContact = versionTestMatrix2 runTestAddContact
           -- pagination
           alice #$> ("/_get chat @2 after=" <> itemId 1 <> " count=100", chat, [(0, "hello there"), (0, "how are you?")])
           alice #$> ("/_get chat @2 before=" <> itemId 2 <> " count=100", chat, features <> [(1, "hello there 🙂")])
+          alice #$> ("/_get chat @2 around=" <> itemId 2 <> " count=3", chat, [(1, "hello there 🙂"), (0, "hello there"), (0, "how are you?")])
           -- search
           alice #$> ("/_get chat @2 count=100 search=ello ther", chat, [(1, "hello there 🙂"), (0, "hello there")])
           -- read messages
@@ -360,6 +362,49 @@ testMarkReadDirect = testChat2 aliceProfile bobProfile $ \alice bob -> do
   let itemIds = intercalate "," $ map show [i - 3 .. i]
   bob #$> ("/_read chat items @2 " <> itemIds, id, "ok")
 
+testChatPaginationInitial :: HasCallStack => FilePath -> IO ()
+testChatPaginationInitial = testChatOpts2 opts aliceProfile bobProfile $ \alice bob -> do
+  connectUsers alice bob
+  -- Wait, otherwise ids are going to be wrong.
+  threadDelay 1000000
+
+  -- Send messages from alice to bob
+  forM_ ([1 .. 10] :: [Int]) $ \n -> alice #> ("@bob " <> show n)
+
+  -- Bob receives the messages.
+  forM_ ([1 .. 10] :: [Int]) $ \n -> bob <# ("alice> " <> show n)
+
+  -- All messages are unread for bob, should return area around unread
+  bob #$> ("/_get chat @2 initial=3", chat, [(0, "Audio/video calls: enabled"), (0, "1"), (0, "2")])
+
+  -- Read next 2 items
+  let itemIds = intercalate "," $ map itemId [1 .. 2]
+  bob #$> ("/_read chat items @2 " <> itemIds, id, "ok")
+  bob #$> ("/_get chat @2 initial=3", chat, [(0, "2"), (0, "3"), (0, "4")])
+
+  -- Read all items
+  bob #$> ("/_read chat @2", id, "ok")
+  bob #$> ("/_get chat @2 initial=3", chat, [(0, "8"), (0, "9"), (0, "10")])
+  bob #$> ("/_get chat @2 initial=5", chat, [(0, "6"), (0, "7"), (0, "8"), (0, "9"), (0, "10")])
+
+  -- Clear chat, send a few extra message and assert page size is consistent
+  bob #$> ("/clear alice", id, "alice: all messages are removed locally ONLY")
+  forM_ ([1 .. 10] :: [Int]) $ \n -> alice #> ("@bob " <> show n)
+  forM_ ([1 .. 10] :: [Int]) $ \n -> bob <# ("alice> " <> show n)
+
+  bob #$> ("/_get chat @2 initial=5", chat, [(0, "1"), (0, "2"), (0, "3"), (0, "4"), (0, "5")])
+  let newItemIds = intercalate "," $ map itemId [11 .. 12] -- Read, 1, 2
+  bob #$> ("/_read chat items @2 " <> newItemIds, id, "ok")
+  bob #$> ("/_get chat @2 initial=5", chat, [(0, "1"), (0, "2"), (0, "3"), (0, "4"), (0, "5")])
+  let allButLastId = intercalate "," $ map itemId [13 .. 19] -- Read all but last
+  bob #$> ("/_read chat items @2 " <> allButLastId, id, "ok")
+  bob #$> ("/_get chat @2 initial=5", chat, [(0, "6"), (0, "7"), (0, "8"), (0, "9"), (0, "10")])
+  where
+    opts =
+      testOpts
+        { markRead = False
+        }
+
 testDuplicateContactsSeparate :: HasCallStack => FilePath -> IO ()
 testDuplicateContactsSeparate =
   testChat2 aliceProfile bobProfile $
@@ -791,7 +836,7 @@ testDirectMessageDelete =
       alice @@@ [("@bob", lastChatFeature)]
       alice #$> ("/_get chat @2 count=100", chat, chatFeatures)
 
-      -- alice: msg id 1
+      -- alice: msg id 3
       bob ##> ("/_update item @2 " <> itemId 2 <> " text hey alice")
       bob <# "@alice [edited] > hello 🙂"
       bob <## "      hey alice"
@@ -806,12 +851,12 @@ testDirectMessageDelete =
       alice @@@ [("@bob", "hey alice [marked deleted]")]
       alice #$> ("/_get chat @2 count=100", chat, chatFeatures <> [(0, "hey alice [marked deleted]")])
 
-      -- alice: deletes msg id 1 that was broadcast deleted by bob
-      alice #$> ("/_delete item @2 " <> itemId 1 <> " internal", id, "message deleted")
+      -- alice: deletes msg id 3 that was broadcast deleted by bob
+      alice #$> ("/_delete item @2 " <> itemId 3 <> " internal", id, "message deleted")
       alice @@@ [("@bob", lastChatFeature)]
       alice #$> ("/_get chat @2 count=100", chat, chatFeatures)
 
-      -- alice: msg id 1, bob: msg id 3 (quoting message alice deleted locally)
+      -- alice: msg id 4, bob: msg id 3 (quoting message alice deleted locally)
       bob `send` "> @alice (hello 🙂) do you receive my messages?"
       bob <# "@alice > hello 🙂"
       bob <## "      do you receive my messages?"
@@ -819,14 +864,14 @@ testDirectMessageDelete =
       alice <## "      do you receive my messages?"
       alice @@@ [("@bob", "do you receive my messages?")]
       alice #$> ("/_get chat @2 count=100", chat', chatFeatures' <> [((0, "do you receive my messages?"), Just (1, "hello 🙂"))])
-      alice #$> ("/_delete item @2 " <> itemId 1 <> " broadcast", id, "cannot delete this item")
+      alice #$> ("/_delete item @2 " <> itemId 4 <> " broadcast", id, "cannot delete this item")
 
-      -- alice: msg id 2, bob: msg id 4
+      -- alice: msg id 5, bob: msg id 4
       bob #> "@alice how are you?"
       alice <# "bob> how are you?"
 
-      -- alice: deletes msg id 2
-      alice #$> ("/_delete item @2 " <> itemId 2 <> " internal", id, "message deleted")
+      -- alice: deletes msg id 5
+      alice #$> ("/_delete item @2 " <> itemId 5 <> " internal", id, "message deleted")
 
       -- bob: marks deleted msg id 4 (that alice deleted locally)
       bob #$> ("/_delete item @2 " <> itemId 4 <> " broadcast", id, "message marked deleted")
@@ -2340,6 +2385,12 @@ testUserPrivacy =
                "bob> Voice messages: enabled",
                "bob> Audio/video calls: enabled"
              ]
+      alice ##> "/_get items around=11 count=3"
+      alice
+        <##? [ "bob> Message reactions: enabled",
+               "bob> Voice messages: enabled",
+               "bob> Audio/video calls: enabled"
+             ]
       alice ##> "/_get items after=12 count=10"
       alice
         <##? [ "@bob hello",

tests/ChatTests/Groups.hs

@@ -36,6 +36,7 @@ chatGroupTests = do
   describe "chat groups" $ do
     describe "add contacts, create group and send/receive messages" testGroupMatrix
     it "mark multiple messages as read" testMarkReadGroup
+    it "initial chat pagination" testChatPaginationInitial
     it "v1: add contacts, create group and send/receive messages" testGroup
     it "v1: add contacts, create group and send/receive messages, check messages" testGroupCheckMessages
     it "send large message" testGroupLargeMessage
@@ -344,6 +345,7 @@ testGroupShared alice bob cath checkMessages directConnections = do
       -- so we take into account group event items as well as sent group invitations in direct chats
       alice #$> ("/_get chat #1 after=" <> msgItem1 <> " count=100", chat, [(0, "hi there"), (0, "hey team")])
       alice #$> ("/_get chat #1 before=" <> msgItem2 <> " count=100", chat, [(1, e2eeInfoNoPQStr), (0, "connected"), (0, "connected"), (1, "hello"), (0, "hi there")])
+      alice #$> ("/_get chat #1 around=" <> msgItem2 <> " count=6", chat, [(0, "connected"), (1, "hello"), (0, "hi there"), (0, "hey team")]) -- expecting only 4 items since there is only 1 item after
       alice #$> ("/_get chat #1 count=100 search=team", chat, [(0, "hey team")])
       bob @@@ [("@cath", "hey"), ("#team", "hey team"), ("@alice", "received invitation to join group team as admin")]
       bob #$> ("/_get chat #1 count=100", chat, groupFeatures <> [(0, "connected"), (0, "added cath (Catherine)"), (0, "connected"), (0, "hello"), (1, "hi there"), (0, "hey team")])
@@ -374,6 +376,51 @@ testMarkReadGroup = testChat2 aliceProfile bobProfile $ \alice bob -> do
   let itemIds = intercalate "," $ map show [i - 3 .. i]
   bob #$> ("/_read chat items #1 " <> itemIds, id, "ok")
 
+testChatPaginationInitial :: HasCallStack => FilePath -> IO ()
+testChatPaginationInitial = testChatOpts2 opts aliceProfile bobProfile $ \alice bob -> do
+  createGroup2 "team" alice bob
+  -- Wait, otherwise ids are going to be wrong.
+  threadDelay 1000000
+  lastEventId <- (read :: String -> Int) <$> lastItemId bob
+  let groupItemId n = show $ lastEventId + n
+
+  -- Send messages from alice to bob
+  forM_ ([1 .. 10] :: [Int]) $ \n -> alice #> ("#team " <> show n)
+
+  -- Bob receives the messages.
+  forM_ ([1 .. 10] :: [Int]) $ \n -> bob <# ("#team alice> " <> show n)
+
+  -- All messages are unread for bob, should return area around unread
+  bob #$> ("/_get chat #1 initial=3", chat, [(0, "connected"), (0, "1"), (0, "2")])
+
+  -- Read next 2 items
+  let itemIds = intercalate "," $ map groupItemId [1 .. 2]
+  bob #$> ("/_read chat items #1 " <> itemIds, id, "ok")
+  bob #$> ("/_get chat #1 initial=3", chat, [(0, "2"), (0, "3"), (0, "4")])
+
+  -- Read all items
+  bob #$> ("/_read chat #1", id, "ok")
+  bob #$> ("/_get chat #1 initial=3", chat, [(0, "8"), (0, "9"), (0, "10")])
+  bob #$> ("/_get chat #1 initial=5", chat, [(0, "6"), (0, "7"), (0, "8"), (0, "9"), (0, "10")])
+
+  -- Clear chat, send a few extra message and assert page size is consistent
+  bob #$> ("/clear #team", id, "#team: all messages are removed locally ONLY")
+  forM_ ([1 .. 10] :: [Int]) $ \n -> alice #> ("#team " <> show n)
+  forM_ ([1 .. 10] :: [Int]) $ \n -> bob <# ("#team alice> " <> show n)
+
+  bob #$> ("/_get chat #1 initial=5", chat, [(0, "1"), (0, "2"), (0, "3"), (0, "4"), (0, "5")])
+  let newItemIds = intercalate "," $ map groupItemId [11 .. 12] -- Read, 1, 2
+  bob #$> ("/_read chat items #1 " <> newItemIds, id, "ok")
+  bob #$> ("/_get chat #1 initial=5", chat, [(0, "1"), (0, "2"), (0, "3"), (0, "4"), (0, "5")])
+  let allButLastId = intercalate "," $ map groupItemId [13 .. 19] -- Read all but last
+  bob #$> ("/_read chat items #1 " <> allButLastId, id, "ok")
+  bob #$> ("/_get chat #1 initial=5", chat, [(0, "6"), (0, "7"), (0, "8"), (0, "9"), (0, "10")])
+  where
+    opts =
+      testOpts
+        { markRead = False
+        }
+
 testGroupLargeMessage :: HasCallStack => FilePath -> IO ()
 testGroupLargeMessage =
   testChat2 aliceProfile bobProfile $

tests/ChatTests/Local.hs

@@ -51,7 +51,7 @@ testNotes tmp = withNewTestChat tmp "alice" aliceProfile $ \alice -> do
   alice ##> "/chats"
 
   alice /* "ahoy!"
-  alice ##> "/_update item *1 1 text Greetings."
+  alice ##> "/_update item *1 2 text Greetings."
   alice ##> "/tail *"
   alice <# "* Greetings."
 
@@ -102,6 +102,10 @@ testChatPagination tmp = withNewTestChat tmp "alice" aliceProfile $ \alice -> do
 
   alice #$> ("/_get chat *1 count=100", chat, [(1, "hello world"), (1, "memento mori"), (1, "knock-knock"), (1, "who's there?")])
   alice #$> ("/_get chat *1 count=1", chat, [(1, "who's there?")])
+  alice #$> ("/_get chat *1 around=2 count=1", chat, [(1, "memento mori")])
+  alice #$> ("/_get chat *1 around=2 count=3", chat, [(1, "hello world"), (1, "memento mori"), (1, "knock-knock")])
+  alice #$> ("/_get chat *1 around=3 count=10", chat, [(1, "hello world"), (1, "memento mori"), (1, "knock-knock"), (1, "who's there?")])
+  alice #$> ("/_get chat *1 around=4 count=3", chat, [(1, "knock-knock"), (1, "who's there?")])
   alice #$> ("/_get chat *1 after=2 count=10", chat, [(1, "knock-knock"), (1, "who's there?")])
   alice #$> ("/_get chat *1 after=2 count=2", chat, [(1, "knock-knock"), (1, "who's there?")])
   alice #$> ("/_get chat *1 after=1 count=2", chat, [(1, "memento mori"), (1, "knock-knock")])

tests/SchemaDump.hs

@@ -102,7 +102,9 @@ skipComparisonForDownMigrations =
     -- table and indexes move down to the end of the file
     "20231215_recreate_msg_deliveries",
     -- on down migration idx_msg_deliveries_agent_ack_cmd_id index moves down to the end of the file
-    "20240313_drop_agent_ack_cmd_id"
+    "20240313_drop_agent_ack_cmd_id",
+    -- on down migration chat_item_autoincrement_id makes sequence table creation move down on the file
+    "20241023_chat_item_autoincrement_id"
   ]
 
 getSchema :: FilePath -> FilePath -> IO String