wip putIntoGroups refactor

* remove gaps

* consistent pagination size

* proper fix and around fix too

* optimize

* refactor

apps/ios/Shared/Model/ChatModel.swift

@@ -58,6 +58,7 @@ class ItemsModel: ObservableObject {
     // this will cause reversedChatItems to be rendered without throttling
     @Published var isLoading = false
     @Published var showLoadingProgress = false
+    @State var anchors: [ChatItem.ID] = []
 
     init() {
         publisher

apps/ios/Shared/Model/SimpleXAPI.swift

@@ -318,10 +318,12 @@ private func apiChatsResponse(_ r: ChatResponse) throws -> [ChatData] {
     throw r
 }
 
-let loadItemsPerPage = 50
+let loadItemsPerPage = 100
+let preloadItem = 25
+let idealChatListSize = 300
 
 func apiGetChat(type: ChatType, id: Int64, search: String = "") async throws -> Chat {
-    let r = await chatSendCmd(.apiGetChat(type: type, id: id, pagination: .last(count: loadItemsPerPage), search: search))
+    let r = await chatSendCmd(.apiGetChat(type: type, id: id, pagination: .initial(count: loadItemsPerPage), search: search))
     if case let .apiChat(_, chat) = r { return Chat.init(chat) }
     throw r
 }
@@ -329,6 +331,15 @@ func apiGetChat(type: ChatType, id: Int64, search: String = "") async throws ->
 func apiGetChatItems(type: ChatType, id: Int64, pagination: ChatPagination, search: String = "") async throws -> [ChatItem] {
     let r = await chatSendCmd(.apiGetChat(type: type, id: id, pagination: pagination, search: search))
     if case let .apiChat(_, chat) = r { return chat.chatItems }
+    if case .chatCmdError(_, _) = r {
+        if case .chatError(_, let chatError) = r {
+            if case .errorStore(let storeError) = chatError {
+                if case .chatItemNotFound(_) = storeError {
+                    itemNotFoundAlert()
+                }
+            }
+        }
+    }
     throw r
 }
 
@@ -339,7 +350,9 @@ func loadChat(chat: Chat, search: String = "", clearItems: Bool = true) async {
         let im = ItemsModel.shared
         m.chatItemStatuses = [:]
         if clearItems {
-            await MainActor.run { im.reversedChatItems = [] }
+            await MainActor.run {
+                im.reversedChatItems = []
+            }
         }
         let chat = try await apiGetChat(type: cInfo.chatType, id: cInfo.apiId, search: search)
         await MainActor.run {
@@ -418,6 +431,13 @@ func apiCreateChatItems(noteFolderId: Int64, composedMessages: [ComposedMessage]
     return nil
 }
 
+func itemNotFoundAlert() {
+    AlertManager.shared.showAlertMsg(
+        title: "Message no longer available",
+        message: "The quoted message you are trying to view has been deleted."
+    )
+}
+
 private func sendMessageErrorAlert(_ r: ChatResponse) {
     logger.error("send message error: \(String(describing: r))")
     AlertManager.shared.showAlertMsg(

apps/ios/Shared/Views/Chat/ChatItem/FramedItemView.swift

@@ -48,10 +48,18 @@ struct FramedItemView: View {
                 if let qi = chatItem.quotedItem {
                     ciQuoteView(qi)
                         .onTapGesture {
-                            if let ci = ItemsModel.shared.reversedChatItems.first(where: { $0.id == qi.itemId }) {
-                                withAnimation {
-                                    scrollModel.scrollToItem(id: ci.id)
+                            if let itemId = qi.itemId {
+                                if !scrollToItem(itemId) {
+                                    Task {
+                                        //if await loadItemsAround(chat.chatInfo, itemId) != nil {
+                                          //  await MainActor.run {
+                                            //    let _ = scrollToItem(itemId)
+                                            //}
+                                        //}
+                                    }
                                 }
+                            } else {
+                                itemNotFoundAlert()
                             }
                         }
                 } else if let itemForwarded = chatItem.meta.itemForwarded {
@@ -323,6 +331,18 @@ struct FramedItemView: View {
             return videoWidth
         }
     }
+    
+    // Scroll to an item, if success returns true otherwise false
+    private func scrollToItem(_ itemId: Int64) -> Bool {
+        if let ci = ItemsModel.shared.reversedChatItems.first(where: { $0.id == itemId }) {
+            withAnimation {
+                scrollModel.scrollToItem(id: ci.id)
+            }
+            return true
+        } else {
+            return false
+        }
+    }    
 }
 
 @ViewBuilder func toggleSecrets<V: View>(_ ft: [FormattedText]?, _ showSecrets: Binding<Bool>, _ v: V) -> some View {

apps/ios/Shared/Views/Chat/ChatItemGroups.swift

@@ -0,0 +1,219 @@
+//
+//  ChatItemGroups.swift
+//  SimpleX (iOS)
+//
+//  Created by Diogo Cunha on 11/11/2024.
+//  Copyright © 2024 SimpleX Chat. All rights reserved.
+//
+
+import Foundation
+import SwiftUI
+import SimpleXChat
+
+/// Represents an anchor  in a list of chat items, indicating where data is missing and should be loaded.
+///
+/// - Parameters:
+///   - itemId: The unique identifier of the last item in the loaded list before the anchor.
+///     This ID corresponds to an item in the chat history, ordered from older to newer items.
+///     It is typically used when loading items via .around or .initial pagination when loading items
+///   - indexRange: The range of indexes within `reversedChatItems` array that
+///     represents the anchor. The first index in this range is the position of the anchor itself.
+///     For instance, if the array `[0, 1, 2, -100-, 101]` has an anchor at index 3, `indexRange`
+///     would be `3..<5`, indicating the anchor starts at index 3.
+///   - indexRangeInParentItems: The range of indexes in the `ReverseList` or parent UI component
+///     that considers revealed or hidden items, showing where the anchor appears in the visible list.
+///     The first index in this range points to where the anchor starts in the UI.
+struct AnchoredRange {
+    let itemId: Int64
+    let indexRange: Range<Int>
+    let indexRangeInParentItems: Range<Int>
+}
+struct SectionGroups {
+    let sections: [SectionItems]
+    let anchoredRanges: [AnchoredRange]
+}
+
+struct ListItem: Hashable, Equatable {
+    let item: ChatItem
+    let separation: ItemSeparation
+    let prevItemSeparationLargeGap: Bool
+}
+
+class SectionItems: ObservableObject {    
+    var mergeCategory: CIMergeCategory?
+    var items: [ListItem]
+    @Published var revealed: Bool
+    var showAvatar: Set<Int64>
+    var startIndexInParentItems: Int
+
+    init(mergeCategory: CIMergeCategory?, items: [ListItem], revealed: Bool, showAvatar: Set<Int64>, startIndexInParentItems: Int) {
+        self.mergeCategory = mergeCategory
+        self.items = items
+        self.revealed = revealed
+        self.showAvatar = showAvatar
+        self.startIndexInParentItems = startIndexInParentItems
+    }
+
+    func reveal(_ reveal: Bool, revealedItems: inout Set<Int64>) {
+        if reveal {
+            for item in items {
+                revealedItems.insert(item.item.id)
+            }
+        } else {
+            for item in items {
+                revealedItems.remove(item.item.id)
+            }
+        }
+        self.revealed = reveal
+    }
+}
+
+func putIntoGroups(chatItems: [ChatItem], revealedItems: Set<ChatItem.ID>, itemAnchors: Array<ChatItem.ID>) -> SectionGroups {
+    guard !chatItems.isEmpty else { return SectionGroups(sections: [], anchoredRanges: []) }
+
+    var groups: [SectionItems] = []
+    var anchoredRanges: [AnchoredRange] = []
+    var index = 0
+    var unclosedAnchorIndex: Int?
+    var unclosedAnchorIndexInParent: Int?
+    var unclosedAnchorItemId: Int64?
+    var visibleItemIndexInParent = -1
+    var recent: SectionItems?
+
+    while index < chatItems.count {
+        let item = chatItems[index]
+        let next = index + 1 < chatItems.count ? chatItems[index + 1] : nil
+        let category = item.mergeCategory
+        let itemIsAnchor = itemAnchors.contains(item.id)
+
+        let itemSeparation: ItemSeparation
+        let prevItemSeparationLargeGap: Bool
+
+        if let recentSection = recent, index > 0, recentSection.mergeCategory == category, !itemIsAnchor {
+            if recentSection.revealed {
+                let prev = index > 0 ? chatItems[index - 1] : nil
+                itemSeparation = getItemSeparation(item, at: index)
+                let nextForGap = (category != nil && category == prev?.mergeCategory) || index + 1 == chatItems.count ? nil : next
+                prevItemSeparationLargeGap = nextForGap == nil ? false : getItemSeparationLargeGap(item, at: index)
+                visibleItemIndexInParent += 1
+            } else {
+                itemSeparation = getItemSeparation(item, at: index)
+                prevItemSeparationLargeGap = false
+            }
+
+            let listItem = ListItem(item: item, separation: itemSeparation, prevItemSeparationLargeGap: prevItemSeparationLargeGap)
+            recentSection.items.append(listItem)
+            if shouldShowAvatar(current: item, older: next) {
+                recentSection.showAvatar.insert(item.id)
+            }
+        } else {
+            let revealed = item.mergeCategory == nil || revealedItems.contains(item.id)
+            visibleItemIndexInParent += 1
+
+            if revealed {
+                let prev = index > 0 ? chatItems[index - 1] : nil
+                itemSeparation = getItemSeparation(item, at: index)
+                let nextForGap = (category != nil && category == prev?.mergeCategory) || index + 1 == chatItems.count ? nil : next
+                prevItemSeparationLargeGap = nextForGap == nil ? false : getItemSeparationLargeGap(item, at: index)
+            } else {
+                itemSeparation = getItemSeparation(item, at: index)
+                prevItemSeparationLargeGap = false
+            }
+
+            let listItem = ListItem(item: item, separation: itemSeparation, prevItemSeparationLargeGap: prevItemSeparationLargeGap)
+            let newSection = SectionItems(
+                mergeCategory: item.mergeCategory,
+                items: [listItem],
+                revealed: revealed,
+                showAvatar: shouldShowAvatar(current: item, older: next) ? [item.id] : [],
+                startIndexInParentItems: visibleItemIndexInParent
+            )
+            groups.append(newSection)
+            recent = newSection
+        }
+
+        if itemIsAnchor {
+            if let unclosedIndex = unclosedAnchorIndex, let unclosedId = unclosedAnchorItemId, let unclosedIndexInParent = unclosedAnchorIndexInParent {
+                anchoredRanges.append(
+                    AnchoredRange(
+                        itemId: unclosedId,
+                        indexRange: unclosedIndex..<index,
+                        indexRangeInParentItems: unclosedIndexInParent..<visibleItemIndexInParent
+                    )
+                )
+            }
+            unclosedAnchorIndex = index
+            unclosedAnchorIndexInParent = visibleItemIndexInParent
+            unclosedAnchorItemId = item.id
+        } else if index + 1 == chatItems.count, let unclosedIndex = unclosedAnchorIndex, let unclosedId = unclosedAnchorItemId, let unclosedIndexInParent = unclosedAnchorIndexInParent {
+            anchoredRanges.append(
+                AnchoredRange(
+                    itemId: unclosedId,
+                    indexRange: unclosedIndex..<index + 1,
+                    indexRangeInParentItems: unclosedIndexInParent..<visibleItemIndexInParent + 1
+                )
+            )
+        }
+        index += 1
+    }
+    
+    return SectionGroups(sections: groups, anchoredRanges: anchoredRanges)
+}
+
+func getItemSectionItems(sections: Array<SectionItems>, itemId: ChatItem.ID) -> SectionItems? {
+    for sec in sections {
+        if sec.items.firstIndex(where: { $0.item.id == itemId }) != nil {
+            return sec
+        }
+    }
+    return nil
+}
+
+func getIndexInParentItems(sections: Array<SectionItems>, itemId: ChatItem.ID) -> Int {
+    for sec in sections {
+        if let index = sec.items.firstIndex(where: { $0.item.id == itemId }) {
+            return sec.startIndexInParentItems + (sec.revealed ? index : 0)
+        }
+    }
+    return -1
+}
+
+func getNewestItemAtParentIndexOrNull(sections: [SectionItems], parentIndex: Int) -> ChatItem? {
+    for group in sections {
+        let range = group.startIndexInParentItems...(group.startIndexInParentItems + group.items.count - 1)
+        if range.contains(parentIndex) {
+            if group.revealed {
+                return group.items[parentIndex - group.startIndexInParentItems].item
+            } else {
+                return group.items.first?.item
+            }
+        }
+    }
+    return nil
+}
+
+
+private func shouldShowAvatar(current: ChatItem, older: ChatItem?) -> Bool {
+    if case let .groupRcv(currentMember) = current.chatDir {
+        if let older = older, case let .groupRcv(olderMember) = older.chatDir {
+            return olderMember.memberId != currentMember.memberId
+        }
+        return true // Show avatar if there is no older item or if older is not a GroupRcv
+    }
+    return false
+}
+
+private func getItemSeparationLargeGap(_ chatItem: ChatItem, at index: Int?) -> Bool {
+    let im = ItemsModel.shared
+
+    if let index = index, index > 0, index < im.reversedChatItems.count {
+        let nextItem = im.reversedChatItems[index - 1]
+        let sameMemberAndDirection = nextItem.chatDir.sameDirection(chatItem.chatDir)
+        
+        // Return true if they are not the same direction or the time interval is more than 60 seconds.
+        return !sameMemberAndDirection || nextItem.meta.itemTs.timeIntervalSince(chatItem.meta.itemTs) > 60
+    } else {
+        // If there is no next item or it's out of bounds, consider it a large gap.
+        return true
+    }
+}

apps/ios/Shared/Views/Chat/ChatView.swift

@@ -13,6 +13,27 @@ import Combine
 
 private let memberImageSize: CGFloat = 34
 
+struct ItemSeparation: Equatable, Hashable {
+    let timestamp: Bool;
+    let largeGap: Bool;
+    let date: Date?
+}
+
+func getItemSeparation(_ chatItem: ChatItem, at i: Int?) -> ItemSeparation {
+    let im = ItemsModel.shared
+    if let i, i > 0 && im.reversedChatItems.count >= i {
+        let nextItem = im.reversedChatItems[i - 1]
+        let largeGap = !nextItem.chatDir.sameDirection(chatItem.chatDir) || nextItem.meta.itemTs.timeIntervalSince(chatItem.meta.itemTs) > 60
+        return ItemSeparation(
+            timestamp: largeGap || formatTimestampMeta(chatItem.meta.itemTs) != formatTimestampMeta(nextItem.meta.itemTs),
+            largeGap: largeGap,
+            date: Calendar.current.isDate(chatItem.meta.itemTs, inSameDayAs: nextItem.meta.itemTs) ? nil : nextItem.meta.itemTs
+        )
+    } else {
+        return ItemSeparation(timestamp: true, largeGap: true, date: nil)
+    }
+}
+
 struct ChatView: View {
     @EnvironmentObject var chatModel: ChatModel
     @ObservedObject var im = ItemsModel.shared
@@ -32,7 +53,6 @@ struct ChatView: View {
     @State private var connectionCode: String?
     @State private var loadingItems = false
     @State private var firstPage = false
-    @State private var revealedChatItem: ChatItem?
     @State private var searchMode = false
     @State private var searchText: String = ""
     @FocusState private var searchFocussed
@@ -46,6 +66,9 @@ struct ChatView: View {
     @State private var selectedChatItems: Set<Int64>? = nil
     @State private var showDeleteSelectedMessages: Bool = false
     @State private var allowToDeleteSelectedMessagesForAll: Bool = false
+    @State private var initialChatItem: ChatItem? = nil
+    @State private var revealedItems: Set<ChatItem.ID> = []
+    @State private var anchors: Array<ChatItem.ID> = []
 
     @AppStorage(DEFAULT_TOOLBAR_MATERIAL) private var toolbarMaterial = ToolbarMaterial.defaultMaterial
 
@@ -167,6 +190,7 @@ struct ChatView: View {
         }
         .onChange(of: chatModel.chatId) { cId in
             showChatInfoSheet = false
+            firstPage = false
             selectedChatItems = nil
             scrollModel.scrollToBottom()
             stopAudioPlayer()
@@ -180,7 +204,7 @@ struct ChatView: View {
                 dismiss()
             }
         }
-        .onChange(of: revealedChatItem) { _ in
+        .onChange(of: revealedItems.count) { _ in
             NotificationCenter.postReverseListNeedsLayout()
         }
         .onChange(of: im.isLoading) { isLoading in
@@ -362,7 +386,18 @@ struct ChatView: View {
                 await markChatUnread(chat, unreadChat: false)
             }
         }
+        if im.reversedChatItems.count == loadItemsPerPage {
+            loadChatItems(chat.chatInfo, .last(count: loadItemsPerPage))
+        }
+
         ChatView.FloatingButtonModel.shared.totalUnread = chat.chatStats.unreadCount
+        Task {
+            DispatchQueue.main.async {
+                if let firstunreadItem = self.getFirstUnreadItem() {
+                    initialChatItem = firstunreadItem
+                }
+            }
+        }
     }
 
     private func searchToolbar() -> some View {
@@ -416,9 +451,10 @@ struct ChatView: View {
 
     private func chatItemsList() -> some View {
         let cInfo = chat.chatInfo
-        let mergedItems = filtered(im.reversedChatItems)
+        let groups = putIntoGroups(chatItems: im.reversedChatItems, revealedItems: self.revealedItems, itemAnchors: self.anchors)
         return GeometryReader { g in
-            ReverseList(items: mergedItems, scrollState: $scrollModel.state) { ci in
+            ReverseList(groups: groups, scrollState: $scrollModel.state, initialChatItem: $initialChatItem) { li in
+                let ci = li.item
                 let voiceNoFrame = voiceWithoutFrame(ci)
                 let maxWidth = cInfo.chatType == .group
                                 ? voiceNoFrame
@@ -433,13 +469,19 @@ struct ChatView: View {
                     maxWidth: maxWidth,
                     composeState: $composeState,
                     selectedMember: $selectedMember,
-                    revealedChatItem: $revealedChatItem,
+                    revealedChatItems: $revealedItems,
                     selectedChatItems: $selectedChatItems,
-                    forwardedChatItems: $forwardedChatItems
+                    forwardedChatItems: $forwardedChatItems,
+                    onReveal: { revealState in
+                        if let sec = getItemSectionItems(sections: groups.sections, itemId: ci.id) {
+                            sec.reveal(revealState, revealedItems: &self.revealedItems)
+                        }
+                    }
                 )
                 .id(ci.id) // Required to trigger `onAppear` on iOS15
-            } loadPage: {
-                loadChatItems(cInfo)
+                
+            } loadPage: { pagination in
+                loadChatItems(cInfo, pagination)
             }
             .opacity(ItemsModel.shared.isLoading ? 0 : 1)
             .padding(.vertical, -InvertedTableView.inset)
@@ -471,6 +513,23 @@ struct ChatView: View {
             EmptyView()
         }
     }
+    
+    private func getFirstUnreadItem() -> ChatItem? {
+        var maybeItem: ChatItem? = nil
+        for i in stride(from: im.reversedChatItems.count - 1, through: 0, by: -1) {
+            let item = im.reversedChatItems[i]
+            if item.isRcvNew {
+                if item.mergeCategory == nil {
+                    return maybeItem ?? item
+                } else {
+                    maybeItem = item
+                }
+            } else if maybeItem != nil {
+                return maybeItem
+            }
+        }
+        return nil
+    }
 
     class FloatingButtonModel: ObservableObject {
         static let shared = FloatingButtonModel()
@@ -478,17 +537,24 @@ struct ChatView: View {
         @Published var isNearBottom: Bool = true
         @Published var date: Date?
         @Published var isDateVisible: Bool = false
+        @Published var bottomItemIndex: Int = 0
         var totalUnread: Int = 0
         var isReallyNearBottom: Bool = true
         var hideDateWorkItem: DispatchWorkItem?
 
         func updateOnListChange(_ listState: ListState) {
             let im = ItemsModel.shared
-            let unreadBelow =
+            let bottomItemIndex =
                 if let id = listState.bottomItemId,
-                   let index = im.reversedChatItems.firstIndex(where: { $0.id == id })
-                {
-                 im.reversedChatItems[..<index].reduce(into: 0) { unread, chatItem in
+                   let index = im.reversedChatItems.firstIndex(where: { $0.id == id }) {
+                    index
+                } else {
+                    -1
+                }
+            
+            var unreadBelow =
+                if bottomItemIndex != -1 {
+                 im.reversedChatItems[..<bottomItemIndex].reduce(into: 0) { unread, chatItem in
                      if chatItem.isRcvNew { unread += 1 }
                  }
                 } else {
@@ -508,6 +574,7 @@ struct ChatView: View {
                 it.unreadBelow = unreadBelow
                 it.date = date
                 it.isReallyNearBottom = listState.scrollOffset > 0 && listState.scrollOffset < 500
+                it.bottomItemIndex = bottomItemIndex
             }
             
             // set floating button indication mode
@@ -839,38 +906,133 @@ struct ChatView: View {
         }
     }
 
-    private func loadChatItems(_ cInfo: ChatInfo) {
+    private func loadChatItems(_ cInfo: ChatInfo, _ pagination: ChatPagination = .initial(count: loadItemsPerPage)) {
         Task {
-            if loadingItems || firstPage { return }
+            if loadingItems { return }
             loadingItems = true
+            
             do {
-                var reversedPage = Array<ChatItem>()
-                var chatItemsAvailable = true
-                // Load additional items until the page is +50 large after merging
-                while chatItemsAvailable && filtered(reversedPage).count < loadItemsPerPage {
-                    let pagination: ChatPagination =
-                        if let lastItem = reversedPage.last ?? im.reversedChatItems.last {
-                            .before(chatItemId: lastItem.id, count: loadItemsPerPage)
-                        } else {
-                            .last(count: loadItemsPerPage)
+                let chatItems = try await apiGetChatItems(
+                    type: cInfo.chatType,
+                    id: cInfo.apiId,
+                    pagination: pagination,
+                    search: searchText
+                )
+                
+                if (cInfo.id != chatModel.chatId) {
+                    await MainActor.run { loadingItems = false }
+                    return
+                }
+                
+                let im = ItemsModel.shared
+                var newItems = im.reversedChatItems
+                
+                switch pagination {
+                case .last:
+                    await MainActor.run {
+                        let newItemIds = Set(chatItems.map { $0.id })
+                        var duplicateFound = false
+                        newItems.removeAll {
+                            let isDuplicate = newItemIds.contains($0.id)
+                            duplicateFound = duplicateFound || isDuplicate
+                            return isDuplicate
                         }
-                    let chatItems = try await apiGetChatItems(
-                        type: cInfo.chatType,
-                        id: cInfo.apiId,
-                        pagination: pagination,
-                        search: searchText
-                    )
-                    chatItemsAvailable = !chatItems.isEmpty
-                    reversedPage.append(contentsOf: chatItems.reversed())
-                }
-                await MainActor.run {
-                    if reversedPage.count == 0 {
-                        firstPage = true
-                    } else {
-                        im.reversedChatItems.append(contentsOf: reversedPage)
+                        
+                        if !duplicateFound {
+                            if let existingItem = im.reversedChatItems.first {
+                                anchors = [existingItem.id]
+                            }
+                        }
+                        
+                        newItems.insert(contentsOf: chatItems.reversed(), at: 0)
+                        im.reversedChatItems = newItems
+                        loadingItems = false
+                    }
+                case .initial:
+                    await MainActor.run {
+                        im.reversedChatItems = chatItems.reversed()
+                        anchors = []
+                        loadingItems = false
+                    }
+                case let .after(chatItemId, _):
+                    guard let indexInCurrentItems = im.reversedChatItems.firstIndex(where: { $0.id == chatItemId }) else {
+                        return
+                    }
+                    
+                    let wasSize = newItems.count
+                    let newItemIds = Set(chatItems.map { $0.id })
+                    let indexInAnchors = anchors.firstIndex { $0 == chatItemId }
+                    var anchorAfterChatItem: [Int64] = []
+                    if let indexInAnchors = indexInAnchors, indexInAnchors + 1 <= anchors.count {
+                        anchorAfterChatItem = Array(anchors[indexInAnchors + 1..<anchors.count])
+                    }
+                    var anchorsToRemove = Set<Int64>()
+                    var reachedBottom: Bool = false
+                    
+                    newItems.removeAll { item in
+                        let isDuplicate = newItemIds.contains(item.id)
+                        if indexInAnchors != nil && newItemIds.contains(item.id) {
+                            if anchorAfterChatItem.contains(item.id) {
+                                anchorAfterChatItem.removeAll { $0 == item.id }
+                                anchorsToRemove.insert(item.id)
+                            } else if reachedBottom == false && anchorAfterChatItem.isEmpty {
+                                // We passed all anchors and found a duplicated item below all of them, indicating no more anchors below the loaded items.
+                                reachedBottom = true
+                            }
+                        }
+                        return isDuplicate
+                    }
+                    
+                    let insertAt = indexInCurrentItems - (wasSize - newItems.count)
+                    newItems.insert(contentsOf: chatItems.reversed(), at: insertAt)
+                    
+                    await MainActor.run {
+                        im.reversedChatItems = newItems
+                        var newAnchors = anchors.filter { !anchorsToRemove.contains($0) }
+                        
+                        if reachedBottom {
+                            newAnchors = []
+                        } else {
+                            if let enlargedAnchorIndex = anchors.firstIndex(where: { $0 == chatItemId }) {
+                                // Move the anchor to the end of the loaded items.
+                                newAnchors[enlargedAnchorIndex] = chatItems.last?.id ?? newAnchors[enlargedAnchorIndex]
+                            }
+                        }
+                        
+                        anchors = newAnchors
+                        loadingItems = false
+                    }
+                case let .before(chatItemId, _):
+                    guard let indexInCurrentItems = im.reversedChatItems.firstIndex(where: { $0.id == chatItemId }) else {
+                        return
+                    }
+                    let newItemIds = Set(chatItems.map { $0.id })
+                    newItems.removeAll { newItemIds.contains($0.id) }
+                    newItems.insert(contentsOf: chatItems.reversed(), at: min(indexInCurrentItems + 1, newItems.count))
+                                        
+                    await MainActor.run {
+                        if chatItems.count == 0 || newItems.count == im.reversedChatItems.count  {
+                            firstPage = true
+                        } else {
+                            im.reversedChatItems = newItems
+                        }
+                        anchors = anchors.filter { !newItemIds.contains($0) }
+                        loadingItems = false
+                    }
+                case .around(_, _):
+                    let newItemIds = Set(chatItems.map { $0.id })
+                    newItems.removeAll { newItemIds.contains($0.id) }
+                    newItems.insert(contentsOf: chatItems, at: 0)
+                    
+                    await MainActor.run {
+                        im.reversedChatItems = newItems
+                        if let lastItemId = chatItems.last?.id {
+                            anchors.insert(lastItemId, at: 0)
+                        }
+                        loadingItems = false
                     }
-                    loadingItems = false
                 }
+                
             } catch let error {
                 logger.error("apiGetChat error: \(responseError(error))")
                 await MainActor.run { loadingItems = false }
@@ -892,7 +1054,7 @@ struct ChatView: View {
         let maxWidth: CGFloat
         @Binding var composeState: ComposeState
         @Binding var selectedMember: GMember?
-        @Binding var revealedChatItem: ChatItem?
+        @Binding var revealedChatItems: Set<ChatItem.ID>
 
         @State private var deletingItem: ChatItem? = nil
         @State private var showDeleteMessage = false
@@ -907,25 +1069,8 @@ struct ChatView: View {
 
         @State private var allowMenu: Bool = true
         @State private var markedRead = false
-
-        var revealed: Bool { chatItem == revealedChatItem }
-
-        typealias ItemSeparation = (timestamp: Bool, largeGap: Bool, date: Date?)
-
-        func getItemSeparation(_ chatItem: ChatItem, at i: Int?) -> ItemSeparation {
-            let im = ItemsModel.shared
-            if let i, i > 0 && im.reversedChatItems.count >= i {
-                let nextItem = im.reversedChatItems[i - 1]
-                let largeGap = !nextItem.chatDir.sameDirection(chatItem.chatDir) || nextItem.meta.itemTs.timeIntervalSince(chatItem.meta.itemTs) > 60
-                return (
-                    timestamp: largeGap || formatTimestampMeta(chatItem.meta.itemTs) != formatTimestampMeta(nextItem.meta.itemTs),
-                    largeGap: largeGap,
-                    date: Calendar.current.isDate(chatItem.meta.itemTs, inSameDayAs: nextItem.meta.itemTs) ? nil : nextItem.meta.itemTs
-                )
-            } else {
-                return (timestamp: true, largeGap: true, date: nil)
-            }
-        }
+        let onReveal: (Bool) -> Void
+        var revealed: Bool { revealedChatItems.contains(chatItem.id) }
 
         var body: some View {
             let currIndex = m.getChatItemIndex(chatItem)
@@ -933,46 +1078,7 @@ struct ChatView: View {
             let (prevHidden, prevItem) = m.getPrevShownChatItem(currIndex, ciCategory)
             let range = itemsRange(currIndex, prevHidden)
             let timeSeparation = getItemSeparation(chatItem, at: currIndex)
-            let im = ItemsModel.shared
-            Group {
-                if revealed, let range = range {
-                    let items = Array(zip(Array(range), im.reversedChatItems[range]))
-                    VStack(spacing: 0) {
-                        ForEach(items.reversed(), id: \.1.viewId) { (i: Int, ci: ChatItem) in
-                            let prev = i == prevHidden ? prevItem : im.reversedChatItems[i + 1]
-                            chatItemView(ci, nil, prev, getItemSeparation(ci, at: i))
-                                .overlay {
-                                    if let selected = selectedChatItems, ci.canBeDeletedForSelf {
-                                        Color.clear
-                                            .contentShape(Rectangle())
-                                            .onTapGesture {
-                                                let checked = selected.contains(ci.id)
-                                                selectUnselectChatItem(select: !checked, ci)
-                                            }
-                                    }
-                                }
-                        }
-                    }
-                } else {
-                    VStack(spacing: 0) {
-                        chatItemView(chatItem, range, prevItem, timeSeparation)
-                        if let date = timeSeparation.date {
-                            DateSeparator(date: date).padding(8)
-                        }
-                    }
-                    .overlay {
-                        if let selected = selectedChatItems, chatItem.canBeDeletedForSelf {
-                            Color.clear
-                                .contentShape(Rectangle())
-                                .onTapGesture {
-                                    let checked = selected.contains(chatItem.id)
-                                    selectUnselectChatItem(select: !checked, chatItem)
-                                }
-                        }
-                    }
-                }
-            }
-            .onAppear {
+            func markAsRead() {
                 if markedRead {
                     return
                 } else {
@@ -991,6 +1097,30 @@ struct ChatView: View {
                     }
                 }
             }
+            return Group {
+                VStack(spacing: 0) {
+                    chatItemView(chatItem, range, prevItem, timeSeparation)
+                    if let date = timeSeparation.date {
+                        DateSeparator(date: date).padding(8)
+                    }
+                }
+                .overlay {
+                    if let selected = selectedChatItems, chatItem.canBeDeletedForSelf {
+                        Color.clear
+                            .contentShape(Rectangle())
+                            .onTapGesture {
+                                let checked = selected.contains(chatItem.id)
+                                selectUnselectChatItem(select: !checked, chatItem)
+                            }
+                    }
+                }
+            }
+            .onAppear {
+                markAsRead()
+            }
+            .onChange(of: ChatView.FloatingButtonModel.shared.bottomItemIndex) { _ in
+                markAsRead()
+            }
         }
 
         private func unreadItemIds(_ range: ClosedRange<Int>) -> [ChatItem.ID] {
@@ -1008,8 +1138,13 @@ struct ChatView: View {
         private func waitToMarkRead(_ op: @Sendable @escaping () async -> Void) {
             Task {
                 _ = try? await Task.sleep(nanoseconds: 600_000000)
-                if m.chatId == chat.chatInfo.id {
-                    await op()
+                let currIndex = m.getChatItemIndex(chatItem)                
+                if let currIndex = currIndex, currIndex >= ChatView.FloatingButtonModel.shared.bottomItemIndex - 3 {
+                    if m.chatId == chat.chatInfo.id {
+                        await op()
+                    }
+                } else {
+                    markedRead = false
                 }
             }
         }
@@ -1573,7 +1708,7 @@ struct ChatView: View {
         private func hideButton() -> Button<some View> {
             Button {
                 withConditionalAnimation {
-                    revealedChatItem = nil
+                    onReveal(false)
                 }
             } label: {
                 Label(
@@ -1648,7 +1783,7 @@ struct ChatView: View {
         private func revealButton(_ ci: ChatItem) -> Button<some View> {
             Button {
                 withConditionalAnimation {
-                    revealedChatItem = ci
+                    onReveal(true)
                 }
             } label: {
                 Label(
@@ -1661,7 +1796,7 @@ struct ChatView: View {
         private func expandButton() -> Button<some View> {
             Button {
                 withConditionalAnimation {
-                    revealedChatItem = chatItem
+                    onReveal(true)
                 }
             } label: {
                 Label(
@@ -1674,7 +1809,7 @@ struct ChatView: View {
         private func shrinkButton() -> Button<some View> {
             Button {
                 withConditionalAnimation {
-                    revealedChatItem = nil
+                    onReveal(false)
                 }
             } label: {
                 Label (

apps/ios/Shared/Views/Chat/ReverseList.swift

@@ -12,14 +12,15 @@ import SimpleXChat
 
 /// A List, which displays it's items in reverse order - from bottom to top
 struct ReverseList<Content: View>: UIViewControllerRepresentable {
-    let items: Array<ChatItem>
+    let groups: SectionGroups
 
     @Binding var scrollState: ReverseListScrollModel.State
+    @Binding var initialChatItem: ChatItem?
 
     /// Closure, that returns user interface for a given item
-    let content: (ChatItem) -> Content
+    let content: (ListItem) -> Content
 
-    let loadPage: () -> Void
+    let loadPage: (_ pagination: ChatPagination) -> Void
 
     func makeUIViewController(context: Context) -> Controller {
         Controller(representer: self)
@@ -27,18 +28,18 @@ struct ReverseList<Content: View>: UIViewControllerRepresentable {
 
     func updateUIViewController(_ controller: Controller, context: Context) {
         controller.representer = self
-        if case let .scrollingTo(destination) = scrollState, !items.isEmpty {
+        if case let .scrollingTo(destination) = scrollState, !groups.sections.isEmpty {
             controller.view.layer.removeAllAnimations()
             switch destination {
             case .nextPage:
                 controller.scrollToNextPage()
             case let .item(id):
-                controller.scroll(to: items.firstIndex(where: { $0.id == id }), position: .bottom)
+                controller.scroll(to: getIndexInParentItems(sections: groups.sections, itemId: id), position: .bottom)
             case .bottom:
                 controller.scroll(to: 0, position: .top)
             }
         } else {
-            controller.update(items: items)
+            controller.update(groups: groups)
         }
     }
 
@@ -46,10 +47,11 @@ struct ReverseList<Content: View>: UIViewControllerRepresentable {
     class Controller: UITableViewController {
         private enum Section { case main }
         var representer: ReverseList
-        private var dataSource: UITableViewDiffableDataSource<Section, ChatItem>!
+        private var dataSource: UITableViewDiffableDataSource<Section, ListItem>!
         private var itemCount: Int = 0
         private let updateFloatingButtons = PassthroughSubject<Void, Never>()
         private var bag = Set<AnyCancellable>()
+        private var revealedItems: Array<ListItem> = []
 
         init(representer: ReverseList) {
             self.representer = representer
@@ -75,11 +77,16 @@ struct ReverseList<Content: View>: UIViewControllerRepresentable {
             }
 
             // 3. Configure data source
-            self.dataSource = UITableViewDiffableDataSource<Section, ChatItem>(
+            self.dataSource = UITableViewDiffableDataSource<Section, ListItem>(
                 tableView: tableView
             ) { (tableView, indexPath, item) -> UITableViewCell? in
-                if indexPath.item > self.itemCount - 8 {
-                    self.representer.loadPage()
+                if self.representer.scrollState == .atDestination, self.representer.initialChatItem == nil {
+                    if indexPath.item > self.itemCount - preloadItem,
+                       let item = getNewestItemAtParentIndexOrNull(sections: self.representer.groups.sections, parentIndex: self.itemCount - 1) {
+                        self.representer.loadPage(.before(chatItemId: item.id, count: loadItemsPerPage))
+                    } else if let item = self.getFirstItemAfterPlacholder(indexPath) {
+                        self.representer.loadPage(.after(chatItemId: item.id, count: loadItemsPerPage))
+                    }
                 }
                 let cell = tableView.dequeueReusableCell(withIdentifier: cellReuseId, for: indexPath)
                 if #available(iOS 16.0, *) {
@@ -149,6 +156,29 @@ struct ReverseList<Content: View>: UIViewControllerRepresentable {
             tableView.clipsToBounds = false
             parent?.viewIfLoaded?.clipsToBounds = false
         }
+        
+        override func viewDidLayoutSubviews() {
+            super.viewDidLayoutSubviews()
+            if let cItem = self.representer.initialChatItem {
+                let index = getIndexInParentItems(sections: self.representer.groups.sections, itemId: cItem.id)
+                
+                if index == -1 {
+                    return
+                }
+                let indexPath = IndexPath(row: index, section: 0)
+                if !isVisible(indexPath: indexPath) {
+                    if tableView.numberOfRows(inSection: indexPath.section) > indexPath.row {
+                        let cellRect = tableView.rectForRow(at: indexPath)
+                        tableView.setContentOffset(CGPoint(x: 0, y: cellRect.maxY - tableView.bounds.height), animated: false)
+                    }
+                }
+                Task {
+                    DispatchQueue.main.async {
+                        self.representer.initialChatItem = nil
+                    }
+                }
+            }
+        }
 
         /// Scrolls up
         func scrollToNextPage() {
@@ -169,6 +199,7 @@ struct ReverseList<Content: View>: UIViewControllerRepresentable {
             if #available(iOS 16.0, *) {
                 animated = true
             }
+
             if let index, tableView.numberOfRows(inSection: 0) != 0 {
                 tableView.scrollToRow(
                     at: IndexPath(row: index, section: 0),
@@ -181,18 +212,49 @@ struct ReverseList<Content: View>: UIViewControllerRepresentable {
                     animated: animated
                 )
             }
-            Task { representer.scrollState = .atDestination }
+            
+            DispatchQueue.main.asyncAfter(deadline: .now() + 0.5) {
+                Task {
+                    self.representer.scrollState = .atDestination
+                }
+            }
         }
 
-        func update(items: [ChatItem]) {
-            var snapshot = NSDiffableDataSourceSnapshot<Section, ChatItem>()
+        func update(groups: SectionGroups) {
+            var snapshot = NSDiffableDataSourceSnapshot<Section, ListItem>()
+            revealedItems.removeAll()
+            groups.sections.forEach { sc in
+                if sc.revealed {
+                    revealedItems.append(contentsOf: sc.items)
+                } else if let item = sc.items.first {
+                    revealedItems.append(item)
+                }
+            }
             snapshot.appendSections([.main])
-            snapshot.appendItems(items)
+            snapshot.appendItems(revealedItems, toSection: .main)
             dataSource.defaultRowAnimation = .none
-            dataSource.apply(
-                snapshot,
-                animatingDifferences: itemCount != 0 && abs(items.count - itemCount) == 1
-            )
+            
+            let countDiff = max(0, revealedItems.count - itemCount)
+            if tableView.contentOffset.y == 100, itemCount < revealedItems.count, itemCount > 0 {
+                dataSource.apply(
+                    snapshot,
+                    animatingDifferences: false
+                )
+                
+                tableView.scrollToRow(
+                    at: IndexPath(row: countDiff, section: 0),
+                    at: .top,
+                    animated: false
+                )
+            } else {
+                tableView.beginUpdates()
+                dataSource.apply(
+                    snapshot,
+                    animatingDifferences: false
+                )
+                tableView.endUpdates()
+            }
+
             // Sets content offset on initial load
             if itemCount == 0 {
                 tableView.setContentOffset(
@@ -200,7 +262,7 @@ struct ReverseList<Content: View>: UIViewControllerRepresentable {
                     animated: false
                 )
             }
-            itemCount = items.count
+            itemCount = revealedItems.count
             updateFloatingButtons.send()
         }
 
@@ -210,17 +272,18 @@ struct ReverseList<Content: View>: UIViewControllerRepresentable {
 
         func getListState() -> ListState? {
             if let visibleRows = tableView.indexPathsForVisibleRows,
-                visibleRows.last?.item ?? 0 < representer.items.count {
+               visibleRows.last?.item ?? 0 < revealedItems.count {
                 let scrollOffset: Double = tableView.contentOffset.y + InvertedTableView.inset
+                
                 let topItemDate: Date? =
                     if let lastVisible = visibleRows.last(where: { isVisible(indexPath: $0) }) {
-                        representer.items[lastVisible.item].meta.itemTs
+                        revealedItems[lastVisible.item].item.meta.itemTs
                     } else {
                         nil
                     }
                 let bottomItemId: ChatItem.ID? =
                     if let firstVisible = visibleRows.first(where: { isVisible(indexPath: $0) }) {
-                        representer.items[firstVisible.item].id
+                        revealedItems[firstVisible.item].item.id
                     } else {
                         nil
                     }
@@ -238,6 +301,10 @@ struct ReverseList<Content: View>: UIViewControllerRepresentable {
                 relativeFrame.minY < tableView.frame.height - InvertedTableView.inset
             } else { false }
         }
+        
+        private func getFirstItemAfterPlacholder(_ indexPath: IndexPath) -> ChatItem? {
+            return nil
+        }
     }
 
     /// `UIHostingConfiguration` back-port for iOS14 and iOS15

apps/ios/SimpleX.xcodeproj/project.pbxproj

@@ -200,6 +200,7 @@
 		8CC4ED902BD7B8530078AEE8 /* CallAudioDeviceManager.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8CC4ED8F2BD7B8530078AEE8 /* CallAudioDeviceManager.swift */; };
 		8CC956EE2BC0041000412A11 /* NetworkObserver.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8CC956ED2BC0041000412A11 /* NetworkObserver.swift */; };
 		8CE848A32C5A0FA000D5C7C8 /* SelectableChatItemToolbars.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8CE848A22C5A0FA000D5C7C8 /* SelectableChatItemToolbars.swift */; };
+		B72540EB2CE277AC0041D1B4 /* ChatItemGroups.swift in Sources */ = {isa = PBXBuildFile; fileRef = B72540EA2CE277AC0041D1B4 /* ChatItemGroups.swift */; };
 		B76E6C312C5C41D900EC11AA /* ContactListNavLink.swift in Sources */ = {isa = PBXBuildFile; fileRef = B76E6C302C5C41D900EC11AA /* ContactListNavLink.swift */; };
 		CE176F202C87014C00145DBC /* InvertedForegroundStyle.swift in Sources */ = {isa = PBXBuildFile; fileRef = CE176F1F2C87014C00145DBC /* InvertedForegroundStyle.swift */; };
 		CE1EB0E42C459A660099D896 /* ShareAPI.swift in Sources */ = {isa = PBXBuildFile; fileRef = CE1EB0E32C459A660099D896 /* ShareAPI.swift */; };
@@ -544,6 +545,7 @@
 		8CC4ED8F2BD7B8530078AEE8 /* CallAudioDeviceManager.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CallAudioDeviceManager.swift; sourceTree = "<group>"; };
 		8CC956ED2BC0041000412A11 /* NetworkObserver.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = NetworkObserver.swift; sourceTree = "<group>"; };
 		8CE848A22C5A0FA000D5C7C8 /* SelectableChatItemToolbars.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SelectableChatItemToolbars.swift; sourceTree = "<group>"; };
+		B72540EA2CE277AC0041D1B4 /* ChatItemGroups.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ChatItemGroups.swift; sourceTree = "<group>"; };
 		B76E6C302C5C41D900EC11AA /* ContactListNavLink.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ContactListNavLink.swift; sourceTree = "<group>"; };
 		CE176F1F2C87014C00145DBC /* InvertedForegroundStyle.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = InvertedForegroundStyle.swift; sourceTree = "<group>"; };
 		CE1EB0E32C459A660099D896 /* ShareAPI.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ShareAPI.swift; sourceTree = "<group>"; };
@@ -734,6 +736,7 @@
 				64C06EB42A0A4A7C00792D4D /* ChatItemInfoView.swift */,
 				648679AA2BC96A74006456E7 /* ChatItemForwardingView.swift */,
 				8CE848A22C5A0FA000D5C7C8 /* SelectableChatItemToolbars.swift */,
+				B72540EA2CE277AC0041D1B4 /* ChatItemGroups.swift */,
 			);
 			path = Chat;
 			sourceTree = "<group>";
@@ -1483,6 +1486,7 @@
 				5CB346E92869E8BA001FD2EF /* PushEnvironment.swift in Sources */,
 				5C55A91F283AD0E400C4E99E /* CallManager.swift in Sources */,
 				5CFA59D12864782E00863A68 /* ChatArchiveView.swift in Sources */,
+				B72540EB2CE277AC0041D1B4 /* ChatItemGroups.swift in Sources */,
 				649BCDA22805D6EF00C3A862 /* CIImageView.swift in Sources */,
 				5CADE79C292131E900072E13 /* ContactPreferencesView.swift in Sources */,
 				CEA6E91C2CBD21B0002B5DB4 /* UserDefault.swift in Sources */,

apps/ios/SimpleXChat/APITypes.swift

@@ -1133,12 +1133,16 @@ public enum ChatPagination {
     case last(count: Int)
     case after(chatItemId: Int64, count: Int)
     case before(chatItemId: Int64, count: Int)
-
+    case around(chatItemId: Int64, count: Int)
+    case initial(count: Int)
+    
     var cmdString: String {
         switch self {
         case let .last(count): return "count=\(count)"
         case let .after(chatItemId, count): return "after=\(chatItemId) count=\(count)"
         case let .before(chatItemId, count): return "before=\(chatItemId) count=\(count)"
+        case let .around(chatItemId, count): return "around=\(chatItemId) count=\(count)"
+        case let .initial(count): return "initial=\(count)"
         }
     }
 }

apps/ios/SimpleXChat/ChatTypes.swift

@@ -2663,7 +2663,7 @@ public struct ChatItem: Identifiable, Decodable, Hashable {
         item.isLiveDummy = true
         return item
     }
-
+    
     public static func invalidJSON(chatDir: CIDirection?, meta: CIMeta?, json: String) -> ChatItem {
         ChatItem(
             chatDir: chatDir ?? .directSnd,

apps/multiplatform/common/src/commonMain/kotlin/chat/simplex/common/views/chat/ComposeView.kt

@@ -381,6 +381,14 @@ fun ComposeView(
 
   suspend fun send(chat: Chat, mc: MsgContent, quoted: Long?, file: CryptoFile? = null, live: Boolean = false, ttl: Int?): ChatItem? {
     val cInfo = chat.chatInfo
+
+//    val composedMessages = Array(300) { index ->
+//      ComposedMessage(
+//        file,
+//        quoted,
+//        MsgContent.MCText("$index")
+//      )
+//    }.toList()
     val chatItems = if (chat.chatInfo.chatType == ChatType.Local)
       chatModel.controller.apiCreateChatItems(
         rh = chat.remoteHostId,

apps/simplex-directory-service/src/Directory/Service.hs

@@ -605,7 +605,7 @@ directoryService st DirectoryOpts {superUsers, serviceName, searchResults, testi
         listGroups count pending =
           readTVarIO (groupRegs st) >>= \groups -> do
             grs <-
-              if pending 
+              if pending
                 then filterM (fmap pendingApproval . readTVarIO . groupRegStatus) groups
                 else pure groups
             sendReply $ show (length grs) <> " registered group(s)" <> (if length grs > count then ", showing the last " <> show count else "")

simplex-chat.cabal

@@ -150,6 +150,7 @@ library
       Simplex.Chat.Migrations.M20240920_user_order
       Simplex.Chat.Migrations.M20241008_indexes
       Simplex.Chat.Migrations.M20241010_contact_requests_contact_id
+      Simplex.Chat.Migrations.M20241023_chat_item_autoincrement_id
       Simplex.Chat.Mobile
       Simplex.Chat.Mobile.File
       Simplex.Chat.Mobile.Shared

src/Simplex/Chat.hs

@@ -8301,6 +8301,8 @@ chatCommandP =
       (CPLast <$ "count=" <*> A.decimal)
         <|> (CPAfter <$ "after=" <*> A.decimal <* A.space <* "count=" <*> A.decimal)
         <|> (CPBefore <$ "before=" <*> A.decimal <* A.space <* "count=" <*> A.decimal)
+        <|> (CPAround <$ "around=" <*> A.decimal <* A.space <* "count=" <*> A.decimal)
+        <|> (CPInitial <$ "initial=" <*> A.decimal)
     paginationByTimeP =
       (PTLast <$ "count=" <*> A.decimal)
         <|> (PTAfter <$ "after=" <*> strP <* A.space <* "count=" <*> A.decimal)

src/Simplex/Chat/Controller.hs

@@ -839,6 +839,8 @@ data ChatPagination
   = CPLast Int
   | CPAfter ChatItemId Int
   | CPBefore ChatItemId Int
+  | CPAround ChatItemId Int
+  | CPInitial Int
   deriving (Show)
 
 data PaginationByTime

src/Simplex/Chat/Messages.hs

@@ -239,6 +239,12 @@ chatItemTs (CChatItem _ ci) = chatItemTs' ci
 chatItemTs' :: ChatItem c d -> UTCTime
 chatItemTs' ChatItem {meta = CIMeta {itemTs}} = itemTs
 
+chatItemCreatedAt :: CChatItem c -> UTCTime
+chatItemCreatedAt (CChatItem _ ci) = chatItemCreatedAt' ci
+
+chatItemCreatedAt' :: ChatItem c d -> UTCTime
+chatItemCreatedAt' ChatItem {meta = CIMeta {createdAt}} = createdAt
+
 chatItemTimed :: ChatItem c d -> Maybe CITimed
 chatItemTimed ChatItem {meta = CIMeta {itemTimed}} = itemTimed
 

src/Simplex/Chat/Migrations/M20241023_chat_item_autoincrement_id.hs

@@ -0,0 +1,34 @@
+{-# LANGUAGE QuasiQuotes #-}
+
+module Simplex.Chat.Migrations.M20241023_chat_item_autoincrement_id where
+
+import Database.SQLite.Simple (Query)
+import Database.SQLite.Simple.QQ (sql)
+
+m20241023_chat_item_autoincrement_id :: Query
+m20241023_chat_item_autoincrement_id =
+    [sql|
+INSERT INTO sqlite_sequence (name, seq)
+SELECT 'chat_items', MAX(ROWID) FROM chat_items;
+
+PRAGMA writable_schema=1;
+
+UPDATE sqlite_master SET sql = replace(sql, 'INTEGER PRIMARY KEY', 'INTEGER PRIMARY KEY AUTOINCREMENT')
+WHERE name = 'chat_items' AND type = 'table';
+
+PRAGMA writable_schema=0;
+|]
+
+down_m20241023_chat_item_autoincrement_id :: Query
+down_m20241023_chat_item_autoincrement_id =
+    [sql|
+DELETE FROM sqlite_sequence WHERE name = 'chat_items';
+
+PRAGMA writable_schema=1;
+
+UPDATE sqlite_master
+SET sql = replace(sql, 'INTEGER PRIMARY KEY AUTOINCREMENT', 'INTEGER PRIMARY KEY')
+WHERE name = 'chat_items' AND type = 'table';
+
+PRAGMA writable_schema=0;
+|]

src/Simplex/Chat/Migrations/chat_schema.sql

@@ -360,7 +360,7 @@ CREATE TABLE pending_group_messages(
   updated_at TEXT NOT NULL DEFAULT(datetime('now'))
 );
 CREATE TABLE chat_items(
-  chat_item_id INTEGER PRIMARY KEY,
+  chat_item_id INTEGER PRIMARY KEY AUTOINCREMENT,
   user_id INTEGER NOT NULL REFERENCES users ON DELETE CASCADE,
   contact_id INTEGER REFERENCES contacts ON DELETE CASCADE,
   group_id INTEGER REFERENCES groups ON DELETE CASCADE,
@@ -399,6 +399,7 @@ CREATE TABLE chat_items(
   fwd_from_chat_item_id INTEGER REFERENCES chat_items ON DELETE SET NULL,
   via_proxy INTEGER
 );
+CREATE TABLE sqlite_sequence(name,seq);
 CREATE TABLE chat_item_messages(
   chat_item_id INTEGER NOT NULL REFERENCES chat_items ON DELETE CASCADE,
   message_id INTEGER NOT NULL UNIQUE REFERENCES messages ON DELETE CASCADE,
@@ -429,7 +430,6 @@ CREATE TABLE commands(
   created_at TEXT NOT NULL DEFAULT(datetime('now')),
   updated_at TEXT NOT NULL DEFAULT(datetime('now'))
 );
-CREATE TABLE sqlite_sequence(name,seq);
 CREATE TABLE settings(
   settings_id INTEGER PRIMARY KEY,
   chat_item_ttl INTEGER,

src/Simplex/Chat/Store/Messages.hs

@@ -3,6 +3,7 @@
 {-# LANGUAGE GADTs #-}
 {-# LANGUAGE KindSignatures #-}
 {-# LANGUAGE LambdaCase #-}
+{-# LANGUAGE MultiWayIf #-}
 {-# LANGUAGE NamedFieldPuns #-}
 {-# LANGUAGE OverloadedStrings #-}
 {-# LANGUAGE PatternSynonyms #-}
@@ -951,33 +952,37 @@ getDirectChat :: DB.Connection -> VersionRangeChat -> User -> Int64 -> ChatPagin
 getDirectChat db vr user contactId pagination search_ = do
   let search = fromMaybe "" search_
   ct <- getContact db vr user contactId
-  liftIO $ case pagination of
-    CPLast count -> getDirectChatLast_ db user ct count search
+  case pagination of
+    CPLast count -> liftIO $ getDirectChatLast_ db user ct count search
     CPAfter afterId count -> getDirectChatAfter_ db user ct afterId count search
     CPBefore beforeId count -> getDirectChatBefore_ db user ct beforeId count search
+    CPAround aroundId count -> getDirectChatAround_ db user ct aroundId count search
+    CPInitial count -> do
+      unless (null search) $ throwError $ SEInternalError "initial chat pagination doesn't support search"
+      getDirectChatInitial_ db user ct count
 
 -- the last items in reverse order (the last item in the conversation is the first in the returned list)
 getDirectChatLast_ :: DB.Connection -> User -> Contact -> Int -> String -> IO (Chat 'CTDirect)
-getDirectChatLast_ db user@User {userId} ct@Contact {contactId} count search = do
+getDirectChatLast_ db user ct count search = do
   let stats = ChatStats {unreadCount = 0, minUnreadItemId = 0, unreadChat = False}
-  chatItemIds <- getDirectChatItemIdsLast_
+  chatItemIds <- getDirectChatItemIdsLast_ db user ct count search
   currentTs <- getCurrentTime
   chatItems <- mapM (safeGetDirectItem db user ct currentTs) chatItemIds
   pure $ Chat (DirectChat ct) (reverse chatItems) stats
-  where
-    getDirectChatItemIdsLast_ :: IO [ChatItemId]
-    getDirectChatItemIdsLast_ =
-      map fromOnly
-        <$> DB.query
-          db
-          [sql|
-            SELECT chat_item_id
-            FROM chat_items
-            WHERE user_id = ? AND contact_id = ? AND item_text LIKE '%' || ? || '%'
-            ORDER BY created_at DESC, chat_item_id DESC
-            LIMIT ?
-          |]
-          (userId, contactId, search, count)
+
+getDirectChatItemIdsLast_ :: DB.Connection -> User -> Contact -> Int -> String -> IO [ChatItemId]
+getDirectChatItemIdsLast_ db User {userId} Contact {contactId} count search =
+  map fromOnly
+    <$> DB.query
+      db
+      [sql|
+        SELECT chat_item_id
+        FROM chat_items
+        WHERE user_id = ? AND contact_id = ? AND item_text LIKE '%' || ? || '%'
+        ORDER BY created_at DESC, chat_item_id DESC
+        LIMIT ?
+      |]
+      (userId, contactId, search, count)
 
 safeGetDirectItem :: DB.Connection -> User -> Contact -> UTCTime -> ChatItemId -> IO (CChatItem 'CTDirect)
 safeGetDirectItem db user ct currentTs itemId =
@@ -1021,51 +1026,96 @@ getDirectChatItemLast db user@User {userId} contactId = do
         (userId, contactId)
   getDirectChatItem db user contactId chatItemId
 
-getDirectChatAfter_ :: DB.Connection -> User -> Contact -> ChatItemId -> Int -> String -> IO (Chat 'CTDirect)
-getDirectChatAfter_ db user@User {userId} ct@Contact {contactId} afterChatItemId count search = do
+getDirectChatAfter_ :: DB.Connection -> User -> Contact -> ChatItemId -> Int -> String -> ExceptT StoreError IO (Chat 'CTDirect)
+getDirectChatAfter_ db user ct@Contact {contactId} afterChatItemId count search = do
   let stats = ChatStats {unreadCount = 0, minUnreadItemId = 0, unreadChat = False}
-  chatItemIds <- getDirectChatItemIdsAfter_
-  currentTs <- getCurrentTime
-  chatItems <- mapM (safeGetDirectItem db user ct currentTs) chatItemIds
+  afterChatItem <- getDirectChatItem db user contactId afterChatItemId
+  chatItemIds <- liftIO $ getDirectChatItemIdsAfter_ db user ct afterChatItemId count search (chatItemCreatedAt afterChatItem)
+  currentTs <- liftIO getCurrentTime
+  chatItems <- liftIO $ mapM (safeGetDirectItem db user ct currentTs) chatItemIds
   pure $ Chat (DirectChat ct) chatItems stats
-  where
-    getDirectChatItemIdsAfter_ :: IO [ChatItemId]
-    getDirectChatItemIdsAfter_ =
-      map fromOnly
-        <$> DB.query
-          db
-          [sql|
-            SELECT chat_item_id
-            FROM chat_items
-            WHERE user_id = ? AND contact_id = ? AND item_text LIKE '%' || ? || '%'
-              AND chat_item_id > ?
-            ORDER BY created_at ASC, chat_item_id ASC
-            LIMIT ?
-          |]
-          (userId, contactId, search, afterChatItemId, count)
 
-getDirectChatBefore_ :: DB.Connection -> User -> Contact -> ChatItemId -> Int -> String -> IO (Chat 'CTDirect)
-getDirectChatBefore_ db user@User {userId} ct@Contact {contactId} beforeChatItemId count search = do
+getDirectChatItemIdsAfter_ :: DB.Connection -> User -> Contact -> ChatItemId -> Int -> String -> UTCTime -> IO [ChatItemId]
+getDirectChatItemIdsAfter_ db User {userId} Contact {contactId} afterChatItemId count search afterChatItemCreatedAt =
+  map fromOnly
+    <$> DB.query
+      db
+      [sql|
+        SELECT chat_item_id
+        FROM chat_items
+        WHERE user_id = ? AND contact_id = ? AND item_text LIKE '%' || ? || '%'
+          AND (created_at > ? OR (created_at = ? AND chat_item_id > ?))
+        ORDER BY created_at ASC, chat_item_id ASC
+        LIMIT ?
+      |]
+      (userId, contactId, search, afterChatItemCreatedAt, afterChatItemCreatedAt, afterChatItemId, count)
+
+getDirectChatBefore_ :: DB.Connection -> User -> Contact -> ChatItemId -> Int -> String -> ExceptT StoreError IO (Chat 'CTDirect)
+getDirectChatBefore_ db user ct@Contact {contactId} beforeChatItemId count search = do
   let stats = ChatStats {unreadCount = 0, minUnreadItemId = 0, unreadChat = False}
-  chatItemIds <- getDirectChatItemsIdsBefore_
-  currentTs <- getCurrentTime
-  chatItems <- mapM (safeGetDirectItem db user ct currentTs) chatItemIds
+  beforeChatItem <- getDirectChatItem db user contactId beforeChatItemId
+  chatItemIds <- liftIO $ getDirectChatItemsIdsBefore_ db user ct beforeChatItemId count search (chatItemCreatedAt beforeChatItem)
+  currentTs <- liftIO getCurrentTime
+  chatItems <- liftIO $ mapM (safeGetDirectItem db user ct currentTs) chatItemIds
   pure $ Chat (DirectChat ct) (reverse chatItems) stats
+
+getDirectChatItemsIdsBefore_ :: DB.Connection -> User -> Contact -> ChatItemId -> Int -> String -> UTCTime -> IO [ChatItemId]
+getDirectChatItemsIdsBefore_ db User {userId} Contact {contactId} beforeChatItemId count search beforeChatItemCreatedAt =
+  map fromOnly
+    <$> DB.query
+      db
+      [sql|
+        SELECT chat_item_id
+        FROM chat_items
+        WHERE user_id = ? AND contact_id = ? AND item_text LIKE '%' || ? || '%'
+          AND (created_at < ? OR (created_at = ? AND chat_item_id < ?))
+        ORDER BY created_at DESC, chat_item_id DESC
+        LIMIT ?
+      |]
+      (userId, contactId, search, beforeChatItemCreatedAt, beforeChatItemCreatedAt, beforeChatItemId, count)
+
+getDirectChatAround_ :: DB.Connection -> User -> Contact -> ChatItemId -> Int -> String -> ExceptT StoreError IO (Chat 'CTDirect)
+getDirectChatAround_ db user ct@Contact {contactId} aroundItemId count search = do
+  let stats = ChatStats {unreadCount = 0, minUnreadItemId = 0, unreadChat = False}
+  let (fetchCountBefore, fetchCountAfter) = divideFetchCountAround_ (count - 1)
+  middleChatItem <- getDirectChatItem db user contactId aroundItemId
+  beforeIds <- liftIO $ getDirectChatItemsIdsBefore_ db user ct aroundItemId fetchCountBefore search (chatItemCreatedAt middleChatItem)
+  afterIds <- liftIO $ getDirectChatItemIdsAfter_ db user ct aroundItemId fetchCountAfter search (chatItemCreatedAt middleChatItem)
+  currentTs <- liftIO getCurrentTime
+  beforeChatItems <- liftIO $ mapM (safeGetDirectItem db user ct currentTs) beforeIds
+  afterChatItems <- liftIO $ mapM (safeGetDirectItem db user ct currentTs) afterIds
+  let remainingAfter = fetchCountAfter - length afterIds
+  let remainingBefore = fetchCountBefore - length beforeIds
+  if
+    | remainingBefore > 0 && remainingAfter <= 0 -> do
+        extraAfterIds <- liftIO $ getDirectChatItemIdsAfter_ db user ct (last afterIds) remainingBefore search (chatItemCreatedAt (last afterChatItems))
+        extraAfterItems <- liftIO $ mapM (safeGetDirectItem db user ct currentTs) extraAfterIds
+        pure $ Chat (DirectChat ct) (reverse beforeChatItems <> [middleChatItem] <> afterChatItems <> extraAfterItems) stats
+    | remainingAfter > 0 && remainingBefore <= 0 -> do
+        extraBeforeIds <- liftIO $ getDirectChatItemsIdsBefore_ db user ct (last beforeIds) remainingAfter search (chatItemCreatedAt (last beforeChatItems))
+        extraBeforeItems <- liftIO $ mapM (safeGetDirectItem db user ct currentTs) extraBeforeIds
+        pure $ Chat (DirectChat ct) (reverse (beforeChatItems <> extraBeforeItems) <> [middleChatItem] <> afterChatItems) stats
+    | otherwise ->
+        pure $ Chat (DirectChat ct) (reverse beforeChatItems <> [middleChatItem] <> afterChatItems) stats
+
+getDirectChatInitial_ :: DB.Connection -> User -> Contact -> Int -> ExceptT StoreError IO (Chat 'CTDirect)
+getDirectChatInitial_ db user@User {userId} ct@Contact {contactId} count = do
+  firstUnreadItemId_ <- liftIO getDirectChatMinUnreadItemId_
+  case firstUnreadItemId_ of
+    Just firstUnreadItemId -> getDirectChatAround_ db user ct firstUnreadItemId count ""
+    Nothing -> liftIO $ getDirectChatLast_ db user ct count ""
   where
-    getDirectChatItemsIdsBefore_ :: IO [ChatItemId]
-    getDirectChatItemsIdsBefore_ =
-      map fromOnly
-        <$> DB.query
+    getDirectChatMinUnreadItemId_ :: IO (Maybe ChatItemId)
+    getDirectChatMinUnreadItemId_ =
+      fmap join . maybeFirstRow fromOnly $
+        DB.query
           db
           [sql|
-            SELECT chat_item_id
+            SELECT MIN(chat_item_id)
             FROM chat_items
-            WHERE user_id = ? AND contact_id = ? AND item_text LIKE '%' || ? || '%'
-              AND chat_item_id < ?
-            ORDER BY created_at DESC, chat_item_id DESC
-            LIMIT ?
+            WHERE user_id = ? AND contact_id = ? AND item_status = ?
           |]
-          (userId, contactId, search, beforeChatItemId, count)
+          (userId, contactId, CISRcvNew)
 
 getGroupChat :: DB.Connection -> VersionRangeChat -> User -> Int64 -> ChatPagination -> Maybe String -> ExceptT StoreError IO (Chat 'CTGroup)
 getGroupChat db vr user groupId pagination search_ = do
@@ -1075,28 +1125,32 @@ getGroupChat db vr user groupId pagination search_ = do
     CPLast count -> liftIO $ getGroupChatLast_ db user g count search
     CPAfter afterId count -> getGroupChatAfter_ db user g afterId count search
     CPBefore beforeId count -> getGroupChatBefore_ db user g beforeId count search
+    CPAround aroundId count -> getGroupChatAround_ db user g aroundId count search
+    CPInitial count -> do
+      unless (null search) $ throwError $ SEInternalError "initial chat pagination doesn't support search"
+      getGroupChatInitial_ db user g count
 
 getGroupChatLast_ :: DB.Connection -> User -> GroupInfo -> Int -> String -> IO (Chat 'CTGroup)
-getGroupChatLast_ db user@User {userId} g@GroupInfo {groupId} count search = do
+getGroupChatLast_ db user g count search = do
   let stats = ChatStats {unreadCount = 0, minUnreadItemId = 0, unreadChat = False}
-  chatItemIds <- getGroupChatItemIdsLast_
+  chatItemIds <- getGroupChatItemIdsLast_ db user g count search
   currentTs <- getCurrentTime
   chatItems <- mapM (safeGetGroupItem db user g currentTs) chatItemIds
   pure $ Chat (GroupChat g) (reverse chatItems) stats
-  where
-    getGroupChatItemIdsLast_ :: IO [ChatItemId]
-    getGroupChatItemIdsLast_ =
-      map fromOnly
-        <$> DB.query
-          db
-          [sql|
-            SELECT chat_item_id
-            FROM chat_items
-            WHERE user_id = ? AND group_id = ? AND item_text LIKE '%' || ? || '%'
-            ORDER BY item_ts DESC, chat_item_id DESC
-            LIMIT ?
-          |]
-          (userId, groupId, search, count)
+
+getGroupChatItemIdsLast_ :: DB.Connection -> User -> GroupInfo -> Int -> String -> IO [ChatItemId]
+getGroupChatItemIdsLast_ db User {userId} GroupInfo {groupId} count search =
+  map fromOnly
+    <$> DB.query
+      db
+      [sql|
+        SELECT chat_item_id
+        FROM chat_items
+        WHERE user_id = ? AND group_id = ? AND item_text LIKE '%' || ? || '%'
+        ORDER BY item_ts DESC, chat_item_id DESC
+        LIMIT ?
+      |]
+      (userId, groupId, search, count)
 
 safeGetGroupItem :: DB.Connection -> User -> GroupInfo -> UTCTime -> ChatItemId -> IO (CChatItem 'CTGroup)
 safeGetGroupItem db user g currentTs itemId =
@@ -1141,83 +1195,130 @@ getGroupMemberChatItemLast db user@User {userId} groupId groupMemberId = do
   getGroupChatItem db user groupId chatItemId
 
 getGroupChatAfter_ :: DB.Connection -> User -> GroupInfo -> ChatItemId -> Int -> String -> ExceptT StoreError IO (Chat 'CTGroup)
-getGroupChatAfter_ db user@User {userId} g@GroupInfo {groupId} afterChatItemId count search = do
+getGroupChatAfter_ db user g@GroupInfo {groupId} afterChatItemId count search = do
   let stats = ChatStats {unreadCount = 0, minUnreadItemId = 0, unreadChat = False}
   afterChatItem <- getGroupChatItem db user groupId afterChatItemId
-  chatItemIds <- liftIO $ getGroupChatItemIdsAfter_ (chatItemTs afterChatItem)
+  chatItemIds <- liftIO $ getGroupChatItemIdsAfter_ db user g afterChatItemId count search (chatItemTs afterChatItem)
   currentTs <- liftIO getCurrentTime
   chatItems <- liftIO $ mapM (safeGetGroupItem db user g currentTs) chatItemIds
   pure $ Chat (GroupChat g) chatItems stats
-  where
-    getGroupChatItemIdsAfter_ :: UTCTime -> IO [ChatItemId]
-    getGroupChatItemIdsAfter_ afterChatItemTs =
-      map fromOnly
-        <$> DB.query
-          db
-          [sql|
-            SELECT chat_item_id
-            FROM chat_items
-            WHERE user_id = ? AND group_id = ? AND item_text LIKE '%' || ? || '%'
-              AND (item_ts > ? OR (item_ts = ? AND chat_item_id > ?))
-            ORDER BY item_ts ASC, chat_item_id ASC
-            LIMIT ?
-          |]
-          (userId, groupId, search, afterChatItemTs, afterChatItemTs, afterChatItemId, count)
+
+getGroupChatItemIdsAfter_ :: DB.Connection -> User -> GroupInfo -> ChatItemId -> Int -> String -> UTCTime -> IO [ChatItemId]
+getGroupChatItemIdsAfter_ db User {userId} GroupInfo {groupId} afterChatItemId count search afterChatItemTs =
+  map fromOnly
+    <$> DB.query
+      db
+      [sql|
+        SELECT chat_item_id
+        FROM chat_items
+        WHERE user_id = ? AND group_id = ? AND item_text LIKE '%' || ? || '%'
+          AND (item_ts > ? OR (item_ts = ? AND chat_item_id > ?))
+        ORDER BY item_ts ASC, chat_item_id ASC
+        LIMIT ?
+      |]
+      (userId, groupId, search, afterChatItemTs, afterChatItemTs, afterChatItemId, count)
 
 getGroupChatBefore_ :: DB.Connection -> User -> GroupInfo -> ChatItemId -> Int -> String -> ExceptT StoreError IO (Chat 'CTGroup)
-getGroupChatBefore_ db user@User {userId} g@GroupInfo {groupId} beforeChatItemId count search = do
+getGroupChatBefore_ db user g@GroupInfo {groupId} beforeChatItemId count search = do
   let stats = ChatStats {unreadCount = 0, minUnreadItemId = 0, unreadChat = False}
   beforeChatItem <- getGroupChatItem db user groupId beforeChatItemId
-  chatItemIds <- liftIO $ getGroupChatItemIdsBefore_ (chatItemTs beforeChatItem)
+  chatItemIds <- liftIO $ getGroupChatItemIdsBefore_ db user g beforeChatItemId count search (chatItemTs beforeChatItem)
   currentTs <- liftIO getCurrentTime
   chatItems <- liftIO $ mapM (safeGetGroupItem db user g currentTs) chatItemIds
   pure $ Chat (GroupChat g) (reverse chatItems) stats
+
+getGroupChatItemIdsBefore_ :: DB.Connection -> User -> GroupInfo -> ChatItemId -> Int -> String -> UTCTime -> IO [ChatItemId]
+getGroupChatItemIdsBefore_ db User {userId} GroupInfo {groupId} beforeChatItemId count search beforeChatItemTs =
+  map fromOnly
+    <$> DB.query
+      db
+      [sql|
+        SELECT chat_item_id
+        FROM chat_items
+        WHERE user_id = ? AND group_id = ? AND item_text LIKE '%' || ? || '%'
+          AND (item_ts < ? OR (item_ts = ? AND chat_item_id < ?))
+        ORDER BY item_ts DESC, chat_item_id DESC
+        LIMIT ?
+      |]
+      (userId, groupId, search, beforeChatItemTs, beforeChatItemTs, beforeChatItemId, count)
+
+getGroupChatAround_ :: DB.Connection -> User -> GroupInfo -> ChatItemId -> Int -> String -> ExceptT StoreError IO (Chat 'CTGroup)
+getGroupChatAround_ db user g@GroupInfo {groupId} aroundItemId count search = do
+  let stats = ChatStats {unreadCount = 0, minUnreadItemId = 0, unreadChat = False}
+  let (fetchCountBefore, fetchCountAfter) = divideFetchCountAround_ (count - 1)
+  middleChatItem <- getGroupChatItem db user groupId aroundItemId
+  beforeIds <- liftIO $ getGroupChatItemIdsBefore_ db user g aroundItemId fetchCountBefore search (chatItemTs middleChatItem)
+  afterIds <- liftIO $ getGroupChatItemIdsAfter_ db user g aroundItemId fetchCountAfter search (chatItemTs middleChatItem)
+  currentTs <- liftIO getCurrentTime
+  beforeChatItems <- liftIO $ mapM (safeGetGroupItem db user g currentTs) beforeIds
+  afterChatItems <- liftIO $ mapM (safeGetGroupItem db user g currentTs) afterIds
+  let remainingAfter = fetchCountAfter - length afterIds
+  let remainingBefore = fetchCountBefore - length beforeIds
+  if
+    | remainingBefore > 0 && remainingAfter <= 0 -> do
+        extraAfterIds <- liftIO $ getGroupChatItemIdsAfter_ db user g (last afterIds) remainingBefore search (chatItemTs (last afterChatItems))
+        extraAfterItems <- liftIO $ mapM (safeGetGroupItem db user g currentTs) extraAfterIds
+        pure $ Chat (GroupChat g) (reverse beforeChatItems <> [middleChatItem] <> afterChatItems <> extraAfterItems) stats
+    | remainingAfter > 0 && remainingBefore <= 0 -> do
+        extraBeforeIds <- liftIO $ getGroupChatItemIdsBefore_ db user g (last beforeIds) remainingAfter search (chatItemTs (last beforeChatItems))
+        extraBeforeItems <- liftIO $ mapM (safeGetGroupItem db user g currentTs) extraBeforeIds
+        pure $ Chat (GroupChat g) (reverse (beforeChatItems <> extraBeforeItems) <> [middleChatItem] <> afterChatItems) stats
+    | otherwise ->
+        pure $ Chat (GroupChat g) (reverse beforeChatItems <> [middleChatItem] <> afterChatItems) stats
+
+getGroupChatInitial_ :: DB.Connection -> User -> GroupInfo -> Int -> ExceptT StoreError IO (Chat 'CTGroup)
+getGroupChatInitial_ db user@User {userId} g@GroupInfo {groupId} count = do
+  firstUnreadItemId_ <- liftIO getGroupChatMinUnreadItemId_
+  case firstUnreadItemId_ of
+    Just firstUnreadItemId -> getGroupChatAround_ db user g firstUnreadItemId count ""
+    Nothing -> liftIO $ getGroupChatLast_ db user g count ""
   where
-    getGroupChatItemIdsBefore_ :: UTCTime -> IO [ChatItemId]
-    getGroupChatItemIdsBefore_ beforeChatItemTs =
-      map fromOnly
-        <$> DB.query
+    getGroupChatMinUnreadItemId_ :: IO (Maybe ChatItemId)
+    getGroupChatMinUnreadItemId_ =
+      fmap join . maybeFirstRow fromOnly $
+        DB.query
           db
           [sql|
-            SELECT chat_item_id
+            SELECT MIN(chat_item_id)
             FROM chat_items
-            WHERE user_id = ? AND group_id = ? AND item_text LIKE '%' || ? || '%'
-              AND (item_ts < ? OR (item_ts = ? AND chat_item_id < ?))
-            ORDER BY item_ts DESC, chat_item_id DESC
-            LIMIT ?
+            WHERE user_id = ? AND group_id = ? AND item_status = ?
           |]
-          (userId, groupId, search, beforeChatItemTs, beforeChatItemTs, beforeChatItemId, count)
+          (userId, groupId, CISRcvNew)
 
 getLocalChat :: DB.Connection -> User -> Int64 -> ChatPagination -> Maybe String -> ExceptT StoreError IO (Chat 'CTLocal)
 getLocalChat db user folderId pagination search_ = do
   let search = fromMaybe "" search_
   nf <- getNoteFolder db user folderId
-  liftIO $ case pagination of
-    CPLast count -> getLocalChatLast_ db user nf count search
+  case pagination of
+    CPLast count -> liftIO $ getLocalChatLast_ db user nf count search
     CPAfter afterId count -> getLocalChatAfter_ db user nf afterId count search
     CPBefore beforeId count -> getLocalChatBefore_ db user nf beforeId count search
+    CPAround aroundId count -> getLocalChatAround_ db user nf aroundId count search
+    CPInitial count -> do
+      unless (null search) $ throwError $ SEInternalError "initial chat pagination doesn't support search"
+      getLocalChatInitial_ db user nf count
 
 getLocalChatLast_ :: DB.Connection -> User -> NoteFolder -> Int -> String -> IO (Chat 'CTLocal)
-getLocalChatLast_ db user@User {userId} nf@NoteFolder {noteFolderId} count search = do
+getLocalChatLast_ db user nf count search = do
   let stats = ChatStats {unreadCount = 0, minUnreadItemId = 0, unreadChat = False}
-  chatItemIds <- getLocalChatItemIdsLast_
+  chatItemIds <- getLocalChatItemIdsLast_ db user nf count search
   currentTs <- getCurrentTime
   chatItems <- mapM (safeGetLocalItem db user nf currentTs) chatItemIds
   pure $ Chat (LocalChat nf) (reverse chatItems) stats
-  where
-    getLocalChatItemIdsLast_ :: IO [ChatItemId]
-    getLocalChatItemIdsLast_ =
-      map fromOnly
-        <$> DB.query
-          db
-          [sql|
-            SELECT chat_item_id
-            FROM chat_items
-            WHERE user_id = ? AND note_folder_id = ? AND item_text LIKE '%' || ? || '%'
-            ORDER BY created_at DESC, chat_item_id DESC
-            LIMIT ?
-          |]
-          (userId, noteFolderId, search, count)
+
+getLocalChatItemIdsLast_ :: DB.Connection -> User -> NoteFolder -> Int -> String -> IO [ChatItemId]
+getLocalChatItemIdsLast_ db User {userId} NoteFolder {noteFolderId} count search =
+  map fromOnly
+    <$> DB.query
+      db
+      [sql|
+        SELECT chat_item_id
+        FROM chat_items
+        WHERE user_id = ? AND note_folder_id = ? AND item_text LIKE '%' || ? || '%'
+        ORDER BY created_at DESC, chat_item_id DESC
+        LIMIT ?
+      |]
+      (userId, noteFolderId, search, count)
 
 safeGetLocalItem :: DB.Connection -> User -> NoteFolder -> UTCTime -> ChatItemId -> IO (CChatItem 'CTLocal)
 safeGetLocalItem db user NoteFolder {noteFolderId} currentTs itemId =
@@ -1245,51 +1346,105 @@ safeToLocalItem currentTs itemId = \case
                 file = Nothing
               }
 
-getLocalChatAfter_ :: DB.Connection -> User -> NoteFolder -> ChatItemId -> Int -> String -> IO (Chat 'CTLocal)
-getLocalChatAfter_ db user@User {userId} nf@NoteFolder {noteFolderId} afterChatItemId count search = do
+getLocalChatAfter_ :: DB.Connection -> User -> NoteFolder -> ChatItemId -> Int -> String -> ExceptT StoreError IO (Chat 'CTLocal)
+getLocalChatAfter_ db user nf@NoteFolder {noteFolderId} afterChatItemId count search = do
   let stats = ChatStats {unreadCount = 0, minUnreadItemId = 0, unreadChat = False}
-  chatItemIds <- getLocalChatItemIdsAfter_
-  currentTs <- getCurrentTime
-  chatItems <- mapM (safeGetLocalItem db user nf currentTs) chatItemIds
+  afterChatItem <- getLocalChatItem db user noteFolderId afterChatItemId
+  chatItemIds <- liftIO $ getLocalChatItemIdsAfter_ db user nf afterChatItemId count search (chatItemCreatedAt afterChatItem)
+  currentTs <- liftIO getCurrentTime
+  chatItems <- liftIO $ mapM (safeGetLocalItem db user nf currentTs) chatItemIds
   pure $ Chat (LocalChat nf) chatItems stats
-  where
-    getLocalChatItemIdsAfter_ :: IO [ChatItemId]
-    getLocalChatItemIdsAfter_ =
-      map fromOnly
-        <$> DB.query
-          db
-          [sql|
-            SELECT chat_item_id
-            FROM chat_items
-            WHERE user_id = ? AND note_folder_id = ? AND item_text LIKE '%' || ? || '%'
-              AND chat_item_id > ?
-            ORDER BY created_at ASC, chat_item_id ASC
-            LIMIT ?
-          |]
-          (userId, noteFolderId, search, afterChatItemId, count)
 
-getLocalChatBefore_ :: DB.Connection -> User -> NoteFolder -> ChatItemId -> Int -> String -> IO (Chat 'CTLocal)
-getLocalChatBefore_ db user@User {userId} nf@NoteFolder {noteFolderId} beforeChatItemId count search = do
+getLocalChatItemIdsAfter_ :: DB.Connection -> User -> NoteFolder -> ChatItemId -> Int -> String -> UTCTime -> IO [ChatItemId]
+getLocalChatItemIdsAfter_ db User {userId} NoteFolder {noteFolderId} afterChatItemId count search afterChatItemCreatedAt =
+  map fromOnly
+    <$> DB.query
+      db
+      [sql|
+        SELECT chat_item_id
+        FROM chat_items
+        WHERE user_id = ? AND note_folder_id = ? AND item_text LIKE '%' || ? || '%'
+          AND (created_at > ? OR (created_at = ? AND chat_item_id > ?))
+        ORDER BY created_at ASC, chat_item_id ASC
+        LIMIT ?
+      |]
+      (userId, noteFolderId, search, afterChatItemCreatedAt, afterChatItemCreatedAt, afterChatItemId, count)
+
+getLocalChatBefore_ :: DB.Connection -> User -> NoteFolder -> ChatItemId -> Int -> String -> ExceptT StoreError IO (Chat 'CTLocal)
+getLocalChatBefore_ db user nf@NoteFolder {noteFolderId} beforeChatItemId count search = do
   let stats = ChatStats {unreadCount = 0, minUnreadItemId = 0, unreadChat = False}
-  chatItemIds <- getLocalChatItemIdsBefore_
-  currentTs <- getCurrentTime
-  chatItems <- mapM (safeGetLocalItem db user nf currentTs) chatItemIds
+  beforeChatItem <- getLocalChatItem db user noteFolderId beforeChatItemId
+  chatItemIds <- liftIO $ getLocalChatItemIdsBefore_ db user nf beforeChatItemId count search (chatItemCreatedAt beforeChatItem)
+  currentTs <- liftIO getCurrentTime
+  chatItems <- liftIO $ mapM (safeGetLocalItem db user nf currentTs) chatItemIds
   pure $ Chat (LocalChat nf) (reverse chatItems) stats
+
+getLocalChatItemIdsBefore_ :: DB.Connection -> User -> NoteFolder -> ChatItemId -> Int -> String -> UTCTime -> IO [ChatItemId]
+getLocalChatItemIdsBefore_ db User {userId} NoteFolder {noteFolderId} beforeChatItemId count search beforeChatItemCreatedAt =
+  map fromOnly
+    <$> DB.query
+      db
+      [sql|
+        SELECT chat_item_id
+        FROM chat_items
+        WHERE user_id = ? AND note_folder_id = ? AND item_text LIKE '%' || ? || '%'
+          AND (created_at < ? OR (created_at = ? AND chat_item_id < ?))
+        ORDER BY created_at DESC, chat_item_id DESC
+        LIMIT ?
+      |]
+      (userId, noteFolderId, search, beforeChatItemCreatedAt, beforeChatItemCreatedAt, beforeChatItemId, count)
+
+getLocalChatAround_ :: DB.Connection -> User -> NoteFolder -> ChatItemId -> Int -> String -> ExceptT StoreError IO (Chat 'CTLocal)
+getLocalChatAround_ db user nf@NoteFolder {noteFolderId} aroundItemId count search = do
+  let stats = ChatStats {unreadCount = 0, minUnreadItemId = 0, unreadChat = False}
+  let (fetchCountBefore, fetchCountAfter) = divideFetchCountAround_ (count - 1)
+  middleChatItem <- getLocalChatItem db user noteFolderId aroundItemId
+  beforeIds <- liftIO $ getLocalChatItemIdsBefore_ db user nf aroundItemId fetchCountBefore search (chatItemCreatedAt middleChatItem)
+  afterIds <- liftIO $ getLocalChatItemIdsAfter_ db user nf aroundItemId fetchCountAfter search (chatItemCreatedAt middleChatItem)
+  currentTs <- liftIO getCurrentTime
+  beforeChatItems <- liftIO $ mapM (safeGetLocalItem db user nf currentTs) beforeIds
+  afterChatItems <- liftIO $ mapM (safeGetLocalItem db user nf currentTs) afterIds
+  let remainingAfter = fetchCountAfter - length afterIds
+  let remainingBefore = fetchCountBefore - length beforeIds
+  if
+    | remainingBefore > 0 && remainingAfter <= 0 -> do
+        extraAfterIds <- liftIO $ getLocalChatItemIdsAfter_ db user nf (last afterIds) remainingBefore search (chatItemCreatedAt (last afterChatItems))
+        extraAfterItems <- liftIO $ mapM (safeGetLocalItem db user nf currentTs) extraAfterIds
+        pure $ Chat (LocalChat nf) (reverse beforeChatItems <> [middleChatItem] <> afterChatItems <> extraAfterItems) stats
+    | remainingAfter > 0 && remainingBefore <= 0 -> do
+        extraBeforeIds <- liftIO $ getLocalChatItemIdsBefore_ db user nf (last beforeIds) remainingAfter search (chatItemCreatedAt (last beforeChatItems))
+        extraBeforeItems <- liftIO $ mapM (safeGetLocalItem db user nf currentTs) extraBeforeIds
+        pure $ Chat (LocalChat nf) (reverse (beforeChatItems <> extraBeforeItems) <> [middleChatItem] <> afterChatItems) stats
+    | otherwise ->
+        pure $ Chat (LocalChat nf) (reverse beforeChatItems <> [middleChatItem] <> afterChatItems) stats
+
+getLocalChatInitial_ :: DB.Connection -> User -> NoteFolder -> Int -> ExceptT StoreError IO (Chat 'CTLocal)
+getLocalChatInitial_ db user@User {userId} nf@NoteFolder {noteFolderId} count = do
+  firstUnreadItemId_ <- liftIO getLocalChatMinUnreadItemId_
+  case firstUnreadItemId_ of
+    Just firstUnreadItemId -> do
+      chat <- getLocalChatAround_ db user nf firstUnreadItemId count ""
+      let items = chatItems chat
+      if null items || length items == count
+        then pure chat
+        else do
+          let remainingCount = count - length items
+          let afterId = cchatItemId $ last items
+          after <- getLocalChatAfter_ db user nf afterId remainingCount ""
+          pure $ chat {chatItems = chatItems chat <> chatItems after}
+    Nothing -> liftIO $ getLocalChatLast_ db user nf count ""
   where
-    getLocalChatItemIdsBefore_ :: IO [ChatItemId]
-    getLocalChatItemIdsBefore_ =
-      map fromOnly
-        <$> DB.query
+    getLocalChatMinUnreadItemId_ :: IO (Maybe ChatItemId)
+    getLocalChatMinUnreadItemId_ =
+      fmap join . maybeFirstRow fromOnly $
+        DB.query
           db
           [sql|
-            SELECT chat_item_id
+            SELECT MIN(chat_item_id)
             FROM chat_items
-            WHERE user_id = ? AND note_folder_id = ? AND item_text LIKE '%' || ? || '%'
-              AND chat_item_id < ?
-            ORDER BY created_at DESC, chat_item_id DESC
-            LIMIT ?
+            WHERE user_id = ? AND note_folder_id = ? AND item_status = ?
           |]
-          (userId, noteFolderId, search, beforeChatItemId, count)
+          (userId, noteFolderId, CISRcvNew)
 
 toChatItemRef :: (ChatItemId, Maybe Int64, Maybe Int64, Maybe Int64) -> Either StoreError (ChatRef, ChatItemId)
 toChatItemRef = \case
@@ -1581,6 +1736,13 @@ getAllChatItems db vr user@User {userId} pagination search_ = do
       CPLast count -> liftIO $ getAllChatItemsLast_ count
       CPAfter afterId count -> liftIO . getAllChatItemsAfter_ afterId count . aChatItemTs =<< getAChatItem_ afterId
       CPBefore beforeId count -> liftIO . getAllChatItemsBefore_ beforeId count . aChatItemTs =<< getAChatItem_ beforeId
+      CPAround aroundId count -> liftIO . getAllChatItemsAround_ aroundId count . aChatItemTs =<< getAChatItem_ aroundId
+      CPInitial count -> do
+        unless (null search) $ throwError $ SEInternalError "initial chat pagination doesn't support search"
+        firstUnreadItemId <- liftIO getFirstUnreadItemId_
+        case firstUnreadItemId of
+          Just itemId -> liftIO . getAllChatItemsAround_ itemId count . aChatItemTs =<< getAChatItem_ itemId
+          Nothing -> liftIO $ getAllChatItemsLast_ count
   mapM (uncurry (getAChatItem db vr user)) itemRefs
   where
     search = fromMaybe "" search_
@@ -1624,6 +1786,31 @@ getAllChatItems db vr user@User {userId} pagination search_ = do
             LIMIT ?
           |]
           (userId, search, beforeTs, beforeTs, beforeId, count)
+    getChatItem chatId =
+      DB.query
+        db
+        [sql|
+          SELECT chat_item_id, contact_id, group_id, note_folder_id
+          FROM chat_items
+          WHERE chat_item_id = ?
+        |]
+        (Only chatId)
+    getAllChatItemsAround_ aroundId count aroundTs = do
+      let (fetchCountBefore, fetchCountAfter) = divideFetchCountAround_ (count - 1)
+      itemsBefore <- getAllChatItemsBefore_ aroundId fetchCountBefore aroundTs
+      item <- getChatItem aroundId
+      itemsAfter <- getAllChatItemsAfter_ aroundId fetchCountAfter aroundTs
+      pure $ itemsBefore <> item <> itemsAfter
+    getFirstUnreadItemId_ =
+      fmap join . maybeFirstRow fromOnly $
+        DB.query
+          db
+          [sql|
+            SELECT MIN(chat_item_id)
+            FROM chat_items
+            WHERE user_id = ? AND item_status = ?
+          |]
+          (userId, CISRcvNew)
 
 getChatItemIdsByAgentMsgId :: DB.Connection -> Int64 -> AgentMsgId -> IO [ChatItemId]
 getChatItemIdsByAgentMsgId db connId msgId =
@@ -2650,3 +2837,10 @@ getGroupHistoryItems db user@User {userId} GroupInfo {groupId} count = do
             LIMIT ?
           |]
           (userId, groupId, rcvMsgContentTag, sndMsgContentTag, count)
+
+divideFetchCountAround_ :: Int -> (Int, Int)
+divideFetchCountAround_ count =
+  let fetchCountEachSide = count `div` 2
+      fetchCountBefore = fetchCountEachSide + if count `mod` 2 /= 0 then 1 else 0
+      fetchCountAfter = fetchCountEachSide
+   in (fetchCountBefore, fetchCountAfter)

src/Simplex/Chat/Store/Migrations.hs

@@ -114,6 +114,7 @@ import Simplex.Chat.Migrations.M20240827_calls_uuid
 import Simplex.Chat.Migrations.M20240920_user_order
 import Simplex.Chat.Migrations.M20241008_indexes
 import Simplex.Chat.Migrations.M20241010_contact_requests_contact_id
+import Simplex.Chat.Migrations.M20241023_chat_item_autoincrement_id
 import Simplex.Messaging.Agent.Store.SQLite.Migrations (Migration (..))
 
 schemaMigrations :: [(String, Query, Maybe Query)]
@@ -227,7 +228,8 @@ schemaMigrations =
     ("20240827_calls_uuid", m20240827_calls_uuid, Just down_m20240827_calls_uuid),
     ("20240920_user_order", m20240920_user_order, Just down_m20240920_user_order),
     ("20241008_indexes", m20241008_indexes, Just down_m20241008_indexes),
-    ("20241010_contact_requests_contact_id", m20241010_contact_requests_contact_id, Just down_m20241010_contact_requests_contact_id)
+    ("20241010_contact_requests_contact_id", m20241010_contact_requests_contact_id, Just down_m20241010_contact_requests_contact_id),
+    ("20241023_chat_item_autoincrement_id", m20241023_chat_item_autoincrement_id, Just down_m20241023_chat_item_autoincrement_id)
   ]
 
 -- | The list of migrations in ascending order by date

tests/ChatTests/Direct.hs

@@ -66,6 +66,7 @@ chatDirectTests = do
     it "repeat AUTH errors disable contact" testRepeatAuthErrorsDisableContact
     it "should send multiline message" testMultilineMessage
     it "send large message" testLargeMessage
+    it "initial chat pagination" testChatPaginationInitial
   describe "batch send messages" $ do
     it "send multiple messages api" testSendMulti
     it "send multiple timed messages" testSendMultiTimed
@@ -210,6 +211,7 @@ testAddContact = versionTestMatrix2 runTestAddContact
           -- pagination
           alice #$> ("/_get chat @2 after=" <> itemId 1 <> " count=100", chat, [(0, "hello there"), (0, "how are you?")])
           alice #$> ("/_get chat @2 before=" <> itemId 2 <> " count=100", chat, features <> [(1, "hello there 🙂")])
+          alice #$> ("/_get chat @2 around=" <> itemId 2 <> " count=3", chat, [(1, "hello there 🙂"), (0, "hello there"), (0, "how are you?")])
           -- search
           alice #$> ("/_get chat @2 count=100 search=ello ther", chat, [(1, "hello there 🙂"), (0, "hello there")])
           -- read messages
@@ -360,6 +362,49 @@ testMarkReadDirect = testChat2 aliceProfile bobProfile $ \alice bob -> do
   let itemIds = intercalate "," $ map show [i - 3 .. i]
   bob #$> ("/_read chat items @2 " <> itemIds, id, "ok")
 
+testChatPaginationInitial :: HasCallStack => FilePath -> IO ()
+testChatPaginationInitial = testChatOpts2 opts aliceProfile bobProfile $ \alice bob -> do
+  connectUsers alice bob
+  -- Wait, otherwise ids are going to be wrong.
+  threadDelay 1000000
+
+  -- Send messages from alice to bob
+  forM_ ([1 .. 10] :: [Int]) $ \n -> alice #> ("@bob " <> show n)
+
+  -- Bob receives the messages.
+  forM_ ([1 .. 10] :: [Int]) $ \n -> bob <# ("alice> " <> show n)
+
+  -- All messages are unread for bob, should return area around unread
+  bob #$> ("/_get chat @2 initial=3", chat, [(0, "Audio/video calls: enabled"), (0, "1"), (0, "2")])
+
+  -- Read next 2 items
+  let itemIds = intercalate "," $ map itemId [1 .. 2]
+  bob #$> ("/_read chat items @2 " <> itemIds, id, "ok")
+  bob #$> ("/_get chat @2 initial=3", chat, [(0, "2"), (0, "3"), (0, "4")])
+
+  -- Read all items
+  bob #$> ("/_read chat @2", id, "ok")
+  bob #$> ("/_get chat @2 initial=3", chat, [(0, "8"), (0, "9"), (0, "10")])
+  bob #$> ("/_get chat @2 initial=5", chat, [(0, "6"), (0, "7"), (0, "8"), (0, "9"), (0, "10")])
+
+  -- Clear chat, send a few extra message and assert page size is consistent
+  bob #$> ("/clear alice", id, "alice: all messages are removed locally ONLY")
+  forM_ ([1 .. 10] :: [Int]) $ \n -> alice #> ("@bob " <> show n)
+  forM_ ([1 .. 10] :: [Int]) $ \n -> bob <# ("alice> " <> show n)
+
+  bob #$> ("/_get chat @2 initial=5", chat, [(0, "1"), (0, "2"), (0, "3"), (0, "4"), (0, "5")])
+  let newItemIds = intercalate "," $ map itemId [11 .. 12] -- Read, 1, 2
+  bob #$> ("/_read chat items @2 " <> newItemIds, id, "ok")
+  bob #$> ("/_get chat @2 initial=5", chat, [(0, "1"), (0, "2"), (0, "3"), (0, "4"), (0, "5")])
+  let allButLastId = intercalate "," $ map itemId [13 .. 19] -- Read all but last
+  bob #$> ("/_read chat items @2 " <> allButLastId, id, "ok")
+  bob #$> ("/_get chat @2 initial=5", chat, [(0, "6"), (0, "7"), (0, "8"), (0, "9"), (0, "10")])
+  where
+    opts =
+      testOpts
+        { markRead = False
+        }
+
 testDuplicateContactsSeparate :: HasCallStack => FilePath -> IO ()
 testDuplicateContactsSeparate =
   testChat2 aliceProfile bobProfile $
@@ -791,7 +836,7 @@ testDirectMessageDelete =
       alice @@@ [("@bob", lastChatFeature)]
       alice #$> ("/_get chat @2 count=100", chat, chatFeatures)
 
-      -- alice: msg id 1
+      -- alice: msg id 3
       bob ##> ("/_update item @2 " <> itemId 2 <> " text hey alice")
       bob <# "@alice [edited] > hello 🙂"
       bob <## "      hey alice"
@@ -806,12 +851,12 @@ testDirectMessageDelete =
       alice @@@ [("@bob", "hey alice [marked deleted]")]
       alice #$> ("/_get chat @2 count=100", chat, chatFeatures <> [(0, "hey alice [marked deleted]")])
 
-      -- alice: deletes msg id 1 that was broadcast deleted by bob
-      alice #$> ("/_delete item @2 " <> itemId 1 <> " internal", id, "message deleted")
+      -- alice: deletes msg id 3 that was broadcast deleted by bob
+      alice #$> ("/_delete item @2 " <> itemId 3 <> " internal", id, "message deleted")
       alice @@@ [("@bob", lastChatFeature)]
       alice #$> ("/_get chat @2 count=100", chat, chatFeatures)
 
-      -- alice: msg id 1, bob: msg id 3 (quoting message alice deleted locally)
+      -- alice: msg id 4, bob: msg id 3 (quoting message alice deleted locally)
       bob `send` "> @alice (hello 🙂) do you receive my messages?"
       bob <# "@alice > hello 🙂"
       bob <## "      do you receive my messages?"
@@ -819,14 +864,14 @@ testDirectMessageDelete =
       alice <## "      do you receive my messages?"
       alice @@@ [("@bob", "do you receive my messages?")]
       alice #$> ("/_get chat @2 count=100", chat', chatFeatures' <> [((0, "do you receive my messages?"), Just (1, "hello 🙂"))])
-      alice #$> ("/_delete item @2 " <> itemId 1 <> " broadcast", id, "cannot delete this item")
+      alice #$> ("/_delete item @2 " <> itemId 4 <> " broadcast", id, "cannot delete this item")
 
-      -- alice: msg id 2, bob: msg id 4
+      -- alice: msg id 5, bob: msg id 4
       bob #> "@alice how are you?"
       alice <# "bob> how are you?"
 
-      -- alice: deletes msg id 2
-      alice #$> ("/_delete item @2 " <> itemId 2 <> " internal", id, "message deleted")
+      -- alice: deletes msg id 5
+      alice #$> ("/_delete item @2 " <> itemId 5 <> " internal", id, "message deleted")
 
       -- bob: marks deleted msg id 4 (that alice deleted locally)
       bob #$> ("/_delete item @2 " <> itemId 4 <> " broadcast", id, "message marked deleted")
@@ -2340,6 +2385,12 @@ testUserPrivacy =
                "bob> Voice messages: enabled",
                "bob> Audio/video calls: enabled"
              ]
+      alice ##> "/_get items around=11 count=3"
+      alice
+        <##? [ "bob> Message reactions: enabled",
+               "bob> Voice messages: enabled",
+               "bob> Audio/video calls: enabled"
+             ]
       alice ##> "/_get items after=12 count=10"
       alice
         <##? [ "@bob hello",

tests/ChatTests/Groups.hs

@@ -36,6 +36,7 @@ chatGroupTests = do
   describe "chat groups" $ do
     describe "add contacts, create group and send/receive messages" testGroupMatrix
     it "mark multiple messages as read" testMarkReadGroup
+    it "initial chat pagination" testChatPaginationInitial
     it "v1: add contacts, create group and send/receive messages" testGroup
     it "v1: add contacts, create group and send/receive messages, check messages" testGroupCheckMessages
     it "send large message" testGroupLargeMessage
@@ -344,6 +345,7 @@ testGroupShared alice bob cath checkMessages directConnections = do
       -- so we take into account group event items as well as sent group invitations in direct chats
       alice #$> ("/_get chat #1 after=" <> msgItem1 <> " count=100", chat, [(0, "hi there"), (0, "hey team")])
       alice #$> ("/_get chat #1 before=" <> msgItem2 <> " count=100", chat, [(1, e2eeInfoNoPQStr), (0, "connected"), (0, "connected"), (1, "hello"), (0, "hi there")])
+      alice #$> ("/_get chat #1 around=" <> msgItem2 <> " count=6", chat, [(0, "connected"), (1, "hello"), (0, "hi there"), (0, "hey team")]) -- expecting only 4 items since there is only 1 item after
       alice #$> ("/_get chat #1 count=100 search=team", chat, [(0, "hey team")])
       bob @@@ [("@cath", "hey"), ("#team", "hey team"), ("@alice", "received invitation to join group team as admin")]
       bob #$> ("/_get chat #1 count=100", chat, groupFeatures <> [(0, "connected"), (0, "added cath (Catherine)"), (0, "connected"), (0, "hello"), (1, "hi there"), (0, "hey team")])
@@ -374,6 +376,51 @@ testMarkReadGroup = testChat2 aliceProfile bobProfile $ \alice bob -> do
   let itemIds = intercalate "," $ map show [i - 3 .. i]
   bob #$> ("/_read chat items #1 " <> itemIds, id, "ok")
 
+testChatPaginationInitial :: HasCallStack => FilePath -> IO ()
+testChatPaginationInitial = testChatOpts2 opts aliceProfile bobProfile $ \alice bob -> do
+  createGroup2 "team" alice bob
+  -- Wait, otherwise ids are going to be wrong.
+  threadDelay 1000000
+  lastEventId <- (read :: String -> Int) <$> lastItemId bob
+  let groupItemId n = show $ lastEventId + n
+
+  -- Send messages from alice to bob
+  forM_ ([1 .. 10] :: [Int]) $ \n -> alice #> ("#team " <> show n)
+
+  -- Bob receives the messages.
+  forM_ ([1 .. 10] :: [Int]) $ \n -> bob <# ("#team alice> " <> show n)
+
+  -- All messages are unread for bob, should return area around unread
+  bob #$> ("/_get chat #1 initial=3", chat, [(0, "connected"), (0, "1"), (0, "2")])
+
+  -- Read next 2 items
+  let itemIds = intercalate "," $ map groupItemId [1 .. 2]
+  bob #$> ("/_read chat items #1 " <> itemIds, id, "ok")
+  bob #$> ("/_get chat #1 initial=3", chat, [(0, "2"), (0, "3"), (0, "4")])
+
+  -- Read all items
+  bob #$> ("/_read chat #1", id, "ok")
+  bob #$> ("/_get chat #1 initial=3", chat, [(0, "8"), (0, "9"), (0, "10")])
+  bob #$> ("/_get chat #1 initial=5", chat, [(0, "6"), (0, "7"), (0, "8"), (0, "9"), (0, "10")])
+
+  -- Clear chat, send a few extra message and assert page size is consistent
+  bob #$> ("/clear #team", id, "#team: all messages are removed locally ONLY")
+  forM_ ([1 .. 10] :: [Int]) $ \n -> alice #> ("#team " <> show n)
+  forM_ ([1 .. 10] :: [Int]) $ \n -> bob <# ("#team alice> " <> show n)
+
+  bob #$> ("/_get chat #1 initial=5", chat, [(0, "1"), (0, "2"), (0, "3"), (0, "4"), (0, "5")])
+  let newItemIds = intercalate "," $ map groupItemId [11 .. 12] -- Read, 1, 2
+  bob #$> ("/_read chat items #1 " <> newItemIds, id, "ok")
+  bob #$> ("/_get chat #1 initial=5", chat, [(0, "1"), (0, "2"), (0, "3"), (0, "4"), (0, "5")])
+  let allButLastId = intercalate "," $ map groupItemId [13 .. 19] -- Read all but last
+  bob #$> ("/_read chat items #1 " <> allButLastId, id, "ok")
+  bob #$> ("/_get chat #1 initial=5", chat, [(0, "6"), (0, "7"), (0, "8"), (0, "9"), (0, "10")])
+  where
+    opts =
+      testOpts
+        { markRead = False
+        }
+
 testGroupLargeMessage :: HasCallStack => FilePath -> IO ()
 testGroupLargeMessage =
   testChat2 aliceProfile bobProfile $

tests/ChatTests/Local.hs

@@ -51,7 +51,7 @@ testNotes tmp = withNewTestChat tmp "alice" aliceProfile $ \alice -> do
   alice ##> "/chats"
 
   alice /* "ahoy!"
-  alice ##> "/_update item *1 1 text Greetings."
+  alice ##> "/_update item *1 2 text Greetings."
   alice ##> "/tail *"
   alice <# "* Greetings."
 
@@ -102,6 +102,10 @@ testChatPagination tmp = withNewTestChat tmp "alice" aliceProfile $ \alice -> do
 
   alice #$> ("/_get chat *1 count=100", chat, [(1, "hello world"), (1, "memento mori"), (1, "knock-knock"), (1, "who's there?")])
   alice #$> ("/_get chat *1 count=1", chat, [(1, "who's there?")])
+  alice #$> ("/_get chat *1 around=2 count=1", chat, [(1, "memento mori")])
+  alice #$> ("/_get chat *1 around=2 count=3", chat, [(1, "hello world"), (1, "memento mori"), (1, "knock-knock")])
+  alice #$> ("/_get chat *1 around=3 count=10", chat, [(1, "hello world"), (1, "memento mori"), (1, "knock-knock"), (1, "who's there?")])
+  alice #$> ("/_get chat *1 around=4 count=3", chat, [(1, "knock-knock"), (1, "who's there?")])
   alice #$> ("/_get chat *1 after=2 count=10", chat, [(1, "knock-knock"), (1, "who's there?")])
   alice #$> ("/_get chat *1 after=2 count=2", chat, [(1, "knock-knock"), (1, "who's there?")])
   alice #$> ("/_get chat *1 after=1 count=2", chat, [(1, "memento mori"), (1, "knock-knock")])

tests/SchemaDump.hs

@@ -102,7 +102,9 @@ skipComparisonForDownMigrations =
     -- table and indexes move down to the end of the file
     "20231215_recreate_msg_deliveries",
     -- on down migration idx_msg_deliveries_agent_ack_cmd_id index moves down to the end of the file
-    "20240313_drop_agent_ack_cmd_id"
+    "20240313_drop_agent_ack_cmd_id",
+    -- on down migration chat_item_autoincrement_id makes sequence table creation move down on the file
+    "20241023_chat_item_autoincrement_id"
   ]
 
 getSchema :: FilePath -> FilePath -> IO String