core: fix pagination indexes (#5241)

simplex-chat.cabal

@@ -152,6 +152,7 @@ library
       Simplex.Chat.Migrations.M20241010_contact_requests_contact_id
       Simplex.Chat.Migrations.M20241023_chat_item_autoincrement_id
       Simplex.Chat.Migrations.M20241027_server_operators
+      Simplex.Chat.Migrations.M20241125_indexes
       Simplex.Chat.Mobile
       Simplex.Chat.Mobile.File
       Simplex.Chat.Mobile.Shared

src/Simplex/Chat/Migrations/M20241125_indexes.hs

@@ -0,0 +1,50 @@
+{-# LANGUAGE QuasiQuotes #-}
+
+module Simplex.Chat.Migrations.M20241125_indexes where
+
+import Database.SQLite.Simple (Query)
+import Database.SQLite.Simple.QQ (sql)
+
+m20241125_indexes :: Query
+m20241125_indexes =
+  [sql|
+-- contacts
+DROP INDEX idx_chat_items_contacts;
+DROP INDEX idx_chat_items_contacts_item_status;
+
+CREATE INDEX idx_chat_items_contacts ON chat_items(user_id, contact_id, item_status, created_at);
+
+-- groups
+DROP INDEX idx_chat_items_groups;
+DROP INDEX idx_chat_items_groups_item_status;
+
+CREATE INDEX idx_chat_items_groups ON chat_items(user_id, group_id, item_status, item_ts);
+CREATE INDEX idx_chat_items_groups_item_ts ON chat_items(user_id, group_id, item_ts);
+
+-- notes
+DROP INDEX idx_chat_items_notes_item_status;
+
+CREATE INDEX idx_chat_items_notes ON chat_items(user_id, note_folder_id, item_status, created_at);
+|]
+
+down_m20241125_indexes :: Query
+down_m20241125_indexes =
+  [sql|
+-- contacts
+DROP INDEX idx_chat_items_contacts;
+
+CREATE INDEX idx_chat_items_contacts ON chat_items(user_id, contact_id, chat_item_id);
+CREATE INDEX idx_chat_items_contacts_item_status on chat_items (user_id, contact_id, item_status);
+
+-- groups
+DROP INDEX idx_chat_items_groups;
+DROP INDEX idx_chat_items_groups_item_ts;
+
+CREATE INDEX idx_chat_items_groups ON chat_items(user_id, group_id, item_ts, chat_item_id);
+CREATE INDEX idx_chat_items_groups_item_status on chat_items (user_id, group_id, item_status);
+
+-- notes
+DROP INDEX idx_chat_items_notes;
+
+CREATE INDEX idx_chat_items_notes_item_status on chat_items (user_id, note_folder_id, item_status);
+|]

src/Simplex/Chat/Migrations/chat_schema.sql

@@ -627,17 +627,6 @@ CREATE INDEX idx_contact_requests_xcontact_id ON contact_requests(xcontact_id);
 CREATE INDEX idx_contacts_xcontact_id ON contacts(xcontact_id);
 CREATE INDEX idx_messages_shared_msg_id ON messages(shared_msg_id);
 CREATE INDEX idx_chat_items_shared_msg_id ON chat_items(shared_msg_id);
-CREATE INDEX idx_chat_items_groups ON chat_items(
-  user_id,
-  group_id,
-  item_ts,
-  chat_item_id
-);
-CREATE INDEX idx_chat_items_contacts ON chat_items(
-  user_id,
-  contact_id,
-  chat_item_id
-);
 CREATE UNIQUE INDEX idx_chat_items_direct_shared_msg_id ON chat_items(
   user_id,
   contact_id,
@@ -887,26 +876,11 @@ CREATE INDEX idx_chat_items_contacts_created_at on chat_items(
   contact_id,
   created_at
 );
-CREATE INDEX idx_chat_items_contacts_item_status on chat_items(
-  user_id,
-  contact_id,
-  item_status
-);
-CREATE INDEX idx_chat_items_groups_item_status on chat_items(
-  user_id,
-  group_id,
-  item_status
-);
 CREATE INDEX idx_chat_items_notes_created_at on chat_items(
   user_id,
   note_folder_id,
   created_at
 );
-CREATE INDEX idx_chat_items_notes_item_status on chat_items(
-  user_id,
-  note_folder_id,
-  item_status
-);
 CREATE INDEX idx_files_redirect_file_id on files(redirect_file_id);
 CREATE INDEX idx_chat_items_fwd_from_contact_id ON chat_items(
   fwd_from_contact_id
@@ -926,3 +900,26 @@ CREATE UNIQUE INDEX idx_operator_usage_conditions_conditions_commit ON operator_
   conditions_commit,
   server_operator_id
 );
+CREATE INDEX idx_chat_items_contacts ON chat_items(
+  user_id,
+  contact_id,
+  item_status,
+  created_at
+);
+CREATE INDEX idx_chat_items_groups ON chat_items(
+  user_id,
+  group_id,
+  item_status,
+  item_ts
+);
+CREATE INDEX idx_chat_items_groups_item_ts ON chat_items(
+  user_id,
+  group_id,
+  item_ts
+);
+CREATE INDEX idx_chat_items_notes ON chat_items(
+  user_id,
+  note_folder_id,
+  item_status,
+  created_at
+);

src/Simplex/Chat/Store/Messages.hs

@@ -1145,27 +1145,52 @@ getContactNavInfo_ db User {userId} Contact {contactId} afterCI = do
     getAfterUnreadCount :: IO Int
     getAfterUnreadCount =
       fromOnly . head
-        <$> DB.query
+        <$> DB.queryNamed
           db
           [sql|
             SELECT COUNT(1)
-            FROM chat_items
-            WHERE user_id = ? AND contact_id = ? AND item_status = ?
-              AND (created_at > ? OR (created_at = ? AND chat_item_id > ?))
+            FROM (
+              SELECT 1
+              FROM chat_items
+              WHERE user_id = :user_id AND contact_id = :contact_id AND item_status = :rcv_new
+                AND created_at > :created_at
+              UNION ALL
+              SELECT 1
+              FROM chat_items
+              WHERE user_id = :user_id AND contact_id = :contact_id AND item_status = :rcv_new
+                AND created_at = :created_at AND chat_item_id > :item_id
+            )
           |]
-          (userId, contactId, CISRcvNew, ciCreatedAt afterCI, ciCreatedAt afterCI, cChatItemId afterCI)
+          [ ":user_id" := userId,
+            ":contact_id" := contactId,
+            ":rcv_new" := CISRcvNew,
+            ":created_at" := ciCreatedAt afterCI,
+            ":item_id" := cChatItemId afterCI
+          ]
     getAfterTotalCount :: IO Int
     getAfterTotalCount =
       fromOnly . head
-        <$> DB.query
+        <$> DB.queryNamed
           db
           [sql|
             SELECT COUNT(1)
-            FROM chat_items
-            WHERE user_id = ? AND contact_id = ?
-              AND (created_at > ? OR (created_at = ? AND chat_item_id > ?))
+            FROM (
+              SELECT 1
+              FROM chat_items
+              WHERE user_id = :user_id AND contact_id = :contact_id
+                AND created_at > :created_at
+              UNION ALL
+              SELECT 1
+              FROM chat_items
+              WHERE user_id = :user_id AND contact_id = :contact_id
+                AND created_at = :created_at AND chat_item_id > :item_id
+            )
           |]
-          (userId, contactId, ciCreatedAt afterCI, ciCreatedAt afterCI, cChatItemId afterCI)
+          [ ":user_id" := userId,
+            ":contact_id" := contactId,
+            ":created_at" := ciCreatedAt afterCI,
+            ":item_id" := cChatItemId afterCI
+          ]
 
 getGroupChat :: DB.Connection -> VersionRangeChat -> User -> Int64 -> ChatPagination -> Maybe String -> ExceptT StoreError IO (Chat 'CTGroup, Maybe NavigationInfo)
 getGroupChat db vr user groupId pagination search_ = do
@@ -1363,27 +1388,52 @@ getGroupNavInfo_ db User {userId} GroupInfo {groupId} afterCI = do
     getAfterUnreadCount :: IO Int
     getAfterUnreadCount =
       fromOnly . head
-        <$> DB.query
+        <$> DB.queryNamed
           db
           [sql|
             SELECT COUNT(1)
-            FROM chat_items
-            WHERE user_id = ? AND group_id = ? AND item_status = ?
-              AND (item_ts > ? OR (item_ts = ? AND chat_item_id > ?))
+            FROM (
+              SELECT 1
+              FROM chat_items
+              WHERE user_id = :user_id AND group_id = :group_id AND item_status = :rcv_new
+                AND item_ts > :item_ts
+              UNION ALL
+              SELECT 1
+              FROM chat_items
+              WHERE user_id = :user_id AND group_id = :group_id AND item_status = :rcv_new
+                AND item_ts = :item_ts AND chat_item_id > :item_id
+            )
           |]
-          (userId, groupId, CISRcvNew, chatItemTs afterCI, chatItemTs afterCI, cChatItemId afterCI)
+          [ ":user_id" := userId,
+            ":group_id" := groupId,
+            ":rcv_new" := CISRcvNew,
+            ":item_ts" := chatItemTs afterCI,
+            ":item_id" := cChatItemId afterCI
+          ]
     getAfterTotalCount :: IO Int
     getAfterTotalCount =
       fromOnly . head
-        <$> DB.query
+        <$> DB.queryNamed
           db
           [sql|
             SELECT COUNT(1)
-            FROM chat_items
-            WHERE user_id = ? AND group_id = ?
-              AND (item_ts > ? OR (item_ts = ? AND chat_item_id > ?))
+            FROM (
+              SELECT 1
+              FROM chat_items
+              WHERE user_id = :user_id AND group_id = :group_id
+                AND item_ts > :item_ts
+              UNION ALL
+              SELECT 1
+              FROM chat_items
+              WHERE user_id = :user_id AND group_id = :group_id
+                AND item_ts = :item_ts AND chat_item_id > :item_id
+            )
           |]
-          (userId, groupId, chatItemTs afterCI, chatItemTs afterCI, cChatItemId afterCI)
+          [ ":user_id" := userId,
+            ":group_id" := groupId,
+            ":item_ts" := chatItemTs afterCI,
+            ":item_id" := cChatItemId afterCI
+          ]
 
 getLocalChat :: DB.Connection -> User -> Int64 -> ChatPagination -> Maybe String -> ExceptT StoreError IO (Chat 'CTLocal, Maybe NavigationInfo)
 getLocalChat db user folderId pagination search_ = do
@@ -1565,27 +1615,52 @@ getLocalNavInfo_ db User {userId} NoteFolder {noteFolderId} afterCI = do
     getAfterUnreadCount :: IO Int
     getAfterUnreadCount =
       fromOnly . head
-        <$> DB.query
+        <$> DB.queryNamed
           db
           [sql|
             SELECT COUNT(1)
-            FROM chat_items
-            WHERE user_id = ? AND note_folder_id = ? AND item_status = ?
-              AND (created_at > ? OR (created_at = ? AND chat_item_id > ?))
+            FROM (
+              SELECT 1
+              FROM chat_items
+              WHERE user_id = :user_id AND note_folder_id = :note_folder_id AND item_status = :rcv_new
+                AND created_at > :created_at
+              UNION ALL
+              SELECT 1
+              FROM chat_items
+              WHERE user_id = :user_id AND note_folder_id = :note_folder_id AND item_status = :rcv_new
+                AND created_at = :created_at AND chat_item_id > :item_id
+            )
           |]
-          (userId, noteFolderId, CISRcvNew, ciCreatedAt afterCI, ciCreatedAt afterCI, cChatItemId afterCI)
+          [ ":user_id" := userId,
+            ":note_folder_id" := noteFolderId,
+            ":rcv_new" := CISRcvNew,
+            ":created_at" := ciCreatedAt afterCI,
+            ":item_id" := cChatItemId afterCI
+          ]
     getAfterTotalCount :: IO Int
     getAfterTotalCount =
       fromOnly . head
-        <$> DB.query
+        <$> DB.queryNamed
           db
           [sql|
             SELECT COUNT(1)
-            FROM chat_items
-            WHERE user_id = ? AND note_folder_id = ?
-              AND (created_at > ? OR (created_at = ? AND chat_item_id > ?))
+            FROM (
+              SELECT 1
+              FROM chat_items
+              WHERE user_id = :user_id AND note_folder_id = :note_folder_id
+                AND created_at > :created_at
+              UNION ALL
+              SELECT 1
+              FROM chat_items
+              WHERE user_id = :user_id AND note_folder_id = :note_folder_id
+                AND created_at = :created_at AND chat_item_id > :item_id
+            )
           |]
-          (userId, noteFolderId, ciCreatedAt afterCI, ciCreatedAt afterCI, cChatItemId afterCI)
+          [ ":user_id" := userId,
+            ":note_folder_id" := noteFolderId,
+            ":created_at" := ciCreatedAt afterCI,
+            ":item_id" := cChatItemId afterCI
+          ]
 
 toChatItemRef :: (ChatItemId, Maybe Int64, Maybe Int64, Maybe Int64) -> Either StoreError (ChatRef, ChatItemId)
 toChatItemRef = \case

src/Simplex/Chat/Store/Migrations.hs

@@ -116,6 +116,7 @@ import Simplex.Chat.Migrations.M20241008_indexes
 import Simplex.Chat.Migrations.M20241010_contact_requests_contact_id
 import Simplex.Chat.Migrations.M20241023_chat_item_autoincrement_id
 import Simplex.Chat.Migrations.M20241027_server_operators
+import Simplex.Chat.Migrations.M20241125_indexes
 import Simplex.Messaging.Agent.Store.SQLite.Migrations (Migration (..))
 
 schemaMigrations :: [(String, Query, Maybe Query)]
@@ -231,7 +232,8 @@ schemaMigrations =
     ("20241008_indexes", m20241008_indexes, Just down_m20241008_indexes),
     ("20241010_contact_requests_contact_id", m20241010_contact_requests_contact_id, Just down_m20241010_contact_requests_contact_id),
     ("20241023_chat_item_autoincrement_id", m20241023_chat_item_autoincrement_id, Just down_m20241023_chat_item_autoincrement_id),
-    ("20241027_server_operators", m20241027_server_operators, Just down_m20241027_server_operators)
+    ("20241027_server_operators", m20241027_server_operators, Just down_m20241027_server_operators),
+    ("20241125_indexes", m20241125_indexes, Just down_m20241125_indexes)
   ]
 
 -- | The list of migrations in ascending order by date

tests/SchemaDump.hs

@@ -103,8 +103,10 @@ skipComparisonForDownMigrations =
     "20231215_recreate_msg_deliveries",
     -- on down migration idx_msg_deliveries_agent_ack_cmd_id index moves down to the end of the file
     "20240313_drop_agent_ack_cmd_id",
-    -- on down migration chat_item_autoincrement_id makes sequence table creation move down on the file
-    "20241023_chat_item_autoincrement_id"
+    -- sequence table moves down to the end of the file
+    "20241023_chat_item_autoincrement_id",
+    -- indexes move down to the end of the file
+    "20241125_indexes"
   ]
 
 getSchema :: FilePath -> FilePath -> IO String