From d912fe07a1f37a59ad8ede0883e7ba4c61185675 Mon Sep 17 00:00:00 2001 From: spaced4ndy <8711996+spaced4ndy@users.noreply.github.com> Date: Mon, 25 Nov 2024 18:51:49 +0400 Subject: [PATCH] core: fix pagination indexes (#5241) --- simplex-chat.cabal | 1 + .../Chat/Migrations/M20241125_indexes.hs | 50 +++++++ src/Simplex/Chat/Migrations/chat_schema.sql | 49 +++---- src/Simplex/Chat/Store/Messages.hs | 135 ++++++++++++++---- src/Simplex/Chat/Store/Migrations.hs | 4 +- tests/SchemaDump.hs | 6 +- 6 files changed, 186 insertions(+), 59 deletions(-) create mode 100644 src/Simplex/Chat/Migrations/M20241125_indexes.hs diff --git a/simplex-chat.cabal b/simplex-chat.cabal index 1a65b87d0b..23071423b8 100644 --- a/simplex-chat.cabal +++ b/simplex-chat.cabal @@ -152,6 +152,7 @@ library Simplex.Chat.Migrations.M20241010_contact_requests_contact_id Simplex.Chat.Migrations.M20241023_chat_item_autoincrement_id Simplex.Chat.Migrations.M20241027_server_operators + Simplex.Chat.Migrations.M20241125_indexes Simplex.Chat.Mobile Simplex.Chat.Mobile.File Simplex.Chat.Mobile.Shared diff --git a/src/Simplex/Chat/Migrations/M20241125_indexes.hs b/src/Simplex/Chat/Migrations/M20241125_indexes.hs new file mode 100644 index 0000000000..2115de09a3 --- /dev/null +++ b/src/Simplex/Chat/Migrations/M20241125_indexes.hs @@ -0,0 +1,50 @@ +{-# LANGUAGE QuasiQuotes #-} + +module Simplex.Chat.Migrations.M20241125_indexes where + +import Database.SQLite.Simple (Query) +import Database.SQLite.Simple.QQ (sql) + +m20241125_indexes :: Query +m20241125_indexes = + [sql| +-- contacts +DROP INDEX idx_chat_items_contacts; +DROP INDEX idx_chat_items_contacts_item_status; + +CREATE INDEX idx_chat_items_contacts ON chat_items(user_id, contact_id, item_status, created_at); + +-- groups +DROP INDEX idx_chat_items_groups; +DROP INDEX idx_chat_items_groups_item_status; + +CREATE INDEX idx_chat_items_groups ON chat_items(user_id, group_id, item_status, item_ts); +CREATE INDEX idx_chat_items_groups_item_ts ON chat_items(user_id, group_id, item_ts); + +-- notes +DROP INDEX idx_chat_items_notes_item_status; + +CREATE INDEX idx_chat_items_notes ON chat_items(user_id, note_folder_id, item_status, created_at); +|] + +down_m20241125_indexes :: Query +down_m20241125_indexes = + [sql| +-- contacts +DROP INDEX idx_chat_items_contacts; + +CREATE INDEX idx_chat_items_contacts ON chat_items(user_id, contact_id, chat_item_id); +CREATE INDEX idx_chat_items_contacts_item_status on chat_items (user_id, contact_id, item_status); + +-- groups +DROP INDEX idx_chat_items_groups; +DROP INDEX idx_chat_items_groups_item_ts; + +CREATE INDEX idx_chat_items_groups ON chat_items(user_id, group_id, item_ts, chat_item_id); +CREATE INDEX idx_chat_items_groups_item_status on chat_items (user_id, group_id, item_status); + +-- notes +DROP INDEX idx_chat_items_notes; + +CREATE INDEX idx_chat_items_notes_item_status on chat_items (user_id, note_folder_id, item_status); +|] diff --git a/src/Simplex/Chat/Migrations/chat_schema.sql b/src/Simplex/Chat/Migrations/chat_schema.sql index 0dc68034e7..6f944157c1 100644 --- a/src/Simplex/Chat/Migrations/chat_schema.sql +++ b/src/Simplex/Chat/Migrations/chat_schema.sql @@ -627,17 +627,6 @@ CREATE INDEX idx_contact_requests_xcontact_id ON contact_requests(xcontact_id); CREATE INDEX idx_contacts_xcontact_id ON contacts(xcontact_id); CREATE INDEX idx_messages_shared_msg_id ON messages(shared_msg_id); CREATE INDEX idx_chat_items_shared_msg_id ON chat_items(shared_msg_id); -CREATE INDEX idx_chat_items_groups ON chat_items( - user_id, - group_id, - item_ts, - chat_item_id -); -CREATE INDEX idx_chat_items_contacts ON chat_items( - user_id, - contact_id, - chat_item_id -); CREATE UNIQUE INDEX idx_chat_items_direct_shared_msg_id ON chat_items( user_id, contact_id, @@ -887,26 +876,11 @@ CREATE INDEX idx_chat_items_contacts_created_at on chat_items( contact_id, created_at ); -CREATE INDEX idx_chat_items_contacts_item_status on chat_items( - user_id, - contact_id, - item_status -); -CREATE INDEX idx_chat_items_groups_item_status on chat_items( - user_id, - group_id, - item_status -); CREATE INDEX idx_chat_items_notes_created_at on chat_items( user_id, note_folder_id, created_at ); -CREATE INDEX idx_chat_items_notes_item_status on chat_items( - user_id, - note_folder_id, - item_status -); CREATE INDEX idx_files_redirect_file_id on files(redirect_file_id); CREATE INDEX idx_chat_items_fwd_from_contact_id ON chat_items( fwd_from_contact_id @@ -926,3 +900,26 @@ CREATE UNIQUE INDEX idx_operator_usage_conditions_conditions_commit ON operator_ conditions_commit, server_operator_id ); +CREATE INDEX idx_chat_items_contacts ON chat_items( + user_id, + contact_id, + item_status, + created_at +); +CREATE INDEX idx_chat_items_groups ON chat_items( + user_id, + group_id, + item_status, + item_ts +); +CREATE INDEX idx_chat_items_groups_item_ts ON chat_items( + user_id, + group_id, + item_ts +); +CREATE INDEX idx_chat_items_notes ON chat_items( + user_id, + note_folder_id, + item_status, + created_at +); diff --git a/src/Simplex/Chat/Store/Messages.hs b/src/Simplex/Chat/Store/Messages.hs index ab8a52a98a..a79eb98f14 100644 --- a/src/Simplex/Chat/Store/Messages.hs +++ b/src/Simplex/Chat/Store/Messages.hs @@ -1145,27 +1145,52 @@ getContactNavInfo_ db User {userId} Contact {contactId} afterCI = do getAfterUnreadCount :: IO Int getAfterUnreadCount = fromOnly . head - <$> DB.query + <$> DB.queryNamed db [sql| SELECT COUNT(1) - FROM chat_items - WHERE user_id = ? AND contact_id = ? AND item_status = ? - AND (created_at > ? OR (created_at = ? AND chat_item_id > ?)) + FROM ( + SELECT 1 + FROM chat_items + WHERE user_id = :user_id AND contact_id = :contact_id AND item_status = :rcv_new + AND created_at > :created_at + UNION ALL + SELECT 1 + FROM chat_items + WHERE user_id = :user_id AND contact_id = :contact_id AND item_status = :rcv_new + AND created_at = :created_at AND chat_item_id > :item_id + ) |] - (userId, contactId, CISRcvNew, ciCreatedAt afterCI, ciCreatedAt afterCI, cChatItemId afterCI) + [ ":user_id" := userId, + ":contact_id" := contactId, + ":rcv_new" := CISRcvNew, + ":created_at" := ciCreatedAt afterCI, + ":item_id" := cChatItemId afterCI + ] getAfterTotalCount :: IO Int getAfterTotalCount = fromOnly . head - <$> DB.query + <$> DB.queryNamed db [sql| SELECT COUNT(1) - FROM chat_items - WHERE user_id = ? AND contact_id = ? - AND (created_at > ? OR (created_at = ? AND chat_item_id > ?)) + FROM ( + SELECT 1 + FROM chat_items + WHERE user_id = :user_id AND contact_id = :contact_id + AND created_at > :created_at + UNION ALL + SELECT 1 + FROM chat_items + WHERE user_id = :user_id AND contact_id = :contact_id + AND created_at = :created_at AND chat_item_id > :item_id + ) |] - (userId, contactId, ciCreatedAt afterCI, ciCreatedAt afterCI, cChatItemId afterCI) + [ ":user_id" := userId, + ":contact_id" := contactId, + ":created_at" := ciCreatedAt afterCI, + ":item_id" := cChatItemId afterCI + ] getGroupChat :: DB.Connection -> VersionRangeChat -> User -> Int64 -> ChatPagination -> Maybe String -> ExceptT StoreError IO (Chat 'CTGroup, Maybe NavigationInfo) getGroupChat db vr user groupId pagination search_ = do @@ -1363,27 +1388,52 @@ getGroupNavInfo_ db User {userId} GroupInfo {groupId} afterCI = do getAfterUnreadCount :: IO Int getAfterUnreadCount = fromOnly . head - <$> DB.query + <$> DB.queryNamed db [sql| SELECT COUNT(1) - FROM chat_items - WHERE user_id = ? AND group_id = ? AND item_status = ? - AND (item_ts > ? OR (item_ts = ? AND chat_item_id > ?)) + FROM ( + SELECT 1 + FROM chat_items + WHERE user_id = :user_id AND group_id = :group_id AND item_status = :rcv_new + AND item_ts > :item_ts + UNION ALL + SELECT 1 + FROM chat_items + WHERE user_id = :user_id AND group_id = :group_id AND item_status = :rcv_new + AND item_ts = :item_ts AND chat_item_id > :item_id + ) |] - (userId, groupId, CISRcvNew, chatItemTs afterCI, chatItemTs afterCI, cChatItemId afterCI) + [ ":user_id" := userId, + ":group_id" := groupId, + ":rcv_new" := CISRcvNew, + ":item_ts" := chatItemTs afterCI, + ":item_id" := cChatItemId afterCI + ] getAfterTotalCount :: IO Int getAfterTotalCount = fromOnly . head - <$> DB.query + <$> DB.queryNamed db [sql| SELECT COUNT(1) - FROM chat_items - WHERE user_id = ? AND group_id = ? - AND (item_ts > ? OR (item_ts = ? AND chat_item_id > ?)) + FROM ( + SELECT 1 + FROM chat_items + WHERE user_id = :user_id AND group_id = :group_id + AND item_ts > :item_ts + UNION ALL + SELECT 1 + FROM chat_items + WHERE user_id = :user_id AND group_id = :group_id + AND item_ts = :item_ts AND chat_item_id > :item_id + ) |] - (userId, groupId, chatItemTs afterCI, chatItemTs afterCI, cChatItemId afterCI) + [ ":user_id" := userId, + ":group_id" := groupId, + ":item_ts" := chatItemTs afterCI, + ":item_id" := cChatItemId afterCI + ] getLocalChat :: DB.Connection -> User -> Int64 -> ChatPagination -> Maybe String -> ExceptT StoreError IO (Chat 'CTLocal, Maybe NavigationInfo) getLocalChat db user folderId pagination search_ = do @@ -1565,27 +1615,52 @@ getLocalNavInfo_ db User {userId} NoteFolder {noteFolderId} afterCI = do getAfterUnreadCount :: IO Int getAfterUnreadCount = fromOnly . head - <$> DB.query + <$> DB.queryNamed db [sql| SELECT COUNT(1) - FROM chat_items - WHERE user_id = ? AND note_folder_id = ? AND item_status = ? - AND (created_at > ? OR (created_at = ? AND chat_item_id > ?)) + FROM ( + SELECT 1 + FROM chat_items + WHERE user_id = :user_id AND note_folder_id = :note_folder_id AND item_status = :rcv_new + AND created_at > :created_at + UNION ALL + SELECT 1 + FROM chat_items + WHERE user_id = :user_id AND note_folder_id = :note_folder_id AND item_status = :rcv_new + AND created_at = :created_at AND chat_item_id > :item_id + ) |] - (userId, noteFolderId, CISRcvNew, ciCreatedAt afterCI, ciCreatedAt afterCI, cChatItemId afterCI) + [ ":user_id" := userId, + ":note_folder_id" := noteFolderId, + ":rcv_new" := CISRcvNew, + ":created_at" := ciCreatedAt afterCI, + ":item_id" := cChatItemId afterCI + ] getAfterTotalCount :: IO Int getAfterTotalCount = fromOnly . head - <$> DB.query + <$> DB.queryNamed db [sql| SELECT COUNT(1) - FROM chat_items - WHERE user_id = ? AND note_folder_id = ? - AND (created_at > ? OR (created_at = ? AND chat_item_id > ?)) + FROM ( + SELECT 1 + FROM chat_items + WHERE user_id = :user_id AND note_folder_id = :note_folder_id + AND created_at > :created_at + UNION ALL + SELECT 1 + FROM chat_items + WHERE user_id = :user_id AND note_folder_id = :note_folder_id + AND created_at = :created_at AND chat_item_id > :item_id + ) |] - (userId, noteFolderId, ciCreatedAt afterCI, ciCreatedAt afterCI, cChatItemId afterCI) + [ ":user_id" := userId, + ":note_folder_id" := noteFolderId, + ":created_at" := ciCreatedAt afterCI, + ":item_id" := cChatItemId afterCI + ] toChatItemRef :: (ChatItemId, Maybe Int64, Maybe Int64, Maybe Int64) -> Either StoreError (ChatRef, ChatItemId) toChatItemRef = \case diff --git a/src/Simplex/Chat/Store/Migrations.hs b/src/Simplex/Chat/Store/Migrations.hs index 7218706239..9a91c7f970 100644 --- a/src/Simplex/Chat/Store/Migrations.hs +++ b/src/Simplex/Chat/Store/Migrations.hs @@ -116,6 +116,7 @@ import Simplex.Chat.Migrations.M20241008_indexes import Simplex.Chat.Migrations.M20241010_contact_requests_contact_id import Simplex.Chat.Migrations.M20241023_chat_item_autoincrement_id import Simplex.Chat.Migrations.M20241027_server_operators +import Simplex.Chat.Migrations.M20241125_indexes import Simplex.Messaging.Agent.Store.SQLite.Migrations (Migration (..)) schemaMigrations :: [(String, Query, Maybe Query)] @@ -231,7 +232,8 @@ schemaMigrations = ("20241008_indexes", m20241008_indexes, Just down_m20241008_indexes), ("20241010_contact_requests_contact_id", m20241010_contact_requests_contact_id, Just down_m20241010_contact_requests_contact_id), ("20241023_chat_item_autoincrement_id", m20241023_chat_item_autoincrement_id, Just down_m20241023_chat_item_autoincrement_id), - ("20241027_server_operators", m20241027_server_operators, Just down_m20241027_server_operators) + ("20241027_server_operators", m20241027_server_operators, Just down_m20241027_server_operators), + ("20241125_indexes", m20241125_indexes, Just down_m20241125_indexes) ] -- | The list of migrations in ascending order by date diff --git a/tests/SchemaDump.hs b/tests/SchemaDump.hs index 4e63a31001..d13dc94b63 100644 --- a/tests/SchemaDump.hs +++ b/tests/SchemaDump.hs @@ -103,8 +103,10 @@ skipComparisonForDownMigrations = "20231215_recreate_msg_deliveries", -- on down migration idx_msg_deliveries_agent_ack_cmd_id index moves down to the end of the file "20240313_drop_agent_ack_cmd_id", - -- on down migration chat_item_autoincrement_id makes sequence table creation move down on the file - "20241023_chat_item_autoincrement_id" + -- sequence table moves down to the end of the file + "20241023_chat_item_autoincrement_id", + -- indexes move down to the end of the file + "20241125_indexes" ] getSchema :: FilePath -> FilePath -> IO String