core: support down migrations to allow reverting to the previous version (#2072)

* core: support down migrations to allow reverting to the previous version * update schema * update simplexmq * rename errors * remove unused functions * migration UI, test migration * update migration UI * return current migrations in CRVersionInfo * update simplexmq * test down migrations * cleanup ios * show migrations in log
2024-12-17 17:20:21 +01:00 · 2023-03-27 18:34:48 +01:00
parent f5c11b8faf
commit c96ba30018
26 changed files with 365 additions and 222 deletions
@@ -972,7 +972,7 @@ func apiGetGroupLink(_ groupId: Int64) throws -> (String, GroupMemberRole)? {

 func apiGetVersion() throws -> CoreVersionInfo {
    let r = chatSendCmdSync(.showVersion)
-    if case let .versionInfo(info) = r { return info }
+    if case let .versionInfo(info, _, _) = r { return info }
    throw r
 }

@@ -983,10 +983,10 @@ private func currentUserId(_ funcName: String) throws -> Int64 {
    throw RuntimeError("\(funcName): no current user")
 }

-func initializeChat(start: Bool, dbKey: String? = nil, refreshInvitations: Bool = true) throws {
+func initializeChat(start: Bool, dbKey: String? = nil, refreshInvitations: Bool = true, confirmMigrations: MigrationConfirmation? = nil) throws {
    logger.debug("initializeChat")
    let m = ChatModel.shared
-    (m.chatDbEncrypted, m.chatDbStatus) = chatMigrateInit(dbKey)
+    (m.chatDbEncrypted, m.chatDbStatus) = chatMigrateInit(dbKey, confirmMigrations: confirmMigrations)
    if  m.chatDbStatus != .ok { return }
    // If we migrated successfully means previous re-encryption process on database level finished successfully too
    if encryptionStartedDefault.get() {
@@ -16,40 +16,68 @@ struct DatabaseErrorView: View {
    @State private var storedDBKey = getDatabaseKey()
    @State private var useKeychain = storeDBPassphraseGroupDefault.get()
    @State private var showRestoreDbButton = false
+    @State private var starting = false

    var body: some View {
+        ZStack {
+            databaseErrorView().disabled(starting)
+            if starting {
+                ProgressView().scaleEffect(2)
+            }
+        }
+    }
+
+    @ViewBuilder private func databaseErrorView() -> some View {
        VStack(alignment: .leading, spacing: 16) {
            switch status {
            case let .errorNotADatabase(dbFile):
                if useKeychain && storedDBKey != nil && storedDBKey != "" {
-                    Text("Wrong database passphrase").font(.title)
+                    titleText("Wrong database passphrase")
                    Text("Database passphrase is different from saved in the keychain.")
                    databaseKeyField(onSubmit: saveAndRunChat)
                    saveAndOpenButton()
-                    Text("File: \(dbFile)")
+                    fileNameText(dbFile)
                } else {
-                    Text("Encrypted database").font(.title)
+                    titleText("Encrypted database")
                    Text("Database passphrase is required to open chat.")
                    if useKeychain {
                        databaseKeyField(onSubmit: saveAndRunChat)
                        saveAndOpenButton()
                    } else {
-                        databaseKeyField(onSubmit: runChat)
+                        databaseKeyField(onSubmit: { runChat() })
                        openChatButton()
                    }
                }
-            case let .error(dbFile, migrationError):
-                Text("Database error")
-                    .font(.title)
-                Text("File: \(dbFile)")
-                Text("Error: \(migrationError)")
+            case let .errorMigration(dbFile, migrationError):
+                switch migrationError {
+                case let .upgrade(upMigrations):
+                    titleText("Database upgrade")
+                    Button("Upgrade and open chat") { runChat(confirmMigrations: .yesUp) }
+                    fileNameText(dbFile)
+                    migrationsText(upMigrations.map(\.upName))
+                case let .downgrade(downMigrations):
+                    titleText("Database downgrade")
+                    Text("Warning: you may lose some data!").bold()
+                    Button("Downgrade and open chat") { runChat(confirmMigrations: .yesUpDown) }
+                    fileNameText(dbFile)
+                    migrationsText(downMigrations)
+                case let .migrationError(mtrError):
+                    titleText("Incompatible database version")
+                    fileNameText(dbFile)
+                    Text("Error: ") + Text(mtrErrorDescription(mtrError))
+                }
+            case let .errorSQL(dbFile, migrationSQLError):
+                titleText("Database error")
+                fileNameText(dbFile)
+                Text("Error: \(migrationSQLError)")
            case .errorKeychain:
-                Text("Keychain error")
-                    .font(.title)
+                titleText("Keychain error")
                Text("Cannot access keychain to save database password")
+            case .invalidConfirmation:
+                // this can only happen if incorrect parameter is passed
+                Text(String("Invalid migration confirmation")).font(.title)
            case let .unknown(json):
-                Text("Database error")
-                    .font(.title)
+                titleText("Database error")
                Text("Unknown database error: \(json)")
            case .ok:
                EmptyView()
@@ -61,10 +89,31 @@ struct DatabaseErrorView: View {
            }
        }
        .padding()
-        .frame(maxHeight: .infinity, alignment: .topLeading)
+        .frame(maxWidth: .infinity, maxHeight: .infinity, alignment: .topLeading)
        .onAppear() { showRestoreDbButton = shouldShowRestoreDbButton() }
    }

+    private func titleText(_ s: LocalizedStringKey) -> Text {
+        Text(s).font(.title)
+    }
+
+    private func fileNameText(_ f: String) -> Text {
+        Text("File: \((f as NSString).lastPathComponent)")
+    }
+
+    private func migrationsText(_ ms: [String]) -> Text {
+        Text("Migrations: \(ms.joined(separator: ", "))")
+    }
+
+    private func mtrErrorDescription(_ err: MTRError) -> LocalizedStringKey {
+        switch err {
+        case let .noDown(dbMigrations):
+            return "database version is newer than the app, but no down migration for: \(dbMigrations.joined(separator: ", "))"
+        case let .different(appMigration, dbMigration):
+            return "different migration in the app/database: \(appMigration) / \(dbMigration)"
+        }
+    }
+
    private func databaseKeyField(onSubmit: @escaping () -> Void) -> some View {
        PassphraseField(key: $dbKey, placeholder: "Enter passphrase…", valid: validKey(dbKey), onSubmit: onSubmit)
    }
@@ -89,14 +138,24 @@ struct DatabaseErrorView: View {
        runChat()
    }

-    private func runChat() {
+    private func runChat(confirmMigrations: MigrationConfirmation? = nil) {
+        starting = true
+        DispatchQueue.main.asyncAfter(deadline: .now() + 0.1) {
+            runChatSync(confirmMigrations: confirmMigrations)
+            starting = false
+        }
+    }
+
+    private func runChatSync(confirmMigrations: MigrationConfirmation? = nil) {
        do {
            resetChatCtrl()
-            try initializeChat(start: m.v3DBMigration.startChat, dbKey: dbKey)
+            try initializeChat(start: m.v3DBMigration.startChat, dbKey: useKeychain ? nil : dbKey, confirmMigrations: confirmMigrations)
            if let s = m.chatDbStatus {
                status = s
                let am = AlertManager.shared
                switch s {
+                case .invalidConfirmation:
+                    am.showAlert(Alert(title: Text(String("Invalid migration confirmation"))))
                case .errorNotADatabase:
                    am.showAlertMsg(
                        title: "Wrong passphrase!",
@@ -104,7 +163,7 @@ struct DatabaseErrorView: View {
                    )
                case .errorKeychain:
                    am.showAlertMsg(title: "Keychain error")
-                case let .error(_, error):
+                case let .errorSQL(_, error):
                    am.showAlert(Alert(
                        title: Text("Database error"),
                        message: Text(error)
@@ -114,6 +173,7 @@ struct DatabaseErrorView: View {
                        title: Text("Unknown error"),
                        message: Text(error)
                    ))
+                case .errorMigration: ()
                case .ok: ()
                }
            }
@@ -11,7 +11,7 @@ import SimpleXChat

 struct CallSettings: View {
    @AppStorage(DEFAULT_WEBRTC_POLICY_RELAY) private var webrtcPolicyRelay = true
-    @AppStorage(GROUP_DEFAULT_CALL_KIT_ENABLED, store: UserDefaults(suiteName: APP_GROUP_NAME)!) private var callKitEnabled = true
+    @AppStorage(GROUP_DEFAULT_CALL_KIT_ENABLED, store: groupDefaults) private var callKitEnabled = true
    @AppStorage(DEFAULT_CALL_KIT_CALLS_IN_RECENTS) private var callKitCallsInRecents = false
    @AppStorage(DEFAULT_DEVELOPER_TOOLS) private var developerTools = false
    private let allowChangingCallsHistory = false
@@ -0,0 +1,50 @@
+//
+//  DeveloperView.swift
+//  SimpleX (iOS)
+//
+//  Created by Evgeny on 26/03/2023.
+//  Copyright © 2023 SimpleX Chat. All rights reserved.
+//
+
+import SwiftUI
+import SimpleXChat
+
+struct DeveloperView: View {
+    @AppStorage(DEFAULT_DEVELOPER_TOOLS) private var developerTools = false
+    @AppStorage(GROUP_DEFAULT_CONFIRM_DB_UPGRADES, store: groupDefaults) private var confirmDatabaseUpgrades = false
+    @Environment(\.colorScheme) var colorScheme
+
+    var body: some View {
+        VStack {
+            List {
+                Section {
+                    NavigationLink {
+                        TerminalView()
+                    } label: {
+                        settingsRow("terminal") { Text("Chat console") }
+                    }
+                    settingsRow("chevron.left.forwardslash.chevron.right") {
+                        Toggle("Show developer options", isOn: $developerTools)
+                    }
+                    settingsRow("internaldrive") {
+                        Toggle("Confirm database upgrades", isOn: $confirmDatabaseUpgrades)
+                    }
+                    ZStack(alignment: .leading) {
+                        Image(colorScheme == .dark ? "github_light" : "github")
+                            .resizable()
+                            .frame(width: 24, height: 24)
+                            .opacity(0.5)
+                        Text("Install [SimpleX Chat for terminal](https://github.com/simplex-chat/simplex-chat)")
+                            .padding(.leading, 36)
+                    }
+                }
+            }
+        }
+    }
+}
+
+struct DeveloperView_Previews: PreviewProvider {
+    static var previews: some View {
+        DeveloperView()
+    }
+}
@@ -108,7 +108,6 @@ struct SettingsView: View {
    @EnvironmentObject var chatModel: ChatModel
    @EnvironmentObject var sceneDelegate: SceneDelegate
    @Binding var showSettings: Bool
-    @AppStorage(DEFAULT_DEVELOPER_TOOLS) private var developerTools = false
    @State private var settingsSheet: SettingsSheet?

    var body: some View {
@@ -259,23 +258,11 @@ struct SettingsView: View {
                }

                Section("Develop") {
-                    settingsRow("chevron.left.forwardslash.chevron.right") {
-                        Toggle("Developer tools", isOn: $developerTools)
-                    }
-                    if developerTools {
-                        NavigationLink {
-                            TerminalView()
-                        } label: {
-                            settingsRow("terminal") { Text("Chat console") }
-                        }
-                        ZStack(alignment: .leading) {
-                            Image(colorScheme == .dark ? "github_light" : "github")
-                                .resizable()
-                                .frame(width: 24, height: 24)
-                                .opacity(0.5)
-                            Text("Install [SimpleX Chat for terminal](https://github.com/simplex-chat/simplex-chat)")
-                                .padding(.leading, indent)
-                        }
+                    NavigationLink {
+                        DeveloperView()
+                            .navigationTitle("Developer tools")
+                    } label: {
+                        settingsRow("chevron.left.forwardslash.chevron.right") { Text("Developer tools") }
                    }
 //                    NavigationLink {
 //                        ExperimentalFeaturesView()
@@ -203,7 +203,7 @@ var networkConfig: NetCfg = getNetCfg()
 func startChat() -> DBMigrationResult? {
    hs_init(0, nil)
    if chatStarted { return .ok }
-    let (_, dbStatus) = chatMigrateInit()
+    let (_, dbStatus) = chatMigrateInit(confirmMigrations: defaultMigrationConfirmation())
    if dbStatus != .ok {
        resetChatCtrl()
        return dbStatus
@@ -58,6 +58,7 @@
 		5C65DAF529CBA429003CEE45 /* libHSsimplex-chat-4.6.0.0-KxI2qGrpKDHEZQGy0eoUXU.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 5C65DAF029CBA429003CEE45 /* libHSsimplex-chat-4.6.0.0-KxI2qGrpKDHEZQGy0eoUXU.a */; };
 		5C65DAF629CBA429003CEE45 /* libgmpxx.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 5C65DAF129CBA429003CEE45 /* libgmpxx.a */; };
 		5C65DAF729CBA429003CEE45 /* libffi.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 5C65DAF229CBA429003CEE45 /* libffi.a */; };
+		5C65DAF929D0CC20003CEE45 /* DeveloperView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5C65DAF829D0CC20003CEE45 /* DeveloperView.swift */; };
 		5C65F343297D45E100B67AF3 /* VersionView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5C65F341297D3F3600B67AF3 /* VersionView.swift */; };
 		5C6AD81327A834E300348BD7 /* NewChatButton.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5C6AD81227A834E300348BD7 /* NewChatButton.swift */; };
 		5C6BA667289BD954009B8ECC /* DismissSheets.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5C6BA666289BD954009B8ECC /* DismissSheets.swift */; };
@@ -294,6 +295,7 @@
 		5C65DAF029CBA429003CEE45 /* libHSsimplex-chat-4.6.0.0-KxI2qGrpKDHEZQGy0eoUXU.a */ = {isa = PBXFileReference; lastKnownFileType = archive.ar; path = "libHSsimplex-chat-4.6.0.0-KxI2qGrpKDHEZQGy0eoUXU.a"; sourceTree = "<group>"; };
 		5C65DAF129CBA429003CEE45 /* libgmpxx.a */ = {isa = PBXFileReference; lastKnownFileType = archive.ar; path = libgmpxx.a; sourceTree = "<group>"; };
 		5C65DAF229CBA429003CEE45 /* libffi.a */ = {isa = PBXFileReference; lastKnownFileType = archive.ar; path = libffi.a; sourceTree = "<group>"; };
+		5C65DAF829D0CC20003CEE45 /* DeveloperView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DeveloperView.swift; sourceTree = "<group>"; };
 		5C65F341297D3F3600B67AF3 /* VersionView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = VersionView.swift; sourceTree = "<group>"; };
 		5C6AD81227A834E300348BD7 /* NewChatButton.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = NewChatButton.swift; sourceTree = "<group>"; };
 		5C6BA666289BD954009B8ECC /* DismissSheets.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DismissSheets.swift; sourceTree = "<group>"; };
@@ -691,6 +693,7 @@
 				64F1CC3A28B39D8600CD1FB1 /* IncognitoHelp.swift */,
 				18415845648CA4F5A8BCA272 /* UserProfilesView.swift */,
 				5C65F341297D3F3600B67AF3 /* VersionView.swift */,
+				5C65DAF829D0CC20003CEE45 /* DeveloperView.swift */,
 			);
 			path = UserSettings;
 			sourceTree = "<group>";
@@ -1022,6 +1025,7 @@
 				5C93292F29239A170090FFF9 /* SMPServersView.swift in Sources */,
 				5CB924D727A8563F00ACCCDD /* SettingsView.swift in Sources */,
 				5CEACCE327DE9246000BD591 /* ComposeView.swift in Sources */,
+				5C65DAF929D0CC20003CEE45 /* DeveloperView.swift in Sources */,
 				5C36027327F47AD5009F19D9 /* AppDelegate.swift in Sources */,
 				5CB924E127A867BA00ACCCDD /* UserProfile.swift in Sources */,
 				5CB0BA9A2827FD8800B3292C /* HowItWorks.swift in Sources */,
@@ -17,7 +17,7 @@ public func getChatCtrl(_ useKey: String? = nil) -> chat_ctrl {
    fatalError("chat controller not initialized")
 }

-public func chatMigrateInit(_ useKey: String? = nil) -> (Bool, DBMigrationResult) {
+public func chatMigrateInit(_ useKey: String? = nil, confirmMigrations: MigrationConfirmation? = nil) -> (Bool, DBMigrationResult) {
    if let res = migrationResult { return res }
    let dbPath = getAppDatabasePath().path
    var dbKey = ""
@@ -34,12 +34,14 @@ public func chatMigrateInit(_ useKey: String? = nil) -> (Bool, DBMigrationResult
            dbKey = key
        }
    }
-    logger.debug("chatMigrateInit DB path: \(dbPath)")
+    let confirm = confirmMigrations ?? defaultMigrationConfirmation()
+    logger.debug("chatMigrateInit DB path: \(dbPath), confirm: \(confirm.rawValue)")
 //    logger.debug("chatMigrateInit DB key: \(dbKey)")
    var cPath = dbPath.cString(using: .utf8)!
    var cKey = dbKey.cString(using: .utf8)!
+    var cConfirm = confirm.rawValue.cString(using: .utf8)!
    // the last parameter of chat_migrate_init is used to return the pointer to chat controller
-    let cjson = chat_migrate_init(&cPath, &cKey, &chatController)!
+    let cjson = chat_migrate_init(&cPath, &cKey, &cConfirm, &chatController)!
    let dbRes = dbMigrationResult(fromCString(cjson))
    let encrypted = dbKey != ""
    let keychainErr = dbRes == .ok && useKeychain && encrypted && !setDatabaseKey(dbKey)
@@ -207,12 +209,40 @@ func chatErrorString(_ err: ChatError) -> String {

 public enum DBMigrationResult: Decodable, Equatable {
    case ok
+    case invalidConfirmation
    case errorNotADatabase(dbFile: String)
-    case error(dbFile: String, migrationError: String)
+    case errorMigration(dbFile: String, migrationError: MigrationError)
+    case errorSQL(dbFile: String, migrationSQLError: String)
    case errorKeychain
    case unknown(json: String)
 }

+public enum MigrationConfirmation: String {
+    case yesUp
+    case yesUpDown
+    case error
+}
+
+public func defaultMigrationConfirmation() -> MigrationConfirmation {
+    confirmDBUpgradesGroupDefault.get() ? .error : .yesUp
+}
+
+public enum MigrationError: Decodable, Equatable {
+    case upgrade(upMigrations: [UpMigration])
+    case downgrade(downMigrations: [String])
+    case migrationError(mtrError: MTRError)
+}
+
+public struct UpMigration: Decodable, Equatable {
+    public var upName: String
+//    public var withDown: Bool
+}
+
+public enum MTRError: Decodable, Equatable {
+    case noDown(dbMigrations: [String])
+    case different(appMigration: String, dbMigration: String)
+}
+
 func dbMigrationResult(_ s: String) -> DBMigrationResult {
    let d = s.data(using: .utf8)!
 // TODO is there a way to do it without copying the data? e.g:
@@ -459,7 +459,7 @@ public enum ChatResponse: Decodable, Error {
    case ntfMessages(user_: User?, connEntity: ConnectionEntity?, msgTs: Date?, ntfMessages: [NtfMsgInfo])
    case newContactConnection(user: User, connection: PendingContactConnection)
    case contactConnectionDeleted(user: User, connection: PendingContactConnection)
-    case versionInfo(versionInfo: CoreVersionInfo)
+    case versionInfo(versionInfo: CoreVersionInfo, chatMigrations: [UpMigration], agentMigrations: [UpMigration])
    case cmdOk(user: User?)
    case chatCmdError(user_: User?, chatError: ChatError)
    case chatError(user_: User?, chatError: ChatError)
@@ -674,7 +674,7 @@ public enum ChatResponse: Decodable, Error {
            case let .ntfMessages(u, connEntity, msgTs, ntfMessages): return withUser(u, "connEntity: \(String(describing: connEntity))\nmsgTs: \(String(describing: msgTs))\nntfMessages: \(String(describing: ntfMessages))")
            case let .newContactConnection(u, connection): return withUser(u, String(describing: connection))
            case let .contactConnectionDeleted(u, connection): return withUser(u, String(describing: connection))
-            case let .versionInfo(versionInfo): return String(describing: versionInfo)
+            case let .versionInfo(versionInfo, chatMigrations, agentMigrations): return "\(String(describing: versionInfo))\n\nchat migrations: \(chatMigrations.map(\.upName))\n\nagent migrations: \(agentMigrations.map(\.upName))"
            case .cmdOk: return noDetails
            case let .chatCmdError(u, chatError): return withUser(u, String(describing: chatError))
            case let .chatError(u, chatError): return withUser(u, String(describing: chatError))
@@ -29,6 +29,7 @@ let GROUP_DEFAULT_NETWORK_TCP_KEEP_CNT = "networkTCPKeepCnt"
 let GROUP_DEFAULT_INCOGNITO = "incognito"
 let GROUP_DEFAULT_STORE_DB_PASSPHRASE = "storeDBPassphrase"
 let GROUP_DEFAULT_INITIAL_RANDOM_DB_PASSPHRASE = "initialRandomDBPassphrase"
+public let GROUP_DEFAULT_CONFIRM_DB_UPGRADES = "confirmDBUpgrades"
 public let GROUP_DEFAULT_CALL_KIT_ENABLED = "callKitEnabled"

 public let APP_GROUP_NAME = "group.chat.simplex.app"
@@ -52,6 +53,7 @@ public func registerGroupDefaults() {
        GROUP_DEFAULT_INITIAL_RANDOM_DB_PASSPHRASE: false,
        GROUP_DEFAULT_PRIVACY_ACCEPT_IMAGES: true,
        GROUP_DEFAULT_PRIVACY_TRANSFER_IMAGES_INLINE: false,
+        GROUP_DEFAULT_CONFIRM_DB_UPGRADES: false,
        GROUP_DEFAULT_CALL_KIT_ENABLED: true
    ])
 }
@@ -121,6 +123,8 @@ public let storeDBPassphraseGroupDefault = BoolDefault(defaults: groupDefaults,

 public let initialRandomDBPassphraseGroupDefault = BoolDefault(defaults: groupDefaults, forKey: GROUP_DEFAULT_INITIAL_RANDOM_DB_PASSPHRASE)

+public let confirmDBUpgradesGroupDefault = BoolDefault(defaults: groupDefaults, forKey: GROUP_DEFAULT_CONFIRM_DB_UPGRADES)
+
 public let callKitEnabledGroupDefault = BoolDefault(defaults: groupDefaults, forKey: GROUP_DEFAULT_CALL_KIT_ENABLED)

 public class DateDefault {
@@ -127,12 +127,16 @@ public func createErrorNtf(_ dbStatus: DBMigrationResult) -> UNMutableNotificati
    switch dbStatus {
    case .errorNotADatabase:
        title = NSLocalizedString("Encrypted message: no passphrase", comment: "notification")
-    case .error:
+    case .errorMigration:
+        title = NSLocalizedString("Encrypted message: database migration error", comment: "notification")
+    case .errorSQL:
        title = NSLocalizedString("Encrypted message: database error", comment: "notification")
    case .errorKeychain:
        title = NSLocalizedString("Encrypted message: keychain error", comment: "notification")
    case .unknown:
        title = NSLocalizedString("Encrypted message: unexpected error", comment: "notification")
+    case .invalidConfirmation:
+        title = NSLocalizedString("Encrypted message or another event", comment: "notification")
    case .ok:
        title = NSLocalizedString("Encrypted message or another event", comment: "notification")
    }
@@ -16,7 +16,7 @@ extern void hs_init(int argc, char **argv[]);
 typedef void* chat_ctrl;

 // the last parameter is used to return the pointer to chat controller
-extern char *chat_migrate_init(char *path, char *key, chat_ctrl *ctrl);
+extern char *chat_migrate_init(char *path, char *key, char *confirm, chat_ctrl *ctrl);
 extern char *chat_send_cmd(chat_ctrl ctl, char *cmd);
 extern char *chat_recv_msg(chat_ctrl ctl);
 extern char *chat_recv_msg_wait(chat_ctrl ctl, int wait);