ios: link previews (#503)

* refactor image utils and initial link metadata tools

* remove LPMetadata conversion as we will build our own view to avoid network calls

* initial very basic preview outline, remove icon loading

* connect preview view to compose view

* v0.1 barely working

* minor refactor

* refactor

* collect images effectively

* link up to api for send/receive

* rework async get metadata logic

* show previews in chat

* refactor resizing logic

* checkpoint before view editing

* ui changes

* housekeeping

* ui tweaks

* typo

* improve link preview design/logic

* resize image to target data size

* fix link preview state machine

* tidy up

* fix typo

Co-authored-by: Evgeny Poberezkin <2769109+epoberezkin@users.noreply.github.com>
This commit is contained in:
IanRDavies
2022-04-08 18:17:10 +01:00
committed by GitHub
parent 9fda89d0db
commit d90c4261b8
12 changed files with 361 additions and 50 deletions
+29 -1
View File
@@ -726,6 +726,15 @@ enum CIContent: Decodable, ItemContent {
}
}
}
var msgContent: MsgContent? {
get {
switch self {
case let .sndMsgContent(mc): return mc
case let .rcvMsgContent(mc): return mc
default: return nil
}
}
}
}
struct RcvFileTransfer: Decodable {
@@ -761,6 +770,7 @@ struct CIQuote: Decodable, ItemContent {
enum MsgContent {
case text(String)
case link(text: String, preview: LinkPreview)
// TODO include original JSON, possibly using https://github.com/zoul/generic-json-swift
case unknown(type: String, text: String)
@@ -768,6 +778,7 @@ enum MsgContent {
get {
switch self {
case let .text(text): return text
case let .link(text, _): return text
case let .unknown(_, text): return text
}
}
@@ -777,6 +788,8 @@ enum MsgContent {
get {
switch self {
case let .text(text): return "text \(text)"
case let .link(text: text, preview: preview):
return "json {\"type\":\"link\",\"text\":\(encodeJSON(text)),\"preview\":\(encodeJSON(preview))}"
default: return ""
}
}
@@ -785,9 +798,11 @@ enum MsgContent {
enum CodingKeys: String, CodingKey {
case type
case text
case preview
}
}
// TODO define Encodable
extension MsgContent: Decodable {
init(from decoder: Decoder) throws {
do {
@@ -797,6 +812,10 @@ extension MsgContent: Decodable {
case "text":
let text = try container.decode(String.self, forKey: CodingKeys.text)
self = .text(text)
case "link":
let text = try container.decode(String.self, forKey: CodingKeys.text)
let preview = try container.decode(LinkPreview.self, forKey: CodingKeys.preview)
self = .link(text: text, preview: preview)
default:
let text = try? container.decode(String.self, forKey: CodingKeys.text)
self = .unknown(type: type, text: text ?? "unknown message format")
@@ -812,7 +831,7 @@ struct FormattedText: Decodable {
var format: Format?
}
enum Format: Decodable {
enum Format: Decodable, Equatable {
case bold
case italic
case strikeThrough
@@ -849,3 +868,12 @@ enum FormatColor: String, Decodable {
}
}
}
// Struct to use with simplex API
struct LinkPreview: Codable {
var uri: URL
var title: String
// TODO remove once optional in haskell
var description: String = ""
var image: String
}
+14 -2
View File
@@ -13,7 +13,7 @@ import BackgroundTasks
private var chatController: chat_ctrl?
private let jsonDecoder = getJSONDecoder()
private let jsonEncoder = getJSONEncoder()
let jsonEncoder = getJSONEncoder()
enum ChatCommand {
case showActiveUser
@@ -31,6 +31,7 @@ enum ChatCommand {
case connect(connReq: String)
case apiDeleteChat(type: ChatType, id: Int64)
case apiUpdateProfile(profile: Profile)
case apiParseMarkdown(text: String)
case createMyAddress
case deleteMyAddress
case showMyAddress
@@ -57,6 +58,7 @@ enum ChatCommand {
case let .connect(connReq): return "/connect \(connReq)"
case let .apiDeleteChat(type, id): return "/_delete \(ref(type, id))"
case let .apiUpdateProfile(profile): return "/_profile \(encodeJSON(profile))"
case let .apiParseMarkdown(text): return "/_parse \(text)"
case .createMyAddress: return "/address"
case .deleteMyAddress: return "/delete_address"
case .showMyAddress: return "/show_address"
@@ -86,6 +88,7 @@ enum ChatCommand {
case .connect: return "connect"
case .apiDeleteChat: return "apiDeleteChat"
case .apiUpdateProfile: return "apiUpdateProfile"
case .apiParseMarkdown: return "apiParseMarkdown"
case .createMyAddress: return "createMyAddress"
case .deleteMyAddress: return "deleteMyAddress"
case .showMyAddress: return "showMyAddress"
@@ -125,6 +128,7 @@ enum ChatResponse: Decodable, Error {
case contactDeleted(contact: Contact)
case userProfileNoChange
case userProfileUpdated(fromProfile: Profile, toProfile: Profile)
case apiParsedMarkdown(formattedText: [FormattedText]?)
case userContactLink(connReqContact: String)
case userContactLinkCreated(connReqContact: String)
case userContactLinkDeleted
@@ -166,6 +170,7 @@ enum ChatResponse: Decodable, Error {
case .contactDeleted: return "contactDeleted"
case .userProfileNoChange: return "userProfileNoChange"
case .userProfileUpdated: return "userProfileUpdated"
case .apiParsedMarkdown: return "apiParsedMarkdown"
case .userContactLink: return "userContactLink"
case .userContactLinkCreated: return "userContactLinkCreated"
case .userContactLinkDeleted: return "userContactLinkDeleted"
@@ -210,6 +215,7 @@ enum ChatResponse: Decodable, Error {
case let .contactDeleted(contact): return String(describing: contact)
case .userProfileNoChange: return noDetails
case let .userProfileUpdated(_, toProfile): return String(describing: toProfile)
case let .apiParsedMarkdown(formattedText): return String(describing: formattedText)
case let .userContactLink(connReq): return connReq
case let .userContactLinkCreated(connReq): return connReq
case .userContactLinkDeleted: return noDetails
@@ -487,6 +493,12 @@ func apiUpdateProfile(profile: Profile) async throws -> Profile? {
}
}
func apiParseMarkdown(text: String) throws -> [FormattedText]? {
let r = chatSendCmdSync(.apiParseMarkdown(text: text))
if case let .apiParsedMarkdown(formattedText) = r { return formattedText }
throw r
}
func apiCreateUserAddress() async throws -> String {
let r = await chatSendCmd(.createMyAddress)
if case let .userContactLinkCreated(connReq) = r { return connReq }
@@ -774,7 +786,7 @@ private func getJSONObject(_ cjson: UnsafePointer<CChar>) -> NSDictionary? {
return try? JSONSerialization.jsonObject(with: d) as? NSDictionary
}
private func encodeJSON<T: Encodable>(_ value: T) -> String {
func encodeJSON<T: Encodable>(_ value: T) -> String {
let data = try! jsonEncoder.encode(value)
return String(decoding: data, as: UTF8.self)
}
@@ -8,8 +8,8 @@
import SwiftUI
private let sentColorLight = Color(.sRGB, red: 0.27, green: 0.72, blue: 1, opacity: 0.12)
private let sentColorDark = Color(.sRGB, red: 0.27, green: 0.72, blue: 1, opacity: 0.17)
let sentColorLight = Color(.sRGB, red: 0.27, green: 0.72, blue: 1, opacity: 0.12)
let sentColorDark = Color(.sRGB, red: 0.27, green: 0.72, blue: 1, opacity: 0.17)
private let sentQuoteColorLight = Color(.sRGB, red: 0.27, green: 0.72, blue: 1, opacity: 0.11)
private let sentQuoteColorDark = Color(.sRGB, red: 0.27, green: 0.72, blue: 1, opacity: 0.09)
@@ -51,6 +51,11 @@ struct FramedItemView: View {
.frame(minWidth: msgWidth, alignment: .center)
.padding(.bottom, 2)
} else {
if case let .link(_, preview) = chatItem.content.msgContent {
ChatItemLinkView(linkPreview: preview, width: msgWidth)
.overlay(DetermineWidth())
.frame(minWidth: msgWidth, alignment: .leading)
}
MsgContentView(
content: chatItem.content,
formattedText: chatItem.formattedText,
+13 -3
View File
@@ -17,6 +17,7 @@ struct ChatView: View {
@State var message: String = ""
@State var quotedItem: ChatItem? = nil
@State var editingItem: ChatItem? = nil
@State var linkPreview: LinkPreview? = nil
@State var deletingItem: ChatItem? = nil
@State private var inProgress: Bool = false
@FocusState private var keyboardVisible: Bool
@@ -85,6 +86,7 @@ struct ChatView: View {
message: $message,
quotedItem: $quotedItem,
editingItem: $editingItem,
linkPreview: $linkPreview,
sendMessage: sendMessage,
resetMessage: { message = "" },
inProgress: inProgress,
@@ -200,7 +202,7 @@ struct ChatView: View {
}
}
func sendMessage(_ msg: String) {
func sendMessage(_ text: String) {
logger.debug("ChatView sendMessage")
Task {
logger.debug("ChatView sendMessage: in Task")
@@ -210,21 +212,29 @@ struct ChatView: View {
type: chat.chatInfo.chatType,
id: chat.chatInfo.apiId,
itemId: ei.id,
msg: .text(msg)
msg: .text(text)
)
DispatchQueue.main.async {
editingItem = nil
linkPreview = nil
let _ = chatModel.upsertChatItem(chat.chatInfo, chatItem)
}
} else {
let mc: MsgContent
if let preview = linkPreview {
mc = .link(text: text, preview: preview)
} else {
mc = .text(text)
}
let chatItem = try await apiSendMessage(
type: chat.chatInfo.chatType,
id: chat.chatInfo.apiId,
quotedItemId: quotedItem?.meta.itemId,
msg: .text(msg)
msg: mc
)
DispatchQueue.main.async {
quotedItem = nil
linkPreview = nil
chatModel.addChatItem(chat.chatInfo, chatItem)
}
}
@@ -19,21 +19,62 @@ struct ComposeView: View {
@Binding var message: String
@Binding var quotedItem: ChatItem?
@Binding var editingItem: ChatItem?
@Binding var linkPreview: LinkPreview?
var sendMessage: (String) -> Void
var resetMessage: () -> Void
var inProgress: Bool = false
@FocusState.Binding var keyboardVisible: Bool
@State var editing: Bool = false
@State var linkUrl: URL? = nil
@State var prevLinkUrl: URL? = nil
@State var pendingLinkUrl: URL? = nil
@State var cancelledLinks: Set<String> = []
private func isValidLink(link: String) -> Bool {
return !(link.starts(with: "https://simplex.chat") || link.starts(with: "http://simplex.chat"))
}
func cancelPreview() {
if let uri = linkPreview?.uri.absoluteString {
cancelledLinks.insert(uri)
}
linkPreview = nil
}
func parseMessage(_ msg: String) -> URL? {
do {
if let parsedMsg = try apiParseMarkdown(text: msg),
let link = parsedMsg.first(where: {
$0.format == .uri && !cancelledLinks.contains($0.text)
}),
isValidLink(link: link.text) {
return URL(string: link.text)
} else {
return nil
}
} catch {
logger.error("apiParseMarkdown error: \(error.localizedDescription)")
return nil
}
}
var body: some View {
VStack(spacing: 0) {
if let metadata = linkPreview {
ComposeLinkView(linkPreview: metadata, cancelPreview: cancelPreview)
}
if (quotedItem != nil) {
ContextItemView(contextItem: $quotedItem, editing: $editing)
} else if (editingItem != nil) {
ContextItemView(contextItem: $editingItem, editing: $editing, resetMessage: resetMessage)
}
SendMessageView(
sendMessage: sendMessage,
sendMessage: { text in
sendMessage(text)
resetLinkPreview()
},
inProgress: inProgress,
message: $message,
keyboardVisible: $keyboardVisible,
@@ -41,10 +82,48 @@ struct ComposeView: View {
)
.background(.background)
}
.onChange(of: message) { _ in
if message.count > 0 {
prevLinkUrl = linkUrl
linkUrl = parseMessage(message)
if let url = linkUrl {
if prevLinkUrl == linkUrl {
loadLinkPreview(url)
} else {
DispatchQueue.main.asyncAfter(deadline: .now() + 1.5) {
loadLinkPreview(url)
}
}
} else {
linkPreview = nil
}
} else {
resetLinkPreview()
}
}
.onChange(of: editingItem == nil) { _ in
editing = (editingItem != nil)
}
}
func loadLinkPreview(_ url: URL) {
if url != linkPreview?.uri && url != pendingLinkUrl {
pendingLinkUrl = url
getLinkPreview(url: url) { lp in
if pendingLinkUrl == url {
linkPreview = lp
pendingLinkUrl = nil
}
}
}
}
func resetLinkPreview() {
linkUrl = nil
prevLinkUrl = nil
pendingLinkUrl = nil
cancelledLinks = []
}
}
struct ComposeView_Previews: PreviewProvider {
@@ -53,12 +132,14 @@ struct ComposeView_Previews: PreviewProvider {
@FocusState var keyboardVisible: Bool
@State var item: ChatItem? = ChatItem.getSample(1, .directSnd, .now, "hello")
@State var nilItem: ChatItem? = nil
@State var linkPreview: LinkPreview? = nil
return Group {
ComposeView(
message: $message,
quotedItem: $item,
editingItem: $nilItem,
linkPreview: $linkPreview,
sendMessage: { print ($0) },
resetMessage: {},
keyboardVisible: $keyboardVisible
@@ -67,6 +148,7 @@ struct ComposeView_Previews: PreviewProvider {
message: $message,
quotedItem: $nilItem,
editingItem: $item,
linkPreview: $linkPreview,
sendMessage: { print ($0) },
resetMessage: {},
keyboardVisible: $keyboardVisible
@@ -91,7 +91,6 @@ struct SendMessageView_Previews: PreviewProvider {
@State var editingOff: Bool = false
@State var editingOn: Bool = true
@State var item: ChatItem? = ChatItem.getSample(1, .directSnd, .now, "hello")
@State var nilItem: ChatItem? = nil
return Group {
VStack {
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -9,6 +9,64 @@
import SwiftUI
import PhotosUI
func dropPrefix(_ s: String, _ prefix: String) -> String {
s.hasPrefix(prefix) ? String(s.dropFirst(prefix.count)) : s
}
func dropImagePrefix(_ s: String) -> String {
dropPrefix(dropPrefix(s, "data:image/png;base64,"), "data:image/jpg;base64,")
}
func resizeAndCrop(_ image: UIImage, to newSize: CGSize) -> UIImage {
let format = UIGraphicsImageRendererFormat()
format.scale = 1.0
format.opaque = true
return UIGraphicsImageRenderer(bounds: CGRect(origin: .zero, size: newSize), format: format).image { _ in
let size = image.size
let hScale = newSize.height / size.height
let vScale = newSize.width / size.width
let scale = max(hScale, vScale) // scaleToFill
let resizeSize = CGSize(width: size.width * scale, height: size.height * scale)
var middle = CGPoint.zero
if resizeSize.width > newSize.width {
middle.x -= (resizeSize.width - newSize.width) / 2
} else if resizeSize.height > newSize.height {
middle.y -= (resizeSize.height - newSize.height) / 2
}
image.draw(in: CGRect(origin: middle, size: resizeSize))
}
}
func cropToSquare(_ image: UIImage) -> UIImage {
let side = min(image.size.width, image.size.height)
return resizeAndCrop(image, to: CGSize(width: side, height: side))
}
func resizeImageToDataSize(_ image: UIImage, maxSize: Int) -> String? {
let size = image.size
var imageStr = compressImage(image)
var resized = image
var ratio: CGFloat = 1
var dataSize = imageStr?.count ?? 0
logger.debug("resizeImageToDataSize: initial size \(String(describing: size)), data size \(dataSize)")
while dataSize != 0 && dataSize > maxSize {
ratio *= sqrt(CGFloat(dataSize / maxSize) * 1.2)
resized = resizeAndCrop(resized, to: CGSize(width: size.width / ratio, height: size.height / ratio))
imageStr = compressImage(resized)
dataSize = imageStr?.count ?? 0
logger.debug("resizeImageToDataSize: ratio \(ratio)")
}
logger.debug("resizeImageToDataSize: final size \(String(describing: resized.size)), data size \(dataSize)")
return imageStr
}
func compressImage(_ image: UIImage, _ compressionQuality: CGFloat = 0.85) -> String? {
if let data = image.jpegData(compressionQuality: compressionQuality) {
return "data:image/jpg;base64,\(data.base64EncodedString())"
}
return nil
}
enum ImageSource {
case imageLibrary
case camera
@@ -26,14 +26,6 @@ struct ProfileImage: View {
.foregroundColor(color)
}
}
func dropPrefix(_ s: String, _ prefix: String) -> String {
s.hasPrefix(prefix) ? String(s.dropFirst(prefix.count)) : s
}
func dropImagePrefix(_ s: String) -> String {
dropPrefix(dropPrefix(s, "data:image/png;base64,"), "data:image/jpg;base64,")
}
}
struct ProfileImage_Previews: PreviewProvider {
@@ -98,14 +98,8 @@ struct UserProfile: View {
}
}
.onChange(of: chosenImage) { image in
if let image = image,
let data = resizeToSquare(image, 104).jpegData(compressionQuality: 0.85) {
let imageStr = "data:image/jpg;base64,\(data.base64EncodedString())"
if imageStr.count <= 12500 {
profile.image = imageStr
} else {
logger.error("UserProfile: resized image is too big \(imageStr.count)")
}
if let image = image {
profile.image = resizeImageToDataSize(cropToSquare(image), maxSize: 12500)
} else {
profile.image = nil
}
@@ -167,30 +161,6 @@ struct UserProfile: View {
}
}
func resize(_ image: UIImage, to newSize: CGSize) -> UIImage {
let format = UIGraphicsImageRendererFormat()
format.scale = 1.0
format.opaque = true
return UIGraphicsImageRenderer(bounds: CGRect(origin: .zero, size: newSize), format: format).image { _ in
let size = image.size
let hScale = newSize.height / size.height
let vScale = newSize.width / size.width
let scale = max(hScale, vScale) // scaleToFill
let resizeSize = CGSize(width: size.width * scale, height: size.height * scale)
var middle = CGPoint.zero
if resizeSize.width > newSize.width {
middle.x -= (resizeSize.width - newSize.width) / 2
} else if resizeSize.height > newSize.height {
middle.y -= (resizeSize.height - newSize.height) / 2
}
image.draw(in: CGRect(origin: middle, size: resizeSize))
}
}
func resizeToSquare(_ image: UIImage, _ side: CGFloat) -> UIImage {
resize(image, to: CGSize(width: side, height: side))
}
struct UserProfile_Previews: PreviewProvider {
static var previews: some View {
let chatModel1 = ChatModel()
@@ -7,6 +7,10 @@
objects = {
/* Begin PBXBuildFile section */
3CDBCF4227FAE51000354CDD /* ComposeLinkView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3CDBCF4127FAE51000354CDD /* ComposeLinkView.swift */; };
3CDBCF4327FAE51000354CDD /* ComposeLinkView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3CDBCF4127FAE51000354CDD /* ComposeLinkView.swift */; };
3CDBCF4827FF621E00354CDD /* ChatItemLinkView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3CDBCF4727FF621E00354CDD /* ChatItemLinkView.swift */; };
3CDBCF4927FF621E00354CDD /* ChatItemLinkView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3CDBCF4727FF621E00354CDD /* ChatItemLinkView.swift */; };
5C063D2727A4564100AEC577 /* ChatPreviewView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5C063D2627A4564100AEC577 /* ChatPreviewView.swift */; };
5C063D2827A4564100AEC577 /* ChatPreviewView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5C063D2627A4564100AEC577 /* ChatPreviewView.swift */; };
5C116CDC27AABE0400E66D01 /* ContactRequestView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5C116CDB27AABE0400E66D01 /* ContactRequestView.swift */; };
@@ -138,6 +142,8 @@
/* End PBXContainerItemProxy section */
/* Begin PBXFileReference section */
3CDBCF4127FAE51000354CDD /* ComposeLinkView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ComposeLinkView.swift; sourceTree = "<group>"; };
3CDBCF4727FF621E00354CDD /* ChatItemLinkView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ChatItemLinkView.swift; sourceTree = "<group>"; };
5C063D2627A4564100AEC577 /* ChatPreviewView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ChatPreviewView.swift; sourceTree = "<group>"; };
5C116CDB27AABE0400E66D01 /* ContactRequestView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ContactRequestView.swift; sourceTree = "<group>"; };
5C1A4C1D27A715B700EAD5AD /* ChatItemView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ChatItemView.swift; sourceTree = "<group>"; };
@@ -321,6 +327,8 @@
5C3A88CD27DF50170060F1C2 /* DetermineWidth.swift */,
5C5F2B6C27EBC3FE006A9D5F /* ImagePicker.swift */,
5C5F2B6F27EBC704006A9D5F /* ProfileImage.swift */,
3CDBCF4127FAE51000354CDD /* ComposeLinkView.swift */,
3CDBCF4727FF621E00354CDD /* ChatItemLinkView.swift */,
);
path = Helpers;
sourceTree = "<group>";
@@ -622,6 +630,8 @@
5CB924E127A867BA00ACCCDD /* UserProfile.swift in Sources */,
5CE4407927ADB701007B033A /* EmojiItemView.swift in Sources */,
5C5346A827B59A6A004DF848 /* ChatHelp.swift in Sources */,
3CDBCF4227FAE51000354CDD /* ComposeLinkView.swift in Sources */,
3CDBCF4827FF621E00354CDD /* ChatItemLinkView.swift in Sources */,
5C764E80279C7276000C6508 /* dummy.m in Sources */,
5C7505A827B6D34800BE3227 /* ChatInfoToolbar.swift in Sources */,
5C3A88D127DF57800060F1C2 /* FramedItemView.swift in Sources */,
@@ -674,6 +684,8 @@
5CB924E227A867BA00ACCCDD /* UserProfile.swift in Sources */,
5CE4407A27ADB701007B033A /* EmojiItemView.swift in Sources */,
5C5346A927B59A6A004DF848 /* ChatHelp.swift in Sources */,
3CDBCF4327FAE51000354CDD /* ComposeLinkView.swift in Sources */,
3CDBCF4927FF621E00354CDD /* ChatItemLinkView.swift in Sources */,
5C764E81279C7276000C6508 /* dummy.m in Sources */,
5C7505A927B6D34800BE3227 /* ChatInfoToolbar.swift in Sources */,
5C3A88D227DF57800060F1C2 /* FramedItemView.swift in Sources */,