diff --git a/Dockerfile b/Dockerfile index 7b9641777a..cdcbc40d7d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -29,7 +29,7 @@ RUN cp ./scripts/cabal.project.local.linux ./cabal.project.local # Compile simplex-chat RUN cabal update -RUN cabal build exe:simplex-chat --constraint 'simplexmq +client_library' +RUN cabal build exe:simplex-chat --constraint 'simplexmq +client_library' --constraint 'simplex-chat +client_library' # Strip the binary from debug symbols to reduce size RUN bin=$(find /project/dist-newstyle -name "simplex-chat" -type f -executable) && \ diff --git a/README.md b/README.md index 936667da8c..1830228370 100644 --- a/README.md +++ b/README.md @@ -234,6 +234,10 @@ You can use SimpleX with your own servers and still communicate with people usin Recent and important updates: +[Mar 8, 2025. SimpleX Chat v6.3: new user experience and safety in public groups](./blog/20250308-simplex-chat-v6-3-new-user-experience-safety-in-public-groups.md) + +[Jan 14, 2025. SimpleX network: large groups and privacy-preserving content moderation](./blog/20250114-simplex-network-large-groups-privacy-preserving-content-moderation.md) + [Dec 10, 2024. SimpleX network: preset servers operated by Flux, business chats and more with v6.2 of the apps](./20241210-simplex-network-v6-2-servers-by-flux-business-chats.md) [Oct 14, 2024. SimpleX network: security review of protocols design by Trail of Bits, v6.1 released with better calls and user experience.](./blog/20241014-simplex-network-v6-1-security-review-better-calls-user-experience.md) @@ -305,12 +309,13 @@ What is already implemented: 15. Manual messaging queue rotations to move conversation to another SMP relay. 16. Sending end-to-end encrypted files using [XFTP protocol](https://simplex.chat/blog/20230301-simplex-file-transfer-protocol.html). 17. Local files encryption. +18. [Reproducible server builds](./docs/SERVER.md#reproduce-builds). We plan to add: 1. Automatic message queue rotation and redundancy. Currently the queues created between two users are used until the queue is manually changed by the user or contact is deleted. We are planning to add automatic queue rotation to make these identifiers temporary and rotate based on some schedule TBC (e.g., every X messages, or every X hours/days). 2. Message "mixing" - adding latency to message delivery, to protect against traffic correlation by message time. -3. Reproducible builds – this is the limitation of the development stack, but we will be investing into solving this problem. Users can still build all applications and services from the source code. +3. Reproducible clients builds – this is a complex problem, but we are aiming to have it in 2025 at least partially. 4. Recipients' XFTP relays to reduce traffic and conceal IP addresses from the relays chosen, and potentially controlled, by another party. ## For developers diff --git a/apps/ios/Shared/Model/ChatModel.swift b/apps/ios/Shared/Model/ChatModel.swift index 23b167a9ff..00260cc05e 100644 --- a/apps/ios/Shared/Model/ChatModel.swift +++ b/apps/ios/Shared/Model/ChatModel.swift @@ -66,6 +66,10 @@ class ItemsModel: ObservableObject { private var navigationTimeoutTask: Task? = nil private var loadChatTask: Task? = nil + var lastItemsLoaded: Bool { + chatState.splits.isEmpty || chatState.splits.first != reversedChatItems.first?.id + } + init() { publisher .throttle(for: 0.2, scheduler: DispatchQueue.main, latest: true) diff --git a/apps/ios/Shared/Views/Chat/ChatItemsLoader.swift b/apps/ios/Shared/Views/Chat/ChatItemsLoader.swift index add28cd7f9..07034cf8ec 100644 --- a/apps/ios/Shared/Views/Chat/ChatItemsLoader.swift +++ b/apps/ios/Shared/Views/Chat/ChatItemsLoader.swift @@ -60,6 +60,8 @@ func apiLoadMessages( chatState.unreadTotal = chat.chatStats.unreadCount chatState.unreadAfter = navInfo.afterUnread chatState.unreadAfterNewestLoaded = navInfo.afterUnread + + PreloadState.shared.clear() } case let .before(paginationChatItemId, _): newItems.append(contentsOf: oldItems) @@ -104,19 +106,22 @@ func apiLoadMessages( } } case .around: - let newSplits: [Int64] + var newSplits: [Int64] if openAroundItemId == nil { newItems.append(contentsOf: oldItems) newSplits = await removeDuplicatesAndUpperSplits(&newItems, chat, chatState.splits, visibleItemIndexesNonReversed) } else { newSplits = [] } - // currently, items will always be added on top, which is index 0 - newItems.insert(contentsOf: chat.chatItems, at: 0) + let (itemIndex, splitIndex) = indexToInsertAround(chat.chatInfo.chatType, chat.chatItems.last, to: newItems, Set(newSplits)) + //indexToInsertAroundTest() + newItems.insert(contentsOf: chat.chatItems, at: itemIndex) + newSplits.insert(chat.chatItems.last!.id, at: splitIndex) let newReversed: [ChatItem] = newItems.reversed() + let orderedSplits = newSplits await MainActor.run { ItemsModel.shared.reversedChatItems = newReversed - chatState.splits = [chat.chatItems.last!.id] + newSplits + chatState.splits = orderedSplits chatState.unreadAfterItemId = chat.chatItems.last!.id chatState.totalAfter = navInfo.afterTotal chatState.unreadTotal = chat.chatStats.unreadCount @@ -130,14 +135,16 @@ func apiLoadMessages( // no need to set it, count will be wrong // chatState.unreadAfterNewestLoaded = navInfo.afterUnread } + PreloadState.shared.clear() } case .last: newItems.append(contentsOf: oldItems) - removeDuplicates(&newItems, chat) + let newSplits = await removeDuplicatesAndUnusedSplits(&newItems, chat, chatState.splits) newItems.append(contentsOf: chat.chatItems) let items = newItems await MainActor.run { ItemsModel.shared.reversedChatItems = items.reversed() + chatState.splits = newSplits chatModel.updateChatInfo(chat.chatInfo) chatState.unreadAfterNewestLoaded = 0 } @@ -234,10 +241,14 @@ private func removeDuplicatesAndModifySplitsOnAfterPagination( let indexInSplitRanges = splits.firstIndex(of: paginationChatItemId) // Currently, it should always load from split range let loadingFromSplitRange = indexInSplitRanges != nil - var splitsToMerge: [Int64] = if let indexInSplitRanges, loadingFromSplitRange && indexInSplitRanges + 1 <= splits.count { - Array(splits[indexInSplitRanges + 1 ..< splits.count]) + let topSplits: [Int64] + var splitsToMerge: [Int64] + if let indexInSplitRanges, loadingFromSplitRange && indexInSplitRanges + 1 <= splits.count { + splitsToMerge = Array(splits[indexInSplitRanges + 1 ..< splits.count]) + topSplits = Array(splits[0 ..< indexInSplitRanges + 1]) } else { - [] + splitsToMerge = [] + topSplits = [] } newItems.removeAll(where: { new in let duplicate = newIds.contains(new.id) @@ -257,8 +268,8 @@ private func removeDuplicatesAndModifySplitsOnAfterPagination( }) var newSplits: [Int64] = [] if firstItemIdBelowAllSplits != nil { - // no splits anymore, all were merged with bottom items - newSplits = [] + // no splits below anymore, all were merged with bottom items + newSplits = topSplits } else { if !splitsToRemove.isEmpty { var new = splits @@ -320,6 +331,28 @@ private func removeDuplicatesAndUpperSplits( return newSplits } +private func removeDuplicatesAndUnusedSplits( + _ newItems: inout [ChatItem], + _ chat: Chat, + _ splits: [Int64] +) async -> [Int64] { + if splits.isEmpty { + removeDuplicates(&newItems, chat) + return splits + } + + var newSplits = splits + let (newIds, _) = mapItemsToIds(chat.chatItems) + newItems.removeAll(where: { + let duplicate = newIds.contains($0.id) + if duplicate, let firstIndex = newSplits.firstIndex(of: $0.id) { + newSplits.remove(at: firstIndex) + } + return duplicate + }) + return newSplits +} + // ids, number of unread items private func mapItemsToIds(_ items: [ChatItem]) -> (Set, Int) { var unreadInLoaded = 0 @@ -340,3 +373,139 @@ private func removeDuplicates(_ newItems: inout [ChatItem], _ chat: Chat) { let (newIds, _) = mapItemsToIds(chat.chatItems) newItems.removeAll { newIds.contains($0.id) } } + +private typealias SameTimeItem = (index: Int, item: ChatItem) + +// return (item index, split index) +private func indexToInsertAround(_ chatType: ChatType, _ lastNew: ChatItem?, to: [ChatItem], _ splits: Set) -> (Int, Int) { + guard to.count > 0, let lastNew = lastNew else { return (0, 0) } + // group sorting: item_ts, item_id + // everything else: created_at, item_id + let compareByTimeTs = chatType == .group + // in case several items have the same time as another item in the `to` array + var sameTime: [SameTimeItem] = [] + + // trying to find new split index for item looks difficult but allows to not use one more loop. + // The idea is to memorize how many splits were till any index (map number of splits until index) + // and use resulting itemIndex to decide new split index position. + // Because of the possibility to have many items with the same timestamp, it's possible to see `itemIndex < || == || > i`. + var splitsTillIndex: [Int] = [] + var splitsPerPrevIndex = 0 + + for i in 0 ..< to.count { + let item = to[i] + + splitsPerPrevIndex = splits.contains(item.id) ? splitsPerPrevIndex + 1 : splitsPerPrevIndex + splitsTillIndex.append(splitsPerPrevIndex) + + let itemIsNewer = (compareByTimeTs ? item.meta.itemTs > lastNew.meta.itemTs : item.meta.createdAt > lastNew.meta.createdAt) + if itemIsNewer || i + 1 == to.count { + if (compareByTimeTs ? lastNew.meta.itemTs == item.meta.itemTs : lastNew.meta.createdAt == item.meta.createdAt) { + sameTime.append((i, item)) + } + // time to stop the loop. Item is newer or it's the last item in `to` array, taking previous items and checking position inside them + let itemIndex: Int + if sameTime.count > 1, let first = sameTime.sorted(by: { prev, next in prev.item.meta.itemId < next.item.id }).first(where: { same in same.item.id > lastNew.id }) { + itemIndex = first.index + } else if sameTime.count == 1 { + itemIndex = sameTime[0].item.id > lastNew.id ? sameTime[0].index : sameTime[0].index + 1 + } else { + itemIndex = itemIsNewer ? i : i + 1 + } + let splitIndex = splitsTillIndex[min(itemIndex, splitsTillIndex.count - 1)] + let prevItemSplitIndex = itemIndex == 0 ? 0 : splitsTillIndex[min(itemIndex - 1, splitsTillIndex.count - 1)] + return (itemIndex, splitIndex == prevItemSplitIndex ? splitIndex : prevItemSplitIndex) + } + + if (compareByTimeTs ? lastNew.meta.itemTs == item.meta.itemTs : lastNew.meta.createdAt == item.meta.createdAt) { + sameTime.append(SameTimeItem(index: i, item: item)) + } else { + sameTime = [] + } + } + // shouldn't be here + return (to.count, splits.count) +} + +private func indexToInsertAroundTest() { + func assert(_ one: (Int, Int), _ two: (Int, Int)) { + if one != two { + logger.debug("\(String(describing: one)) != \(String(describing: two))") + fatalError() + } + } + + let itemsToInsert = [ChatItem.getSample(3, .groupSnd, Date.init(timeIntervalSince1970: 3), "")] + let items1 = [ + ChatItem.getSample(0, .groupSnd, Date.init(timeIntervalSince1970: 0), ""), + ChatItem.getSample(1, .groupSnd, Date.init(timeIntervalSince1970: 1), ""), + ChatItem.getSample(2, .groupSnd, Date.init(timeIntervalSince1970: 2), "") + ] + assert(indexToInsertAround(.group, itemsToInsert.last, to: items1, Set([1])), (3, 1)) + + let items2 = [ + ChatItem.getSample(0, .groupSnd, Date.init(timeIntervalSince1970: 0), ""), + ChatItem.getSample(1, .groupSnd, Date.init(timeIntervalSince1970: 1), ""), + ChatItem.getSample(2, .groupSnd, Date.init(timeIntervalSince1970: 3), "") + ] + assert(indexToInsertAround(.group, itemsToInsert.last, to: items2, Set([2])), (3, 1)) + + let items3 = [ + ChatItem.getSample(0, .groupSnd, Date.init(timeIntervalSince1970: 0), ""), + ChatItem.getSample(1, .groupSnd, Date.init(timeIntervalSince1970: 3), ""), + ChatItem.getSample(2, .groupSnd, Date.init(timeIntervalSince1970: 3), "") + ] + assert(indexToInsertAround(.group, itemsToInsert.last, to: items3, Set([1])), (3, 1)) + + let items4 = [ + ChatItem.getSample(0, .groupSnd, Date.init(timeIntervalSince1970: 0), ""), + ChatItem.getSample(4, .groupSnd, Date.init(timeIntervalSince1970: 3), ""), + ChatItem.getSample(5, .groupSnd, Date.init(timeIntervalSince1970: 3), "") + ] + assert(indexToInsertAround(.group, itemsToInsert.last, to: items4, Set([4])), (1, 0)) + + let items5 = [ + ChatItem.getSample(0, .groupSnd, Date.init(timeIntervalSince1970: 0), ""), + ChatItem.getSample(2, .groupSnd, Date.init(timeIntervalSince1970: 3), ""), + ChatItem.getSample(4, .groupSnd, Date.init(timeIntervalSince1970: 3), "") + ] + assert(indexToInsertAround(.group, itemsToInsert.last, to: items5, Set([2])), (2, 1)) + + let items6 = [ + ChatItem.getSample(4, .groupSnd, Date.init(timeIntervalSince1970: 4), ""), + ChatItem.getSample(5, .groupSnd, Date.init(timeIntervalSince1970: 4), ""), + ChatItem.getSample(6, .groupSnd, Date.init(timeIntervalSince1970: 4), "") + ] + assert(indexToInsertAround(.group, itemsToInsert.last, to: items6, Set([5])), (0, 0)) + + let items7 = [ + ChatItem.getSample(4, .groupSnd, Date.init(timeIntervalSince1970: 4), ""), + ChatItem.getSample(5, .groupSnd, Date.init(timeIntervalSince1970: 4), ""), + ChatItem.getSample(6, .groupSnd, Date.init(timeIntervalSince1970: 4), "") + ] + assert(indexToInsertAround(.group, nil, to: items7, Set([6])), (0, 0)) + + let items8 = [ + ChatItem.getSample(2, .groupSnd, Date.init(timeIntervalSince1970: 4), ""), + ChatItem.getSample(4, .groupSnd, Date.init(timeIntervalSince1970: 3), ""), + ChatItem.getSample(5, .groupSnd, Date.init(timeIntervalSince1970: 4), "") + ] + assert(indexToInsertAround(.group, itemsToInsert.last, to: items8, Set([2])), (0, 0)) + + let items9 = [ + ChatItem.getSample(2, .groupSnd, Date.init(timeIntervalSince1970: 3), ""), + ChatItem.getSample(4, .groupSnd, Date.init(timeIntervalSince1970: 3), ""), + ChatItem.getSample(5, .groupSnd, Date.init(timeIntervalSince1970: 4), "") + ] + assert(indexToInsertAround(.group, itemsToInsert.last, to: items9, Set([5])), (1, 0)) + + let items10 = [ + ChatItem.getSample(4, .groupSnd, Date.init(timeIntervalSince1970: 3), ""), + ChatItem.getSample(5, .groupSnd, Date.init(timeIntervalSince1970: 3), ""), + ChatItem.getSample(6, .groupSnd, Date.init(timeIntervalSince1970: 4), "") + ] + assert(indexToInsertAround(.group, itemsToInsert.last, to: items10, Set([4])), (0, 0)) + + let items11: [ChatItem] = [] + assert(indexToInsertAround(.group, itemsToInsert.last, to: items11, Set([])), (0, 0)) +} diff --git a/apps/ios/Shared/Views/Chat/ChatScrollHelpers.swift b/apps/ios/Shared/Views/Chat/ChatScrollHelpers.swift index 873f24d5c3..c1a1eec7d2 100644 --- a/apps/ios/Shared/Views/Chat/ChatScrollHelpers.swift +++ b/apps/ios/Shared/Views/Chat/ChatScrollHelpers.swift @@ -9,25 +9,23 @@ import SwiftUI import SimpleXChat -func loadLastItems(_ loadingMoreItems: Binding, loadingBottomItems: Binding, _ chat: Chat) { - if ItemsModel.shared.chatState.totalAfter == 0 { - return +func loadLastItems(_ loadingMoreItems: Binding, loadingBottomItems: Binding, _ chat: Chat) async { + await MainActor.run { + loadingMoreItems.wrappedValue = true + loadingBottomItems.wrappedValue = true } - loadingMoreItems.wrappedValue = true - loadingBottomItems.wrappedValue = true - Task { - try? await Task.sleep(nanoseconds: 500_000000) - if ChatModel.shared.chatId != chat.chatInfo.id { - await MainActor.run { - loadingMoreItems.wrappedValue = false - } - return - } - await apiLoadMessages(chat.chatInfo.id, ChatPagination.last(count: 50), ItemsModel.shared.chatState) + try? await Task.sleep(nanoseconds: 500_000000) + if ChatModel.shared.chatId != chat.chatInfo.id { await MainActor.run { loadingMoreItems.wrappedValue = false loadingBottomItems.wrappedValue = false } + return + } + await apiLoadMessages(chat.chatInfo.id, ChatPagination.last(count: 50), ItemsModel.shared.chatState) + await MainActor.run { + loadingMoreItems.wrappedValue = false + loadingBottomItems.wrappedValue = false } } @@ -36,6 +34,12 @@ class PreloadState { var prevFirstVisible: Int64 = Int64.min var prevItemsCount: Int = 0 var preloading: Bool = false + + func clear() { + prevFirstVisible = Int64.min + prevItemsCount = 0 + preloading = false + } } func preloadIfNeeded( @@ -43,26 +47,41 @@ func preloadIfNeeded( _ ignoreLoadingRequests: Binding, _ listState: EndlessScrollView.ListState, _ mergedItems: BoxedValue, - loadItems: @escaping (Bool, ChatPagination) async -> Bool + loadItems: @escaping (Bool, ChatPagination) async -> Bool, + loadLastItems: @escaping () async -> Void ) { let state = PreloadState.shared guard !listState.isScrolling && !listState.isAnimatedScrolling, - state.prevFirstVisible != listState.firstVisibleItemIndex || state.prevItemsCount != mergedItems.boxedValue.indexInParentItems.count, !state.preloading, listState.totalItemsCount > 0 else { return } - state.prevFirstVisible = listState.firstVisibleItemId as! Int64 - state.prevItemsCount = mergedItems.boxedValue.indexInParentItems.count - state.preloading = true - let allowLoadMore = allowLoadMoreItems.wrappedValue - Task { - defer { - state.preloading = false + if state.prevFirstVisible != listState.firstVisibleItemId as! Int64 || state.prevItemsCount != mergedItems.boxedValue.indexInParentItems.count { + state.preloading = true + let allowLoadMore = allowLoadMoreItems.wrappedValue + Task { + defer { state.preloading = false } + var triedToLoad = true + await preloadItems(mergedItems.boxedValue, allowLoadMore, listState, ignoreLoadingRequests) { pagination in + triedToLoad = await loadItems(false, pagination) + return triedToLoad + } + if triedToLoad { + state.prevFirstVisible = listState.firstVisibleItemId as! Int64 + state.prevItemsCount = mergedItems.boxedValue.indexInParentItems.count + } + // it's important to ask last items when the view is fully covered with items. Otherwise, visible items from one + // split will be merged with last items and position of scroll will change unexpectedly. + if listState.itemsCanCoverScreen && !ItemsModel.shared.lastItemsLoaded { + await loadLastItems() + } } - await preloadItems(mergedItems.boxedValue, allowLoadMore, listState, ignoreLoadingRequests) { pagination in - await loadItems(false, pagination) + } else if listState.itemsCanCoverScreen && !ItemsModel.shared.lastItemsLoaded { + state.preloading = true + Task { + defer { state.preloading = false } + await loadLastItems() } } } @@ -105,6 +124,7 @@ async { let triedToLoad = await loadItems(ChatPagination.before(chatItemId: loadFromItemId, count: ChatPagination.PRELOAD_COUNT)) if triedToLoad && sizeWas == ItemsModel.shared.reversedChatItems.count && firstItemIdWas == ItemsModel.shared.reversedChatItems.last?.id { ignoreLoadingRequests.wrappedValue = loadFromItemId + return false } return triedToLoad } diff --git a/apps/ios/Shared/Views/Chat/ChatView.swift b/apps/ios/Shared/Views/Chat/ChatView.swift index 1e4c16b036..693efcfbb5 100644 --- a/apps/ios/Shared/Views/Chat/ChatView.swift +++ b/apps/ios/Shared/Views/Chat/ChatView.swift @@ -91,7 +91,11 @@ struct ChatView: View { if let groupInfo = chat.chatInfo.groupInfo, !composeState.message.isEmpty { GroupMentionsView(groupInfo: groupInfo, composeState: $composeState, selectedRange: $selectedRange, keyboardVisible: $keyboardVisible) } - FloatingButtons(theme: theme, scrollView: scrollView, chat: chat, loadingTopItems: $loadingTopItems, requestedTopScroll: $requestedTopScroll, loadingBottomItems: $loadingBottomItems, requestedBottomScroll: $requestedBottomScroll, animatedScrollingInProgress: $animatedScrollingInProgress, listState: scrollView.listState, model: floatingButtonModel) + FloatingButtons(theme: theme, scrollView: scrollView, chat: chat, loadingMoreItems: $loadingMoreItems, loadingTopItems: $loadingTopItems, requestedTopScroll: $requestedTopScroll, loadingBottomItems: $loadingBottomItems, requestedBottomScroll: $requestedBottomScroll, animatedScrollingInProgress: $animatedScrollingInProgress, listState: scrollView.listState, model: floatingButtonModel, reloadItems: { + mergedItems.boxedValue = MergedItems.create(im.reversedChatItems, revealedItems, im.chatState) + scrollView.updateItems(mergedItems.boxedValue.items) + } + ) } connectingText() if selectedChatItems == nil { @@ -262,7 +266,6 @@ struct ChatView: View { // this may already being loading because of changed chat id (see .onChange(of: chat.id) if !loadingBottomItems { - loadLastItems($loadingMoreItems, loadingBottomItems: $loadingBottomItems, chat) allowLoadMoreItems = false DispatchQueue.main.asyncAfter(deadline: .now() + 0.5) { allowLoadMoreItems = true @@ -584,7 +587,6 @@ struct ChatView: View { scrollView.updateItems(mergedItems.boxedValue.items) } .onChange(of: chat.id) { _ in - loadLastItems($loadingMoreItems, loadingBottomItems: $loadingBottomItems, chat) allowLoadMoreItems = false DispatchQueue.main.asyncAfter(deadline: .now() + 0.5) { allowLoadMoreItems = true @@ -629,7 +631,6 @@ struct ChatView: View { if let unreadIndex { scrollView.scrollToItem(unreadIndex) } - loadLastItems($loadingMoreItems, loadingBottomItems: $loadingBottomItems, chat) DispatchQueue.main.asyncAfter(deadline: .now() + 0.5) { allowLoadMoreItems = true } @@ -647,10 +648,8 @@ struct ChatView: View { } else if let index = scrollView.listState.items.lastIndex(where: { $0.hasUnread() }) { // scroll to the top unread item scrollView.scrollToItem(index) - loadLastItems($loadingMoreItems, loadingBottomItems: $loadingBottomItems, chat) } else { scrollView.scrollToBottom() - loadLastItems($loadingMoreItems, loadingBottomItems: $loadingBottomItems, chat) } } } @@ -731,6 +730,7 @@ struct ChatView: View { let theme: AppTheme let scrollView: EndlessScrollView let chat: Chat + @Binding var loadingMoreItems: Bool @Binding var loadingTopItems: Bool @Binding var requestedTopScroll: Bool @Binding var loadingBottomItems: Bool @@ -738,6 +738,7 @@ struct ChatView: View { @Binding var animatedScrollingInProgress: Bool let listState: EndlessScrollView.ListState @ObservedObject var model: FloatingButtonModel + let reloadItems: () -> Void var body: some View { ZStack(alignment: .top) { @@ -795,7 +796,7 @@ struct ChatView: View { } } .onTapGesture { - if loadingBottomItems { + if loadingBottomItems || !ItemsModel.shared.lastItemsLoaded { requestedTopScroll = false requestedBottomScroll = true } else { @@ -815,7 +816,7 @@ struct ChatView: View { } } .onChange(of: loadingBottomItems) { loading in - if !loading && requestedBottomScroll { + if !loading && requestedBottomScroll && ItemsModel.shared.lastItemsLoaded { requestedBottomScroll = false scrollToBottom() } @@ -824,15 +825,25 @@ struct ChatView: View { } private func scrollToTopUnread() { - if let index = listState.items.lastIndex(where: { $0.hasUnread() }) { - animatedScrollingInProgress = true - // scroll to the top unread item - Task { + Task { + if !ItemsModel.shared.chatState.splits.isEmpty { + await MainActor.run { loadingMoreItems = true } + await loadChat(chatId: chat.id, openAroundItemId: nil, clearItems: false) + await MainActor.run { reloadItems() } + if let index = listState.items.lastIndex(where: { $0.hasUnread() }) { + await MainActor.run { animatedScrollingInProgress = true } + await scrollView.scrollToItemAnimated(index) + await MainActor.run { animatedScrollingInProgress = false } + } + await MainActor.run { loadingMoreItems = false } + } else if let index = listState.items.lastIndex(where: { $0.hasUnread() }) { + await MainActor.run { animatedScrollingInProgress = true } + // scroll to the top unread item await scrollView.scrollToItemAnimated(index) await MainActor.run { animatedScrollingInProgress = false } + } else { + logger.debug("No more unread items, total: \(listState.items.count)") } - } else { - logger.debug("No more unread items, total: \(listState.items.count)") } } @@ -1147,6 +1158,11 @@ struct ChatView: View { } else { await loadChatItems(chat, pagination) } + }, + loadLastItems: { + if !loadingMoreItems { + await loadLastItems($loadingMoreItems, loadingBottomItems: $loadingBottomItems, chat) + } } ) } @@ -1247,18 +1263,11 @@ struct ChatView: View { nil } let showAvatar = shouldShowAvatar(item, listItem.nextItem) - let itemSeparation: ItemSeparation let single = switch merged { case .single: true default: false } - if single || revealed { - let prev = listItem.prevItem - itemSeparation = getItemSeparation(item, prev) - let nextForGap = (item.mergeCategory != nil && item.mergeCategory == prev?.mergeCategory) || isLastItem ? nil : listItem.nextItem - } else { - itemSeparation = getItemSeparation(item, nil) - } + let itemSeparation = getItemSeparation(item, single || revealed ? listItem.prevItem: nil) return VStack(spacing: 0) { if let last { DateSeparator(date: last.meta.itemTs).padding(8) diff --git a/apps/ios/Shared/Views/Chat/EndlessScrollView.swift b/apps/ios/Shared/Views/Chat/EndlessScrollView.swift index 9756489628..cc61754b26 100644 --- a/apps/ios/Shared/Views/Chat/EndlessScrollView.swift +++ b/apps/ios/Shared/Views/Chat/EndlessScrollView.swift @@ -171,6 +171,9 @@ class EndlessScrollView: UIScrollView, UIScrollViewDelegate, UIGestu visibleItems.last?.index ?? 0 } + /// Specifies if visible items cover the whole screen or can cover it (if overscrolled) + var itemsCanCoverScreen: Bool = false + /// Whether there is a non-animated scroll to item in progress or not var isScrolling: Bool = false /// Whether there is an animated scroll to item in progress or not @@ -284,7 +287,8 @@ class EndlessScrollView: UIScrollView, UIScrollViewDelegate, UIGestu func updateItems(_ items: [ScrollItem], _ forceReloadVisible: Bool = false) { if !Thread.isMainThread { - fatalError("Use main thread to update items") + logger.error("Use main thread to update items") + return } if bounds.height == 0 { self.listState.items = items @@ -302,6 +306,7 @@ class EndlessScrollView: UIScrollView, UIScrollViewDelegate, UIGestu if items.isEmpty { listState.visibleItems.forEach { item in item.view.removeFromSuperview() } listState.visibleItems = [] + listState.itemsCanCoverScreen = false listState.firstVisibleItemId = EndlessScrollView.DEFAULT_ITEM_ID listState.firstVisibleItemIndex = 0 listState.firstVisibleItemOffset = -insetTop @@ -322,6 +327,7 @@ class EndlessScrollView: UIScrollView, UIScrollViewDelegate, UIGestu var oldVisible = listState.visibleItems var newVisible: [VisibleItem] = [] + var visibleItemsHeight: CGFloat = 0 let offsetsDiff = contentOffsetY - prevProcessedOffset var shouldBeFirstVisible = items.firstIndex(where: { item in item.id == listState.firstVisibleItemId as! ScrollItem.ID }) ?? 0 @@ -339,7 +345,11 @@ class EndlessScrollView: UIScrollView, UIScrollViewDelegate, UIGestu if let visibleIndex { let v = oldVisible.remove(at: visibleIndex) if forceReloadVisible || v.view.bounds.width != bounds.width || v.item.hashValue != item.hashValue { + let wasHeight = v.view.bounds.height updateCell(v.view, i, items) + if wasHeight < v.view.bounds.height && i == 0 && shouldBeFirstVisible == i { + v.view.frame.origin.y -= v.view.bounds.height - wasHeight + } } visible = v } else { @@ -389,6 +399,7 @@ class EndlessScrollView: UIScrollView, UIScrollViewDelegate, UIGestu addSubview(vis.view) } newVisible.append(vis) + visibleItemsHeight += vis.view.frame.height nextOffsetY = vis.view.frame.origin.y } else { let vis: VisibleItem @@ -406,6 +417,7 @@ class EndlessScrollView: UIScrollView, UIScrollViewDelegate, UIGestu addSubview(vis.view) } newVisible.append(vis) + visibleItemsHeight += vis.view.frame.height } if abs(nextOffsetY) < contentOffsetY && !allowOneMore { break @@ -435,6 +447,7 @@ class EndlessScrollView: UIScrollView, UIScrollViewDelegate, UIGestu } offset += vis.view.frame.height newVisible.insert(vis, at: 0) + visibleItemsHeight += vis.view.frame.height if offset >= contentOffsetY + bounds.height { break } @@ -450,11 +463,15 @@ class EndlessScrollView: UIScrollView, UIScrollViewDelegate, UIGestu prevProcessedOffset = contentOffsetY listState.visibleItems = newVisible - listState.items = items + // bottom drawing starts from 0 until top visible area at least (bound.height - insetTop) or above top bar (bounds.height). + // For visible items to preserve offset after adding more items having such height is enough + listState.itemsCanCoverScreen = visibleItemsHeight >= bounds.height - insetTop listState.firstVisibleItemId = listState.visibleItems.first?.item.id ?? EndlessScrollView.DEFAULT_ITEM_ID listState.firstVisibleItemIndex = listState.visibleItems.first?.index ?? 0 listState.firstVisibleItemOffset = listState.visibleItems.first?.offset ?? -insetTop + // updating the items with the last step in order to call listener with fully updated state + listState.items = items estimatedContentHeight.update(contentOffset, listState, averageItemHeight, itemsCountChanged) scrollBarView.contentSize = CGSizeMake(bounds.width, estimatedContentHeight.virtualOverscrolledHeight) diff --git a/apps/ios/SimpleX.xcodeproj/project.pbxproj b/apps/ios/SimpleX.xcodeproj/project.pbxproj index efc2bad450..31c8ab0b47 100644 --- a/apps/ios/SimpleX.xcodeproj/project.pbxproj +++ b/apps/ios/SimpleX.xcodeproj/project.pbxproj @@ -179,8 +179,8 @@ 64C3B0212A0D359700E19930 /* CustomTimePicker.swift in Sources */ = {isa = PBXBuildFile; fileRef = 64C3B0202A0D359700E19930 /* CustomTimePicker.swift */; }; 64C8299D2D54AEEE006B9E89 /* libgmp.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 64C829982D54AEED006B9E89 /* libgmp.a */; }; 64C8299E2D54AEEE006B9E89 /* libffi.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 64C829992D54AEEE006B9E89 /* libffi.a */; }; - 64C8299F2D54AEEE006B9E89 /* libHSsimplex-chat-6.3.0.8-3RDt4h0Fq4hJV00CU7V8H-ghc9.6.3.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 64C8299A2D54AEEE006B9E89 /* libHSsimplex-chat-6.3.0.8-3RDt4h0Fq4hJV00CU7V8H-ghc9.6.3.a */; }; - 64C829A02D54AEEE006B9E89 /* libHSsimplex-chat-6.3.0.8-3RDt4h0Fq4hJV00CU7V8H.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 64C8299B2D54AEEE006B9E89 /* libHSsimplex-chat-6.3.0.8-3RDt4h0Fq4hJV00CU7V8H.a */; }; + 64C8299F2D54AEEE006B9E89 /* libHSsimplex-chat-6.3.0.8-CNSVk2Y5c4gAUnxeodOxfm-ghc9.6.3.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 64C8299A2D54AEEE006B9E89 /* libHSsimplex-chat-6.3.0.8-CNSVk2Y5c4gAUnxeodOxfm-ghc9.6.3.a */; }; + 64C829A02D54AEEE006B9E89 /* libHSsimplex-chat-6.3.0.8-CNSVk2Y5c4gAUnxeodOxfm.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 64C8299B2D54AEEE006B9E89 /* libHSsimplex-chat-6.3.0.8-CNSVk2Y5c4gAUnxeodOxfm.a */; }; 64C829A12D54AEEE006B9E89 /* libgmpxx.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 64C8299C2D54AEEE006B9E89 /* libgmpxx.a */; }; 64D0C2C029F9688300B38D5F /* UserAddressView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 64D0C2BF29F9688300B38D5F /* UserAddressView.swift */; }; 64D0C2C229FA57AB00B38D5F /* UserAddressLearnMore.swift in Sources */ = {isa = PBXBuildFile; fileRef = 64D0C2C129FA57AB00B38D5F /* UserAddressLearnMore.swift */; }; @@ -541,8 +541,8 @@ 64C3B0202A0D359700E19930 /* CustomTimePicker.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CustomTimePicker.swift; sourceTree = ""; }; 64C829982D54AEED006B9E89 /* libgmp.a */ = {isa = PBXFileReference; lastKnownFileType = archive.ar; path = libgmp.a; sourceTree = ""; }; 64C829992D54AEEE006B9E89 /* libffi.a */ = {isa = PBXFileReference; lastKnownFileType = archive.ar; path = libffi.a; sourceTree = ""; }; - 64C8299A2D54AEEE006B9E89 /* libHSsimplex-chat-6.3.0.8-3RDt4h0Fq4hJV00CU7V8H-ghc9.6.3.a */ = {isa = PBXFileReference; lastKnownFileType = archive.ar; path = "libHSsimplex-chat-6.3.0.8-3RDt4h0Fq4hJV00CU7V8H-ghc9.6.3.a"; sourceTree = ""; }; - 64C8299B2D54AEEE006B9E89 /* libHSsimplex-chat-6.3.0.8-3RDt4h0Fq4hJV00CU7V8H.a */ = {isa = PBXFileReference; lastKnownFileType = archive.ar; path = "libHSsimplex-chat-6.3.0.8-3RDt4h0Fq4hJV00CU7V8H.a"; sourceTree = ""; }; + 64C8299A2D54AEEE006B9E89 /* libHSsimplex-chat-6.3.0.8-CNSVk2Y5c4gAUnxeodOxfm-ghc9.6.3.a */ = {isa = PBXFileReference; lastKnownFileType = archive.ar; path = "libHSsimplex-chat-6.3.0.8-CNSVk2Y5c4gAUnxeodOxfm-ghc9.6.3.a"; sourceTree = ""; }; + 64C8299B2D54AEEE006B9E89 /* libHSsimplex-chat-6.3.0.8-CNSVk2Y5c4gAUnxeodOxfm.a */ = {isa = PBXFileReference; lastKnownFileType = archive.ar; path = "libHSsimplex-chat-6.3.0.8-CNSVk2Y5c4gAUnxeodOxfm.a"; sourceTree = ""; }; 64C8299C2D54AEEE006B9E89 /* libgmpxx.a */ = {isa = PBXFileReference; lastKnownFileType = archive.ar; path = libgmpxx.a; sourceTree = ""; }; 64D0C2BF29F9688300B38D5F /* UserAddressView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = UserAddressView.swift; sourceTree = ""; }; 64D0C2C129FA57AB00B38D5F /* UserAddressLearnMore.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = UserAddressLearnMore.swift; sourceTree = ""; }; @@ -698,8 +698,8 @@ 64C8299D2D54AEEE006B9E89 /* libgmp.a in Frameworks */, 64C8299E2D54AEEE006B9E89 /* libffi.a in Frameworks */, 64C829A12D54AEEE006B9E89 /* libgmpxx.a in Frameworks */, - 64C8299F2D54AEEE006B9E89 /* libHSsimplex-chat-6.3.0.8-3RDt4h0Fq4hJV00CU7V8H-ghc9.6.3.a in Frameworks */, - 64C829A02D54AEEE006B9E89 /* libHSsimplex-chat-6.3.0.8-3RDt4h0Fq4hJV00CU7V8H.a in Frameworks */, + 64C8299F2D54AEEE006B9E89 /* libHSsimplex-chat-6.3.0.8-CNSVk2Y5c4gAUnxeodOxfm-ghc9.6.3.a in Frameworks */, + 64C829A02D54AEEE006B9E89 /* libHSsimplex-chat-6.3.0.8-CNSVk2Y5c4gAUnxeodOxfm.a in Frameworks */, CE38A29C2C3FCD72005ED185 /* SwiftyGif in Frameworks */, ); runOnlyForDeploymentPostprocessing = 0; @@ -784,8 +784,8 @@ 64C829992D54AEEE006B9E89 /* libffi.a */, 64C829982D54AEED006B9E89 /* libgmp.a */, 64C8299C2D54AEEE006B9E89 /* libgmpxx.a */, - 64C8299A2D54AEEE006B9E89 /* libHSsimplex-chat-6.3.0.8-3RDt4h0Fq4hJV00CU7V8H-ghc9.6.3.a */, - 64C8299B2D54AEEE006B9E89 /* libHSsimplex-chat-6.3.0.8-3RDt4h0Fq4hJV00CU7V8H.a */, + 64C8299A2D54AEEE006B9E89 /* libHSsimplex-chat-6.3.0.8-CNSVk2Y5c4gAUnxeodOxfm-ghc9.6.3.a */, + 64C8299B2D54AEEE006B9E89 /* libHSsimplex-chat-6.3.0.8-CNSVk2Y5c4gAUnxeodOxfm.a */, ); path = Libraries; sourceTree = ""; @@ -1973,7 +1973,7 @@ CLANG_TIDY_MISC_REDUNDANT_EXPRESSION = YES; CODE_SIGN_ENTITLEMENTS = "SimpleX (iOS).entitlements"; CODE_SIGN_STYLE = Automatic; - CURRENT_PROJECT_VERSION = 268; + CURRENT_PROJECT_VERSION = 269; DEAD_CODE_STRIPPING = YES; DEVELOPMENT_TEAM = 5NN7GUYB6T; ENABLE_BITCODE = NO; @@ -1999,6 +1999,7 @@ ); LLVM_LTO = YES_THIN; MARKETING_VERSION = 6.3; + OTHER_LDFLAGS = "-Wl,-stack_size,0x1000000"; PRODUCT_BUNDLE_IDENTIFIER = chat.simplex.app; PRODUCT_NAME = SimpleX; SDKROOT = iphoneos; @@ -2022,7 +2023,7 @@ CLANG_TIDY_MISC_REDUNDANT_EXPRESSION = YES; CODE_SIGN_ENTITLEMENTS = "SimpleX (iOS).entitlements"; CODE_SIGN_STYLE = Automatic; - CURRENT_PROJECT_VERSION = 268; + CURRENT_PROJECT_VERSION = 269; DEAD_CODE_STRIPPING = YES; DEVELOPMENT_TEAM = 5NN7GUYB6T; ENABLE_BITCODE = NO; @@ -2048,6 +2049,7 @@ ); LLVM_LTO = YES; MARKETING_VERSION = 6.3; + OTHER_LDFLAGS = "-Wl,-stack_size,0x1000000"; PRODUCT_BUNDLE_IDENTIFIER = chat.simplex.app; PRODUCT_NAME = SimpleX; SDKROOT = iphoneos; @@ -2063,7 +2065,7 @@ buildSettings = { ALWAYS_EMBED_SWIFT_STANDARD_LIBRARIES = YES; CODE_SIGN_STYLE = Automatic; - CURRENT_PROJECT_VERSION = 268; + CURRENT_PROJECT_VERSION = 269; DEVELOPMENT_TEAM = 5NN7GUYB6T; GENERATE_INFOPLIST_FILE = YES; IPHONEOS_DEPLOYMENT_TARGET = 15.0; @@ -2083,7 +2085,7 @@ buildSettings = { ALWAYS_EMBED_SWIFT_STANDARD_LIBRARIES = YES; CODE_SIGN_STYLE = Automatic; - CURRENT_PROJECT_VERSION = 268; + CURRENT_PROJECT_VERSION = 269; DEVELOPMENT_TEAM = 5NN7GUYB6T; GENERATE_INFOPLIST_FILE = YES; IPHONEOS_DEPLOYMENT_TARGET = 15.0; @@ -2108,7 +2110,7 @@ CODE_SIGN_ENTITLEMENTS = "SimpleX NSE/SimpleX NSE.entitlements"; CODE_SIGN_IDENTITY = "Apple Development"; CODE_SIGN_STYLE = Automatic; - CURRENT_PROJECT_VERSION = 268; + CURRENT_PROJECT_VERSION = 269; DEVELOPMENT_TEAM = 5NN7GUYB6T; ENABLE_BITCODE = NO; GCC_OPTIMIZATION_LEVEL = s; @@ -2145,7 +2147,7 @@ CODE_SIGN_ENTITLEMENTS = "SimpleX NSE/SimpleX NSE.entitlements"; CODE_SIGN_IDENTITY = "Apple Development"; CODE_SIGN_STYLE = Automatic; - CURRENT_PROJECT_VERSION = 268; + CURRENT_PROJECT_VERSION = 269; DEVELOPMENT_TEAM = 5NN7GUYB6T; ENABLE_BITCODE = NO; ENABLE_CODE_COVERAGE = NO; @@ -2182,7 +2184,7 @@ CLANG_TIDY_BUGPRONE_REDUNDANT_BRANCH_CONDITION = YES; CLANG_TIDY_MISC_REDUNDANT_EXPRESSION = YES; CODE_SIGN_STYLE = Automatic; - CURRENT_PROJECT_VERSION = 268; + CURRENT_PROJECT_VERSION = 269; DEFINES_MODULE = YES; DEVELOPMENT_TEAM = 5NN7GUYB6T; DYLIB_COMPATIBILITY_VERSION = 1; @@ -2233,7 +2235,7 @@ CLANG_TIDY_BUGPRONE_REDUNDANT_BRANCH_CONDITION = YES; CLANG_TIDY_MISC_REDUNDANT_EXPRESSION = YES; CODE_SIGN_STYLE = Automatic; - CURRENT_PROJECT_VERSION = 268; + CURRENT_PROJECT_VERSION = 269; DEFINES_MODULE = YES; DEVELOPMENT_TEAM = 5NN7GUYB6T; DYLIB_COMPATIBILITY_VERSION = 1; @@ -2284,7 +2286,7 @@ CLANG_CXX_LANGUAGE_STANDARD = "gnu++20"; CODE_SIGN_ENTITLEMENTS = "SimpleX SE/SimpleX SE.entitlements"; CODE_SIGN_STYLE = Automatic; - CURRENT_PROJECT_VERSION = 268; + CURRENT_PROJECT_VERSION = 269; DEVELOPMENT_TEAM = 5NN7GUYB6T; ENABLE_USER_SCRIPT_SANDBOXING = YES; GCC_C_LANGUAGE_STANDARD = gnu17; @@ -2318,7 +2320,7 @@ CLANG_CXX_LANGUAGE_STANDARD = "gnu++20"; CODE_SIGN_ENTITLEMENTS = "SimpleX SE/SimpleX SE.entitlements"; CODE_SIGN_STYLE = Automatic; - CURRENT_PROJECT_VERSION = 268; + CURRENT_PROJECT_VERSION = 269; DEVELOPMENT_TEAM = 5NN7GUYB6T; ENABLE_USER_SCRIPT_SANDBOXING = YES; GCC_C_LANGUAGE_STANDARD = gnu17; diff --git a/apps/ios/SimpleXChat/API.swift b/apps/ios/SimpleXChat/API.swift index 5e5f047611..e439cd337b 100644 --- a/apps/ios/SimpleXChat/API.swift +++ b/apps/ios/SimpleXChat/API.swift @@ -183,7 +183,9 @@ public func chatResponse(_ s: String) -> ChatResponse { // let p = UnsafeMutableRawPointer.init(mutating: UnsafeRawPointer(cjson)) // let d = Data.init(bytesNoCopy: p, count: strlen(cjson), deallocator: .free) do { - let r = try jsonDecoder.decode(APIResponse.self, from: d) + let r = try callWithLargeStack { + try jsonDecoder.decode(APIResponse.self, from: d) + } return r.resp } catch { logger.error("chatResponse jsonDecoder.decode error: \(error.localizedDescription)") @@ -231,6 +233,32 @@ public func chatResponse(_ s: String) -> ChatResponse { return ChatResponse.response(type: type ?? "invalid", json: json ?? s) } +private let largeStackSize: Int = 2 * 1024 * 1024 + +private func callWithLargeStack(_ f: @escaping () throws -> T) throws -> T { + let semaphore = DispatchSemaphore(value: 0) + var result: Result? + let thread = Thread { + do { + result = .success(try f()) + } catch { + result = .failure(error) + } + semaphore.signal() + } + + thread.stackSize = largeStackSize + thread.qualityOfService = Thread.current.qualityOfService + thread.start() + + semaphore.wait() + + switch result! { + case let .success(r): return r + case let .failure(e): throw e + } +} + private func decodeUser_(_ jDict: NSDictionary) -> UserRef? { if let user_ = jDict["user_"] { try? decodeObject(user_ as Any) diff --git a/apps/multiplatform/common/src/commonMain/kotlin/chat/simplex/common/platform/Core.kt b/apps/multiplatform/common/src/commonMain/kotlin/chat/simplex/common/platform/Core.kt index 276d3e3909..a9f2dcaffc 100644 --- a/apps/multiplatform/common/src/commonMain/kotlin/chat/simplex/common/platform/Core.kt +++ b/apps/multiplatform/common/src/commonMain/kotlin/chat/simplex/common/platform/Core.kt @@ -79,7 +79,11 @@ suspend fun initChatController(useKey: String? = null, confirmMigrations: Migrat } if (rerunMigration) { chatModel.dbMigrationInProgress.value = true - migrated = chatMigrateInit(dbAbsolutePrefixPath, dbKey, confirm.value) + migrated = if (databaseBackend == "postgres") { + chatMigrateInit("simplex_v1", "postgresql://simplex@/simplex_v1", confirm.value) + } else { + chatMigrateInit(dbAbsolutePrefixPath, dbKey, confirm.value) + } res = runCatching { json.decodeFromString(migrated[0] as String) }.getOrElse { DBMigrationResult.Unknown(migrated[0] as String) } diff --git a/apps/multiplatform/common/src/commonMain/kotlin/chat/simplex/common/views/chat/ChatItemsLoader.kt b/apps/multiplatform/common/src/commonMain/kotlin/chat/simplex/common/views/chat/ChatItemsLoader.kt index 9050804db4..385bf42397 100644 --- a/apps/multiplatform/common/src/commonMain/kotlin/chat/simplex/common/views/chat/ChatItemsLoader.kt +++ b/apps/multiplatform/common/src/commonMain/kotlin/chat/simplex/common/views/chat/ChatItemsLoader.kt @@ -6,6 +6,7 @@ import chat.simplex.common.model.ChatModel.withChats import chat.simplex.common.platform.chatModel import kotlinx.coroutines.* import kotlinx.coroutines.flow.StateFlow +import kotlinx.datetime.Instant import kotlin.math.min const val TRIM_KEEP_COUNT = 200 @@ -122,18 +123,20 @@ suspend fun processLoadedChat( } } is ChatPagination.Around -> { - val newSplits = if (openAroundItemId == null) { + val newSplits: ArrayList = if (openAroundItemId == null) { newItems.addAll(oldItems) - removeDuplicatesAndUpperSplits(newItems, chat, splits, visibleItemIndexesNonReversed) + ArrayList(removeDuplicatesAndUpperSplits(newItems, chat, splits, visibleItemIndexesNonReversed)) } else { - emptyList() + arrayListOf() } - // currently, items will always be added on top, which is index 0 - newItems.addAll(0, chat.chatItems) + val (itemIndex, splitIndex) = indexToInsertAround(chat.chatInfo.chatType, chat.chatItems.lastOrNull(), to = newItems, newSplits.toSet()) + //indexToInsertAroundTest() + newItems.addAll(itemIndex, chat.chatItems) + newSplits.add(splitIndex, chat.chatItems.last().id) withChats(contentTag) { chatItems.replaceAll(newItems) - splits.value = listOf(chat.chatItems.last().id) + newSplits + splits.value = newSplits unreadAfterItemId.value = chat.chatItems.last().id totalAfter.value = navInfo.afterTotal unreadTotal.value = chat.chatStats.unreadCount @@ -151,10 +154,12 @@ suspend fun processLoadedChat( } is ChatPagination.Last -> { newItems.addAll(oldItems) + val newSplits = removeDuplicatesAndUnusedSplits(newItems, chat, chatState.splits.value) removeDuplicates(newItems, chat) newItems.addAll(chat.chatItems) withChats(contentTag) { chatItems.replaceAll(newItems) + chatState.splits.value = newSplits unreadAfterNewestLoaded.value = 0 } } @@ -240,7 +245,15 @@ private fun removeDuplicatesAndModifySplitsOnAfterPagination( val indexInSplitRanges = splits.value.indexOf(paginationChatItemId) // Currently, it should always load from split range val loadingFromSplitRange = indexInSplitRanges != -1 - val splitsToMerge = if (loadingFromSplitRange && indexInSplitRanges + 1 <= splits.value.size) ArrayList(splits.value.subList(indexInSplitRanges + 1, splits.value.size)) else ArrayList() + val topSplits: List + val splitsToMerge: ArrayList + if (loadingFromSplitRange && indexInSplitRanges + 1 <= splits.value.size) { + splitsToMerge = ArrayList(splits.value.subList(indexInSplitRanges + 1, splits.value.size)) + topSplits = splits.value.take(indexInSplitRanges + 1) + } else { + splitsToMerge = ArrayList() + topSplits = emptyList() + } newItems.removeAll { val duplicate = newIds.contains(it.id) if (loadingFromSplitRange && duplicate) { @@ -259,8 +272,8 @@ private fun removeDuplicatesAndModifySplitsOnAfterPagination( } var newSplits: List = emptyList() if (firstItemIdBelowAllSplits != null) { - // no splits anymore, all were merged with bottom items - newSplits = emptyList() + // no splits below anymore, all were merged with bottom items + newSplits = topSplits } else { if (splitsToRemove.isNotEmpty()) { val new = ArrayList(splits.value) @@ -323,6 +336,31 @@ private fun removeDuplicatesAndUpperSplits( return newSplits } +private fun removeDuplicatesAndUnusedSplits( + newItems: SnapshotStateList, + chat: Chat, + splits: List +): List { + if (splits.isEmpty()) { + removeDuplicates(newItems, chat) + return splits + } + + val newSplits = splits.toMutableList() + val (newIds, _) = mapItemsToIds(chat.chatItems) + newItems.removeAll { + val duplicate = newIds.contains(it.id) + if (duplicate) { + val firstIndex = newSplits.indexOf(it.id) + if (firstIndex != -1) { + newSplits.removeAt(firstIndex) + } + } + duplicate + } + return newSplits +} + // ids, number of unread items private fun mapItemsToIds(items: List): Pair, Int> { var unreadInLoaded = 0 @@ -343,3 +381,141 @@ private fun removeDuplicates(newItems: SnapshotStateList, chat: Chat) val (newIds, _) = mapItemsToIds(chat.chatItems) newItems.removeAll { newIds.contains(it.id) } } + +private data class SameTimeItem(val index: Int, val item: ChatItem) + +// return (item index, split index) +private fun indexToInsertAround(chatType: ChatType, lastNew: ChatItem?, to: List, splits: Set): Pair { + if (to.size <= 0 || lastNew == null) { + return 0 to 0 + } + // group sorting: item_ts, item_id + // everything else: created_at, item_id + val compareByTimeTs = chatType == ChatType.Group + // in case several items have the same time as another item in the `to` array + var sameTime: ArrayList = arrayListOf() + + // trying to find new split index for item looks difficult but allows to not use one more loop. + // The idea is to memorize how many splits were till any index (map number of splits until index) + // and use resulting itemIndex to decide new split index position. + // Because of the possibility to have many items with the same timestamp, it's possible to see `itemIndex < || == || > i`. + val splitsTillIndex: ArrayList = arrayListOf() + var splitsPerPrevIndex = 0 + + for (i in to.indices) { + val item = to[i] + + splitsPerPrevIndex = if (splits.contains(item.id)) splitsPerPrevIndex + 1 else splitsPerPrevIndex + splitsTillIndex.add(splitsPerPrevIndex) + val itemIsNewer = (if (compareByTimeTs) item.meta.itemTs > lastNew.meta.itemTs else item.meta.createdAt > lastNew.meta.createdAt) + if (itemIsNewer || i + 1 == to.size) { + val same = if (compareByTimeTs) lastNew.meta.itemTs == item.meta.itemTs else lastNew.meta.createdAt == item.meta.createdAt + if (same) { + sameTime.add(SameTimeItem(i, item)) + } + // time to stop the loop. Item is newer, or it's the last item in `to` array, taking previous items and checking position inside them + val itemIndex: Int + val first = if (sameTime.size > 1) sameTime.sortedWith { prev, next -> prev.item.meta.itemId.compareTo(next.item.id) }.firstOrNull { same -> same.item.id > lastNew.id } else null + if (sameTime.size > 1 && first != null) { + itemIndex = first.index + } else if (sameTime.size == 1) { + itemIndex = if (sameTime[0].item.id > lastNew.id) sameTime[0].index else sameTime[0].index + 1 + } else { + itemIndex = if (itemIsNewer) i else i + 1 + } + val splitIndex = splitsTillIndex[min(itemIndex, splitsTillIndex.size - 1)] + val prevItemSplitIndex = if (itemIndex == 0) 0 else splitsTillIndex[min(itemIndex - 1, splitsTillIndex.size - 1)] + return Pair(itemIndex, if (splitIndex == prevItemSplitIndex) splitIndex else prevItemSplitIndex) + } + val same = if (compareByTimeTs) lastNew.meta.itemTs == item.meta.itemTs else lastNew.meta.createdAt == item.meta.createdAt + if (same) { + sameTime.add(SameTimeItem(index = i, item = item)) + } else { + sameTime = arrayListOf() + } + } + // shouldn't be here + return Pair(to.size, splits.size) +} + +private fun indexToInsertAroundTest() { + fun assert(one: Pair, two: Pair) { + if (one != two) { + throw Exception("$one != $two") + } + } + + val itemsToInsert = listOf(ChatItem.getSampleData(3, CIDirection.GroupSnd(), Instant.fromEpochMilliseconds( 3), "")) + val items1 = listOf( + ChatItem.getSampleData(0, CIDirection.GroupSnd(), Instant.fromEpochMilliseconds( 0), ""), + ChatItem.getSampleData(1, CIDirection.GroupSnd(), Instant.fromEpochMilliseconds( 1), ""), + ChatItem.getSampleData(2, CIDirection.GroupSnd(), Instant.fromEpochMilliseconds( 2), "") + ) + assert(indexToInsertAround(ChatType.Group, itemsToInsert.lastOrNull(), to = items1, setOf(1)), Pair(3, 1)) + + val items2 = listOf( + ChatItem.getSampleData(0, CIDirection.GroupSnd(), Instant.fromEpochMilliseconds(0), ""), + ChatItem.getSampleData(1, CIDirection.GroupSnd(), Instant.fromEpochMilliseconds(1), ""), + ChatItem.getSampleData(2, CIDirection.GroupSnd(), Instant.fromEpochMilliseconds(3), "") + ) + assert(indexToInsertAround(ChatType.Group, itemsToInsert.lastOrNull(), to = items2, setOf(2)), Pair(3, 1)) + + val items3 = listOf( + ChatItem.getSampleData(0, CIDirection.GroupSnd(), Instant.fromEpochMilliseconds(0), ""), + ChatItem.getSampleData(1, CIDirection.GroupSnd(), Instant.fromEpochMilliseconds(3), ""), + ChatItem.getSampleData(2, CIDirection.GroupSnd(), Instant.fromEpochMilliseconds(3), "") + ) + assert(indexToInsertAround(ChatType.Group, itemsToInsert.lastOrNull(), to = items3, setOf(1)), Pair(3, 1)) + + val items4 = listOf( + ChatItem.getSampleData(0, CIDirection.GroupSnd(), Instant.fromEpochMilliseconds(0), ""), + ChatItem.getSampleData(4, CIDirection.GroupSnd(), Instant.fromEpochMilliseconds(3), ""), + ChatItem.getSampleData(5, CIDirection.GroupSnd(), Instant.fromEpochMilliseconds(3), "") + ) + assert(indexToInsertAround(ChatType.Group, itemsToInsert.lastOrNull(), to = items4, setOf(4)), Pair(1, 0)) + + val items5 = listOf( + ChatItem.getSampleData(0, CIDirection.GroupSnd(), Instant.fromEpochMilliseconds(0), ""), + ChatItem.getSampleData(2, CIDirection.GroupSnd(), Instant.fromEpochMilliseconds(3), ""), + ChatItem.getSampleData(4, CIDirection.GroupSnd(), Instant.fromEpochMilliseconds(3), "") + ) + assert(indexToInsertAround(ChatType.Group, itemsToInsert.lastOrNull(), to = items5, setOf(2)), Pair(2, 1)) + + val items6 = listOf( + ChatItem.getSampleData(4, CIDirection.GroupSnd(), Instant.fromEpochMilliseconds(4), ""), + ChatItem.getSampleData(5, CIDirection.GroupSnd(), Instant.fromEpochMilliseconds(4), ""), + ChatItem.getSampleData(6, CIDirection.GroupSnd(), Instant.fromEpochMilliseconds(4), "") + ) + assert(indexToInsertAround(ChatType.Group, itemsToInsert.lastOrNull(), to = items6, setOf(5)), Pair(0, 0)) + + val items7 = listOf( + ChatItem.getSampleData(4, CIDirection.GroupSnd(), Instant.fromEpochMilliseconds(4), ""), + ChatItem.getSampleData(5, CIDirection.GroupSnd(), Instant.fromEpochMilliseconds(4), ""), + ChatItem.getSampleData(6, CIDirection.GroupSnd(), Instant.fromEpochMilliseconds(4), "") + ) + assert(indexToInsertAround(ChatType.Group, null, to = items7, setOf(6)), Pair(0, 0)) + + val items8 = listOf( + ChatItem.getSampleData(2, CIDirection.GroupSnd(), Instant.fromEpochMilliseconds(4), ""), + ChatItem.getSampleData(4, CIDirection.GroupSnd(), Instant.fromEpochMilliseconds(3), ""), + ChatItem.getSampleData(5, CIDirection.GroupSnd(), Instant.fromEpochMilliseconds(4), "") + ) + assert(indexToInsertAround(ChatType.Group, itemsToInsert.lastOrNull(), to = items8, setOf(2)), Pair(0, 0)) + + val items9 = listOf( + ChatItem.getSampleData(2, CIDirection.GroupSnd(), Instant.fromEpochMilliseconds(3), ""), + ChatItem.getSampleData(4, CIDirection.GroupSnd(), Instant.fromEpochMilliseconds(3), ""), + ChatItem.getSampleData(5, CIDirection.GroupSnd(), Instant.fromEpochMilliseconds(4), "") + ) + assert(indexToInsertAround(ChatType.Group, itemsToInsert.lastOrNull(), to = items9, setOf(5)), Pair(1, 0)) + + val items10 = listOf( + ChatItem.getSampleData(4, CIDirection.GroupSnd(), Instant.fromEpochMilliseconds(3), ""), + ChatItem.getSampleData(5, CIDirection.GroupSnd(), Instant.fromEpochMilliseconds(3), ""), + ChatItem.getSampleData(6, CIDirection.GroupSnd(), Instant.fromEpochMilliseconds(4), "") + ) + assert(indexToInsertAround(ChatType.Group, itemsToInsert.lastOrNull(), to = items10, setOf(4)), Pair(0, 0)) + + val items11: List = listOf() + assert(indexToInsertAround(ChatType.Group, itemsToInsert.lastOrNull(), to = items11, emptySet()), Pair(0, 0)) +} diff --git a/apps/multiplatform/common/src/commonMain/kotlin/chat/simplex/common/views/chat/ChatView.kt b/apps/multiplatform/common/src/commonMain/kotlin/chat/simplex/common/views/chat/ChatView.kt index 1c3f4b7749..94a5fd3549 100644 --- a/apps/multiplatform/common/src/commonMain/kotlin/chat/simplex/common/views/chat/ChatView.kt +++ b/apps/multiplatform/common/src/commonMain/kotlin/chat/simplex/common/views/chat/ChatView.kt @@ -1185,6 +1185,22 @@ fun BoxScope.ChatItemsList( developerTools: Boolean, showViaProxy: Boolean ) { + val loadingTopItems = remember { mutableStateOf(false) } + val loadingBottomItems = remember { mutableStateOf(false) } + // just for changing local var here based on request + val loadMessages: suspend (ChatId, ChatPagination, visibleItemIndexesNonReversed: () -> IntRange) -> Unit = { chatId, pagination, visibleItemIndexesNonReversed -> + val loadingSide = when (pagination) { + is ChatPagination.Before -> loadingTopItems + is ChatPagination.Last -> loadingBottomItems + is ChatPagination.After, is ChatPagination.Around, is ChatPagination.Initial -> null + } + loadingSide?.value = true + try { + loadMessages(chatId, pagination, visibleItemIndexesNonReversed) + } finally { + loadingSide?.value = false + } + } val searchValueIsEmpty = remember { derivedStateOf { searchValue.value.isEmpty() } } val searchValueIsNotBlank = remember { derivedStateOf { searchValue.value.isNotBlank() } } val revealedItems = rememberSaveable(stateSaver = serializableSaver()) { mutableStateOf(setOf()) } @@ -1227,7 +1243,7 @@ fun BoxScope.ChatItemsList( if (reportsState != null) { reportsListState = null reportsState - } else if (index <= 0) { + } else if (index <= 0 || !searchValueIsEmpty.value) { LazyListState(0, 0) } else { LazyListState(index + 1, -maxHeightForList.value) @@ -1242,19 +1258,19 @@ fun BoxScope.ChatItemsList( if (searchValueIsEmpty.value && reversedChatItems.value.size < ChatPagination.INITIAL_COUNT) ignoreLoadingRequests.add(reversedChatItems.value.lastOrNull()?.id ?: return@LaunchedEffect) } - if (!loadingMoreItems.value) { - PreloadItems(chatInfo.id, if (searchValueIsEmpty.value) ignoreLoadingRequests else mutableSetOf(), contentTag, mergedItems, listState, ChatPagination.UNTIL_PRELOAD_COUNT) { chatId, pagination -> - if (loadingMoreItems.value) return@PreloadItems false + PreloadItems(chatInfo.id, if (searchValueIsEmpty.value) ignoreLoadingRequests else mutableSetOf(), loadingMoreItems, resetListState, contentTag, mergedItems, listState, ChatPagination.UNTIL_PRELOAD_COUNT) { chatId, pagination -> + if (loadingMoreItems.value || chatId != chatModel.chatId.value) return@PreloadItems false + loadingMoreItems.value = true + withContext(NonCancellable) { try { - loadingMoreItems.value = true loadMessages(chatId, pagination) { visibleItemIndexesNonReversed(mergedItems, reversedChatItems.value.size, listState.value) } } finally { loadingMoreItems.value = false } - true } + true } val remoteHostIdUpdated = rememberUpdatedState(remoteHostId) @@ -1275,7 +1291,6 @@ fun BoxScope.ChatItemsList( scrollToItemId.value = null } } } - LoadLastItems(loadingMoreItems, resetListState, remoteHostId, chatInfo) SmallScrollOnNewMessage(listState, reversedChatItems) val finishedInitialComposition = remember { mutableStateOf(false) } NotifyChatListOnFinishingComposition(finishedInitialComposition, chatInfo, revealedItems, listState, onComposed) @@ -1583,7 +1598,25 @@ fun BoxScope.ChatItemsList( } } } - FloatingButtons(topPaddingToContent, topPaddingToContentPx, loadingMoreItems, animatedScrollingInProgress, mergedItems, unreadCount, maxHeight, composeViewHeight, searchValue, markChatRead, listState) + FloatingButtons( + reversedChatItems, + chatInfoUpdated, + topPaddingToContent, + topPaddingToContentPx, + contentTag, + loadingMoreItems, + loadingTopItems, + loadingBottomItems, + animatedScrollingInProgress, + mergedItems, + unreadCount, + maxHeight, + composeViewHeight, + searchValue, + markChatRead, + listState, + loadMessages + ) FloatingDate(Modifier.padding(top = 10.dp + topPaddingToContent).align(Alignment.TopCenter), topPaddingToContentPx, mergedItems, listState) LaunchedEffect(Unit) { @@ -1603,21 +1636,20 @@ fun BoxScope.ChatItemsList( } } -@Composable -private fun LoadLastItems(loadingMoreItems: MutableState, resetListState: State, remoteHostId: Long?, chatInfo: ChatInfo) { - val contentTag = LocalContentTag.current - LaunchedEffect(remoteHostId, chatInfo.id, resetListState.value) { - try { - loadingMoreItems.value = true - if (chatModel.chatStateForContent(contentTag).totalAfter.value <= 0) return@LaunchedEffect - delay(500) - withContext(Dispatchers.Default) { - apiLoadMessages(remoteHostId, chatInfo.chatType, chatInfo.apiId, contentTag, ChatPagination.Last(ChatPagination.INITIAL_COUNT)) - } - } finally { - loadingMoreItems.value = false - } - } +private suspend fun loadLastItems(chatId: State, contentTag: MsgContentTag?, listState: State, loadItems: State Boolean>) { + val lastVisible = listState.value.layoutInfo.visibleItemsInfo.lastOrNull() + val itemsCanCoverScreen = lastVisible != null && listState.value.layoutInfo.viewportEndOffset - listState.value.layoutInfo.afterContentPadding <= lastVisible.offset + lastVisible.size + if (!itemsCanCoverScreen) return + + if (lastItemsLoaded(contentTag)) return + + delay(500) + loadItems.value(chatId.value, ChatPagination.Last(ChatPagination.INITIAL_COUNT)) +} + +private fun lastItemsLoaded(contentTag: MsgContentTag?): Boolean { + val chatState = chatModel.chatStateForContent(contentTag) + return chatState.splits.value.isEmpty() || chatState.splits.value.firstOrNull() != chatModel.chatItemsForContent(contentTag).value.lastOrNull()?.id } // TODO: in extra rare case when after loading last items only 1 item is loaded, the view will jump like when receiving new message @@ -1680,9 +1712,14 @@ private fun NotifyChatListOnFinishingComposition( @Composable fun BoxScope.FloatingButtons( + reversedChatItems: State>, + chatInfo: State, topPaddingToContent: Dp, topPaddingToContentPx: State, + contentTag: MsgContentTag?, loadingMoreItems: MutableState, + loadingTopItems: MutableState, + loadingBottomItems: MutableState, animatedScrollingInProgress: MutableState, mergedItems: State, unreadCount: State, @@ -1690,9 +1727,44 @@ fun BoxScope.FloatingButtons( composeViewHeight: State, searchValue: State, markChatRead: () -> Unit, - listState: State + listState: State, + loadMessages: suspend (ChatId, ChatPagination, visibleItemIndexesNonReversed: () -> IntRange) -> Unit ) { val scope = rememberCoroutineScope() + fun scrollToBottom() { + scope.launch { + animatedScrollingInProgress.value = true + tryBlockAndSetLoadingMore(loadingMoreItems) { listState.value.animateScrollToItem(0) } + } + } + fun scrollToTopUnread() { + scope.launch { + tryBlockAndSetLoadingMore(loadingMoreItems) { + if (chatModel.chatStateForContent(contentTag).splits.value.isNotEmpty()) { + val pagination = ChatPagination.Initial(ChatPagination.INITIAL_COUNT) + val oldSize = reversedChatItems.value.size + loadMessages(chatInfo.value.id, pagination) { + visibleItemIndexesNonReversed(mergedItems, reversedChatItems.value.size, listState.value) + } + var repeatsLeft = 100 + while (oldSize == reversedChatItems.value.size && repeatsLeft > 0) { + delay(10) + repeatsLeft-- + } + if (oldSize == reversedChatItems.value.size) { + return@tryBlockAndSetLoadingMore + } + } + val index = mergedItems.value.items.indexOfLast { it.hasUnread() } + if (index != -1) { + // scroll to the top unread item + animatedScrollingInProgress.value = true + listState.value.animateScrollToItem(index + 1, -maxHeight.value) + } + } + } + } + val bottomUnreadCount = remember { derivedStateOf { if (unreadCount.value == 0) return@derivedStateOf 0 @@ -1718,43 +1790,73 @@ fun BoxScope.FloatingButtons( allowToShowBottomWithArrow.value = shouldShow shouldShow && allow } } + + val requestedTopScroll = remember { mutableStateOf(false) } + val requestedBottomScroll = remember { mutableStateOf(false) } + BottomEndFloatingButton( bottomUnreadCount, showBottomButtonWithCounter, showBottomButtonWithArrow, + requestedBottomScroll, animatedScrollingInProgress, composeViewHeight, onClick = { - scope.launch { - animatedScrollingInProgress.value = true - tryBlockAndSetLoadingMore(loadingMoreItems) { listState.value.animateScrollToItem(0) } + if (loadingBottomItems.value || !lastItemsLoaded(contentTag)) { + requestedTopScroll.value = false + requestedBottomScroll.value = true + } else { + scrollToBottom() } } ) + LaunchedEffect(Unit) { + launch { + snapshotFlow { loadingTopItems.value } + .drop(1) + .collect { top -> + if (!top && requestedTopScroll.value) { + requestedTopScroll.value = false + scrollToTopUnread() + } + } + } + launch { + snapshotFlow { loadingBottomItems.value } + .drop(1) + .collect { bottom -> + if (!bottom && requestedBottomScroll.value) { + requestedBottomScroll.value = false + scrollToBottom() + } + } + } + } // Don't show top FAB if is in search if (searchValue.value.isNotEmpty()) return val fabSize = 56.dp - val topUnreadCount = remember { derivedStateOf { if (bottomUnreadCount.value >= 0) (unreadCount.value - bottomUnreadCount.value).coerceAtLeast(0) else 0 } } + val topUnreadCount = remember { derivedStateOf { + if (bottomUnreadCount.value >= 0) (unreadCount.value - bottomUnreadCount.value).coerceAtLeast(0) else 0 } + } val showDropDown = remember { mutableStateOf(false) } TopEndFloatingButton( Modifier.padding(end = DEFAULT_PADDING, top = 24.dp + topPaddingToContent).align(Alignment.TopEnd), topUnreadCount, + requestedTopScroll, animatedScrollingInProgress, onClick = { - val index = mergedItems.value.items.indexOfLast { it.hasUnread() } - if (index != -1) { - // scroll to the top unread item - scope.launch { - animatedScrollingInProgress.value = true - tryBlockAndSetLoadingMore(loadingMoreItems) { listState.value.animateScrollToItem(index + 1, -maxHeight.value) } - } + if (loadingTopItems.value) { + requestedBottomScroll.value = false + requestedTopScroll.value = true + } else { + scrollToTopUnread() } }, onLongClick = { showDropDown.value = true } ) - Box(Modifier.fillMaxWidth().wrapContentSize(Alignment.TopEnd)) { + Box(Modifier.fillMaxWidth().wrapContentSize(Alignment.TopEnd).align(Alignment.TopEnd)) { val density = LocalDensity.current val width = remember { mutableStateOf(250.dp) } DefaultDropdownMenu( @@ -1777,6 +1879,8 @@ fun BoxScope.FloatingButtons( fun PreloadItems( chatId: String, ignoreLoadingRequests: MutableSet, + loadingMoreItems: State, + resetListState: State, contentTag: MsgContentTag?, mergedItems: State, listState: State, @@ -1788,13 +1892,32 @@ fun PreloadItems( val chatId = rememberUpdatedState(chatId) val loadItems = rememberUpdatedState(loadItems) val ignoreLoadingRequests = rememberUpdatedState(ignoreLoadingRequests) - PreloadItemsBefore(allowLoad, chatId, ignoreLoadingRequests, contentTag, mergedItems, listState, remaining, loadItems) - PreloadItemsAfter(allowLoad, chatId, contentTag, mergedItems, listState, remaining, loadItems) + LaunchedEffect(Unit) { + snapshotFlow { chatId.value } + .distinctUntilChanged() + .filterNotNull() + .collect { + allowLoad.value = false + delay(500) + allowLoad.value = true + } + } + if (allowLoad.value && !loadingMoreItems.value) { + LaunchedEffect(chatId.value, resetListState.value) { + snapshotFlow { listState.value.firstVisibleItemIndex } + .distinctUntilChanged() + .collect { firstVisibleIndex -> + if (!preloadItemsBefore(firstVisibleIndex, chatId, ignoreLoadingRequests, contentTag, mergedItems, listState, remaining, loadItems)) { + preloadItemsAfter(firstVisibleIndex, chatId, contentTag, mergedItems, remaining, loadItems) + } + loadLastItems(chatId, contentTag, listState, loadItems) + } + } + } } -@Composable -private fun PreloadItemsBefore( - allowLoad: State, +private suspend fun preloadItemsBefore( + firstVisibleIndex: Int, chatId: State, ignoreLoadingRequests: State>, contentTag: MsgContentTag?, @@ -1802,83 +1925,47 @@ private fun PreloadItemsBefore( listState: State, remaining: Int, loadItems: State Boolean>, -) { - KeyChangeEffect(allowLoad.value, chatId.value) { - snapshotFlow { listState.value.firstVisibleItemIndex } - .distinctUntilChanged() - .map { firstVisibleIndex -> - val splits = mergedItems.value.splits - val lastVisibleIndex = (listState.value.layoutInfo.visibleItemsInfo.lastOrNull()?.index ?: 0) - var lastIndexToLoadFrom: Int? = findLastIndexToLoadFromInSplits(firstVisibleIndex, lastVisibleIndex, remaining, splits) - val items = reversedChatItemsStatic(contentTag) - if (splits.isEmpty() && items.isNotEmpty() && lastVisibleIndex > mergedItems.value.items.size - remaining) { - lastIndexToLoadFrom = items.lastIndex - } - if (allowLoad.value && lastIndexToLoadFrom != null) { - items.getOrNull(lastIndexToLoadFrom)?.id - } else { - null - } - } - .filterNotNull() - .filter { !ignoreLoadingRequests.value.contains(it) } - .collect { loadFromItemId -> - withBGApi { - val items = reversedChatItemsStatic(contentTag) - val sizeWas = items.size - val oldestItemIdWas = items.lastOrNull()?.id - val triedToLoad = loadItems.value(chatId.value, ChatPagination.Before(loadFromItemId, ChatPagination.PRELOAD_COUNT)) - val itemsUpdated = reversedChatItemsStatic(contentTag) - if (triedToLoad && sizeWas == itemsUpdated.size && oldestItemIdWas == itemsUpdated.lastOrNull()?.id) { - ignoreLoadingRequests.value.add(loadFromItemId) - } - } - } +): Boolean { + val splits = mergedItems.value.splits + val lastVisibleIndex = (listState.value.layoutInfo.visibleItemsInfo.lastOrNull()?.index ?: 0) + var lastIndexToLoadFrom: Int? = findLastIndexToLoadFromInSplits(firstVisibleIndex, lastVisibleIndex, remaining, splits) + val items = reversedChatItemsStatic(contentTag) + if (splits.isEmpty() && items.isNotEmpty() && lastVisibleIndex > mergedItems.value.items.size - remaining) { + lastIndexToLoadFrom = items.lastIndex } + if (lastIndexToLoadFrom != null) { + val loadFromItemId = items.getOrNull(lastIndexToLoadFrom)?.id ?: return false + if (!ignoreLoadingRequests.value.contains(loadFromItemId)) { + val items = reversedChatItemsStatic(contentTag) + val sizeWas = items.size + val oldestItemIdWas = items.lastOrNull()?.id + val triedToLoad = loadItems.value(chatId.value, ChatPagination.Before(loadFromItemId, ChatPagination.PRELOAD_COUNT)) + val itemsUpdated = reversedChatItemsStatic(contentTag) + if (triedToLoad && sizeWas == itemsUpdated.size && oldestItemIdWas == itemsUpdated.lastOrNull()?.id) { + ignoreLoadingRequests.value.add(loadFromItemId) + return false + } + return triedToLoad + } + } + return false } -@Composable -private fun PreloadItemsAfter( - allowLoad: MutableState, +private suspend fun preloadItemsAfter( + firstVisibleIndex: Int, chatId: State, contentTag: MsgContentTag?, mergedItems: State, - listState: State, remaining: Int, loadItems: State Boolean>, ) { - LaunchedEffect(Unit) { - snapshotFlow { chatId.value } - .distinctUntilChanged() - .filterNotNull() - .collect { - allowLoad.value = listState.value.layoutInfo.totalItemsCount == listState.value.layoutInfo.visibleItemsInfo.size - delay(500) - allowLoad.value = true - } - } - LaunchedEffect(chatId.value) { - launch { - snapshotFlow { listState.value.firstVisibleItemIndex } - .distinctUntilChanged() - .map { firstVisibleIndex -> - val items = reversedChatItemsStatic(contentTag) - val splits = mergedItems.value.splits - val split = splits.lastOrNull { it.indexRangeInParentItems.contains(firstVisibleIndex) } - // we're inside a splitRange (top --- [end of the splitRange --- we're here --- start of the splitRange] --- bottom) - if (split != null && split.indexRangeInParentItems.first + remaining > firstVisibleIndex) { - items.getOrNull(split.indexRangeInReversed.first)?.id - } else { - null - } - } - .filterNotNull() - .collect { loadFromItemId -> - withBGApi { - loadItems.value(chatId.value, ChatPagination.After(loadFromItemId, ChatPagination.PRELOAD_COUNT)) - } - } - } + val items = reversedChatItemsStatic(contentTag) + val splits = mergedItems.value.splits + val split = splits.lastOrNull { it.indexRangeInParentItems.contains(firstVisibleIndex) } + // we're inside a splitRange (top --- [end of the splitRange --- we're here --- start of the splitRange] --- bottom) + if (split != null && split.indexRangeInParentItems.first + remaining > firstVisibleIndex) { + val loadFromItemId = items.getOrNull(split.indexRangeInReversed.first)?.id ?: return + loadItems.value(chatId.value, ChatPagination.After(loadFromItemId, ChatPagination.PRELOAD_COUNT)) } } @@ -1893,6 +1980,7 @@ fun MemberImage(member: GroupMember) { private fun TopEndFloatingButton( modifier: Modifier = Modifier, unreadCount: State, + requestedTopScroll: State, animatedScrollingInProgress: State, onClick: () -> Unit, onLongClick: () -> Unit @@ -1906,11 +1994,15 @@ private fun TopEndFloatingButton( elevation = FloatingActionButtonDefaults.elevation(0.dp, 0.dp), interactionSource = interactionSource, ) { - Text( - unreadCountStr(unreadCount.value), - color = MaterialTheme.colors.primary, - fontSize = 14.sp, - ) + if (requestedTopScroll.value) { + LoadingProgressIndicator() + } else { + Text( + unreadCountStr(unreadCount.value), + color = MaterialTheme.colors.primary, + fontSize = 14.sp, + ) + } } } } @@ -2115,7 +2207,7 @@ private fun DateSeparator(date: Instant) { @Composable private fun MarkItemsReadAfterDelay( - itemKey: String, + itemKey: ChatViewItemKey, itemIds: List, finishedInitialComposition: State, chatId: ChatId, @@ -2153,18 +2245,20 @@ private fun reversedChatItemsStatic(contentTag: MsgContentTag?): List private fun oldestPartiallyVisibleListItemInListStateOrNull(topPaddingToContentPx: State, mergedItems: State, listState: State): ListItem? { val lastFullyVisibleOffset = listState.value.layoutInfo.viewportEndOffset - topPaddingToContentPx.value - return mergedItems.value.items.getOrNull((listState.value.layoutInfo.visibleItemsInfo.lastOrNull { item -> + val visibleKey: ChatViewItemKey? = listState.value.layoutInfo.visibleItemsInfo.lastOrNull { item -> item.offset <= lastFullyVisibleOffset - }?.index ?: listState.value.layoutInfo.visibleItemsInfo.lastOrNull()?.index) ?: -1)?.oldest() + }?.key as? ChatViewItemKey + return mergedItems.value.items.getOrNull((mergedItems.value.indexInParentItems[visibleKey?.first] ?: listState.value.layoutInfo.visibleItemsInfo.lastOrNull()?.index) ?: -1)?.oldest() } private fun lastFullyVisibleIemInListState(topPaddingToContentPx: State, density: Float, fontSizeSqrtMultiplier: Float, mergedItems: State, listState: State): ChatItem? { val lastFullyVisibleOffsetMinusFloatingHeight = listState.value.layoutInfo.viewportEndOffset - topPaddingToContentPx.value - 50 * density * fontSizeSqrtMultiplier + val visibleKey: ChatViewItemKey? = listState.value.layoutInfo.visibleItemsInfo.lastOrNull { item -> + item.offset <= lastFullyVisibleOffsetMinusFloatingHeight && item.size > 0 + }?.key as? ChatViewItemKey + return mergedItems.value.items.getOrNull( - (listState.value.layoutInfo.visibleItemsInfo.lastOrNull { item -> - item.offset <= lastFullyVisibleOffsetMinusFloatingHeight && item.size > 0 - } - ?.index + (mergedItems.value.indexInParentItems[visibleKey?.first] ?: listState.value.layoutInfo.visibleItemsInfo.lastOrNull()?.index) ?: -1)?.newest()?.item } @@ -2276,39 +2370,50 @@ private fun BoxScope.BottomEndFloatingButton( unreadCount: State, showButtonWithCounter: State, showButtonWithArrow: State, + requestedBottomScroll: State, animatedScrollingInProgress: State, composeViewHeight: State, onClick: () -> Unit -) = when { - showButtonWithCounter.value && !animatedScrollingInProgress.value -> { - FloatingActionButton( - onClick = onClick, - elevation = FloatingActionButtonDefaults.elevation(0.dp, 0.dp, 0.dp, 0.dp), - modifier = Modifier.padding(end = DEFAULT_PADDING, bottom = DEFAULT_PADDING + composeViewHeight.value).align(Alignment.BottomEnd).size(48.dp), - backgroundColor = MaterialTheme.colors.secondaryVariant, - ) { - Text( - unreadCountStr(unreadCount.value), - color = MaterialTheme.colors.primary, - fontSize = 14.sp, - ) +) { + when { + showButtonWithCounter.value && !animatedScrollingInProgress.value -> { + FloatingActionButton( + onClick = onClick, + elevation = FloatingActionButtonDefaults.elevation(0.dp, 0.dp, 0.dp, 0.dp), + modifier = Modifier.padding(end = DEFAULT_PADDING, bottom = DEFAULT_PADDING + composeViewHeight.value).align(Alignment.BottomEnd).size(48.dp), + backgroundColor = MaterialTheme.colors.secondaryVariant, + ) { + if (requestedBottomScroll.value) { + LoadingProgressIndicator() + } else { + Text( + unreadCountStr(unreadCount.value), + color = MaterialTheme.colors.primary, + fontSize = 14.sp, + ) + } + } } - } - showButtonWithArrow.value && !animatedScrollingInProgress.value -> { - FloatingActionButton( - onClick = onClick, - elevation = FloatingActionButtonDefaults.elevation(0.dp, 0.dp, 0.dp, 0.dp), - modifier = Modifier.padding(end = DEFAULT_PADDING, bottom = DEFAULT_PADDING + composeViewHeight.value).align(Alignment.BottomEnd).size(48.dp), - backgroundColor = MaterialTheme.colors.secondaryVariant, - ) { - Icon( - painter = painterResource(MR.images.ic_keyboard_arrow_down), - contentDescription = null, - tint = MaterialTheme.colors.primary - ) + showButtonWithArrow.value && !animatedScrollingInProgress.value -> { + FloatingActionButton( + onClick = onClick, + elevation = FloatingActionButtonDefaults.elevation(0.dp, 0.dp, 0.dp, 0.dp), + modifier = Modifier.padding(end = DEFAULT_PADDING, bottom = DEFAULT_PADDING + composeViewHeight.value).align(Alignment.BottomEnd).size(48.dp), + backgroundColor = MaterialTheme.colors.secondaryVariant, + ) { + if (requestedBottomScroll.value) { + LoadingProgressIndicator() + } else { + Icon( + painter = painterResource(MR.images.ic_keyboard_arrow_down), + contentDescription = null, + tint = MaterialTheme.colors.primary + ) + } + } } + else -> {} } - else -> {} } @Composable @@ -2334,6 +2439,20 @@ fun SelectedListItem( ) } +@Composable +private fun LoadingProgressIndicator() { + Box( + Modifier.fillMaxSize(), + contentAlignment = Alignment.Center + ) { + CircularProgressIndicator( + Modifier.size(30.dp), + color = MaterialTheme.colors.secondary, + strokeWidth = 2.dp + ) + } +} + private fun selectUnselectChatItem( select: Boolean, ci: ChatItem, @@ -2652,7 +2771,9 @@ fun providerForGallery( } } -private fun keyForItem(item: ChatItem): String = (item.id to item.meta.createdAt.toEpochMilliseconds()).toString() +typealias ChatViewItemKey = Pair + +private fun keyForItem(item: ChatItem): ChatViewItemKey = ChatViewItemKey(item.id, item.meta.createdAt.toEpochMilliseconds()) private fun ViewConfiguration.bigTouchSlop(slop: Float = 50f) = object: ViewConfiguration { override val longPressTimeoutMillis diff --git a/apps/multiplatform/common/src/commonMain/kotlin/chat/simplex/common/views/chat/group/GroupChatInfoView.kt b/apps/multiplatform/common/src/commonMain/kotlin/chat/simplex/common/views/chat/group/GroupChatInfoView.kt index a94c787eec..f38cd972f3 100644 --- a/apps/multiplatform/common/src/commonMain/kotlin/chat/simplex/common/views/chat/group/GroupChatInfoView.kt +++ b/apps/multiplatform/common/src/commonMain/kotlin/chat/simplex/common/views/chat/group/GroupChatInfoView.kt @@ -379,10 +379,11 @@ fun ModalData.GroupChatInfoLayout( } else { PaddingValues( top = topPaddingToContent(false), - bottom = navBarPadding + - imePadding + - selectedItemsBarHeight + - (if (navBarPadding > 0.dp && imePadding > 0.dp) -AppBarHeight * fontSizeSqrtMultiplier else 0.dp) + bottom = if (imePadding > 0.dp) { + imePadding + selectedItemsBarHeight + } else { + navBarPadding + selectedItemsBarHeight + } ) } ) { diff --git a/apps/multiplatform/common/src/commonMain/kotlin/chat/simplex/common/views/chat/item/FramedItemView.kt b/apps/multiplatform/common/src/commonMain/kotlin/chat/simplex/common/views/chat/item/FramedItemView.kt index 689bf195f5..fd8a32af64 100644 --- a/apps/multiplatform/common/src/commonMain/kotlin/chat/simplex/common/views/chat/item/FramedItemView.kt +++ b/apps/multiplatform/common/src/commonMain/kotlin/chat/simplex/common/views/chat/item/FramedItemView.kt @@ -66,7 +66,11 @@ fun FramedItemView( @Composable fun ciQuotedMsgView(qi: CIQuote) { Box( - Modifier.padding(vertical = 6.dp, horizontal = 12.dp), + Modifier + // this width limitation prevents crash on calculating constraints that may happen if you post veeeery long message and then quote it. + // Top level layout wants `IntrinsicWidth.Max` and very long layout makes the crash in this case + .widthIn(max = 50000.dp) + .padding(vertical = 6.dp, horizontal = 12.dp), contentAlignment = Alignment.TopStart ) { val sender = qi.sender(membership()) diff --git a/apps/multiplatform/common/src/commonMain/kotlin/chat/simplex/common/views/chatlist/ChatListNavLinkView.kt b/apps/multiplatform/common/src/commonMain/kotlin/chat/simplex/common/views/chatlist/ChatListNavLinkView.kt index 89dc84a1fc..8b4b2bb1d2 100644 --- a/apps/multiplatform/common/src/commonMain/kotlin/chat/simplex/common/views/chatlist/ChatListNavLinkView.kt +++ b/apps/multiplatform/common/src/commonMain/kotlin/chat/simplex/common/views/chatlist/ChatListNavLinkView.kt @@ -105,13 +105,14 @@ fun ChatListNavLinkView(chat: Chat, nextChatSelected: State) { ) } is ChatInfo.Local -> { + val defaultClickAction = { if (chatModel.chatId.value != chat.id) scope.launch { noteFolderChatAction(chat.remoteHostId, chat.chatInfo.noteFolder) } } ChatListNavLinkLayout( chatLinkPreview = { tryOrShowError("${chat.id}ChatListNavLink", error = { ErrorChatListItem() }) { - ChatPreviewView(chat, showChatPreviews, chatModel.draft.value, chatModel.draftChatId.value, chatModel.currentUser.value?.profile?.displayName, null, disabled, linkMode, inProgress = false, progressByTimeout = false, {}) + ChatPreviewView(chat, showChatPreviews, chatModel.draft.value, chatModel.draftChatId.value, chatModel.currentUser.value?.profile?.displayName, null, disabled, linkMode, inProgress = false, progressByTimeout = false, defaultClickAction) } }, - click = { if (chatModel.chatId.value != chat.id) scope.launch { noteFolderChatAction(chat.remoteHostId, chat.chatInfo.noteFolder) } }, + click = defaultClickAction, dropdownMenuItems = { tryOrShowError("${chat.id}ChatListNavLinkDropdown", error = {}) { NoteFolderMenuItems(chat, showMenu, showMarkRead) @@ -244,7 +245,7 @@ suspend fun apiFindMessages(ch: Chat, search: String, contentTag: MsgContentTag? withChats(contentTag) { chatItems.clearAndNotify() } - apiLoadMessages(ch.remoteHostId, ch.chatInfo.chatType, ch.chatInfo.apiId, contentTag, pagination = ChatPagination.Last(ChatPagination.INITIAL_COUNT), search = search) + apiLoadMessages(ch.remoteHostId, ch.chatInfo.chatType, ch.chatInfo.apiId, contentTag, pagination = if (search.isNotEmpty()) ChatPagination.Last(ChatPagination.INITIAL_COUNT) else ChatPagination.Initial(ChatPagination.INITIAL_COUNT), search = search) } suspend fun setGroupMembers(rhId: Long?, groupInfo: GroupInfo, chatModel: ChatModel) = coroutineScope { diff --git a/apps/simplex-directory-service/src/Directory/Captcha.hs b/apps/simplex-directory-service/src/Directory/Captcha.hs new file mode 100644 index 0000000000..d60b09df83 --- /dev/null +++ b/apps/simplex-directory-service/src/Directory/Captcha.hs @@ -0,0 +1,37 @@ +module Directory.Captcha (getCaptchaStr, matchCaptchaStr) where + +import qualified Data.Map.Strict as M +import Data.Maybe (fromMaybe) +import qualified Data.Text as T +import System.Random (randomRIO) + +getCaptchaStr :: Int -> String -> IO String +getCaptchaStr 0 s = pure s +getCaptchaStr n s = do + i <- randomRIO (0, length captchaChars - 1) + let c = captchaChars !! i + getCaptchaStr (n - 1) (c : s) + where + captchaChars = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" + +matchCaptchaStr :: T.Text -> T.Text -> Bool +matchCaptchaStr captcha guess = T.length captcha == T.length guess && matchChars (T.zip captcha guess) + where + matchChars [] = True + matchChars ((c, g) : cs) = matchChar c == matchChar g && matchChars cs + matchChar c = fromMaybe c $ M.lookup c captchaMatches + captchaMatches = + M.fromList + [ ('0', 'O'), + ('1', 'I'), + ('c', 'C'), + ('l', 'I'), + ('o', 'O'), + ('p', 'P'), + ('s', 'S'), + ('u', 'U'), + ('v', 'V'), + ('w', 'W'), + ('x', 'X'), + ('z', 'Z') + ] diff --git a/apps/simplex-directory-service/src/Directory/Service.hs b/apps/simplex-directory-service/src/Directory/Service.hs index 79654b4da6..575c7ca738 100644 --- a/apps/simplex-directory-service/src/Directory/Service.hs +++ b/apps/simplex-directory-service/src/Directory/Service.hs @@ -39,6 +39,7 @@ import qualified Data.Text.IO as T import Data.Time.Clock (NominalDiffTime, UTCTime, diffUTCTime, getCurrentTime) import Data.Time.LocalTime (getCurrentTimeZone) import Directory.BlockedWords +import Directory.Captcha import Directory.Events import Directory.Options import Directory.Search @@ -67,7 +68,6 @@ import qualified Simplex.Messaging.TMap as TM import Simplex.Messaging.Util (safeDecodeUtf8, tshow, ($>>=), (<$$>)) import System.Directory (getAppUserDataDirectory) import System.Process (readProcess) -import System.Random (randomRIO) data GroupProfileUpdate = GPNoServiceLink | GPServiceLinkAdded | GPServiceLinkRemoved | GPHasServiceLink | GPServiceLinkError @@ -455,12 +455,6 @@ directoryServiceEvent st opts@DirectoryOpts {adminUsers, superUsers, serviceName atomically $ TM.insert gmId captcha $ pendingCaptchas env sendCaptcha mc where - getCaptchaStr 0 s = pure s - getCaptchaStr n s = do - i <- randomRIO (0, length chars - 1) - let c = chars !! i - getCaptchaStr (n - 1) (c : s) - chars = "23456789ABCDEFGHIJKLMNOPQRSTUVWXYZabdefghijkmnpqrsty" getCaptcha s = case captchaGenerator opts of Nothing -> pure textMsg Just script -> content <$> readProcess script [s] "" @@ -491,7 +485,7 @@ directoryServiceEvent st opts@DirectoryOpts {adminUsers, superUsers, serviceName atomically (TM.lookup (groupMemberId' m) $ pendingCaptchas env) >>= \case Just PendingCaptcha {captchaText, sentAt, attempts} | ts `diffUTCTime` sentAt > captchaTTL -> sendMemberCaptcha g m (Just ciId) captchaExpired $ attempts - 1 - | captchaText == msgText -> do + | matchCaptchaStr captchaText msgText -> do sendComposedMessages_ cc (SRGroup groupId $ Just $ groupMemberId' m) [(Just ciId, MCText $ "Correct, you joined the group " <> n)] approvePendingMember a g m | attempts >= maxCaptchaAttempts -> rejectPendingMember tooManyAttempts diff --git a/blog/20250308-simplex-chat-v6-3-new-user-experience-safety-in-public-groups.md b/blog/20250308-simplex-chat-v6-3-new-user-experience-safety-in-public-groups.md index 1e201825ce..f1aa3a243f 100644 --- a/blog/20250308-simplex-chat-v6-3-new-user-experience-safety-in-public-groups.md +++ b/blog/20250308-simplex-chat-v6-3-new-user-experience-safety-in-public-groups.md @@ -2,31 +2,157 @@ layout: layouts/article.html title: "SimpleX Chat v6.3: new user experience and safety in public groups" date: 2025-03-08 -# previewBody: blog_previews/20241210.html -# image: images/20241210-operators-1.png -draft: true +previewBody: blog_previews/20250308.html +image: images/20250308-captcha.png +imageBottom: true permalink: "/blog/20250308-simplex-chat-v6-3-new-user-experience-safety-in-public-groups.html" --- # SimpleX Chat v6.3: new user experience and safety in public groups -**Will be published:** Mar 8, 2025 +**Published:** Mar 8, 2025 -This is a permalink for release announcement. +**What's new in v6.3**: +- [preventing spam and abuse in public groups](#preventing-spam-and-abuse-in-public-groups). +- [group improvements](#group-improvements): [mention other members](#mention-other-members-and-get-notified-when-mentioned), [improved performance](#better-group-performance). +- [better chat navigation](#better-chat-navigation): [organize chats into lists](#organize-chats-into-lists) and [jump to found and forwarded messages](#jump-to-found-and-forwarded-messages). +- [privacy and security improvements](#privacy-and-security-improvements): [chat retention period](#set-message-retention-period-in-chats) and [private media file names](#private-media-file-names). -What's new in v6.3: +Also, we added Catalan interface language to Android and desktop apps, thanks to [our users and Weblate](https://github.com/simplex-chat/simplex-chat#help-translating-simplex-chat). -- improved groups: - - [mention other members](#mention-members). - - [private reports](#private-reports) to group admins. - - [preventing spam and abuse](#preventing-spam-and-abuse). - - [better performance](#better-privacy-and-security). -- [chat lists](#chat-lists) to keep track of what's important. -- [jump to found and forwarded messages](#jump-to-found-and-forwarded-messages) -- [privacy and security improvements](#privacy-and-security-improvements). +The last but not the least - [server builds are now reproducible](#reproducible-server-builds). ## What's new in v6.3 +## Preventing spam and abuse in public groups + +[We wrote before](./20250114-simplex-network-large-groups-privacy-preserving-content-moderation.md): as the network grows, it becomes more attractive to attackers. This release adds several features that reduce the possibility of attacks and abuse. + +### Spam in groups that are listed in our group directory + +There is no built-in group discovery in SimpleX Chat apps. Instead, we offer an experimental chat bot that allows to submit and to discover public groups. Not so long ago, spammers started sending messages via bots attempting to disrupt these groups. + +We released several changes to the groups directory to protect from spam attacks. + +**Optional captcha verification** + + + +Group owners can enable the requirement to pass captcha challenge before joining the group. Captcha is generated in the directory bot itself, without any 3rd party servers, and is sent to the joining member. The new member must reply with the text in the image to be accepted to the group. While not a perfect protection, this basic measure complicates programming automatic bots to join public groups. It also provides a foundation to implement "knocking" - a conversation with dedicated group admins prior to joining the group. We plan to release support for knocking in March. + +**Profanity filter for member names** + +While group settings support giving all joining member an "observer" role - that is, without the right to send messages - the attackers tried spaming groups by joining and leaving. We added an optional filter for member names that group owners can enable for groups listed in directory - if a member name contains profanity, they will be rejected. Further improvements will be released in March as well. + +The current SimpleX directory chatbot is a hybrid of [future chat relays](./20250114-simplex-network-large-groups-privacy-preserving-content-moderation.md#can-large-groups-scale) (a.k.a. super-peers) we are now developing to support large groups, and of a directory service that will be embedded in the app UI later this year, allowing to search and to discover public groups. Anybody is able to run their own directory bots now, and there will be possibility to use third party directories via the app UI in the future too. + +Read more about [SimpleX group directory](../docs/DIRECTORY.md), how to submit your groups, and which groups we now accept. Currently we accept groups related to a limited list of topics that will be expanded once we have better moderation functionality for the groups. + +### More power to group owners and moderators + +This release includes two new features to help group moderators. + + + +**Private reports** + +Group members can privately bring to group moderators attention specific messages and members, even if the group does not allow direct messages. The simply need to choose report in the message context menu and choose the report reason. This report will be visible to all group owners and moderators, but not to other members. + +Group moderators can see all member reports in a separate view, and quickly find the problematic messages, making moderation much easier in public groups. These reports are private to groups, they are not sent to server operators. + +Please note: in the groups listed in our directory, the directory bot acts as admin, so it will receive all reports as well. + +**Acting on multiple members at once** + +When attackers come, they often use multiple profiles. This version allows selecting multiple members at once and perform these actions on all selected members: +- switch members role between "observer" and "member". +- block and unblock members - this is a "shadow" block, so when you block multiple members who you believe are attackers, their messages will be blocked for all other members but not for them. +- remove members from the group. + +The next version will also allow to remove members together with all messages they sent - for example, if a spam bot joined and sent a lot of spam, but nothing of value. + +## Group improvements + +### Mention other members and get notified when mentioned + + + +This feature allows you to mention other members in the group in the usual way - type `@` character, and choose the member you want to mention from the menu. Even that there is no user accounts and persistent identities we made it work by referencing members by their random group ID that is also used for replies and all other interactions in the group. + +You can also now switch message notifications in the group to "mentions only" mode. You will be notified only when you are mentioned in a message, or when somebody replies to your message. Simply choose "Mute" in the context menu of the group in the list of chats to switch group notifications to "mentions only" mode. After that you can choose "Mute all" to disable all notifications, including mentions. + +### Better group performance + +**Send messages faster** + +We didn't reduce the required network traffic to send messages to large groups yet - your client still has to send message to each member individually. But we redesigned the process of sending a message, reducing temporary storage required to schedule the message for delivery by about 100x. This creates a significant storage saving - e.g, to send one message to a group of 1,000 members previously required ~20Mb, and now it is reduced to ~200kb. It also reduces the time and battery used to send a message. + +**Faster group deletion** + +When you leave the group, the app preserves a copy of all your communications in the group. You can choose to keep it or to delete it completely. This final group deletion was very slow prior to this release - depending on the number of groups on your device it could sometimes take several minutes. + +This release solved this problem – the time it takes to delete the group is reduced to seconds, and even in cases when the app is terminated half-way, it either rolls back or completes, but it cannot leave the group in a partially deleted state. It improves both user experience and privacy, as gives you better control over your data. + +## Better chat navigation + +### Organize chats into lists + + + +It is a common feature in many messengers – it helps organizing your conversations. + +The lists also show a blue mark when any chat in the list has new messages. + +There are several preset lists: contacts, groups, private notes, business chats, favourite chats and also groups with member reports - the last list is automatically shown if members of any groups where you are the moderator or the owner sent private reports, until these reports are acted on or archived. + +### Jump to found and forwarded messages + +This version allows to quickly navigate from message in the search results to the point in the conversation when it was sent. + +You can also navigate from the forwarded message (or from the message saved to private notes) to the original message in the chat where it was forwarded or saved from. + +## Privacy and security improvements + +### Set message retention period in chats + +Before this version, you could enable message retention period for all chats in your profile. While helpful in some cases, many of us have conversations that we want to keep for a long time, and some other conversations that we want to remove quicker. + +This version allows it - you can set different retention periods in different conversations. It can be 1 day, 1 week, 1 month or 1 year. We may allow custom retention time in the future. + +### Private media file names + +Previously there were scenarios when original media file names were preserved - e.g., when sending a video file or when forwarding any media file. The latter problem was worse, as media file name is generated automatically, and includes timestamp. So the same name could have been used to correlate files between conversations, as one of our users pointed out. + +This version fixes this problem - media file name is now changed when forwarding it to match the time of forwarding, so no additional metadata is revealed. + +Please also note: +- the apps remove metadata from all static images, +- iOS app removes metadata from videos, but android and desktop apps do not do it yet, +- animated images are sent as is, +- other file types are sent as is, and their names are left unchanged - we believe that for ordinary files their name is part of their content. + +We plan further improvements to reduce metadata in files in the near future – please let us know what you believe is the most important to reduce first. + +## Reproducible server builds + +Starting from v6.3 server releases are reproducible! + +**Why it is important** + +With reproducible builds anybody can build servers from our code following the same process, and the build would produce identical binaries. + +This also allows us to sign releases, as we reproduce GitHub builds ourselves and by signing them we attest that our builds resulted in identical binaries. + +**How to reproduce builds** + +You can reproduce our builds on Linux with x86 CPU in docker container - please follow the instructions [here](../docs/SERVER.md#reproduce-builds). + +We are looking for support from open-source contributors or security researchers who would also reproduce and sign our releases. + +**How to verify release signature** + +Please see the instructions [here](../docs/SERVER.md#verifying-server-binaries). + ## SimpleX network Some links to answer the most common questions: diff --git a/blog/README.md b/blog/README.md index 1432d95de5..00a84eca6c 100644 --- a/blog/README.md +++ b/blog/README.md @@ -1,5 +1,25 @@ # Blog +Mar 3, 2025 [SimpleX Chat v6.3: new user experience and safety in public groups](./20241014-simplex-network-v6-1-security-review-better-calls-user-experience.md) + +What's new in v6.3: +- preventing spam and abuse in public groups. +- group improvements: mention other members and improved performance. +- better chat navigation: organize chats into lists and jump to found and forwarded messages. +- privacy and security improvements: chat retention period and private media file names. + +Also, we added Catalan interface language to Android and desktop apps, thanks to our users and Weblate. + +The last but not the least - server builds are now reproducible! + +-- + +Jan 14, 2025 [SimpleX network: large groups and privacy-preserving content moderation](./20250114-simplex-network-large-groups-privacy-preserving-content-moderation.md) + +This post explains how server operators can moderate end-to-end encrypted conversations without compromising user privacy or end-to-end encryption. + +-- + Dec 10, 2024 [SimpleX network: preset servers operated by Flux, business chats and more with v6.2 of the apps](./20241210-simplex-network-v6-2-servers-by-flux-business-chats.md) - SimpleX Chat and Flux made an agreement to include Flux-operated servers into the app to improve metadata privacy in SimpleX network. @@ -19,7 +39,7 @@ Nov 25, 2024 [Servers operated by Flux - true privacy and decentralization for a --- -Oct 14, 2024 [SimpleX network: security review of protocols design by Trail of Bits, v6.1 released with better calls and user experience.](./20241014-simplex-network-v6-1-security-review-better-calls-user-experience.md) +Oct 14, 2024 [SimpleX network: security review of protocols design by Trail of Bits, v6.1 released with better calls and user experience](./20241014-simplex-network-v6-1-security-review-better-calls-user-experience.md) New security audit: Trail of Bits reviewed the cryptographic design of protocols used in SimpleX network and apps. diff --git a/blog/images/20250308-captcha.png b/blog/images/20250308-captcha.png new file mode 100644 index 0000000000..7f43b47bd1 Binary files /dev/null and b/blog/images/20250308-captcha.png differ diff --git a/blog/images/20250308-lists.png b/blog/images/20250308-lists.png new file mode 100644 index 0000000000..cd15adbcbc Binary files /dev/null and b/blog/images/20250308-lists.png differ diff --git a/blog/images/20250308-mentions.png b/blog/images/20250308-mentions.png new file mode 100644 index 0000000000..658c7e0011 Binary files /dev/null and b/blog/images/20250308-mentions.png differ diff --git a/blog/images/20250308-reports.png b/blog/images/20250308-reports.png new file mode 100644 index 0000000000..def06bf83e Binary files /dev/null and b/blog/images/20250308-reports.png differ diff --git a/cabal.project b/cabal.project index 2e1b43d41f..0554d14ec8 100644 --- a/cabal.project +++ b/cabal.project @@ -12,7 +12,7 @@ constraints: zip +disable-bzip2 +disable-zstd source-repository-package type: git location: https://github.com/simplex-chat/simplexmq.git - tag: a491a1d8780054432542611f540317a6090b9360 + tag: aace3fd2fb146097304b09ef07ff613a8b255f67 source-repository-package type: git diff --git a/docs/SERVER.md b/docs/SERVER.md index 88ef74c058..3bfd064c4e 100644 --- a/docs/SERVER.md +++ b/docs/SERVER.md @@ -15,6 +15,7 @@ revision: 12.10.2024 - [systemd service](#systemd-service) with [installation script](#installation-script) or [manually](#manual-deployment) - [docker container](#docker-container) - [Linode marketplace](#linode-marketplace) +- [Verifying server binaries] - [Configuration](#configuration) - [Interactively](#interactively) - [Via command line options](#via-command-line-options) @@ -32,6 +33,7 @@ revision: 12.10.2024 - [Systemd commands](#systemd-commands) - [Control port](#control-port) - [Daily statistics](#daily-statistics) +- [Reproduce builds](#reproduce-builds) - [Updating your SMP server](#updating-your-smp-server) - [Configuring the app to use the server](#configuring-the-app-to-use-the-server) @@ -515,6 +517,28 @@ This configuration allows you to retain the ability to manage 80 and 443 ports y You can deploy smp-server upon creating new Linode VM. Please refer to: [Linode Marketplace](https://www.linode.com/marketplace/apps/simplex-chat/simplex-chat/) +## Verifying server binaries + +Starting from v6.3 server builds are [reproducible](#reproduce-builds). + +That also allows us to sign server releases, confirming the integrity of GitHub builds. + +To verify server binaries after you downloaded them: + +1. Download `_sha256sums` (hashes of all server binaries) and `_sha256sums.asc` (signature). + +2. Download our key FB44AF81A45BDE327319797C85107E357D4A17FC from [openpgp.org](https://keys.openpgp.org/search?q=chat%40simplex.chat) + +3. Import the key with `gpg --import FB44AF81A45BDE327319797C85107E357D4A17FC`. Key filename should be the same as its fingerprint, but please change it if necessary. + +4. Run `gpg --verify --trusted-key _sha256sums.asc _sha256sums`. It should print: + +> Good signature from "SimpleX Chat " + +5. Compute the hashes of the binaries you plan to use with `shu256sum ` or with `openssl sha256 ` and compare them with the hashes in the file `_sha256sums` - they must be the same. + +That is it - you now verified authenticity of our GitHub server binaries. + ## Configuration To see which options are available, execute `smp-server` without flags: @@ -1564,6 +1588,78 @@ To update your smp-server to latest version, choose your installation method and docker image prune ``` +## Reproduce builds + +You can locally reproduce server binaries, following these instructions. + +You must have: + +- Linux machine +- `x86-64` architecture +- Installed `docker`, `curl` and `git` + +1. Download script: + + ```sh + curl -LO 'https://raw.githubusercontent.com/simplex-chat/simplexmq/refs/heads/master/scripts/reproduce-builds.sh' + ``` + +2. Make it executable: + + ```sh + chmod +x reproduce-builds.sh + ``` + +3. Execute the script with the required tag: + + ```sh + ./reproduce-builds.sh 'v6.3.0' + ``` + + This will take a while. + +4. After compilation, you should see the following folders: + + ```sh + ls out* + ``` + + ```sh + out-20.04: + ntf-server smp-server xftp xftp-server + + out-20.04-github: + ntf-server smp-server xftp xftp-server + + out-22.04: + ntf-server smp-server xftp xftp-server + + out-22.04-github: + ntf-server smp-server xftp xftp-server + + out-24.04: + ntf-server smp-server xftp xftp-server + + out-24.04-github: + ntf-server smp-server xftp xftp-server + ``` + +5. Compare the hashes from github release with locally build binaries: + + ```sh + sha256sum out*-github/* + ``` + + ```sh + sha256sum out*[0-9]/* + ``` + + You can safely delete cloned repository: + + ```sh + cd ../ && rm -rf simplexmq + ``` + ## Configuring the app to use the server To configure the app to use your messaging server copy it's full address, including password, and add it to the app. You have an option to use your server together with preset servers or without them - you can remove or disable them. diff --git a/flake.nix b/flake.nix index 05859ce563..9e7e2bf521 100644 --- a/flake.nix +++ b/flake.nix @@ -165,6 +165,7 @@ packages.direct-sqlcipher.patches = [ ./scripts/nix/direct-sqlcipher-android-log.patch ]; + packages.simplex-chat.flags.client_library = true; packages.simplexmq.flags.client_library = true; packages.simplexmq.components.library.libs = pkgs.lib.mkForce [ (android32Pkgs.openssl.override { static = true; enableKTLS = false; }) @@ -230,6 +231,7 @@ packages.direct-sqlcipher.patches = [ ./scripts/nix/direct-sqlcipher-android-log.patch ]; + packages.simplex-chat.flags.client_library = true; packages.simplexmq.flags.client_library = true; packages.simplexmq.components.library.libs = pkgs.lib.mkForce [ (androidPkgs.openssl.override { static = true; }) @@ -297,6 +299,7 @@ packages.simplexmq.flags.swift = true; packages.direct-sqlcipher.flags.commoncrypto = true; packages.entropy.flags.DoNotGetEntropy = true; + packages.simplex-chat.flags.client_library = true; packages.simplexmq.flags.client_library = true; packages.simplexmq.components.library.libs = pkgs.lib.mkForce [ (pkgs.openssl.override { static = true; }) @@ -311,6 +314,7 @@ extra-modules = [{ packages.direct-sqlcipher.flags.commoncrypto = true; packages.entropy.flags.DoNotGetEntropy = true; + packages.simplex-chat.flags.client_library = true; packages.simplexmq.flags.client_library = true; packages.simplexmq.components.library.libs = pkgs.lib.mkForce [ ((pkgs.openssl.override { static = true; }).overrideDerivation (old: { CFLAGS = "-mcpu=apple-a7 -march=armv8-a+norcpc" ;})) @@ -329,6 +333,7 @@ packages.simplexmq.flags.swift = true; packages.direct-sqlcipher.flags.commoncrypto = true; packages.entropy.flags.DoNotGetEntropy = true; + packages.simplex-chat.flags.client_library = true; packages.simplexmq.flags.client_library = true; packages.simplexmq.components.library.libs = pkgs.lib.mkForce [ (pkgs.openssl.override { static = true; }) @@ -343,6 +348,7 @@ extra-modules = [{ packages.direct-sqlcipher.flags.commoncrypto = true; packages.entropy.flags.DoNotGetEntropy = true; + packages.simplex-chat.flags.client_library = true; packages.simplexmq.flags.client_library = true; packages.simplexmq.components.library.libs = pkgs.lib.mkForce [ (pkgs.openssl.override { static = true; }) diff --git a/scripts/desktop/build-lib-linux.sh b/scripts/desktop/build-lib-linux.sh index c8f887a9af..61a4a0ea43 100755 --- a/scripts/desktop/build-lib-linux.sh +++ b/scripts/desktop/build-lib-linux.sh @@ -25,7 +25,7 @@ for elem in "${exports[@]}"; do count=$(grep -R "$elem$" libsimplex.dll.def | wc for elem in "${exports[@]}"; do count=$(grep -R "\"$elem\"" flake.nix | wc -l); if [ $count -ne 2 ]; then echo Wrong exports in flake.nix. Add \"$elem\" in two places of the file; exit 1; fi ; done rm -rf $BUILD_DIR -cabal build lib:simplex-chat --ghc-options='-optl-Wl,-rpath,$ORIGIN' --ghc-options="-optl-L$(ghc --print-libdir)/rts -optl-Wl,--as-needed,-lHSrts_thr-ghc$GHC_VERSION" --constraint 'simplexmq +client_library' +cabal build lib:simplex-chat --ghc-options='-optl-Wl,-rpath,$ORIGIN' --ghc-options="-optl-L$(ghc --print-libdir)/rts -optl-Wl,--as-needed,-lHSrts_thr-ghc$GHC_VERSION" --constraint 'simplexmq +client_library' --constraint 'simplex-chat +client_library' cd $BUILD_DIR/build #patchelf --add-needed libHSrts_thr-ghc${GHC_VERSION}.so libHSsimplex-chat-*-inplace-ghc${GHC_VERSION}.so #patchelf --add-rpath '$ORIGIN' libHSsimplex-chat-*-inplace-ghc${GHC_VERSION}.so diff --git a/scripts/desktop/build-lib-mac.sh b/scripts/desktop/build-lib-mac.sh index 06fad69613..f234dfac25 100755 --- a/scripts/desktop/build-lib-mac.sh +++ b/scripts/desktop/build-lib-mac.sh @@ -27,10 +27,10 @@ rm -rf $BUILD_DIR if [[ "$DATABASE_BACKEND" == "postgres" ]]; then echo "Building with postgres backend..." - cabal build -f client_postgres lib:simplex-chat lib:simplex-chat --ghc-options="-optl-Wl,-rpath,@loader_path -optl-Wl,-L$GHC_LIBS_DIR/rts -optl-lHSrts_thr-ghc8.10.7 -optl-lffi" --constraint 'simplexmq +client_library' + cabal build -f client_postgres lib:simplex-chat lib:simplex-chat --ghc-options="-optl-Wl,-rpath,@loader_path -optl-Wl,-L$GHC_LIBS_DIR/rts -optl-lHSrts_thr-ghc8.10.7 -optl-lffi" --constraint 'simplexmq +client_library' --constraint 'simplex-chat +client_library' else echo "Building with sqlite backend..." - cabal build lib:simplex-chat lib:simplex-chat --ghc-options="-optl-Wl,-rpath,@loader_path -optl-Wl,-L$GHC_LIBS_DIR/rts -optl-lHSrts_thr-ghc8.10.7 -optl-lffi" --constraint 'simplexmq +client_library' + cabal build lib:simplex-chat lib:simplex-chat --ghc-options="-optl-Wl,-rpath,@loader_path -optl-Wl,-L$GHC_LIBS_DIR/rts -optl-lHSrts_thr-ghc8.10.7 -optl-lffi" --constraint 'simplexmq +client_library' --constraint 'simplex-chat +client_library' fi cd $BUILD_DIR/build diff --git a/scripts/desktop/build-lib-windows.sh b/scripts/desktop/build-lib-windows.sh index cbb886ccb3..af408d4054 100755 --- a/scripts/desktop/build-lib-windows.sh +++ b/scripts/desktop/build-lib-windows.sh @@ -51,7 +51,7 @@ echo " ghc-options: -shared -threaded -optl-L$openssl_windows_style_path -opt # Very important! Without it the build fails on linking step since the linker can't find exported symbols. # It looks like GHC bug because with such random path the build ends successfully sed -i "s/ld.lld.exe/abracadabra.exe/" `ghc --print-libdir`/settings -cabal build lib:simplex-chat --constraint 'simplexmq +client_library' +cabal build lib:simplex-chat --constraint 'simplexmq +client_library' --constraint 'simplex-chat +client_library' rm -rf apps/multiplatform/common/src/commonMain/cpp/desktop/libs/$OS-$ARCH/ rm -rf apps/multiplatform/desktop/build/cmake diff --git a/scripts/flatpak/chat.simplex.simplex.metainfo.xml b/scripts/flatpak/chat.simplex.simplex.metainfo.xml index db84b67e63..45e0f9a418 100644 --- a/scripts/flatpak/chat.simplex.simplex.metainfo.xml +++ b/scripts/flatpak/chat.simplex.simplex.metainfo.xml @@ -38,6 +38,22 @@ + + https://simplex.chat/blog/20250308-simplex-chat-v6-3-new-user-experience-safety-in-public-groups.html + +

New in v6.3.0:

+
    +
  • Mention members and get notified when mentioned.
    • +
  • Send private reports to moderators.
    • +
  • Delete, block and change role for multiple members at once
    • +
  • Faster sending messages and faster deletion.
    • +
  • Organize chats into lists to keep track of what's important.
    • +
  • Jump to found and forwarded messages.
    • +
  • Private media file names.
    • +
  • Message expiration in chats.
    • +
+ + https://simplex.chat/blog/20241210-simplex-network-v6-2-servers-by-flux-business-chats.html diff --git a/scripts/nix/sha256map.nix b/scripts/nix/sha256map.nix index 53ccda920f..41b45e9ddc 100644 --- a/scripts/nix/sha256map.nix +++ b/scripts/nix/sha256map.nix @@ -1,5 +1,5 @@ { - "https://github.com/simplex-chat/simplexmq.git"."a491a1d8780054432542611f540317a6090b9360" = "183wmraa25rxcf3b07apimsdvamccc3qx3p5rr726qzvpkvrxpab"; + "https://github.com/simplex-chat/simplexmq.git"."aace3fd2fb146097304b09ef07ff613a8b255f67" = "0iqdarkvlakk4xmrqsg62z0vhs3kwm02l8vpr383vf8q2hd7ky75"; "https://github.com/simplex-chat/hs-socks.git"."a30cc7a79a08d8108316094f8f2f82a0c5e1ac51" = "0yasvnr7g91k76mjkamvzab2kvlb1g5pspjyjn2fr6v83swjhj38"; "https://github.com/simplex-chat/direct-sqlcipher.git"."f814ee68b16a9447fbb467ccc8f29bdd3546bfd9" = "1ql13f4kfwkbaq7nygkxgw84213i0zm7c1a8hwvramayxl38dq5d"; "https://github.com/simplex-chat/sqlcipher-simple.git"."a46bd361a19376c5211f1058908fc0ae6bf42446" = "1z0r78d8f0812kxbgsm735qf6xx8lvaz27k1a0b4a2m0sshpd5gl"; diff --git a/simplex-chat.cabal b/simplex-chat.cabal index bbdd766a7b..f2b3582722 100644 --- a/simplex-chat.cabal +++ b/simplex-chat.cabal @@ -5,7 +5,7 @@ cabal-version: 1.12 -- see: https://github.com/sol/hpack name: simplex-chat -version: 6.3.0.8 +version: 6.3.1.0 category: Web, System, Services, Cryptography homepage: https://github.com/simplex-chat/simplex-chat#readme author: simplex.chat @@ -24,6 +24,11 @@ flag swift manual: True default: False +flag client_library + description: Don't build server- and CLI-related code. + manual: True + default: False + flag client_postgres description: Build with PostgreSQL instead of SQLite. manual: True @@ -33,13 +38,9 @@ library exposed-modules: Simplex.Chat Simplex.Chat.AppSettings - Simplex.Chat.Bot - Simplex.Chat.Bot.KnownContacts Simplex.Chat.Call Simplex.Chat.Controller - Simplex.Chat.Core Simplex.Chat.Files - Simplex.Chat.Help Simplex.Chat.Library.Commands Simplex.Chat.Library.Internal Simplex.Chat.Library.Subscriber @@ -78,18 +79,24 @@ library Simplex.Chat.Store.Remote Simplex.Chat.Store.Shared Simplex.Chat.Styled - Simplex.Chat.Terminal - Simplex.Chat.Terminal.Input - Simplex.Chat.Terminal.Main - Simplex.Chat.Terminal.Notification - Simplex.Chat.Terminal.Output Simplex.Chat.Types Simplex.Chat.Types.Preferences Simplex.Chat.Types.Shared Simplex.Chat.Types.UITheme Simplex.Chat.Types.Util Simplex.Chat.Util - Simplex.Chat.View + if !flag(client_library) + exposed-modules: + Simplex.Chat.Bot + Simplex.Chat.Bot.KnownContacts + Simplex.Chat.Core + Simplex.Chat.Help + Simplex.Chat.Terminal + Simplex.Chat.Terminal.Input + Simplex.Chat.Terminal.Main + Simplex.Chat.Terminal.Notification + Simplex.Chat.Terminal.Output + Simplex.Chat.View if flag(client_postgres) exposed-modules: Simplex.Chat.Options.Postgres @@ -296,6 +303,8 @@ library , text >=1.2.4.0 && <1.3 executable simplex-bot + if flag(client_library) + buildable: False main-is: Main.hs other-modules: Paths_simplex_chat @@ -313,6 +322,8 @@ executable simplex-bot cpp-options: -DdbPostgres executable simplex-bot-advanced + if flag(client_library) + buildable: False main-is: Main.hs other-modules: Paths_simplex_chat @@ -339,6 +350,8 @@ executable simplex-bot-advanced text >=1.2.4.0 && <1.3 executable simplex-broadcast-bot + if flag(client_library) + buildable: False main-is: Main.hs hs-source-dirs: apps/simplex-broadcast-bot @@ -369,6 +382,8 @@ executable simplex-broadcast-bot text >=1.2.4.0 && <1.3 executable simplex-chat + if flag(client_library) + buildable: False main-is: Main.hs other-modules: Server @@ -400,6 +415,8 @@ executable simplex-chat text >=1.2.4.0 && <1.3 executable simplex-directory-service + if flag(client_library) + buildable: False main-is: Main.hs hs-source-dirs: apps/simplex-directory-service @@ -408,6 +425,7 @@ executable simplex-directory-service StrictData other-modules: Directory.BlockedWords + Directory.Captcha Directory.Events Directory.Options Directory.Search @@ -446,6 +464,8 @@ executable simplex-directory-service , text >=1.2.4.0 && <1.3 test-suite simplex-chat-test + if flag(client_library) + buildable: False type: exitcode-stdio-1.0 main-is: Test.hs other-modules: @@ -475,6 +495,7 @@ test-suite simplex-chat-test Broadcast.Bot Broadcast.Options Directory.BlockedWords + Directory.Captcha Directory.Events Directory.Options Directory.Search diff --git a/src/Simplex/Chat/Call.hs b/src/Simplex/Chat/Call.hs index 3b1f28dd27..18398ecb3e 100644 --- a/src/Simplex/Chat/Call.hs +++ b/src/Simplex/Chat/Call.hs @@ -23,10 +23,10 @@ import Data.Text (Text) import Data.Time.Clock (UTCTime) import Simplex.Chat.Options.DB (FromField (..), ToField (..)) import Simplex.Chat.Types (Contact, ContactId, User) -import Simplex.Messaging.Agent.Store.DB (Binary (..)) +import Simplex.Messaging.Agent.Store.DB (Binary (..), fromTextField_) import qualified Simplex.Messaging.Crypto as C import Simplex.Messaging.Encoding.String -import Simplex.Messaging.Parsers (defaultJSON, dropPrefix, enumJSON, fromTextField_, fstToLower, singleFieldJSON) +import Simplex.Messaging.Parsers (defaultJSON, dropPrefix, enumJSON, fstToLower, singleFieldJSON) import Simplex.Messaging.Util (decodeJSON, encodeJSON) data Call = Call diff --git a/src/Simplex/Chat/Messages.hs b/src/Simplex/Chat/Messages.hs index ae88bc796b..79d416dee5 100644 --- a/src/Simplex/Chat/Messages.hs +++ b/src/Simplex/Chat/Messages.hs @@ -50,10 +50,11 @@ import Simplex.Chat.Types.Preferences import Simplex.Chat.Types.Shared import Simplex.Chat.Types.Util (textParseJSON) import Simplex.Messaging.Agent.Protocol (AgentMsgId, MsgMeta (..), MsgReceiptStatus (..)) +import Simplex.Messaging.Agent.Store.DB (fromTextField_) import Simplex.Messaging.Crypto.File (CryptoFile (..)) import qualified Simplex.Messaging.Crypto.File as CF import Simplex.Messaging.Encoding.String -import Simplex.Messaging.Parsers (defaultJSON, dropPrefix, enumJSON, fromTextField_, parseAll, sumTypeJSON) +import Simplex.Messaging.Parsers (defaultJSON, dropPrefix, enumJSON, parseAll, sumTypeJSON) import Simplex.Messaging.Protocol (BlockingInfo, MsgBody) import Simplex.Messaging.Util (eitherToMaybe, safeDecodeUtf8, (<$?>)) diff --git a/src/Simplex/Chat/Operators.hs b/src/Simplex/Chat/Operators.hs index 15135d4e9e..5240460c9c 100644 --- a/src/Simplex/Chat/Operators.hs +++ b/src/Simplex/Chat/Operators.hs @@ -50,8 +50,9 @@ import Simplex.Chat.Options.DB (FromField (..), ToField (..)) import Simplex.Chat.Types (User) import Simplex.Chat.Types.Util (textParseJSON) import Simplex.Messaging.Agent.Env.SQLite (ServerCfg (..), ServerRoles (..), allRoles) +import Simplex.Messaging.Agent.Store.DB (fromTextField_) import Simplex.Messaging.Encoding.String -import Simplex.Messaging.Parsers (defaultJSON, dropPrefix, fromTextField_, sumTypeJSON) +import Simplex.Messaging.Parsers (defaultJSON, dropPrefix, sumTypeJSON) import Simplex.Messaging.Protocol (AProtocolType (..), ProtoServerWithAuth (..), ProtocolServer (..), ProtocolType (..), ProtocolTypeI, SProtocolType (..), UserProtocol) import Simplex.Messaging.Transport.Client (TransportHost (..)) import Simplex.Messaging.Util (atomicModifyIORef'_, safeDecodeUtf8) diff --git a/src/Simplex/Chat/Protocol.hs b/src/Simplex/Chat/Protocol.hs index 566cd04003..2281c1aefa 100644 --- a/src/Simplex/Chat/Protocol.hs +++ b/src/Simplex/Chat/Protocol.hs @@ -29,7 +29,6 @@ import qualified Data.Aeson.KeyMap as JM import qualified Data.Aeson.TH as JQ import qualified Data.Aeson.Types as JT import qualified Data.Attoparsec.ByteString.Char8 as A -import Data.Bifunctor (first) import Data.ByteString.Char8 (ByteString) import qualified Data.ByteString.Char8 as B import Data.ByteString.Internal (c2w, w2c) @@ -54,11 +53,12 @@ import Simplex.Chat.Types import Simplex.Chat.Types.Preferences import Simplex.Chat.Types.Shared import Simplex.Messaging.Agent.Protocol (VersionSMPA, pqdrSMPAgentVersion) +import Simplex.Messaging.Agent.Store.DB (fromTextField_) import qualified Simplex.Messaging.Agent.Store.DB as DB import Simplex.Messaging.Compression (Compressed, compress1, decompress1) import Simplex.Messaging.Encoding import Simplex.Messaging.Encoding.String -import Simplex.Messaging.Parsers (defaultJSON, dropPrefix, fromTextField_, fstToLower, parseAll, sumTypeJSON, taggedObjectJSON) +import Simplex.Messaging.Parsers (defaultJSON, dropPrefix, fstToLower, parseAll, sumTypeJSON, taggedObjectJSON) import Simplex.Messaging.Protocol (MsgBody) import Simplex.Messaging.Util (decodeJSON, eitherToMaybe, encodeJSON, safeDecodeUtf8, (<$?>)) import Simplex.Messaging.Version hiding (version) diff --git a/src/Simplex/Chat/Types.hs b/src/Simplex/Chat/Types.hs index 5ba7565611..2bc9e725e5 100644 --- a/src/Simplex/Chat/Types.hs +++ b/src/Simplex/Chat/Types.hs @@ -52,11 +52,11 @@ import Simplex.Chat.Types.Util import Simplex.FileTransfer.Description (FileDigest) import Simplex.FileTransfer.Types (RcvFileId, SndFileId) import Simplex.Messaging.Agent.Protocol (ACorrId, AEventTag (..), AEvtTag (..), ConnId, ConnectionMode (..), ConnectionRequestUri, InvitationId, SAEntity (..), UserId) -import Simplex.Messaging.Agent.Store.DB (Binary (..)) +import Simplex.Messaging.Agent.Store.DB (Binary (..), blobFieldDecoder, fromTextField_) import Simplex.Messaging.Crypto.File (CryptoFileArgs (..)) import Simplex.Messaging.Crypto.Ratchet (PQEncryption (..), PQSupport, pattern PQEncOff) import Simplex.Messaging.Encoding.String -import Simplex.Messaging.Parsers (blobFieldDecoder, defaultJSON, dropPrefix, enumJSON, fromTextField_, sumTypeJSON) +import Simplex.Messaging.Parsers (defaultJSON, dropPrefix, enumJSON, sumTypeJSON) import Simplex.Messaging.Util (safeDecodeUtf8, (<$?>)) import Simplex.Messaging.Version import Simplex.Messaging.Version.Internal diff --git a/src/Simplex/Chat/Types/Preferences.hs b/src/Simplex/Chat/Types/Preferences.hs index 2e704c5cf5..63d80657dc 100644 --- a/src/Simplex/Chat/Types/Preferences.hs +++ b/src/Simplex/Chat/Types/Preferences.hs @@ -32,8 +32,9 @@ import qualified Data.Text as T import GHC.Records.Compat import Simplex.Chat.Options.DB (FromField (..), ToField (..)) import Simplex.Chat.Types.Shared +import Simplex.Messaging.Agent.Store.DB (blobFieldDecoder, fromTextField_) import Simplex.Messaging.Encoding.String -import Simplex.Messaging.Parsers (blobFieldDecoder, defaultJSON, dropPrefix, enumJSON, fromTextField_, sumTypeJSON) +import Simplex.Messaging.Parsers (defaultJSON, dropPrefix, enumJSON, sumTypeJSON) import Simplex.Messaging.Util (decodeJSON, encodeJSON, safeDecodeUtf8, (<$?>)) data ChatFeature diff --git a/src/Simplex/Chat/Types/Shared.hs b/src/Simplex/Chat/Types/Shared.hs index e22610cfe5..4c6adae4e9 100644 --- a/src/Simplex/Chat/Types/Shared.hs +++ b/src/Simplex/Chat/Types/Shared.hs @@ -7,8 +7,8 @@ import Data.Aeson (FromJSON (..), ToJSON (..)) import qualified Data.Attoparsec.ByteString.Char8 as A import qualified Data.ByteString.Char8 as B import Simplex.Chat.Options.DB (FromField (..), ToField (..)) +import Simplex.Messaging.Agent.Store.DB (blobFieldDecoder) import Simplex.Messaging.Encoding.String -import Simplex.Messaging.Parsers (blobFieldDecoder) import Simplex.Messaging.Util ((<$?>)) data GroupMemberRole diff --git a/src/Simplex/Chat/Types/UITheme.hs b/src/Simplex/Chat/Types/UITheme.hs index f2512a3a5a..d0f23a7307 100644 --- a/src/Simplex/Chat/Types/UITheme.hs +++ b/src/Simplex/Chat/Types/UITheme.hs @@ -15,8 +15,9 @@ import Data.Maybe (fromMaybe) import Data.Text (Text) import Simplex.Chat.Options.DB (FromField (..), ToField (..)) import Simplex.Chat.Types.Util +import Simplex.Messaging.Agent.Store.DB (fromTextField_) import Simplex.Messaging.Encoding.String -import Simplex.Messaging.Parsers (defaultJSON, dropPrefix, enumJSON, fromTextField_) +import Simplex.Messaging.Parsers (defaultJSON, dropPrefix, enumJSON) import Simplex.Messaging.Util (decodeJSON, encodeJSON) data UITheme = UITheme diff --git a/tests/Bots/DirectoryTests.hs b/tests/Bots/DirectoryTests.hs index 31b36159fd..e9db100e8d 100644 --- a/tests/Bots/DirectoryTests.hs +++ b/tests/Bots/DirectoryTests.hs @@ -13,6 +13,7 @@ import Control.Concurrent (forkIO, killThread, threadDelay) import Control.Exception (finally) import Control.Monad (forM_, when) import qualified Data.Text as T +import Directory.Captcha import qualified Directory.Events as DE import Directory.Options import Directory.Service @@ -65,6 +66,8 @@ directoryServiceTests = do it "should list user's groups" testListUserGroups describe "store log" $ do it "should restore directory service state" testRestoreDirectory + describe "captcha" $ do + it "should accept some incorrect spellings" testCaptcha directoryProfile :: Profile directoryProfile = Profile {displayName = "SimpleX-Directory", fullName = "", image = Nothing, contactLink = Nothing, preferences = Nothing} @@ -974,6 +977,19 @@ testRestoreDirectory ps = do cath #> "@SimpleX-Directory security" groupFoundN' 2 cath "security" +testCaptcha :: HasCallStack => TestParams -> IO () +testCaptcha _ps = do + let captcha = "23456789ABCDEFGHIJKLMNOPQRSTUVWXYZabdefghijkmnpqrty" + matchCaptchaStr captcha captcha `shouldBe` True + matchCaptchaStr captcha "23456789ABcDEFGH1JKLMNoPQRsTuvwxYzabdefghijkmnpqrty" `shouldBe` True + matchCaptchaStr "23456789ABcDEFGH1JKLMNoPQRsTuvwxYzabdefghijkmnpqrty" captcha `shouldBe` True + matchCaptchaStr "OOIICPSUVWXZ" "OOIICPSUVWXZ" `shouldBe` True + matchCaptchaStr "OOIICPSUVWXZ" "0o1lcpsuvwxz" `shouldBe` True + matchCaptchaStr "0o1lcpsuvwxz" "OOIICPSUVWXZ" `shouldBe` True + matchCaptchaStr "OOIICPSUVWXZ" "" `shouldBe` False + matchCaptchaStr "OOIICPSUVWXZ" "0o1lcpsuvwx" `shouldBe` False + matchCaptchaStr "OOIICPSUVWXZ" "0o1lcpsuvwxzz" `shouldBe` False + listGroups :: HasCallStack => TestCC -> TestCC -> TestCC -> IO () listGroups superUser bob cath = do bob #> "@SimpleX-Directory /list" diff --git a/tests/ChatClient.hs b/tests/ChatClient.hs index 28f7befebe..c41c155697 100644 --- a/tests/ChatClient.hs +++ b/tests/ChatClient.hs @@ -54,8 +54,8 @@ import Simplex.Messaging.Crypto.Ratchet (supportedE2EEncryptVRange) import qualified Simplex.Messaging.Crypto.Ratchet as CR import Simplex.Messaging.Protocol (srvHostnamesSMPClientVersion) import Simplex.Messaging.Server (runSMPServerBlocking) -import Simplex.Messaging.Server.Env.STM (ServerConfig (..), StartOptions (..), defaultMessageExpiration, defaultIdleQueueInterval, defaultNtfExpiration, defaultInactiveClientExpiration) -import Simplex.Messaging.Server.MsgStore.Types (AMSType (..), SMSType (..)) +import Simplex.Messaging.Server.Env.STM (AServerStoreCfg (..), ServerConfig (..), ServerStoreCfg (..), StartOptions (..), StorePaths (..), defaultMessageExpiration, defaultIdleQueueInterval, defaultNtfExpiration, defaultInactiveClientExpiration) +import Simplex.Messaging.Server.MsgStore.Types (SQSType (..), SMSType (..)) import Simplex.Messaging.Transport import Simplex.Messaging.Transport.Server (ServerCredentials (..), defaultTransportServerConfig) import Simplex.Messaging.Version @@ -476,14 +476,12 @@ smpServerCfg = ServerConfig { transports = [(serverPort, transport @TLS, False)], tbqSize = 1, - msgStoreType = AMSType SMSMemory, msgQueueQuota = 16, maxJournalMsgCount = 24, maxJournalStateLines = 4, queueIdBytes = 12, msgIdBytes = 6, - storeLogFile = Nothing, - storeMsgsFile = Nothing, + serverStoreCfg = ASSCfg SQSMemory SMSMemory $ SSCMemory Nothing, storeNtfsFile = Nothing, allowNewQueues = True, -- server password is disabled as otherwise v1 tests fail @@ -518,9 +516,12 @@ smpServerCfg = allowSMPProxy = True, serverClientConcurrency = 16, information = Nothing, - startOptions = StartOptions False False + startOptions = StartOptions {maintenance = False, compactLog = False, skipWarnings = False, confirmMigrations = MCYesUp} } +persistentServerStoreCfg :: FilePath -> AServerStoreCfg +persistentServerStoreCfg tmp = ASSCfg SQSMemory SMSMemory $ SSCMemory $ Just StorePaths {storeLogFile = tmp <> "/smp-server-store.log", storeMsgsFile = Just $ tmp <> "/smp-server-messages.log"} + withSmpServer :: IO () -> IO () withSmpServer = withSmpServer' smpServerCfg diff --git a/tests/ChatTests/Direct.hs b/tests/ChatTests/Direct.hs index 998835b867..5c18088b76 100644 --- a/tests/ChatTests/Direct.hs +++ b/tests/ChatTests/Direct.hs @@ -270,8 +270,7 @@ testRetryConnecting ps = testChatCfgOpts2 cfg' opts' aliceProfile bobProfile tes smpServerCfg { transports = [("7003", transport @TLS, False)], msgQueueQuota = 2, - storeLogFile = Just $ tmp <> "/smp-server-store.log", - storeMsgsFile = Just $ tmp <> "/smp-server-messages.log" + serverStoreCfg = persistentServerStoreCfg tmp } fastRetryInterval = defaultReconnectInterval {initialInterval = 50000} -- same as in agent tests cfg' = @@ -329,8 +328,7 @@ testRetryConnectingClientTimeout ps = do smpServerCfg { transports = [("7003", transport @TLS, False)], msgQueueQuota = 2, - storeLogFile = Just $ tmp <> "/smp-server-store.log", - storeMsgsFile = Just $ tmp <> "/smp-server-messages.log" + serverStoreCfg = persistentServerStoreCfg tmp } fastRetryInterval = defaultReconnectInterval {initialInterval = 50000} -- same as in agent tests cfg' = diff --git a/tests/ChatTests/Groups.hs b/tests/ChatTests/Groups.hs index 2e99de4a82..73a8735f63 100644 --- a/tests/ChatTests/Groups.hs +++ b/tests/ChatTests/Groups.hs @@ -1988,8 +1988,7 @@ testSharedMessageBody ps = serverCfg' = smpServerCfg { transports = [("7003", transport @TLS, False)], - storeLogFile = Just $ tmp <> "/smp-server-store.log", - storeMsgsFile = Just $ tmp <> "/smp-server-messages.log" + serverStoreCfg = persistentServerStoreCfg tmp } opts' = testOpts @@ -2045,8 +2044,7 @@ testSharedBatchBody ps = serverCfg' = smpServerCfg { transports = [("7003", transport @TLS, False)], - storeLogFile = Just $ tmp <> "/smp-server-store.log", - storeMsgsFile = Just $ tmp <> "/smp-server-messages.log" + serverStoreCfg = persistentServerStoreCfg tmp } opts' = testOpts diff --git a/tests/ChatTests/Profiles.hs b/tests/ChatTests/Profiles.hs index 5d2b9f5ba8..72d502392e 100644 --- a/tests/ChatTests/Profiles.hs +++ b/tests/ChatTests/Profiles.hs @@ -311,8 +311,7 @@ testRetryAcceptingViaContactLink ps = testChatCfgOpts2 cfg' opts' aliceProfile b smpServerCfg { transports = [("7003", transport @TLS, False)], msgQueueQuota = 2, - storeLogFile = Just $ tmp <> "/smp-server-store.log", - storeMsgsFile = Just $ tmp <> "/smp-server-messages.log" + serverStoreCfg = persistentServerStoreCfg tmp } fastRetryInterval = defaultReconnectInterval {initialInterval = 50000} -- same as in agent tests cfg' = diff --git a/website/src/_includes/blog_previews/20250308.html b/website/src/_includes/blog_previews/20250308.html new file mode 100644 index 0000000000..ab55ecdbca --- /dev/null +++ b/website/src/_includes/blog_previews/20250308.html @@ -0,0 +1,12 @@ +

v6.3 is released:

+ +
    +
  • Preventing spam and abuse in public groups.
    • +
  • Group improvements: mention other members and improved performance.
    • +
  • Better chat navigation: organize chats into lists and jump to found and forwarded messages.
    • +
  • Privacy and security improvements: chat retention period and private media file names.
    • +
+ +

Also, we added Catalan interface language, thanks to our users and Weblate.

+ +

The last but not the least - server builds are now reproducible!