Files
simplex-chat/apps/ios/Shared/Views/Helpers/ImagePicker.swift
T
fbca2cae81 core, ui: resize images in the core for sharper previews and images (#7941)
* WIP: Haskell impl of resizeImageToStrSize

* WIP: add FFI export wrapper

* WIP: switch ios resizeImageToStrSizeSync to backend

* WIP: fix resizeImageToStrSizeSync

* WIP: add JNI wrappers

* Add non-base64 resize and cWriteImage

* WIP: add chat_write_image

* fix resizeToStr

* fixes

* fix syntax, raise jpeg quality back

* add chatWriteImage

* Fix cfArgs in tests

* Use jpeg-turbo encoder

* add nix pins

* bump jpeg-turbo

* add CI deps

* add encoding funs to android lib

* bump jpeg-turbo for 20.04 fix

* jpeg-turbo fix for 8.10

* android fixes

* Bump jpeg-turbo

* set static flag for jpeg-turbo

* Fix nix for aarch64/android

* Fix android C api

* Fix resize in android

* Bump hs-libjpeg-turbo

* target correct override

* override android lib from android pkgs

* add macos build tweaks

* bump hs-jpeg-turbo for 8.10 fixes

* more build fixes

* make ios compile

* Always use pkgconfig for libjpeg-turbo

* WIP: debug libjpeg in ubuntu pkg-config CI

* Add pkg-config to dockerfiles

* Add libturbojpeg to dockerfiles

* Use -dev version of libturbojpeg in dockerfiles

* Install pkg-config in windows CI

* Disable flag for custom libturbojpeg linking options

Workarounds relocation error on x86_64-linux with gcc:
  /usr/bin/ld: /usr/lib/gcc/x86_64-linux-gnu/13/../../../x86_64-linux-gnu/libturbojpeg.a(turbojpeg.c.o): relocation R_X86_64_TPOFF32 against `errStr' can not be used when making a shared object; recompile with -fPIC
  /usr/bin/ld: failed to set dynamic section sizes: bad value

* WIP: debug libjpeg in windows pkg-config CI

* WIP: debug libjpeg package in windows CI

* core: image resize module and buffer FFI

* ci: link libjpeg-turbo statically on all platforms

* ui: resize images in the core

* core: rename image resize functions to chat_resize_image and chat_resize_image_uri

* ci: restore whitespace in build.yml

* ci: build libjpeg-turbo 3.1.4.1 from a submodule

* core: hs-jpeg-turbo from github.com/simplex-chat

* nodejs: resize images

* python: resize images

* core: hs-jpeg-turbo from gitlab.com, the GitHub import lacks LFS objects

* core: generate /img previews

* ci: link libjpeg-turbo on Windows without pkg-config

* ci: build libjpeg-turbo on Windows with the GHC toolchain

* docs: plan for image resizing in the core

* docs: update image resizing plan

* ios: prevent crash when wallpaper image cannot be resized

* android, desktop: raise an error when an image cannot be resized

* docs: update spec references for image resizing

* docker: build the CLI image from a clone without submodules

* docs: apply review fixes to image resizing plan

* docker: build the CLI from a git worktree with the submodule present

* docs: apply second review to image resizing plan

* docs: point image resizing plan at the rewritten commit

* core: choose image format by transparency in chat_resize_image

* nodejs: resize images to the app sizes

* python: resize images to the app sizes

* nodejs, python: describe the core modules as used for image resizing

* docs: add license notices of the image libraries

* ci: describe the avatar method without attributing it to the apps

* ci, ios, nix: remove the libjpeg-turbo build outside cabal

* python: name resize_image in the core module text

* nodejs, python: describe resizing to the app image sizes

* core: refuse images larger than the supported size

* core: resize to JPEG images with a side above 65500 pixels

* docs: record image resizing decisions for the bundled libjpeg-turbo

* core, docs: build libjpeg-turbo and stb from the fork submodules

* docs: record the fork pins in the image resizing plan

* core: resize images to at most 4320 pixels per side, refuse above 16384

* docs: record the image side limits in the image resizing plan

* core: pin the rebased hs-jpeg-turbo fork

* core: pin the hs-jpeg-turbo fork without package.yaml

* docs: plan scaled JPEG decoding and EXIF orientation of images

* docs: copy the JuicyPixels-stbir licence of the pinned fork

* core: pin the hs-jpeg-turbo fork with scaled JPEG decoding

* core: decode JPEG at the display size, up to 65500 pixels per side

* core: rotate JPEG images by their EXIF orientation

* docs: refer the image plan to the JPEG decoding code

* docs: record the approved image processing decisions

* core: limit JPEG decoding memory to 1024 MB outside iOS and Android

* docs: name JPEG decoding in the libjpeg-turbo notice

* docs: plan image resizing from files and the decisions of 26 Oct

* core: scale images in one Catmull-Rom pass with sharpening

* docs: plan image sizes at a fixed quality

* core: resize images at a fixed quality to fill the size limit

* core: resize images from a file path or a file descriptor

* ios, android, desktop: resize picked images from their files

* core: encode resized images with mozjpeg

* tests: check descriptor closing by the file it refers to

* core: raise JPEG quality only for a full-size image below 97% of the limit

* ci: name the mozjpeg submodule in the Windows cleanup comments

* docs: refer the image plan to the current code

* docs: refer the image plan to the code merged with master

* core: export every image resizing function from each library

* core: close the file descriptor when its handle cannot be created

* docs: count the C sources of mozjpeg with the arithmetic decoder

* core: use the reviewed hs-jpeg-turbo and JuicyPixels-stbir forks

* ci: build the armv7a Android core library on request

* core: round image heights half up in the size search

* ios: describe the file image resizing functions in SimpleX.h

* ci: pick, preview and send images in the Claude app tests

* core: decode PNG images with libspng and refuse other formats

* core: encode PNG with libspng, keeping the smaller filter choice

* ci: pick, preview and send images in the scripted app smoke tests

* flake: build the PNG decoder package in nix builds

* ios: remove the copy of a picked image after sending it

* core: refuse image files larger than the decode memory limit

* core: scale a JPEG image to its largest size once per resize

* core: bisect JPEG quality without a step function

* core: reduce the memory used to sharpen and to decode PNG images

* android, desktop: pass the image format to the core as an integer

* android, desktop: fuse the preview conditions of picked media

* android, desktop, ci: check that the core resized the smoke test images

* python: remove the docstrings of the explicit image resizing functions

* docs: add the second review fixes to the image plan

* ci: tap the iOS attachment button by its accessibility label

* core: keep the colour profiles of resized images

* docs: add the colour profiles to the image plan

* ci: wait up to 30 s for the new group screens in the iOS smoke test

* ci: pick the Android smoke test images from their own photo folders

* ci: tap the iOS photo picker by position, as its tree is not shown

* ci: pick the test card and the logo by their iOS picker positions

* core: update JuicyPixels-stbir with the stb_image_resize licence terms

* core: write a fixed Display P3 profile for Display P3 images only

* core: do not keep the input image in memory for its colour profile

* core: read the JPEG segments once for the orientation and the profile

* core: read an image file into one buffer of its size

* docs: add the third review fixes to the image plan

* core: resize images with Display P3 or no profile, not profile bytes

* ci: check the pixel sizes of the images sent in the iOS smoke test

* core: update hs-jpeg-turbo with PNG chunk limits and encoder fixes

* docs: list Compact ICC Profiles, libspng and miniz in the dependencies

* android, desktop: reword the picked media preview comment

* docs: refer the image plan to the code merged with master

* ci: render the iOS snapshots before the images case

* ci: dismiss the snapshot tests' camera alert before the iOS images case

* core: update hs-jpeg-turbo with tests and the JuicyPixels-spng synopsis

* core: pin hs-jpeg-turbo and JuicyPixels-stbir in simplex-chat repositories

* update ios core library

* core, ui: accept only JPEG and PNG image data

* core: update JuicyPixels-stbir

* docs: plan for the iOS copy of picked images

---------

Co-authored-by: IC Rainbow <aenor.realm@gmail.com>
Co-authored-by: Evgeny Poberezkin <evgeny@poberezkin.com>
Co-authored-by: Alexandre Esteves <alexfmpe@proton.me>
2026-10-30 12:14:51 +00:00

223 lines
8.6 KiB
Swift

//
// ImagePicker.swift
// SimpleX
//
// Created by Evgeny on 23/03/2022.
// Copyright © 2022 SimpleX Chat. All rights reserved.
//
import SwiftUI
import PhotosUI
import SwiftyGif
import SimpleXChat
struct LibraryImagePicker: View {
@Binding var image: UIImage?
var didFinishPicking: (_ didSelectImage: Bool) async -> Void
@State var mediaAdded = false
var body: some View {
LibraryMediaListPicker(addMedia: addMedia, selectionLimit: 1, didFinishPicking: didFinishPicking)
}
private func addMedia(_ content: UploadContent) async {
if mediaAdded { return }
await MainActor.run {
mediaAdded = true
image = content.uiImage
}
}
}
struct LibraryMediaListPicker: UIViewControllerRepresentable {
typealias UIViewControllerType = PHPickerViewController
var addMedia: (_ content: UploadContent) async -> Void
var selectionLimit: Int
var filter: PHPickerFilter = .any(of: [.images, .videos])
var finishedPreprocessing: () -> Void = {}
var didFinishPicking: (_ didSelectItems: Bool) async -> Void
class Coordinator: PHPickerViewControllerDelegate {
let parent: LibraryMediaListPicker
let dispatchQueue = DispatchQueue(label: "chat.simplex.app.LibraryMediaListPicker")
init(_ parent: LibraryMediaListPicker) {
self.parent = parent
}
func picker(_ picker: PHPickerViewController, didFinishPicking results: [PHPickerResult]) {
Task {
await parent.didFinishPicking(!results.isEmpty)
if results.isEmpty { return }
for r in results {
await loadItem(r.itemProvider)
}
parent.finishedPreprocessing()
}
}
private func loadItem(_ p: NSItemProvider) async {
logger.debug("LibraryMediaListPicker result")
if p.hasItemConformingToTypeIdentifier(UTType.movie.identifier) {
if let video = await loadVideo(p) {
await self.parent.addMedia(video)
logger.debug("LibraryMediaListPicker: added video")
}
} else if p.hasItemConformingToTypeIdentifier(UTType.data.identifier) {
if let img = await loadImageData(p) {
await self.parent.addMedia(img)
logger.debug("LibraryMediaListPicker: added image")
}
} else if p.canLoadObject(ofClass: UIImage.self) {
if let img = await loadImage(p) {
await self.parent.addMedia(.simpleImage(image: img))
logger.debug("LibraryMediaListPicker: added image")
}
}
}
private func loadImageData(_ p: NSItemProvider) async -> UploadContent? {
await withCheckedContinuation { cont in
loadFileURL(p, type: imageFileType(p)) { url in
if let url = url {
let img = UploadContent.loadImageFileFromURL(url: url)
cont.resume(returning: img)
} else {
cont.resume(returning: nil)
}
}
}
}
private func imageFileType(_ p: NSItemProvider) -> UTType {
let original = p.hasItemConformingToTypeIdentifier(UTType.png.identifier) || p.hasItemConformingToTypeIdentifier(UTType.gif.identifier)
return original || !p.hasItemConformingToTypeIdentifier(UTType.jpeg.identifier) ? UTType.data : UTType.jpeg
}
private func loadImage(_ p: NSItemProvider) async -> UIImage? {
await withCheckedContinuation { cont in
p.loadObject(ofClass: UIImage.self) { obj, err in
if let err = err {
logger.error("LibraryMediaListPicker result image error: \(err.localizedDescription)")
cont.resume(returning: nil)
} else {
cont.resume(returning: obj as? UIImage)
}
}
}
}
private func loadVideo(_ p: NSItemProvider) async -> UploadContent? {
await withCheckedContinuation { cont in
loadFileURL(p, type: UTType.movie) { url in
if let url = url {
let tempUrl = URL(fileURLWithPath: generateNewFileName(getTempFilesDirectory().path + "/" + "rawvideo", url.pathExtension, fullPath: true))
let convertedVideoUrl = URL(fileURLWithPath: generateNewFileName(getTempFilesDirectory().path + "/" + "video", "mp4", fullPath: true))
do {
// logger.debug("LibraryMediaListPicker copyItem \(url) to \(tempUrl)")
try FileManager.default.copyItem(at: url, to: tempUrl)
} catch let err {
logger.error("LibraryMediaListPicker copyItem error: \(err.localizedDescription)")
return cont.resume(returning: nil)
}
Task {
let success = await makeVideoQualityLower(tempUrl, outputUrl: convertedVideoUrl)
try? FileManager.default.removeItem(at: tempUrl)
if success {
_ = ChatModel.shared.filesToDelete.insert(convertedVideoUrl)
let video = UploadContent.loadVideoFromURL(url: convertedVideoUrl)
return cont.resume(returning: video)
}
try? FileManager.default.removeItem(at: convertedVideoUrl)
cont.resume(returning: nil)
}
}
}
}
}
private func loadFileURL(_ p: NSItemProvider, type: UTType, completion: @escaping (URL?) -> Void) {
p.loadFileRepresentation(forTypeIdentifier: type.identifier) { url, err in
if let err = err {
logger.error("LibraryMediaListPicker loadFileURL error: \(err.localizedDescription)")
completion(nil)
} else {
completion(url)
}
}
}
}
func makeCoordinator() -> Coordinator {
Coordinator(self)
}
func makeUIViewController(context: Context) -> PHPickerViewController {
var config = PHPickerConfiguration()
config.filter = filter
config.selectionLimit = selectionLimit
config.selection = .ordered
config.preferredAssetRepresentationMode = .current
let controller = PHPickerViewController(configuration: config)
controller.delegate = context.coordinator
return controller
}
func updateUIViewController(_ uiViewController: PHPickerViewController, context: Context) {
}
}
struct CameraImageListPicker: View {
@Binding var images: [UploadContent]
@State var image: UIImage?
var body: some View {
CameraImagePicker(image: $image)
.onChange(of: image) { img in
if let img = img {
images = [UploadContent.simpleImage(image: img)]
} else {
images = []
}
}
}
}
struct CameraImagePicker: UIViewControllerRepresentable {
@Environment(\.presentationMode) var presentationMode
@Binding var image: UIImage?
class Coordinator: NSObject, UINavigationControllerDelegate, UIImagePickerControllerDelegate {
let parent: CameraImagePicker
init(_ parent: CameraImagePicker) {
self.parent = parent
}
func imagePickerController(_ picker: UIImagePickerController,
didFinishPickingMediaWithInfo info: [UIImagePickerController.InfoKey: Any]) {
if let uiImage = info[.originalImage] as? UIImage {
parent.image = uiImage
}
parent.presentationMode.wrappedValue.dismiss()
}
}
func makeCoordinator() -> Coordinator {
Coordinator(self)
}
func makeUIViewController(context: UIViewControllerRepresentableContext<CameraImagePicker>) -> UIImagePickerController {
let picker = UIImagePickerController()
picker.sourceType = .camera
picker.allowsEditing = false
picker.delegate = context.coordinator
return picker
}
func updateUIViewController(_ uiViewController: UIImagePickerController, context: UIViewControllerRepresentableContext<CameraImagePicker>) {
}
}