mirror of
https://github.com/simplex-chat/simplex-chat.git
synced 2026-11-04 06:32:59 +00:00
core, ui: resize images in the core for sharper previews and images (#7941)
* WIP: Haskell impl of resizeImageToStrSize * WIP: add FFI export wrapper * WIP: switch ios resizeImageToStrSizeSync to backend * WIP: fix resizeImageToStrSizeSync * WIP: add JNI wrappers * Add non-base64 resize and cWriteImage * WIP: add chat_write_image * fix resizeToStr * fixes * fix syntax, raise jpeg quality back * add chatWriteImage * Fix cfArgs in tests * Use jpeg-turbo encoder * add nix pins * bump jpeg-turbo * add CI deps * add encoding funs to android lib * bump jpeg-turbo for 20.04 fix * jpeg-turbo fix for 8.10 * android fixes * Bump jpeg-turbo * set static flag for jpeg-turbo * Fix nix for aarch64/android * Fix android C api * Fix resize in android * Bump hs-libjpeg-turbo * target correct override * override android lib from android pkgs * add macos build tweaks * bump hs-jpeg-turbo for 8.10 fixes * more build fixes * make ios compile * Always use pkgconfig for libjpeg-turbo * WIP: debug libjpeg in ubuntu pkg-config CI * Add pkg-config to dockerfiles * Add libturbojpeg to dockerfiles * Use -dev version of libturbojpeg in dockerfiles * Install pkg-config in windows CI * Disable flag for custom libturbojpeg linking options Workarounds relocation error on x86_64-linux with gcc: /usr/bin/ld: /usr/lib/gcc/x86_64-linux-gnu/13/../../../x86_64-linux-gnu/libturbojpeg.a(turbojpeg.c.o): relocation R_X86_64_TPOFF32 against `errStr' can not be used when making a shared object; recompile with -fPIC /usr/bin/ld: failed to set dynamic section sizes: bad value * WIP: debug libjpeg in windows pkg-config CI * WIP: debug libjpeg package in windows CI * core: image resize module and buffer FFI * ci: link libjpeg-turbo statically on all platforms * ui: resize images in the core * core: rename image resize functions to chat_resize_image and chat_resize_image_uri * ci: restore whitespace in build.yml * ci: build libjpeg-turbo 3.1.4.1 from a submodule * core: hs-jpeg-turbo from github.com/simplex-chat * nodejs: resize images * python: resize images * core: hs-jpeg-turbo from gitlab.com, the GitHub import lacks LFS objects * core: generate /img previews * ci: link libjpeg-turbo on Windows without pkg-config * ci: build libjpeg-turbo on Windows with the GHC toolchain * docs: plan for image resizing in the core * docs: update image resizing plan * ios: prevent crash when wallpaper image cannot be resized * android, desktop: raise an error when an image cannot be resized * docs: update spec references for image resizing * docker: build the CLI image from a clone without submodules * docs: apply review fixes to image resizing plan * docker: build the CLI from a git worktree with the submodule present * docs: apply second review to image resizing plan * docs: point image resizing plan at the rewritten commit * core: choose image format by transparency in chat_resize_image * nodejs: resize images to the app sizes * python: resize images to the app sizes * nodejs, python: describe the core modules as used for image resizing * docs: add license notices of the image libraries * ci: describe the avatar method without attributing it to the apps * ci, ios, nix: remove the libjpeg-turbo build outside cabal * python: name resize_image in the core module text * nodejs, python: describe resizing to the app image sizes * core: refuse images larger than the supported size * core: resize to JPEG images with a side above 65500 pixels * docs: record image resizing decisions for the bundled libjpeg-turbo * core, docs: build libjpeg-turbo and stb from the fork submodules * docs: record the fork pins in the image resizing plan * core: resize images to at most 4320 pixels per side, refuse above 16384 * docs: record the image side limits in the image resizing plan * core: pin the rebased hs-jpeg-turbo fork * core: pin the hs-jpeg-turbo fork without package.yaml * docs: plan scaled JPEG decoding and EXIF orientation of images * docs: copy the JuicyPixels-stbir licence of the pinned fork * core: pin the hs-jpeg-turbo fork with scaled JPEG decoding * core: decode JPEG at the display size, up to 65500 pixels per side * core: rotate JPEG images by their EXIF orientation * docs: refer the image plan to the JPEG decoding code * docs: record the approved image processing decisions * core: limit JPEG decoding memory to 1024 MB outside iOS and Android * docs: name JPEG decoding in the libjpeg-turbo notice * docs: plan image resizing from files and the decisions of 26 Oct * core: scale images in one Catmull-Rom pass with sharpening * docs: plan image sizes at a fixed quality * core: resize images at a fixed quality to fill the size limit * core: resize images from a file path or a file descriptor * ios, android, desktop: resize picked images from their files * core: encode resized images with mozjpeg * tests: check descriptor closing by the file it refers to * core: raise JPEG quality only for a full-size image below 97% of the limit * ci: name the mozjpeg submodule in the Windows cleanup comments * docs: refer the image plan to the current code * docs: refer the image plan to the code merged with master * core: export every image resizing function from each library * core: close the file descriptor when its handle cannot be created * docs: count the C sources of mozjpeg with the arithmetic decoder * core: use the reviewed hs-jpeg-turbo and JuicyPixels-stbir forks * ci: build the armv7a Android core library on request * core: round image heights half up in the size search * ios: describe the file image resizing functions in SimpleX.h * ci: pick, preview and send images in the Claude app tests * core: decode PNG images with libspng and refuse other formats * core: encode PNG with libspng, keeping the smaller filter choice * ci: pick, preview and send images in the scripted app smoke tests * flake: build the PNG decoder package in nix builds * ios: remove the copy of a picked image after sending it * core: refuse image files larger than the decode memory limit * core: scale a JPEG image to its largest size once per resize * core: bisect JPEG quality without a step function * core: reduce the memory used to sharpen and to decode PNG images * android, desktop: pass the image format to the core as an integer * android, desktop: fuse the preview conditions of picked media * android, desktop, ci: check that the core resized the smoke test images * python: remove the docstrings of the explicit image resizing functions * docs: add the second review fixes to the image plan * ci: tap the iOS attachment button by its accessibility label * core: keep the colour profiles of resized images * docs: add the colour profiles to the image plan * ci: wait up to 30 s for the new group screens in the iOS smoke test * ci: pick the Android smoke test images from their own photo folders * ci: tap the iOS photo picker by position, as its tree is not shown * ci: pick the test card and the logo by their iOS picker positions * core: update JuicyPixels-stbir with the stb_image_resize licence terms * core: write a fixed Display P3 profile for Display P3 images only * core: do not keep the input image in memory for its colour profile * core: read the JPEG segments once for the orientation and the profile * core: read an image file into one buffer of its size * docs: add the third review fixes to the image plan * core: resize images with Display P3 or no profile, not profile bytes * ci: check the pixel sizes of the images sent in the iOS smoke test * core: update hs-jpeg-turbo with PNG chunk limits and encoder fixes * docs: list Compact ICC Profiles, libspng and miniz in the dependencies * android, desktop: reword the picked media preview comment * docs: refer the image plan to the code merged with master * ci: render the iOS snapshots before the images case * ci: dismiss the snapshot tests' camera alert before the iOS images case * core: update hs-jpeg-turbo with tests and the JuicyPixels-spng synopsis * core: pin hs-jpeg-turbo and JuicyPixels-stbir in simplex-chat repositories * update ios core library * core, ui: accept only JPEG and PNG image data * core: update JuicyPixels-stbir * docs: plan for the iOS copy of picked images --------- Co-authored-by: IC Rainbow <aenor.realm@gmail.com> Co-authored-by: Evgeny Poberezkin <evgeny@poberezkin.com> Co-authored-by: Alexandre Esteves <alexfmpe@proton.me>
This commit is contained in:
118 files changed
+4227
-348
No files matched your search
@@ -36,6 +36,8 @@ struct Library {
|
||||
chat_read_file_fn chat_read_file = nullptr;
|
||||
chat_encrypt_file_fn chat_encrypt_file = nullptr;
|
||||
chat_decrypt_file_fn chat_decrypt_file = nullptr;
|
||||
chat_resize_image_fn chat_resize_image = nullptr;
|
||||
chat_resize_image_uri_fn chat_resize_image_uri = nullptr;
|
||||
};
|
||||
|
||||
static Library lib;
|
||||
@@ -136,6 +138,45 @@ static bool RequireLoaded(Env env) {
|
||||
return false;
|
||||
}
|
||||
|
||||
template <typename F>
|
||||
static bool RequireExport(Env env, F fn, const std::string& name, const std::string& feature) {
|
||||
if (fn != nullptr) return true;
|
||||
Error::New(env, "loaded libsimplex does not export " + name + "; " + feature + " needs a newer libsimplex").ThrowAsJavaScriptException();
|
||||
return false;
|
||||
}
|
||||
|
||||
static bool GetInt32Arg(Env env, const Value& value, const std::string& name, int& result) {
|
||||
Number number = value.As<Number>();
|
||||
result = number.Int32Value();
|
||||
if (static_cast<double>(result) == number.DoubleValue()) return true;
|
||||
RangeError::New(env, "Expected 32-bit integer " + name).ThrowAsJavaScriptException();
|
||||
return false;
|
||||
}
|
||||
|
||||
struct BufferArg {
|
||||
char* data;
|
||||
int len;
|
||||
};
|
||||
|
||||
static bool GetBufferArg(Env env, const Value& value, BufferArg& result) {
|
||||
size_t len;
|
||||
if (value.IsArrayBuffer()) {
|
||||
ArrayBuffer ab = value.As<ArrayBuffer>();
|
||||
result.data = static_cast<char*>(ab.Data());
|
||||
len = ab.ByteLength();
|
||||
} else {
|
||||
TypedArray view = value.As<TypedArray>();
|
||||
result.data = static_cast<char*>(view.ArrayBuffer().Data()) + view.ByteOffset();
|
||||
len = view.ByteLength();
|
||||
}
|
||||
if (len > static_cast<size_t>(INT_MAX)) {
|
||||
RangeError::New(env, "Buffer is too large").ThrowAsJavaScriptException();
|
||||
return false;
|
||||
}
|
||||
result.len = static_cast<int>(len);
|
||||
return true;
|
||||
}
|
||||
|
||||
Value Load(const CallbackInfo& args) {
|
||||
Env env = args.Env();
|
||||
if (args.Length() < 1 || !args[0].IsString()) {
|
||||
@@ -173,6 +214,8 @@ Value Load(const CallbackInfo& args) {
|
||||
return env.Undefined();
|
||||
}
|
||||
Resolve(h, "chat_migrate_init_queue", l.chat_migrate_init_queue);
|
||||
Resolve(h, "chat_resize_image", l.chat_resize_image);
|
||||
Resolve(h, "chat_resize_image_uri", l.chat_resize_image_uri);
|
||||
Resolve(h, "hs_thread_done", l.hs_thread_done);
|
||||
lib = l;
|
||||
haskell_init();
|
||||
@@ -527,10 +570,7 @@ Value ChatMigrateInit(const CallbackInfo& args) {
|
||||
Value ChatMigrateInitQueue(const CallbackInfo& args) {
|
||||
Env env = args.Env();
|
||||
if (!RequireLoaded(env)) return env.Undefined();
|
||||
if (lib.chat_migrate_init_queue == nullptr) {
|
||||
Error::New(env, "loaded libsimplex does not export chat_migrate_init_queue; queue size needs a newer libsimplex").ThrowAsJavaScriptException();
|
||||
return env.Undefined();
|
||||
}
|
||||
if (!RequireExport(env, lib.chat_migrate_init_queue, "chat_migrate_init_queue", "queue size")) return env.Undefined();
|
||||
if (args.Length() < 4 || !args[0].IsString() || !args[1].IsString() || !args[2].IsString() || !args[3].IsNumber()) {
|
||||
TypeError::New(env, "Expected three string arguments and number").ThrowAsJavaScriptException();
|
||||
return env.Undefined();
|
||||
@@ -539,12 +579,8 @@ Value ChatMigrateInitQueue(const CallbackInfo& args) {
|
||||
std::string path = args[0].As<String>().Utf8Value();
|
||||
std::string key = args[1].As<String>().Utf8Value();
|
||||
std::string confirm = args[2].As<String>().Utf8Value();
|
||||
Number queue_size_arg = args[3].As<Number>();
|
||||
int queue_size = queue_size_arg.Int32Value();
|
||||
if (static_cast<double>(queue_size) != queue_size_arg.DoubleValue()) {
|
||||
RangeError::New(env, "Expected 32-bit integer queue size").ThrowAsJavaScriptException();
|
||||
return env.Undefined();
|
||||
}
|
||||
int queue_size;
|
||||
if (!GetInt32Arg(env, args[3], "queue size", queue_size)) return env.Undefined();
|
||||
|
||||
Function cb;
|
||||
Promise promise = CreatePromiseAndCallback(env, cb);
|
||||
@@ -657,27 +693,14 @@ Value ChatWriteFile(const CallbackInfo& args) {
|
||||
|
||||
chat_ctrl ctrl = FromChatCtrlBigInt(args[0]);
|
||||
std::string path = args[1].As<String>().Utf8Value();
|
||||
char* data;
|
||||
size_t len;
|
||||
if (args[2].IsArrayBuffer()) {
|
||||
ArrayBuffer ab = args[2].As<ArrayBuffer>();
|
||||
data = static_cast<char*>(ab.Data());
|
||||
len = ab.ByteLength();
|
||||
} else {
|
||||
TypedArray view = args[2].As<TypedArray>();
|
||||
data = static_cast<char*>(view.ArrayBuffer().Data()) + view.ByteOffset();
|
||||
len = view.ByteLength();
|
||||
}
|
||||
if (len > static_cast<size_t>(INT_MAX)) {
|
||||
RangeError::New(env, "Buffer is too large").ThrowAsJavaScriptException();
|
||||
return env.Undefined();
|
||||
}
|
||||
BufferArg buffer;
|
||||
if (!GetBufferArg(env, args[2], buffer)) return env.Undefined();
|
||||
|
||||
Function cb;
|
||||
Promise promise = CreatePromiseAndCallback(env, cb);
|
||||
|
||||
auto execute_fn = [ctrl, path, data, len](ResultAsyncWorker* worker) {
|
||||
char* c_res = lib.chat_write_file(ctrl, path.c_str(), data, static_cast<int>(len));
|
||||
auto execute_fn = [ctrl, path, buffer](ResultAsyncWorker* worker) {
|
||||
char* c_res = lib.chat_write_file(ctrl, path.c_str(), buffer.data, buffer.len);
|
||||
HandleCResult(worker, c_res, "chat_write_file");
|
||||
};
|
||||
|
||||
@@ -785,6 +808,66 @@ Value ChatDecryptFile(const CallbackInfo& args) {
|
||||
return promise;
|
||||
}
|
||||
|
||||
Value ChatResizeImage(const CallbackInfo& args) {
|
||||
Env env = args.Env();
|
||||
if (!RequireLoaded(env)) return env.Undefined();
|
||||
if (!RequireExport(env, lib.chat_resize_image, "chat_resize_image", "image resizing")) return env.Undefined();
|
||||
if (args.Length() < 3 || !(args[0].IsArrayBuffer() || args[0].IsTypedArray()) || !args[1].IsNumber() || !args[2].IsNumber()) {
|
||||
TypeError::New(env, "Expected ArrayBuffer or Uint8Array, number (maxSize), number (png)").ThrowAsJavaScriptException();
|
||||
return env.Undefined();
|
||||
}
|
||||
|
||||
BufferArg image;
|
||||
if (!GetBufferArg(env, args[0], image)) return env.Undefined();
|
||||
int max_size;
|
||||
if (!GetInt32Arg(env, args[1], "maxSize", max_size)) return env.Undefined();
|
||||
int png;
|
||||
if (!GetInt32Arg(env, args[2], "png", png)) return env.Undefined();
|
||||
|
||||
Function cb;
|
||||
Promise promise = CreatePromiseAndCallback(env, cb);
|
||||
|
||||
auto execute_fn = [image, max_size, png](ResultAsyncWorker* worker) {
|
||||
char* c_res = lib.chat_resize_image(image.data, image.len, max_size, png);
|
||||
HandleCResult(worker, c_res, "chat_resize_image");
|
||||
};
|
||||
|
||||
ResultAsyncWorker* worker = new ResultAsyncWorker(cb, std::move(execute_fn));
|
||||
worker->KeepAlive(args[0].As<Object>());
|
||||
worker->Queue();
|
||||
|
||||
return promise;
|
||||
}
|
||||
|
||||
Value ChatResizeImageUri(const CallbackInfo& args) {
|
||||
Env env = args.Env();
|
||||
if (!RequireLoaded(env)) return env.Undefined();
|
||||
if (!RequireExport(env, lib.chat_resize_image_uri, "chat_resize_image_uri", "image resizing")) return env.Undefined();
|
||||
if (args.Length() < 2 || !(args[0].IsArrayBuffer() || args[0].IsTypedArray()) || !args[1].IsNumber()) {
|
||||
TypeError::New(env, "Expected ArrayBuffer or Uint8Array, number (maxSize)").ThrowAsJavaScriptException();
|
||||
return env.Undefined();
|
||||
}
|
||||
|
||||
BufferArg image;
|
||||
if (!GetBufferArg(env, args[0], image)) return env.Undefined();
|
||||
int max_size;
|
||||
if (!GetInt32Arg(env, args[1], "maxSize", max_size)) return env.Undefined();
|
||||
|
||||
Function cb;
|
||||
Promise promise = CreatePromiseAndCallback(env, cb);
|
||||
|
||||
auto execute_fn = [image, max_size](ResultAsyncWorker* worker) {
|
||||
char* c_res = lib.chat_resize_image_uri(image.data, image.len, max_size);
|
||||
HandleCResult(worker, c_res, "chat_resize_image_uri");
|
||||
};
|
||||
|
||||
ResultAsyncWorker* worker = new ResultAsyncWorker(cb, std::move(execute_fn));
|
||||
worker->KeepAlive(args[0].As<Object>());
|
||||
worker->Queue();
|
||||
|
||||
return promise;
|
||||
}
|
||||
|
||||
Object Init(Env env, Object exports) {
|
||||
exports.Set("load", Function::New(env, Load));
|
||||
auto* receivers = new Receivers();
|
||||
@@ -804,6 +887,8 @@ Object Init(Env env, Object exports) {
|
||||
exports.Set("chat_read_file", Function::New(env, ChatReadFile));
|
||||
exports.Set("chat_encrypt_file", Function::New(env, ChatEncryptFile));
|
||||
exports.Set("chat_decrypt_file", Function::New(env, ChatDecryptFile));
|
||||
exports.Set("chat_resize_image", Function::New(env, ChatResizeImage));
|
||||
exports.Set("chat_resize_image_uri", Function::New(env, ChatResizeImageUri));
|
||||
return exports;
|
||||
}
|
||||
|
||||
|
||||
@@ -35,4 +35,10 @@ typedef char *(*chat_encrypt_file_fn)(chat_ctrl ctrl, const char *fromPath, cons
|
||||
// chat_decrypt_file returns null-terminated string with the error message
|
||||
typedef char *(*chat_decrypt_file_fn)(const char *fromPath, const char *key, const char *nonce, const char *toPath);
|
||||
|
||||
// chat_resize_image returns base64 of at most maxSize bytes of PNG (png > 0), JPEG (png == 0), or PNG for an image with transparency, otherwise JPEG (png < 0), or an empty string
|
||||
typedef char *(*chat_resize_image_fn)(const char *data, int len, int maxSize, int png);
|
||||
|
||||
// chat_resize_image_uri returns a data URI of at most maxSize characters, or an empty string
|
||||
typedef char *(*chat_resize_image_uri_fn)(const char *data, int len, int maxSize);
|
||||
|
||||
#endif /* simplex_h */
|
||||
Reference in new issue
Block a user