From 6b429fcde9939571f777dc5182a418a900112cb8 Mon Sep 17 00:00:00 2001 From: Rory& Date: Sat, 11 Apr 2026 15:41:18 +0200 Subject: [PATCH] Functioning CDN-CS workers --- .../FilesystemFileSource.cs | 21 +++-- .../IFileSource.cs | 2 +- .../ProxyFileSource.cs | 5 +- .../DiscordImageResizeService.cs | 91 +++++++++++++++++++ .../admin-api/Spacebar.Cdn.Worker/Program.cs | 16 +++- .../Spacebar.Cdn.Worker/emmatests.sh | 7 ++ .../admin-api/Spacebar.Cdn.Worker/outputs.nix | 13 ++- extra/admin-api/Spacebar.Cdn.Worker/test.html | 91 +++++++++++++++++++ extra/admin-api/Spacebar.Cdn/Mimes.cs | 3 +- nix/lib/buildSpacebarDotnetModule.nix | 81 ++++++++++------- 10 files changed, 283 insertions(+), 47 deletions(-) create mode 100644 extra/admin-api/Spacebar.Cdn.Worker/DiscordImageResizeService.cs create mode 100755 extra/admin-api/Spacebar.Cdn.Worker/emmatests.sh create mode 100644 extra/admin-api/Spacebar.Cdn.Worker/test.html diff --git a/extra/admin-api/Interop/Spacebar.Interop.Cdn.Abstractions/FilesystemFileSource.cs b/extra/admin-api/Interop/Spacebar.Interop.Cdn.Abstractions/FilesystemFileSource.cs index a3f0a7d87..8a67dfe38 100644 --- a/extra/admin-api/Interop/Spacebar.Interop.Cdn.Abstractions/FilesystemFileSource.cs +++ b/extra/admin-api/Interop/Spacebar.Interop.Cdn.Abstractions/FilesystemFileSource.cs @@ -1,20 +1,27 @@ -using ArcaneLibs; - namespace Spacebar.Interop.Cdn.Abstractions; public class FilesystemFileSource(string baseUrl) : IFileSource { public string BaseUrl => baseUrl; - public async Task Init(CancellationToken? cancellationToken = null) { + private string GetPathSafe(string path) { + var p = Path.Join(baseUrl, path); + if (!p.StartsWith(baseUrl)) throw new UnauthorizedAccessException("Invalid path: " + p); + Console.WriteLine($"Resolved \"safe\" path: {p}"); + return p; + } + + public async Task Init(CancellationToken? cancellationToken = null) { foreach (var dir in new[] { "avatars", "banners", "icons", "stickers", "emojis" }) { var fullPath = Path.Join(baseUrl, dir); if (!Directory.Exists(fullPath)) Directory.CreateDirectory(fullPath); } + + return this; } public async Task GetFile(string path, CancellationToken? cancellationToken = null) { - await using var rs = File.OpenRead(Path.Join(baseUrl, path)); + await using var rs = File.OpenRead(GetPathSafe(path)); var ms = new MemoryStream(); await rs.CopyToAsync(ms); return new() { @@ -24,11 +31,11 @@ public class FilesystemFileSource(string baseUrl) : IFileSource { } public Task FileExists(string path, CancellationToken? cancellationToken = null) { - return Task.FromResult(File.Exists(Path.Join(baseUrl, path))); + return Task.FromResult(File.Exists(GetPathSafe(path))); } public async Task WriteFile(string path, Stream stream) { - var fullPath = Path.Join(baseUrl, path); + var fullPath = GetPathSafe(path); // Console.WriteLine($"Writing file to {fullPath}... "); if (!Directory.Exists(Path.GetDirectoryName(fullPath)!)) Directory.CreateDirectory(Path.GetDirectoryName(fullPath)!); @@ -43,6 +50,6 @@ public class FilesystemFileSource(string baseUrl) : IFileSource { // return Mimes.GetMime(mic.First().Format); // } public Task DirectoryExists(string path) { - return Task.FromResult(Directory.Exists(Path.Join(baseUrl, path))); + return Task.FromResult(Directory.Exists(GetPathSafe(path))); } } \ No newline at end of file diff --git a/extra/admin-api/Interop/Spacebar.Interop.Cdn.Abstractions/IFileSource.cs b/extra/admin-api/Interop/Spacebar.Interop.Cdn.Abstractions/IFileSource.cs index a69764d5b..41f27c3f5 100644 --- a/extra/admin-api/Interop/Spacebar.Interop.Cdn.Abstractions/IFileSource.cs +++ b/extra/admin-api/Interop/Spacebar.Interop.Cdn.Abstractions/IFileSource.cs @@ -2,7 +2,7 @@ namespace Spacebar.Interop.Cdn.Abstractions; public interface IFileSource { public string BaseUrl { get; } - public Task Init(CancellationToken? cancellationToken = null); + public Task Init(CancellationToken? cancellationToken = null); public Task GetFile(string path, CancellationToken? cancellationToken = null); public Task FileExists(string path, CancellationToken? cancellationToken = null); public Task WriteFile(string path, Stream stream); diff --git a/extra/admin-api/Interop/Spacebar.Interop.Cdn.Abstractions/ProxyFileSource.cs b/extra/admin-api/Interop/Spacebar.Interop.Cdn.Abstractions/ProxyFileSource.cs index 6a84bd4df..b0990706d 100644 --- a/extra/admin-api/Interop/Spacebar.Interop.Cdn.Abstractions/ProxyFileSource.cs +++ b/extra/admin-api/Interop/Spacebar.Interop.Cdn.Abstractions/ProxyFileSource.cs @@ -18,7 +18,9 @@ public class ProxyFileSource(string baseUrl) : IFileSource { public string BaseUrl => baseUrl; - public async Task Init(CancellationToken? cancellationToken = null) { } + public async Task Init(CancellationToken? cancellationToken = null) { + return this; + } public async Task GetFile(string path, CancellationToken? cancellationToken = null) { var res = await _cache.GetOrAdd(path, async () => { @@ -44,6 +46,7 @@ public class ProxyFileSource(string baseUrl) : IFileSource { await using var s = await res.Content.ReadAsStreamAsync(); Console.WriteLine($"Got {res.StatusCode}: ({res.Content.Headers.ContentLength}b of {res.Content.Headers.ContentType})\n{s.ReadToEnd().AsString()}"); } + return res.IsSuccessStatusCode; } diff --git a/extra/admin-api/Spacebar.Cdn.Worker/DiscordImageResizeService.cs b/extra/admin-api/Spacebar.Cdn.Worker/DiscordImageResizeService.cs new file mode 100644 index 000000000..64e726602 --- /dev/null +++ b/extra/admin-api/Spacebar.Cdn.Worker/DiscordImageResizeService.cs @@ -0,0 +1,91 @@ +using System.Runtime.Serialization; +using ImageMagick; +using Microsoft.AspNetCore.Mvc; + +namespace Spacebar.AdminApi.TestClient.Services.Services; + +public class DiscordImageResizeParams { + public uint? Size { get; set; } + public DiscordImageResizeQuality Quality { get; set; } = DiscordImageResizeQuality.High; + public bool KeepAspectRatio { get; set; } = true; + public bool Passthrough { get; set; } = true; + public bool Animated { get; set; } = true; + + public bool SpacebarAllowUpscale { get; set; } = false; + public bool SpacebarOptimiseGif { get; set; } = true; + + public string ToSerializedName() { + return $"{(Animated ? "a_" : "")}{Size}px_{Quality.ToString()}_u.{SpacebarAllowUpscale}_o.{SpacebarOptimiseGif}"; + } +} + +public static class HttpRequestExtensions { + extension(HttpRequest request) { + public DiscordImageResizeParams GetResizeParams() { + return new() { + Size = request.Query.ContainsKey("size") && uint.TryParse(request.Query["size"], out uint size) ? size : null, + Quality = request.Query.ContainsKey("quality") && Enum.TryParse(request.Query["quality"], true, out var quality) + ? quality + : DiscordImageResizeQuality.High, + KeepAspectRatio = !request.Query.ContainsKey("keepAspectRatio") || !bool.TryParse(request.Query["keepAspectRatio"], out bool kar) || kar, + Passthrough = request.Query.ContainsKey("passthrough") && bool.TryParse(request.Query["passthrough"], out bool pt) && pt, + Animated = request.Query.ContainsKey("animated") && bool.TryParse(request.Query["animated"], out bool an) && an, + SpacebarAllowUpscale = request.Query.ContainsKey("allowUpscale") && bool.TryParse(request.Query["allowUpscale"], out bool au) && au, + SpacebarOptimiseGif = request.Query.ContainsKey("optimiseGif") && bool.TryParse(request.Query["optimiseGif"], out bool og) && og, + }; + } + } + + extension (HttpResponse response) { + public void SetSuccessCacheHeader() { + int cacheDuration = (int)TimeSpan.FromHours(6).TotalSeconds; + response.Headers.CacheControl = $"public, max-age={cacheDuration}, s-maxage={cacheDuration}, immutable"; + } + + public void SetFailureCacheHeader() { + int cacheDuration = (int)TimeSpan.FromMinutes(5).TotalSeconds; + response.Headers.CacheControl = $"public, max-age={cacheDuration}, s-maxage={cacheDuration}, immutable"; + } + } +} + +public enum DiscordImageResizeQuality { + [EnumMember(Value = "low")] Low, + [EnumMember(Value = "high")] High, + [EnumMember(Value = "lossless")] Lossless +} + +public class DiscordImageResizeService { + //(PixelArtDetectionService pads) { + public MagickImageCollection Apply(MagickImageCollection img, DiscordImageResizeParams resizeParams) { + if (resizeParams.Passthrough) return img; + if (img.First().Format == MagickFormat.Gif) { + Console.WriteLine("Coalescing gif for resize"); + img.Coalesce(); + } + + if (resizeParams.Size.HasValue) { + if (resizeParams.Size > 4096) + resizeParams.Size = 4096; + + if (img.Max(x => Math.Max(x.Height, x.Width)) > resizeParams.Size || resizeParams.SpacebarAllowUpscale) { + Parallel.ForEach(img, new ParallelOptions() { MaxDegreeOfParallelism = 8 }, frame => { + if (resizeParams.Size.HasValue) { + uint oldWidth = frame.Width, oldHeight = frame.Height; + // pads.IsPixelArt(frame) + frame.Resize(resizeParams.Size.Value, resizeParams.Size.Value, + resizeParams.Quality == DiscordImageResizeQuality.Low ? FilterType.Point : FilterType.Gaussian); + Console.WriteLine($"Resized frame from {oldWidth}x{oldHeight} to {frame.Width}x{frame.Height}: {img.IndexOf(frame)}/{img.Count}"); + } + }); + } + } + + if (img.First().Format == MagickFormat.Gif && resizeParams.SpacebarOptimiseGif) { + Console.WriteLine("Optimizing gif after resize"); + img.OptimizePlus(); + } + + return img; + } +} \ No newline at end of file diff --git a/extra/admin-api/Spacebar.Cdn.Worker/Program.cs b/extra/admin-api/Spacebar.Cdn.Worker/Program.cs index c75151b50..a722e666b 100644 --- a/extra/admin-api/Spacebar.Cdn.Worker/Program.cs +++ b/extra/admin-api/Spacebar.Cdn.Worker/Program.cs @@ -2,8 +2,11 @@ using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.Globalization; using ArcaneLibs; +using ArcaneLibs.Extensions.Streams; using ImageMagick; +using Microsoft.AspNetCore.Mvc; using Spacebar.AdminApi.TestClient.Services.Helpers; +using Spacebar.AdminApi.TestClient.Services.Services; using Spacebar.Cdn.Worker; using Spacebar.Interop.Cdn.Abstractions; @@ -57,8 +60,8 @@ MagickNET.Initialize(); // builder.WebHost.ConfigureKestrel(opts => opts.ListenUnixSocket(Environment.GetEnvironmentVariable("SOCKET_PATH")!)); -builder.Services.AddSingleton(new FilesystemFileSource(Environment.GetEnvironmentVariable("STORAGE_PATH") ?? throw new InvalidOperationException("STORAGE_PATH not set!"))); -// builder.Services.AddSingleton(); +builder.Services.AddSingleton(await new FilesystemFileSource(Environment.GetEnvironmentVariable("STORAGE_PATH") ?? throw new InvalidOperationException("STORAGE_PATH not set!")).Init()); +builder.Services.AddSingleton(); builder.Services.AddControllers(); var app = builder.Build(); @@ -87,6 +90,13 @@ app.MapGet("/defaultAvatar/_{bg:length(6)}_{fg:length(6)}.{ext}", async (HttpCon return Results.File(res, Mimes.GetMime(Mimes.GetFormatForExtension(ext))); }); -app.MapGet("/*", async (ctx) => { }); +app.MapGet("/scale/{*path}", async (HttpContext ctx, IFileSource ifs, DiscordImageResizeService dirs, string path) => { + var f = await ifs.GetFile(path); + f.Stream.Position = 0; + var res = dirs.Apply(new MagickImageCollection(f.Stream), ctx.Request.GetResizeParams()); + await ctx.Response.StartAsync(); + await res.WriteAsync(ctx.Response.Body); + await ctx.Response.CompleteAsync(); +}); app.Run(); \ No newline at end of file diff --git a/extra/admin-api/Spacebar.Cdn.Worker/emmatests.sh b/extra/admin-api/Spacebar.Cdn.Worker/emmatests.sh new file mode 100755 index 000000000..88a97ae1c --- /dev/null +++ b/extra/admin-api/Spacebar.Cdn.Worker/emmatests.sh @@ -0,0 +1,7 @@ +#! /usr/bin/env sh +#for s in {16,32,48,64,96,128,256,512,4096}; do time curl http://localhost:5000/scale/avatars/1006598230156341276/cc0a23b479ef23641480d53a49f1b3db\?size=$s\&quality\=high | kitty icat; done +for i in {1..6}; do + for s in {16,32,48,64,96,128,256,512,1024,2048,4096}; do + echo curl http://localhost:5000/defaultAvatar/${i}.png\?size=$s\&quality\=low #| kitty icat + done +done diff --git a/extra/admin-api/Spacebar.Cdn.Worker/outputs.nix b/extra/admin-api/Spacebar.Cdn.Worker/outputs.nix index ad451920a..12d51d9d6 100644 --- a/extra/admin-api/Spacebar.Cdn.Worker/outputs.nix +++ b/extra/admin-api/Spacebar.Cdn.Worker/outputs.nix @@ -28,7 +28,7 @@ flake-utils.lib.eachSystem flake-utils.lib.allSystems ( makeWorkerPackage = arch: variant: buildSpacebarDotnetModule { - name = "Spacebar.Cdn.Worker-${variant}.${arch}"; + name = "Spacebar.Cdn.Worker.${variant}.${arch}"; nugetDeps = ./deps.${variant}.${arch}.json; projectFile = "Spacebar.Cdn.Worker.${variant}.${arch}.csproj"; srcRoot = ./.; @@ -36,6 +36,17 @@ flake-utils.lib.eachSystem flake-utils.lib.allSystems ( projectReferences = [ proj.Spacebar-Interop-Cdn-Abstractions ]; + nativeBuildInputs = [ + pkgs.autoPatchelfHook + pkgs.libgcc.lib + ]; + runtimeDependencies = [ (lib.getLib pkgs.libgcc) ]; + #makeWrapperArgs = [ + # "--prefix" + # "LD_PRELOAD" + # ":" + # "${pkgs.libgcc}/lib/libgomp.so" + #]; }; in { diff --git a/extra/admin-api/Spacebar.Cdn.Worker/test.html b/extra/admin-api/Spacebar.Cdn.Worker/test.html new file mode 100644 index 000000000..fbd2f9e9f --- /dev/null +++ b/extra/admin-api/Spacebar.Cdn.Worker/test.html @@ -0,0 +1,91 @@ +1 s=16 q=low +1 s=32 q=low +1 s=48 q=low +1 s=64 q=low +1 s=96 q=low +1 s=128 q=low +1 s=256 q=low +1 s=512 q=low +1 s=1024 q=low +1 s=2048 q=low +1 s=4096 q=low +2 s=16 q=low +2 s=32 q=low +2 s=48 q=low +2 s=64 q=low +2 s=96 q=low +2 s=128 q=low +2 s=256 q=low +2 s=512 q=low +2 s=1024 q=low +2 s=2048 q=low +2 s=4096 q=low +3 s=16 q=low +3 s=32 q=low +3 s=48 q=low +3 s=64 q=low +3 s=96 q=low +3 s=128 q=low +3 s=256 q=low +3 s=512 q=low +3 s=1024 q=low +3 s=2048 q=low +3 s=4096 q=low +4 s=16 q=low +4 s=32 q=low +4 s=48 q=low +4 s=64 q=low +4 s=96 q=low +4 s=128 q=low +4 s=256 q=low +4 s=512 q=low +4 s=1024 q=low +4 s=2048 q=low +4 s=4096 q=low +5 s=16 q=low +5 s=32 q=low +5 s=48 q=low +5 s=64 q=low +5 s=96 q=low +5 s=128 q=low +5 s=256 q=low +5 s=512 q=low +5 s=1024 q=low +5 s=2048 q=low +5 s=4096 q=low +6 s=16 q=low +6 s=32 q=low +6 s=48 q=low +6 s=64 q=low +6 s=96 q=low +6 s=128 q=low +6 s=256 q=low +6 s=512 q=low +6 s=1024 q=low +6 s=2048 q=low +6 s=4096 q=low + + +a s=16 q=low +a s=32 q=low +a s=48 q=low +a s=64 q=low +a s=96 q=low +a s=128 q=low +a s=256 q=low +a s=512 q=low +a s=1024 q=low +a s=2048 q=low +a s=4096 q=low + +a s=16 q=high +a s=32 q=high +a s=48 q=high +a s=64 q=high +a s=96 q=high +a s=128 q=high +a s=256 q=high +a s=512 q=high +a s=1024 q=high +a s=2048 q=high +a s=4096 q=high diff --git a/extra/admin-api/Spacebar.Cdn/Mimes.cs b/extra/admin-api/Spacebar.Cdn/Mimes.cs index 273df2dd1..efef1e808 100644 --- a/extra/admin-api/Spacebar.Cdn/Mimes.cs +++ b/extra/admin-api/Spacebar.Cdn/Mimes.cs @@ -31,8 +31,7 @@ public static class Mimes { or "emf" // some microsoft meta format, windows only or "mpr" // Magick Persistent Registry - basically a resident in-memory image ) throw new AccessViolationException("Disallowed extension: " + extension); - - MagickFormat + var matchingFormat = Enum.GetNames().FirstOrDefault(f => f.ToLower() == extension); if (string.IsNullOrWhiteSpace(matchingFormat)) throw new InvalidEnumArgumentException("Unknown format: " + extension); return Enum.TryParse(matchingFormat, out MagickFormat fmt) ? fmt : throw new InvalidEnumArgumentException("Unknown format: " + extension); diff --git a/nix/lib/buildSpacebarDotnetModule.nix b/nix/lib/buildSpacebarDotnetModule.nix index c76ab4e29..7b7c02aed 100644 --- a/nix/lib/buildSpacebarDotnetModule.nix +++ b/nix/lib/buildSpacebarDotnetModule.nix @@ -8,37 +8,54 @@ useAppHost ? null, packNupkg ? true, srcRoot ? ./., + ... }@args: -pkgs.buildDotnetModule rec { - inherit - projectReferences - nugetDeps - projectFile - runtimeId - useAppHost - srcRoot - packNupkg - ; +pkgs.buildDotnetModule (pkgs.lib.recursiveUpdate + ( + rec { + inherit + projectReferences + nugetDeps + projectFile + runtimeId + useAppHost + srcRoot + packNupkg + ; - pname = "${name}"; - version = "1.0.0-" + rVersion; - dotnetPackFlags = [ - "--include-symbols" - "--include-source" - "--version-suffix ${rVersion}" - ]; - # dotnetFlags = [ "-v:n" ]; # diag - dotnet-sdk = pkgs.dotnet-sdk_10; - dotnet-runtime = pkgs.dotnet-aspnetcore_10; - src = pkgs.lib.cleanSource srcRoot; - meta = with pkgs.lib; { - description = "Spacebar Server, Typescript Edition (C# extensions)"; - homepage = "https://github.com/spacebarchat/server"; - license = licenses.agpl3Plus; - maintainers = with maintainers; [ RorySys ]; - mainProgram = name; - }; -} -// { - __nugetDeps = args.nugetDeps; -} + pname = "${name}"; + version = "1.0.0-" + rVersion; + dotnetPackFlags = [ + "--include-symbols" + "--include-source" + "--version-suffix ${rVersion}" + ]; + # dotnetFlags = [ "-v:n" ]; # diag + dotnet-sdk = pkgs.dotnet-sdk_10; + dotnet-runtime = pkgs.dotnet-aspnetcore_10; + src = pkgs.lib.cleanSource srcRoot; + meta = with pkgs.lib; { + description = "Spacebar Server, Typescript Edition (C# extensions)"; + homepage = "https://github.com/spacebarchat/server"; + license = licenses.agpl3Plus; + maintainers = with maintainers; [ RorySys ]; + mainProgram = name; + }; + } + // { + __nugetDeps = args.nugetDeps; + } + ) + ( + builtins.removeAttrs args [ + "name" + "nugetDeps" + "projectReferences" + "projectFile" + "runtimeId" + "useAppHost" + "packNupkg" + "srcRoot" + ] + ) +) \ No newline at end of file