From c24ef4c8a316f236b98c72f92dbf47b3bec75e0d Mon Sep 17 00:00:00 2001 From: Rory& Date: Sun, 12 Apr 2026 21:04:22 +0200 Subject: [PATCH] CDN-CS: support passing a bin path as cdn worker -> forks a new proc --- .../DiscordImageResizeService.cs | 20 +++++--- .../admin-api/Spacebar.Cdn.Worker/Program.cs | 6 ++- .../ImagesAndStickersController.cs | 2 +- .../Controllers/StaticAssetController.cs | 2 +- .../Controllers/UserController.cs | 2 +- .../Spacebar.Cdn/Services/CdnWorkerService.cs | 48 ++++++++++++++++--- 6 files changed, 63 insertions(+), 17 deletions(-) diff --git a/extra/admin-api/Spacebar.Cdn.Worker/DiscordImageResizeService.cs b/extra/admin-api/Spacebar.Cdn.Worker/DiscordImageResizeService.cs index 51ee1aa5e..1295baddf 100644 --- a/extra/admin-api/Spacebar.Cdn.Worker/DiscordImageResizeService.cs +++ b/extra/admin-api/Spacebar.Cdn.Worker/DiscordImageResizeService.cs @@ -57,11 +57,15 @@ public enum DiscordImageResizeQuality { public class DiscordImageResizeService { //(PixelArtDetectionService pads) { - public MagickImageCollection Apply(MagickImageCollection img, DiscordImageResizeParams resizeParams) { + public async Task Apply(MagickImageCollection img, DiscordImageResizeParams resizeParams) { if (resizeParams.Passthrough) return img; if (img.First().Format == MagickFormat.Gif) { - Console.WriteLine("Coalescing gif for resize"); - img.Coalesce(); + var t = new Thread(() => { + Console.WriteLine("Coalescing gif for resize"); + img.Coalesce(); + }); + t.Start(); + while (t.IsAlive) await Task.Delay(100); } if (!resizeParams.Animated) { @@ -75,7 +79,7 @@ public class DiscordImageResizeService { resizeParams.Size = 4096; if (img.Max(x => Math.Max(x.Height, x.Width)) > resizeParams.Size || resizeParams.SpacebarAllowUpscale) { - Parallel.ForEach(img, new ParallelOptions() { MaxDegreeOfParallelism = 8 }, frame => { + Parallel.ForEach(img, new ParallelOptions() { MaxDegreeOfParallelism = 16 }, frame => { if (resizeParams.Size.HasValue) { uint oldWidth = frame.Width, oldHeight = frame.Height; // pads.IsPixelArt(frame) @@ -88,8 +92,12 @@ public class DiscordImageResizeService { } if (img.First().Format == MagickFormat.Gif && resizeParams.SpacebarOptimiseGif) { - Console.WriteLine("Optimizing gif after resize"); - img.OptimizePlus(); + var t = new Thread(() => { + Console.WriteLine("Optimizing gif after resize"); + img.OptimizePlus(); + }); + t.Start(); + while (t.IsAlive) await Task.Delay(100); } return img; diff --git a/extra/admin-api/Spacebar.Cdn.Worker/Program.cs b/extra/admin-api/Spacebar.Cdn.Worker/Program.cs index a7da7392b..289b869b9 100644 --- a/extra/admin-api/Spacebar.Cdn.Worker/Program.cs +++ b/extra/admin-api/Spacebar.Cdn.Worker/Program.cs @@ -55,6 +55,10 @@ MagickNET.Initialize(); // builder.WebHost.ConfigureKestrel(opts => opts.ListenUnixSocket(Environment.GetEnvironmentVariable("SOCKET_PATH")!)); +builder.WebHost.ConfigureKestrel(o => { + o.UseSystemd(); // Socket activation if wanted +}); + builder.Services.AddSingleton(await new FilesystemFileSource(Environment.GetEnvironmentVariable("STORAGE_PATH") ?? throw new InvalidOperationException("STORAGE_PATH not set!")).Init()); builder.Services.AddSingleton(); @@ -90,7 +94,7 @@ app.MapGet("/scale/{*path}", async (HttpContext ctx, IFileSource ifs, DiscordIma f.Stream.Position = 0; var mig = new MagickImageCollection(); await mig.ReadAsync(f.Stream, ctx.RequestAborted); - var res = dirs.Apply(mig, ctx.Request.GetResizeParams()); + var res = await dirs.Apply(mig, ctx.Request.GetResizeParams()); await ctx.Response.StartAsync(); await res.WriteAsync(ctx.Response.Body); await ctx.Response.CompleteAsync(); diff --git a/extra/admin-api/Spacebar.Cdn/Controllers/ImagesAndStickersController.cs b/extra/admin-api/Spacebar.Cdn/Controllers/ImagesAndStickersController.cs index 1f28f9bd4..6c0409dca 100644 --- a/extra/admin-api/Spacebar.Cdn/Controllers/ImagesAndStickersController.cs +++ b/extra/admin-api/Spacebar.Cdn/Controllers/ImagesAndStickersController.cs @@ -13,7 +13,7 @@ public class ImagesAndStickerController(LruFileCache lfc, IFileSource fs, CdnWor [HttpGet("/stickers/{id}.{ext}")] [HttpGet("/emojis/{id}")] [HttpGet("/emojis/{id}.{ext}")] - public async Task GetUserAvatar(string id, string ext = "png") { + public async Task GetUserAvatar(string id, string ext = "webp") { DiscordImageResizeParams resizeParams = Request.GetResizeParams(); var cacheKey = Request.Path + resizeParams.ToSerializedName(); LruFileCache.Entry? entry; diff --git a/extra/admin-api/Spacebar.Cdn/Controllers/StaticAssetController.cs b/extra/admin-api/Spacebar.Cdn/Controllers/StaticAssetController.cs index 52b802a59..ce64bc476 100644 --- a/extra/admin-api/Spacebar.Cdn/Controllers/StaticAssetController.cs +++ b/extra/admin-api/Spacebar.Cdn/Controllers/StaticAssetController.cs @@ -12,7 +12,7 @@ namespace Spacebar.Cdn.Controllers; public class StaticAssetController(LruFileCache lfc, IFileSource fs, CdnWorkerService cws) : ControllerBase { [HttpGet("/embed/avatars/{avatarIdx}")] [HttpGet("/embed/avatars/{avatarIdx}.{ext}")] - public async Task GetUserAvatar(string avatarIdx, string ext = "png") { + public async Task GetUserAvatar(string avatarIdx, string ext = "webp") { DiscordImageResizeParams resizeParams = Request.GetResizeParams(); var cacheKey = Request.Path + resizeParams.ToSerializedName(); LruFileCache.Entry? entry; diff --git a/extra/admin-api/Spacebar.Cdn/Controllers/UserController.cs b/extra/admin-api/Spacebar.Cdn/Controllers/UserController.cs index ee9ced719..eaf8feb70 100644 --- a/extra/admin-api/Spacebar.Cdn/Controllers/UserController.cs +++ b/extra/admin-api/Spacebar.Cdn/Controllers/UserController.cs @@ -11,7 +11,7 @@ namespace Spacebar.Cdn.Controllers; public class UserController(LruFileCache lfc, IFileSource fs, CdnWorkerService cws) : ControllerBase { [HttpGet("/avatars/{userId}/{hash}")] [HttpGet("/avatars/{userId}/{hash}.{ext}")] - public async Task GetUserAvatar(string userId, string hash, string ext = "png") { + public async Task GetUserAvatar(string userId, string hash, string ext = "webp") { DiscordImageResizeParams resizeParams = Request.GetResizeParams(); var originalKey = fs.BaseUrl + Request.Path; var cacheKey = Request.Path + resizeParams.ToSerializedName(); diff --git a/extra/admin-api/Spacebar.Cdn/Services/CdnWorkerService.cs b/extra/admin-api/Spacebar.Cdn/Services/CdnWorkerService.cs index 2eabf129a..1238705ee 100644 --- a/extra/admin-api/Spacebar.Cdn/Services/CdnWorkerService.cs +++ b/extra/admin-api/Spacebar.Cdn/Services/CdnWorkerService.cs @@ -1,10 +1,11 @@ using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; using System.Net.Sockets; using ArcaneLibs.Extensions; namespace Spacebar.Cdn.Services; -public class CdnWorkerService(SpacebarCdnWorkerConfiguration cfg) : IDisposable { +public class CdnWorkerService(SpacebarCdnWorkerConfiguration cfg, IHostApplicationLifetime lifetime) : IDisposable { private int _q8Idx = 0; private int _q16Idx = 0; private int _q16HdriIdx = 0; @@ -24,13 +25,23 @@ public class CdnWorkerService(SpacebarCdnWorkerConfiguration cfg) : IDisposable Console.WriteLine("Done initializing CDN worker store!"); } - private static HttpClient[] GetWorkerHttpClients(List urls) { + [SuppressMessage("Interoperability", "CA1416:Validate platform compatibility", Justification = "Unix is presumed by the developers - depends on unix sockets anyhow")] + private HttpClient[] GetWorkerHttpClients(List urls) { List results = []; foreach (var url in urls) { Console.WriteLine(" - Handling worker URI/path: " + url); - if (url.StartsWith("http://unix:")) results.Add(UnixSocketHttpClientFactory.GetHttpClientForSocket(url)); - else if (url.StartsWith("http://") || url.StartsWith("https://")) results.Add(new HttpClient() { BaseAddress = new(url) }); - // else if (File.Exists(url)) { } + if (url.StartsWith("http://unix:")) results.Add(HttpClientFactory.GetHttpClientForSocket(url)); + else if (url.StartsWith("http://") || url.StartsWith("https://")) results.Add(HttpClientFactory.GetHttpClientForUrl(url)); + else if (File.Exists(url) && File.GetUnixFileMode(url).HasFlag(UnixFileMode.OtherExecute)) { + var res = HttpClientFactory.GetHttpClientForExec(url); + results.Add(res.client); + lifetime.ApplicationStopped.Register(() => { + Console.WriteLine("Killing CDN worker..."); + res.p.Kill(); + res.p.WaitForExit(); + Console.WriteLine("CDN worker killed!"); + }); + } else throw new NotImplementedException($"Don't know how to handle worker URL \"{url}\""); } @@ -60,7 +71,7 @@ public class CdnWorkerService(SpacebarCdnWorkerConfiguration cfg) : IDisposable } } -internal class UnixSocketHttpClientFactory { +internal class HttpClientFactory { internal static HttpClient GetHttpClientForSocket(string url) { var socketPath = new Uri(url).LocalPath; var httpHandler = new SocketsHttpHandler { @@ -73,7 +84,30 @@ internal class UnixSocketHttpClientFactory { } }; return new HttpClient(httpHandler) { - BaseAddress = new Uri("http://localhost") // just a dummy value, since dotnet still wants :) + BaseAddress = new Uri("http://localhost"), // just a dummy value, since dotnet still wants it :) + Timeout = TimeSpan.FromMinutes(15) // because stuff can get slow, we want caching to at least attempt to succeed }; } + + public static HttpClient GetHttpClientForUrl(string url) { + return new HttpClient { + BaseAddress = new(url), + Timeout = TimeSpan.FromMinutes(15) + }; + } + + public static (HttpClient client, Process p) GetHttpClientForExec(string path) { + var url = $"http://unix:{Path.GetTempPath()}sb-cdn-worker-{Random.Shared.GetHexString(32)}.sock"; + var psi = new ProcessStartInfo() { + FileName = path, + RedirectStandardError = true, RedirectStandardOutput = true + }; + psi.Environment["DOTNET_URLS"] = url; + var p = Process.Start(psi); + p.OutputDataReceived += (_, args) => Console.WriteLine("[CDN Worker/OUT] " + args.Data); + p.ErrorDataReceived += (_, args) => Console.WriteLine("[CDN Worker/ERR] " + args.Data); + p.BeginErrorReadLine(); + p.BeginOutputReadLine(); + return (GetHttpClientForSocket(url), p); + } } \ No newline at end of file