From a92d127ff60c04b700728ffe84c3185ca1f06d14 Mon Sep 17 00:00:00 2001 From: Rory& Date: Sun, 12 Apr 2026 02:21:09 +0200 Subject: [PATCH] CDN-CS: it works!!! --- .../LruFileCache.cs | 41 ++- .../DiscordImageResizeService.cs | 6 + .../admin-api/Spacebar.Cdn.Worker/Program.cs | 10 +- .../Spacebar.Cdn.Worker/appsettings.json | 4 +- extra/admin-api/Spacebar.Cdn.Worker/test.html | 176 +++++----- .../Controllers/GetImageController.cs | 2 - .../ImagesAndStickersController.cs | 41 +++ .../Internal/GetImageController.cs | 324 +++++++++--------- .../Controllers/StaticAssetController.cs | 98 ++---- .../Controllers/UserController.cs | 151 ++++---- .../Extensions/FileSourceExtensions.cs | 50 +-- extra/admin-api/Spacebar.Cdn/Program.cs | 13 +- .../Spacebar.Cdn/Services/CdnWorkerService.cs | 79 +++++ .../Services/DiscordImageResizeService.cs | 34 -- .../SpacebarCdnWorkerConfiguration.cs | 18 + extra/admin-api/Spacebar.Cdn/gcc-lib-lib | 1 - extra/admin-api/Spacebar.Cdn/opencl-lib | 1 - extra/admin-api/Spacebar.Cdn/result-lib | 1 - 18 files changed, 570 insertions(+), 480 deletions(-) create mode 100644 extra/admin-api/Spacebar.Cdn/Controllers/ImagesAndStickersController.cs create mode 100644 extra/admin-api/Spacebar.Cdn/Services/CdnWorkerService.cs create mode 100644 extra/admin-api/Spacebar.Cdn/Services/SpacebarCdnWorkerConfiguration.cs delete mode 120000 extra/admin-api/Spacebar.Cdn/gcc-lib-lib delete mode 120000 extra/admin-api/Spacebar.Cdn/opencl-lib delete mode 120000 extra/admin-api/Spacebar.Cdn/result-lib diff --git a/extra/admin-api/Interop/Spacebar.Interop.Cdn.Abstractions/LruFileCache.cs b/extra/admin-api/Interop/Spacebar.Interop.Cdn.Abstractions/LruFileCache.cs index 3bd4ec095..27b554005 100644 --- a/extra/admin-api/Interop/Spacebar.Interop.Cdn.Abstractions/LruFileCache.cs +++ b/extra/admin-api/Interop/Spacebar.Interop.Cdn.Abstractions/LruFileCache.cs @@ -1,7 +1,4 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading.Tasks; +using ArcaneLibs; namespace Spacebar.Interop.Cdn.Abstractions; @@ -9,21 +6,36 @@ public class LruFileCache(int maxSizeBytes) { private readonly Dictionary _entries = new(); public async Task GetOrAdd(string key, Func> factory) { - if (_entries.TryGetValue(key, out var entry)) { - entry.LastAccessed = DateTimeOffset.UtcNow; - return entry; + lock (_entries) { + if (_entries.TryGetValue(key, out var entry)) { + entry.LastAccessed = DateTimeOffset.UtcNow; + return entry; + } } - entry = await factory(); - if (entry.Data.Length > 0 && entry.Data.Length <= maxSizeBytes) - _entries[key] = entry; + var newEntryTask = factory(); + var newEntry = await newEntryTask; + int oldSize; + lock (_entries) + oldSize = _entries.Sum(kv => kv.Value.Data.Length); + if (newEntry.Data.Length > 0 && newEntry.Data.Length <= maxSizeBytes) + lock (_entries) + _entries[key] = newEntry; - if (_entries.Sum(kv => kv.Value.Data.Length) > maxSizeBytes) { - var oldestKey = _entries.OrderBy(kv => kv.Value.LastAccessed).First().Key; - _entries.Remove(oldestKey); + lock (_entries) { + var newSize = _entries.Sum(kv => kv.Value.Data.Length); + if (newSize > maxSizeBytes) { + var oldestKey = _entries.OrderBy(kv => kv.Value.LastAccessed).First().Key; + _entries.Remove(oldestKey); + newSize = _entries.Sum(kv => kv.Value.Data.Length); + } + + var diffSize = newSize - oldSize; + Console.WriteLine( + $"LruCache: {Util.BytesToString(oldSize)} -> {Util.BytesToString(newSize)} / {Util.BytesToString(maxSizeBytes)} ({(diffSize > 0 ? "+ " : "- ") + Util.BytesToString(Math.Abs(diffSize))})"); } - return entry; + return newEntry; } public class Entry { @@ -33,3 +45,4 @@ public class LruFileCache(int maxSizeBytes) { } } +// https://cdn.old.server.spacebar.chat/emojis/1444515631118282917.webp?size=80&animated=true \ No newline at end of file diff --git a/extra/admin-api/Spacebar.Cdn.Worker/DiscordImageResizeService.cs b/extra/admin-api/Spacebar.Cdn.Worker/DiscordImageResizeService.cs index 7aaf312e3..51ee1aa5e 100644 --- a/extra/admin-api/Spacebar.Cdn.Worker/DiscordImageResizeService.cs +++ b/extra/admin-api/Spacebar.Cdn.Worker/DiscordImageResizeService.cs @@ -64,6 +64,12 @@ public class DiscordImageResizeService { img.Coalesce(); } + if (!resizeParams.Animated) { + var oldImg = img; + img = new MagickImageCollection([oldImg.First()]); + oldImg.Dispose(); + } + if (resizeParams.Size.HasValue) { if (resizeParams.Size > 4096) resizeParams.Size = 4096; diff --git a/extra/admin-api/Spacebar.Cdn.Worker/Program.cs b/extra/admin-api/Spacebar.Cdn.Worker/Program.cs index 9c60d7413..a7da7392b 100644 --- a/extra/admin-api/Spacebar.Cdn.Worker/Program.cs +++ b/extra/admin-api/Spacebar.Cdn.Worker/Program.cs @@ -62,7 +62,7 @@ builder.Services.AddControllers(); var app = builder.Build(); app.MapControllers(); -app.MapGet("/defaultAvatar/{idx:int}.{ext}", async (HttpContext ctx, int idx, string ext) => { +app.MapGet("/embed/avatars/{idx:int}.{ext}", async (HttpContext ctx, int idx, string ext) => { var (r, g, b) = DefaultAvatarRenderer.DefaultAvatarColors[idx % DefaultAvatarRenderer.DefaultAvatarColors.Length]; var res = await DefaultAvatarRenderer.GetDefaultAvatar(r, g, b, size: ctx.Request.Query.ContainsKey("size") ? int.Parse(ctx.Request.Query["size"]!) : 4096, format: Mimes.GetFormatForExtension(ext)); @@ -70,14 +70,14 @@ app.MapGet("/defaultAvatar/{idx:int}.{ext}", async (HttpContext ctx, int idx, st }); // small easter egg internal stuff, maybe used someday :) -app.MapGet("/defaultAvatar/_{bg:length(6)}.{ext}", async (HttpContext ctx, string bg, string ext) => { +app.MapGet("/embed/avatars/_{bg:length(6)}.{ext}", async (HttpContext ctx, string bg, string ext) => { var (r, g, b) = (byte.Parse(bg[..2], NumberStyles.HexNumber), byte.Parse(bg[2..4], NumberStyles.HexNumber), byte.Parse(bg[4..6], NumberStyles.HexNumber)); var res = await DefaultAvatarRenderer.GetDefaultAvatar(r, g, b, size: ctx.Request.Query.ContainsKey("size") ? int.Parse(ctx.Request.Query["size"]!) : 4096, format: Mimes.GetFormatForExtension(ext)); return Results.File(res, Mimes.GetMime(Mimes.GetFormatForExtension(ext))); }); -app.MapGet("/defaultAvatar/_{bg:length(6)}_{fg:length(6)}.{ext}", async (HttpContext ctx, string bg, string fg, string ext) => { +app.MapGet("/embed/avatars/_{bg:length(6)}_{fg:length(6)}.{ext}", async (HttpContext ctx, string bg, string fg, string ext) => { var (r, g, b) = (byte.Parse(bg[..2], NumberStyles.HexNumber), byte.Parse(bg[2..4], NumberStyles.HexNumber), byte.Parse(bg[4..6], NumberStyles.HexNumber)); var (rf, gf, bf) = (byte.Parse(fg[..2], NumberStyles.HexNumber), byte.Parse(fg[2..4], NumberStyles.HexNumber), byte.Parse(fg[4..6], NumberStyles.HexNumber)); var res = await DefaultAvatarRenderer.GetDefaultAvatar(r, g, b, rf, gf, bf, size: ctx.Request.Query.ContainsKey("size") ? int.Parse(ctx.Request.Query["size"]!) : 4096, @@ -88,7 +88,9 @@ app.MapGet("/defaultAvatar/_{bg:length(6)}_{fg:length(6)}.{ext}", async (HttpCon app.MapGet("/scale/{*path}", async (HttpContext ctx, IFileSource ifs, DiscordImageResizeService dirs, string path) => { var f = await ifs.GetFile(path); f.Stream.Position = 0; - var res = dirs.Apply(new MagickImageCollection(f.Stream), ctx.Request.GetResizeParams()); + var mig = new MagickImageCollection(); + await mig.ReadAsync(f.Stream, ctx.RequestAborted); + var res = dirs.Apply(mig, ctx.Request.GetResizeParams()); await ctx.Response.StartAsync(); await res.WriteAsync(ctx.Response.Body); await ctx.Response.CompleteAsync(); diff --git a/extra/admin-api/Spacebar.Cdn.Worker/appsettings.json b/extra/admin-api/Spacebar.Cdn.Worker/appsettings.json index 10f68b8c8..81d2855d8 100644 --- a/extra/admin-api/Spacebar.Cdn.Worker/appsettings.json +++ b/extra/admin-api/Spacebar.Cdn.Worker/appsettings.json @@ -1,8 +1,8 @@ { "Logging": { "LogLevel": { - "Default": "Information", - "Microsoft.AspNetCore": "Warning" + "Default": "Debug", + "Microsoft.AspNetCore": "Debug" } }, "AllowedHosts": "*" diff --git a/extra/admin-api/Spacebar.Cdn.Worker/test.html b/extra/admin-api/Spacebar.Cdn.Worker/test.html index fbd2f9e9f..bebf4f55c 100644 --- a/extra/admin-api/Spacebar.Cdn.Worker/test.html +++ b/extra/admin-api/Spacebar.Cdn.Worker/test.html @@ -1,91 +1,91 @@ -1 s=16 q=low -1 s=32 q=low -1 s=48 q=low -1 s=64 q=low -1 s=96 q=low -1 s=128 q=low -1 s=256 q=low -1 s=512 q=low -1 s=1024 q=low -1 s=2048 q=low -1 s=4096 q=low -2 s=16 q=low -2 s=32 q=low -2 s=48 q=low -2 s=64 q=low -2 s=96 q=low -2 s=128 q=low -2 s=256 q=low -2 s=512 q=low -2 s=1024 q=low -2 s=2048 q=low -2 s=4096 q=low -3 s=16 q=low -3 s=32 q=low -3 s=48 q=low -3 s=64 q=low -3 s=96 q=low -3 s=128 q=low -3 s=256 q=low -3 s=512 q=low -3 s=1024 q=low -3 s=2048 q=low -3 s=4096 q=low -4 s=16 q=low -4 s=32 q=low -4 s=48 q=low -4 s=64 q=low -4 s=96 q=low -4 s=128 q=low -4 s=256 q=low -4 s=512 q=low -4 s=1024 q=low -4 s=2048 q=low -4 s=4096 q=low -5 s=16 q=low -5 s=32 q=low -5 s=48 q=low -5 s=64 q=low -5 s=96 q=low -5 s=128 q=low -5 s=256 q=low -5 s=512 q=low -5 s=1024 q=low -5 s=2048 q=low -5 s=4096 q=low -6 s=16 q=low -6 s=32 q=low -6 s=48 q=low -6 s=64 q=low -6 s=96 q=low -6 s=128 q=low -6 s=256 q=low -6 s=512 q=low -6 s=1024 q=low -6 s=2048 q=low -6 s=4096 q=low +1 s=16 q=low +1 s=32 q=low +1 s=48 q=low +1 s=64 q=low +1 s=96 q=low +1 s=128 q=low +1 s=256 q=low +1 s=512 q=low +1 s=1024 q=low +1 s=2048 q=low +1 s=4096 q=low +2 s=16 q=low +2 s=32 q=low +2 s=48 q=low +2 s=64 q=low +2 s=96 q=low +2 s=128 q=low +2 s=256 q=low +2 s=512 q=low +2 s=1024 q=low +2 s=2048 q=low +2 s=4096 q=low +3 s=16 q=low +3 s=32 q=low +3 s=48 q=low +3 s=64 q=low +3 s=96 q=low +3 s=128 q=low +3 s=256 q=low +3 s=512 q=low +3 s=1024 q=low +3 s=2048 q=low +3 s=4096 q=low +4 s=16 q=low +4 s=32 q=low +4 s=48 q=low +4 s=64 q=low +4 s=96 q=low +4 s=128 q=low +4 s=256 q=low +4 s=512 q=low +4 s=1024 q=low +4 s=2048 q=low +4 s=4096 q=low +5 s=16 q=low +5 s=32 q=low +5 s=48 q=low +5 s=64 q=low +5 s=96 q=low +5 s=128 q=low +5 s=256 q=low +5 s=512 q=low +5 s=1024 q=low +5 s=2048 q=low +5 s=4096 q=low +6 s=16 q=low +6 s=32 q=low +6 s=48 q=low +6 s=64 q=low +6 s=96 q=low +6 s=128 q=low +6 s=256 q=low +6 s=512 q=low +6 s=1024 q=low +6 s=2048 q=low +6 s=4096 q=low -a s=16 q=low -a s=32 q=low -a s=48 q=low -a s=64 q=low -a s=96 q=low -a s=128 q=low -a s=256 q=low -a s=512 q=low -a s=1024 q=low -a s=2048 q=low -a s=4096 q=low +a s=16 q=low +a s=32 q=low +a s=48 q=low +a s=64 q=low +a s=96 q=low +a s=128 q=low +a s=256 q=low +a s=512 q=low +a s=1024 q=low +a s=2048 q=low +a s=4096 q=low -a s=16 q=high -a s=32 q=high -a s=48 q=high -a s=64 q=high -a s=96 q=high -a s=128 q=high -a s=256 q=high -a s=512 q=high -a s=1024 q=high -a s=2048 q=high -a s=4096 q=high +a s=16 q=high +a s=32 q=high +a s=48 q=high +a s=64 q=high +a s=96 q=high +a s=128 q=high +a s=256 q=high +a s=512 q=high +a s=1024 q=high +a s=2048 q=high +a s=4096 q=high diff --git a/extra/admin-api/Spacebar.Cdn/Controllers/GetImageController.cs b/extra/admin-api/Spacebar.Cdn/Controllers/GetImageController.cs index c4473932b..cf37b83ba 100644 --- a/extra/admin-api/Spacebar.Cdn/Controllers/GetImageController.cs +++ b/extra/admin-api/Spacebar.Cdn/Controllers/GetImageController.cs @@ -8,10 +8,8 @@ // // [ApiController] // public class GetImageController(LruFileCache lfc, IFileSource fs, DiscordImageResizeService dirs) : ImageController { -// // [HttpGet("/avatars/{_:required}")] // [HttpGet("/emojis/{emoji_id:required}.{ext:required}")] // [HttpGet("/stickers/{sticker_id:required}.{ext:required}")] -// // [HttpGet("/avatars/{user_id:required}/{avatar_hash:required}.{ext:required}")] // [HttpGet("/banners/{user_id:required}/{user_banner:required}.{ext:required}")] // public async Task GetImage(string? ext) { // var originalKey = fs.BaseUrl + Request.Path; diff --git a/extra/admin-api/Spacebar.Cdn/Controllers/ImagesAndStickersController.cs b/extra/admin-api/Spacebar.Cdn/Controllers/ImagesAndStickersController.cs new file mode 100644 index 000000000..1f28f9bd4 --- /dev/null +++ b/extra/admin-api/Spacebar.Cdn/Controllers/ImagesAndStickersController.cs @@ -0,0 +1,41 @@ +using ArcaneLibs.Extensions.Streams; +using Microsoft.AspNetCore.Mvc; +using Spacebar.AdminApi.TestClient.Services.Services; +using Spacebar.Cdn.Extensions; +using Spacebar.Cdn.Services; +using Spacebar.Interop.Cdn.Abstractions; + +namespace Spacebar.Cdn.Controllers; + +[ApiController] +public class ImagesAndStickerController(LruFileCache lfc, IFileSource fs, CdnWorkerService cws) : ControllerBase { + [HttpGet("/stickers/{id}")] + [HttpGet("/stickers/{id}.{ext}")] + [HttpGet("/emojis/{id}")] + [HttpGet("/emojis/{id}.{ext}")] + public async Task GetUserAvatar(string id, string ext = "png") { + DiscordImageResizeParams resizeParams = Request.GetResizeParams(); + var cacheKey = Request.Path + resizeParams.ToSerializedName(); + LruFileCache.Entry? entry; + if (!Request.Query.Any() || resizeParams.Passthrough) { + var original = await fs.GetFile(Request.Path.ToString().Replace("."+ext, "")); + entry = new LruFileCache.Entry() { + Data = original.Stream.ReadToEnd().ToArray(), + MimeType = original.MimeType + }; + } + else + entry = await lfc.GetOrAdd(cacheKey, async () => { + var original = await fs.GetFile(Request.Path.ToString().Replace("."+ext, "")); + var res = await cws.GetRawClient("q8").GetAsync("/scale" + Request.Path.ToString().Replace("."+ext, "") + Request.QueryString); + var outStream = await res.Content.ReadAsStreamAsync(); + + return new LruFileCache.Entry() { + Data = outStream.ReadToEnd().ToArray(), + MimeType = res.Content.Headers.ContentType?.ToString() ?? original.MimeType + }; + }); + + return new FileContentResult(entry.Data, entry.MimeType); + } +} \ No newline at end of file diff --git a/extra/admin-api/Spacebar.Cdn/Controllers/Internal/GetImageController.cs b/extra/admin-api/Spacebar.Cdn/Controllers/Internal/GetImageController.cs index f595fba97..3b7df118e 100644 --- a/extra/admin-api/Spacebar.Cdn/Controllers/Internal/GetImageController.cs +++ b/extra/admin-api/Spacebar.Cdn/Controllers/Internal/GetImageController.cs @@ -1,162 +1,162 @@ -using ArcaneLibs; -using ArcaneLibs.Collections; -using ImageMagick; -using Microsoft.AspNetCore.Mvc; -using Spacebar.AdminApi.TestClient.Services.Services; -using Spacebar.Cdn.Extensions; -using Spacebar.Interop.Cdn.Abstractions; - -namespace Spacebar.Cdn.Controllers.Internal; - -[ApiController] -public class IsPixelArtController(LruFileCache lfc, IFileSource fs, PixelArtDetectionService pads, DiscordImageResizeService dirs) : ControllerBase { - private static readonly LruCache _isPixelArtCache = new(100_000); - private static readonly LruFileCache _edgeCache = new(100_000_000); - - [HttpGet("/isPixelArt/{*_:required}")] - public async Task IsPixelArt() { - return await _isPixelArtCache.GetOrAddAsync(Request.Path.ToString(), async () => { - await using var original = await fs.GetFile(Request.Path.ToString().Replace("/isPixelArt", "")); - using var img = await original.ToMagickImageCollectionAsync(); - return pads.IsPixelArt(img); - }); - } - - [HttpGet("/isCartoonArt/{*_:required}")] - public async Task IsCartoonArt() { - return await _isPixelArtCache.GetOrAddAsync(Request.Path.ToString(), async () => { - await using var original = await fs.GetFile(Request.Path.ToString().Replace("/isCartoonArt", "")); - using var img = await original.ToMagickImageCollectionAsync(); - return pads.IsCartoonArt(img); - }); - } - - [HttpGet("/edges/{*_:required}")] - public async Task GetEdges([FromQuery] string applyMode = "pre") { - return await _edgeCache.GetOrAdd(Request.Path.ToString() + Request.QueryString, async () => { - var original = await fs.GetFile(Request.Path.ToString().Replace("/edges", "")); - - DiscordImageResizeParams resizeParams = new() { - Size = Request.Query.ContainsKey("size") && uint.TryParse(Request.Query["size"], out uint size) ? size : null, - Quality = Request.Query.ContainsKey("quality") && Enum.TryParse(Request.Query["quality"], true, out var quality) - ? quality - : DiscordImageResizeQuality.High, - KeepAspectRatio = !Request.Query.ContainsKey("keepAspectRatio") || !bool.TryParse(Request.Query["keepAspectRatio"], out bool kar) || kar, - Passthrough = Request.Query.ContainsKey("passthrough") && bool.TryParse(Request.Query["passthrough"], out bool pt) && pt, - Animated = Request.Query.ContainsKey("animated") && bool.TryParse(Request.Query["animated"], out bool an) && an, - SpacebarAllowUpscale = Request.Query.ContainsKey("allowUpscale") && bool.TryParse(Request.Query["allowUpscale"], out bool au) && au, - SpacebarOptimiseGif = Request.Query.ContainsKey("optimiseGif") && bool.TryParse(Request.Query["optimiseGif"], out bool og) && og - }; - - double radius = 1; - if (Request.Query.ContainsKey("radius")) double.TryParse(Request.Query["radius"], out radius); - - var img = await original.ToMagickImageCollectionAsync(); - - // if (applyMode == "pre") img = dirs.Apply(img,resizeParams); - - int inFrames = img.Count; - using var edged = pads.RenderEdges(img, radius); - // if (applyMode == "post") img = dirs.Apply(img,resizeParams); - - Console.WriteLine($"Generated edges for {Request.Path}, radius={radius}, inFrames={inFrames}, outFrames={edged.Count}"); - return new LruFileCache.Entry { - Data = edged.ToByteArray(), - // MimeType = Mimes.GetMime(img.First().Format), - MimeType = "image/apng" - }; - }).ContinueWith(t => File(t.Result.Data, t.Result.MimeType)); - } - - [HttpGet("/posterize/{*_:required}")] - public async Task Posterize() { - return await _edgeCache.GetOrAdd(Request.Path.ToString() + Request.QueryString, async () => { - var original = await fs.GetFile(Request.Path.ToString().Replace("/posterize", "")); - DiscordImageResizeParams resizeParams = new() { - Size = Request.Query.ContainsKey("size") && uint.TryParse(Request.Query["size"], out uint size) ? size : null, - Quality = Request.Query.ContainsKey("quality") && Enum.TryParse(Request.Query["quality"], true, out var quality) - ? quality - : DiscordImageResizeQuality.High, - KeepAspectRatio = !Request.Query.ContainsKey("keepAspectRatio") || !bool.TryParse(Request.Query["keepAspectRatio"], out bool kar) || kar, - Passthrough = Request.Query.ContainsKey("passthrough") && bool.TryParse(Request.Query["passthrough"], out bool pt) && pt, - Animated = Request.Query.ContainsKey("animated") && bool.TryParse(Request.Query["animated"], out bool an) && an, - SpacebarAllowUpscale = Request.Query.ContainsKey("allowUpscale") && bool.TryParse(Request.Query["allowUpscale"], out bool au) && au - }; - - double radius = 1; - if (Request.Query.ContainsKey("radius")) double.TryParse(Request.Query["radius"], out radius); - - var img = await original.ToMagickImageCollectionAsync(); - - int inFrames = img.Count; - // using var edged = pads.RenderEdges(img, radius); - foreach (var frame in img) { - // get major color count - frame.ColorFuzz = new Percentage(25); - frame.Posterize(16, DitherMethod.No); - frame.ColorFuzz = new Percentage(0); - } - - if (resizeParams.Size.HasValue) - img = dirs.Apply(img, resizeParams); - - Console.WriteLine($"Generated edges for {Request.Path}, radius={radius}, inFrames={inFrames}, outFrames={img.Count}"); - return new LruFileCache.Entry { - Data = img.ToByteArray(), - // MimeType = Mimes.GetMime(img.First().Format), - MimeType = "image/apng" - }; - }).ContinueWith(t => File(t.Result.Data, t.Result.MimeType)); - } - - [HttpGet("/colorFuzz/{*_:required}")] - public async Task ColorFuzz() { - return await _edgeCache.GetOrAdd(Request.Path.ToString() + Request.QueryString, async () => { - var original = await fs.GetFile(Request.Path.ToString().Replace("/colorFuzz", "")); - DiscordImageResizeParams resizeParams = new() { - Size = Request.Query.ContainsKey("size") && uint.TryParse(Request.Query["size"], out uint size) - ? size - : null, - Quality = Request.Query.ContainsKey("quality") && Enum.TryParse(Request.Query["quality"], true, out var quality) - ? quality - : DiscordImageResizeQuality.High, - KeepAspectRatio = !Request.Query.ContainsKey("keepAspectRatio") || !bool.TryParse(Request.Query["keepAspectRatio"], out bool kar) || kar, - Passthrough = Request.Query.ContainsKey("passthrough") && bool.TryParse(Request.Query["passthrough"], out bool pt) && pt, - Animated = Request.Query.ContainsKey("animated") && bool.TryParse(Request.Query["animated"], out bool an) && an, - SpacebarAllowUpscale = Request.Query.ContainsKey("allowUpscale") && bool.TryParse(Request.Query["allowUpscale"], out bool au) && au, - }; - - double colorFuzz = 1; - if (Request.Query.ContainsKey("colorFuzz")) double.TryParse(Request.Query["colorFuzz"], out colorFuzz); - - var img = await original.ToMagickImageCollectionAsync(); - - int inFrames = img.Count; - // using var edged = pads.RenderEdges(img, radius); - foreach (var frame in img) { - // get major color count - frame.ColorFuzz = new Percentage(colorFuzz); - } - - if (resizeParams.Size.HasValue) - img = dirs.Apply(img, resizeParams); - - Console.WriteLine($"Generated colorFuzz for {Request.Path}, fuzz={colorFuzz}, inFrames={inFrames}, outFrames={img.Count}"); - return new LruFileCache.Entry { - Data = img.ToByteArray(), - // MimeType = Mimes.GetMime(img.First().Format), - MimeType = "image/apng" - }; - }).ContinueWith(t => File(t.Result.Data, t.Result.MimeType)); - } - - [HttpGet("/defaultAvatar")] - public async Task DefaultAvatar() { - var re = new RainbowEnumerator(); - var img = new MagickImageCollection(); - - var ms = new MemoryStream(); - - return new FileContentResult(ms.ToArray(), "image/gif"); - } -} \ No newline at end of file +// using ArcaneLibs; +// using ArcaneLibs.Collections; +// using ImageMagick; +// using Microsoft.AspNetCore.Mvc; +// using Spacebar.AdminApi.TestClient.Services.Services; +// using Spacebar.Cdn.Extensions; +// using Spacebar.Interop.Cdn.Abstractions; +// +// namespace Spacebar.Cdn.Controllers.Internal; +// +// [ApiController] +// public class IsPixelArtController(LruFileCache lfc, IFileSource fs, PixelArtDetectionService pads, DiscordImageResizeService dirs) : ControllerBase { +// private static readonly LruCache _isPixelArtCache = new(100_000); +// private static readonly LruFileCache _edgeCache = new(100_000_000); +// +// [HttpGet("/isPixelArt/{*_:required}")] +// public async Task IsPixelArt() { +// return await _isPixelArtCache.GetOrAddAsync(Request.Path.ToString(), async () => { +// await using var original = await fs.GetFile(Request.Path.ToString().Replace("/isPixelArt", "")); +// using var img = await original.ToMagickImageCollectionAsync(); +// return pads.IsPixelArt(img); +// }); +// } +// +// [HttpGet("/isCartoonArt/{*_:required}")] +// public async Task IsCartoonArt() { +// return await _isPixelArtCache.GetOrAddAsync(Request.Path.ToString(), async () => { +// await using var original = await fs.GetFile(Request.Path.ToString().Replace("/isCartoonArt", "")); +// using var img = await original.ToMagickImageCollectionAsync(); +// return pads.IsCartoonArt(img); +// }); +// } +// +// [HttpGet("/edges/{*_:required}")] +// public async Task GetEdges([FromQuery] string applyMode = "pre") { +// return await _edgeCache.GetOrAdd(Request.Path.ToString() + Request.QueryString, async () => { +// var original = await fs.GetFile(Request.Path.ToString().Replace("/edges", "")); +// +// DiscordImageResizeParams resizeParams = new() { +// Size = Request.Query.ContainsKey("size") && uint.TryParse(Request.Query["size"], out uint size) ? size : null, +// Quality = Request.Query.ContainsKey("quality") && Enum.TryParse(Request.Query["quality"], true, out var quality) +// ? quality +// : DiscordImageResizeQuality.High, +// KeepAspectRatio = !Request.Query.ContainsKey("keepAspectRatio") || !bool.TryParse(Request.Query["keepAspectRatio"], out bool kar) || kar, +// Passthrough = Request.Query.ContainsKey("passthrough") && bool.TryParse(Request.Query["passthrough"], out bool pt) && pt, +// Animated = Request.Query.ContainsKey("animated") && bool.TryParse(Request.Query["animated"], out bool an) && an, +// SpacebarAllowUpscale = Request.Query.ContainsKey("allowUpscale") && bool.TryParse(Request.Query["allowUpscale"], out bool au) && au, +// SpacebarOptimiseGif = Request.Query.ContainsKey("optimiseGif") && bool.TryParse(Request.Query["optimiseGif"], out bool og) && og +// }; +// +// double radius = 1; +// if (Request.Query.ContainsKey("radius")) double.TryParse(Request.Query["radius"], out radius); +// +// var img = await original.ToMagickImageCollectionAsync(); +// +// // if (applyMode == "pre") img = dirs.Apply(img,resizeParams); +// +// int inFrames = img.Count; +// using var edged = pads.RenderEdges(img, radius); +// // if (applyMode == "post") img = dirs.Apply(img,resizeParams); +// +// Console.WriteLine($"Generated edges for {Request.Path}, radius={radius}, inFrames={inFrames}, outFrames={edged.Count}"); +// return new LruFileCache.Entry { +// Data = edged.ToByteArray(), +// // MimeType = Mimes.GetMime(img.First().Format), +// MimeType = "image/apng" +// }; +// }).ContinueWith(t => File(t.Result.Data, t.Result.MimeType)); +// } +// +// [HttpGet("/posterize/{*_:required}")] +// public async Task Posterize() { +// return await _edgeCache.GetOrAdd(Request.Path.ToString() + Request.QueryString, async () => { +// var original = await fs.GetFile(Request.Path.ToString().Replace("/posterize", "")); +// DiscordImageResizeParams resizeParams = new() { +// Size = Request.Query.ContainsKey("size") && uint.TryParse(Request.Query["size"], out uint size) ? size : null, +// Quality = Request.Query.ContainsKey("quality") && Enum.TryParse(Request.Query["quality"], true, out var quality) +// ? quality +// : DiscordImageResizeQuality.High, +// KeepAspectRatio = !Request.Query.ContainsKey("keepAspectRatio") || !bool.TryParse(Request.Query["keepAspectRatio"], out bool kar) || kar, +// Passthrough = Request.Query.ContainsKey("passthrough") && bool.TryParse(Request.Query["passthrough"], out bool pt) && pt, +// Animated = Request.Query.ContainsKey("animated") && bool.TryParse(Request.Query["animated"], out bool an) && an, +// SpacebarAllowUpscale = Request.Query.ContainsKey("allowUpscale") && bool.TryParse(Request.Query["allowUpscale"], out bool au) && au +// }; +// +// double radius = 1; +// if (Request.Query.ContainsKey("radius")) double.TryParse(Request.Query["radius"], out radius); +// +// var img = await original.ToMagickImageCollectionAsync(); +// +// int inFrames = img.Count; +// // using var edged = pads.RenderEdges(img, radius); +// foreach (var frame in img) { +// // get major color count +// frame.ColorFuzz = new Percentage(25); +// frame.Posterize(16, DitherMethod.No); +// frame.ColorFuzz = new Percentage(0); +// } +// +// if (resizeParams.Size.HasValue) +// img = dirs.Apply(img, resizeParams); +// +// Console.WriteLine($"Generated edges for {Request.Path}, radius={radius}, inFrames={inFrames}, outFrames={img.Count}"); +// return new LruFileCache.Entry { +// Data = img.ToByteArray(), +// // MimeType = Mimes.GetMime(img.First().Format), +// MimeType = "image/apng" +// }; +// }).ContinueWith(t => File(t.Result.Data, t.Result.MimeType)); +// } +// +// [HttpGet("/colorFuzz/{*_:required}")] +// public async Task ColorFuzz() { +// return await _edgeCache.GetOrAdd(Request.Path.ToString() + Request.QueryString, async () => { +// var original = await fs.GetFile(Request.Path.ToString().Replace("/colorFuzz", "")); +// DiscordImageResizeParams resizeParams = new() { +// Size = Request.Query.ContainsKey("size") && uint.TryParse(Request.Query["size"], out uint size) +// ? size +// : null, +// Quality = Request.Query.ContainsKey("quality") && Enum.TryParse(Request.Query["quality"], true, out var quality) +// ? quality +// : DiscordImageResizeQuality.High, +// KeepAspectRatio = !Request.Query.ContainsKey("keepAspectRatio") || !bool.TryParse(Request.Query["keepAspectRatio"], out bool kar) || kar, +// Passthrough = Request.Query.ContainsKey("passthrough") && bool.TryParse(Request.Query["passthrough"], out bool pt) && pt, +// Animated = Request.Query.ContainsKey("animated") && bool.TryParse(Request.Query["animated"], out bool an) && an, +// SpacebarAllowUpscale = Request.Query.ContainsKey("allowUpscale") && bool.TryParse(Request.Query["allowUpscale"], out bool au) && au, +// }; +// +// double colorFuzz = 1; +// if (Request.Query.ContainsKey("colorFuzz")) double.TryParse(Request.Query["colorFuzz"], out colorFuzz); +// +// var img = await original.ToMagickImageCollectionAsync(); +// +// int inFrames = img.Count; +// // using var edged = pads.RenderEdges(img, radius); +// foreach (var frame in img) { +// // get major color count +// frame.ColorFuzz = new Percentage(colorFuzz); +// } +// +// if (resizeParams.Size.HasValue) +// img = dirs.Apply(img, resizeParams); +// +// Console.WriteLine($"Generated colorFuzz for {Request.Path}, fuzz={colorFuzz}, inFrames={inFrames}, outFrames={img.Count}"); +// return new LruFileCache.Entry { +// Data = img.ToByteArray(), +// // MimeType = Mimes.GetMime(img.First().Format), +// MimeType = "image/apng" +// }; +// }).ContinueWith(t => File(t.Result.Data, t.Result.MimeType)); +// } +// +// [HttpGet("/defaultAvatar")] +// public async Task DefaultAvatar() { +// var re = new RainbowEnumerator(); +// var img = new MagickImageCollection(); +// +// var ms = new MemoryStream(); +// +// return new FileContentResult(ms.ToArray(), "image/gif"); +// } +// } \ No newline at end of file diff --git a/extra/admin-api/Spacebar.Cdn/Controllers/StaticAssetController.cs b/extra/admin-api/Spacebar.Cdn/Controllers/StaticAssetController.cs index 48ae22534..52b802a59 100644 --- a/extra/admin-api/Spacebar.Cdn/Controllers/StaticAssetController.cs +++ b/extra/admin-api/Spacebar.Cdn/Controllers/StaticAssetController.cs @@ -1,66 +1,32 @@ -// using ArcaneLibs.Extensions.Streams; -// using Microsoft.AspNetCore.Mvc; -// using Microsoft.OpenApi; -// using Spacebar.AdminApi.TestClient.Services.Services; -// using Spacebar.Cdn.Extensions; -// using Spacebar.Interop.Cdn.Abstractions; -// -// namespace Spacebar.Cdn.Controllers; -// -// [ApiController] -// public class StaticAssetController(LruFileCache lfc, IFileSource fs, DiscordImageResizeService dirs) : ImageController { -// private static readonly Dictionary defaultAvatarHashMap = new() { -// { "0", "4a8562cf00887030c416d3ec2d46385a" }, -// { "1", "9b0bb198936784c45c72833cc426cc55" }, -// { "2", "22341bdb500c7b63a93bbce957d1601e" }, -// { "3", "d9977836b82058bf2f74eebd50edc095" }, -// { "4", "9d6ddb4e4d899a533a8cc617011351c9" }, -// { "5", "7213ab6677377974697dfdfbaf5f6a6f" }, -// }; -// -// private static readonly Dictionary defaultGroupDMAvatarHashMap = new() { -// { "0", "3b70bb66089c60f8be5e214bf8574c9d" }, -// { "1", "9581acd31832465bdeaa5385b0e919a3" }, -// { "2", "a8a4727cf2dc2939bd3c657fad4463fa" }, -// { "3", "2e46fe14586f8e95471c0917f56726b5" }, -// { "4", "fac7e78de9753d4a37083bba74c1d9ef" }, -// { "5", "4ab900144b0865430dc9be825c838faa" }, -// { "6", "1276374a404452756f3c9cc2601508a5" }, -// { "7", "904bf9f1b61f53ef4a3b7a893afeabe3" }, -// }; -// -// // png only -// [HttpGet("/embed/avatars/{userIndex}.{ext}")] -// public async Task GetDefaultUserAvatar(string userIndex, string ext) { -// -// var cacheKey = Request.Path + Request.QueryString; -// -// DiscordImageResizeParams resizeParams = GetResizeParams(); -// -// var entry = await lfc.GetOrAdd(cacheKey, async () => { -// var original = await fs.GetFile(Request.Path); -// -// if (Request.Query.Any()) { -// using var img = await original.ToMagickImageCollectionAsync(); -// dirs.Apply(img, resizeParams); -// -// var outStream = new MemoryStream(); -// await img.WriteAsync(outStream, img.First().Format); -// outStream.Position = 0; -// -// return new LruFileCache.Entry() { -// Data = outStream.ReadToEnd().ToArray(), -// MimeType = original.MimeType -// }; -// } -// -// return new LruFileCache.Entry() { -// Data = original.Stream.ReadToEnd().ToArray(), -// MimeType = original.MimeType -// }; -// }); -// -// // byte array with mime type result -// return new FileContentResult(entry.Data, entry.MimeType); -// } -// } \ No newline at end of file +using ArcaneLibs.Extensions.Streams; +using Microsoft.AspNetCore.Mvc; +using Microsoft.OpenApi; +using Spacebar.AdminApi.TestClient.Services.Services; +using Spacebar.Cdn.Extensions; +using Spacebar.Cdn.Services; +using Spacebar.Interop.Cdn.Abstractions; + +namespace Spacebar.Cdn.Controllers; + +[ApiController] +public class StaticAssetController(LruFileCache lfc, IFileSource fs, CdnWorkerService cws) : ControllerBase { + [HttpGet("/embed/avatars/{avatarIdx}")] + [HttpGet("/embed/avatars/{avatarIdx}.{ext}")] + public async Task GetUserAvatar(string avatarIdx, string ext = "png") { + DiscordImageResizeParams resizeParams = Request.GetResizeParams(); + var cacheKey = Request.Path + resizeParams.ToSerializedName(); + LruFileCache.Entry? entry; + entry = await lfc.GetOrAdd(cacheKey, async () => { + + var res = await cws.GetRawClient("q8").GetAsync(Request.Path + Request.QueryString); + var outStream = await res.Content.ReadAsStreamAsync(); + + return new LruFileCache.Entry() { + Data = outStream.ReadToEnd().ToArray(), + MimeType = res.Content.Headers.ContentType?.ToString() ?? Mimes.GetMime(Mimes.GetFormatForExtension(ext)) + }; + }); + + return new FileContentResult(entry.Data, entry.MimeType); + } +} \ No newline at end of file diff --git a/extra/admin-api/Spacebar.Cdn/Controllers/UserController.cs b/extra/admin-api/Spacebar.Cdn/Controllers/UserController.cs index a6c470369..ee9ced719 100644 --- a/extra/admin-api/Spacebar.Cdn/Controllers/UserController.cs +++ b/extra/admin-api/Spacebar.Cdn/Controllers/UserController.cs @@ -1,77 +1,74 @@ -// using ArcaneLibs.Extensions.Streams; -// using Microsoft.AspNetCore.Mvc; -// using Spacebar.AdminApi.TestClient.Services.Services; -// using Spacebar.Cdn.Extensions; -// using Spacebar.Interop.Cdn.Abstractions; -// -// namespace Spacebar.Cdn.Controllers; -// -// [ApiController] -// public class UserController(LruFileCache lfc, IFileSource fs, DiscordImageResizeService dirs) : ImageController { -// [HttpGet("/avatars/{userId}/{hash}.{ext}")] -// public async Task GetUserAvatar(string userId, string hash, string ext) { -// var originalKey = fs.BaseUrl + Request.Path; -// var cacheKey = Request.Path + Request.QueryString; -// -// DiscordImageResizeParams resizeParams = GetResizeParams(); -// -// var entry = await lfc.GetOrAdd(cacheKey, async () => { -// var original = await fs.GetFile(Request.Path); -// -// if (Request.Query.Any()) { -// using var img = await original.ToMagickImageCollectionAsync(); -// dirs.Apply(img, resizeParams); -// -// var outStream = new MemoryStream(); -// await img.WriteAsync(outStream, img.First().Format); -// outStream.Position = 0; -// -// return new LruFileCache.Entry() { -// Data = outStream.ReadToEnd().ToArray(), -// MimeType = original.MimeType -// }; -// } -// -// return new LruFileCache.Entry() { -// Data = original.Stream.ReadToEnd().ToArray(), -// MimeType = original.MimeType -// }; -// }); -// -// // byte array with mime type result -// return new FileContentResult(entry.Data, entry.MimeType); -// } -// [HttpGet("/banners/{userId}/{hash}.{ext}")] -// public async Task GetUserBanner(string userId, string hash, string ext) { -// var originalKey = fs.BaseUrl + Request.Path; -// var cacheKey = Request.Path + Request.QueryString; -// -// DiscordImageResizeParams resizeParams = GetResizeParams(); -// -// var entry = await lfc.GetOrAdd(cacheKey, async () => { -// var original = await fs.GetFile(Request.Path); -// -// if (Request.Query.Any()) { -// using var img = await original.ToMagickImageCollectionAsync(); -// dirs.Apply(img, resizeParams); -// -// var outStream = new MemoryStream(); -// await img.WriteAsync(outStream, img.First().Format); -// outStream.Position = 0; -// -// return new LruFileCache.Entry() { -// Data = outStream.ReadToEnd().ToArray(), -// MimeType = original.MimeType -// }; -// } -// -// return new LruFileCache.Entry() { -// Data = original.Stream.ReadToEnd().ToArray(), -// MimeType = original.MimeType -// }; -// }); -// -// // byte array with mime type result -// return new FileContentResult(entry.Data, entry.MimeType); -// } -// } \ No newline at end of file +using ArcaneLibs.Extensions.Streams; +using Microsoft.AspNetCore.Mvc; +using Spacebar.AdminApi.TestClient.Services.Services; +using Spacebar.Cdn.Extensions; +using Spacebar.Cdn.Services; +using Spacebar.Interop.Cdn.Abstractions; + +namespace Spacebar.Cdn.Controllers; + +[ApiController] +public class UserController(LruFileCache lfc, IFileSource fs, CdnWorkerService cws) : ControllerBase { + [HttpGet("/avatars/{userId}/{hash}")] + [HttpGet("/avatars/{userId}/{hash}.{ext}")] + public async Task GetUserAvatar(string userId, string hash, string ext = "png") { + DiscordImageResizeParams resizeParams = Request.GetResizeParams(); + var originalKey = fs.BaseUrl + Request.Path; + var cacheKey = Request.Path + resizeParams.ToSerializedName(); + LruFileCache.Entry? entry; + if (!Request.Query.Any() || resizeParams.Passthrough) { + var original = await fs.GetFile(Request.Path); + entry = new LruFileCache.Entry() { + Data = original.Stream.ReadToEnd().ToArray(), + MimeType = original.MimeType + }; + } + else + entry = await lfc.GetOrAdd(cacheKey, async () => { + var original = await fs.GetFile(Request.Path); + var res = await cws.GetRawClient("q8").GetAsync("/scale" + Request.Path + Request.QueryString); + var outStream = await res.Content.ReadAsStreamAsync(); + + return new LruFileCache.Entry() { + Data = outStream.ReadToEnd().ToArray(), + MimeType = res.Content.Headers.ContentType?.ToString() ?? original.MimeType + }; + }); + + // byte array with mime type result + return new FileContentResult(entry.Data, entry.MimeType); + } + // [HttpGet("/banners/{userId}/{hash}.{ext}")] + // public async Task GetUserBanner(string userId, string hash, string ext) { + // var originalKey = fs.BaseUrl + Request.Path; + // var cacheKey = Request.Path + Request.QueryString; + // + // DiscordImageResizeParams resizeParams = GetResizeParams(); + // + // var entry = await lfc.GetOrAdd(cacheKey, async () => { + // var original = await fs.GetFile(Request.Path); + // + // if (Request.Query.Any()) { + // using var img = await original.ToMagickImageCollectionAsync(); + // dirs.Apply(img, resizeParams); + // + // var outStream = new MemoryStream(); + // await img.WriteAsync(outStream, img.First().Format); + // outStream.Position = 0; + // + // return new LruFileCache.Entry() { + // Data = outStream.ReadToEnd().ToArray(), + // MimeType = original.MimeType + // }; + // } + // + // return new LruFileCache.Entry() { + // Data = original.Stream.ReadToEnd().ToArray(), + // MimeType = original.MimeType + // }; + // }); + // + // // byte array with mime type result + // return new FileContentResult(entry.Data, entry.MimeType); + // } +} \ No newline at end of file diff --git a/extra/admin-api/Spacebar.Cdn/Extensions/FileSourceExtensions.cs b/extra/admin-api/Spacebar.Cdn/Extensions/FileSourceExtensions.cs index 218ba6a23..2a11be388 100644 --- a/extra/admin-api/Spacebar.Cdn/Extensions/FileSourceExtensions.cs +++ b/extra/admin-api/Spacebar.Cdn/Extensions/FileSourceExtensions.cs @@ -1,25 +1,25 @@ -using ImageMagick; -using FileInfo = Spacebar.Interop.Cdn.Abstractions.FileInfo; - -namespace Spacebar.Cdn.Extensions; - -public static class FileSourceExtensions { - public static async Task ToMagickImageCollectionAsync(this FileInfo fileInfo) { - var ms = new MemoryStream(); - fileInfo.Stream.Position = 0; - await fileInfo.Stream.CopyToAsync(ms); - ms.Position = 0; - var img = fileInfo.MimeType switch { - "image/apng" => new MagickImageCollection(ms, MagickFormat.APng), - _ => new MagickImageCollection(ms) - }; - - // if (img.First().Format == MagickFormat.Png) { - // img.Dispose(); - // ms.Position = 0; - // img = new MagickImageCollection(ms, MagickFormat.APng); - // } - - return img; - } -} \ No newline at end of file +// using ImageMagick; +// using FileInfo = Spacebar.Interop.Cdn.Abstractions.FileInfo; +// +// namespace Spacebar.Cdn.Extensions; +// +// public static class FileSourceExtensions { +// public static async Task ToMagickImageCollectionAsync(this FileInfo fileInfo) { +// var ms = new MemoryStream(); +// fileInfo.Stream.Position = 0; +// await fileInfo.Stream.CopyToAsync(ms); +// ms.Position = 0; +// var img = fileInfo.MimeType switch { +// "image/apng" => new MagickImageCollection(ms, MagickFormat.APng), +// _ => new MagickImageCollection(ms) +// }; +// +// // if (img.First().Format == MagickFormat.Png) { +// // img.Dispose(); +// // ms.Position = 0; +// // img = new MagickImageCollection(ms, MagickFormat.APng); +// // } +// +// return img; +// } +// } \ No newline at end of file diff --git a/extra/admin-api/Spacebar.Cdn/Program.cs b/extra/admin-api/Spacebar.Cdn/Program.cs index 911c72593..1cb87d07d 100644 --- a/extra/admin-api/Spacebar.Cdn/Program.cs +++ b/extra/admin-api/Spacebar.Cdn/Program.cs @@ -2,19 +2,25 @@ using ArcaneLibs; using ImageMagick; using Microsoft.EntityFrameworkCore; using Spacebar.AdminApi.TestClient.Services.Services; +using Spacebar.Cdn.Services; using Spacebar.Interop.Cdn.Abstractions; using Spacebar.Models.Db.Contexts; var builder = WebApplication.CreateBuilder(args); +var u = new Uri("http://unix:/var/x/y/z/x/z"); +Console.WriteLine(u.LocalPath); if (!string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("APPSETTINGS_PATH"))) builder.Configuration.AddJsonFile(Environment.GetEnvironmentVariable("APPSETTINGS_PATH")!); // Add services to the container. // builder.Services.AddSingleton(new ProxyFileSource("http://cdn.old.server.spacebar.chat")); -builder.Services.AddSingleton(new FilesystemFileSource(Environment.GetEnvironmentVariable("STORAGE_PATH") ?? throw new InvalidOperationException("STORAGE_PATH not set!"))); -builder.Services.AddSingleton(new LruFileCache(1 * 1024 * 1024 * 1024)); -builder.Services.AddSingleton(); +builder.Services.AddSingleton(new FilesystemFileSource(Environment.GetEnvironmentVariable("STORAGE_PATH") ?? + throw new InvalidOperationException("STORAGE_PATH not set!"))); +builder.Services.AddSingleton(sp => + new LruFileCache(sp.GetRequiredService().GetSection("Spacebar:Cdn:LruFileCache:Size").Value is { } val ? int.Parse(val) : 1 * 1024 * 1024 * 1024)); builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); builder.Services.AddDbContextPool(options => { options @@ -27,6 +33,7 @@ builder.Services.AddControllers(); builder.Services.AddOpenApi(); var app = builder.Build(); +app.Services.GetRequiredService().Initialize(); // Configure the HTTP request pipeline. if (app.Environment.IsDevelopment()) app.MapOpenApi(); diff --git a/extra/admin-api/Spacebar.Cdn/Services/CdnWorkerService.cs b/extra/admin-api/Spacebar.Cdn/Services/CdnWorkerService.cs new file mode 100644 index 000000000..2eabf129a --- /dev/null +++ b/extra/admin-api/Spacebar.Cdn/Services/CdnWorkerService.cs @@ -0,0 +1,79 @@ +using System.Diagnostics; +using System.Net.Sockets; +using ArcaneLibs.Extensions; + +namespace Spacebar.Cdn.Services; + +public class CdnWorkerService(SpacebarCdnWorkerConfiguration cfg) : IDisposable { + private int _q8Idx = 0; + private int _q16Idx = 0; + private int _q16HdriIdx = 0; + private HttpClient[] _q8HttpClients = []; + private HttpClient[] _q16HttpClients = []; + private HttpClient[] _q16HdriHttpClients = []; + private List _workerProcesses = []; + + public void Initialize() { + Console.WriteLine("Initializing CDN worker store..."); + Console.WriteLine(" - Q8"); + _q8HttpClients = GetWorkerHttpClients(cfg.Q8Workers); + Console.WriteLine(" - Q16"); + _q16HttpClients = GetWorkerHttpClients(cfg.Q16Workers); + Console.WriteLine(" - Q16-HDRI"); + _q16HdriHttpClients = GetWorkerHttpClients(cfg.Q16HdriWorkers); + Console.WriteLine("Done initializing CDN worker store!"); + } + + private static HttpClient[] GetWorkerHttpClients(List urls) { + List results = []; + foreach (var url in urls) { + Console.WriteLine(" - Handling worker URI/path: " + url); + if (url.StartsWith("http://unix:")) results.Add(UnixSocketHttpClientFactory.GetHttpClientForSocket(url)); + else if (url.StartsWith("http://") || url.StartsWith("https://")) results.Add(new HttpClient() { BaseAddress = new(url) }); + // else if (File.Exists(url)) { } + else throw new NotImplementedException($"Don't know how to handle worker URL \"{url}\""); + } + + Console.WriteLine($" => {results.Count} results"); + return results.ToArray(); + } + + public HttpClient GetRawClient(string variant) { + Console.WriteLine($"GetRawClient: q8={_q8Idx}/{_q8HttpClients.Length} q16={_q16Idx}/{_q16HttpClients.Length} q16Hdri={_q16HdriIdx}/{_q16HdriHttpClients.Length} "); + return variant switch { + "q8" => _q8HttpClients[_q8Idx++ % _q8HttpClients.Length], + "q16" => _q16HttpClients[_q16Idx++ % _q16HttpClients.Length], + "q16-hdr" => _q16HdriHttpClients[_q16HdriIdx++ % _q16HdriHttpClients.Length], + _ => throw new ArgumentException("Variant must be q8/q16/q16-hdri") + }; + } + + public void Dispose() { + foreach (var hc in _q8HttpClients) hc.Dispose(); + foreach (var hc in _q16HttpClients) hc.Dispose(); + foreach (var hc in _q16HdriHttpClients) hc.Dispose(); + foreach (var proc in _workerProcesses) { + if (proc.HasExited) continue; + proc.Kill(true); + proc.Dispose(); + } + } +} + +internal class UnixSocketHttpClientFactory { + internal static HttpClient GetHttpClientForSocket(string url) { + var socketPath = new Uri(url).LocalPath; + var httpHandler = new SocketsHttpHandler { + ConnectCallback = async (ctx, ct) => { + var socket = new Socket(AddressFamily.Unix, SocketType.Stream, ProtocolType.IP); + var endpoint = new UnixDomainSocketEndPoint(socketPath); + await socket.ConnectAsync(endpoint, ct); + // tell the socket handler it owns the stream and can dispose it + return new NetworkStream(socket, ownsSocket: true); + } + }; + return new HttpClient(httpHandler) { + BaseAddress = new Uri("http://localhost") // just a dummy value, since dotnet still wants :) + }; + } +} \ No newline at end of file diff --git a/extra/admin-api/Spacebar.Cdn/Services/DiscordImageResizeService.cs b/extra/admin-api/Spacebar.Cdn/Services/DiscordImageResizeService.cs index 382845e27..d3b10b087 100644 --- a/extra/admin-api/Spacebar.Cdn/Services/DiscordImageResizeService.cs +++ b/extra/admin-api/Spacebar.Cdn/Services/DiscordImageResizeService.cs @@ -1,6 +1,4 @@ using System.Runtime.Serialization; -using ImageMagick; -using Microsoft.AspNetCore.Mvc; namespace Spacebar.AdminApi.TestClient.Services.Services; @@ -23,36 +21,4 @@ public enum DiscordImageResizeQuality { [EnumMember(Value = "low")] Low, [EnumMember(Value = "high")] High, [EnumMember(Value = "lossless")] Lossless -} - -public class DiscordImageResizeService(PixelArtDetectionService pads) { - public MagickImageCollection Apply(MagickImageCollection img, DiscordImageResizeParams resizeParams) { - if (resizeParams.Passthrough) return img; - if (img.First().Format == MagickFormat.Gif) { - Console.WriteLine("Coalescing gif for resize"); - img.Coalesce(); - } - - if (resizeParams.Size.HasValue) { - if (resizeParams.Size > 4096) - resizeParams.Size = 4096; - - if (img.Max(x => Math.Max(x.Height, x.Width)) > resizeParams.Size || resizeParams.SpacebarAllowUpscale) { - Parallel.ForEach(img, new ParallelOptions() { MaxDegreeOfParallelism = 8 }, frame => { - if (resizeParams.Size.HasValue) { - uint oldWidth = frame.Width, oldHeight = frame.Height; - frame.Resize(resizeParams.Size.Value, resizeParams.Size.Value, pads.IsPixelArt(frame) ? FilterType.Point : FilterType.Gaussian); - Console.WriteLine($"Resized frame from {oldWidth}x{oldHeight} to {frame.Width}x{frame.Height}: {img.IndexOf(frame)}/{img.Count}"); - } - }); - } - } - - if (img.First().Format == MagickFormat.Gif && resizeParams.SpacebarOptimiseGif) { - Console.WriteLine("Optimizing gif after resize"); - img.OptimizePlus(); - } - - return img; - } } \ No newline at end of file diff --git a/extra/admin-api/Spacebar.Cdn/Services/SpacebarCdnWorkerConfiguration.cs b/extra/admin-api/Spacebar.Cdn/Services/SpacebarCdnWorkerConfiguration.cs new file mode 100644 index 000000000..68e9a300f --- /dev/null +++ b/extra/admin-api/Spacebar.Cdn/Services/SpacebarCdnWorkerConfiguration.cs @@ -0,0 +1,18 @@ +using Newtonsoft.Json; + +namespace Spacebar.Cdn.Services; + +public class SpacebarCdnWorkerConfiguration { + public SpacebarCdnWorkerConfiguration(IConfiguration config) { + config.GetRequiredSection("Spacebar").GetRequiredSection("Cdn").GetRequiredSection("Workers").Bind(this); + } + + [JsonProperty("q8")] + public List Q8Workers { get; set; } = []; + + [JsonProperty("q16")] + public List Q16Workers { get; set; } = []; + + [JsonProperty("q16-hdri")] + public List Q16HdriWorkers { get; set; } = []; +} \ No newline at end of file diff --git a/extra/admin-api/Spacebar.Cdn/gcc-lib-lib b/extra/admin-api/Spacebar.Cdn/gcc-lib-lib deleted file mode 120000 index 84ce47f59..000000000 --- a/extra/admin-api/Spacebar.Cdn/gcc-lib-lib +++ /dev/null @@ -1 +0,0 @@ -/nix/store/xm08aqdd7pxcdhm0ak6aqb1v7hw5q6ri-gcc-14.3.0-lib \ No newline at end of file diff --git a/extra/admin-api/Spacebar.Cdn/opencl-lib b/extra/admin-api/Spacebar.Cdn/opencl-lib deleted file mode 120000 index 0c99ced3d..000000000 --- a/extra/admin-api/Spacebar.Cdn/opencl-lib +++ /dev/null @@ -1 +0,0 @@ -/nix/store/iayix7sf8n2mcjihl16xxpppv6j2syjm-clr-6.4.3 \ No newline at end of file diff --git a/extra/admin-api/Spacebar.Cdn/result-lib b/extra/admin-api/Spacebar.Cdn/result-lib deleted file mode 120000 index 84ce47f59..000000000 --- a/extra/admin-api/Spacebar.Cdn/result-lib +++ /dev/null @@ -1 +0,0 @@ -/nix/store/xm08aqdd7pxcdhm0ak6aqb1v7hw5q6ri-gcc-14.3.0-lib \ No newline at end of file