Cdn cs worker cleanup

This commit is contained in:
Rory&
2026-04-11 21:38:36 +02:00
parent 6b429fcde9
commit 1d76274546
19 changed files with 220 additions and 253 deletions
@@ -1,8 +0,0 @@
using Microsoft.AspNetCore.Mvc;
namespace Spacebar.Cdn.Worker.Controllers;
[ApiController]
public class ImageResizeController : ControllerBase {
}
@@ -75,7 +75,7 @@ public class DiscordImageResizeService {
// pads.IsPixelArt(frame)
frame.Resize(resizeParams.Size.Value, resizeParams.Size.Value,
resizeParams.Quality == DiscordImageResizeQuality.Low ? FilterType.Point : FilterType.Gaussian);
Console.WriteLine($"Resized frame from {oldWidth}x{oldHeight} to {frame.Width}x{frame.Height}: {img.IndexOf(frame)}/{img.Count}");
Console.WriteLine($"Resized frame from {oldWidth}x{oldHeight} to {frame.Width}x{frame.Height}: {img.IndexOf(frame)+1}/{img.Count}");
}
});
}
+1 -1
View File
@@ -40,7 +40,7 @@ public static class Mimes {
public static string GetMime(MagickFormat fmt) => fmt switch {
MagickFormat.Png => "image/png",
MagickFormat.Jpeg => "image/jpeg",
MagickFormat.Jpeg or MagickFormat.Jpg => "image/jpeg",
MagickFormat.Gif => "image/gif",
MagickFormat.Bmp => "image/bmp",
MagickFormat.Tiff => "image/tiff",
@@ -1,10 +1,5 @@
using System.Diagnostics;
using System.Diagnostics.CodeAnalysis;
using System.Globalization;
using ArcaneLibs;
using ArcaneLibs.Extensions.Streams;
using ImageMagick;
using Microsoft.AspNetCore.Mvc;
using Spacebar.AdminApi.TestClient.Services.Helpers;
using Spacebar.AdminApi.TestClient.Services.Services;
using Spacebar.Cdn.Worker;
@@ -12,7 +7,7 @@ using Spacebar.Interop.Cdn.Abstractions;
var builder = WebApplication.CreateBuilder(args);
var sw = Stopwatch.StartNew();
// var sw = Stopwatch.StartNew();
Console.WriteLine("Pre-initializing Magick.NET...");
// OpenCL.IsEnabled = true;
MagickNET.Initialize();
@@ -17,8 +17,4 @@
<ProjectReference Include="..\Interop\Spacebar.Interop.Cdn.Abstractions\Spacebar.Interop.Cdn.Abstractions.csproj" Condition="'$(ContinuousIntegrationBuild)'!='true'"/>
<PackageReference Include="Spacebar.Interop.Cdn.Abstractions" Version="*-preview*" Condition="'$(ContinuousIntegrationBuild)'=='true'"/>
</ItemGroup>
<ItemGroup>
<Folder Include="Renderers\" />
</ItemGroup>
</Project>
@@ -17,8 +17,4 @@
<ProjectReference Include="..\Interop\Spacebar.Interop.Cdn.Abstractions\Spacebar.Interop.Cdn.Abstractions.csproj" Condition="'$(ContinuousIntegrationBuild)'!='true'"/>
<PackageReference Include="Spacebar.Interop.Cdn.Abstractions" Version="*-preview*" Condition="'$(ContinuousIntegrationBuild)'=='true'"/>
</ItemGroup>
<ItemGroup>
<Folder Include="Renderers\" />
</ItemGroup>
</Project>
@@ -17,8 +17,4 @@
<ProjectReference Include="..\Interop\Spacebar.Interop.Cdn.Abstractions\Spacebar.Interop.Cdn.Abstractions.csproj" Condition="'$(ContinuousIntegrationBuild)'!='true'"/>
<PackageReference Include="Spacebar.Interop.Cdn.Abstractions" Version="*-preview*" Condition="'$(ContinuousIntegrationBuild)'=='true'"/>
</ItemGroup>
<ItemGroup>
<Folder Include="Renderers\" />
</ItemGroup>
</Project>
@@ -17,8 +17,4 @@
<ProjectReference Include="..\Interop\Spacebar.Interop.Cdn.Abstractions\Spacebar.Interop.Cdn.Abstractions.csproj" Condition="'$(ContinuousIntegrationBuild)'!='true'"/>
<PackageReference Include="Spacebar.Interop.Cdn.Abstractions" Version="*-preview*" Condition="'$(ContinuousIntegrationBuild)'=='true'"/>
</ItemGroup>
<ItemGroup>
<Folder Include="Renderers\" />
</ItemGroup>
</Project>
@@ -17,8 +17,4 @@
<ProjectReference Include="..\Interop\Spacebar.Interop.Cdn.Abstractions\Spacebar.Interop.Cdn.Abstractions.csproj" Condition="'$(ContinuousIntegrationBuild)'!='true'"/>
<PackageReference Include="Spacebar.Interop.Cdn.Abstractions" Version="*-preview*" Condition="'$(ContinuousIntegrationBuild)'=='true'"/>
</ItemGroup>
<ItemGroup>
<Folder Include="Renderers\" />
</ItemGroup>
</Project>
@@ -17,8 +17,4 @@
<ProjectReference Include="..\Interop\Spacebar.Interop.Cdn.Abstractions\Spacebar.Interop.Cdn.Abstractions.csproj" Condition="'$(ContinuousIntegrationBuild)'!='true'"/>
<PackageReference Include="Spacebar.Interop.Cdn.Abstractions" Version="*-preview*" Condition="'$(ContinuousIntegrationBuild)'=='true'"/>
</ItemGroup>
<ItemGroup>
<Folder Include="Renderers\" />
</ItemGroup>
</Project>
@@ -17,8 +17,4 @@
<ProjectReference Include="..\Interop\Spacebar.Interop.Cdn.Abstractions\Spacebar.Interop.Cdn.Abstractions.csproj" Condition="'$(ContinuousIntegrationBuild)'!='true'"/>
<PackageReference Include="Spacebar.Interop.Cdn.Abstractions" Version="*-preview*" Condition="'$(ContinuousIntegrationBuild)'=='true'"/>
</ItemGroup>
<ItemGroup>
<Folder Include="Renderers\" />
</ItemGroup>
</Project>
@@ -17,8 +17,4 @@
<ProjectReference Include="..\Interop\Spacebar.Interop.Cdn.Abstractions\Spacebar.Interop.Cdn.Abstractions.csproj" Condition="'$(ContinuousIntegrationBuild)'!='true'"/>
<PackageReference Include="Spacebar.Interop.Cdn.Abstractions" Version="*-preview*" Condition="'$(ContinuousIntegrationBuild)'=='true'"/>
</ItemGroup>
<ItemGroup>
<Folder Include="Renderers\" />
</ItemGroup>
</Project>
@@ -17,8 +17,4 @@
<ProjectReference Include="..\Interop\Spacebar.Interop.Cdn.Abstractions\Spacebar.Interop.Cdn.Abstractions.csproj" Condition="'$(ContinuousIntegrationBuild)'!='true'"/>
<PackageReference Include="Spacebar.Interop.Cdn.Abstractions" Version="*-preview*" Condition="'$(ContinuousIntegrationBuild)'=='true'"/>
</ItemGroup>
<ItemGroup>
<Folder Include="Renderers\" />
</ItemGroup>
</Project>
@@ -0,0 +1,16 @@
namespace Spacebar.Cdn;
public class SpacebarCdnConfiguration
{
public SpacebarCdnConfiguration(IConfiguration configuration) {
configuration.GetRequiredSection("Spacebar").GetRequiredSection("Cdn").Bind(this);
}
// public
}
// public class SpacebarCdnWorkerConfiguration
// {
// public
// }
@@ -1,56 +1,56 @@
using ArcaneLibs.Extensions.Streams;
using Microsoft.AspNetCore.Mvc;
using Spacebar.AdminApi.TestClient.Services.Services;
using Spacebar.Cdn.Extensions;
using Spacebar.Interop.Cdn.Abstractions;
namespace Spacebar.Cdn.Controllers;
[ApiController]
public class GetImageController(LruFileCache lfc, IFileSource fs, DiscordImageResizeService dirs) : ImageController {
// [HttpGet("/avatars/{_:required}")]
[HttpGet("/emojis/{emoji_id:required}.{ext:required}")]
[HttpGet("/stickers/{sticker_id:required}.{ext:required}")]
// [HttpGet("/avatars/{user_id:required}/{avatar_hash:required}.{ext:required}")]
[HttpGet("/banners/{user_id:required}/{user_banner:required}.{ext:required}")]
public async Task<IActionResult> GetImage(string? ext) {
var originalKey = fs.BaseUrl + Request.Path;
var cacheKey = Request.Path + Request.QueryString;
DiscordImageResizeParams resizeParams = GetResizeParams();
var entry = await lfc.GetOrAdd(cacheKey, async () => {
var original = await fs.GetFile(Request.Path);
if (Request.Query.Any()) {
using var img = await original.ToMagickImageCollectionAsync();
dirs.Apply(img, resizeParams);
var outStream = new MemoryStream();
await img.WriteAsync(outStream, img.First().Format);
outStream.Position = 0;
return new LruFileCache.Entry() {
Data = outStream.ReadToEnd().ToArray(),
MimeType = original.MimeType
};
}
return new LruFileCache.Entry() {
Data = original.Stream.ReadToEnd().ToArray(),
MimeType = original.MimeType
};
});
// byte array with mime type result
return new FileContentResult(entry.Data, entry.MimeType);
}
// TODO: is message_id required? Can't tell from discord.food: /attachments/{channel_id}/[{message_id]/attachment_id}/{attachment_filename}
[HttpGet("/attachments/{channel_id:required}/{message_id:required}/{attachment_id:required}/{filename:required}")]
[HttpGet("/ephemeral-attachments/{application_id:required}/{attachment_id:required}/{attachment_filename:required}")]
public async Task<IActionResult> GetAttachmentImage() {
// TODO: url signing, file type checks
return await GetImage("");
}
}
// using ArcaneLibs.Extensions.Streams;
// using Microsoft.AspNetCore.Mvc;
// using Spacebar.AdminApi.TestClient.Services.Services;
// using Spacebar.Cdn.Extensions;
// using Spacebar.Interop.Cdn.Abstractions;
//
// namespace Spacebar.Cdn.Controllers;
//
// [ApiController]
// public class GetImageController(LruFileCache lfc, IFileSource fs, DiscordImageResizeService dirs) : ImageController {
// // [HttpGet("/avatars/{_:required}")]
// [HttpGet("/emojis/{emoji_id:required}.{ext:required}")]
// [HttpGet("/stickers/{sticker_id:required}.{ext:required}")]
// // [HttpGet("/avatars/{user_id:required}/{avatar_hash:required}.{ext:required}")]
// [HttpGet("/banners/{user_id:required}/{user_banner:required}.{ext:required}")]
// public async Task<IActionResult> GetImage(string? ext) {
// var originalKey = fs.BaseUrl + Request.Path;
// var cacheKey = Request.Path + Request.QueryString;
//
// DiscordImageResizeParams resizeParams = Request.GetResizeParams();
//
// var entry = await lfc.GetOrAdd(cacheKey, async () => {
// var original = await fs.GetFile(Request.Path);
//
// if (Request.Query.Any()) {
// using var img = await original.ToMagickImageCollectionAsync();
// dirs.Apply(img, resizeParams);
//
// var outStream = new MemoryStream();
// await img.WriteAsync(outStream, img.First().Format);
// outStream.Position = 0;
//
// return new LruFileCache.Entry() {
// Data = outStream.ReadToEnd().ToArray(),
// MimeType = original.MimeType
// };
// }
//
// return new LruFileCache.Entry() {
// Data = original.Stream.ReadToEnd().ToArray(),
// MimeType = original.MimeType
// };
// });
//
// // byte array with mime type result
// return new FileContentResult(entry.Data, entry.MimeType);
// }
//
// // TODO: is message_id required? Can't tell from discord.food: /attachments/{channel_id}/[{message_id]/attachment_id}/{attachment_filename}
// [HttpGet("/attachments/{channel_id:required}/{message_id:required}/{attachment_id:required}/{filename:required}")]
// [HttpGet("/ephemeral-attachments/{application_id:required}/{attachment_id:required}/{attachment_filename:required}")]
// public async Task<IActionResult> GetAttachmentImage() {
// // TODO: url signing, file type checks
// return await GetImage("");
// }
// }
@@ -1,66 +1,66 @@
using ArcaneLibs.Extensions.Streams;
using Microsoft.AspNetCore.Mvc;
using Microsoft.OpenApi;
using Spacebar.AdminApi.TestClient.Services.Services;
using Spacebar.Cdn.Extensions;
using Spacebar.Interop.Cdn.Abstractions;
namespace Spacebar.Cdn.Controllers;
[ApiController]
public class StaticAssetController(LruFileCache lfc, IFileSource fs, DiscordImageResizeService dirs) : ImageController {
private static readonly Dictionary<string, string> defaultAvatarHashMap = new() {
{ "0", "4a8562cf00887030c416d3ec2d46385a" },
{ "1", "9b0bb198936784c45c72833cc426cc55" },
{ "2", "22341bdb500c7b63a93bbce957d1601e" },
{ "3", "d9977836b82058bf2f74eebd50edc095" },
{ "4", "9d6ddb4e4d899a533a8cc617011351c9" },
{ "5", "7213ab6677377974697dfdfbaf5f6a6f" },
};
private static readonly Dictionary<string, string> defaultGroupDMAvatarHashMap = new() {
{ "0", "3b70bb66089c60f8be5e214bf8574c9d" },
{ "1", "9581acd31832465bdeaa5385b0e919a3" },
{ "2", "a8a4727cf2dc2939bd3c657fad4463fa" },
{ "3", "2e46fe14586f8e95471c0917f56726b5" },
{ "4", "fac7e78de9753d4a37083bba74c1d9ef" },
{ "5", "4ab900144b0865430dc9be825c838faa" },
{ "6", "1276374a404452756f3c9cc2601508a5" },
{ "7", "904bf9f1b61f53ef4a3b7a893afeabe3" },
};
// png only
[HttpGet("/embed/avatars/{userIndex}.{ext}")]
public async Task<IActionResult> GetDefaultUserAvatar(string userIndex, string ext) {
var cacheKey = Request.Path + Request.QueryString;
DiscordImageResizeParams resizeParams = GetResizeParams();
var entry = await lfc.GetOrAdd(cacheKey, async () => {
var original = await fs.GetFile(Request.Path);
if (Request.Query.Any()) {
using var img = await original.ToMagickImageCollectionAsync();
dirs.Apply(img, resizeParams);
var outStream = new MemoryStream();
await img.WriteAsync(outStream, img.First().Format);
outStream.Position = 0;
return new LruFileCache.Entry() {
Data = outStream.ReadToEnd().ToArray(),
MimeType = original.MimeType
};
}
return new LruFileCache.Entry() {
Data = original.Stream.ReadToEnd().ToArray(),
MimeType = original.MimeType
};
});
// byte array with mime type result
return new FileContentResult(entry.Data, entry.MimeType);
}
}
// using ArcaneLibs.Extensions.Streams;
// using Microsoft.AspNetCore.Mvc;
// using Microsoft.OpenApi;
// using Spacebar.AdminApi.TestClient.Services.Services;
// using Spacebar.Cdn.Extensions;
// using Spacebar.Interop.Cdn.Abstractions;
//
// namespace Spacebar.Cdn.Controllers;
//
// [ApiController]
// public class StaticAssetController(LruFileCache lfc, IFileSource fs, DiscordImageResizeService dirs) : ImageController {
// private static readonly Dictionary<string, string> defaultAvatarHashMap = new() {
// { "0", "4a8562cf00887030c416d3ec2d46385a" },
// { "1", "9b0bb198936784c45c72833cc426cc55" },
// { "2", "22341bdb500c7b63a93bbce957d1601e" },
// { "3", "d9977836b82058bf2f74eebd50edc095" },
// { "4", "9d6ddb4e4d899a533a8cc617011351c9" },
// { "5", "7213ab6677377974697dfdfbaf5f6a6f" },
// };
//
// private static readonly Dictionary<string, string> defaultGroupDMAvatarHashMap = new() {
// { "0", "3b70bb66089c60f8be5e214bf8574c9d" },
// { "1", "9581acd31832465bdeaa5385b0e919a3" },
// { "2", "a8a4727cf2dc2939bd3c657fad4463fa" },
// { "3", "2e46fe14586f8e95471c0917f56726b5" },
// { "4", "fac7e78de9753d4a37083bba74c1d9ef" },
// { "5", "4ab900144b0865430dc9be825c838faa" },
// { "6", "1276374a404452756f3c9cc2601508a5" },
// { "7", "904bf9f1b61f53ef4a3b7a893afeabe3" },
// };
//
// // png only
// [HttpGet("/embed/avatars/{userIndex}.{ext}")]
// public async Task<IActionResult> GetDefaultUserAvatar(string userIndex, string ext) {
//
// var cacheKey = Request.Path + Request.QueryString;
//
// DiscordImageResizeParams resizeParams = GetResizeParams();
//
// var entry = await lfc.GetOrAdd(cacheKey, async () => {
// var original = await fs.GetFile(Request.Path);
//
// if (Request.Query.Any()) {
// using var img = await original.ToMagickImageCollectionAsync();
// dirs.Apply(img, resizeParams);
//
// var outStream = new MemoryStream();
// await img.WriteAsync(outStream, img.First().Format);
// outStream.Position = 0;
//
// return new LruFileCache.Entry() {
// Data = outStream.ReadToEnd().ToArray(),
// MimeType = original.MimeType
// };
// }
//
// return new LruFileCache.Entry() {
// Data = original.Stream.ReadToEnd().ToArray(),
// MimeType = original.MimeType
// };
// });
//
// // byte array with mime type result
// return new FileContentResult(entry.Data, entry.MimeType);
// }
// }
@@ -1,77 +1,77 @@
using ArcaneLibs.Extensions.Streams;
using Microsoft.AspNetCore.Mvc;
using Spacebar.AdminApi.TestClient.Services.Services;
using Spacebar.Cdn.Extensions;
using Spacebar.Interop.Cdn.Abstractions;
namespace Spacebar.Cdn.Controllers;
[ApiController]
public class UserController(LruFileCache lfc, IFileSource fs, DiscordImageResizeService dirs) : ImageController {
[HttpGet("/avatars/{userId}/{hash}.{ext}")]
public async Task<IActionResult> GetUserAvatar(string userId, string hash, string ext) {
var originalKey = fs.BaseUrl + Request.Path;
var cacheKey = Request.Path + Request.QueryString;
DiscordImageResizeParams resizeParams = GetResizeParams();
var entry = await lfc.GetOrAdd(cacheKey, async () => {
var original = await fs.GetFile(Request.Path);
if (Request.Query.Any()) {
using var img = await original.ToMagickImageCollectionAsync();
dirs.Apply(img, resizeParams);
var outStream = new MemoryStream();
await img.WriteAsync(outStream, img.First().Format);
outStream.Position = 0;
return new LruFileCache.Entry() {
Data = outStream.ReadToEnd().ToArray(),
MimeType = original.MimeType
};
}
return new LruFileCache.Entry() {
Data = original.Stream.ReadToEnd().ToArray(),
MimeType = original.MimeType
};
});
// byte array with mime type result
return new FileContentResult(entry.Data, entry.MimeType);
}
[HttpGet("/banners/{userId}/{hash}.{ext}")]
public async Task<IActionResult> GetUserBanner(string userId, string hash, string ext) {
var originalKey = fs.BaseUrl + Request.Path;
var cacheKey = Request.Path + Request.QueryString;
DiscordImageResizeParams resizeParams = GetResizeParams();
var entry = await lfc.GetOrAdd(cacheKey, async () => {
var original = await fs.GetFile(Request.Path);
if (Request.Query.Any()) {
using var img = await original.ToMagickImageCollectionAsync();
dirs.Apply(img, resizeParams);
var outStream = new MemoryStream();
await img.WriteAsync(outStream, img.First().Format);
outStream.Position = 0;
return new LruFileCache.Entry() {
Data = outStream.ReadToEnd().ToArray(),
MimeType = original.MimeType
};
}
return new LruFileCache.Entry() {
Data = original.Stream.ReadToEnd().ToArray(),
MimeType = original.MimeType
};
});
// byte array with mime type result
return new FileContentResult(entry.Data, entry.MimeType);
}
}
// using ArcaneLibs.Extensions.Streams;
// using Microsoft.AspNetCore.Mvc;
// using Spacebar.AdminApi.TestClient.Services.Services;
// using Spacebar.Cdn.Extensions;
// using Spacebar.Interop.Cdn.Abstractions;
//
// namespace Spacebar.Cdn.Controllers;
//
// [ApiController]
// public class UserController(LruFileCache lfc, IFileSource fs, DiscordImageResizeService dirs) : ImageController {
// [HttpGet("/avatars/{userId}/{hash}.{ext}")]
// public async Task<IActionResult> GetUserAvatar(string userId, string hash, string ext) {
// var originalKey = fs.BaseUrl + Request.Path;
// var cacheKey = Request.Path + Request.QueryString;
//
// DiscordImageResizeParams resizeParams = GetResizeParams();
//
// var entry = await lfc.GetOrAdd(cacheKey, async () => {
// var original = await fs.GetFile(Request.Path);
//
// if (Request.Query.Any()) {
// using var img = await original.ToMagickImageCollectionAsync();
// dirs.Apply(img, resizeParams);
//
// var outStream = new MemoryStream();
// await img.WriteAsync(outStream, img.First().Format);
// outStream.Position = 0;
//
// return new LruFileCache.Entry() {
// Data = outStream.ReadToEnd().ToArray(),
// MimeType = original.MimeType
// };
// }
//
// return new LruFileCache.Entry() {
// Data = original.Stream.ReadToEnd().ToArray(),
// MimeType = original.MimeType
// };
// });
//
// // byte array with mime type result
// return new FileContentResult(entry.Data, entry.MimeType);
// }
// [HttpGet("/banners/{userId}/{hash}.{ext}")]
// public async Task<IActionResult> GetUserBanner(string userId, string hash, string ext) {
// var originalKey = fs.BaseUrl + Request.Path;
// var cacheKey = Request.Path + Request.QueryString;
//
// DiscordImageResizeParams resizeParams = GetResizeParams();
//
// var entry = await lfc.GetOrAdd(cacheKey, async () => {
// var original = await fs.GetFile(Request.Path);
//
// if (Request.Query.Any()) {
// using var img = await original.ToMagickImageCollectionAsync();
// dirs.Apply(img, resizeParams);
//
// var outStream = new MemoryStream();
// await img.WriteAsync(outStream, img.First().Format);
// outStream.Position = 0;
//
// return new LruFileCache.Entry() {
// Data = outStream.ReadToEnd().ToArray(),
// MimeType = original.MimeType
// };
// }
//
// return new LruFileCache.Entry() {
// Data = original.Stream.ReadToEnd().ToArray(),
// MimeType = original.MimeType
// };
// });
//
// // byte array with mime type result
// return new FileContentResult(entry.Data, entry.MimeType);
// }
// }
+1 -1
View File
@@ -39,7 +39,7 @@ public static class Mimes {
public static string GetMime(MagickFormat fmt) => fmt switch {
MagickFormat.Png => "image/png",
MagickFormat.Jpeg => "image/jpeg",
MagickFormat.Jpeg or MagickFormat.Jpg => "image/jpeg",
MagickFormat.Gif => "image/gif",
MagickFormat.Bmp => "image/bmp",
MagickFormat.Tiff => "image/tiff",
+1 -1
View File
@@ -13,8 +13,8 @@ if (!string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("APPSETTINGS_P
// builder.Services.AddSingleton<IFileSource>(new ProxyFileSource("http://cdn.old.server.spacebar.chat"));
builder.Services.AddSingleton<IFileSource>(new FilesystemFileSource(Environment.GetEnvironmentVariable("STORAGE_PATH") ?? throw new InvalidOperationException("STORAGE_PATH not set!")));
builder.Services.AddSingleton<LruFileCache>(new LruFileCache(1 * 1024 * 1024 * 1024));
builder.Services.AddSingleton<PixelArtDetectionService>();
builder.Services.AddSingleton<DiscordImageResizeService>();
builder.Services.AddSingleton<PixelArtDetectionService>();
builder.Services.AddDbContextPool<SpacebarDbContext>(options => {
options