using System.Diagnostics; using ArcaneLibs.Extensions; using Microsoft.EntityFrameworkCore; using Spacebar.Interop.Cdn.Signing; using Spacebar.Models.Db.Contexts; namespace Spacebar.Cdn.Fsck; public class FsckService(ILogger logger, IServiceScopeFactory serviceScopeFactory, MigrationFileStores stores, CdnSigningService sigService) : IHostedService { private SpacebarDbContext _db = null!; public async Task StartAsync(CancellationToken cancellationToken) { var sw = Stopwatch.StartNew(); await using var scope = serviceScopeFactory.CreateAsyncScope(); _db = scope.ServiceProvider.GetRequiredService(); logger.LogInformation("Starting migrations from {source} to {dest}...", $"{stores.From.GetType().FullName}({stores.From.BaseUrl})", $"{stores.To.GetType().FullName}({stores.To.BaseUrl})"); await RunFsckAsync("User Avatars", "/avatars", EnumerateUserAvatarFilesAsync(), cancellationToken); await RunFsckAsync("User Banners", "/banners", EnumerateUserBannerPathsAsync(), cancellationToken); await RunFsckAsync("Guild Icons", "/icons", EnumerateGuildIconPathsAsync(), cancellationToken); await RunFsckAsync("Stickers", "/stickers", EnumerateStickerPathsAsync(), cancellationToken); await RunFsckAsync("Emojis", "/emojis", EnumerateEmojiPathsAsync(), cancellationToken); // var atts = EnumerateAttachmentPathsAsync(); // var refreshedAtts = new List(); // var attRefreshedTasks = atts.Chunk(40).Select(async x => { // var req = // return new FsckItem[10]; // }); // await foreach (var attRefreshedChunk in attRefreshedTasks.ToAsyncResultEnumerable()) { // refreshedAtts.AddRange(attRefreshedChunk); // } await RunFsckAsync("Attachments", "/attachments", EnumerateAttachmentPathsAsync(), cancellationToken); logger.LogInformation("Fsck complete in {time}.", sw.Elapsed); } public async Task StopAsync(CancellationToken cancellationToken) { } private readonly Stopwatch _lastUpdateSw = Stopwatch.StartNew(); private readonly SemaphoreSlim _fsckSemaphore = new(32, 32); public struct FsckItem { public string Path; public string ItemId; } private async Task RunFsckAsync(string name, string path, IQueryable items, CancellationToken? cancellationToken = null) { int i = 0, notFound = 0, alreadyLocal = 0, count = await items.CountAsync(); List tasks = []; await foreach (var item in items.AsAsyncEnumerable()) { tasks.Add(Task.Run(async () => { try { await _fsckSemaphore.WaitAsync(); if (cancellationToken?.IsCancellationRequested ?? false) return; if (await stores.To.FileExists(item.Path)) { alreadyLocal++; logger.LogInformation("TO: {itemType} {itemId} already exists at {path}, skipping.", name, item.ItemId, item.Path); } else if (!await stores.From.FileExists(item.Path)) { notFound++; logger.LogWarning("FROM: {itemType} {itemId} is missing at {path}", name, item.ItemId, stores.From.BaseUrl + item.Path); } else { // logger.LogInformation("Migrating {itemType} {itemId} at {path}", name, item.ItemId, item.Path); await using var f = await stores.From.GetFile(item.Path); // logger.LogInformation("Got file {itemType} {itemId} at {path}, writing to destination...", name, item.ItemId, item.Path); await stores.To.WriteFile(item.Path, f.Stream); } if (true || _lastUpdateSw.ElapsedMilliseconds >= 1000 / 30 || i == 0) { _lastUpdateSw.Restart(); Console.Write($"{name} download: {i}/{count}: {item.Path,-64}\r"); } i++; } catch (Exception ex) { logger.LogError(ex, "Error processing {itemType} {itemId} at {path}: {message}", name, item.ItemId, item.Path, ex.Message); } finally { _fsckSemaphore.Release(); } })); } await Task.WhenAll(tasks); logger.LogInformation("Validated {count} items for {path}: {alreadyLocal} already local, {notFound} not found", i, path, alreadyLocal, notFound); } #region User Assets public IQueryable EnumerateUserAvatarFilesAsync() => _db.Users .Where(x => !string.IsNullOrWhiteSpace(x.Avatar)) .OrderBy(x => x.Id) .Select(x => new FsckItem { Path = $"/avatars/{x.Id}/{x.Avatar}", ItemId = x.Id }); public IQueryable EnumerateUserBannerPathsAsync() => _db.Users .Where(x => !string.IsNullOrWhiteSpace(x.Banner)) .OrderBy(x => x.Id) .Select(x => new FsckItem { Path = $"/banners/{x.Id}/{x.Banner}", ItemId = x.Id }); #endregion #region Guild Assets public IQueryable EnumerateGuildIconPathsAsync() => _db.Guilds .Where(x => !string.IsNullOrWhiteSpace(x.Icon)) .OrderBy(x => x.Id) .Select(x => new FsckItem { Path = $"/icons/{x.Id}/{x.Icon}", ItemId = x.Id }); public IQueryable EnumerateRoleIconPathsAsync() => _db.Roles .Where(x => !string.IsNullOrWhiteSpace(x.Icon)) .OrderBy(x => x.Id) .Select(x => new FsckItem { Path = $"/role-icons/{x.Id}/{x.Icon}", ItemId = x.Id }); public IQueryable EnumerateStickerPathsAsync() => _db.Stickers .OrderBy(x => x.Id) .Select(x => new FsckItem { Path = $"/stickers/{x.Id}.png", ItemId = x.Id }); public IQueryable EnumerateEmojiPathsAsync() => _db.Emojis .OrderBy(x => x.Id) .Select(x => new FsckItem { Path = $"/emojis/{x.Id}", ItemId = x.Id }); #endregion #region Application Assets public IQueryable EnumerateApplicationIconPathsAsync() => _db.Applications .Where(x => !string.IsNullOrWhiteSpace(x.Icon)) .OrderBy(x => x.Id) .Select(x => new FsckItem { Path = $"/app-icons/{x.Id}/{x.Icon}", ItemId = x.Id }); public IQueryable EnumerateApplicationCoverPathsAsync() => _db.Applications .Where(x => !string.IsNullOrWhiteSpace(x.Icon)) .OrderBy(x => x.Id) .Select(x => new FsckItem { Path = $"/app-icons/{x.Id}/{x.CoverImage}", ItemId = x.Id }); // TODO: not implemented? // public IQueryable EnumerateApplicationSplashPathsAsync() => // _db.Applications // .Where(x => !string.IsNullOrWhiteSpace(x.Icon)) // .OrderBy(x => x.Id) // .Select(x => new FsckItem { // Path = $"/app-icons/{x.Id}/{x.OwnerId}", // TODO - no db property for splash? // ItemId = x.Id // }); // // public IQueryable EnumerateApplicationAssets() => // _db.Applications // .Where(x => !string.IsNullOrWhiteSpace(x.Icon)) // .OrderBy(x => x.Id) // .Select(x => new FsckItem { // Path = $"/app-icons/{x.Id}/{x.Icon}", // ItemId = x.Id // }); #endregion #region Attachments public IQueryable EnumerateAttachmentPathsAsync() => _db.Attachments .OrderBy(x => x.Id) .Select(x => new FsckItem { Path = sigService.Sign(new() { Path = $"/attachments/{x.Message!.ChannelId}/{x.Id}/{x.Filename}", IpAddress = "109.128.185.4" }).GetSignedPath(), // Path = $"/attachments/{x.Message!.ChannelId}/{x.Id}/{x.Filename}", ItemId = x.Id }); #endregion }