diff --git a/extra/admin-api/Spacebar.GatewayOffload/Controllers/Op12Controller.cs b/extra/admin-api/Spacebar.GatewayOffload/Controllers/Op12Controller.cs new file mode 100644 index 000000000..2c417bf80 --- /dev/null +++ b/extra/admin-api/Spacebar.GatewayOffload/Controllers/Op12Controller.cs @@ -0,0 +1,219 @@ +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Text.Json; +using System.Text.Json.Serialization; +using ArcaneLibs.Extensions; +using Microsoft.AspNetCore.Mvc; +using Microsoft.EntityFrameworkCore; +using Spacebar.Interop.Authentication.AspNetCore; +using Spacebar.Interop.Replication.Abstractions; +using Spacebar.Models.Db.Contexts; +using Spacebar.Models.Db.Models; + +namespace Spacebar.GatewayOffload.Controllers; + +[ApiController] +[Route("/_spacebar/offload/gateway/GuildSync")] +public class Op12Controller(ILogger logger, SpacebarAspNetAuthenticationService authService, SpacebarDbContext db, IServiceProvider sp) : ControllerBase { + [HttpPost("")] + public async IAsyncEnumerable DoGuildSync(List guildIds) { + var user = await authService.GetCurrentUserAsync(Request); + guildIds = (await db.Members.Where(x => x.Id == user.Id).Select(x => x.GuildId).ToListAsync()) + .Intersect(guildIds) + .OrderByDescending(gi => db.Members.Count(m => m.GuildId == gi)) + .ToList(); + + var syncs = guildIds.Select(GetGuildSyncAsync).ToList().ToAsyncResultEnumerable(); + await foreach (var res in syncs) { + yield return new() { + Origin = "OFFLOAD_GUILD_SYNC", + UserId = user.Id, + Event = "GUILD_SYNC", + CreatedAt = DateTime.Now, + Payload = res + }; + } + } + + private async Task GetGuildSyncAsync(string guildId) { + await using var sc = sp.CreateAsyncScope(); + var offlineTreshold = DateTime.Now.Subtract(TimeSpan.FromDays(14)); + var _db = sc.ServiceProvider.GetRequiredService(); + var memberCount = await _db.Members.Where(x => x.GuildId == guildId).CountAsync(); + var members = await _db.Members.Where(x => x.GuildId == guildId) + .Include(x => x.IdNavigation) + .ThenInclude(x => x.Sessions.Where(s => + !s.IsAdminSession && (s.Status != "offline" && s.Status != "invisible") && (memberCount < 1000 || s.LastSeen >= offlineTreshold))) + .Where(x => x.IdNavigation.Sessions.Count > 0) // ignore members without sessions + .ToListAsync(); + + var mappedPartialUsers = members.Select(x => x.IdNavigation).ToDictionary(x => x.Id, x => new PartialUser() { + Id = x.Id, + Discriminator = x.Discriminator, + Username = x.Username, + AccentColor = x.AccentColor, + Avatar = x.Avatar, + AvatarDecorationData = x.AvatarDecorationData, + Banner = x.Banner, + Bot = x.Bot, + Collectibles = x.Collectibles, + DisplayNameStyles = x.DisplayNameStyles, + // GlobalName = x.GlobalName, + PrimaryGuild = x.PrimaryGuild, + PublicFlags = x.PublicFlags, + System = x.System, + }); + var mappedMembers = members.ToDictionary(m => m.Id, m => new Member() { + User = mappedPartialUsers[m.Id], + AvatarDecorationData = m.AvatarDecorationData, + Avatar = m.Avatar, + Banner = m.Banner, + Collectibles = m.Collectibles, + DisplayNameStyles = m.DisplayNameStyles, + Bio = m.Bio, + Nick = m.Nick + }); + var presences = members.Select(x => x.IdNavigation).Where(x => x.Sessions.Count > 0).ToDictionary(x => x.Id, x => { + var sortedSessions = x.Sessions.OrderByDescending(s => s.LastSeen).ToList(); + return new PresenceResponse() { + GuildId = guildId, + User = mappedPartialUsers[x.Id], + Activities = x.Sessions.SelectMany(s => JsonSerializer.Deserialize(s.Activities)).ToList(), + Status = sortedSessions.FirstOrDefault(s => !string.IsNullOrWhiteSpace(s.Status))?.Status ?? "offline", + ClientStatus = JsonSerializer.Deserialize(sortedSessions.First(s => !string.IsNullOrWhiteSpace(s.ClientStatus)).ClientStatus) + }; + }).Where(x => x.Value.Activities.Count > 0).ToDictionary(); + + var r = new GuildSyncResponse() { + GuildId = guildId, + Members = mappedMembers.Values.ToList(), + Presences = presences.Values.ToList() + }; + return r; + } +} + +public class GuildSyncResponse { + [JsonPropertyName("id")] + public string GuildId { get; set; } + + [JsonPropertyName("presences")] + public List Presences { get; set; } + + [JsonPropertyName("members")] + public List Members { get; set; } +} + +public class PresenceResponse { + [JsonPropertyName("user")] + public required PartialUser User { get; set; } + + [JsonPropertyName("guild_id"), JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? GuildId { get; set; } + + [JsonPropertyName("status")] + public string Status { get; set; } = "unknown"; + + [JsonPropertyName("activities")] + public List Activities { get; set; } + + [JsonPropertyName("hidden_activities")] + + public List HiddenActivities { get; set; } + + [JsonPropertyName("client_status")] + public ClientStatuses ClientStatus { get; set; } + + [JsonPropertyName("has_played_game")] + public bool? HasPlayedGame { get; set; } + + [SuppressMessage("ReSharper", "UnusedMember.Local")] + public class ClientStatuses { + [JsonPropertyName("desktop"), JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Desktop { get; set; } + + [JsonPropertyName("mobile"), JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Mobile { get; set; } + + [JsonPropertyName("web"), JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Web { get; set; } + + [JsonPropertyName("embedded"), JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Embedded { get; set; } + + [JsonPropertyName("vr"), JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Vr { get; set; } + } +} + +[DebuggerDisplay("{Id} ({Username}#{Discriminator})")] +public class PartialUser { + [JsonPropertyName("id")] + public required string Id { get; set; } + + [JsonPropertyName("username")] + public string Username { get; set; } + + [JsonPropertyName("discriminator")] + public string Discriminator { get; set; } + + [JsonPropertyName("global_name")] + public string? GlobalName { get; set; } + + [JsonPropertyName("avatar")] + public string? Avatar { get; set; } + + [JsonPropertyName("avatar_decoration_data")] + public object? AvatarDecorationData { get; set; } + + [JsonPropertyName("collectibles")] + public object? Collectibles { get; set; } + + [JsonPropertyName("display_name_styles")] + public object? DisplayNameStyles { get; set; } + + [JsonPropertyName("primary_guild")] + public object? PrimaryGuild { get; set; } + + [JsonPropertyName("bot")] + public bool? Bot { get; set; } + + [JsonPropertyName("system")] + public bool? System { get; set; } + + [JsonPropertyName("banner")] + public string? Banner { get; set; } + + [JsonPropertyName("accent_color")] + public int? AccentColor { get; set; } + + [JsonPropertyName("public_flags")] + public ulong? PublicFlags { get; set; } +} + +[DebuggerDisplay("{User.Id} ({User.Username}#{User.Discriminator})")] +public class Member { + [JsonPropertyName("user")] + public required PartialUser User { get; set; } + + [JsonPropertyName("nick")] + public string? Nick { get; set; } + + [JsonPropertyName("avatar")] + public string? Avatar { get; set; } + + [JsonPropertyName("avatar_decoration_data")] + public object? AvatarDecorationData { get; set; } + + [JsonPropertyName("collectibles")] + public object? Collectibles { get; set; } + + [JsonPropertyName("display_name_styles")] + public object? DisplayNameStyles { get; set; } + + [JsonPropertyName("banner")] + public string? Banner { get; set; } + + [JsonPropertyName("bio")] + public string? Bio { get; set; } +} \ No newline at end of file diff --git a/extra/admin-api/Spacebar.GatewayOffload/Controllers/Op14Controller.cs b/extra/admin-api/Spacebar.GatewayOffload/Controllers/Op14Controller.cs new file mode 100644 index 000000000..84f03b892 --- /dev/null +++ b/extra/admin-api/Spacebar.GatewayOffload/Controllers/Op14Controller.cs @@ -0,0 +1,13 @@ +using Microsoft.AspNetCore.Mvc; +using Spacebar.Interop.Replication.Abstractions; + +namespace Spacebar.GatewayOffload.Controllers; + +[ApiController] +[Route("/_spacebar/offload/gateway/LazyRequest")] +public class Op14Controller : ControllerBase { + [HttpPost] + public async IAsyncEnumerable DoLazyRequest() { + yield break; + } +} \ No newline at end of file diff --git a/extra/admin-api/Spacebar.GatewayOffload/Program.cs b/extra/admin-api/Spacebar.GatewayOffload/Program.cs new file mode 100644 index 000000000..208302404 --- /dev/null +++ b/extra/admin-api/Spacebar.GatewayOffload/Program.cs @@ -0,0 +1,46 @@ +using System.Text.Json.Serialization; +using Microsoft.EntityFrameworkCore; +using Spacebar.Interop.Authentication; +using Spacebar.Interop.Authentication.AspNetCore; +using Spacebar.Models.Db.Contexts; + +var builder = WebApplication.CreateBuilder(args); + +// Add services to the container. + +builder.Services.AddControllers(options => { + options.MaxValidationDepth = null; + // options.MaxIAsyncEnumerableBufferLimit = 1; +}).AddJsonOptions(options => { + // options.JsonSerializerOptions.DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull; + options.JsonSerializerOptions.WriteIndented = true; + options.JsonSerializerOptions.MaxDepth = 100; + // options.JsonSerializerOptions.DefaultBufferSize = ; +}).AddMvcOptions(o=> { + o.SuppressOutputFormatterBuffering = true; +}); +// Learn more about configuring OpenAPI at https://aka.ms/aspnet/openapi +builder.Services.AddOpenApi(); + +builder.Services.AddDbContextPool(options => { + options + .UseNpgsql(builder.Configuration.GetConnectionString("Spacebar")) + .EnableDetailedErrors(); +}); + +builder.Services.AddSingleton(); +builder.Services.AddScoped(); +builder.Services.AddScoped(); + +var app = builder.Build(); + +// Configure the HTTP request pipeline. +if (app.Environment.IsDevelopment()) { + app.MapOpenApi(); +} + +app.UseAuthorization(); + +app.MapControllers(); + +app.Run(); \ No newline at end of file diff --git a/extra/admin-api/Spacebar.GatewayOffload/Properties/launchSettings.json b/extra/admin-api/Spacebar.GatewayOffload/Properties/launchSettings.json new file mode 100644 index 000000000..385142c62 --- /dev/null +++ b/extra/admin-api/Spacebar.GatewayOffload/Properties/launchSettings.json @@ -0,0 +1,14 @@ +{ + "$schema": "https://json.schemastore.org/launchsettings.json", + "profiles": { + "http": { + "commandName": "Project", + "dotnetRunMessages": true, + "launchBrowser": false, + "applicationUrl": "http://localhost:5019", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + } + } +} diff --git a/extra/admin-api/Spacebar.GatewayOffload/Spacebar.GatewayOffload.csproj b/extra/admin-api/Spacebar.GatewayOffload/Spacebar.GatewayOffload.csproj new file mode 100644 index 000000000..ca9424836 --- /dev/null +++ b/extra/admin-api/Spacebar.GatewayOffload/Spacebar.GatewayOffload.csproj @@ -0,0 +1,19 @@ + + + + net10.0 + enable + enable + + + + + + + + + + + + + diff --git a/extra/admin-api/Spacebar.GatewayOffload/Spacebar.GatewayOffload.http b/extra/admin-api/Spacebar.GatewayOffload/Spacebar.GatewayOffload.http new file mode 100644 index 000000000..dc84f632e --- /dev/null +++ b/extra/admin-api/Spacebar.GatewayOffload/Spacebar.GatewayOffload.http @@ -0,0 +1,34 @@ +@UpstreamApi_HostAddress = https://api.rory.server.spacebar.chat +@GatewayOffload_HostAddress = http://localhost:5019 + +GET {{UpstreamApi_HostAddress}}/api/v10/users/@me/guilds +Accept: application/json +Authorization: Bearer {{AccessToken}} + +### + +POST {{GatewayOffload_HostAddress}}/_spacebar/offload/gateway/GuildSync +Accept: application/json +Authorization: Bearer {{AccessToken}} +Content-Type: application/json + +[ + "1006649183970562092", + "1006688661691001233", + "1006724118650845953", + "1006725777779159397", + "1006929437075185067", + "1064985958721777984", + "1069727354182414384", + "1244318971698665103", + "1294758139447835873", + "1370456872691392086", + "1391838091291000099", + "1412646927261544598", + "1425141280262418979", + "1429366715866774152", + "1430867450484749537", + "1439953071347150967" +] + +### diff --git a/extra/admin-api/Spacebar.GatewayOffload/appsettings.Development.json b/extra/admin-api/Spacebar.GatewayOffload/appsettings.Development.json new file mode 100644 index 000000000..12d6d238d --- /dev/null +++ b/extra/admin-api/Spacebar.GatewayOffload/appsettings.Development.json @@ -0,0 +1,29 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.AspNetCore": "Trace", + //Warning + "Microsoft.AspNetCore.Mvc": "Warning", + //Warning + "Microsoft.AspNetCore.HostFiltering": "Warning", + //Warning + "Microsoft.AspNetCore.Cors": "Warning", + //Warning + // "Microsoft.EntityFrameworkCore": "Warning" + "Microsoft.EntityFrameworkCore.Database.Command": "Information" + } + }, + "ConnectionStrings": { + "Spacebar": "Host=127.0.0.1; Username=postgres; Database=spacebar; Port=5433; Include Error Detail=true; Maximum Pool Size=1000; Command Timeout=6000; Timeout=600;" + }, + "Spacebar": { + "Authentication": { + "PrivateKeyPath": "../../../jwt.key", + "PublicKeyPath": "../../../jwt.key.pub" +// "Enforce2FA": false, +// "OverrideUid": "1006598230156341276", +// "DisableAuthentication": true + } + } +} diff --git a/extra/admin-api/Spacebar.GatewayOffload/appsettings.json b/extra/admin-api/Spacebar.GatewayOffload/appsettings.json new file mode 100644 index 000000000..10f68b8c8 --- /dev/null +++ b/extra/admin-api/Spacebar.GatewayOffload/appsettings.json @@ -0,0 +1,9 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.AspNetCore": "Warning" + } + }, + "AllowedHosts": "*" +} diff --git a/extra/admin-api/SpacebarAdminAPI.sln b/extra/admin-api/SpacebarAdminAPI.sln index 2d76cb824..a273842be 100644 --- a/extra/admin-api/SpacebarAdminAPI.sln +++ b/extra/admin-api/SpacebarAdminAPI.sln @@ -45,6 +45,12 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Spacebar.Cdn.Migration", "U EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Spacebar.Interop.Cdn.Signing", "Interop\Spacebar.Interop.Cdn.Signing\Spacebar.Interop.Cdn.Signing.csproj", "{95A8E8CC-53B2-4FC6-B41E-196BE5F96BB5}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Spacebar.Interop.Authentication", "Interop\Spacebar.Interop.Authentication\Spacebar.Interop.Authentication.csproj", "{8864E79A-2FF3-4B08-8AE9-F9D877CC69B3}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Spacebar.GatewayOffload", "Spacebar.GatewayOffload\Spacebar.GatewayOffload.csproj", "{CFD6ADD1-29D8-414A-B4E4-E2F74CC7D27B}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Spacebar.Interop.Authentication.AspNetCore", "Interop\Spacebar.Interop.Authentication.AspNetCore\Spacebar.Interop.Authentication.AspNetCore.csproj", "{BB961FD8-61C2-4443-AB68-B8088CBC3D43}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -271,6 +277,42 @@ Global {95A8E8CC-53B2-4FC6-B41E-196BE5F96BB5}.Release|x64.Build.0 = Release|Any CPU {95A8E8CC-53B2-4FC6-B41E-196BE5F96BB5}.Release|x86.ActiveCfg = Release|Any CPU {95A8E8CC-53B2-4FC6-B41E-196BE5F96BB5}.Release|x86.Build.0 = Release|Any CPU + {8864E79A-2FF3-4B08-8AE9-F9D877CC69B3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {8864E79A-2FF3-4B08-8AE9-F9D877CC69B3}.Debug|Any CPU.Build.0 = Debug|Any CPU + {8864E79A-2FF3-4B08-8AE9-F9D877CC69B3}.Debug|x64.ActiveCfg = Debug|Any CPU + {8864E79A-2FF3-4B08-8AE9-F9D877CC69B3}.Debug|x64.Build.0 = Debug|Any CPU + {8864E79A-2FF3-4B08-8AE9-F9D877CC69B3}.Debug|x86.ActiveCfg = Debug|Any CPU + {8864E79A-2FF3-4B08-8AE9-F9D877CC69B3}.Debug|x86.Build.0 = Debug|Any CPU + {8864E79A-2FF3-4B08-8AE9-F9D877CC69B3}.Release|Any CPU.ActiveCfg = Release|Any CPU + {8864E79A-2FF3-4B08-8AE9-F9D877CC69B3}.Release|Any CPU.Build.0 = Release|Any CPU + {8864E79A-2FF3-4B08-8AE9-F9D877CC69B3}.Release|x64.ActiveCfg = Release|Any CPU + {8864E79A-2FF3-4B08-8AE9-F9D877CC69B3}.Release|x64.Build.0 = Release|Any CPU + {8864E79A-2FF3-4B08-8AE9-F9D877CC69B3}.Release|x86.ActiveCfg = Release|Any CPU + {8864E79A-2FF3-4B08-8AE9-F9D877CC69B3}.Release|x86.Build.0 = Release|Any CPU + {CFD6ADD1-29D8-414A-B4E4-E2F74CC7D27B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {CFD6ADD1-29D8-414A-B4E4-E2F74CC7D27B}.Debug|Any CPU.Build.0 = Debug|Any CPU + {CFD6ADD1-29D8-414A-B4E4-E2F74CC7D27B}.Debug|x64.ActiveCfg = Debug|Any CPU + {CFD6ADD1-29D8-414A-B4E4-E2F74CC7D27B}.Debug|x64.Build.0 = Debug|Any CPU + {CFD6ADD1-29D8-414A-B4E4-E2F74CC7D27B}.Debug|x86.ActiveCfg = Debug|Any CPU + {CFD6ADD1-29D8-414A-B4E4-E2F74CC7D27B}.Debug|x86.Build.0 = Debug|Any CPU + {CFD6ADD1-29D8-414A-B4E4-E2F74CC7D27B}.Release|Any CPU.ActiveCfg = Release|Any CPU + {CFD6ADD1-29D8-414A-B4E4-E2F74CC7D27B}.Release|Any CPU.Build.0 = Release|Any CPU + {CFD6ADD1-29D8-414A-B4E4-E2F74CC7D27B}.Release|x64.ActiveCfg = Release|Any CPU + {CFD6ADD1-29D8-414A-B4E4-E2F74CC7D27B}.Release|x64.Build.0 = Release|Any CPU + {CFD6ADD1-29D8-414A-B4E4-E2F74CC7D27B}.Release|x86.ActiveCfg = Release|Any CPU + {CFD6ADD1-29D8-414A-B4E4-E2F74CC7D27B}.Release|x86.Build.0 = Release|Any CPU + {BB961FD8-61C2-4443-AB68-B8088CBC3D43}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {BB961FD8-61C2-4443-AB68-B8088CBC3D43}.Debug|Any CPU.Build.0 = Debug|Any CPU + {BB961FD8-61C2-4443-AB68-B8088CBC3D43}.Debug|x64.ActiveCfg = Debug|Any CPU + {BB961FD8-61C2-4443-AB68-B8088CBC3D43}.Debug|x64.Build.0 = Debug|Any CPU + {BB961FD8-61C2-4443-AB68-B8088CBC3D43}.Debug|x86.ActiveCfg = Debug|Any CPU + {BB961FD8-61C2-4443-AB68-B8088CBC3D43}.Debug|x86.Build.0 = Debug|Any CPU + {BB961FD8-61C2-4443-AB68-B8088CBC3D43}.Release|Any CPU.ActiveCfg = Release|Any CPU + {BB961FD8-61C2-4443-AB68-B8088CBC3D43}.Release|Any CPU.Build.0 = Release|Any CPU + {BB961FD8-61C2-4443-AB68-B8088CBC3D43}.Release|x64.ActiveCfg = Release|Any CPU + {BB961FD8-61C2-4443-AB68-B8088CBC3D43}.Release|x64.Build.0 = Release|Any CPU + {BB961FD8-61C2-4443-AB68-B8088CBC3D43}.Release|x86.ActiveCfg = Release|Any CPU + {BB961FD8-61C2-4443-AB68-B8088CBC3D43}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -292,5 +334,7 @@ Global {B494C58D-2D53-49F4-9E3E-1DCF60828802} = {16DBEA54-D51A-4D91-84DF-C701B6B4786F} {2B18EB6B-331D-41A9-A9D1-C2518E2AD914} = {04787943-EBB6-4DE4-96D5-4CFB4A2CEE99} {95A8E8CC-53B2-4FC6-B41E-196BE5F96BB5} = {16DBEA54-D51A-4D91-84DF-C701B6B4786F} + {8864E79A-2FF3-4B08-8AE9-F9D877CC69B3} = {16DBEA54-D51A-4D91-84DF-C701B6B4786F} + {BB961FD8-61C2-4443-AB68-B8088CBC3D43} = {16DBEA54-D51A-4D91-84DF-C701B6B4786F} EndGlobalSection EndGlobal diff --git a/extra/admin-api/outputs.nix b/extra/admin-api/outputs.nix index 8392629a6..4848147fd 100644 --- a/extra/admin-api/outputs.nix +++ b/extra/admin-api/outputs.nix @@ -65,6 +65,11 @@ flake-utils.lib.eachSystem flake-utils.lib.allSystems ( in { # Interop + Spacebar-Interop-Authentication = makeNupkg { + name = "Spacebar.Interop.Authentication"; + projectFile = "Interop/Spacebar.Interop.Authentication/Spacebar.Interop.Authentication.csproj"; + nugetDeps = Interop/Spacebar.Interop.Authentication/deps.json; + }; Spacebar-Interop-Cdn-Abstractions = makeNupkg { name = "Spacebar.Interop.Cdn.Abstractions"; projectFile = "Interop/Spacebar.Interop.Cdn.Abstractions/Spacebar.Interop.Cdn.Abstractions.csproj";