From 862bfd78038e0f35581ebe5d5416bb6f9bca999e Mon Sep 17 00:00:00 2001 From: Rory& Date: Fri, 6 Feb 2026 05:21:31 +0100 Subject: [PATCH] C# gw offload work, schema-izing stuff --- .../Spacebar.DataMappings.Generic/Member.cs | 21 ++ .../Spacebar.DataMappings.Generic.csproj | 16 ++ .../Spacebar.DataMappings.Generic/User.cs | 26 ++ .../Spacebar.DataMappings.Generic/deps.json | 247 ++++++++++++++++++ .../SpacebarAuthenticationService.cs | 11 + .../IdentifyRequest.cs | 89 +++++++ .../Spacebar.Models.Gateway/LazyRequest.cs | 24 ++ extra/admin-api/Spacebar.AdminApi/Program.cs | 6 +- extra/admin-api/Spacebar.Cdn/Program.cs | 6 +- .../Controllers/IdentifyController.cs | 13 + .../Controllers/Op12Controller.cs | 73 +++--- .../Spacebar.GatewayOffload/Program.cs | 2 + .../Spacebar.GatewayOffload.csproj | 2 + extra/admin-api/SpacebarAdminAPI.sln | 17 ++ .../Utilities/Spacebar.Cdn.Fsck/Program.cs | 5 +- .../Spacebar.Cdn.Migration/Program.cs | 3 + .../Spacebar.CleanSettingsRows/Program.cs | 4 +- extra/admin-api/outputs.nix | 35 ++- flake.lock | Bin 1497 -> 1497 bytes .../default/cs/default-appsettings-json.nix | 16 ++ nix/modules/default/cs/gateway-offload-cs.nix | 173 ++++++++++++ nix/modules/default/default.nix | 38 +-- nix/modules/default/secrets.nix | 37 ++- nix/testVm/configuration.nix | 28 +- nix/tests/test-bundle-starts.nix | 8 +- 25 files changed, 817 insertions(+), 83 deletions(-) create mode 100644 extra/admin-api/DataMappings/Spacebar.DataMappings.Generic/Member.cs create mode 100644 extra/admin-api/DataMappings/Spacebar.DataMappings.Generic/Spacebar.DataMappings.Generic.csproj create mode 100644 extra/admin-api/DataMappings/Spacebar.DataMappings.Generic/User.cs create mode 100644 extra/admin-api/DataMappings/Spacebar.DataMappings.Generic/deps.json create mode 100644 extra/admin-api/Models/Spacebar.Models.Gateway/IdentifyRequest.cs create mode 100644 extra/admin-api/Models/Spacebar.Models.Gateway/LazyRequest.cs create mode 100644 extra/admin-api/Spacebar.GatewayOffload/Controllers/IdentifyController.cs create mode 100644 nix/modules/default/cs/default-appsettings-json.nix create mode 100644 nix/modules/default/cs/gateway-offload-cs.nix diff --git a/extra/admin-api/DataMappings/Spacebar.DataMappings.Generic/Member.cs b/extra/admin-api/DataMappings/Spacebar.DataMappings.Generic/Member.cs new file mode 100644 index 000000000..2650fc383 --- /dev/null +++ b/extra/admin-api/DataMappings/Spacebar.DataMappings.Generic/Member.cs @@ -0,0 +1,21 @@ +using Spacebar.Models.Generic; + +namespace Spacebar.DataMappings.Generic; + +public static class Member +{ + public static Models.Generic.Member ToPublicMember(this Models.Db.Models.Member member, PartialUser? partialUser = null) + { + return new() + { + User = partialUser ?? member.IdNavigation.ToPartialUser(), + AvatarDecorationData = member.AvatarDecorationData, + Avatar = string.IsNullOrWhiteSpace(member.Avatar) ? null : member.Avatar, + Banner = string.IsNullOrWhiteSpace(member.Banner) ? null : member.Banner, + Collectibles = member.Collectibles, + DisplayNameStyles = member.DisplayNameStyles, + Bio = string.IsNullOrWhiteSpace(member.Bio) ? null : member.Bio, + Nick = string.IsNullOrWhiteSpace(member.Nick) ? null : member.Nick + }; + } +} \ No newline at end of file diff --git a/extra/admin-api/DataMappings/Spacebar.DataMappings.Generic/Spacebar.DataMappings.Generic.csproj b/extra/admin-api/DataMappings/Spacebar.DataMappings.Generic/Spacebar.DataMappings.Generic.csproj new file mode 100644 index 000000000..3f8936a6f --- /dev/null +++ b/extra/admin-api/DataMappings/Spacebar.DataMappings.Generic/Spacebar.DataMappings.Generic.csproj @@ -0,0 +1,16 @@ + + + + net10.0 + enable + enable + + + + + + + + + + diff --git a/extra/admin-api/DataMappings/Spacebar.DataMappings.Generic/User.cs b/extra/admin-api/DataMappings/Spacebar.DataMappings.Generic/User.cs new file mode 100644 index 000000000..607af9411 --- /dev/null +++ b/extra/admin-api/DataMappings/Spacebar.DataMappings.Generic/User.cs @@ -0,0 +1,26 @@ +using Spacebar.Models.Generic; + +namespace Spacebar.DataMappings.Generic; + +public static class User +{ + public static PartialUser ToPartialUser(this Models.Db.Models.User user) + { + return new PartialUser() { + Id = user.Id, + Discriminator = user.Discriminator, + Username = user.Username, + AccentColor = user.AccentColor, + Avatar = user.Avatar, + AvatarDecorationData = user.AvatarDecorationData, + Banner = user.Banner, + Bot = user.Bot, + Collectibles = user.Collectibles, + DisplayNameStyles = user.DisplayNameStyles, + // GlobalName = x.GlobalName, + PrimaryGuild = user.PrimaryGuild, + PublicFlags = user.PublicFlags, + System = user.System, + }; + } +} \ No newline at end of file diff --git a/extra/admin-api/DataMappings/Spacebar.DataMappings.Generic/deps.json b/extra/admin-api/DataMappings/Spacebar.DataMappings.Generic/deps.json new file mode 100644 index 000000000..aead01783 --- /dev/null +++ b/extra/admin-api/DataMappings/Spacebar.DataMappings.Generic/deps.json @@ -0,0 +1,247 @@ +[ + { + "pname": "Humanizer.Core", + "version": "2.14.1", + "hash": "sha256-EXvojddPu+9JKgOG9NSQgUTfWq1RpOYw7adxDPKDJ6o=" + }, + { + "pname": "Microsoft.Build.Framework", + "version": "17.11.31", + "hash": "sha256-YS4oASrmC5dmZrx5JPS7SfKmUpIJErlUpVDsU3VrfFE=" + }, + { + "pname": "Microsoft.Build.Framework", + "version": "18.0.2", + "hash": "sha256-fO31KAdDs2J0RUYD1ov9UB3ucsbALan7K0YdWW+yg7A=" + }, + { + "pname": "Microsoft.CodeAnalysis.Analyzers", + "version": "3.11.0", + "hash": "sha256-hQ2l6E6PO4m7i+ZsfFlEx+93UsLPo4IY3wDkNG11/Sw=" + }, + { + "pname": "Microsoft.CodeAnalysis.Common", + "version": "5.0.0", + "hash": "sha256-g4ALvBSNyHEmSb1l5TFtWW7zEkiRmhqLx4XWZu9sr2U=" + }, + { + "pname": "Microsoft.CodeAnalysis.CSharp", + "version": "5.0.0", + "hash": "sha256-ctBCkQGFpH/xT5rRE3xibu9YxPD108RuC4a4Z25koG8=" + }, + { + "pname": "Microsoft.CodeAnalysis.CSharp.Workspaces", + "version": "5.0.0", + "hash": "sha256-yWVcLt/f2CouOfFy966glGdtSFy+RcgrU1dd9UtlL/Q=" + }, + { + "pname": "Microsoft.CodeAnalysis.Workspaces.Common", + "version": "5.0.0", + "hash": "sha256-Bir5e1gEhgQQ6upQmVKQHAKLRfenAu60DAzNupNnZsQ=" + }, + { + "pname": "Microsoft.CodeAnalysis.Workspaces.MSBuild", + "version": "5.0.0", + "hash": "sha256-+58+iqTayTiE0pDaog1U8mjaDA8bNNDLA8gjCQZZudo=" + }, + { + "pname": "Microsoft.EntityFrameworkCore", + "version": "10.0.0", + "hash": "sha256-xfgrlxhtOkQwF5Q7j8gSm41URJiH8IuJ/T/Dh88++hE=" + }, + { + "pname": "Microsoft.EntityFrameworkCore", + "version": "10.0.2", + "hash": "sha256-FS6T8EnaWCMtj4PnZhh+oF8mcM44VlM3wkTSMlpte9A=" + }, + { + "pname": "Microsoft.EntityFrameworkCore.Abstractions", + "version": "10.0.0", + "hash": "sha256-UDgZbRQcGPaKsE53EH6bvJiv+Q4KSxAbnsVhTVFGG4Q=" + }, + { + "pname": "Microsoft.EntityFrameworkCore.Abstractions", + "version": "10.0.2", + "hash": "sha256-qkDfIJpcPO2kk4n5OE/13hI/0mUygpTofInn95XjRZI=" + }, + { + "pname": "Microsoft.EntityFrameworkCore.Analyzers", + "version": "10.0.0", + "hash": "sha256-7Q0jYJO50cqGI+u6gLpootbB8GZvgsgtg0F9FZI1jig=" + }, + { + "pname": "Microsoft.EntityFrameworkCore.Analyzers", + "version": "10.0.2", + "hash": "sha256-yOv78rgAACBz1zjitpcZbQQ3zx8huJongZTHkhN4PQ0=" + }, + { + "pname": "Microsoft.EntityFrameworkCore.Design", + "version": "10.0.2", + "hash": "sha256-bTShsGux0y/49PIIMb/4ZX3x5+rPacvT5/NcooNCI1Y=" + }, + { + "pname": "Microsoft.EntityFrameworkCore.Relational", + "version": "10.0.0", + "hash": "sha256-vOP2CE5YA551BlpbOuIy6RuAiAEPEpCVS1cEE33/zN4=" + }, + { + "pname": "Microsoft.EntityFrameworkCore.Relational", + "version": "10.0.2", + "hash": "sha256-Y4jPpoYhKizg5wF6QfkBX4sYlE2FU1bYhfoDN3xkhKM=" + }, + { + "pname": "Microsoft.Extensions.Caching.Abstractions", + "version": "10.0.2", + "hash": "sha256-nKmQuZTt1g5/8gBajo7wdCV64kdCucdiQR8JTt7ZZb0=" + }, + { + "pname": "Microsoft.Extensions.Caching.Memory", + "version": "10.0.0", + "hash": "sha256-AMgDSm1k6q0s17spGtyR5q8nAqUFDOxl/Fe38f9M+d4=" + }, + { + "pname": "Microsoft.Extensions.Caching.Memory", + "version": "10.0.2", + "hash": "sha256-sRUF7DM0s1yzZnfjM/hF9A/IysE6Er23gZ6jST+RWh0=" + }, + { + "pname": "Microsoft.Extensions.Configuration.Abstractions", + "version": "10.0.2", + "hash": "sha256-P+0kaDGO+xB9KxF9eWHDJ4hzi05sUGM/uMNEX5NdBTE=" + }, + { + "pname": "Microsoft.Extensions.DependencyInjection", + "version": "10.0.2", + "hash": "sha256-/9UWQRAI2eoocnJWWf1ktnAx/1Gt65c16fc0Xqr9+CQ=" + }, + { + "pname": "Microsoft.Extensions.DependencyInjection", + "version": "9.0.0", + "hash": "sha256-dAH52PPlTLn7X+1aI/7npdrDzMEFPMXRv4isV1a+14k=" + }, + { + "pname": "Microsoft.Extensions.DependencyInjection.Abstractions", + "version": "10.0.2", + "hash": "sha256-UF9T13V5SQxJy2msfLmyovLmitZrjJayf8gHH+uK2eg=" + }, + { + "pname": "Microsoft.Extensions.DependencyInjection.Abstractions", + "version": "9.0.0", + "hash": "sha256-CncVwkKZ5CsIG2O0+OM9qXuYXh3p6UGyueTHSLDVL+c=" + }, + { + "pname": "Microsoft.Extensions.DependencyModel", + "version": "10.0.2", + "hash": "sha256-w/dGIjtZiGH+KW3969BPOdQpQEV+WB7RPTa2MK2DavE=" + }, + { + "pname": "Microsoft.Extensions.Logging", + "version": "10.0.0", + "hash": "sha256-P+zPAadLL63k/GqK34/qChqQjY9aIRxZfxlB9lqsSrs=" + }, + { + "pname": "Microsoft.Extensions.Logging", + "version": "10.0.2", + "hash": "sha256-9+gfQwK32JMYscW1YvyCWEzc9mRZOjCACoD9U1vVaJI=" + }, + { + "pname": "Microsoft.Extensions.Logging", + "version": "9.0.0", + "hash": "sha256-kR16c+N8nQrWeYLajqnXPg7RiXjZMSFLnKLEs4VfjcM=" + }, + { + "pname": "Microsoft.Extensions.Logging.Abstractions", + "version": "10.0.0", + "hash": "sha256-BnhgGZc01HwTSxogavq7Ueq4V7iMA3wPnbfRwQ4RhGk=" + }, + { + "pname": "Microsoft.Extensions.Logging.Abstractions", + "version": "10.0.2", + "hash": "sha256-ndKGzq8+2J/hvaIULwBui0L/jDyMQTAY24j+ohX5VX8=" + }, + { + "pname": "Microsoft.Extensions.Logging.Abstractions", + "version": "9.0.0", + "hash": "sha256-iBTs9twjWXFeERt4CErkIIcoJZU1jrd1RWCI8V5j7KU=" + }, + { + "pname": "Microsoft.Extensions.Options", + "version": "10.0.2", + "hash": "sha256-12AfUEDdta/pmZUyEyqSUfOk0YoA7JOfGmIYnZQ//qk=" + }, + { + "pname": "Microsoft.Extensions.Options", + "version": "9.0.0", + "hash": "sha256-DT5euAQY/ItB5LPI8WIp6Dnd0lSvBRP35vFkOXC68ck=" + }, + { + "pname": "Microsoft.Extensions.Primitives", + "version": "10.0.2", + "hash": "sha256-8Ccrjjv9cFVf9RyCc7GS/Byt8+DXdSNea0UX3A5BEdA=" + }, + { + "pname": "Microsoft.Extensions.Primitives", + "version": "9.0.0", + "hash": "sha256-ZNLusK1CRuq5BZYZMDqaz04PIKScE2Z7sS2tehU7EJs=" + }, + { + "pname": "Microsoft.VisualStudio.SolutionPersistence", + "version": "1.0.52", + "hash": "sha256-KZGPtOXe6Hv8RrkcsgoLKTRyaCScIpQEa2NhNB3iOXw=" + }, + { + "pname": "Mono.TextTemplating", + "version": "3.0.0", + "hash": "sha256-VlgGDvgNZb7MeBbIZ4DE2Nn/j2aD9k6XqNHnASUSDr0=" + }, + { + "pname": "Newtonsoft.Json", + "version": "13.0.3", + "hash": "sha256-hy/BieY4qxBWVVsDqqOPaLy1QobiIapkbrESm6v2PHc=" + }, + { + "pname": "Npgsql", + "version": "10.0.0", + "hash": "sha256-UVKz9dH/rVCCbMyFdqA31RYpht1XgDRLIqUy0Dp9ACQ=" + }, + { + "pname": "Npgsql.EntityFrameworkCore.PostgreSQL", + "version": "10.0.0", + "hash": "sha256-XIJxnTMektQVP1qtslEIGbmBGrIQsvjQjCMRTs9UIbg=" + }, + { + "pname": "System.CodeDom", + "version": "6.0.0", + "hash": "sha256-uPetUFZyHfxjScu5x4agjk9pIhbCkt5rG4Axj25npcQ=" + }, + { + "pname": "System.Composition", + "version": "9.0.0", + "hash": "sha256-FehOkQ2u1p8mQ0/wn3cZ+24HjhTLdck8VZYWA1CcgbM=" + }, + { + "pname": "System.Composition.AttributedModel", + "version": "9.0.0", + "hash": "sha256-a7y7H6zj+kmYkllNHA402DoVfY9IaqC3Ooys8Vzl24M=" + }, + { + "pname": "System.Composition.Convention", + "version": "9.0.0", + "hash": "sha256-tw4vE5JRQ60ubTZBbxoMPhtjOQCC3XoDFUH7NHO7o8U=" + }, + { + "pname": "System.Composition.Hosting", + "version": "9.0.0", + "hash": "sha256-oOxU+DPEEfMCuNLgW6wSkZp0JY5gYt44FJNnWt+967s=" + }, + { + "pname": "System.Composition.Runtime", + "version": "9.0.0", + "hash": "sha256-AyIe+di1TqwUBbSJ/sJ8Q8tzsnTN+VBdJw4K8xZz43s=" + }, + { + "pname": "System.Composition.TypedParts", + "version": "9.0.0", + "hash": "sha256-F5fpTUs3Rr7yP/NyIzr+Xn5NdTXXp8rrjBnF9UBBUog=" + } +] diff --git a/extra/admin-api/Interop/Spacebar.Interop.Authentication/SpacebarAuthenticationService.cs b/extra/admin-api/Interop/Spacebar.Interop.Authentication/SpacebarAuthenticationService.cs index 7b4ad1c8b..5f9ff500f 100644 --- a/extra/admin-api/Interop/Spacebar.Interop.Authentication/SpacebarAuthenticationService.cs +++ b/extra/admin-api/Interop/Spacebar.Interop.Authentication/SpacebarAuthenticationService.cs @@ -46,4 +46,15 @@ public class SpacebarAuthenticationService(ILogger GetCurrentSessionAsync(string token) { + var res = await ValidateTokenAsync(token); + return await UserCache.GetOrAdd(token, + async () => { + var uid = config.OverrideUid ?? res?.ClaimsIdentity.Claims.First(x => x.Type == "id").Value; + if (string.IsNullOrWhiteSpace(uid)) throw new InvalidOperationException("No user ID specified, is the access token valid?"); + return await db.Users.FindAsync(uid) ?? throw new InvalidOperationException(); + }, + config.AuthCacheExpiry); + } } \ No newline at end of file diff --git a/extra/admin-api/Models/Spacebar.Models.Gateway/IdentifyRequest.cs b/extra/admin-api/Models/Spacebar.Models.Gateway/IdentifyRequest.cs new file mode 100644 index 000000000..a956655ed --- /dev/null +++ b/extra/admin-api/Models/Spacebar.Models.Gateway/IdentifyRequest.cs @@ -0,0 +1,89 @@ +using System.Text.Json.Nodes; +using System.Text.Json.Serialization; + +namespace Spacebar.Models.Gateway; + +public class IdentifyRequest +{ + [JsonPropertyName("token")] + public string Token { get; set; } + + [JsonPropertyName("properties")] + public JsonObject ClientProperties { get; set; } + + [JsonPropertyName("compress")] + public bool? Compress { get; set; } + + [JsonPropertyName("large_threshold")] + public int? LargeTreshold { get; set; } + + [JsonPropertyName("shard")] + public int[]? Shard { get; set; } + + [JsonPropertyName("presence")] + public JsonObject? Presence { get; set; } + + [JsonPropertyName("intents")] + public GatewayIntentFlags? Intents { get; set; } + + [JsonPropertyName("capabilities")] + public GatewayCapabilityFlags? Capabilities { get; set; } + + [JsonPropertyName("client_state")] + public JsonObject? ClientState { get; set; } +} + +[Flags] +public enum GatewayIntentFlags +{ + Guilds = 1, + GuildMembers = 1 << 1, + GuildModeration = 1 << 2, + GuildEmojisAndStickers = 1 << 3, + GuildIntegrations = 1 << 4, + GuildWebhooks = 1 << 5, + GuildInvites = 1 << 6, + GuildVoiceStates = 1 << 7, + GuildPresences = 1 << 8, + GuildMessages = 1 << 9, + GuildMessageReactions = 1 << 10, + GuildMessageTyping = 1 << 11, + DirectMessages = 1 << 12, + DirectMessageReactions = 1 << 13, + DirectMessageTyping = 1 << 14, + MessageContent = 1 << 15, + GuildScheduledEvents = 1 << 16, + PrivateEmbeddedActivities = 1 << 17, + PrivateChannels = 1 << 18, + Calls = 1 << 19, + AutoModerationConfiguration = 1 << 20, + AutoModerationExecution = 1 << 21, + UserRelationships = 1 << 22, + UserPresence = 1 << 23, + GuildMessagePolls = 1 << 24, + DirectMessagePolls = 1 << 25, + DirectEmbeddedActivities = 1 << 26, + Lobbies = 1 << 27, + LobbyDelete = 1 << 28 +} + +[Flags] +public enum GatewayCapabilityFlags +{ + LazyUserNotes = 1, + NoAffineUserIds = 1 << 1, + VersionedReadStates = 1 << 2, + VersionedUserGuildSettings = 1 << 3, + DedupeUserObjects = 1 << 4, + PrioritizedReadyPayload = 1 << 5, + MultipleGuildExperimentPopulations = 1 << 6, + NonChannelReadStates = 1 << 7, + AuthTokenRefresh = 1 << 8, + UserSettingsProto = 1 << 9, + ClientStateV2 = 1 << 10, + PassiveGuildUpdate = 1 << 11, + AutoCallConnect = 1 << 12, + DebounceMessageReactions = 1 << 13, + PassiveGuildUpdateV2 = 1 << 14, + AutoLobbyConnect = 1 << 16 +} \ No newline at end of file diff --git a/extra/admin-api/Models/Spacebar.Models.Gateway/LazyRequest.cs b/extra/admin-api/Models/Spacebar.Models.Gateway/LazyRequest.cs new file mode 100644 index 000000000..200dccea6 --- /dev/null +++ b/extra/admin-api/Models/Spacebar.Models.Gateway/LazyRequest.cs @@ -0,0 +1,24 @@ +using System.Text.Json.Serialization; + +namespace Spacebar.Models.Gateway; + +public class LazyRequest +{ + [JsonPropertyName("guild_id")] + public string GuildId { get; set; } + + [JsonPropertyName("channels")] + public Dictionary>> Channels { get; set; } + + [JsonPropertyName("members")] + public bool Members { get; set; } + + [JsonPropertyName("threads")] + public bool Threads { get; set; } + + [JsonPropertyName("activities")] + public bool Activities { get; set; } + + [JsonPropertyName("typing")] + public bool Typing { get; set; } +} \ No newline at end of file diff --git a/extra/admin-api/Spacebar.AdminApi/Program.cs b/extra/admin-api/Spacebar.AdminApi/Program.cs index cbc504551..b0f8f2090 100644 --- a/extra/admin-api/Spacebar.AdminApi/Program.cs +++ b/extra/admin-api/Spacebar.AdminApi/Program.cs @@ -9,6 +9,8 @@ using Spacebar.Interop.Replication.UnixSocket; using Spacebar.Models.Db.Contexts; var builder = WebApplication.CreateBuilder(args); +if (!string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("APPSETTINGS_PATH"))) + builder.Configuration.AddJsonFile(Environment.GetEnvironmentVariable("APPSETTINGS_PATH")!); // Add services to the container. @@ -19,9 +21,7 @@ builder.Services.AddControllers(options => { options.JsonSerializerOptions.DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull; options.JsonSerializerOptions.WriteIndented = true; // options.JsonSerializerOptions.DefaultBufferSize = ; -}).AddMvcOptions(o=> { - o.SuppressOutputFormatterBuffering = true; -}); +}).AddMvcOptions(o => { o.SuppressOutputFormatterBuffering = true; }); // Learn more about configuring OpenAPI at https://aka.ms/aspnet/openapi builder.Services.AddOpenApi(); diff --git a/extra/admin-api/Spacebar.Cdn/Program.cs b/extra/admin-api/Spacebar.Cdn/Program.cs index f7a319d30..47b518331 100644 --- a/extra/admin-api/Spacebar.Cdn/Program.cs +++ b/extra/admin-api/Spacebar.Cdn/Program.cs @@ -6,10 +6,12 @@ using Spacebar.Interop.Cdn.Abstractions; using Spacebar.Models.Db.Contexts; var builder = WebApplication.CreateBuilder(args); +if (!string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("APPSETTINGS_PATH"))) + builder.Configuration.AddJsonFile(Environment.GetEnvironmentVariable("APPSETTINGS_PATH")!); // Add services to the container. builder.Services.AddSingleton(new ProxyFileSource("http://cdn.old.server.spacebar.chat")); -builder.Services.AddSingleton(new LruFileCache(1*1024*1024*1024)); +builder.Services.AddSingleton(new LruFileCache(1 * 1024 * 1024 * 1024)); builder.Services.AddSingleton(); builder.Services.AddSingleton(); @@ -45,7 +47,7 @@ app.Use((context, next) => { }); // fallback to proxy in case we dont have a specific endpoint... -app.MapFallback("{*_}",async context => { +app.MapFallback("{*_}", async context => { var client = new StreamingHttpClient(); var requestMessage = new HttpRequestMessage( new HttpMethod(context.Request.Method), diff --git a/extra/admin-api/Spacebar.GatewayOffload/Controllers/IdentifyController.cs b/extra/admin-api/Spacebar.GatewayOffload/Controllers/IdentifyController.cs new file mode 100644 index 000000000..6ce9f9fb8 --- /dev/null +++ b/extra/admin-api/Spacebar.GatewayOffload/Controllers/IdentifyController.cs @@ -0,0 +1,13 @@ +using Microsoft.AspNetCore.Mvc; +using Spacebar.Interop.Authentication; +using Spacebar.Interop.Authentication.AspNetCore; +using Spacebar.Models.Db.Contexts; + +namespace Spacebar.GatewayOffload.Controllers; + +[ApiController] +[Route("/_spacebar/offload/gateway/Identify")] +public class IdentifyController(ILogger logger, SpacebarAuthenticationService authService, SpacebarDbContext db, IServiceProvider sp) : ControllerBase +{ + +} \ No newline at end of file diff --git a/extra/admin-api/Spacebar.GatewayOffload/Controllers/Op12Controller.cs b/extra/admin-api/Spacebar.GatewayOffload/Controllers/Op12Controller.cs index 0b7b5c82f..a2008345e 100644 --- a/extra/admin-api/Spacebar.GatewayOffload/Controllers/Op12Controller.cs +++ b/extra/admin-api/Spacebar.GatewayOffload/Controllers/Op12Controller.cs @@ -1,21 +1,25 @@ +using System.Linq.Expressions; using System.Text.Json; using System.Text.Json.Nodes; using ArcaneLibs.Extensions; using Microsoft.AspNetCore.Mvc; using Microsoft.EntityFrameworkCore; +using Spacebar.DataMappings.Generic; using Spacebar.Interop.Authentication.AspNetCore; using Spacebar.Interop.Replication.Abstractions; using Spacebar.Models.Db.Contexts; +using Spacebar.Models.Db.Models; using Spacebar.Models.Gateway; -using Spacebar.Models.Generic; namespace Spacebar.GatewayOffload.Controllers; [ApiController] [Route("/_spacebar/offload/gateway/GuildSync")] -public class Op12Controller(ILogger logger, SpacebarAspNetAuthenticationService authService, SpacebarDbContext db, IServiceProvider sp) : ControllerBase { +public class Op12Controller(ILogger logger, SpacebarAspNetAuthenticationService authService, SpacebarDbContext db, IServiceProvider sp) : ControllerBase +{ [HttpPost("")] - public async IAsyncEnumerable DoGuildSync(List guildIds) { + public async IAsyncEnumerable DoGuildSync(List guildIds) + { var user = await authService.GetCurrentUserAsync(Request); guildIds = (await db.Members.Where(x => x.Id == user.Id).Select(x => x.GuildId).ToListAsync()) .Intersect(guildIds) @@ -23,8 +27,10 @@ public class Op12Controller(ILogger logger, SpacebarAspNetAuthen .ToList(); var syncs = guildIds.Select(GetGuildSyncAsync).ToList().ToAsyncResultEnumerable(); - await foreach (var res in syncs) { - yield return new() { + await foreach (var res in syncs) + { + yield return new() + { Origin = "OFFLOAD_GUILD_SYNC", UserId = user.Id, Event = "GUILD_SYNC", @@ -34,57 +40,48 @@ public class Op12Controller(ILogger logger, SpacebarAspNetAuthen } } - private async Task GetGuildSyncAsync(string guildId) { + // TODO: figure out how to abstract this to a function without EFCore complaining about not being translatable... + //private static Func IsOnline = (Session session) => session.Status != "offline" && session.Status != "invisible" && session.Status != "unknown"; + + private async Task GetGuildSyncAsync(string guildId) + { await using var sc = sp.CreateAsyncScope(); - var offlineTreshold = DateTime.Now.Subtract(TimeSpan.FromDays(14)); var _db = sc.ServiceProvider.GetRequiredService(); var memberCount = await _db.Members.Where(x => x.GuildId == guildId).CountAsync(); + + var offlineTreshold = DateTime.Now.Subtract(TimeSpan.FromDays(14)); + var isLargeGuild = memberCount > 10000; + var members = await _db.Members.Where(x => x.GuildId == guildId) .Include(x => x.IdNavigation) .ThenInclude(x => x.Sessions.Where(s => - !s.IsAdminSession && (s.Status != "offline" && s.Status != "invisible") && (memberCount < 1000 || s.LastSeen >= offlineTreshold))) + !s.IsAdminSession && ( + // see TODO on IsOnline + s.Status != "offline" && s.Status != "invisible" && s.Status != "unknown" + ) && (!isLargeGuild || s.LastSeen >= offlineTreshold))) .Where(x => x.IdNavigation.Sessions.Count > 0) // ignore members without sessions .ToListAsync(); - var mappedPartialUsers = members.Select(x => x.IdNavigation).ToDictionary(x => x.Id, x => new PartialUser() { - Id = x.Id, - Discriminator = x.Discriminator, - Username = x.Username, - AccentColor = x.AccentColor, - Avatar = x.Avatar, - AvatarDecorationData = x.AvatarDecorationData, - Banner = x.Banner, - Bot = x.Bot, - Collectibles = x.Collectibles, - DisplayNameStyles = x.DisplayNameStyles, - // GlobalName = x.GlobalName, - PrimaryGuild = x.PrimaryGuild, - PublicFlags = x.PublicFlags, - System = x.System, - }); - var mappedMembers = members.ToDictionary(m => m.Id, m => new Member() { - User = mappedPartialUsers[m.Id], - AvatarDecorationData = m.AvatarDecorationData, - Avatar = string.IsNullOrWhiteSpace(m.Avatar) ? null : m.Avatar, - Banner = string.IsNullOrWhiteSpace(m.Banner) ? null : m.Banner, - Collectibles = m.Collectibles, - DisplayNameStyles = m.DisplayNameStyles, - Bio = string.IsNullOrWhiteSpace(m.Bio) ? null : m.Bio, - Nick = string.IsNullOrWhiteSpace(m.Nick) ? null : m.Nick - }); - var presences = members.Select(x => x.IdNavigation).Where(x => x.Sessions.Count > 0).ToDictionary(x => x.Id, x => { + var mappedPartialUsers = members.Select(x => x.IdNavigation).ToDictionary(x => x.Id, x => x.ToPartialUser()); + var mappedMembers = members.ToDictionary(m => m.Id, m => m.ToPublicMember(mappedPartialUsers[m.Id])); + + var presences = members.Select(x => x.IdNavigation).Where(x => x.Sessions.Count > 0).ToDictionary(x => x.Id, x => + { var sortedSessions = x.Sessions.OrderByDescending(s => s.LastSeen).ToList(); - return new PresenceResponse() { + return new PresenceResponse() + { GuildId = guildId, User = mappedPartialUsers[x.Id], Activities = x.Sessions.Where(s => s.Status is not ("offline" or "invisible" or "unknown")) .SelectMany(s => JsonSerializer.Deserialize(s.Activities) ?? []).ToList(), Status = sortedSessions.FirstOrDefault(s => !string.IsNullOrWhiteSpace(s.Status))?.Status ?? "offline", - ClientStatus = JsonSerializer.Deserialize(sortedSessions.First(s => !string.IsNullOrWhiteSpace(s.ClientStatus)).ClientStatus) ?? new() + ClientStatus = JsonSerializer.Deserialize(sortedSessions.First(s => !string.IsNullOrWhiteSpace(s.ClientStatus)).ClientStatus) ?? + new() }; }).Where(x => x.Value.Activities.Count > 0).ToDictionary(); - var r = new GuildSyncResponse() { + var r = new GuildSyncResponse() + { GuildId = guildId, Members = mappedMembers.Values.ToList(), Presences = presences.Values.ToList() diff --git a/extra/admin-api/Spacebar.GatewayOffload/Program.cs b/extra/admin-api/Spacebar.GatewayOffload/Program.cs index 76e7797c4..5cfe28d4e 100644 --- a/extra/admin-api/Spacebar.GatewayOffload/Program.cs +++ b/extra/admin-api/Spacebar.GatewayOffload/Program.cs @@ -5,6 +5,8 @@ using Spacebar.Interop.Authentication.AspNetCore; using Spacebar.Models.Db.Contexts; var builder = WebApplication.CreateBuilder(args); +if (!string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("APPSETTINGS_PATH"))) + builder.Configuration.AddJsonFile(Environment.GetEnvironmentVariable("APPSETTINGS_PATH")!); // Add services to the container. diff --git a/extra/admin-api/Spacebar.GatewayOffload/Spacebar.GatewayOffload.csproj b/extra/admin-api/Spacebar.GatewayOffload/Spacebar.GatewayOffload.csproj index e6c22cd47..423b7b440 100644 --- a/extra/admin-api/Spacebar.GatewayOffload/Spacebar.GatewayOffload.csproj +++ b/extra/admin-api/Spacebar.GatewayOffload/Spacebar.GatewayOffload.csproj @@ -11,6 +11,8 @@ + + diff --git a/extra/admin-api/SpacebarAdminAPI.sln b/extra/admin-api/SpacebarAdminAPI.sln index e28cf2abc..e6dfb5272 100644 --- a/extra/admin-api/SpacebarAdminAPI.sln +++ b/extra/admin-api/SpacebarAdminAPI.sln @@ -55,6 +55,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Spacebar.Models.Generic", " EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Spacebar.Models.Gateway", "Models\Spacebar.Models.Gateway\Spacebar.Models.Gateway.csproj", "{0057DA5E-3183-4A7A-B092-167137872FF8}" EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "DataMappings", "DataMappings", "{98939AC0-3ADA-4DB2-8B21-FFB6AF29A2D2}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Spacebar.DataMappings.Generic", "DataMappings\Spacebar.DataMappings.Generic\Spacebar.DataMappings.Generic.csproj", "{7ACE56E6-7A6C-459C-8A7E-5F572DDBB00E}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -341,6 +345,18 @@ Global {0057DA5E-3183-4A7A-B092-167137872FF8}.Release|x64.Build.0 = Release|Any CPU {0057DA5E-3183-4A7A-B092-167137872FF8}.Release|x86.ActiveCfg = Release|Any CPU {0057DA5E-3183-4A7A-B092-167137872FF8}.Release|x86.Build.0 = Release|Any CPU + {7ACE56E6-7A6C-459C-8A7E-5F572DDBB00E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {7ACE56E6-7A6C-459C-8A7E-5F572DDBB00E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {7ACE56E6-7A6C-459C-8A7E-5F572DDBB00E}.Debug|x64.ActiveCfg = Debug|Any CPU + {7ACE56E6-7A6C-459C-8A7E-5F572DDBB00E}.Debug|x64.Build.0 = Debug|Any CPU + {7ACE56E6-7A6C-459C-8A7E-5F572DDBB00E}.Debug|x86.ActiveCfg = Debug|Any CPU + {7ACE56E6-7A6C-459C-8A7E-5F572DDBB00E}.Debug|x86.Build.0 = Debug|Any CPU + {7ACE56E6-7A6C-459C-8A7E-5F572DDBB00E}.Release|Any CPU.ActiveCfg = Release|Any CPU + {7ACE56E6-7A6C-459C-8A7E-5F572DDBB00E}.Release|Any CPU.Build.0 = Release|Any CPU + {7ACE56E6-7A6C-459C-8A7E-5F572DDBB00E}.Release|x64.ActiveCfg = Release|Any CPU + {7ACE56E6-7A6C-459C-8A7E-5F572DDBB00E}.Release|x64.Build.0 = Release|Any CPU + {7ACE56E6-7A6C-459C-8A7E-5F572DDBB00E}.Release|x86.ActiveCfg = Release|Any CPU + {7ACE56E6-7A6C-459C-8A7E-5F572DDBB00E}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -366,5 +382,6 @@ Global {BB961FD8-61C2-4443-AB68-B8088CBC3D43} = {16DBEA54-D51A-4D91-84DF-C701B6B4786F} {58766A5F-BA91-41F1-8A09-44E96685E361} = {259D1A9B-2927-4571-A366-68C3BB30C2B2} {0057DA5E-3183-4A7A-B092-167137872FF8} = {259D1A9B-2927-4571-A366-68C3BB30C2B2} + {7ACE56E6-7A6C-459C-8A7E-5F572DDBB00E} = {98939AC0-3ADA-4DB2-8B21-FFB6AF29A2D2} EndGlobalSection EndGlobal diff --git a/extra/admin-api/Utilities/Spacebar.Cdn.Fsck/Program.cs b/extra/admin-api/Utilities/Spacebar.Cdn.Fsck/Program.cs index 2f9e2c576..df7653dbc 100644 --- a/extra/admin-api/Utilities/Spacebar.Cdn.Fsck/Program.cs +++ b/extra/admin-api/Utilities/Spacebar.Cdn.Fsck/Program.cs @@ -5,9 +5,12 @@ using Spacebar.Interop.Cdn.Abstractions; using Spacebar.Models.Db.Contexts; var builder = Host.CreateApplicationBuilder(args); +if (!string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("APPSETTINGS_PATH"))) + builder.Configuration.AddJsonFile(Environment.GetEnvironmentVariable("APPSETTINGS_PATH")!); + // builder.Services.AddSingleton(new ProxyFileSource("http://cdn.old.server.spacebar.chat")); builder.Services.AddSingleton(new FilesystemFileSource("/mnt/data/dedicated/spacebar-storage")); -builder.Services.AddSingleton(new LruFileCache(1*1024*1024*1024)); +builder.Services.AddSingleton(new LruFileCache(1 * 1024 * 1024 * 1024)); builder.Services.AddHostedService(); builder.Services.AddDbContextPool(options => { diff --git a/extra/admin-api/Utilities/Spacebar.Cdn.Migration/Program.cs b/extra/admin-api/Utilities/Spacebar.Cdn.Migration/Program.cs index 280ffe94b..ca684e0c2 100644 --- a/extra/admin-api/Utilities/Spacebar.Cdn.Migration/Program.cs +++ b/extra/admin-api/Utilities/Spacebar.Cdn.Migration/Program.cs @@ -7,6 +7,9 @@ using Spacebar.Interop.Cdn.Signing; using Spacebar.Models.Db.Contexts; var builder = Host.CreateApplicationBuilder(args); +if (!string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("APPSETTINGS_PATH"))) + builder.Configuration.AddJsonFile(Environment.GetEnvironmentVariable("APPSETTINGS_PATH")!); + // builder.Services.AddSingleton(new ProxyFileSource("http://cdn.old.server.spacebar.chat")); IFileSource fromSrc, toSrc; diff --git a/extra/admin-api/Utilities/Spacebar.CleanSettingsRows/Program.cs b/extra/admin-api/Utilities/Spacebar.CleanSettingsRows/Program.cs index 0f573f55e..8aa73707a 100644 --- a/extra/admin-api/Utilities/Spacebar.CleanSettingsRows/Program.cs +++ b/extra/admin-api/Utilities/Spacebar.CleanSettingsRows/Program.cs @@ -3,6 +3,9 @@ using Spacebar.CleanSettingsRows; using Spacebar.Models.Db.Contexts; var builder = Host.CreateApplicationBuilder(args); +if (!string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("APPSETTINGS_PATH"))) + builder.Configuration.AddJsonFile(Environment.GetEnvironmentVariable("APPSETTINGS_PATH")!); + builder.Services.AddHostedService(); builder.Services.AddDbContextPool(options => { @@ -11,6 +14,5 @@ builder.Services.AddDbContextPool(options => { .EnableDetailedErrors(); }); - var host = builder.Build(); host.Run(); \ No newline at end of file diff --git a/extra/admin-api/outputs.nix b/extra/admin-api/outputs.nix index 63f082499..434301009 100644 --- a/extra/admin-api/outputs.nix +++ b/extra/admin-api/outputs.nix @@ -66,6 +66,18 @@ flake-utils.lib.eachSystem flake-utils.lib.allSystems ( proj = self.packages.${system}; in { + # Data mappings + Spacebar-DataMappings-Generic = makeNupkg { + name = "Spacebar.DataMappings.Generic"; + projectFile = "Spacebar.DataMappings.Generic.csproj"; + nugetDeps = DataMappings/Spacebar.DataMappings.Generic/deps.json; + srcRoot = DataMappings/Spacebar.DataMappings.Generic; + projectReferences = [ + proj.Spacebar-Models-Db + proj.Spacebar-Models-Generic + ]; + }; + # Interop Spacebar-Interop-Authentication = makeNupkg { name = "Spacebar.Interop.Authentication"; @@ -197,6 +209,7 @@ flake-utils.lib.eachSystem flake-utils.lib.allSystems ( srcRoot = ./Spacebar.GatewayOffload; packNupkg = false; projectReferences = [ + proj.Spacebar-DataMappings-Generic proj.Spacebar-Interop-Authentication proj.Spacebar-Interop-Authentication-AspNetCore proj.Spacebar-Interop-Replication-Abstractions @@ -223,7 +236,25 @@ flake-utils.lib.eachSystem flake-utils.lib.allSystems ( contents = [ self.packages.${system}.Spacebar-AdminApi ]; config = { Cmd = [ "${self.outputs.packages.${system}.Spacebar-AdminApi}/bin/Spacebar.AdminApi" ]; - Expose = [ "3001" ]; + Expose = [ "5000" ]; + }; + }; + containers.docker.gateway-offload = pkgs.dockerTools.buildLayeredImage { + name = "spacebar-server-ts-gateway-offload"; + tag = builtins.replaceStrings [ "+" ] [ "_" ] self.packages.${system}.Spacebar-AdminApi.version; + contents = [ self.packages.${system}.Spacebar-AdminApi ]; + config = { + Cmd = [ "${lib.getExe self.outputs.packages.${system}.Spacebar-GatewayOffload}" ]; + Expose = [ "5000" ]; + }; + }; + containers.docker.cdn-cs = pkgs.dockerTools.buildLayeredImage { + name = "spacebar-server-ts-cdn-cs"; + tag = builtins.replaceStrings [ "+" ] [ "_" ] self.packages.${system}.Spacebar-AdminApi.version; + contents = [ self.packages.${system}.Spacebar-AdminApi ]; + config = { + Cmd = [ "${lib.getExe self.outputs.packages.${system}.Spacebar-AdminApi}" ]; + Expose = [ "5000" ]; }; }; } @@ -238,6 +269,8 @@ flake-utils.lib.eachSystem flake-utils.lib.allSystems ( x86_64-linux = { # spacebar-server-tests = self.packages.x86_64-linux.default.passthru.tests; docker-admin-api = self.containers.x86_64-linux.docker.admin-api; + docker-gateway-offload = self.containers.x86_64-linux.docker.gateway-offload; + docker-cdn-cs = self.containers.x86_64-linux.docker.cdn-cs; }; }; } diff --git a/flake.lock b/flake.lock index 6bae7c7ca2c568613c5ac12f4f9dab930b0e0324..c75943333ac6699b8dace9af6fdfa9876a4367d1 100644 GIT binary patch delta 121 zcmcb~eUp2G1`~(5fuW_jskz0(39{Obk%c+oMG*#;+8$-@#THe*W~oV0h6UxJg<08Y z1$ku_xvAxm<&OEa delta 121 zcmcb~eUp2G1`~&wrHPrLg@MV$39{O`#qJitp{C)*PN8XGVTt;dK>