diff --git a/extra/admin-api/DataMappings/Spacebar.DataMappings.Generic/Member.cs b/extra/admin-api/DataMappings/Spacebar.DataMappings.Generic/Member.cs new file mode 100644 index 000000000..2650fc383 --- /dev/null +++ b/extra/admin-api/DataMappings/Spacebar.DataMappings.Generic/Member.cs @@ -0,0 +1,21 @@ +using Spacebar.Models.Generic; + +namespace Spacebar.DataMappings.Generic; + +public static class Member +{ + public static Models.Generic.Member ToPublicMember(this Models.Db.Models.Member member, PartialUser? partialUser = null) + { + return new() + { + User = partialUser ?? member.IdNavigation.ToPartialUser(), + AvatarDecorationData = member.AvatarDecorationData, + Avatar = string.IsNullOrWhiteSpace(member.Avatar) ? null : member.Avatar, + Banner = string.IsNullOrWhiteSpace(member.Banner) ? null : member.Banner, + Collectibles = member.Collectibles, + DisplayNameStyles = member.DisplayNameStyles, + Bio = string.IsNullOrWhiteSpace(member.Bio) ? null : member.Bio, + Nick = string.IsNullOrWhiteSpace(member.Nick) ? null : member.Nick + }; + } +} \ No newline at end of file diff --git a/extra/admin-api/DataMappings/Spacebar.DataMappings.Generic/Spacebar.DataMappings.Generic.csproj b/extra/admin-api/DataMappings/Spacebar.DataMappings.Generic/Spacebar.DataMappings.Generic.csproj new file mode 100644 index 000000000..3f8936a6f --- /dev/null +++ b/extra/admin-api/DataMappings/Spacebar.DataMappings.Generic/Spacebar.DataMappings.Generic.csproj @@ -0,0 +1,16 @@ + + + + net10.0 + enable + enable + + + + + + + + + + diff --git a/extra/admin-api/DataMappings/Spacebar.DataMappings.Generic/User.cs b/extra/admin-api/DataMappings/Spacebar.DataMappings.Generic/User.cs new file mode 100644 index 000000000..607af9411 --- /dev/null +++ b/extra/admin-api/DataMappings/Spacebar.DataMappings.Generic/User.cs @@ -0,0 +1,26 @@ +using Spacebar.Models.Generic; + +namespace Spacebar.DataMappings.Generic; + +public static class User +{ + public static PartialUser ToPartialUser(this Models.Db.Models.User user) + { + return new PartialUser() { + Id = user.Id, + Discriminator = user.Discriminator, + Username = user.Username, + AccentColor = user.AccentColor, + Avatar = user.Avatar, + AvatarDecorationData = user.AvatarDecorationData, + Banner = user.Banner, + Bot = user.Bot, + Collectibles = user.Collectibles, + DisplayNameStyles = user.DisplayNameStyles, + // GlobalName = x.GlobalName, + PrimaryGuild = user.PrimaryGuild, + PublicFlags = user.PublicFlags, + System = user.System, + }; + } +} \ No newline at end of file diff --git a/extra/admin-api/DataMappings/Spacebar.DataMappings.Generic/deps.json b/extra/admin-api/DataMappings/Spacebar.DataMappings.Generic/deps.json new file mode 100644 index 000000000..aead01783 --- /dev/null +++ b/extra/admin-api/DataMappings/Spacebar.DataMappings.Generic/deps.json @@ -0,0 +1,247 @@ +[ + { + "pname": "Humanizer.Core", + "version": "2.14.1", + "hash": "sha256-EXvojddPu+9JKgOG9NSQgUTfWq1RpOYw7adxDPKDJ6o=" + }, + { + "pname": "Microsoft.Build.Framework", + "version": "17.11.31", + "hash": "sha256-YS4oASrmC5dmZrx5JPS7SfKmUpIJErlUpVDsU3VrfFE=" + }, + { + "pname": "Microsoft.Build.Framework", + "version": "18.0.2", + "hash": "sha256-fO31KAdDs2J0RUYD1ov9UB3ucsbALan7K0YdWW+yg7A=" + }, + { + "pname": "Microsoft.CodeAnalysis.Analyzers", + "version": "3.11.0", + "hash": "sha256-hQ2l6E6PO4m7i+ZsfFlEx+93UsLPo4IY3wDkNG11/Sw=" + }, + { + "pname": "Microsoft.CodeAnalysis.Common", + "version": "5.0.0", + "hash": "sha256-g4ALvBSNyHEmSb1l5TFtWW7zEkiRmhqLx4XWZu9sr2U=" + }, + { + "pname": "Microsoft.CodeAnalysis.CSharp", + "version": "5.0.0", + "hash": "sha256-ctBCkQGFpH/xT5rRE3xibu9YxPD108RuC4a4Z25koG8=" + }, + { + "pname": "Microsoft.CodeAnalysis.CSharp.Workspaces", + "version": "5.0.0", + "hash": "sha256-yWVcLt/f2CouOfFy966glGdtSFy+RcgrU1dd9UtlL/Q=" + }, + { + "pname": "Microsoft.CodeAnalysis.Workspaces.Common", + "version": "5.0.0", + "hash": "sha256-Bir5e1gEhgQQ6upQmVKQHAKLRfenAu60DAzNupNnZsQ=" + }, + { + "pname": "Microsoft.CodeAnalysis.Workspaces.MSBuild", + "version": "5.0.0", + "hash": "sha256-+58+iqTayTiE0pDaog1U8mjaDA8bNNDLA8gjCQZZudo=" + }, + { + "pname": "Microsoft.EntityFrameworkCore", + "version": "10.0.0", + "hash": "sha256-xfgrlxhtOkQwF5Q7j8gSm41URJiH8IuJ/T/Dh88++hE=" + }, + { + "pname": "Microsoft.EntityFrameworkCore", + "version": "10.0.2", + "hash": "sha256-FS6T8EnaWCMtj4PnZhh+oF8mcM44VlM3wkTSMlpte9A=" + }, + { + "pname": "Microsoft.EntityFrameworkCore.Abstractions", + "version": "10.0.0", + "hash": "sha256-UDgZbRQcGPaKsE53EH6bvJiv+Q4KSxAbnsVhTVFGG4Q=" + }, + { + "pname": "Microsoft.EntityFrameworkCore.Abstractions", + "version": "10.0.2", + "hash": "sha256-qkDfIJpcPO2kk4n5OE/13hI/0mUygpTofInn95XjRZI=" + }, + { + "pname": "Microsoft.EntityFrameworkCore.Analyzers", + "version": "10.0.0", + "hash": "sha256-7Q0jYJO50cqGI+u6gLpootbB8GZvgsgtg0F9FZI1jig=" + }, + { + "pname": "Microsoft.EntityFrameworkCore.Analyzers", + "version": "10.0.2", + "hash": "sha256-yOv78rgAACBz1zjitpcZbQQ3zx8huJongZTHkhN4PQ0=" + }, + { + "pname": "Microsoft.EntityFrameworkCore.Design", + "version": "10.0.2", + "hash": "sha256-bTShsGux0y/49PIIMb/4ZX3x5+rPacvT5/NcooNCI1Y=" + }, + { + "pname": "Microsoft.EntityFrameworkCore.Relational", + "version": "10.0.0", + "hash": "sha256-vOP2CE5YA551BlpbOuIy6RuAiAEPEpCVS1cEE33/zN4=" + }, + { + "pname": "Microsoft.EntityFrameworkCore.Relational", + "version": "10.0.2", + "hash": "sha256-Y4jPpoYhKizg5wF6QfkBX4sYlE2FU1bYhfoDN3xkhKM=" + }, + { + "pname": "Microsoft.Extensions.Caching.Abstractions", + "version": "10.0.2", + "hash": "sha256-nKmQuZTt1g5/8gBajo7wdCV64kdCucdiQR8JTt7ZZb0=" + }, + { + "pname": "Microsoft.Extensions.Caching.Memory", + "version": "10.0.0", + "hash": "sha256-AMgDSm1k6q0s17spGtyR5q8nAqUFDOxl/Fe38f9M+d4=" + }, + { + "pname": "Microsoft.Extensions.Caching.Memory", + "version": "10.0.2", + "hash": "sha256-sRUF7DM0s1yzZnfjM/hF9A/IysE6Er23gZ6jST+RWh0=" + }, + { + "pname": "Microsoft.Extensions.Configuration.Abstractions", + "version": "10.0.2", + "hash": "sha256-P+0kaDGO+xB9KxF9eWHDJ4hzi05sUGM/uMNEX5NdBTE=" + }, + { + "pname": "Microsoft.Extensions.DependencyInjection", + "version": "10.0.2", + "hash": "sha256-/9UWQRAI2eoocnJWWf1ktnAx/1Gt65c16fc0Xqr9+CQ=" + }, + { + "pname": "Microsoft.Extensions.DependencyInjection", + "version": "9.0.0", + "hash": "sha256-dAH52PPlTLn7X+1aI/7npdrDzMEFPMXRv4isV1a+14k=" + }, + { + "pname": "Microsoft.Extensions.DependencyInjection.Abstractions", + "version": "10.0.2", + "hash": "sha256-UF9T13V5SQxJy2msfLmyovLmitZrjJayf8gHH+uK2eg=" + }, + { + "pname": "Microsoft.Extensions.DependencyInjection.Abstractions", + "version": "9.0.0", + "hash": "sha256-CncVwkKZ5CsIG2O0+OM9qXuYXh3p6UGyueTHSLDVL+c=" + }, + { + "pname": "Microsoft.Extensions.DependencyModel", + "version": "10.0.2", + "hash": "sha256-w/dGIjtZiGH+KW3969BPOdQpQEV+WB7RPTa2MK2DavE=" + }, + { + "pname": "Microsoft.Extensions.Logging", + "version": "10.0.0", + "hash": "sha256-P+zPAadLL63k/GqK34/qChqQjY9aIRxZfxlB9lqsSrs=" + }, + { + "pname": "Microsoft.Extensions.Logging", + "version": "10.0.2", + "hash": "sha256-9+gfQwK32JMYscW1YvyCWEzc9mRZOjCACoD9U1vVaJI=" + }, + { + "pname": "Microsoft.Extensions.Logging", + "version": "9.0.0", + "hash": "sha256-kR16c+N8nQrWeYLajqnXPg7RiXjZMSFLnKLEs4VfjcM=" + }, + { + "pname": "Microsoft.Extensions.Logging.Abstractions", + "version": "10.0.0", + "hash": "sha256-BnhgGZc01HwTSxogavq7Ueq4V7iMA3wPnbfRwQ4RhGk=" + }, + { + "pname": "Microsoft.Extensions.Logging.Abstractions", + "version": "10.0.2", + "hash": "sha256-ndKGzq8+2J/hvaIULwBui0L/jDyMQTAY24j+ohX5VX8=" + }, + { + "pname": "Microsoft.Extensions.Logging.Abstractions", + "version": "9.0.0", + "hash": "sha256-iBTs9twjWXFeERt4CErkIIcoJZU1jrd1RWCI8V5j7KU=" + }, + { + "pname": "Microsoft.Extensions.Options", + "version": "10.0.2", + "hash": "sha256-12AfUEDdta/pmZUyEyqSUfOk0YoA7JOfGmIYnZQ//qk=" + }, + { + "pname": "Microsoft.Extensions.Options", + "version": "9.0.0", + "hash": "sha256-DT5euAQY/ItB5LPI8WIp6Dnd0lSvBRP35vFkOXC68ck=" + }, + { + "pname": "Microsoft.Extensions.Primitives", + "version": "10.0.2", + "hash": "sha256-8Ccrjjv9cFVf9RyCc7GS/Byt8+DXdSNea0UX3A5BEdA=" + }, + { + "pname": "Microsoft.Extensions.Primitives", + "version": "9.0.0", + "hash": "sha256-ZNLusK1CRuq5BZYZMDqaz04PIKScE2Z7sS2tehU7EJs=" + }, + { + "pname": "Microsoft.VisualStudio.SolutionPersistence", + "version": "1.0.52", + "hash": "sha256-KZGPtOXe6Hv8RrkcsgoLKTRyaCScIpQEa2NhNB3iOXw=" + }, + { + "pname": "Mono.TextTemplating", + "version": "3.0.0", + "hash": "sha256-VlgGDvgNZb7MeBbIZ4DE2Nn/j2aD9k6XqNHnASUSDr0=" + }, + { + "pname": "Newtonsoft.Json", + "version": "13.0.3", + "hash": "sha256-hy/BieY4qxBWVVsDqqOPaLy1QobiIapkbrESm6v2PHc=" + }, + { + "pname": "Npgsql", + "version": "10.0.0", + "hash": "sha256-UVKz9dH/rVCCbMyFdqA31RYpht1XgDRLIqUy0Dp9ACQ=" + }, + { + "pname": "Npgsql.EntityFrameworkCore.PostgreSQL", + "version": "10.0.0", + "hash": "sha256-XIJxnTMektQVP1qtslEIGbmBGrIQsvjQjCMRTs9UIbg=" + }, + { + "pname": "System.CodeDom", + "version": "6.0.0", + "hash": "sha256-uPetUFZyHfxjScu5x4agjk9pIhbCkt5rG4Axj25npcQ=" + }, + { + "pname": "System.Composition", + "version": "9.0.0", + "hash": "sha256-FehOkQ2u1p8mQ0/wn3cZ+24HjhTLdck8VZYWA1CcgbM=" + }, + { + "pname": "System.Composition.AttributedModel", + "version": "9.0.0", + "hash": "sha256-a7y7H6zj+kmYkllNHA402DoVfY9IaqC3Ooys8Vzl24M=" + }, + { + "pname": "System.Composition.Convention", + "version": "9.0.0", + "hash": "sha256-tw4vE5JRQ60ubTZBbxoMPhtjOQCC3XoDFUH7NHO7o8U=" + }, + { + "pname": "System.Composition.Hosting", + "version": "9.0.0", + "hash": "sha256-oOxU+DPEEfMCuNLgW6wSkZp0JY5gYt44FJNnWt+967s=" + }, + { + "pname": "System.Composition.Runtime", + "version": "9.0.0", + "hash": "sha256-AyIe+di1TqwUBbSJ/sJ8Q8tzsnTN+VBdJw4K8xZz43s=" + }, + { + "pname": "System.Composition.TypedParts", + "version": "9.0.0", + "hash": "sha256-F5fpTUs3Rr7yP/NyIzr+Xn5NdTXXp8rrjBnF9UBBUog=" + } +] diff --git a/extra/admin-api/Interop/Spacebar.Interop.Authentication/SpacebarAuthenticationService.cs b/extra/admin-api/Interop/Spacebar.Interop.Authentication/SpacebarAuthenticationService.cs index 7b4ad1c8b..5f9ff500f 100644 --- a/extra/admin-api/Interop/Spacebar.Interop.Authentication/SpacebarAuthenticationService.cs +++ b/extra/admin-api/Interop/Spacebar.Interop.Authentication/SpacebarAuthenticationService.cs @@ -46,4 +46,15 @@ public class SpacebarAuthenticationService(ILogger GetCurrentSessionAsync(string token) { + var res = await ValidateTokenAsync(token); + return await UserCache.GetOrAdd(token, + async () => { + var uid = config.OverrideUid ?? res?.ClaimsIdentity.Claims.First(x => x.Type == "id").Value; + if (string.IsNullOrWhiteSpace(uid)) throw new InvalidOperationException("No user ID specified, is the access token valid?"); + return await db.Users.FindAsync(uid) ?? throw new InvalidOperationException(); + }, + config.AuthCacheExpiry); + } } \ No newline at end of file diff --git a/extra/admin-api/Models/Spacebar.Models.Gateway/IdentifyRequest.cs b/extra/admin-api/Models/Spacebar.Models.Gateway/IdentifyRequest.cs new file mode 100644 index 000000000..a956655ed --- /dev/null +++ b/extra/admin-api/Models/Spacebar.Models.Gateway/IdentifyRequest.cs @@ -0,0 +1,89 @@ +using System.Text.Json.Nodes; +using System.Text.Json.Serialization; + +namespace Spacebar.Models.Gateway; + +public class IdentifyRequest +{ + [JsonPropertyName("token")] + public string Token { get; set; } + + [JsonPropertyName("properties")] + public JsonObject ClientProperties { get; set; } + + [JsonPropertyName("compress")] + public bool? Compress { get; set; } + + [JsonPropertyName("large_threshold")] + public int? LargeTreshold { get; set; } + + [JsonPropertyName("shard")] + public int[]? Shard { get; set; } + + [JsonPropertyName("presence")] + public JsonObject? Presence { get; set; } + + [JsonPropertyName("intents")] + public GatewayIntentFlags? Intents { get; set; } + + [JsonPropertyName("capabilities")] + public GatewayCapabilityFlags? Capabilities { get; set; } + + [JsonPropertyName("client_state")] + public JsonObject? ClientState { get; set; } +} + +[Flags] +public enum GatewayIntentFlags +{ + Guilds = 1, + GuildMembers = 1 << 1, + GuildModeration = 1 << 2, + GuildEmojisAndStickers = 1 << 3, + GuildIntegrations = 1 << 4, + GuildWebhooks = 1 << 5, + GuildInvites = 1 << 6, + GuildVoiceStates = 1 << 7, + GuildPresences = 1 << 8, + GuildMessages = 1 << 9, + GuildMessageReactions = 1 << 10, + GuildMessageTyping = 1 << 11, + DirectMessages = 1 << 12, + DirectMessageReactions = 1 << 13, + DirectMessageTyping = 1 << 14, + MessageContent = 1 << 15, + GuildScheduledEvents = 1 << 16, + PrivateEmbeddedActivities = 1 << 17, + PrivateChannels = 1 << 18, + Calls = 1 << 19, + AutoModerationConfiguration = 1 << 20, + AutoModerationExecution = 1 << 21, + UserRelationships = 1 << 22, + UserPresence = 1 << 23, + GuildMessagePolls = 1 << 24, + DirectMessagePolls = 1 << 25, + DirectEmbeddedActivities = 1 << 26, + Lobbies = 1 << 27, + LobbyDelete = 1 << 28 +} + +[Flags] +public enum GatewayCapabilityFlags +{ + LazyUserNotes = 1, + NoAffineUserIds = 1 << 1, + VersionedReadStates = 1 << 2, + VersionedUserGuildSettings = 1 << 3, + DedupeUserObjects = 1 << 4, + PrioritizedReadyPayload = 1 << 5, + MultipleGuildExperimentPopulations = 1 << 6, + NonChannelReadStates = 1 << 7, + AuthTokenRefresh = 1 << 8, + UserSettingsProto = 1 << 9, + ClientStateV2 = 1 << 10, + PassiveGuildUpdate = 1 << 11, + AutoCallConnect = 1 << 12, + DebounceMessageReactions = 1 << 13, + PassiveGuildUpdateV2 = 1 << 14, + AutoLobbyConnect = 1 << 16 +} \ No newline at end of file diff --git a/extra/admin-api/Models/Spacebar.Models.Gateway/LazyRequest.cs b/extra/admin-api/Models/Spacebar.Models.Gateway/LazyRequest.cs new file mode 100644 index 000000000..200dccea6 --- /dev/null +++ b/extra/admin-api/Models/Spacebar.Models.Gateway/LazyRequest.cs @@ -0,0 +1,24 @@ +using System.Text.Json.Serialization; + +namespace Spacebar.Models.Gateway; + +public class LazyRequest +{ + [JsonPropertyName("guild_id")] + public string GuildId { get; set; } + + [JsonPropertyName("channels")] + public Dictionary>> Channels { get; set; } + + [JsonPropertyName("members")] + public bool Members { get; set; } + + [JsonPropertyName("threads")] + public bool Threads { get; set; } + + [JsonPropertyName("activities")] + public bool Activities { get; set; } + + [JsonPropertyName("typing")] + public bool Typing { get; set; } +} \ No newline at end of file diff --git a/extra/admin-api/Spacebar.AdminApi/Program.cs b/extra/admin-api/Spacebar.AdminApi/Program.cs index cbc504551..b0f8f2090 100644 --- a/extra/admin-api/Spacebar.AdminApi/Program.cs +++ b/extra/admin-api/Spacebar.AdminApi/Program.cs @@ -9,6 +9,8 @@ using Spacebar.Interop.Replication.UnixSocket; using Spacebar.Models.Db.Contexts; var builder = WebApplication.CreateBuilder(args); +if (!string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("APPSETTINGS_PATH"))) + builder.Configuration.AddJsonFile(Environment.GetEnvironmentVariable("APPSETTINGS_PATH")!); // Add services to the container. @@ -19,9 +21,7 @@ builder.Services.AddControllers(options => { options.JsonSerializerOptions.DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull; options.JsonSerializerOptions.WriteIndented = true; // options.JsonSerializerOptions.DefaultBufferSize = ; -}).AddMvcOptions(o=> { - o.SuppressOutputFormatterBuffering = true; -}); +}).AddMvcOptions(o => { o.SuppressOutputFormatterBuffering = true; }); // Learn more about configuring OpenAPI at https://aka.ms/aspnet/openapi builder.Services.AddOpenApi(); diff --git a/extra/admin-api/Spacebar.Cdn/Program.cs b/extra/admin-api/Spacebar.Cdn/Program.cs index f7a319d30..47b518331 100644 --- a/extra/admin-api/Spacebar.Cdn/Program.cs +++ b/extra/admin-api/Spacebar.Cdn/Program.cs @@ -6,10 +6,12 @@ using Spacebar.Interop.Cdn.Abstractions; using Spacebar.Models.Db.Contexts; var builder = WebApplication.CreateBuilder(args); +if (!string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("APPSETTINGS_PATH"))) + builder.Configuration.AddJsonFile(Environment.GetEnvironmentVariable("APPSETTINGS_PATH")!); // Add services to the container. builder.Services.AddSingleton(new ProxyFileSource("http://cdn.old.server.spacebar.chat")); -builder.Services.AddSingleton(new LruFileCache(1*1024*1024*1024)); +builder.Services.AddSingleton(new LruFileCache(1 * 1024 * 1024 * 1024)); builder.Services.AddSingleton(); builder.Services.AddSingleton(); @@ -45,7 +47,7 @@ app.Use((context, next) => { }); // fallback to proxy in case we dont have a specific endpoint... -app.MapFallback("{*_}",async context => { +app.MapFallback("{*_}", async context => { var client = new StreamingHttpClient(); var requestMessage = new HttpRequestMessage( new HttpMethod(context.Request.Method), diff --git a/extra/admin-api/Spacebar.GatewayOffload/Controllers/IdentifyController.cs b/extra/admin-api/Spacebar.GatewayOffload/Controllers/IdentifyController.cs new file mode 100644 index 000000000..6ce9f9fb8 --- /dev/null +++ b/extra/admin-api/Spacebar.GatewayOffload/Controllers/IdentifyController.cs @@ -0,0 +1,13 @@ +using Microsoft.AspNetCore.Mvc; +using Spacebar.Interop.Authentication; +using Spacebar.Interop.Authentication.AspNetCore; +using Spacebar.Models.Db.Contexts; + +namespace Spacebar.GatewayOffload.Controllers; + +[ApiController] +[Route("/_spacebar/offload/gateway/Identify")] +public class IdentifyController(ILogger logger, SpacebarAuthenticationService authService, SpacebarDbContext db, IServiceProvider sp) : ControllerBase +{ + +} \ No newline at end of file diff --git a/extra/admin-api/Spacebar.GatewayOffload/Controllers/Op12Controller.cs b/extra/admin-api/Spacebar.GatewayOffload/Controllers/Op12Controller.cs index 0b7b5c82f..a2008345e 100644 --- a/extra/admin-api/Spacebar.GatewayOffload/Controllers/Op12Controller.cs +++ b/extra/admin-api/Spacebar.GatewayOffload/Controllers/Op12Controller.cs @@ -1,21 +1,25 @@ +using System.Linq.Expressions; using System.Text.Json; using System.Text.Json.Nodes; using ArcaneLibs.Extensions; using Microsoft.AspNetCore.Mvc; using Microsoft.EntityFrameworkCore; +using Spacebar.DataMappings.Generic; using Spacebar.Interop.Authentication.AspNetCore; using Spacebar.Interop.Replication.Abstractions; using Spacebar.Models.Db.Contexts; +using Spacebar.Models.Db.Models; using Spacebar.Models.Gateway; -using Spacebar.Models.Generic; namespace Spacebar.GatewayOffload.Controllers; [ApiController] [Route("/_spacebar/offload/gateway/GuildSync")] -public class Op12Controller(ILogger logger, SpacebarAspNetAuthenticationService authService, SpacebarDbContext db, IServiceProvider sp) : ControllerBase { +public class Op12Controller(ILogger logger, SpacebarAspNetAuthenticationService authService, SpacebarDbContext db, IServiceProvider sp) : ControllerBase +{ [HttpPost("")] - public async IAsyncEnumerable DoGuildSync(List guildIds) { + public async IAsyncEnumerable DoGuildSync(List guildIds) + { var user = await authService.GetCurrentUserAsync(Request); guildIds = (await db.Members.Where(x => x.Id == user.Id).Select(x => x.GuildId).ToListAsync()) .Intersect(guildIds) @@ -23,8 +27,10 @@ public class Op12Controller(ILogger logger, SpacebarAspNetAuthen .ToList(); var syncs = guildIds.Select(GetGuildSyncAsync).ToList().ToAsyncResultEnumerable(); - await foreach (var res in syncs) { - yield return new() { + await foreach (var res in syncs) + { + yield return new() + { Origin = "OFFLOAD_GUILD_SYNC", UserId = user.Id, Event = "GUILD_SYNC", @@ -34,57 +40,48 @@ public class Op12Controller(ILogger logger, SpacebarAspNetAuthen } } - private async Task GetGuildSyncAsync(string guildId) { + // TODO: figure out how to abstract this to a function without EFCore complaining about not being translatable... + //private static Func IsOnline = (Session session) => session.Status != "offline" && session.Status != "invisible" && session.Status != "unknown"; + + private async Task GetGuildSyncAsync(string guildId) + { await using var sc = sp.CreateAsyncScope(); - var offlineTreshold = DateTime.Now.Subtract(TimeSpan.FromDays(14)); var _db = sc.ServiceProvider.GetRequiredService(); var memberCount = await _db.Members.Where(x => x.GuildId == guildId).CountAsync(); + + var offlineTreshold = DateTime.Now.Subtract(TimeSpan.FromDays(14)); + var isLargeGuild = memberCount > 10000; + var members = await _db.Members.Where(x => x.GuildId == guildId) .Include(x => x.IdNavigation) .ThenInclude(x => x.Sessions.Where(s => - !s.IsAdminSession && (s.Status != "offline" && s.Status != "invisible") && (memberCount < 1000 || s.LastSeen >= offlineTreshold))) + !s.IsAdminSession && ( + // see TODO on IsOnline + s.Status != "offline" && s.Status != "invisible" && s.Status != "unknown" + ) && (!isLargeGuild || s.LastSeen >= offlineTreshold))) .Where(x => x.IdNavigation.Sessions.Count > 0) // ignore members without sessions .ToListAsync(); - var mappedPartialUsers = members.Select(x => x.IdNavigation).ToDictionary(x => x.Id, x => new PartialUser() { - Id = x.Id, - Discriminator = x.Discriminator, - Username = x.Username, - AccentColor = x.AccentColor, - Avatar = x.Avatar, - AvatarDecorationData = x.AvatarDecorationData, - Banner = x.Banner, - Bot = x.Bot, - Collectibles = x.Collectibles, - DisplayNameStyles = x.DisplayNameStyles, - // GlobalName = x.GlobalName, - PrimaryGuild = x.PrimaryGuild, - PublicFlags = x.PublicFlags, - System = x.System, - }); - var mappedMembers = members.ToDictionary(m => m.Id, m => new Member() { - User = mappedPartialUsers[m.Id], - AvatarDecorationData = m.AvatarDecorationData, - Avatar = string.IsNullOrWhiteSpace(m.Avatar) ? null : m.Avatar, - Banner = string.IsNullOrWhiteSpace(m.Banner) ? null : m.Banner, - Collectibles = m.Collectibles, - DisplayNameStyles = m.DisplayNameStyles, - Bio = string.IsNullOrWhiteSpace(m.Bio) ? null : m.Bio, - Nick = string.IsNullOrWhiteSpace(m.Nick) ? null : m.Nick - }); - var presences = members.Select(x => x.IdNavigation).Where(x => x.Sessions.Count > 0).ToDictionary(x => x.Id, x => { + var mappedPartialUsers = members.Select(x => x.IdNavigation).ToDictionary(x => x.Id, x => x.ToPartialUser()); + var mappedMembers = members.ToDictionary(m => m.Id, m => m.ToPublicMember(mappedPartialUsers[m.Id])); + + var presences = members.Select(x => x.IdNavigation).Where(x => x.Sessions.Count > 0).ToDictionary(x => x.Id, x => + { var sortedSessions = x.Sessions.OrderByDescending(s => s.LastSeen).ToList(); - return new PresenceResponse() { + return new PresenceResponse() + { GuildId = guildId, User = mappedPartialUsers[x.Id], Activities = x.Sessions.Where(s => s.Status is not ("offline" or "invisible" or "unknown")) .SelectMany(s => JsonSerializer.Deserialize(s.Activities) ?? []).ToList(), Status = sortedSessions.FirstOrDefault(s => !string.IsNullOrWhiteSpace(s.Status))?.Status ?? "offline", - ClientStatus = JsonSerializer.Deserialize(sortedSessions.First(s => !string.IsNullOrWhiteSpace(s.ClientStatus)).ClientStatus) ?? new() + ClientStatus = JsonSerializer.Deserialize(sortedSessions.First(s => !string.IsNullOrWhiteSpace(s.ClientStatus)).ClientStatus) ?? + new() }; }).Where(x => x.Value.Activities.Count > 0).ToDictionary(); - var r = new GuildSyncResponse() { + var r = new GuildSyncResponse() + { GuildId = guildId, Members = mappedMembers.Values.ToList(), Presences = presences.Values.ToList() diff --git a/extra/admin-api/Spacebar.GatewayOffload/Program.cs b/extra/admin-api/Spacebar.GatewayOffload/Program.cs index 76e7797c4..5cfe28d4e 100644 --- a/extra/admin-api/Spacebar.GatewayOffload/Program.cs +++ b/extra/admin-api/Spacebar.GatewayOffload/Program.cs @@ -5,6 +5,8 @@ using Spacebar.Interop.Authentication.AspNetCore; using Spacebar.Models.Db.Contexts; var builder = WebApplication.CreateBuilder(args); +if (!string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("APPSETTINGS_PATH"))) + builder.Configuration.AddJsonFile(Environment.GetEnvironmentVariable("APPSETTINGS_PATH")!); // Add services to the container. diff --git a/extra/admin-api/Spacebar.GatewayOffload/Spacebar.GatewayOffload.csproj b/extra/admin-api/Spacebar.GatewayOffload/Spacebar.GatewayOffload.csproj index e6c22cd47..423b7b440 100644 --- a/extra/admin-api/Spacebar.GatewayOffload/Spacebar.GatewayOffload.csproj +++ b/extra/admin-api/Spacebar.GatewayOffload/Spacebar.GatewayOffload.csproj @@ -11,6 +11,8 @@ + + diff --git a/extra/admin-api/SpacebarAdminAPI.sln b/extra/admin-api/SpacebarAdminAPI.sln index e28cf2abc..e6dfb5272 100644 --- a/extra/admin-api/SpacebarAdminAPI.sln +++ b/extra/admin-api/SpacebarAdminAPI.sln @@ -55,6 +55,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Spacebar.Models.Generic", " EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Spacebar.Models.Gateway", "Models\Spacebar.Models.Gateway\Spacebar.Models.Gateway.csproj", "{0057DA5E-3183-4A7A-B092-167137872FF8}" EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "DataMappings", "DataMappings", "{98939AC0-3ADA-4DB2-8B21-FFB6AF29A2D2}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Spacebar.DataMappings.Generic", "DataMappings\Spacebar.DataMappings.Generic\Spacebar.DataMappings.Generic.csproj", "{7ACE56E6-7A6C-459C-8A7E-5F572DDBB00E}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -341,6 +345,18 @@ Global {0057DA5E-3183-4A7A-B092-167137872FF8}.Release|x64.Build.0 = Release|Any CPU {0057DA5E-3183-4A7A-B092-167137872FF8}.Release|x86.ActiveCfg = Release|Any CPU {0057DA5E-3183-4A7A-B092-167137872FF8}.Release|x86.Build.0 = Release|Any CPU + {7ACE56E6-7A6C-459C-8A7E-5F572DDBB00E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {7ACE56E6-7A6C-459C-8A7E-5F572DDBB00E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {7ACE56E6-7A6C-459C-8A7E-5F572DDBB00E}.Debug|x64.ActiveCfg = Debug|Any CPU + {7ACE56E6-7A6C-459C-8A7E-5F572DDBB00E}.Debug|x64.Build.0 = Debug|Any CPU + {7ACE56E6-7A6C-459C-8A7E-5F572DDBB00E}.Debug|x86.ActiveCfg = Debug|Any CPU + {7ACE56E6-7A6C-459C-8A7E-5F572DDBB00E}.Debug|x86.Build.0 = Debug|Any CPU + {7ACE56E6-7A6C-459C-8A7E-5F572DDBB00E}.Release|Any CPU.ActiveCfg = Release|Any CPU + {7ACE56E6-7A6C-459C-8A7E-5F572DDBB00E}.Release|Any CPU.Build.0 = Release|Any CPU + {7ACE56E6-7A6C-459C-8A7E-5F572DDBB00E}.Release|x64.ActiveCfg = Release|Any CPU + {7ACE56E6-7A6C-459C-8A7E-5F572DDBB00E}.Release|x64.Build.0 = Release|Any CPU + {7ACE56E6-7A6C-459C-8A7E-5F572DDBB00E}.Release|x86.ActiveCfg = Release|Any CPU + {7ACE56E6-7A6C-459C-8A7E-5F572DDBB00E}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -366,5 +382,6 @@ Global {BB961FD8-61C2-4443-AB68-B8088CBC3D43} = {16DBEA54-D51A-4D91-84DF-C701B6B4786F} {58766A5F-BA91-41F1-8A09-44E96685E361} = {259D1A9B-2927-4571-A366-68C3BB30C2B2} {0057DA5E-3183-4A7A-B092-167137872FF8} = {259D1A9B-2927-4571-A366-68C3BB30C2B2} + {7ACE56E6-7A6C-459C-8A7E-5F572DDBB00E} = {98939AC0-3ADA-4DB2-8B21-FFB6AF29A2D2} EndGlobalSection EndGlobal diff --git a/extra/admin-api/Utilities/Spacebar.Cdn.Fsck/Program.cs b/extra/admin-api/Utilities/Spacebar.Cdn.Fsck/Program.cs index 2f9e2c576..df7653dbc 100644 --- a/extra/admin-api/Utilities/Spacebar.Cdn.Fsck/Program.cs +++ b/extra/admin-api/Utilities/Spacebar.Cdn.Fsck/Program.cs @@ -5,9 +5,12 @@ using Spacebar.Interop.Cdn.Abstractions; using Spacebar.Models.Db.Contexts; var builder = Host.CreateApplicationBuilder(args); +if (!string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("APPSETTINGS_PATH"))) + builder.Configuration.AddJsonFile(Environment.GetEnvironmentVariable("APPSETTINGS_PATH")!); + // builder.Services.AddSingleton(new ProxyFileSource("http://cdn.old.server.spacebar.chat")); builder.Services.AddSingleton(new FilesystemFileSource("/mnt/data/dedicated/spacebar-storage")); -builder.Services.AddSingleton(new LruFileCache(1*1024*1024*1024)); +builder.Services.AddSingleton(new LruFileCache(1 * 1024 * 1024 * 1024)); builder.Services.AddHostedService(); builder.Services.AddDbContextPool(options => { diff --git a/extra/admin-api/Utilities/Spacebar.Cdn.Migration/Program.cs b/extra/admin-api/Utilities/Spacebar.Cdn.Migration/Program.cs index 280ffe94b..ca684e0c2 100644 --- a/extra/admin-api/Utilities/Spacebar.Cdn.Migration/Program.cs +++ b/extra/admin-api/Utilities/Spacebar.Cdn.Migration/Program.cs @@ -7,6 +7,9 @@ using Spacebar.Interop.Cdn.Signing; using Spacebar.Models.Db.Contexts; var builder = Host.CreateApplicationBuilder(args); +if (!string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("APPSETTINGS_PATH"))) + builder.Configuration.AddJsonFile(Environment.GetEnvironmentVariable("APPSETTINGS_PATH")!); + // builder.Services.AddSingleton(new ProxyFileSource("http://cdn.old.server.spacebar.chat")); IFileSource fromSrc, toSrc; diff --git a/extra/admin-api/Utilities/Spacebar.CleanSettingsRows/Program.cs b/extra/admin-api/Utilities/Spacebar.CleanSettingsRows/Program.cs index 0f573f55e..8aa73707a 100644 --- a/extra/admin-api/Utilities/Spacebar.CleanSettingsRows/Program.cs +++ b/extra/admin-api/Utilities/Spacebar.CleanSettingsRows/Program.cs @@ -3,6 +3,9 @@ using Spacebar.CleanSettingsRows; using Spacebar.Models.Db.Contexts; var builder = Host.CreateApplicationBuilder(args); +if (!string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("APPSETTINGS_PATH"))) + builder.Configuration.AddJsonFile(Environment.GetEnvironmentVariable("APPSETTINGS_PATH")!); + builder.Services.AddHostedService(); builder.Services.AddDbContextPool(options => { @@ -11,6 +14,5 @@ builder.Services.AddDbContextPool(options => { .EnableDetailedErrors(); }); - var host = builder.Build(); host.Run(); \ No newline at end of file diff --git a/extra/admin-api/outputs.nix b/extra/admin-api/outputs.nix index 63f082499..434301009 100644 --- a/extra/admin-api/outputs.nix +++ b/extra/admin-api/outputs.nix @@ -66,6 +66,18 @@ flake-utils.lib.eachSystem flake-utils.lib.allSystems ( proj = self.packages.${system}; in { + # Data mappings + Spacebar-DataMappings-Generic = makeNupkg { + name = "Spacebar.DataMappings.Generic"; + projectFile = "Spacebar.DataMappings.Generic.csproj"; + nugetDeps = DataMappings/Spacebar.DataMappings.Generic/deps.json; + srcRoot = DataMappings/Spacebar.DataMappings.Generic; + projectReferences = [ + proj.Spacebar-Models-Db + proj.Spacebar-Models-Generic + ]; + }; + # Interop Spacebar-Interop-Authentication = makeNupkg { name = "Spacebar.Interop.Authentication"; @@ -197,6 +209,7 @@ flake-utils.lib.eachSystem flake-utils.lib.allSystems ( srcRoot = ./Spacebar.GatewayOffload; packNupkg = false; projectReferences = [ + proj.Spacebar-DataMappings-Generic proj.Spacebar-Interop-Authentication proj.Spacebar-Interop-Authentication-AspNetCore proj.Spacebar-Interop-Replication-Abstractions @@ -223,7 +236,25 @@ flake-utils.lib.eachSystem flake-utils.lib.allSystems ( contents = [ self.packages.${system}.Spacebar-AdminApi ]; config = { Cmd = [ "${self.outputs.packages.${system}.Spacebar-AdminApi}/bin/Spacebar.AdminApi" ]; - Expose = [ "3001" ]; + Expose = [ "5000" ]; + }; + }; + containers.docker.gateway-offload = pkgs.dockerTools.buildLayeredImage { + name = "spacebar-server-ts-gateway-offload"; + tag = builtins.replaceStrings [ "+" ] [ "_" ] self.packages.${system}.Spacebar-AdminApi.version; + contents = [ self.packages.${system}.Spacebar-AdminApi ]; + config = { + Cmd = [ "${lib.getExe self.outputs.packages.${system}.Spacebar-GatewayOffload}" ]; + Expose = [ "5000" ]; + }; + }; + containers.docker.cdn-cs = pkgs.dockerTools.buildLayeredImage { + name = "spacebar-server-ts-cdn-cs"; + tag = builtins.replaceStrings [ "+" ] [ "_" ] self.packages.${system}.Spacebar-AdminApi.version; + contents = [ self.packages.${system}.Spacebar-AdminApi ]; + config = { + Cmd = [ "${lib.getExe self.outputs.packages.${system}.Spacebar-AdminApi}" ]; + Expose = [ "5000" ]; }; }; } @@ -238,6 +269,8 @@ flake-utils.lib.eachSystem flake-utils.lib.allSystems ( x86_64-linux = { # spacebar-server-tests = self.packages.x86_64-linux.default.passthru.tests; docker-admin-api = self.containers.x86_64-linux.docker.admin-api; + docker-gateway-offload = self.containers.x86_64-linux.docker.gateway-offload; + docker-cdn-cs = self.containers.x86_64-linux.docker.cdn-cs; }; }; } diff --git a/flake.lock b/flake.lock index 6bae7c7ca..c75943333 100644 Binary files a/flake.lock and b/flake.lock differ diff --git a/nix/modules/default/cs/default-appsettings-json.nix b/nix/modules/default/cs/default-appsettings-json.nix new file mode 100644 index 000000000..e6bb6323f --- /dev/null +++ b/nix/modules/default/cs/default-appsettings-json.nix @@ -0,0 +1,16 @@ +{ + Logging.LogLevel = { + Default = "Information"; + "Microsoft.AspNetCore" = "Trace"; + "Microsoft.AspNetCore.Mvc" = "Warning"; + "Microsoft.AspNetCore.HostFiltering" = "Warning"; + "Microsoft.AspNetCore.Cors" = "Warning"; + "Microsoft.EntityFrameworkCore.Database.Command" = "Information"; + }; + Spacebar = { + Authentication = { + PrivateKeyPath = "./jwt.key"; + PublicKeyPath = "./jwt.key.pub"; + }; + }; +} diff --git a/nix/modules/default/cs/gateway-offload-cs.nix b/nix/modules/default/cs/gateway-offload-cs.nix new file mode 100644 index 000000000..54e84bc37 --- /dev/null +++ b/nix/modules/default/cs/gateway-offload-cs.nix @@ -0,0 +1,173 @@ +self: +{ + config, + lib, + pkgs, + spacebar, + ... +}: + +let + secrets = import ../secrets.nix { inherit lib config; }; + cfg = config.services.spacebarchat-server; + jsonFormat = pkgs.formats.json { }; +in +{ + imports = [ ]; + options.services.spacebarchat-server.gatewayOffload = lib.mkOption { + default = { }; + description = "Configuration for C# gateway offload daemon."; + type = lib.types.submodule { + options = { + enable = lib.mkEnableOption "Enable gateway offload daemon (C#)."; + listenPort = lib.mkOption { + type = lib.types.port; + default = 3011; + description = "Port for the gateway offload daemon to listen on."; + }; + extraConfiguration = lib.mkOption { + type = jsonFormat.type; + default = import ./default-appsettings-json.nix; + description = "Extra appsettings.json configuration for the gateway offload daemon."; + }; + enableIdentify = lib.mkEnableOption "Enable offloading gateway opcode 2 (IDENTIFY)."; + enableGuildSync = lib.mkEnableOption "Enable offloading gateway opcode 12 (GUILD_SYNC)."; + enableLazyRequest = lib.mkEnableOption "Enable offloading gateway opcode 12 (LAZY_REQUEST)."; + }; + }; + }; + + config = lib.mkIf cfg.gatewayOffload.enable ( + let + makeServerTsService = ( + conf: + lib.recursiveUpdate + (lib.recursiveUpdate { + documentation = [ "https://docs.spacebar.chat/" ]; + wantedBy = [ "multi-user.target" ]; + wants = [ "network-online.target" ]; + after = [ "network-online.target" ]; + environment = secrets.systemdEnvironment; + serviceConfig = { + LoadCredential = secrets.systemdLoadCredentials; + + User = "spacebarchat"; + Group = "spacebarchat"; + DynamicUser = false; + + LockPersonality = true; + NoNewPrivileges = true; + + ProtectClock = true; + ProtectControlGroups = true; + ProtectHostname = true; + ProtectKernelLogs = true; + ProtectKernelModules = true; + ProtectKernelTunables = true; + PrivateDevices = true; + PrivateMounts = true; + PrivateUsers = true; + RestrictAddressFamilies = [ + "AF_INET" + "AF_INET6" + "AF_UNIX" + ]; + RestrictNamespaces = true; + RestrictRealtime = true; + SystemCallArchitectures = "native"; + SystemCallFilter = [ + "@system-service" + "~@privileged" + "@chown" # Required for copying files with FICLONE, apparently. + ]; + CapabilityBoundingSet = [ + "~CAP_SYS_ADMIN" + "~CAP_AUDIT_*" + "~CAP_NET_(BIND_SERVICE|BROADCAST|RAW)" + "~CAP_NET_ADMIN" # No use for this as we don't currently use iptables for enforcing instance bans + "~CAP_SYS_TIME" + "~CAP_KILL" + "~CAP_(DAC_*|FOWNER|IPC_OWNER)" + "~CAP_LINUX_IMMUTABLE" + "~CAP_IPC_LOCK" + "~CAP_BPF" + "~CAP_SYS_TTY_CONFIG" + "~CAP_SYS_BOOT" + "~CAP_SYS_CHROOT" + "~CAP_BLOCK_SUSPEND" + "~CAP_LEASE" + "~CAP_(CHOWN|FSETID|FSETFCAP)" # Check if we need CAP_CHOWN for `fchown()` (FICLONE)? + "~CAP_SET(UID|GID|PCAP)" + "~CAP_MAC_*" + "~CAP_SYS_PTRACE" + "~CAP_SYS_(NICE|RESOURCE)" + "~CAP_SYS_RAWIO" + "~CAP_SYSLOG" + ]; + RestrictSUIDSGID = true; + + WorkingDirectory = "/var/lib/spacebar"; + StateDirectory = "spacebar"; + StateDirectoryMode = "0750"; + RuntimeDirectory = "spacebar"; + RuntimeDirectoryMode = "0750"; + ReadWritePaths = [ cfg.cdnPath ]; + NoExecPaths = [ cfg.cdnPath ]; + + Restart = "on-failure"; + RestartSec = 10; + StartLimitBurst = 5; + UMask = "077"; + } + // lib.optionalAttrs (cfg.databaseFile != null) { EnvironmentFile = cfg.databaseFile; }; + } conf) + { + } + ); + in + { + assertions = [ + { + assertion = + cfg.gatewayOffload.extraConfiguration ? ConnectionStrings + && cfg.gatewayOffload.extraConfiguration.ConnectionStrings ? Spacebar + && cfg.gatewayOffload.extraConfiguration.ConnectionStrings.Spacebar != null; + message = '' + Setting a database connection string in extraConfiguration (`extraConfiguration.ConnectionStrings.Spacebar`) is required when using C# services. + Example: Host=127.0.0.1; Username=Spacebar; Password=SuperSecurePassword12; Database=spacebar; Port=5432; Include Error Detail=true; Maximum Pool Size=1000; Command Timeout=6000; Timeout=600; + ''; + } + ]; + + services.spacebarchat-server.settings.offload = { + gateway = { + op2BaseUrl = lib.mkIf cfg.gatewayOffload.enableIdentify "http://127.0.0.1:${builtins.toString cfg.gatewayOffload.listenPort}"; + op12BaseUrl = lib.mkIf cfg.gatewayOffload.enableGuildSync "http://127.0.0.1:${builtins.toString cfg.gatewayOffload.listenPort}"; + op14BaseUrl = lib.mkIf cfg.gatewayOffload.enableLazyRequest "http://127.0.0.1:${builtins.toString cfg.gatewayOffload.listenPort}"; + }; + }; + + systemd.services.spacebar-cs-gateway-offload = makeServerTsService { + description = "Spacebar Server - C# Gateway offload"; + environment = builtins.mapAttrs (_: val: builtins.toString val) ( + { + # things we set by default... + EVENT_TRANSMISSION = "unix"; + EVENT_SOCKET_PATH = "/run/spacebar/"; + } + // cfg.extraEnvironment + // { + # things we force... + # CONFIG_PATH = configFile; + CONFIG_READONLY = 1; + ASPNETCORE_URLS = "http://127.0.0.1:${toString cfg.gatewayOffload.listenPort}"; + STORAGE_LOCATION = cfg.cdnPath; + } + ); + serviceConfig = { + ExecStart = "${self.packages.${pkgs.stdenv.hostPlatform.system}.Spacebar-GatewayOffload}/bin/Spacebar.GatewayOffload"; + }; + }; + } + ); +} diff --git a/nix/modules/default/default.nix b/nix/modules/default/default.nix index f20edcb2d..c4f9e4cd8 100644 --- a/nix/modules/default/default.nix +++ b/nix/modules/default/default.nix @@ -8,6 +8,7 @@ self: }: let + secrets = import ./secrets.nix { inherit lib config; }; cfg = config.services.spacebarchat-server; jsonFormat = pkgs.formats.json { }; configFile = @@ -43,8 +44,8 @@ in { imports = [ ./integration-nginx.nix - ./secrets.nix ./users.nix + (import ./cs/gateway-offload-cs.nix self) ]; options.services.spacebarchat-server = let @@ -111,7 +112,8 @@ in See https://docs.spacebar.chat/setup/server/configuration for supported values. ''; }; - }; + } + // secrets.options; config = lib.mkIf cfg.enable ( let @@ -123,35 +125,9 @@ in wantedBy = [ "multi-user.target" ]; wants = [ "network-online.target" ]; after = [ "network-online.target" ]; - environment = - { } - // (if cfg.cdnSignaturePath != null then { CDN_SIGNATURE_PATH = "%d/cdnSignature"; } else { }) - // (if cfg.legacyJwtSecretPath != null then { LEGACY_JWT_SECRET_PATH = "%d/legacyJwtSecret"; } else { }) - // (if cfg.mailjetApiKeyPath != null then { MAILJET_API_KEY_PATH = "%d/mailjetApiKey"; } else { }) - // (if cfg.mailjetApiSecretPath != null then { MAILJET_API_SECRET_PATH = "%d/mailjetApiSecret"; } else { }) - // (if cfg.smtpPasswordPath != null then { SMTP_PASSWORD_PATH = "%d/smtpPassword"; } else { }) - // (if cfg.gifApiKeyPath != null then { GIF_API_KEY_PATH = "%d/gifApiKey"; } else { }) - // (if cfg.rabbitmqHostPath != null then { RABBITMQ_HOST_PATH = "%d/rabbitmqHost"; } else { }) - // (if cfg.abuseIpDbApiKeyPath != null then { ABUSE_IP_DB_API_KEY_PATH = "%d/abuseIpDbApiKey"; } else { }) - // (if cfg.captchaSecretKeyPath != null then { CAPTCHA_SECRET_KEY_PATH = "%d/captchaSecretKey"; } else { }) - // (if cfg.captchaSiteKeyPath != null then { CAPTCHA_SITE_KEY_PATH = "%d/captchaSiteKey"; } else { }) - // (if cfg.ipdataApiKeyPath != null then { IPDATA_API_KEY_PATH = "%d/ipdataApiKey"; } else { }) - // (if cfg.requestSignaturePath != null then { REQUEST_SIGNATURE_PATH = "%d/requestSignature"; } else { }); + environment = secrets.systemdEnvironment; serviceConfig = { - LoadCredential = - [ ] - ++ (if cfg.cdnSignaturePath != null then [ "cdnSignature:${cfg.cdnSignaturePath}" ] else [ ]) - ++ (if cfg.legacyJwtSecretPath != null then [ "legacyJwtSecret:${cfg.legacyJwtSecretPath}" ] else [ ]) - ++ (if cfg.mailjetApiKeyPath != null then [ "mailjetApiKey:${cfg.mailjetApiKeyPath}" ] else [ ]) - ++ (if cfg.mailjetApiSecretPath != null then [ "mailjetApiSecret:${cfg.mailjetApiSecretPath}" ] else [ ]) - ++ (if cfg.smtpPasswordPath != null then [ "smtpPassword:${cfg.smtpPasswordPath}" ] else [ ]) - ++ (if cfg.gifApiKeyPath != null then [ "gifApiKey:${cfg.gifApiKeyPath}" ] else [ ]) - ++ (if cfg.rabbitmqHostPath != null then [ "rabbitmqHost:${cfg.rabbitmqHostPath}" ] else [ ]) - ++ (if cfg.abuseIpDbApiKeyPath != null then [ "abuseIpDbApiKey:${cfg.abuseIpDbApiKeyPath}" ] else [ ]) - ++ (if cfg.captchaSecretKeyPath != null then [ "captchaSecretKey:${cfg.captchaSecretKeyPath}" ] else [ ]) - ++ (if cfg.captchaSiteKeyPath != null then [ "captchaSiteKey:${cfg.captchaSiteKeyPath}" ] else [ ]) - ++ (if cfg.ipdataApiKeyPath != null then [ "ipdataApiKey:${cfg.ipdataApiKeyPath}" ] else [ ]) - ++ (if cfg.requestSignaturePath != null then [ "requestSignature:${cfg.requestSignaturePath}" ] else [ ]); + LoadCredential = secrets.systemdLoadCredentials; User = "spacebarchat"; Group = "spacebarchat"; @@ -304,4 +280,4 @@ in }); } ); -} \ No newline at end of file +} diff --git a/nix/modules/default/secrets.nix b/nix/modules/default/secrets.nix index 7b6a6ccfd..f28720964 100644 --- a/nix/modules/default/secrets.nix +++ b/nix/modules/default/secrets.nix @@ -1,6 +1,9 @@ -{ lib, ... }: +{ lib, config, ... }: +let + cfg = config.services.spacebarchat-server; +in { - options.services.spacebarchat-server = { + options = { cdnSignaturePath = lib.mkOption { type = lib.types.nullOr lib.types.str; default = null; @@ -67,4 +70,34 @@ description = "Path to the secret"; }; }; + + systemdLoadCredentials = + [ ] + ++ (if cfg.cdnSignaturePath != null then [ "cdnSignature:${cfg.cdnSignaturePath}" ] else [ ]) + ++ (if cfg.legacyJwtSecretPath != null then [ "legacyJwtSecret:${cfg.legacyJwtSecretPath}" ] else [ ]) + ++ (if cfg.mailjetApiKeyPath != null then [ "mailjetApiKey:${cfg.mailjetApiKeyPath}" ] else [ ]) + ++ (if cfg.mailjetApiSecretPath != null then [ "mailjetApiSecret:${cfg.mailjetApiSecretPath}" ] else [ ]) + ++ (if cfg.smtpPasswordPath != null then [ "smtpPassword:${cfg.smtpPasswordPath}" ] else [ ]) + ++ (if cfg.gifApiKeyPath != null then [ "gifApiKey:${cfg.gifApiKeyPath}" ] else [ ]) + ++ (if cfg.rabbitmqHostPath != null then [ "rabbitmqHost:${cfg.rabbitmqHostPath}" ] else [ ]) + ++ (if cfg.abuseIpDbApiKeyPath != null then [ "abuseIpDbApiKey:${cfg.abuseIpDbApiKeyPath}" ] else [ ]) + ++ (if cfg.captchaSecretKeyPath != null then [ "captchaSecretKey:${cfg.captchaSecretKeyPath}" ] else [ ]) + ++ (if cfg.captchaSiteKeyPath != null then [ "captchaSiteKey:${cfg.captchaSiteKeyPath}" ] else [ ]) + ++ (if cfg.ipdataApiKeyPath != null then [ "ipdataApiKey:${cfg.ipdataApiKeyPath}" ] else [ ]) + ++ (if cfg.requestSignaturePath != null then [ "requestSignature:${cfg.requestSignaturePath}" ] else [ ]); + + systemdEnvironment = + { } + // (if cfg.cdnSignaturePath != null then { CDN_SIGNATURE_PATH = "%d/cdnSignature"; } else { }) + // (if cfg.legacyJwtSecretPath != null then { LEGACY_JWT_SECRET_PATH = "%d/legacyJwtSecret"; } else { }) + // (if cfg.mailjetApiKeyPath != null then { MAILJET_API_KEY_PATH = "%d/mailjetApiKey"; } else { }) + // (if cfg.mailjetApiSecretPath != null then { MAILJET_API_SECRET_PATH = "%d/mailjetApiSecret"; } else { }) + // (if cfg.smtpPasswordPath != null then { SMTP_PASSWORD_PATH = "%d/smtpPassword"; } else { }) + // (if cfg.gifApiKeyPath != null then { GIF_API_KEY_PATH = "%d/gifApiKey"; } else { }) + // (if cfg.rabbitmqHostPath != null then { RABBITMQ_HOST_PATH = "%d/rabbitmqHost"; } else { }) + // (if cfg.abuseIpDbApiKeyPath != null then { ABUSE_IP_DB_API_KEY_PATH = "%d/abuseIpDbApiKey"; } else { }) + // (if cfg.captchaSecretKeyPath != null then { CAPTCHA_SECRET_KEY_PATH = "%d/captchaSecretKey"; } else { }) + // (if cfg.captchaSiteKeyPath != null then { CAPTCHA_SITE_KEY_PATH = "%d/captchaSiteKey"; } else { }) + // (if cfg.ipdataApiKeyPath != null then { IPDATA_API_KEY_PATH = "%d/ipdataApiKey"; } else { }) + // (if cfg.requestSignaturePath != null then { REQUEST_SIGNATURE_PATH = "%d/requestSignature"; } else { }); } diff --git a/nix/testVm/configuration.nix b/nix/testVm/configuration.nix index c826ca0a5..a39531c2b 100644 --- a/nix/testVm/configuration.nix +++ b/nix/testVm/configuration.nix @@ -6,11 +6,31 @@ let cfg = { enable = true; - apiEndpoint = { useSsl = false; host = "api.sb.localhost"; localPort = 3001; publicPort = 8080; }; - gatewayEndpoint = { useSsl = false; host = "gw.sb.localhost"; localPort = 3002; publicPort = 8080; }; - cdnEndpoint = { useSsl = false; host = "cdn.sb.localhost"; localPort = 3003; publicPort = 8080; }; + apiEndpoint = { + useSsl = false; + host = "api.sb.localhost"; + localPort = 3001; + publicPort = 8080; + }; + gatewayEndpoint = { + useSsl = false; + host = "gw.sb.localhost"; + localPort = 3002; + publicPort = 8080; + }; + cdnEndpoint = { + useSsl = false; + host = "cdn.sb.localhost"; + localPort = 3003; + publicPort = 8080; + }; nginx.enable = true; serverName = "sb.localhost"; + gatewayOffload = { + enable = true; + enableGuildSync = true; + extraConfiguration.ConnectionStrings.Spacebar = "Host=127.0.0.1; Username=Spacebar; Password=postgres; Database=spacebar; Port=5432; Include Error Detail=true; Maximum Pool Size=1000; Command Timeout=6000; Timeout=600;"; + }; extraEnvironment = { DATABASE = "postgres://postgres:postgres@127.0.0.1/spacebar"; #WEBRTC_PORT_RANGE=60000-61000; @@ -41,5 +61,7 @@ btop duf lnav + net-tools + nethogs ]; } diff --git a/nix/tests/test-bundle-starts.nix b/nix/tests/test-bundle-starts.nix index a90fe37ba..3beedaae2 100644 --- a/nix/tests/test-bundle-starts.nix +++ b/nix/tests/test-bundle-starts.nix @@ -22,13 +22,19 @@ in apiEndpoint = sb.mkEndpoint "api.sb.localhost" 3001 false; gatewayEndpoint = sb.mkEndpoint "gw.sb.localhost" 3002 false; cdnEndpoint = sb.mkEndpoint "cdn.sb.localhost" 3003 false; - nginx.enable = true; serverName = "sb.localhost"; extraEnvironment = { DATABASE = "postgres://postgres:postgres@127.0.0.1/spacebar"; LOG_REQUESTS = "-"; # Log all requests LOG_VALIDATION_ERRORS = true; }; + + nginx.enable = true; + gatewayOffload = { + enable = true; + enableGuildSync = true; + extraConfiguration.ConnectionStrings.Spacebar = "Host=127.0.0.1; Username=Spacebar; Password=postgres; Database=spacebar; Port=5432; Include Error Detail=true; Maximum Pool Size=1000; Command Timeout=6000; Timeout=600;"; + }; }; in lib.trace ("Testing with config: " + builtins.toJSON cfg) cfg;