diff --git a/extra/admin-api/Interop/Spacebar.Interop.Replication.Abstractions/ReplicationMessage.cs b/extra/admin-api/Interop/Spacebar.Interop.Replication.Abstractions/ReplicationMessage.cs index 5cffa012b..8696b29e2 100644 --- a/extra/admin-api/Interop/Spacebar.Interop.Replication.Abstractions/ReplicationMessage.cs +++ b/extra/admin-api/Interop/Spacebar.Interop.Replication.Abstractions/ReplicationMessage.cs @@ -1,7 +1,12 @@ +using System.Reflection; +using System.Text.Json; +using System.Text.Json.Nodes; using System.Text.Json.Serialization; +using System.Transactions; namespace Spacebar.Interop.Replication.Abstractions; +[JsonConverter(typeof(ReplicationMessageJsonConverter))] public class ContentlessReplicationMessage { [JsonPropertyName("channel_id"), JsonNumberHandling(JsonNumberHandling.AllowReadingFromString | JsonNumberHandling.WriteAsString)] public long? ChannelId { get; set; } @@ -12,8 +17,8 @@ public class ContentlessReplicationMessage { [JsonPropertyName("user_id"), JsonNumberHandling(JsonNumberHandling.AllowReadingFromString | JsonNumberHandling.WriteAsString)] public long? UserId { get; set; } - [JsonPropertyName("session_id"), JsonNumberHandling(JsonNumberHandling.AllowReadingFromString | JsonNumberHandling.WriteAsString)] - public long? SessionId { get; set; } + [JsonPropertyName("session_id")] + public string? SessionId { get; set; } [JsonPropertyName("created_at")] public DateTime? CreatedAt { get; set; } @@ -31,4 +36,37 @@ public class ContentlessReplicationMessage { public class ReplicationMessage : ContentlessReplicationMessage { [JsonPropertyName("data")] public TPayload Payload { get; set; } = default!; +} + +public class ReplicationMessageJsonConverter : JsonConverter { + public override ContentlessReplicationMessage? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) { + // return JsonSerializer.Deserialize(ref reader, typeToConvert, options) as ContentlessReplicationMessage; + Console.WriteLine("TODO: write ReplicationMessage deserialization"); //TODO + return null; + } + + public override void Write(Utf8JsonWriter writer, ContentlessReplicationMessage value, JsonSerializerOptions options) { + writer.WriteStartObject(); + writer.WriteString("event", value.Event); + if (value.ChannelId.HasValue) writer.WriteString("channel_id", value.ChannelId.ToString()); + if (value.GuildId.HasValue) writer.WriteString("guild_id", value.GuildId.ToString()); + if (value.UserId.HasValue) writer.WriteString("user_id", value.UserId.ToString()); + if (value.SessionId != null) writer.WriteString("session_id", value.SessionId); + + if (value.CreatedAt.HasValue) writer.WriteString("created_at", value.CreatedAt.Value.ToString("O")); + if (value.Origin != null) writer.WriteString("origin", value.Origin); + if (value.ReconnectDelay.HasValue) writer.WriteNumber("reconnect_delay", value.ReconnectDelay.Value); + + if (value.GetType().IsGenericType && value.GetType().GetGenericTypeDefinition() == typeof(ReplicationMessage<>)) { + var dataType = value.GetType().GenericTypeArguments[0]; + + var dataProperty = value.GetType().GetProperty("Payload", BindingFlags.Public | BindingFlags.Instance); + var dataValue = dataProperty!.GetValue(value); + + writer.WritePropertyName("data"); + JsonSerializer.Serialize(writer, dataValue, dataType, options); + } + + writer.WriteEndObject(); + } } \ No newline at end of file diff --git a/extra/admin-api/Models/Spacebar.Models.Gateway/GuildMemberListUpdate.cs b/extra/admin-api/Models/Spacebar.Models.Gateway/GuildMemberListUpdate.cs index 3d81c3609..45d28467f 100644 --- a/extra/admin-api/Models/Spacebar.Models.Gateway/GuildMemberListUpdate.cs +++ b/extra/admin-api/Models/Spacebar.Models.Gateway/GuildMemberListUpdate.cs @@ -1,12 +1,16 @@ +using System.Text.Json; +using System.Text.Json.Nodes; using System.Text.Json.Serialization; using Spacebar.Models.Generic; public class GuildMemberListUpdate { + public const string EventId = "GUILD_MEMBER_LIST_UPDATE"; + [JsonPropertyName("id")] public string ListId { get; set; } = null!; - [JsonPropertyName("guild_id")] - public string GuildId { get; set; } = null!; + [JsonPropertyName("guild_id"), JsonNumberHandling(JsonNumberHandling.AllowReadingFromString | JsonNumberHandling.WriteAsString)] + public long GuildId { get; set; } [JsonPropertyName("online_count")] public int OnlineCount { get; set; } @@ -15,10 +19,10 @@ public class GuildMemberListUpdate { public int MemberCount { get; set; } [JsonPropertyName("ops")] - public List Operations { get; set; } = null!; + public List Operations { get; set; } = null!; [JsonPropertyName("groups")] - public List Groups { get; set; } = null!; + public List Groups { get; set; } = null!; } // i cba to write a dictionary converter for this... @@ -48,26 +52,61 @@ public enum GuildMemberListUpdateOperationType { #region Operations -public class BaseGuildMemberListUpdateOperation { +[JsonConverter(typeof(GuildMemberListUpdateOperationJsonConverter))] +public class GuildMemberListUpdateOperation { [JsonPropertyName("op")] public GuildMemberListUpdateOperationType Operation { get; set; } + + public class SyncOperation : GuildMemberListUpdateOperation { + [JsonPropertyName("range")] + public int[] Range { get; set; } = null!; + + [JsonPropertyName("items")] + public List Items { get; set; } = null!; + } } -public class GuildMemberListSyncOperation : BaseGuildMemberListUpdateOperation { - [JsonPropertyName("range")] - public int[] Range { get; set; } = null!; - - [JsonPropertyName("items")] - public List Items { get; set; } = null!; -} - +[JsonConverter(typeof(GuildMemberListSyncItemJsonConverter))] public class GuildMemberListSyncItem { - public class GuildMemberListMemberSyncItem : GuildMemberListSyncItem { + public class RoleEntry : GuildMemberListSyncItem { + [JsonPropertyName("id")] + public string Id { get; set; } + + [JsonPropertyName("count")] + public long Count { get; set; } + } + + public class MemberEntry : GuildMemberListSyncItem { [JsonPropertyName("member")] public Member Member { get; set; } = null!; } } +public class GuildMemberListSyncItemJsonConverter : JsonConverter { + public override GuildMemberListSyncItem? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) { + var n = JsonSerializer.Deserialize(ref reader, options); + if (n.ContainsKey("id") && n.ContainsKey("count")) return n.Deserialize(); + if (n.ContainsKey("member")) return n.Deserialize(); + throw new InvalidOperationException("Could not determine sync item type for keys " + string.Join(", ", n.Select(x => x.Key))); + } + + public override void Write(Utf8JsonWriter writer, GuildMemberListSyncItem value, JsonSerializerOptions options) => + JsonSerializer.Serialize(writer, value, value.GetType(), options); +} + +public class GuildMemberListUpdateOperationJsonConverter : JsonConverter { + public override GuildMemberListUpdateOperation? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) { + var n = JsonSerializer.Deserialize(ref reader, options); + return n!["op"]!.GetValue() switch { + "sync" => n.Deserialize(), + _ => throw new InvalidCastException("Unknown operation: " + n["op"]!.GetValue()) + }; + } + + public override void Write(Utf8JsonWriter writer, GuildMemberListUpdateOperation value, JsonSerializerOptions options) => + JsonSerializer.Serialize(writer, value, value.GetType(), options); +} + #endregion // TODO: diff algo diff --git a/extra/admin-api/Spacebar.Offload/Controllers/Op14Controller.cs b/extra/admin-api/Spacebar.Offload/Controllers/Op14Controller.cs index 95998151e..f34455542 100644 --- a/extra/admin-api/Spacebar.Offload/Controllers/Op14Controller.cs +++ b/extra/admin-api/Spacebar.Offload/Controllers/Op14Controller.cs @@ -34,12 +34,56 @@ public class Op14Controller(ILogger logger, SpacebarAspNetAuthen yield break; } + var memberList = await GetGuildMemberListAsync(db, payload.GuildId); + + yield return new ReplicationMessage() { + UserId = user.Result.Id, + Event = GuildMemberListUpdate.EventId, + Origin = "Offload/LazyRequest", + CreatedAt = DateTime.UtcNow, + Payload = new GuildMemberListUpdate() { + GuildId = payload.GuildId, + ListId = payload.GuildId.ToString(), + OnlineCount = memberList.TakeWhile(x => x is not RoleEntry { Id: "offline" }).Count(), + MemberCount = await db.Members.CountAsync(x => x.GuildId == payload.GuildId), + Operations = [ + new GuildMemberListUpdateOperation.SyncOperation() { + Items = memberList.Select(item => item is RoleEntry re + ? new GuildMemberListSyncItem.RoleEntry() { Id = re.Id, Count = re.Count } + : item is MemberEntry me + ? new GuildMemberListSyncItem.MemberEntry() { Member = me.Member } + : throw new InvalidCastException("List item was neither RoleEntry nor MemberEntry???")) + .ToList(), + Range = [0, memberList.Count] + } + ], + Groups = memberList.OfType().Select(re => new GuildMemberListSyncItem.RoleEntry() { Id = re.Id, Count = re.Count }).ToList() + } + // TODO: send presence updates + // TODO: handle subscriptions + // TODO: handle channel permissions + // TODO: handle channels at all + }; + } + + private async Task GetMemberListIdAsync(SpacebarDbContext db, long guildId, long channelId) { + var channel = await db.Channels.AsNoTracking().FirstOrDefaultAsync(c => c.Id == channelId && c.GuildId == guildId); + if (channel == null) return null; + + if (string.IsNullOrWhiteSpace(channel.PermissionOverwrites) || channel.PermissionOverwrites == "[]") { + return "everyone"; + } + + return null; // TODO + } + + private async Task> GetGuildMemberListAsync(SpacebarDbContext db, long guildId) { var memberList = new List(); // Fetch hoisted roles for the guild to define groups var hoistedRoles = await db.Roles .AsNoTracking() - .Where(r => r.GuildId == payload.GuildId && r.Hoist) + .Where(r => r.GuildId == guildId && r.Hoist) .OrderByDescending(r => r.Position) // .Select(r => r.Id) .ToListAsync(); @@ -51,7 +95,7 @@ public class Op14Controller(ILogger logger, SpacebarAspNetAuthen var members = await db.Members.AsNoTracking() .Include(x => x.IdNavigation) .Where(x => - x.GuildId == payload.GuildId + x.GuildId == guildId && x.Roles.Any(r => r.Id == role) && !x.Roles.Any(r => handledRoles.Contains(r.Id)) // and finally, filter by online @@ -74,7 +118,7 @@ public class Op14Controller(ILogger logger, SpacebarAspNetAuthen .Include(x => x.IdNavigation) // .ThenInclude(x=>x.Sessions) .Where(x => - x.GuildId == payload.GuildId + x.GuildId == guildId && !x.Roles.Any(r => handledRoles.Contains(r.Id)) // and finally, filter by online && x.IdNavigation.Sessions.Any(s => s.Status != "offline" && s.Status != "invisible" && s.Status != "unknown") @@ -90,14 +134,13 @@ public class Op14Controller(ILogger logger, SpacebarAspNetAuthen memberList.AddRange(onlineMembers.Select(m => (IMemberListEntry)new MemberEntry() { Member = m.ToPublicMember() })); } - if (memberList.Count < 2000) { logger.LogInformation("Less than 2000 members, including offline members..."); var offlineMembers = await db.Members.AsNoTracking() .Include(x => x.IdNavigation) // .ThenInclude(x=>x.Sessions) .Where(x => - x.GuildId == payload.GuildId + x.GuildId == guildId && !x.Roles.Any(r => handledRoles.Contains(r.Id)) // and finally, filter by online && (x.IdNavigation.Sessions.Any(s => s.Status == "offline" || s.Status == "invisible" || s.Status == "unknown") || !x.IdNavigation.Sessions.Any()) @@ -115,24 +158,14 @@ public class Op14Controller(ILogger logger, SpacebarAspNetAuthen } logger.LogInformation("Got member list with {count} total nodes", memberList.Count); - } - - private async Task GetMemberListIdAsync(SpacebarDbContext db, long guildId, long channelId) { - var channel = await db.Channels.AsNoTracking().FirstOrDefaultAsync(c => c.Id == channelId && c.GuildId == guildId); - if (channel == null) return null; - - if (string.IsNullOrWhiteSpace(channel.PermissionOverwrites) || channel.PermissionOverwrites == "[]") { - return "everyone"; - } - - return null; // TODO + return memberList; } } internal interface IMemberListEntry { } internal struct RoleEntry : IMemberListEntry { - [JsonPropertyName("id"), JsonNumberHandling(JsonNumberHandling.AllowReadingFromString | JsonNumberHandling.WriteAsString)] + [JsonPropertyName("id")] public string Id { get; set; } [JsonPropertyName("count")] diff --git a/src/get-member-list.ts b/src/get-member-list.ts deleted file mode 100644 index 58bd898b5..000000000 --- a/src/get-member-list.ts +++ /dev/null @@ -1,40 +0,0 @@ -// /* -// Spacebar: A FOSS re-implementation and extension of the Discord.com backend. -// Copyright (C) 2026 Spacebar and Spacebar Contributors -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published -// by the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . -// */ -// -// process.on("uncaughtException", console.error); -// process.on("unhandledRejection", console.error); -// -// import moduleAlias from "module-alias"; -// moduleAlias(__dirname + "../../package.json"); -// import { config } from "dotenv"; -// config({ quiet: true }); -// -// // process.env.DB_LOGGING = "true"; -// -// import { closeDatabase, initDatabase } from "@spacebar/database"; -// import { buildFullMemberlistSequential } from "@spacebar/gateway/opcodes/LazyRequest"; -// -// async function main() { -// await initDatabase(); -// -// await buildFullMemberlistSequential("1006649183970562092"); -// -// await closeDatabase(); -// } -// -// main().then(() => console.log("meow"));