dandri/server
1
0
Fork 0
mirror of https://github.com/spacebarchat/server.git synced 2026-06-28 18:31:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

Get basic member list working in offload

Browse Source
This commit is contained in:
Rory&
2026-06-21 20:41:01 +02:00
parent 387cea129d
commit 3d07e05fd0
4 changed files with 143 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files
extra/admin-api/Interop/Spacebar.Interop.Replication.Abstractions/ReplicationMessage.cs
+40 -2
View File
@@ -1,7 +1,12 @@
using System.Reflection;
using System.Text.Json;
using System.Text.Json.Nodes;
using System.Text.Json.Serialization;
using System.Transactions;
namespace Spacebar.Interop.Replication.Abstractions;
[JsonConverter(typeof(ReplicationMessageJsonConverter))]
public class ContentlessReplicationMessage {
[JsonPropertyName("channel_id"), JsonNumberHandling(JsonNumberHandling.AllowReadingFromString | JsonNumberHandling.WriteAsString)]
public long? ChannelId { get; set; }
@@ -12,8 +17,8 @@ public class ContentlessReplicationMessage {
[JsonPropertyName("user_id"), JsonNumberHandling(JsonNumberHandling.AllowReadingFromString | JsonNumberHandling.WriteAsString)]
public long? UserId { get; set; }
[JsonPropertyName("session_id"), JsonNumberHandling(JsonNumberHandling.AllowReadingFromString | JsonNumberHandling.WriteAsString)]
public long? SessionId { get; set; }
[JsonPropertyName("session_id")]
public string? SessionId { get; set; }
[JsonPropertyName("created_at")]
public DateTime? CreatedAt { get; set; }
@@ -31,4 +36,37 @@ public class ContentlessReplicationMessage {
public class ReplicationMessage<TPayload> : ContentlessReplicationMessage {
[JsonPropertyName("data")]
public TPayload Payload { get; set; } = default!;
}
public class ReplicationMessageJsonConverter : JsonConverter<ContentlessReplicationMessage> {
public override ContentlessReplicationMessage? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) {
// return JsonSerializer.Deserialize(ref reader, typeToConvert, options) as ContentlessReplicationMessage;
Console.WriteLine("TODO: write ReplicationMessage deserialization"); //TODO
return null;
}
public override void Write(Utf8JsonWriter writer, ContentlessReplicationMessage value, JsonSerializerOptions options) {
writer.WriteStartObject();
writer.WriteString("event", value.Event);
if (value.ChannelId.HasValue) writer.WriteString("channel_id", value.ChannelId.ToString());
if (value.GuildId.HasValue) writer.WriteString("guild_id", value.GuildId.ToString());
if (value.UserId.HasValue) writer.WriteString("user_id", value.UserId.ToString());
if (value.SessionId != null) writer.WriteString("session_id", value.SessionId);
if (value.CreatedAt.HasValue) writer.WriteString("created_at", value.CreatedAt.Value.ToString("O"));
if (value.Origin != null) writer.WriteString("origin", value.Origin);
if (value.ReconnectDelay.HasValue) writer.WriteNumber("reconnect_delay", value.ReconnectDelay.Value);
if (value.GetType().IsGenericType && value.GetType().GetGenericTypeDefinition() == typeof(ReplicationMessage<>)) {
var dataType = value.GetType().GenericTypeArguments[0];
var dataProperty = value.GetType().GetProperty("Payload", BindingFlags.Public | BindingFlags.Instance);
var dataValue = dataProperty!.GetValue(value);
writer.WritePropertyName("data");
JsonSerializer.Serialize(writer, dataValue, dataType, options);
}
writer.WriteEndObject();
}
}
extra/admin-api/Models/Spacebar.Models.Gateway/GuildMemberListUpdate.cs
+53 -14
View File
@@ -1,12 +1,16 @@
using System.Text.Json;
using System.Text.Json.Nodes;
using System.Text.Json.Serialization;
using Spacebar.Models.Generic;
public class GuildMemberListUpdate {
public const string EventId = "GUILD_MEMBER_LIST_UPDATE";
[JsonPropertyName("id")]
public string ListId { get; set; } = null!;
[JsonPropertyName("guild_id")]
public string GuildId { get; set; } = null!;
[JsonPropertyName("guild_id"), JsonNumberHandling(JsonNumberHandling.AllowReadingFromString | JsonNumberHandling.WriteAsString)]
public long GuildId { get; set; }
[JsonPropertyName("online_count")]
public int OnlineCount { get; set; }
@@ -15,10 +19,10 @@ public class GuildMemberListUpdate {
public int MemberCount { get; set; }
[JsonPropertyName("ops")]
public List<BaseGuildMemberListUpdateOperation> Operations { get; set; } = null!;
public List<GuildMemberListUpdateOperation> Operations { get; set; } = null!;
[JsonPropertyName("groups")]
public List<GuildMemberListGroupCount> Groups { get; set; } = null!;
public List<GuildMemberListSyncItem.RoleEntry> Groups { get; set; } = null!;
}
// i cba to write a dictionary converter for this...
@@ -48,26 +52,61 @@ public enum GuildMemberListUpdateOperationType {
#region Operations
public class BaseGuildMemberListUpdateOperation {
[JsonConverter(typeof(GuildMemberListUpdateOperationJsonConverter))]
public class GuildMemberListUpdateOperation {
[JsonPropertyName("op")]
public GuildMemberListUpdateOperationType Operation { get; set; }
public class SyncOperation : GuildMemberListUpdateOperation {
[JsonPropertyName("range")]
public int[] Range { get; set; } = null!;
[JsonPropertyName("items")]
public List<GuildMemberListSyncItem> Items { get; set; } = null!;
}
}
public class GuildMemberListSyncOperation : BaseGuildMemberListUpdateOperation {
[JsonPropertyName("range")]
public int[] Range { get; set; } = null!;
[JsonPropertyName("items")]
public List<GuildMemberListSyncItem> Items { get; set; } = null!;
}
[JsonConverter(typeof(GuildMemberListSyncItemJsonConverter))]
public class GuildMemberListSyncItem {
public class GuildMemberListMemberSyncItem : GuildMemberListSyncItem {
public class RoleEntry : GuildMemberListSyncItem {
[JsonPropertyName("id")]
public string Id { get; set; }
[JsonPropertyName("count")]
public long Count { get; set; }
}
public class MemberEntry : GuildMemberListSyncItem {
[JsonPropertyName("member")]
public Member Member { get; set; } = null!;
}
}
public class GuildMemberListSyncItemJsonConverter : JsonConverter<GuildMemberListSyncItem> {
public override GuildMemberListSyncItem? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) {
var n = JsonSerializer.Deserialize<JsonObject>(ref reader, options);
if (n.ContainsKey("id") && n.ContainsKey("count")) return n.Deserialize<GuildMemberListSyncItem.RoleEntry>();
if (n.ContainsKey("member")) return n.Deserialize<GuildMemberListSyncItem>();
throw new InvalidOperationException("Could not determine sync item type for keys " + string.Join(", ", n.Select(x => x.Key)));
}
public override void Write(Utf8JsonWriter writer, GuildMemberListSyncItem value, JsonSerializerOptions options) =>
JsonSerializer.Serialize(writer, value, value.GetType(), options);
}
public class GuildMemberListUpdateOperationJsonConverter : JsonConverter<GuildMemberListUpdateOperation> {
public override GuildMemberListUpdateOperation? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) {
var n = JsonSerializer.Deserialize<JsonObject>(ref reader, options);
return n!["op"]!.GetValue<string>() switch {
"sync" => n.Deserialize<GuildMemberListUpdateOperation.SyncOperation>(),
_ => throw new InvalidCastException("Unknown operation: " + n["op"]!.GetValue<string>())
};
}
public override void Write(Utf8JsonWriter writer, GuildMemberListUpdateOperation value, JsonSerializerOptions options) =>
JsonSerializer.Serialize(writer, value, value.GetType(), options);
}
#endregion
// TODO: diff algo
extra/admin-api/Spacebar.Offload/Controllers/Op14Controller.cs
+50 -17
View File
@@ -34,12 +34,56 @@ public class Op14Controller(ILogger<Op12Controller> logger, SpacebarAspNetAuthen
yield break;
}
var memberList = await GetGuildMemberListAsync(db, payload.GuildId);
yield return new ReplicationMessage<GuildMemberListUpdate>() {
UserId = user.Result.Id,
Event = GuildMemberListUpdate.EventId,
Origin = "Offload/LazyRequest",
CreatedAt = DateTime.UtcNow,
Payload = new GuildMemberListUpdate() {
GuildId = payload.GuildId,
ListId = payload.GuildId.ToString(),
OnlineCount = memberList.TakeWhile(x => x is not RoleEntry { Id: "offline" }).Count(),
MemberCount = await db.Members.CountAsync(x => x.GuildId == payload.GuildId),
Operations = [
new GuildMemberListUpdateOperation.SyncOperation() {
Items = memberList.Select<IMemberListEntry, GuildMemberListSyncItem>(item => item is RoleEntry re
? new GuildMemberListSyncItem.RoleEntry() { Id = re.Id, Count = re.Count }
: item is MemberEntry me
? new GuildMemberListSyncItem.MemberEntry() { Member = me.Member }
: throw new InvalidCastException("List item was neither RoleEntry nor MemberEntry???"))
.ToList(),
Range = [0, memberList.Count]
}
],
Groups = memberList.OfType<RoleEntry>().Select(re => new GuildMemberListSyncItem.RoleEntry() { Id = re.Id, Count = re.Count }).ToList()
}
// TODO: send presence updates
// TODO: handle subscriptions
// TODO: handle channel permissions
// TODO: handle channels at all
};
}
private async Task<string?> GetMemberListIdAsync(SpacebarDbContext db, long guildId, long channelId) {
var channel = await db.Channels.AsNoTracking().FirstOrDefaultAsync(c => c.Id == channelId && c.GuildId == guildId);
if (channel == null) return null;
if (string.IsNullOrWhiteSpace(channel.PermissionOverwrites) || channel.PermissionOverwrites == "[]") {
return "everyone";
}
return null; // TODO
}
private async Task<List<IMemberListEntry>> GetGuildMemberListAsync(SpacebarDbContext db, long guildId) {
var memberList = new List<IMemberListEntry>();
// Fetch hoisted roles for the guild to define groups
var hoistedRoles = await db.Roles
.AsNoTracking()
.Where(r => r.GuildId == payload.GuildId && r.Hoist)
.Where(r => r.GuildId == guildId && r.Hoist)
.OrderByDescending(r => r.Position)
// .Select(r => r.Id)
.ToListAsync();
@@ -51,7 +95,7 @@ public class Op14Controller(ILogger<Op12Controller> logger, SpacebarAspNetAuthen
var members = await db.Members.AsNoTracking()
.Include(x => x.IdNavigation)
.Where(x =>
x.GuildId == payload.GuildId
x.GuildId == guildId
&& x.Roles.Any(r => r.Id == role)
&& !x.Roles.Any(r => handledRoles.Contains(r.Id))
// and finally, filter by online
@@ -74,7 +118,7 @@ public class Op14Controller(ILogger<Op12Controller> logger, SpacebarAspNetAuthen
.Include(x => x.IdNavigation)
// .ThenInclude(x=>x.Sessions)
.Where(x =>
x.GuildId == payload.GuildId
x.GuildId == guildId
&& !x.Roles.Any(r => handledRoles.Contains(r.Id))
// and finally, filter by online
&& x.IdNavigation.Sessions.Any(s => s.Status != "offline" && s.Status != "invisible" && s.Status != "unknown")
@@ -90,14 +134,13 @@ public class Op14Controller(ILogger<Op12Controller> logger, SpacebarAspNetAuthen
memberList.AddRange(onlineMembers.Select(m => (IMemberListEntry)new MemberEntry() { Member = m.ToPublicMember() }));
}
if (memberList.Count < 2000) {
logger.LogInformation("Less than 2000 members, including offline members...");
var offlineMembers = await db.Members.AsNoTracking()
.Include(x => x.IdNavigation)
// .ThenInclude(x=>x.Sessions)
.Where(x =>
x.GuildId == payload.GuildId
x.GuildId == guildId
&& !x.Roles.Any(r => handledRoles.Contains(r.Id))
// and finally, filter by online
&& (x.IdNavigation.Sessions.Any(s => s.Status == "offline" || s.Status == "invisible" || s.Status == "unknown") || !x.IdNavigation.Sessions.Any())
@@ -115,24 +158,14 @@ public class Op14Controller(ILogger<Op12Controller> logger, SpacebarAspNetAuthen
}
logger.LogInformation("Got member list with {count} total nodes", memberList.Count);
}
private async Task<string?> GetMemberListIdAsync(SpacebarDbContext db, long guildId, long channelId) {
var channel = await db.Channels.AsNoTracking().FirstOrDefaultAsync(c => c.Id == channelId && c.GuildId == guildId);
if (channel == null) return null;
if (string.IsNullOrWhiteSpace(channel.PermissionOverwrites) || channel.PermissionOverwrites == "[]") {
return "everyone";
}
return null; // TODO
return memberList;
}
}
internal interface IMemberListEntry { }
internal struct RoleEntry : IMemberListEntry {
[JsonPropertyName("id"), JsonNumberHandling(JsonNumberHandling.AllowReadingFromString | JsonNumberHandling.WriteAsString)]
[JsonPropertyName("id")]
public string Id { get; set; }
[JsonPropertyName("count")]
src/get-member-list.ts
-40
View File
@@ -1,40 +0,0 @@
// /*
// Spacebar: A FOSS re-implementation and extension of the Discord.com backend.
// Copyright (C) 2026 Spacebar and Spacebar Contributors
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published
// by the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
// */
//
// process.on("uncaughtException", console.error);
// process.on("unhandledRejection", console.error);
//
// import moduleAlias from "module-alias";
// moduleAlias(__dirname + "../../package.json");
// import { config } from "dotenv";
// config({ quiet: true });
//
// // process.env.DB_LOGGING = "true";
//
// import { closeDatabase, initDatabase } from "@spacebar/database";
// import { buildFullMemberlistSequential } from "@spacebar/gateway/opcodes/LazyRequest";
//
// async function main() {
// await initDatabase();
//
// await buildFullMemberlistSequential("1006649183970562092");
//
// await closeDatabase();
// }
//
// main().then(() => console.log("meow"));