diff --git a/extra/admin-api/Interop/Spacebar.Interop.Authentication/SpacebarAuthenticationService.cs b/extra/admin-api/Interop/Spacebar.Interop.Authentication/SpacebarAuthenticationService.cs index ffd6c4fa3..421c23055 100644 --- a/extra/admin-api/Interop/Spacebar.Interop.Authentication/SpacebarAuthenticationService.cs +++ b/extra/admin-api/Interop/Spacebar.Interop.Authentication/SpacebarAuthenticationService.cs @@ -1,6 +1,7 @@ using System.IdentityModel.Tokens.Jwt; using System.Security.Cryptography; using ArcaneLibs.Collections; +using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; using Microsoft.IdentityModel.Tokens; using Spacebar.Models.Db.Contexts; @@ -53,7 +54,7 @@ public class SpacebarAuthenticationService(ILogger { var uid = config.OverrideUid ?? res?.ClaimsIdentity.Claims.First(x => x.Type == "id").Value; if (string.IsNullOrWhiteSpace(uid)) throw new InvalidOperationException("No user ID specified, is the access token valid?"); - return await db.Users.FindAsync(long.Parse(uid)) ?? throw new InvalidOperationException(); + return await db.Users.FindAsync(long.Parse(uid)) ?? throw new InvalidOperationException($"Could not find user with ID {uid}?"); }, config.AuthCacheExpiry); } @@ -62,9 +63,11 @@ public class SpacebarAuthenticationService(ILogger { + var uid = config.OverrideUid ?? res?.ClaimsIdentity.Claims.First(x => x.Type == "id").Value; var did = config.OverrideDid ?? res?.ClaimsIdentity.Claims.First(x => x.Type == "did").Value; if (string.IsNullOrWhiteSpace(did)) throw new InvalidOperationException("No device ID specified, is the access token valid?"); - return await db.Sessions.FindAsync(long.Parse(did)) ?? throw new InvalidOperationException(); + return await db.Sessions.SingleAsync(s => s.SessionId == did && s.UserId == long.Parse(uid)) + ?? throw new InvalidOperationException($"Could not find device with ID {did}?"); }, config.AuthCacheExpiry); } diff --git a/extra/admin-api/Spacebar.Offload/Controllers/Op14Controller.cs b/extra/admin-api/Spacebar.Offload/Controllers/Op14Controller.cs index 1102bec2c..95998151e 100644 --- a/extra/admin-api/Spacebar.Offload/Controllers/Op14Controller.cs +++ b/extra/admin-api/Spacebar.Offload/Controllers/Op14Controller.cs @@ -1,6 +1,11 @@ +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; using System.Text.Json; +using System.Text.Json.Serialization; +using ArcaneLibs.Extensions; using Microsoft.AspNetCore.Mvc; using Microsoft.EntityFrameworkCore; +using Spacebar.DataMappings.Generic; using Spacebar.GatewayOffload.Extensions.Db; using Spacebar.Interop.Authentication.AspNetCore; using Spacebar.Interop.Replication.Abstractions; @@ -29,13 +34,87 @@ public class Op14Controller(ILogger logger, SpacebarAspNetAuthen yield break; } + var memberList = new List(); + // Fetch hoisted roles for the guild to define groups var hoistedRoles = await db.Roles .AsNoTracking() .Where(r => r.GuildId == payload.GuildId && r.Hoist) .OrderByDescending(r => r.Position) - .Select(r => new { r.Id }) + // .Select(r => r.Id) .ToListAsync(); + + logger.LogDebug("Got hoisted roles: {roleIds}", hoistedRoles.Select(x => x.Id).ToList()); + List handledRoles = []; + foreach (var roleObj in hoistedRoles) { + var role = roleObj.Id; + var members = await db.Members.AsNoTracking() + .Include(x => x.IdNavigation) + .Where(x => + x.GuildId == payload.GuildId + && x.Roles.Any(r => r.Id == role) + && !x.Roles.Any(r => handledRoles.Contains(r.Id)) + // and finally, filter by online + && x.IdNavigation.Sessions.Any(s => s.Status != "offline" && s.Status != "invisible" && s.Status != "unknown") + ) + .OrderBy(x => x.Nick ?? x.IdNavigation.Username).ToListAsync(); + + logger.LogInformation("Got {count} potential members for group {group} ({groupName}):\n - {members}", + members.Count, role, roleObj.Name, string.Join("\n - ", members.Take(10).Select(x => $"{x.Id} {x.Nick ?? x.IdNavigation.Tag}")) + ); + + memberList.Add(new RoleEntry() { Id = role.ToString(), Count = members.Count }); + memberList.AddRange(members.Select(m => (IMemberListEntry)new MemberEntry() { Member = m.ToPublicMember() })); + + handledRoles.Add(role); + } + + // online members + var onlineMembers = await db.Members.AsNoTracking() + .Include(x => x.IdNavigation) + // .ThenInclude(x=>x.Sessions) + .Where(x => + x.GuildId == payload.GuildId + && !x.Roles.Any(r => handledRoles.Contains(r.Id)) + // and finally, filter by online + && x.IdNavigation.Sessions.Any(s => s.Status != "offline" && s.Status != "invisible" && s.Status != "unknown") + ) + .OrderBy(x => x.Nick ?? x.IdNavigation.Username).ToListAsync(); + + logger.LogInformation("Got {count} potential members for group {group} ({groupName}):\n - {members}", + onlineMembers.Count, "online", "online", string.Join("\n - ", onlineMembers.Take(10).Select(x => $"{x.Id} {x.Nick ?? x.IdNavigation.Tag}")) + ); + + if (onlineMembers.Count > 0) { + memberList.Add(new RoleEntry() { Id = "online", Count = onlineMembers.Count }); + memberList.AddRange(onlineMembers.Select(m => (IMemberListEntry)new MemberEntry() { Member = m.ToPublicMember() })); + } + + + if (memberList.Count < 2000) { + logger.LogInformation("Less than 2000 members, including offline members..."); + var offlineMembers = await db.Members.AsNoTracking() + .Include(x => x.IdNavigation) + // .ThenInclude(x=>x.Sessions) + .Where(x => + x.GuildId == payload.GuildId + && !x.Roles.Any(r => handledRoles.Contains(r.Id)) + // and finally, filter by online + && (x.IdNavigation.Sessions.Any(s => s.Status == "offline" || s.Status == "invisible" || s.Status == "unknown") || !x.IdNavigation.Sessions.Any()) + ) + .OrderBy(x => x.Nick ?? x.IdNavigation.Username).ToListAsync(); + + logger.LogInformation("Got {count} potential members for group {group} ({groupName}):\n - {members}", + offlineMembers.Count, "offline", "offline", string.Join("\n - ", offlineMembers.Take(10).Select(x => $"{x.Id} {x.Nick ?? x.IdNavigation.Tag}")) + ); + + if (offlineMembers.Count > 0) { + memberList.Add(new RoleEntry() { Id = "offline", Count = offlineMembers.Count }); + memberList.AddRange(offlineMembers.Select(m => (IMemberListEntry)new MemberEntry() { Member = m.ToPublicMember() })); + } + } + + logger.LogInformation("Got member list with {count} total nodes", memberList.Count); } private async Task GetMemberListIdAsync(SpacebarDbContext db, long guildId, long channelId) { @@ -48,4 +127,19 @@ public class Op14Controller(ILogger logger, SpacebarAspNetAuthen return null; // TODO } +} + +internal interface IMemberListEntry { } + +internal struct RoleEntry : IMemberListEntry { + [JsonPropertyName("id"), JsonNumberHandling(JsonNumberHandling.AllowReadingFromString | JsonNumberHandling.WriteAsString)] + public string Id { get; set; } + + [JsonPropertyName("count")] + public int Count { get; set; } +} + +internal struct MemberEntry : IMemberListEntry { + [JsonPropertyName("member")] + public Member Member { get; set; } } \ No newline at end of file diff --git a/extra/admin-api/Spacebar.Offload/Spacebar.Offload.http b/extra/admin-api/Spacebar.Offload/Spacebar.Offload.http index dc84f632e..9cceb53d0 100644 --- a/extra/admin-api/Spacebar.Offload/Spacebar.Offload.http +++ b/extra/admin-api/Spacebar.Offload/Spacebar.Offload.http @@ -32,3 +32,15 @@ Content-Type: application/json ] ### + +POST {{GatewayOffload_HostAddress}}/_spacebar/offload/gateway/LazyRequest +Accept: application/json +Authorization: Bearer {{AccessToken}} +Content-Type: application/json + +{ + "guild_id": "1006649183970562092", + "channels": { + "1006649184062836783": [[1,2]] + } +} \ No newline at end of file diff --git a/extra/admin-api/Spacebar.Offload/appsettings.Development.json b/extra/admin-api/Spacebar.Offload/appsettings.Development.json index 12d6d238d..0092bb18f 100644 --- a/extra/admin-api/Spacebar.Offload/appsettings.Development.json +++ b/extra/admin-api/Spacebar.Offload/appsettings.Development.json @@ -3,19 +3,17 @@ "LogLevel": { "Default": "Information", "Microsoft.AspNetCore": "Trace", - //Warning + "Microsoft.AspNetCore.Server.Kestrel.Connections": "Information", + "Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets": "Information", "Microsoft.AspNetCore.Mvc": "Warning", - //Warning "Microsoft.AspNetCore.HostFiltering": "Warning", - //Warning "Microsoft.AspNetCore.Cors": "Warning", - //Warning // "Microsoft.EntityFrameworkCore": "Warning" "Microsoft.EntityFrameworkCore.Database.Command": "Information" } }, "ConnectionStrings": { - "Spacebar": "Host=127.0.0.1; Username=postgres; Database=spacebar; Port=5433; Include Error Detail=true; Maximum Pool Size=1000; Command Timeout=6000; Timeout=600;" + "Spacebar": "Host=127.0.0.1; Username=postgres; Database=sb-testing; Port=5432; Include Error Detail=true; Maximum Pool Size=1000; Command Timeout=6000; Timeout=600;" }, "Spacebar": { "Authentication": { diff --git a/extra/admin-api/Spacebar.Offload/http-client.private.env.json b/extra/admin-api/Spacebar.Offload/http-client.private.env.json deleted file mode 100644 index 8e13c5990..000000000 --- a/extra/admin-api/Spacebar.Offload/http-client.private.env.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "dev": { - "AccessToken": "eyJhbGciOiJFUzUxMiIsInR5cCI6IkpXVCJ9.eyJpZCI6IjEwMDY1OTgyMzAxNTYzNDEyNzYiLCJpYXQiOjE3NjU4NjYxNDcsImtpZCI6IjVkZTcwNjZlNWQ5YTkxZDc4NGQ2NTY1Njc2Zjc0ZGY4NGQyMDllNzBjY2M3ZmZmNmFhNjgxNTkwODEwMWRjMWEiLCJ2ZXIiOjMsImRpZCI6IklCWkFSR01YT0UifQ.ALoB-4LXPaHTiUWRSoO3KIIc7CX2tP2vdebxmt10h3DqqBW57Zqx9zNImGxn0tV4cqFB1nZct3cZjJ_XVchtUF61AEgGR54QpV2sHAss2NMqZA_S3WK7UigFJYDddWUt2D_GrvzUYUVJ_WB4gt-tXekKzB2K6dazTEFYPFSY6xINBWed" - } -} \ No newline at end of file diff --git a/src/gateway/opcodes/LazyRequest.ts b/src/gateway/opcodes/LazyRequest.ts index d8f4cb5a7..c050fe985 100644 --- a/src/gateway/opcodes/LazyRequest.ts +++ b/src/gateway/opcodes/LazyRequest.ts @@ -18,11 +18,12 @@ import murmur from "murmurhash-js/murmurhash3_gc"; import { getDatabase, Member, Role, Session, User, Channel } from "@spacebar/database"; -import { arrayPartition } from "@spacebar/extensions"; +import { arrayPartition, Stopwatch } from "@spacebar/extensions"; import { WebSocket, Payload, handlePresenceUpdate, OPCODES, Send } from "@spacebar/gateway"; import { LazyRequestSchema } from "@spacebar/schemas"; import { getPermission, listenEvent, Presence, Permissions, getMostRelevantSession } from "@spacebar/util"; import { check } from "./instanceOf"; +import { And, Any, ArrayContains, In, Not } from "typeorm"; // TODO: only show roles/members that have access to this channel // TODO: config: to list all members (even those who are offline) sorted by role, or just those who are online @@ -258,3 +259,42 @@ export async function onLazyRequest(this: WebSocket, { d }: Payload) { console.log(`[Gateway/${this.user_id}] LAZY_REQUEST ${guild_id} ${channel_id} took ${Date.now() - startTime}ms`); } + +// async function getAllGroups(guild_id: string) { +// const hoistedRoles = await Role.find({ where: { hoist: true, guild_id }, order: { position: "DESC" } }); +// return [...hoistedRoles.map((r) => ({ id: r.id, count: 0 })), { id: "online", count: 0 }, { id: "offline", count: 0 }]; +// } +// +// export async function buildFullMemberlistSequential(guild_id: string) { +// const totalSw = Stopwatch.startNew(); +// const incSw = Stopwatch.startNew(); +// const logTrace = (...data: unknown[]) => { +// if (process.env.LOG_VERBOSE_TRACES !== "true") return; +// console.log("[LazyRequest/buildFullMemberlist]", ...data, `[${totalSw.elapsed().toString()} (+${incSw.getElapsedAndReset().totalMilliseconds}ms)]`); +// }; +// +// const groups = await getAllGroups(guild_id); +// const handledGroups: string[] = []; +// const handledUsers: string[] = []; +// const offlineUsers: string[] = []; +// logTrace("[LazyRequest] Got", groups.length, "groups..."); +// +// for (const group of groups) { +// console.log("[LazyRequest] Building member list for", group.id); +// if (group.id == "offline") { +// } else if (group.id == "online") { +// } else { +// const potentialMembers = await Member.find({ +// where: { roles: And(Any(And({ id: group.id }, Not(Any({ id: In(handledGroups) }))))) }, +// relations: { roles: true, user: true }, +// }); +// console.log( +// "[LazyRequest] Found", +// potentialMembers.length, +// "potential members", +// potentialMembers.map((pm) => ({ id: pm.id, name: pm.nick ?? pm.user.tag ?? pm.id })), +// ); +// } +// logTrace("Built member list for", group.id, "with", 0, "members!"); +// } +// } diff --git a/src/get-member-list.ts b/src/get-member-list.ts new file mode 100644 index 000000000..58bd898b5 --- /dev/null +++ b/src/get-member-list.ts @@ -0,0 +1,40 @@ +// /* +// Spacebar: A FOSS re-implementation and extension of the Discord.com backend. +// Copyright (C) 2026 Spacebar and Spacebar Contributors +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published +// by the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . +// */ +// +// process.on("uncaughtException", console.error); +// process.on("unhandledRejection", console.error); +// +// import moduleAlias from "module-alias"; +// moduleAlias(__dirname + "../../package.json"); +// import { config } from "dotenv"; +// config({ quiet: true }); +// +// // process.env.DB_LOGGING = "true"; +// +// import { closeDatabase, initDatabase } from "@spacebar/database"; +// import { buildFullMemberlistSequential } from "@spacebar/gateway/opcodes/LazyRequest"; +// +// async function main() { +// await initDatabase(); +// +// await buildFullMemberlistSequential("1006649183970562092"); +// +// await closeDatabase(); +// } +// +// main().then(() => console.log("meow"));