From 0a649a0de2cd88d11e16f2c4668b91709957f3a5 Mon Sep 17 00:00:00 2001 From: Rory& Date: Tue, 25 Nov 2025 23:04:25 +0100 Subject: [PATCH 1/8] Expose mor emember keys --- assets/openapi.json | Bin 843980 -> 844877 bytes assets/schemas.json | Bin 3418792 -> 3421605 bytes src/schemas/api/users/Member.ts | 12 +++++++++++- src/util/entities/Member.ts | 6 ++++++ 4 files changed, 17 insertions(+), 1 deletion(-) diff --git a/assets/openapi.json b/assets/openapi.json index 0e21164c49419fa921dd238c58a0bebbd561bd13..b62e2b43429f828d13b3295836a1ee59401b8c99 100644 GIT binary patch delta 558 zcmX?e(D>{L2Wd$|OB`LmkWXy;oT!r@QpBif@n7W{PEjn`F#5{eT!F%k+i< zHl^te3mI*v8>BF&O>Zz@IzOF%4Wq)e6)djP4O&>@rG<>oQ(1Vn*G**EDo(a@5QZ#Zoh%^8HJ$Y#m+kWSxHD1#8jtfU7KQ(=U8sm6;x3!W=mLKo+b0G*4#t z?eng%Ff(osF<^>f-tI7oWd`H)1{Kyl(`K@;OmE0%j+$Oz%ACGE-i^_iaXNo36Knf_ zb0#2W-u~a5CC`@_9YB>#LQLB)bh9+D5TkxNQ2oZ~3V&E6Ccjx?G1*{|-u4SqS)#1(R7awgZFW1|u+-JlfgSS%H{sJG(l2SpjoyYTopX y|Cq$Ke-P%}4h%7OCUzE|%;J*m2N$qUVrs9Q&JM&JK+FlmTtLjdy>>d!1s?zz3Yp;m diff --git a/assets/schemas.json b/assets/schemas.json index bbf6461f5bba97f836d827a8f5213f05c4a8b815..e53b7b5278c2de8d9d99574cca5109feac1d73fc 100644 GIT binary patch delta 1444 zcmdVaPe_w-7zc3n?wxbZy?dwA_J7m4mO6d6cbTb4uasqi`d+dE4GRLH9Tq{@sZpaH zG8&lsf_0dh;GsfH_)#z>T9=?0LJSl%w+>RzY~6>aI(*=L;rD#s=Xv4xEPX0y z7QYuXiKHgUCAF*kPk|*|t{Yk?Y5y>8H$;!2a74QTCBp?E_UQcBPF6m^@rAyn6eYK zB7Qg0f%GPS32MJl59OWFQ7rHvex0{Kdaw7DUOT8Z-)Wmc{4}qFXy@(I388mdwfgq5 zF~nQWBYVcylsOr7K}+bn@#4<^(SC~BCr;ubZsH+c;v@B>f%r)y z3DCHQo^0%Ywv30bMz%wcpstv`Se9fzmg~7IrKiWy0jN7so+&SbCQb1(dc=J$0lD?o z8uTnQhmdPGSlRe$u$C|P(v+@ZYA_!phe;D@CM_gHTG?Q}&7CYau#bazR<2;&fHcRI zineNbwYW)cX1?lqHa0Ac?)A6tZ3EgiI@U5Gy*8KI*55gEF)5Xcg-L|vV)CDM&pyV!@pDk0wF{3hm8IABLIMOe%g8ynI2a- zG2g(G5@I4|Qc5hOjLkP#FNKRsPEazjk#ZssJ8=+^RFF#IBvquE<)>?m+U)_ENANwr zgvVE{UM$2c_1F$rMXZk78~2v_u0+d6774*Y&IA1spKn{fj|oBVwP-0(84Lj1OO=c0{MlZbXw^l}H6#rO=nlomdXIEQ{JoCFZ_X)ge09 zRf5bE?dn|W4zQ+v*9P}b%PE(uozQ3HIw-&VBJ?v>4~u-3#+nl|6E5OrGZRvF_J4Os zPzO>=>WGKblLpdAyrhXVlNQoSo!j276+Wpc)g#5(%d^9rW|B^Jb)u zaN^!wY-N3A$S2(behza^_Tstk<$j;@NFu!MBYx6O4v>T75b0py^-d{X+;jdjC;&~E ziItv~?;7bb*?2@}-Aq_V`2F`U;O0m<@!hEoW1x~h4CFq;+ oRMSfFNx+Xabrtzz-9yZlQ7@G5BZ5)H{VV&}SU8)}+1c;E0ZjZAc>n+a diff --git a/src/schemas/api/users/Member.ts b/src/schemas/api/users/Member.ts index 8436e62aa..1acd441cb 100644 --- a/src/schemas/api/users/Member.ts +++ b/src/schemas/api/users/Member.ts @@ -64,7 +64,12 @@ export type PublicMemberKeys = | "deaf" | "mute" | "premium_since" - | "avatar"; + | "avatar" + | "banner" + | "bio" + | "theme_colors" + | "pronouns" + | "communication_disabled_until"; export const PublicMemberProjection: PublicMemberKeys[] = [ "id", @@ -77,6 +82,11 @@ export const PublicMemberProjection: PublicMemberKeys[] = [ "mute", "premium_since", "avatar", + "banner", + "bio", + "theme_colors", + "pronouns", + "communication_disabled_until", ]; export type PublicMember = Omit, "roles"> & { diff --git a/src/util/entities/Member.ts b/src/util/entities/Member.ts index 2cabd8580..404d2ce73 100644 --- a/src/util/entities/Member.ts +++ b/src/util/entities/Member.ts @@ -44,6 +44,12 @@ export const MemberPrivateProjection: (keyof Member)[] = [ "roles", "settings", "user", + "avatar", + "banner", + "bio", + "theme_colors", + "pronouns", + "communication_disabled_until", ]; @Entity({ From 06e33083556ae750f25bd07abff0670e0efacda7 Mon Sep 17 00:00:00 2001 From: Rory& Date: Wed, 26 Nov 2025 15:27:19 +0100 Subject: [PATCH 2/8] Mention inbox, hopefully --- .idea/data_source_mapping.xml | 6 - .idea/workspace.xml | 95 ++++++++++------ assets/openapi.json | Bin 844877 -> 846010 bytes scripts/openapi.js | 3 +- scripts/util/getRouteDescriptions.js | 3 +- src/api/routes/users/@me/mentions.ts | 161 +++++++++++++++++++++++++++ 6 files changed, 223 insertions(+), 45 deletions(-) delete mode 100644 .idea/data_source_mapping.xml create mode 100644 src/api/routes/users/@me/mentions.ts diff --git a/.idea/data_source_mapping.xml b/.idea/data_source_mapping.xml deleted file mode 100644 index c2a7ad0fd..000000000 --- a/.idea/data_source_mapping.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/workspace.xml b/.idea/workspace.xml index 11e173ef7..f43b99864 100644 --- a/.idea/workspace.xml +++ b/.idea/workspace.xml @@ -43,46 +43,46 @@ + + + + + + + + + @@ -155,6 +165,17 @@ + + + diff --git a/assets/openapi.json b/assets/openapi.json index b62e2b43429f828d13b3295836a1ee59401b8c99..c5e9276405f2cb24065f24f67a7490994394452b 100644 GIT binary patch delta 80 zcmX?m!g$w3F7M2#)7Pc+ywbPk%Q}d>8{Kq7={ev*)cE;%q?o8|~KAFWO f+Yc^apTyK&JDnYfIe?fGh`E56dwcD4o(nzz2S^|Q delta 49 zcmdmW(fI5MF7M2#)7Pc+ywbR>Q&1MH;4j|?PVlE)&-u`Mf&jlX x?.prototype?.OPTS_MARKER == true); if (!opts) return console.error( - `${file} has route without route() description middleware`, + `${bgRedBright("ERROR")} ${file} has route without route() description middleware`, ); console.log(`${method.toUpperCase().padStart("OPTIONS".length)} ${prefix + path}`); diff --git a/src/api/routes/users/@me/mentions.ts b/src/api/routes/users/@me/mentions.ts new file mode 100644 index 000000000..c4491ad08 --- /dev/null +++ b/src/api/routes/users/@me/mentions.ts @@ -0,0 +1,161 @@ +/* + Spacebar: A FOSS re-implementation and extension of the Discord.com backend. + Copyright (C) 2025 Spacebar and Spacebar Contributors + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published + by the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +import { route } from "@spacebar/api"; +import { Snowflake, User, Message, Member, Channel, Permissions, timePromise } from "@spacebar/util"; +import { Request, Response, Router } from "express"; +import { In } from "typeorm"; + +const router: Router = Router({ mergeParams: true }); + +router.get( + "", + route({ + responses: { + 200: { + body: "MessageListResponse", + }, + 404: { + body: "APIErrorResponse", + }, + }, + }), + // AFAICT this endpoint doesn't list DMs + async (req: Request, res: Response) => { + const limit = req.query.limit && !isNaN(Number(req.query.limit)) ? Number(req.query.limit) : 50; + const everyone = !!req.query.everyone; + const roles = !!req.query.roles; + + const user = await User.findOneOrFail({ + where: { id: req.user_id }, + }); + + const memberships = await Member.find({ + where: { id: req.user_id }, + select: { + guild_id: true, + id: true, + communication_disabled_until: true, + roles: { + // We don't want to include all guild roles, as this could cause a lot more explosive behavior + id: true, + position: true, + permissions: true, + mentionable: true, // cause we can skip querying for unmentionable roles + }, + guild: { + id: true, + owner_id: true, + }, + }, + relations: ["guild", "roles"], + }); + + const channels = await Channel.find({ + where: { + guild_id: In(memberships.map((m) => m.guild_id)), + }, + select: { id: true, permission_overwrites: true }, + }); + + const visibleChannels = channels.filter((c) => { + const member = memberships.find((m) => m.guild_id === c.guild_id)!; + return Permissions.finalPermission({ + user: { id: member.id, roles: member.roles.map((r) => r.id), communication_disabled_until: member.communication_disabled_until, flags: 0 }, + guild: { id: member.guild.id, owner_id: member.guild.owner_id!, roles: member.guild.roles }, + channel: c, + }).has("VIEW_CHANNEL"); + }); + const visibleChannelIds = visibleChannels.map((c) => c.id); + const ownedMentionableRoleIds = memberships.reduce((acc, m) => { + acc.push(...m.roles.filter((r) => r.mentionable).map((r) => r.id)); + return acc; + }, [] as Snowflake[]); + + const [ + { result: userMentions, elapsed: userMentionQueryTime }, + { result: roleMentions, elapsed: roleMentionQueryTime }, + { result: everyoneMentions, elapsed: everyoneMentionQueryTime }, + ] = await Promise.all([ + await timePromise(() => + Message.find({ + where: { + channel_id: In(visibleChannelIds), + mentions: { id: user.id }, + }, + select: { + id: true, + timestamp: true, + }, + order: { + timestamp: "DESC", + }, + take: limit ? Number(limit) : 50, + }), + ), + await timePromise(() => + !roles + ? Promise.resolve([]) + : Message.find({ + where: { + channel_id: In(visibleChannelIds), + mention_roles: { id: In(ownedMentionableRoleIds) }, + }, + select: { + id: true, + timestamp: true, + }, + order: { + timestamp: "DESC", + }, + take: limit ? Number(limit) : 50, + }), + ), + await timePromise(() => + !everyone + ? Promise.resolve([]) + : Message.find({ + where: { + channel_id: In(visibleChannelIds), + mention_everyone: true, + }, + select: { + id: true, + timestamp: true, + }, + order: { + timestamp: "DESC", + }, + take: limit ? Number(limit) : 50, + }), + ), + ]); + + const allMentions = [...userMentions, ...roleMentions, ...everyoneMentions]; + console.log(`[Inbox/mentions] User ${user.id} query results: totalRecs=${allMentions.length} | user=${userMentions.length} (took ${userMentionQueryTime}ms), role=${roleMentions.length} (took ${roleMentionQueryTime}ms), everyone=${everyoneMentions.length} (took ${everyoneMentionQueryTime}ms)`); + + return res.json( + allMentions + .sort((a, b) => b.timestamp.getTime() - a.timestamp.getTime()) + .distinctBy((m) => m.id) + .slice(0, limit), + ); + }, +); + +export default router; From f818781b27e5bc8619b0c5761f25d0037ef849f8 Mon Sep 17 00:00:00 2001 From: Rory& Date: Wed, 26 Nov 2025 16:03:54 +0100 Subject: [PATCH 3/8] Cache access token key in memory and write it back to disk if deleted --- src/util/util/Token.ts | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/src/util/util/Token.ts b/src/util/util/Token.ts index 3de05aa2a..576349538 100644 --- a/src/util/util/Token.ts +++ b/src/util/util/Token.ts @@ -147,8 +147,31 @@ export async function generateToken(id: string) { }); } +let lastFsCheck: number; +let cachedKeypair: { + privateKey: crypto.KeyObject; + publicKey: crypto.KeyObject; + fingerprint: string; +} + // Get ECDSA keypair from file or generate it export async function loadOrGenerateKeypair() { + if (cachedKeypair) { + // check for file deletion every minute + if (Date.now() - lastFsCheck > 60000) { + if (!existsSync("jwt.key") || !existsSync("jwt.key.pub")) { + console.log("[JWT] Keypair files disappeared... Saving them again."); + await Promise.all([ + fs.writeFile("jwt.key", cachedKeypair.privateKey.export({ format: "pem", type: "sec1" })), + fs.writeFile("jwt.key.pub", cachedKeypair.publicKey.export({ format: "pem", type: "spki" })), + ]); + } + lastFsCheck = Date.now(); + } + + return cachedKeypair; + } + let privateKey: crypto.KeyObject; let publicKey: crypto.KeyObject; @@ -185,5 +208,6 @@ export async function loadOrGenerateKeypair() { .update(publicKey.export({ format: "pem", type: "spki" })) .digest("hex"); - return { privateKey, publicKey, fingerprint }; + lastFsCheck = Date.now(); + return cachedKeypair = { privateKey, publicKey, fingerprint }; } From 3477f624599332f8e3194acce515b580e9f1ab75 Mon Sep 17 00:00:00 2001 From: Rory& Date: Wed, 26 Nov 2025 16:16:01 +0100 Subject: [PATCH 4/8] Mentions inbox: include channel guild id --- src/api/routes/users/@me/mentions.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/api/routes/users/@me/mentions.ts b/src/api/routes/users/@me/mentions.ts index c4491ad08..5b799956b 100644 --- a/src/api/routes/users/@me/mentions.ts +++ b/src/api/routes/users/@me/mentions.ts @@ -70,7 +70,7 @@ router.get( where: { guild_id: In(memberships.map((m) => m.guild_id)), }, - select: { id: true, permission_overwrites: true }, + select: { id: true, guild_id: true, permission_overwrites: true }, }); const visibleChannels = channels.filter((c) => { From cfb88d7fe971ffc9d1ae2f587f0993ccab5a6814 Mon Sep 17 00:00:00 2001 From: Rory& Date: Wed, 26 Nov 2025 16:18:41 +0100 Subject: [PATCH 5/8] Mentions inbox: use the right roles prop... --- src/api/routes/users/@me/mentions.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/api/routes/users/@me/mentions.ts b/src/api/routes/users/@me/mentions.ts index 5b799956b..6ac64ef2d 100644 --- a/src/api/routes/users/@me/mentions.ts +++ b/src/api/routes/users/@me/mentions.ts @@ -77,10 +77,11 @@ router.get( const member = memberships.find((m) => m.guild_id === c.guild_id)!; return Permissions.finalPermission({ user: { id: member.id, roles: member.roles.map((r) => r.id), communication_disabled_until: member.communication_disabled_until, flags: 0 }, - guild: { id: member.guild.id, owner_id: member.guild.owner_id!, roles: member.guild.roles }, + guild: { id: member.guild.id, owner_id: member.guild.owner_id!, roles: member.roles }, channel: c, }).has("VIEW_CHANNEL"); }); + const visibleChannelIds = visibleChannels.map((c) => c.id); const ownedMentionableRoleIds = memberships.reduce((acc, m) => { acc.push(...m.roles.filter((r) => r.mentionable).map((r) => r.id)); From ad0b0245dd2e1ec7a25f02e5005b3c414f4b611f Mon Sep 17 00:00:00 2001 From: Rory& Date: Wed, 26 Nov 2025 17:04:46 +0100 Subject: [PATCH 6/8] Actually include messages in mentions result --- src/api/routes/users/@me/mentions.ts | 60 +++++++++++++++++++++++----- 1 file changed, 49 insertions(+), 11 deletions(-) diff --git a/src/api/routes/users/@me/mentions.ts b/src/api/routes/users/@me/mentions.ts index 6ac64ef2d..f13edf0a8 100644 --- a/src/api/routes/users/@me/mentions.ts +++ b/src/api/routes/users/@me/mentions.ts @@ -17,7 +17,7 @@ */ import { route } from "@spacebar/api"; -import { Snowflake, User, Message, Member, Channel, Permissions, timePromise } from "@spacebar/util"; +import { Snowflake, User, Message, Member, Channel, Permissions, timePromise, NewUrlUserSignatureData, Stopwatch } from "@spacebar/util"; import { Request, Response, Router } from "express"; import { In } from "typeorm"; @@ -40,6 +40,7 @@ router.get( const limit = req.query.limit && !isNaN(Number(req.query.limit)) ? Number(req.query.limit) : 50; const everyone = !!req.query.everyone; const roles = !!req.query.roles; + const before = req.query.before && BigInt(req.query.before as string); const user = await User.findOneOrFail({ where: { id: req.user_id }, @@ -106,7 +107,7 @@ router.get( order: { timestamp: "DESC", }, - take: limit ? Number(limit) : 50, + take: limit, }), ), await timePromise(() => @@ -124,7 +125,7 @@ router.get( order: { timestamp: "DESC", }, - take: limit ? Number(limit) : 50, + take: limit, }), ), await timePromise(() => @@ -142,20 +143,57 @@ router.get( order: { timestamp: "DESC", }, - take: limit ? Number(limit) : 50, + take: limit, }), ), ]); const allMentions = [...userMentions, ...roleMentions, ...everyoneMentions]; - console.log(`[Inbox/mentions] User ${user.id} query results: totalRecs=${allMentions.length} | user=${userMentions.length} (took ${userMentionQueryTime}ms), role=${roleMentions.length} (took ${roleMentionQueryTime}ms), everyone=${everyoneMentions.length} (took ${everyoneMentionQueryTime}ms)`); - - return res.json( - allMentions - .sort((a, b) => b.timestamp.getTime() - a.timestamp.getTime()) - .distinctBy((m) => m.id) - .slice(0, limit), + console.log( + `[Inbox/mentions] User ${user.id} query results: totalRecs=${allMentions.length} | user=${userMentions.length} (took ${userMentionQueryTime.totalMilliseconds}ms), role=${roleMentions.length} (took ${roleMentionQueryTime.totalMilliseconds}ms), everyone=${everyoneMentions.length} (took ${everyoneMentionQueryTime.totalMilliseconds}ms)`, ); + const messageIdsToReturn = allMentions + .sort((a, b) => b.timestamp.getTime() - a.timestamp.getTime()) + .distinctBy((m) => m.id) + .slice(0, limit); + + const sw = Stopwatch.startNew(); + const finalMessages = ( + await Message.find({ + where: { id: In(messageIdsToReturn.map((m) => m.id)) }, + order: { timestamp: "DESC" }, + relations: [ + "author", + "webhook", + "application", + "mentions", + "mention_roles", + "mention_channels", + "sticker_items", + "attachments", + "referenced_message", + "referenced_message.author", + "referenced_message.webhook", + "referenced_message.application", + "referenced_message.mentions", + "referenced_message.mention_roles", + "referenced_message.mention_channels", + "referenced_message.sticker_items", + "referenced_message.attachments", + ], + }) + ).map((m) => + m.toJSON().withSignedAttachments( + new NewUrlUserSignatureData({ + ip: req.ip, + userAgent: req.headers["user-agent"] as string, + }), + ), + ); + + console.log(`[Inbox/mentions] User ${user.id} fetched full message data for ${finalMessages.length} messages in ${sw.elapsed().totalMilliseconds}ms`); + + return res.json(finalMessages); }, ); From de4a1ed58b51e42d8655faa874cccf11929253ce Mon Sep 17 00:00:00 2001 From: Rory& Date: Wed, 26 Nov 2025 17:06:06 +0100 Subject: [PATCH 7/8] Swap around toJSON and withSignedAttachments (oops) --- src/api/routes/users/@me/mentions.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/api/routes/users/@me/mentions.ts b/src/api/routes/users/@me/mentions.ts index f13edf0a8..0bee6bb0e 100644 --- a/src/api/routes/users/@me/mentions.ts +++ b/src/api/routes/users/@me/mentions.ts @@ -183,12 +183,12 @@ router.get( ], }) ).map((m) => - m.toJSON().withSignedAttachments( + m.withSignedAttachments( new NewUrlUserSignatureData({ ip: req.ip, userAgent: req.headers["user-agent"] as string, }), - ), + ).toJSON(), ); console.log(`[Inbox/mentions] User ${user.id} fetched full message data for ${finalMessages.length} messages in ${sw.elapsed().totalMilliseconds}ms`); From ce3a46ed2c6fd1d45918275e8f39ff3734089ef0 Mon Sep 17 00:00:00 2001 From: Rory& Date: Wed, 26 Nov 2025 17:20:57 +0100 Subject: [PATCH 8/8] Maybe? --- src/api/routes/users/@me/mentions.ts | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/src/api/routes/users/@me/mentions.ts b/src/api/routes/users/@me/mentions.ts index 0bee6bb0e..cd3bb684e 100644 --- a/src/api/routes/users/@me/mentions.ts +++ b/src/api/routes/users/@me/mentions.ts @@ -17,7 +17,7 @@ */ import { route } from "@spacebar/api"; -import { Snowflake, User, Message, Member, Channel, Permissions, timePromise, NewUrlUserSignatureData, Stopwatch } from "@spacebar/util"; +import { Snowflake, User, Message, Member, Channel, Permissions, timePromise, NewUrlUserSignatureData, Stopwatch, Attachment } from "@spacebar/util"; import { Request, Response, Router } from "express"; import { In } from "typeorm"; @@ -182,14 +182,20 @@ router.get( "referenced_message.attachments", ], }) - ).map((m) => - m.withSignedAttachments( - new NewUrlUserSignatureData({ - ip: req.ip, - userAgent: req.headers["user-agent"] as string, - }), - ).toJSON(), - ); + ).map((m) => { + return { + ...m.toJSON(), + attachments: m.attachments?.map((attachment: Attachment) => + Attachment.prototype.signUrls.call( + attachment, + new NewUrlUserSignatureData({ + ip: req.ip, + userAgent: req.headers["user-agent"] as string, + }), + ), + ), + }; + }); console.log(`[Inbox/mentions] User ${user.id} fetched full message data for ${finalMessages.length} messages in ${sw.elapsed().totalMilliseconds}ms`);