From 19edf63e37e84f81e7168e7411285084a30002ba Mon Sep 17 00:00:00 2001 From: Rory& Date: Thu, 25 Dec 2025 05:31:32 +0100 Subject: [PATCH] various crap i was working on --- .idea/inspectionProfiles/Project_Default.xml | 16 ++ .idea/workspace.xml | 16 +- assets/public/invite.html | 165 +++++++++++++++++ src/api/Server.ts | 8 + .../channels/#channel_id/attachments.ts | 6 +- src/api/routes/guilds/#guild_id/widget.png.ts | 2 +- src/api/util/handlers/Message.ts | 2 +- src/cdn/Server.ts | 74 ++++---- src/cdn/index.ts | 4 +- .../routes/_spacebar/cdn/cloud-attachments.ts | 171 ++++++++++++++++++ src/cdn/routes/_spacebar/cdn/upload.ts | 51 ++++++ src/cdn/routes/attachments.ts | 115 +----------- src/cdn/routes/avatars.ts | 15 +- src/cdn/routes/badge-icons.ts | 5 +- src/cdn/routes/embed.ts | 8 +- src/cdn/routes/guild-profiles.ts | 10 +- src/cdn/routes/role-icons.ts | 10 +- src/cdn/util/basicCrdFileRouter.ts | 8 +- src/cdn/util/cache.ts | 7 + .../util/{FileStorage.ts => fileStorage.ts} | 6 +- src/cdn/util/index.ts | 7 +- src/cdn/util/mimetypes.ts | 2 + src/cdn/util/{S3Storage.ts => s3Storage.ts} | 6 +- src/cdn/util/{Storage.ts => storage.ts} | 5 +- src/util/config/types/CdnConfiguration.ts | 9 +- 25 files changed, 515 insertions(+), 213 deletions(-) create mode 100644 assets/public/invite.html create mode 100644 src/cdn/routes/_spacebar/cdn/cloud-attachments.ts create mode 100644 src/cdn/routes/_spacebar/cdn/upload.ts create mode 100644 src/cdn/util/cache.ts rename src/cdn/util/{FileStorage.ts => fileStorage.ts} (87%) create mode 100644 src/cdn/util/mimetypes.ts rename src/cdn/util/{S3Storage.ts => s3Storage.ts} (89%) rename src/cdn/util/{Storage.ts => storage.ts} (94%) diff --git a/.idea/inspectionProfiles/Project_Default.xml b/.idea/inspectionProfiles/Project_Default.xml index 03d9549ea..494575eb4 100644 --- a/.idea/inspectionProfiles/Project_Default.xml +++ b/.idea/inspectionProfiles/Project_Default.xml @@ -2,5 +2,21 @@ \ No newline at end of file diff --git a/.idea/workspace.xml b/.idea/workspace.xml index 25a9866f8..de39b0b9b 100644 --- a/.idea/workspace.xml +++ b/.idea/workspace.xml @@ -65,7 +65,7 @@ "git-widget-placeholder": "master", "javascript.nodejs.core.library.configured.version": "24.11.1", "javascript.nodejs.core.library.typings.version": "24.10.4", - "last_opened_file_path": "/home/Rory/git/spacebar/server-master/src/util/migration/postgres", + "last_opened_file_path": "/home/Rory/git/spacebar/server-master/nix/modules/default", "node.js.detected.package.eslint": "true", "node.js.detected.package.standard": "true", "node.js.selected.package.eslint": "(autodetect)", @@ -96,18 +96,18 @@ }]]> - - - - - + + + + + + - @@ -133,7 +133,7 @@ - + diff --git a/assets/public/invite.html b/assets/public/invite.html new file mode 100644 index 000000000..510b03f29 --- /dev/null +++ b/assets/public/invite.html @@ -0,0 +1,165 @@ + + + + + + + + Spacebar Server + + + + + + + + + +
+ Spacebar Logo + +

You've been invited!

+ +
+
+
+ +

Loading...

+
+

Invited by null.

+
+ 0 Online + 0 Members +
+
+ +

Pick a client to continue.

+

Continue with + Fermi (https://fermi.chat)

+
+ + + \ No newline at end of file diff --git a/src/api/Server.ts b/src/api/Server.ts index d371ed9c4..5039b1542 100644 --- a/src/api/Server.ts +++ b/src/api/Server.ts @@ -24,6 +24,7 @@ import morgan from "morgan"; import path from "path"; import { red } from "picocolors"; import { initInstance } from "./util/handlers/Instance"; +import fs from "fs/promises"; const ASSETS_FOLDER = path.join(__dirname, "..", "..", "assets"); const PUBLIC_ASSETS_FOLDER = path.join(ASSETS_FOLDER, "public"); @@ -129,6 +130,13 @@ export class SpacebarServer extends Server { return res.sendFile(path.join(PUBLIC_ASSETS_FOLDER, "widget.html")); }); + app.get("/invite/:code", async (req, res) => { + const { code } = req.params; + const content = fs.readFile(path.join(PUBLIC_ASSETS_FOLDER, "invite.html"), "utf-8"); + res.set("Cache-Control", "public, max-age=21600"); + return res.send((await content).replace("{code}", code).replace("{serverName}", Config.get().general.serverName)); + }); + app.get("/_spacebar/api/schemas.json", (req, res) => { res.sendFile(path.join(ASSETS_FOLDER, "schemas.json")); }); diff --git a/src/api/routes/channels/#channel_id/attachments.ts b/src/api/routes/channels/#channel_id/attachments.ts index 0d5daa4cc..baf77af37 100644 --- a/src/api/routes/channels/#channel_id/attachments.ts +++ b/src/api/routes/channels/#channel_id/attachments.ts @@ -81,7 +81,7 @@ router.post( userIsClip: attachment.is_clip, userOriginalContentType: attachment.original_content_type, }); - await newAttachment.save(); + await newAttachment.insert(); return newAttachment; }), ); @@ -91,7 +91,7 @@ router.post( return { id: a.userAttachmentId, upload_filename: a.uploadFilename, - upload_url: `${cdnUrl}/attachments/${a.uploadFilename}`, + upload_url: `${cdnUrl}/_spacebar/cdn/cloud-attachments/${a.uploadFilename}`, original_content_type: a.userOriginalContentType, }; }), @@ -119,7 +119,7 @@ router.delete("/:cloud_attachment_url", async (req: Request, res: Response) => { }); } - const response = await fetch(`${Config.get().cdn.endpointPrivate}/attachments/${att.uploadFilename}`, { + const response = await fetch(`${Config.get().cdn.endpointPrivate}/_spacebar/cdn/cloud-attachments/${att.uploadFilename}`, { headers: { signature: Config.get().security.requestSignature, }, diff --git a/src/api/routes/guilds/#guild_id/widget.png.ts b/src/api/routes/guilds/#guild_id/widget.png.ts index 3ff95e995..48c8d22c9 100644 --- a/src/api/routes/guilds/#guild_id/widget.png.ts +++ b/src/api/routes/guilds/#guild_id/widget.png.ts @@ -24,7 +24,7 @@ import { Request, Response, Router } from "express"; import fs from "fs"; import { HTTPError } from "lambert-server"; import path from "path"; -import { storage } from "../../../../cdn/util/Storage"; +import { storage } from "@spacebar/cdn"; const router: Router = Router({ mergeParams: true }); diff --git a/src/api/util/handlers/Message.ts b/src/api/util/handlers/Message.ts index 766abd13f..741fe82cb 100644 --- a/src/api/util/handlers/Message.ts +++ b/src/api/util/handlers/Message.ts @@ -118,7 +118,7 @@ export async function handleMessage(opts: MessageOptions): Promise { }, }); - const cloneResponse = await fetch(`${Config.get().cdn.endpointPrivate}/attachments/${attEnt.uploadFilename}/clone_to_message/${message.id}`, { + const cloneResponse = await fetch(`${Config.get().cdn.endpointPrivate}/_spacebar/cdn/cloud-attachments/${attEnt.uploadFilename}/clone_to_message/${message.id}`, { method: "POST", headers: { signature: Config.get().security.requestSignature || "", diff --git a/src/cdn/Server.ts b/src/cdn/Server.ts index 091621f06..e9115fc76 100644 --- a/src/cdn/Server.ts +++ b/src/cdn/Server.ts @@ -25,6 +25,8 @@ import guildProfilesRoute from "./routes/guild-profiles"; import iconsRoute from "./routes/role-icons"; import morgan from "morgan"; import { Like } from "typeorm"; +import { Router } from "express"; +import { BasicCrdFileRouterOptions, createBasicCrdFileRouter } from "./util/basicCrdFileRouter"; export type CDNServerOptions = ServerOptions; @@ -62,48 +64,40 @@ export class CDNServer extends Server { this.app.use(BodyParser({ inflate: true, limit: "10mb" })); await registerRoutes(this, path.join(__dirname, "routes/")); + const register = (path: string, ...handlers: Router[]) => { + this.app.use(path, ...handlers); + console.log(`[Server] Route ${path} registered`); + }; - this.app.use("/icons/", avatarsRoute); - console.log("[Server] Route /icons registered"); + register("/role-icons/", iconsRoute); + register("/guilds/:guild_id/users/:user_id/avatars", guildProfilesRoute); + register("/guilds/:guild_id/users/:user_id/banners", guildProfilesRoute); - this.app.use("/role-icons/", iconsRoute); - console.log("[Server] Route /role-icons registered"); - - this.app.use("/emojis/", avatarsRoute); - console.log("[Server] Route /emojis registered"); - - this.app.use("/stickers/", avatarsRoute); - console.log("[Server] Route /stickers registered"); - - this.app.use("/banners/", avatarsRoute); - console.log("[Server] Route /banners registered"); - - this.app.use("/splashes/", avatarsRoute); - console.log("[Server] Route /splashes registered"); - - this.app.use("/discovery-splashes/", avatarsRoute); - console.log("[Server] Route /discovery-splashes registered"); - - this.app.use("/app-icons/", avatarsRoute); - console.log("[Server] Route /app-icons registered"); - - this.app.use("/app-assets/", avatarsRoute); - console.log("[Server] Route /app-assets registered"); - - this.app.use("/discover-splashes/", avatarsRoute); - console.log("[Server] Route /discover-splashes registered"); - - this.app.use("/team-icons/", avatarsRoute); - console.log("[Server] Route /team-icons registered"); - - this.app.use("/channel-icons/", avatarsRoute); - console.log("[Server] Route /channel-icons registered"); - - this.app.use("/guilds/:guild_id/users/:user_id/avatars", guildProfilesRoute); - console.log("[Server] Route /guilds/avatars registered"); - - this.app.use("/guilds/:guild_id/users/:user_id/banners", guildProfilesRoute); - console.log("[Server] Route /guilds/banners registered"); + if (!process.env.CDN_CRD_ROUTER) { + register("/icons/", avatarsRoute); + register("/emojis/", avatarsRoute); + register("/stickers/", avatarsRoute); + register("/banners/", avatarsRoute); + register("/splashes/", avatarsRoute); + register("/discovery-splashes/", avatarsRoute); + register("/app-icons/", avatarsRoute); + register("/app-assets/", avatarsRoute); + register("/discover-splashes/", avatarsRoute); + register("/team-icons/", avatarsRoute); + register("/channel-icons/", avatarsRoute); + } else { + register("/icons/", createBasicCrdFileRouter(new BasicCrdFileRouterOptions({ pathPrefix: "icons/", fallbackToAvatarPath: true }))); + register("/emojis/", createBasicCrdFileRouter(new BasicCrdFileRouterOptions({ pathPrefix: "emojis/", fallbackToAvatarPath: true }))); + register("/stickers/", createBasicCrdFileRouter(new BasicCrdFileRouterOptions({ pathPrefix: "stickers/", fallbackToAvatarPath: true }))); + register("/banners/", createBasicCrdFileRouter(new BasicCrdFileRouterOptions({ pathPrefix: "banners/", fallbackToAvatarPath: true }))); + register("/splashes/", createBasicCrdFileRouter(new BasicCrdFileRouterOptions({ pathPrefix: "splashes/", fallbackToAvatarPath: true }))); + register("/discovery-splashes/", createBasicCrdFileRouter(new BasicCrdFileRouterOptions({ pathPrefix: "discovery-splashes/", fallbackToAvatarPath: true }))); + register("/app-icons/", createBasicCrdFileRouter(new BasicCrdFileRouterOptions({ pathPrefix: "app-icons/", fallbackToAvatarPath: true }))); + register("/app-assets/", createBasicCrdFileRouter(new BasicCrdFileRouterOptions({ pathPrefix: "app-assets/", fallbackToAvatarPath: true }))); + register("/discover-splashes/", createBasicCrdFileRouter(new BasicCrdFileRouterOptions({ pathPrefix: "discover-splashes/", fallbackToAvatarPath: true }))); + register("/team-icons/", createBasicCrdFileRouter(new BasicCrdFileRouterOptions({ pathPrefix: "team-icons/", fallbackToAvatarPath: true }))); + register("/channel-icons/", createBasicCrdFileRouter(new BasicCrdFileRouterOptions({ pathPrefix: "channel-icons/", fallbackToAvatarPath: true }))); + } return super.start(); } diff --git a/src/cdn/index.ts b/src/cdn/index.ts index dea9942bc..dc9fd0f7b 100644 --- a/src/cdn/index.ts +++ b/src/cdn/index.ts @@ -17,6 +17,4 @@ */ export * from "./Server"; -export * from "./util/FileStorage"; -export * from "./util/Storage"; -export * from "./util/multer"; +export * from "./util"; diff --git a/src/cdn/routes/_spacebar/cdn/cloud-attachments.ts b/src/cdn/routes/_spacebar/cdn/cloud-attachments.ts new file mode 100644 index 000000000..5da928800 --- /dev/null +++ b/src/cdn/routes/_spacebar/cdn/cloud-attachments.ts @@ -0,0 +1,171 @@ +/* + Spacebar: A FOSS re-implementation and extension of the Discord.com backend. + Copyright (C) 2025 Spacebar and Spacebar Contributors + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published + by the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +import { CloudAttachment, Config, hasValidSignature, NewUrlUserSignatureData, Snowflake, UrlSignResult } from "@spacebar/util"; +import { Request, Response, Router } from "express"; +import imageSize from "image-size"; +import { HTTPError } from "lambert-server"; +import { fileTypeFromBuffer } from "file-type"; +import { cache, multer, storage } from "../../../util"; + +const router = Router({ mergeParams: true }); + +const SANITIZED_CONTENT_TYPE = ["text/html", "text/mhtml", "multipart/related", "application/xhtml+xml"]; + +router.get("/:channel_id/:id/:filename", cache, async (req: Request, res: Response) => { + const { channel_id, id, filename } = req.params; + // const { format } = req.query; + + const path = `attachments/${channel_id}/${id}/${filename}`; + + const fullUrl = (req.headers["x-forwarded-proto"] ?? req.protocol) + "://" + (req.headers["x-forwarded-host"] ?? req.hostname) + req.originalUrl; + + if ( + Config.get().security.cdnSignUrls && + !hasValidSignature( + new NewUrlUserSignatureData({ + ip: req.ip, + userAgent: req.headers["user-agent"] as string, + }), + UrlSignResult.fromUrl(fullUrl), + ) + ) { + return res.status(404).send("This content is no longer available."); + } + + const file = await storage.get(path); + if (!file) throw new HTTPError("File not found"); + const type = await fileTypeFromBuffer(file); + let content_type = type?.mime || "application/octet-stream"; + + if (SANITIZED_CONTENT_TYPE.includes(content_type)) { + content_type = "application/octet-stream"; + } + + res.set("Content-Type", content_type); + + return res.send(file); +}); + +// "cloud attachments" +router.put("/:channel_id/:batch_id/:attachment_id/:filename", multer.single("file"), async (req: Request, res: Response) => { + const { channel_id, batch_id, attachment_id, filename } = req.params; + const att = await CloudAttachment.findOneOrFail({ + where: { + uploadFilename: `${channel_id}/${batch_id}/${attachment_id}/${filename}`, + channelId: channel_id, + userAttachmentId: attachment_id, + userFilename: filename, + }, + }); + + const maxLength = Config.get().cdn.maxAttachmentSize; + + console.log("[Cloud Upload] Uploading attachment", att.id, att.userFilename, `Max size: ${maxLength} bytes`); + + const chunks: Buffer[] = []; + let length = 0; + + req.on("data", (chunk) => { + console.log(`[Cloud Upload] Received chunk of size ${chunk.length} bytes`); + chunks.push(chunk); + length += chunk.length; + if (length > maxLength) { + res.status(413).send("File too large"); + req.destroy(); + } + }); + req.on("end", async () => { + console.log(`[Cloud Upload] Finished receiving file, total size ${length} bytes`); + const buffer = Buffer.concat(chunks); + const path = `attachments/${channel_id}/${batch_id}/${attachment_id}/${filename}`; + + await storage.set(path, buffer); + + let mimeType = att.userOriginalContentType; + if (att.userOriginalContentType === null) { + const ft = await fileTypeFromBuffer(buffer); + mimeType = att.contentType = ft?.mime || "application/octet-stream"; + } + + if (mimeType?.includes("image")) { + const dimensions = imageSize(buffer); + if (dimensions) { + att.width = dimensions.width; + att.height = dimensions.height; + } + } + + att.size = buffer.length; + await att.save(); + + console.log("[Cloud Upload] Saved attachment", att.id, att.userFilename); + res.status(200).end(); + }); +}); + +router.delete("/:channel_id/:batch_id/:attachment_id/:filename", async (req: Request, res: Response) => { + if (req.headers.signature !== Config.get().security.requestSignature) throw new HTTPError("Invalid request signature"); + console.log("[Cloud Delete] Deleting attachment", req.params); + + const { channel_id, batch_id, attachment_id, filename } = req.params; + const path = `attachments/${channel_id}/${batch_id}/${attachment_id}/${filename}`; + + const att = await CloudAttachment.findOne({ + where: { + uploadFilename: `${channel_id}/${batch_id}/${attachment_id}/${filename}`, + channelId: channel_id, + userAttachmentId: attachment_id, + userFilename: filename, + }, + }); + + if (att) { + await att.remove(); + await storage.delete(path); + return res.send({ success: true }); + } + return res.status(404).send("Attachment not found"); +}); + +router.post("/:channel_id/:batch_id/:attachment_id/:filename/clone_to_message/:message_id", async (req: Request, res: Response) => { + if (req.headers.signature !== Config.get().security.requestSignature) throw new HTTPError("Invalid request signature"); + console.log("[Cloud Clone] Cloning attachment to message", req.params); + + const { channel_id, batch_id, attachment_id, filename, message_id } = req.params; + const path = `attachments/${channel_id}/${batch_id}/${attachment_id}/${filename}`; + const newPath = `attachments/${channel_id}/${message_id}/${filename}`; + + const att = await CloudAttachment.findOne({ + where: { + uploadFilename: `${channel_id}/${batch_id}/${attachment_id}/${filename}`, + channelId: channel_id, + userAttachmentId: attachment_id, + userFilename: filename, + }, + }); + + if (att) { + await storage.clone(path, newPath); + return res.send({ success: true, new_path: newPath }); + } + + return res.status(404).send("Attachment not found"); +}); + +export default router; diff --git a/src/cdn/routes/_spacebar/cdn/upload.ts b/src/cdn/routes/_spacebar/cdn/upload.ts new file mode 100644 index 000000000..f706d15c1 --- /dev/null +++ b/src/cdn/routes/_spacebar/cdn/upload.ts @@ -0,0 +1,51 @@ +/* + Spacebar: A FOSS re-implementation and extension of the Discord.com backend. + Copyright (C) 2025 Spacebar and Spacebar Contributors + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published + by the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +import { CloudAttachment, Config, hasValidSignature, NewUrlUserSignatureData, Snowflake, UrlSignResult } from "@spacebar/util"; +import { Request, Response, Router } from "express"; +import imageSize from "image-size"; +import { HTTPError } from "lambert-server"; +import { fileTypeFromBuffer } from "file-type"; +import { cache, multer, storage } from "../../../util"; +import { CdnImageLimitsConfiguration } from "../../../../util/config/types"; + +const router = Router({ mergeParams: true }); + +const SANITIZED_CONTENT_TYPE = ["text/html", "text/mhtml", "multipart/related", "application/xhtml+xml"]; + +const limits = Config.get().cdn.limits; +function createImageUploadRoute(name: string, path: string, limits: CdnImageLimitsConfiguration) { + router.post(`/${name}/:user_id`, multer.single("file"), async (req: Request, res: Response) => {}); + console.log(`Registered image upload /_spacebar/cdn/upload/${name} (-> storage/${path}/) with limits:`, JSON.stringify(limits)); +} + +createImageUploadRoute("icon", "icons", limits.icon); +createImageUploadRoute("role-icon", "role-icons", limits.roleIcon); +createImageUploadRoute("emoji", "emojis", limits.emoji); +createImageUploadRoute("sticker", "stickers", limits.sticker); +createImageUploadRoute("banner", "banners", limits.banner); +createImageUploadRoute("splash", "splashs", limits.splash); +createImageUploadRoute("avatar", "avatars", limits.avatar); +createImageUploadRoute("discovery-splash", "discovery-splashes", limits.discoverySplash); +createImageUploadRoute("app-icon", "app-icons", limits.appIcon); +createImageUploadRoute("discover-splash", "discover-splashes", limits.discoverSplash); +createImageUploadRoute("team-icon", "team-icons", limits.teamIcon); +createImageUploadRoute("channel-icon", "channel-icons", limits.channelIcon); +createImageUploadRoute("guild-avatar", "guild-avatars", limits.guildAvatar); + +export default router; diff --git a/src/cdn/routes/attachments.ts b/src/cdn/routes/attachments.ts index db2b95031..b10b3b0a3 100644 --- a/src/cdn/routes/attachments.ts +++ b/src/cdn/routes/attachments.ts @@ -16,14 +16,12 @@ along with this program. If not, see . */ -import { Config, hasValidSignature, NewUrlUserSignatureData, Snowflake, UrlSignResult } from "@spacebar/util"; +import { CloudAttachment, Config, hasValidSignature, NewUrlUserSignatureData, Snowflake, UrlSignResult } from "@spacebar/util"; import { Request, Response, Router } from "express"; import imageSize from "image-size"; import { HTTPError } from "lambert-server"; -import { multer } from "../util/multer"; -import { storage } from "../util/Storage"; -import { CloudAttachment } from "../../util/entities/CloudAttachment"; import { fileTypeFromBuffer } from "file-type"; +import { cache, multer, storage } from "../util"; const router = Router({ mergeParams: true }); @@ -69,7 +67,7 @@ router.post("/:channel_id", multer.single("file"), async (req: Request, res: Res return res.json(file); }); -router.get("/:channel_id/:id/:filename", async (req: Request, res: Response) => { +router.get("/:channel_id/:id/:filename", cache, async (req: Request, res: Response) => { const { channel_id, id, filename } = req.params; // const { format } = req.query; @@ -100,7 +98,6 @@ router.get("/:channel_id/:id/:filename", async (req: Request, res: Response) => } res.set("Content-Type", content_type); - res.set("Cache-Control", "public, max-age=31536000"); return res.send(file); }); @@ -116,110 +113,4 @@ router.delete("/:channel_id/:id/:filename", async (req: Request, res: Response) return res.send({ success: true }); }); -// "cloud attachments" -router.put("/:channel_id/:batch_id/:attachment_id/:filename", multer.single("file"), async (req: Request, res: Response) => { - const { channel_id, batch_id, attachment_id, filename } = req.params; - const att = await CloudAttachment.findOneOrFail({ - where: { - uploadFilename: `${channel_id}/${batch_id}/${attachment_id}/${filename}`, - channelId: channel_id, - userAttachmentId: attachment_id, - userFilename: filename, - }, - }); - - const maxLength = Config.get().cdn.maxAttachmentSize; - - console.log("[Cloud Upload] Uploading attachment", att.id, att.userFilename, `Max size: ${maxLength} bytes`); - - const chunks: Buffer[] = []; - let length = 0; - - req.on("data", (chunk) => { - console.log(`[Cloud Upload] Received chunk of size ${chunk.length} bytes`); - chunks.push(chunk); - length += chunk.length; - if (length > maxLength) { - res.status(413).send("File too large"); - req.destroy(); - } - }); - req.on("end", async () => { - console.log(`[Cloud Upload] Finished receiving file, total size ${length} bytes`); - const buffer = Buffer.concat(chunks); - const path = `attachments/${channel_id}/${batch_id}/${attachment_id}/${filename}`; - - await storage.set(path, buffer); - - let mimeType = att.userOriginalContentType; - if (att.userOriginalContentType === null) { - const ft = await fileTypeFromBuffer(buffer); - mimeType = att.contentType = ft?.mime || "application/octet-stream"; - } - - if (mimeType?.includes("image")) { - const dimensions = imageSize(buffer); - if (dimensions) { - att.width = dimensions.width; - att.height = dimensions.height; - } - } - - att.size = buffer.length; - await att.save(); - - console.log("[Cloud Upload] Saved attachment", att.id, att.userFilename); - res.status(200).end(); - }); -}); - -router.delete("/:channel_id/:batch_id/:attachment_id/:filename", async (req: Request, res: Response) => { - if (req.headers.signature !== Config.get().security.requestSignature) throw new HTTPError("Invalid request signature"); - console.log("[Cloud Delete] Deleting attachment", req.params); - - const { channel_id, batch_id, attachment_id, filename } = req.params; - const path = `attachments/${channel_id}/${batch_id}/${attachment_id}/${filename}`; - - const att = await CloudAttachment.findOne({ - where: { - uploadFilename: `${channel_id}/${batch_id}/${attachment_id}/${filename}`, - channelId: channel_id, - userAttachmentId: attachment_id, - userFilename: filename, - }, - }); - - if (att) { - await att.remove(); - await storage.delete(path); - return res.send({ success: true }); - } - return res.status(404).send("Attachment not found"); -}); - -router.post("/:channel_id/:batch_id/:attachment_id/:filename/clone_to_message/:message_id", async (req: Request, res: Response) => { - if (req.headers.signature !== Config.get().security.requestSignature) throw new HTTPError("Invalid request signature"); - console.log("[Cloud Clone] Cloning attachment to message", req.params); - - const { channel_id, batch_id, attachment_id, filename, message_id } = req.params; - const path = `attachments/${channel_id}/${batch_id}/${attachment_id}/${filename}`; - const newPath = `attachments/${channel_id}/${message_id}/${filename}`; - - const att = await CloudAttachment.findOne({ - where: { - uploadFilename: `${channel_id}/${batch_id}/${attachment_id}/${filename}`, - channelId: channel_id, - userAttachmentId: attachment_id, - userFilename: filename, - }, - }); - - if (att) { - await storage.clone(path, newPath); - return res.send({ success: true, new_path: newPath }); - } - - return res.status(404).send("Attachment not found"); -}); - export default router; diff --git a/src/cdn/routes/avatars.ts b/src/cdn/routes/avatars.ts index c770007a7..4a433b23e 100644 --- a/src/cdn/routes/avatars.ts +++ b/src/cdn/routes/avatars.ts @@ -18,19 +18,16 @@ import { Router, Response, Request } from "express"; import { Config, Snowflake } from "@spacebar/util"; -import { storage } from "../util/Storage"; import { fileTypeFromBuffer } from "file-type"; import { HTTPError } from "lambert-server"; import crypto from "crypto"; -import { multer } from "../util/multer"; +import { multer, storage, cache, ANIMATED_MIME_TYPES, STATIC_MIME_TYPES } from "../util"; // TODO: check premium and animated pfp are allowed in the config // TODO: generate different sizes of icon // TODO: generate different image types of icon // TODO: delete old icons -const ANIMATED_MIME_TYPES = ["image/apng", "image/gif", "image/gifv"]; -const STATIC_MIME_TYPES = ["image/png", "image/jpeg", "image/webp", "image/svg+xml", "image/svg"]; const ALLOWED_MIME_TYPES = [...ANIMATED_MIME_TYPES, ...STATIC_MIME_TYPES]; const router = Router({ mergeParams: true }); @@ -60,7 +57,7 @@ router.post("/:user_id", multer.single("file"), async (req: Request, res: Respon }); }); -router.get("/:user_id", async (req: Request, res: Response) => { +router.get("/:user_id", cache, async (req: Request, res: Response) => { let { user_id } = req.params; user_id = user_id.split(".")[0]; // remove .file extension const path = `avatars/${user_id}`; @@ -70,12 +67,11 @@ router.get("/:user_id", async (req: Request, res: Response) => { const type = await fileTypeFromBuffer(file); res.set("Content-Type", type?.mime); - res.set("Cache-Control", "public, max-age=31536000"); return res.send(file); }); -export const getAvatar = async (req: Request, res: Response) => { +router.get("/:user_id/:hash", cache, async (req: Request, res: Response) => { const { user_id } = req.params; let { hash } = req.params; hash = hash.split(".")[0]; // remove .file extension @@ -86,12 +82,9 @@ export const getAvatar = async (req: Request, res: Response) => { const type = await fileTypeFromBuffer(file); res.set("Content-Type", type?.mime); - res.set("Cache-Control", "public, max-age=31536000"); return res.send(file); -}; - -router.get("/:user_id/:hash", getAvatar); +}); router.delete("/:user_id/:id", async (req: Request, res: Response) => { if (req.headers.signature !== Config.get().security.requestSignature) throw new HTTPError("Invalid request signature"); diff --git a/src/cdn/routes/badge-icons.ts b/src/cdn/routes/badge-icons.ts index ea7314a94..ac7c93853 100644 --- a/src/cdn/routes/badge-icons.ts +++ b/src/cdn/routes/badge-icons.ts @@ -17,13 +17,13 @@ */ import { Router, Response, Request } from "express"; -import { storage } from "../util/Storage"; import { HTTPError } from "lambert-server"; import { fileTypeFromBuffer } from "file-type"; +import { cache, storage } from "../util"; const router = Router({ mergeParams: true }); -router.get("/:badge_id", async (req: Request, res: Response) => { +router.get("/:badge_id", cache, async (req: Request, res: Response) => { const { badge_id } = req.params; const path = `badge-icons/${badge_id}`; @@ -32,7 +32,6 @@ router.get("/:badge_id", async (req: Request, res: Response) => { const type = await fileTypeFromBuffer(file); res.set("Content-Type", type?.mime); - res.set("Cache-Control", "public, max-age=31536000, must-revalidate"); return res.send(file); }); diff --git a/src/cdn/routes/embed.ts b/src/cdn/routes/embed.ts index 2a08da5a9..f8e11ea45 100644 --- a/src/cdn/routes/embed.ts +++ b/src/cdn/routes/embed.ts @@ -21,6 +21,7 @@ import fs from "fs/promises"; import { HTTPError } from "lambert-server"; import { join } from "path"; import { fileTypeFromBuffer } from "file-type"; +import { cache } from "../util"; const defaultAvatarHashMap = new Map([ ["0", "4a8562cf00887030c416d3ec2d46385a"], @@ -46,6 +47,7 @@ const router = Router({ mergeParams: true }); async function getFile(path: string) { try { + console.log("[CDN/Embed.ts] Trying to read file:", path); return await fs.readFile(path); } catch (error) { try { @@ -58,7 +60,7 @@ async function getFile(path: string) { } } -router.get("/avatars/:id", async (req: Request, res: Response) => { +router.get("/avatars/:id", cache, async (req: Request, res: Response) => { let { id } = req.params; id = id.split(".")[0]; // remove .file extension const hash = defaultAvatarHashMap.get(id); @@ -70,12 +72,11 @@ router.get("/avatars/:id", async (req: Request, res: Response) => { const type = await fileTypeFromBuffer(file); res.set("Content-Type", type?.mime); - res.set("Cache-Control", "public, max-age=31536000"); return res.send(file); }); -router.get("/group-avatars/:id", async (req: Request, res: Response) => { +router.get("/group-avatars/:id", cache, async (req: Request, res: Response) => { let { id } = req.params; id = id.split(".")[0]; // remove .file extension const hash = defaultGroupDMAvatarHashMap.get(id); @@ -87,7 +88,6 @@ router.get("/group-avatars/:id", async (req: Request, res: Response) => { const type = await fileTypeFromBuffer(file); res.set("Content-Type", type?.mime); - res.set("Cache-Control", "public, max-age=31536000"); return res.send(file); }); diff --git a/src/cdn/routes/guild-profiles.ts b/src/cdn/routes/guild-profiles.ts index 6660b7208..3c9eb5fbf 100644 --- a/src/cdn/routes/guild-profiles.ts +++ b/src/cdn/routes/guild-profiles.ts @@ -21,16 +21,14 @@ import crypto from "crypto"; import { Request, Response, Router } from "express"; import { HTTPError } from "lambert-server"; import { multer } from "../util/multer"; -import { storage } from "../util/Storage"; import { fileTypeFromBuffer } from "file-type"; +import { ANIMATED_MIME_TYPES, cache, STATIC_MIME_TYPES, storage } from "../util"; // TODO: check premium and animated pfp are allowed in the config // TODO: generate different sizes of icon // TODO: generate different image types of icon // TODO: delete old icons -const ANIMATED_MIME_TYPES = ["image/apng", "image/gif", "image/gifv"]; -const STATIC_MIME_TYPES = ["image/png", "image/jpeg", "image/webp", "image/svg+xml", "image/svg"]; const ALLOWED_MIME_TYPES = [...ANIMATED_MIME_TYPES, ...STATIC_MIME_TYPES]; const router = Router({ mergeParams: true }); @@ -60,7 +58,7 @@ router.post("/", multer.single("file"), async (req: Request, res: Response) => { }); }); -router.get("/", async (req: Request, res: Response) => { +router.get("/", cache, async (req: Request, res: Response) => { const { guild_id } = req.params; let { user_id } = req.params; user_id = user_id.split(".")[0]; // remove .file extension @@ -71,12 +69,11 @@ router.get("/", async (req: Request, res: Response) => { const type = await fileTypeFromBuffer(file); res.set("Content-Type", type?.mime); - res.set("Cache-Control", "public, max-age=31536000"); return res.send(file); }); -router.get("/:hash", async (req: Request, res: Response) => { +router.get("/:hash", cache, async (req: Request, res: Response) => { const { guild_id, user_id } = req.params; let { hash } = req.params; hash = hash.split(".")[0]; // remove .file extension @@ -87,7 +84,6 @@ router.get("/:hash", async (req: Request, res: Response) => { const type = await fileTypeFromBuffer(file); res.set("Content-Type", type?.mime); - res.set("Cache-Control", "public, max-age=31536000"); return res.send(file); }); diff --git a/src/cdn/routes/role-icons.ts b/src/cdn/routes/role-icons.ts index 1e00f9877..f987479dc 100644 --- a/src/cdn/routes/role-icons.ts +++ b/src/cdn/routes/role-icons.ts @@ -18,11 +18,10 @@ import { Router, Response, Request } from "express"; import { Config, Snowflake } from "@spacebar/util"; -import { storage } from "../util/Storage"; import { fileTypeFromBuffer } from "file-type"; import { HTTPError } from "lambert-server"; import crypto from "crypto"; -import { multer } from "../util/multer"; +import { cache, multer, STATIC_MIME_TYPES, storage } from "../util"; //Role icons ---> avatars.ts modified @@ -30,7 +29,6 @@ import { multer } from "../util/multer"; // TODO: generate different sizes of icon // TODO: generate different image types of icon -const STATIC_MIME_TYPES = ["image/png", "image/jpeg", "image/webp", "image/svg+xml", "image/svg"]; const ALLOWED_MIME_TYPES = [...STATIC_MIME_TYPES]; const router = Router({ mergeParams: true }); @@ -59,7 +57,7 @@ router.post("/:role_id", multer.single("file"), async (req: Request, res: Respon }); }); -router.get("/:role_id", async (req: Request, res: Response) => { +router.get("/:role_id", cache, async (req: Request, res: Response) => { const { role_id } = req.params; //role_id = role_id.split(".")[0]; // remove .file extension const path = `role-icons/${role_id}`; @@ -69,12 +67,11 @@ router.get("/:role_id", async (req: Request, res: Response) => { const type = await fileTypeFromBuffer(file); res.set("Content-Type", type?.mime); - res.set("Cache-Control", "public, max-age=31536000, must-revalidate"); return res.send(file); }); -router.get("/:role_id/:hash", async (req: Request, res: Response) => { +router.get("/:role_id/:hash", cache, async (req: Request, res: Response) => { const { role_id, hash } = req.params; //hash = hash.split(".")[0]; // remove .file extension const requested_extension = hash.split(".")[1]; @@ -92,7 +89,6 @@ router.get("/:role_id/:hash", async (req: Request, res: Response) => { const type = await fileTypeFromBuffer(file); res.set("Content-Type", type?.mime); - res.set("Cache-Control", "public, max-age=31536000, must-revalidate"); return res.send(file); }); diff --git a/src/cdn/util/basicCrdFileRouter.ts b/src/cdn/util/basicCrdFileRouter.ts index 46fcf8823..b05f3dbe4 100644 --- a/src/cdn/util/basicCrdFileRouter.ts +++ b/src/cdn/util/basicCrdFileRouter.ts @@ -1,6 +1,6 @@ /* Spacebar: A FOSS re-implementation and extension of the Discord.com backend. - Copyright (C) 2023 Spacebar and Spacebar Contributors + Copyright (C) 2025 Spacebar and Spacebar Contributors This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as published @@ -18,19 +18,16 @@ import { Router, Response, Request } from "express"; import { Config, Snowflake } from "@spacebar/util"; -import { storage } from "../util/Storage"; import { fileTypeFromBuffer } from "file-type"; import { HTTPError } from "lambert-server"; import crypto from "crypto"; -import { multer } from "../util/multer"; +import { storage, multer, ANIMATED_MIME_TYPES, STATIC_MIME_TYPES } from "../util"; // TODO: check premium and animated pfp are allowed in the config // TODO: generate different sizes of icon // TODO: generate different image types of icon // TODO: delete old icons -const ANIMATED_MIME_TYPES = ["image/apng", "image/gif", "image/gifv"]; -const STATIC_MIME_TYPES = ["image/png", "image/jpeg", "image/webp", "image/svg+xml", "image/svg"]; const ALLOWED_MIME_TYPES = [...ANIMATED_MIME_TYPES, ...STATIC_MIME_TYPES]; export class BasicCrdFileRouterOptions { @@ -44,6 +41,7 @@ export class BasicCrdFileRouterOptions { export function createBasicCrdFileRouter(opts: BasicCrdFileRouterOptions) { const router = Router({ mergeParams: true }); + console.log("Creating Basic CRD File Router with opts:", JSON.stringify(opts)); router.post("/:user_id", multer.single("file"), async (req: Request, res: Response) => { if (req.headers.signature !== Config.get().security.requestSignature) throw new HTTPError("Invalid request signature"); diff --git a/src/cdn/util/cache.ts b/src/cdn/util/cache.ts new file mode 100644 index 000000000..c48087dd9 --- /dev/null +++ b/src/cdn/util/cache.ts @@ -0,0 +1,7 @@ +import { NextFunction, Response, Request } from "express"; + +export function cache(req: Request, res: Response, next: NextFunction) { + const durationInSeconds = 21600; // 6 hours + res.set("Cache-Control", `public, max-age=${durationInSeconds}, s-maxage=${durationInSeconds}, immutable`); + next(); +} diff --git a/src/cdn/util/FileStorage.ts b/src/cdn/util/fileStorage.ts similarity index 87% rename from src/cdn/util/FileStorage.ts rename to src/cdn/util/fileStorage.ts index 6f01b3c5c..e21182287 100644 --- a/src/cdn/util/FileStorage.ts +++ b/src/cdn/util/fileStorage.ts @@ -16,7 +16,7 @@ along with this program. If not, see . */ -import { Storage } from "./Storage"; +import { Storage } from "."; import fs from "fs"; import fsp from "fs/promises"; import { join, dirname } from "path"; @@ -36,6 +36,7 @@ function getPath(path: string) { export class FileStorage implements Storage { async get(path: string): Promise { + if (process.env.CDN_LOG_IO) console.log(`[CDN][IO] read file: ${path}`); path = getPath(path); try { return await fsp.readFile(path); @@ -51,6 +52,7 @@ export class FileStorage implements Storage { } async clone(path: string, newPath: string) { + if (process.env.CDN_LOG_IO) console.log(`[CDN][IO] clone file: ${path} -> ${newPath}`); path = getPath(path); newPath = getPath(newPath); @@ -61,6 +63,7 @@ export class FileStorage implements Storage { } async set(path: string, value: Buffer) { + if (process.env.CDN_LOG_IO) console.log(`[CDN][IO] write file: ${path}`); path = getPath(path); if (!fs.existsSync(dirname(path))) fs.mkdirSync(dirname(path), { recursive: true }); @@ -71,6 +74,7 @@ export class FileStorage implements Storage { } async delete(path: string) { + if (process.env.CDN_LOG_IO) console.log(`[CDN][IO] delete file: ${path}`); //TODO we should delete the parent directory if empty fs.unlinkSync(getPath(path)); } diff --git a/src/cdn/util/index.ts b/src/cdn/util/index.ts index ba249845c..9d58117b1 100644 --- a/src/cdn/util/index.ts +++ b/src/cdn/util/index.ts @@ -16,6 +16,9 @@ along with this program. If not, see . */ -export * from "./FileStorage"; +export * from "./storage"; +export * from "./fileStorage"; +export * from "./s3Storage"; export * from "./multer"; -export * from "./Storage"; +export * from "./cache"; +export * from "./mimetypes"; diff --git a/src/cdn/util/mimetypes.ts b/src/cdn/util/mimetypes.ts new file mode 100644 index 000000000..c56bd7341 --- /dev/null +++ b/src/cdn/util/mimetypes.ts @@ -0,0 +1,2 @@ +export const ANIMATED_MIME_TYPES = ["image/apng", "image/gif", "image/gifv"]; +export const STATIC_MIME_TYPES = ["image/png", "image/jpeg", "image/webp", "image/svg+xml", "image/svg"]; diff --git a/src/cdn/util/S3Storage.ts b/src/cdn/util/s3Storage.ts similarity index 89% rename from src/cdn/util/S3Storage.ts rename to src/cdn/util/s3Storage.ts index f657fd299..2827e974b 100644 --- a/src/cdn/util/S3Storage.ts +++ b/src/cdn/util/s3Storage.ts @@ -17,7 +17,7 @@ */ import { Readable } from "stream"; -import { Storage } from "./Storage"; +import { Storage } from "."; const readableToBuffer = (readable: Readable): Promise => new Promise((resolve, reject) => { @@ -46,6 +46,7 @@ export class S3Storage implements Storage { } async set(path: string, data: Buffer): Promise { + if (process.env.CDN_LOG_IO) console.log(`[CDN][IO] write file: ${path}`); // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-expect-error await this.client.putObject({ @@ -56,6 +57,7 @@ export class S3Storage implements Storage { } async clone(path: string, newPath: string): Promise { + if (process.env.CDN_LOG_IO) console.log(`[CDN][IO] clone file: ${path} -> ${newPath}`); // TODO: does this even work? // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-expect-error @@ -67,6 +69,7 @@ export class S3Storage implements Storage { } async get(path: string): Promise { + if (process.env.CDN_LOG_IO) console.log(`[CDN][IO] read file: ${path}`); try { // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-expect-error @@ -88,6 +91,7 @@ export class S3Storage implements Storage { } async delete(path: string): Promise { + if (process.env.CDN_LOG_IO) console.log(`[CDN][IO] delete file: ${path}`); // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-expect-error await this.client.deleteObject({ diff --git a/src/cdn/util/Storage.ts b/src/cdn/util/storage.ts similarity index 94% rename from src/cdn/util/Storage.ts rename to src/cdn/util/storage.ts index 1184bde8d..f9d9d6f97 100644 --- a/src/cdn/util/Storage.ts +++ b/src/cdn/util/storage.ts @@ -16,7 +16,8 @@ along with this program. If not, see . */ -import { FileStorage } from "./FileStorage"; +import { FileStorage } from "./fileStorage"; +import { S3Storage } from "./s3Storage"; import path from "path"; import fs from "fs"; import { red } from "picocolors"; @@ -39,7 +40,7 @@ if (process.env.STORAGE_PROVIDER === "file" || !process.env.STORAGE_PROVIDER) { location = path.join(process.cwd(), "files"); } // TODO: move this to some start func, so it doesn't run when server is imported - //console.log(`[CDN] storage location: ${bgCyan(`${black(location)}`)}`); + // console.log(`[CDN] storage location: ${bgCyan(`${black(location)}`)}`); if (!fs.existsSync(location)) fs.mkdirSync(location); process.env.STORAGE_LOCATION = location; diff --git a/src/util/config/types/CdnConfiguration.ts b/src/util/config/types/CdnConfiguration.ts index a78af5a98..b8efd6aa7 100644 --- a/src/util/config/types/CdnConfiguration.ts +++ b/src/util/config/types/CdnConfiguration.ts @@ -25,11 +25,10 @@ export class CdnConfiguration extends EndpointConfiguration { proxyCacheHeaderSeconds: number = 60 * 60 * 24; maxAttachmentSize: number = 25 * 1024 * 1024; // 25 MB - // limits: CdnLimitsConfiguration = new CdnLimitsConfiguration(); + limits: CdnLimitsConfiguration = new CdnLimitsConfiguration(); } export class CdnLimitsConfiguration { - // ordered by route register order in CDN... icon: CdnImageLimitsConfiguration = new CdnImageLimitsConfiguration(); roleIcon: CdnImageLimitsConfiguration = new CdnImageLimitsConfiguration(); emoji: CdnImageLimitsConfiguration = new CdnImageLimitsConfiguration(); @@ -46,6 +45,12 @@ export class CdnLimitsConfiguration { } export class CdnImageLimitsConfiguration { + constructor(data?: Partial) { + if (data) { + Object.assign(this, data); + } + } + maxHeight: number = 8192; maxWidth: number = 8192; maxSize: number = 10 * 1024 * 1024; // 10 MB