From 09f81382f18dd0b145ea1f52289922cf11af1b2f Mon Sep 17 00:00:00 2001 From: Rory& Date: Mon, 23 Feb 2026 08:08:41 +0100 Subject: [PATCH] cdn: migrate emojis to not use subdir fallback --- src/cdn/routes/emojis.ts | 56 ++++++++++++++++++++----------------- src/cdn/util/FileStorage.ts | 39 ++++++++++++++------------ src/cdn/util/S3Storage.ts | 3 ++ src/cdn/util/Storage.ts | 1 + 4 files changed, 56 insertions(+), 43 deletions(-) diff --git a/src/cdn/routes/emojis.ts b/src/cdn/routes/emojis.ts index b0e95f6f5..dec95998f 100644 --- a/src/cdn/routes/emojis.ts +++ b/src/cdn/routes/emojis.ts @@ -24,6 +24,8 @@ import { HTTPError } from "lambert-server"; import crypto from "crypto"; import { multer } from "../util/multer"; import { cache } from "../util/cache"; +import { FileStorage } from "@spacebar/cdn*"; +import fs from "fs/promises"; // TODO: check premium and animated pfp are allowed in the config // TODO: generate different sizes of icon @@ -37,11 +39,11 @@ const ALLOWED_MIME_TYPES = [...ANIMATED_MIME_TYPES, ...STATIC_MIME_TYPES]; const router = Router({ mergeParams: true }); const pathPrefix = "emojis"; -router.post("/:guild_id", multer.single("file"), async (req: Request, res: Response) => { +router.post("/:emoji_id", multer.single("file"), async (req: Request, res: Response) => { if (req.headers.signature !== Config.get().security.requestSignature) throw new HTTPError("Invalid request signature"); if (!req.file) throw new HTTPError("Missing file"); const { buffer, size } = req.file; - const { guild_id } = req.params as { [key: string]: string }; + const { emoji_id } = req.params as { [key: string]: string }; let hash = crypto.createHash("md5").update(Snowflake.generate()).digest("hex"); @@ -49,7 +51,7 @@ router.post("/:guild_id", multer.single("file"), async (req: Request, res: Respo if (!type || !ALLOWED_MIME_TYPES.includes(type.mime)) throw new HTTPError("Invalid file type"); if (ANIMATED_MIME_TYPES.includes(type.mime)) hash = `a_${hash}`; // animated icons have a_ infront of the hash - const path = `${pathPrefix}/${guild_id}/${hash}`; + const path = `${pathPrefix}/${emoji_id}`; const endpoint = Config.get().cdn.endpointPublic; await storage.set(path, buffer); @@ -58,16 +60,16 @@ router.post("/:guild_id", multer.single("file"), async (req: Request, res: Respo id: hash, content_type: type.mime, size, - url: `${endpoint}${req.baseUrl}/${guild_id}/${hash}`, + url: `${endpoint}${req.baseUrl}/${emoji_id}`, }); }); router.get("/:emoji_id", cache, async (req: Request, res: Response) => { - let { guild_id } = req.params as { [key: string]: string }; - guild_id = guild_id.split(".")[0]; // remove .file extension - const path = `${pathPrefix}/${guild_id}`; + let { emoji_id } = req.params as { [key: string]: string }; + emoji_id = emoji_id.split(".")[0]; // remove .file extension + const path = `${pathPrefix}/${emoji_id}`; - const file = await getOrMoveFile(path, `avatars/${guild_id}`); + const file = await getOrMoveFile(path, `avatars/${emoji_id}`); const type = await fileTypeFromBuffer(file); res.set("Content-Type", type?.mime); @@ -75,24 +77,10 @@ router.get("/:emoji_id", cache, async (req: Request, res: Response) => { return res.send(file); }); -export const getAvatar = async (req: Request, res: Response) => { - const { guild_id } = req.params as { [key: string]: string }; - let { hash } = req.params as { [key: string]: string }; - hash = hash.split(".")[0]; // remove .file extension - const path = `${pathPrefix}/${guild_id}/${hash}`; - - const file = await getOrMoveFile(path, `avatars/${guild_id}/${hash}`); - const type = await fileTypeFromBuffer(file); - - res.set("Content-Type", type?.mime); - - return res.send(file); -}; - -router.delete("/:guild_id/:id", async (req: Request, res: Response) => { +router.delete("/:emoji_id/:id", async (req: Request, res: Response) => { if (req.headers.signature !== Config.get().security.requestSignature) throw new HTTPError("Invalid request signature"); const { guild_id, id } = req.params as { [key: string]: string }; - const path = `${pathPrefix}/${guild_id}/${id}`; + const path = `${pathPrefix}/${emoji_id}/${id}`; await storage.delete(path); @@ -101,7 +89,25 @@ router.delete("/:guild_id/:id", async (req: Request, res: Response) => { async function getOrMoveFile(newPath: string, oldPath: string) { let file = await storage.get(newPath); - if (!file) { + if (file) { + if (!(await storage.isFile(newPath))) { + console.log(`[CDN] Migrating emoji from subdirectory+fallback to direct path for ${newPath}`); + // noinspection SuspiciousTypeOfGuard -- not sure whats up with this + if (storage instanceof FileStorage) { + const files = await fs.readdir(storage.getFsPath(newPath)); + if (files.length === 1) { + const oldFilePath = storage.getFsPath(`${newPath}/${files[0]}`); + const newFilePath = storage.getFsPath(newPath); + await fs.rename(oldFilePath, newFilePath + ".tmp"); + await fs.rmdir(storage.getFsPath(newPath)); + await fs.rename(newFilePath + ".tmp", newFilePath); + file = await storage.get(newPath); + } else console.log(`[CDN] Warning: not migrating emojis ${newPath}, as there are multiple files in the old directory`); + } else { + console.log("[CDN] Warning: no migration for s3 storage emojis, as it is not a filesystem"); + } + } + } else { if (await storage.exists(oldPath)) { console.log(`[${pathPrefix}] found file at old path ${oldPath}, moving to new path ${newPath}`); await storage.move(oldPath, newPath); diff --git a/src/cdn/util/FileStorage.ts b/src/cdn/util/FileStorage.ts index ff7a56ee8..e56f5fd8f 100644 --- a/src/cdn/util/FileStorage.ts +++ b/src/cdn/util/FileStorage.ts @@ -24,23 +24,26 @@ import { Readable } from "stream"; import ExifTransformer from "exif-be-gone"; // TODO: split stored files into separate folders named after cloned route - -function getPath(path: string) { - // STORAGE_LOCATION has a default value in start.ts - const root = process.env.STORAGE_LOCATION || "../"; - const filename = join(root, path); - - if (path.indexOf("\0") !== -1 || !filename.startsWith(root)) throw new Error("invalid path"); - return filename; -} - export class FileStorage implements Storage { + getFsPath(path: string): string { + // STORAGE_LOCATION has a default value in start.ts + const root = process.env.STORAGE_LOCATION || "../"; + const filename = join(root, path); + + if (path.indexOf("\0") !== -1 || !filename.startsWith(root)) throw new Error("invalid path"); + return filename; + } + isFile(path: string): Promise { + return Promise.resolve(fs.statSync(path).isFile()); + } + async get(path: string): Promise { - path = getPath(path); + path = this.getFsPath(path); try { return await fsp.readFile(path); } catch (error) { try { + console.warn("[CDN] Warning: falling back to first file in dir for path", path); const files = fs.readdirSync(path); if (!files.length) return null; return await fsp.readFile(join(path, files[0])); @@ -51,8 +54,8 @@ export class FileStorage implements Storage { } async clone(path: string, newPath: string) { - path = getPath(path); - newPath = getPath(newPath); + path = this.getFsPath(path); + newPath = this.getFsPath(newPath); if (!fs.existsSync(dirname(newPath))) fs.mkdirSync(dirname(newPath), { recursive: true }); @@ -61,7 +64,7 @@ export class FileStorage implements Storage { } async set(path: string, value: Buffer) { - path = getPath(path); + path = this.getFsPath(path); if (!fs.existsSync(dirname(path))) fs.mkdirSync(dirname(path), { recursive: true }); const ret = Readable.from(value); @@ -72,16 +75,16 @@ export class FileStorage implements Storage { async delete(path: string) { //TODO we should delete the parent directory if empty - fs.unlinkSync(getPath(path)); + fs.unlinkSync(this.getFsPath(path)); } async exists(path: string) { - return fs.existsSync(getPath(path)); + return fs.existsSync(this.getFsPath(path)); } async move(path: string, newPath: string) { - path = getPath(path); - newPath = getPath(newPath); + path = this.getFsPath(path); + newPath = this.getFsPath(newPath); if (!fs.existsSync(dirname(newPath))) fs.mkdirSync(dirname(newPath), { recursive: true }); diff --git a/src/cdn/util/S3Storage.ts b/src/cdn/util/S3Storage.ts index de575e5d2..82d9954ff 100644 --- a/src/cdn/util/S3Storage.ts +++ b/src/cdn/util/S3Storage.ts @@ -38,6 +38,9 @@ export class S3Storage implements Storage { const { S3 } = require("@aws-sdk/client-s3"); this.client = new S3({ region: region, endpoint: endpoint }); } + isFile(path: string): Promise { + return this.exists(path); + } /** * Always return a string, to ensure consistency. diff --git a/src/cdn/util/Storage.ts b/src/cdn/util/Storage.ts index 199c53ff7..03d71d1d9 100644 --- a/src/cdn/util/Storage.ts +++ b/src/cdn/util/Storage.ts @@ -28,6 +28,7 @@ export interface Storage { get(path: string): Promise; delete(path: string): Promise; exists(path: string): Promise; + isFile(path: string): Promise; move(path: string, newPath: string): Promise; }