dandri/server
1
0
Fork 0
mirror of https://github.com/spacebarchat/server.git synced 2026-05-14 18:25:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

cdn: migrate emojis to not use subdir fallback

Browse Source
This commit is contained in:
Rory&
2026-02-23 08:08:41 +01:00
parent 7f5fae417a
commit 09f81382f1
4 changed files with 56 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/cdn/routes/emojis.ts
+31 -25
View File
@@ -24,6 +24,8 @@ import { HTTPError } from "lambert-server";
import crypto from "crypto";
import { multer } from "../util/multer";
import { cache } from "../util/cache";
import { FileStorage } from "@spacebar/cdn*";
import fs from "fs/promises";
// TODO: check premium and animated pfp are allowed in the config
// TODO: generate different sizes of icon
@@ -37,11 +39,11 @@ const ALLOWED_MIME_TYPES = [...ANIMATED_MIME_TYPES, ...STATIC_MIME_TYPES];
const router = Router({ mergeParams: true });
const pathPrefix = "emojis";
router.post("/:guild_id", multer.single("file"), async (req: Request, res: Response) => {
router.post("/:emoji_id", multer.single("file"), async (req: Request, res: Response) => {
if (req.headers.signature !== Config.get().security.requestSignature) throw new HTTPError("Invalid request signature");
if (!req.file) throw new HTTPError("Missing file");
const { buffer, size } = req.file;
const { guild_id } = req.params as { [key: string]: string };
const { emoji_id } = req.params as { [key: string]: string };
let hash = crypto.createHash("md5").update(Snowflake.generate()).digest("hex");
@@ -49,7 +51,7 @@ router.post("/:guild_id", multer.single("file"), async (req: Request, res: Respo
if (!type || !ALLOWED_MIME_TYPES.includes(type.mime)) throw new HTTPError("Invalid file type");
if (ANIMATED_MIME_TYPES.includes(type.mime)) hash = `a_${hash}`; // animated icons have a_ infront of the hash
const path = `${pathPrefix}/${guild_id}/${hash}`;
const path = `${pathPrefix}/${emoji_id}`;
const endpoint = Config.get().cdn.endpointPublic;
await storage.set(path, buffer);
@@ -58,16 +60,16 @@ router.post("/:guild_id", multer.single("file"), async (req: Request, res: Respo
id: hash,
content_type: type.mime,
size,
url: `${endpoint}${req.baseUrl}/${guild_id}/${hash}`,
url: `${endpoint}${req.baseUrl}/${emoji_id}`,
});
});
router.get("/:emoji_id", cache, async (req: Request, res: Response) => {
let { guild_id } = req.params as { [key: string]: string };
guild_id = guild_id.split(".")[0]; // remove .file extension
const path = `${pathPrefix}/${guild_id}`;
let { emoji_id } = req.params as { [key: string]: string };
emoji_id = emoji_id.split(".")[0]; // remove .file extension
const path = `${pathPrefix}/${emoji_id}`;
const file = await getOrMoveFile(path, `avatars/${guild_id}`);
const file = await getOrMoveFile(path, `avatars/${emoji_id}`);
const type = await fileTypeFromBuffer(file);
res.set("Content-Type", type?.mime);
@@ -75,24 +77,10 @@ router.get("/:emoji_id", cache, async (req: Request, res: Response) => {
return res.send(file);
});
export const getAvatar = async (req: Request, res: Response) => {
const { guild_id } = req.params as { [key: string]: string };
let { hash } = req.params as { [key: string]: string };
hash = hash.split(".")[0]; // remove .file extension
const path = `${pathPrefix}/${guild_id}/${hash}`;
const file = await getOrMoveFile(path, `avatars/${guild_id}/${hash}`);
const type = await fileTypeFromBuffer(file);
res.set("Content-Type", type?.mime);
return res.send(file);
};
router.delete("/:guild_id/:id", async (req: Request, res: Response) => {
router.delete("/:emoji_id/:id", async (req: Request, res: Response) => {
if (req.headers.signature !== Config.get().security.requestSignature) throw new HTTPError("Invalid request signature");
const { guild_id, id } = req.params as { [key: string]: string };
const path = `${pathPrefix}/${guild_id}/${id}`;
const path = `${pathPrefix}/${emoji_id}/${id}`;
await storage.delete(path);
@@ -101,7 +89,25 @@ router.delete("/:guild_id/:id", async (req: Request, res: Response) => {
async function getOrMoveFile(newPath: string, oldPath: string) {
let file = await storage.get(newPath);
if (!file) {
if (file) {
if (!(await storage.isFile(newPath))) {
console.log(`[CDN] Migrating emoji from subdirectory+fallback to direct path for ${newPath}`);
// noinspection SuspiciousTypeOfGuard -- not sure whats up with this
if (storage instanceof FileStorage) {
const files = await fs.readdir(storage.getFsPath(newPath));
if (files.length === 1) {
const oldFilePath = storage.getFsPath(`${newPath}/${files[0]}`);
const newFilePath = storage.getFsPath(newPath);
await fs.rename(oldFilePath, newFilePath + ".tmp");
await fs.rmdir(storage.getFsPath(newPath));
await fs.rename(newFilePath + ".tmp", newFilePath);
file = await storage.get(newPath);
} else console.log(`[CDN] Warning: not migrating emojis ${newPath}, as there are multiple files in the old directory`);
} else {
console.log("[CDN] Warning: no migration for s3 storage emojis, as it is not a filesystem");
}
}
} else {
if (await storage.exists(oldPath)) {
console.log(`[${pathPrefix}] found file at old path ${oldPath}, moving to new path ${newPath}`);
await storage.move(oldPath, newPath);
src/cdn/util/FileStorage.ts
+21 -18
View File
@@ -24,23 +24,26 @@ import { Readable } from "stream";
import ExifTransformer from "exif-be-gone";
// TODO: split stored files into separate folders named after cloned route
function getPath(path: string) {
// STORAGE_LOCATION has a default value in start.ts
const root = process.env.STORAGE_LOCATION || "../";
const filename = join(root, path);
if (path.indexOf("\0") !== -1 || !filename.startsWith(root)) throw new Error("invalid path");
return filename;
}
export class FileStorage implements Storage {
getFsPath(path: string): string {
// STORAGE_LOCATION has a default value in start.ts
const root = process.env.STORAGE_LOCATION || "../";
const filename = join(root, path);
if (path.indexOf("\0") !== -1 || !filename.startsWith(root)) throw new Error("invalid path");
return filename;
}
isFile(path: string): Promise<boolean> {
return Promise.resolve(fs.statSync(path).isFile());
}
async get(path: string): Promise<Buffer | null> {
path = getPath(path);
path = this.getFsPath(path);
try {
return await fsp.readFile(path);
} catch (error) {
try {
console.warn("[CDN] Warning: falling back to first file in dir for path", path);
const files = fs.readdirSync(path);
if (!files.length) return null;
return await fsp.readFile(join(path, files[0]));
@@ -51,8 +54,8 @@ export class FileStorage implements Storage {
}
async clone(path: string, newPath: string) {
path = getPath(path);
newPath = getPath(newPath);
path = this.getFsPath(path);
newPath = this.getFsPath(newPath);
if (!fs.existsSync(dirname(newPath))) fs.mkdirSync(dirname(newPath), { recursive: true });
@@ -61,7 +64,7 @@ export class FileStorage implements Storage {
}
async set(path: string, value: Buffer) {
path = getPath(path);
path = this.getFsPath(path);
if (!fs.existsSync(dirname(path))) fs.mkdirSync(dirname(path), { recursive: true });
const ret = Readable.from(value);
@@ -72,16 +75,16 @@ export class FileStorage implements Storage {
async delete(path: string) {
//TODO we should delete the parent directory if empty
fs.unlinkSync(getPath(path));
fs.unlinkSync(this.getFsPath(path));
}
async exists(path: string) {
return fs.existsSync(getPath(path));
return fs.existsSync(this.getFsPath(path));
}
async move(path: string, newPath: string) {
path = getPath(path);
newPath = getPath(newPath);
path = this.getFsPath(path);
newPath = this.getFsPath(newPath);
if (!fs.existsSync(dirname(newPath))) fs.mkdirSync(dirname(newPath), { recursive: true });
src/cdn/util/S3Storage.ts
+3
View File
@@ -38,6 +38,9 @@ export class S3Storage implements Storage {
const { S3 } = require("@aws-sdk/client-s3");
this.client = new S3({ region: region, endpoint: endpoint });
}
isFile(path: string): Promise<boolean> {
return this.exists(path);
}
/**
* Always return a string, to ensure consistency.
src/cdn/util/Storage.ts
+1
View File
@@ -28,6 +28,7 @@ export interface Storage {
get(path: string): Promise<Buffer | null>;
delete(path: string): Promise<void>;
exists(path: string): Promise<boolean>;
isFile(path: string): Promise<boolean>;
move(path: string, newPath: string): Promise<void>;
}