restructure to single project

This commit is contained in:
Flam3rboy
2022-08-13 22:00:55 +02:00
committed by TheArcaneBrony
parent 5abd6bb7e0
commit 5e86d7ab9c
626 changed files with 7256 additions and 2449 deletions
+80
View File
@@ -0,0 +1,80 @@
import { Server, ServerOptions } from "lambert-server";
import { Config, getOrInitialiseDatabase, registerRoutes } from "@fosscord/util";
import path from "path";
import avatarsRoute from "./routes/avatars";
import iconsRoute from "./routes/role-icons";
import bodyParser from "body-parser";
export interface CDNServerOptions extends ServerOptions {}
export class CDNServer extends Server {
public declare options: CDNServerOptions;
constructor(options?: Partial<CDNServerOptions>) {
super(options);
}
async start() {
await getOrInitialiseDatabase();
await Config.init();
this.app.use((req, res, next) => {
res.set("Access-Control-Allow-Origin", "*");
// TODO: use better CSP policy
res.set(
"Content-security-policy",
"default-src * data: blob: filesystem: about: ws: wss: 'unsafe-inline' 'unsafe-eval'; script-src * data: blob: 'unsafe-inline' 'unsafe-eval'; connect-src * data: blob: 'unsafe-inline'; img-src * data: blob: 'unsafe-inline'; frame-src * data: blob: ; style-src * data: blob: 'unsafe-inline'; font-src * data: blob: 'unsafe-inline';"
);
res.set(
"Access-Control-Allow-Headers",
req.header("Access-Control-Request-Headers") || "*"
);
res.set(
"Access-Control-Allow-Methods",
req.header("Access-Control-Request-Methods") || "*"
);
next();
});
this.app.use(bodyParser.json({ inflate: true, limit: "10mb" }));
await registerRoutes(this, path.join(__dirname, "routes/"));
this.app.use("/icons/", avatarsRoute);
this.log("verbose", "[Server] Route /icons registered");
this.app.use("/role-icons/", iconsRoute);
this.log("verbose", "[Server] Route /role-icons registered");
this.app.use("/emojis/", avatarsRoute);
this.log("verbose", "[Server] Route /emojis registered");
this.app.use("/stickers/", avatarsRoute);
this.log("verbose", "[Server] Route /stickers registered");
this.app.use("/banners/", avatarsRoute);
this.log("verbose", "[Server] Route /banners registered");
this.app.use("/splashes/", avatarsRoute);
this.log("verbose", "[Server] Route /splashes registered");
this.app.use("/app-icons/", avatarsRoute);
this.log("verbose", "[Server] Route /app-icons registered");
this.app.use("/app-assets/", avatarsRoute);
this.log("verbose", "[Server] Route /app-assets registered");
this.app.use("/discover-splashes/", avatarsRoute);
this.log("verbose", "[Server] Route /discover-splashes registered");
this.app.use("/team-icons/", avatarsRoute);
this.log("verbose", "[Server] Route /team-icons registered");
this.app.use("/channel-icons/", avatarsRoute);
this.log("verbose", "[Server] Route /channel-icons registered");
return super.start();
}
async stop() {
return super.stop();
}
}
+4
View File
@@ -0,0 +1,4 @@
export * from "./Server";
export * from "./util/FileStorage";
export * from "./util/Storage";
export * from "./util/multer";
+100
View File
@@ -0,0 +1,100 @@
import { Router, Response, Request } from "express";
import { Config, Snowflake } from "@fosscord/util";
import { storage } from "../util/Storage";
import FileType from "file-type";
import { HTTPError } from "@fosscord/util";
import { multer } from "../util/multer";
import imageSize from "image-size";
const router = Router();
const SANITIZED_CONTENT_TYPE = [
"text/html",
"text/mhtml",
"multipart/related",
"application/xhtml+xml",
];
router.post(
"/:channel_id",
multer.single("file"),
async (req: Request, res: Response) => {
if (req.headers.signature !== Config.get().security.requestSignature)
throw new HTTPError("Invalid request signature");
if (!req.file) throw new HTTPError("file missing");
const { buffer, mimetype, size, originalname, fieldname } = req.file;
const { channel_id } = req.params;
const filename = originalname
.replaceAll(" ", "_")
.replace(/[^a-zA-Z0-9._]+/g, "");
const id = Snowflake.generate();
const path = `attachments/${channel_id}/${id}/${filename}`;
const endpoint =
Config.get()?.cdn.endpointPublic || "http://localhost:3003";
await storage.set(path, buffer);
let width;
let height;
if (mimetype.includes("image")) {
const dimensions = imageSize(buffer);
if (dimensions) {
width = dimensions.width;
height = dimensions.height;
}
}
const file = {
id,
content_type: mimetype,
filename: filename,
size,
url: `${endpoint}/${path}`,
width,
height,
};
return res.json(file);
}
);
router.get(
"/:channel_id/:id/:filename",
async (req: Request, res: Response) => {
const { channel_id, id, filename } = req.params;
const file = await storage.get(
`attachments/${channel_id}/${id}/${filename}`
);
if (!file) throw new HTTPError("File not found");
const type = await FileType.fromBuffer(file);
let content_type = type?.mime || "application/octet-stream";
if (SANITIZED_CONTENT_TYPE.includes(content_type)) {
content_type = "application/octet-stream";
}
res.set("Content-Type", content_type);
res.set("Cache-Control", "public, max-age=31536000");
return res.send(file);
}
);
router.delete(
"/:channel_id/:id/:filename",
async (req: Request, res: Response) => {
if (req.headers.signature !== Config.get().security.requestSignature)
throw new HTTPError("Invalid request signature");
const { channel_id, id, filename } = req.params;
const path = `attachments/${channel_id}/${id}/${filename}`;
await storage.delete(path);
return res.send({ success: true });
}
);
export default router;
+102
View File
@@ -0,0 +1,102 @@
import { Router, Response, Request } from "express";
import { Config, Snowflake } from "@fosscord/util";
import { storage } from "../util/Storage";
import FileType from "file-type";
import { HTTPError } from "@fosscord/util";
import crypto from "crypto";
import { multer } from "../util/multer";
// TODO: check premium and animated pfp are allowed in the config
// TODO: generate different sizes of icon
// TODO: generate different image types of icon
// TODO: delete old icons
const ANIMATED_MIME_TYPES = ["image/apng", "image/gif", "image/gifv"];
const STATIC_MIME_TYPES = [
"image/png",
"image/jpeg",
"image/webp",
"image/svg+xml",
"image/svg",
];
const ALLOWED_MIME_TYPES = [...ANIMATED_MIME_TYPES, ...STATIC_MIME_TYPES];
const router = Router();
router.post(
"/:user_id",
multer.single("file"),
async (req: Request, res: Response) => {
if (req.headers.signature !== Config.get().security.requestSignature)
throw new HTTPError("Invalid request signature");
if (!req.file) throw new HTTPError("Missing file");
const { buffer, mimetype, size, originalname, fieldname } = req.file;
const { user_id } = req.params;
let hash = crypto
.createHash("md5")
.update(Snowflake.generate())
.digest("hex");
const type = await FileType.fromBuffer(buffer);
if (!type || !ALLOWED_MIME_TYPES.includes(type.mime))
throw new HTTPError("Invalid file type");
if (ANIMATED_MIME_TYPES.includes(type.mime)) hash = `a_${hash}`; // animated icons have a_ infront of the hash
const path = `avatars/${user_id}/${hash}`;
const endpoint =
Config.get().cdn.endpointPublic || "http://localhost:3003";
await storage.set(path, buffer);
return res.json({
id: hash,
content_type: type.mime,
size,
url: `${endpoint}${req.baseUrl}/${user_id}/${hash}`,
});
}
);
router.get("/:user_id", async (req: Request, res: Response) => {
let { user_id } = req.params;
user_id = user_id.split(".")[0]; // remove .file extension
const path = `avatars/${user_id}`;
const file = await storage.get(path);
if (!file) throw new HTTPError("not found", 404);
const type = await FileType.fromBuffer(file);
res.set("Content-Type", type?.mime);
res.set("Cache-Control", "public, max-age=31536000");
return res.send(file);
});
router.get("/:user_id/:hash", async (req: Request, res: Response) => {
let { user_id, hash } = req.params;
hash = hash.split(".")[0]; // remove .file extension
const path = `avatars/${user_id}/${hash}`;
const file = await storage.get(path);
if (!file) throw new HTTPError("not found", 404);
const type = await FileType.fromBuffer(file);
res.set("Content-Type", type?.mime);
res.set("Cache-Control", "public, max-age=31536000");
return res.send(file);
});
router.delete("/:user_id/:id", async (req: Request, res: Response) => {
if (req.headers.signature !== Config.get().security.requestSignature)
throw new HTTPError("Invalid request signature");
const { user_id, id } = req.params;
const path = `avatars/${user_id}/${id}`;
await storage.delete(path);
return res.send({ success: true });
});
export default router;
+58
View File
@@ -0,0 +1,58 @@
import { Router, Response, Request } from "express";
import fetch from "node-fetch";
import { HTTPError } from "@fosscord/util";
import { Snowflake, Config } from "@fosscord/util";
import { storage } from "../util/Storage";
import FileType from "file-type";
// TODO: somehow handle the deletion of images posted to the /external route
const router = Router();
const DEFAULT_FETCH_OPTIONS: any = {
redirect: "follow",
follow: 1,
headers: {
"user-agent":
"Mozilla/5.0 (compatible Fosscordbot/0.1; +https://fosscord.com)",
},
size: 1024 * 1024 * 8,
compress: true,
method: "GET",
};
router.post("/", async (req: Request, res: Response) => {
if (req.headers.signature !== Config.get().security.requestSignature)
throw new HTTPError("Invalid request signature");
if (!req.body) throw new HTTPError("Invalid Body");
const { url } = req.body;
if (!url || typeof url !== "string") throw new HTTPError("Invalid url");
const id = Snowflake.generate();
try {
const response = await fetch(url, DEFAULT_FETCH_OPTIONS);
const buffer = await response.buffer();
await storage.set(`/external/${id}`, buffer);
res.send({ id });
} catch (error) {
throw new HTTPError("Couldn't fetch website");
}
});
router.get("/:id", async (req: Request, res: Response) => {
const { id } = req.params;
const file = await storage.get(`/external/${id}`);
if (!file) throw new HTTPError("File not found");
const result = await FileType.fromBuffer(file);
res.set("Content-Type", result?.mime);
return res.send(file);
});
export default router;
+9
View File
@@ -0,0 +1,9 @@
import { Router, Response, Request } from "express";
const router = Router();
router.get("/", (req: Request, res: Response) => {
res.send("pong");
});
export default router;
+101
View File
@@ -0,0 +1,101 @@
import { Router, Response, Request } from "express";
import { Config, Snowflake } from "@fosscord/util";
import { storage } from "../util/Storage";
import FileType from "file-type";
import { HTTPError } from "@fosscord/util";
import crypto from "crypto";
import { multer } from "../util/multer";
//Role icons ---> avatars.ts modified
// TODO: check user rights and perks and animated pfp are allowed in the policies
// TODO: generate different sizes of icon
// TODO: generate different image types of icon
const STATIC_MIME_TYPES = [
"image/png",
"image/jpeg",
"image/webp",
"image/svg+xml",
"image/svg",
];
const ALLOWED_MIME_TYPES = [...STATIC_MIME_TYPES];
const router = Router();
router.post(
"/:role_id",
multer.single("file"),
async (req: Request, res: Response) => {
if (req.headers.signature !== Config.get().security.requestSignature)
throw new HTTPError("Invalid request signature");
if (!req.file) throw new HTTPError("Missing file");
const { buffer, mimetype, size, originalname, fieldname } = req.file;
const { role_id } = req.params;
let hash = crypto
.createHash("md5")
.update(Snowflake.generate())
.digest("hex");
const type = await FileType.fromBuffer(buffer);
if (!type || !ALLOWED_MIME_TYPES.includes(type.mime))
throw new HTTPError("Invalid file type");
const path = `role-icons/${role_id}/${hash}.png`;
const endpoint =
Config.get().cdn.endpointPublic || "http://localhost:3003";
await storage.set(path, buffer);
return res.json({
id: hash,
content_type: type.mime,
size,
url: `${endpoint}${req.baseUrl}/${role_id}/${hash}`,
});
}
);
router.get("/:role_id", async (req: Request, res: Response) => {
let { role_id } = req.params;
//role_id = role_id.split(".")[0]; // remove .file extension
const path = `role-icons/${role_id}`;
const file = await storage.get(path);
if (!file) throw new HTTPError("not found", 404);
const type = await FileType.fromBuffer(file);
res.set("Content-Type", type?.mime);
res.set("Cache-Control", "public, max-age=31536000, must-revalidate");
return res.send(file);
});
router.get("/:role_id/:hash", async (req: Request, res: Response) => {
let { role_id, hash } = req.params;
//hash = hash.split(".")[0]; // remove .file extension
const path = `role-icons/${role_id}/${hash}`;
const file = await storage.get(path);
if (!file) throw new HTTPError("not found", 404);
const type = await FileType.fromBuffer(file);
res.set("Content-Type", type?.mime);
res.set("Cache-Control", "public, max-age=31536000, must-revalidate");
return res.send(file);
});
router.delete("/:role_id/:id", async (req: Request, res: Response) => {
if (req.headers.signature !== Config.get().security.requestSignature)
throw new HTTPError("Invalid request signature");
const { role_id, id } = req.params;
const path = `role-icons/${role_id}/${id}`;
await storage.delete(path);
return res.send({ success: true });
});
export default router;
+13
View File
@@ -0,0 +1,13 @@
import dotenv from "dotenv";
dotenv.config();
import { CDNServer } from "./Server";
const server = new CDNServer({ port: Number(process.env.PORT) || 3003 });
server
.start()
.then(() => {
console.log("[Server] started on :" + server.options.port);
})
.catch((e) => console.error("[Server] Error starting: ", e));
module.exports = server;
+51
View File
@@ -0,0 +1,51 @@
import { Storage } from "./Storage";
import fs from "fs";
import { join, relative, dirname } from "path";
import { Readable } from "stream";
//import ExifTransformer = require("exif-be-gone");
import ExifTransformer from "exif-be-gone";
// TODO: split stored files into separate folders named after cloned route
function getPath(path: string) {
// STORAGE_LOCATION has a default value in start.ts
const root = process.env.STORAGE_LOCATION || "../";
let filename = join(root, path);
if (path.indexOf("\0") !== -1 || !filename.startsWith(root))
throw new Error("invalid path");
return filename;
}
export class FileStorage implements Storage {
async get(path: string): Promise<Buffer | null> {
path = getPath(path);
try {
return fs.readFileSync(path);
} catch (error) {
try {
const files = fs.readdirSync(path);
if (!files.length) return null;
return fs.readFileSync(join(path, files[0]));
} catch (error) {
return null;
}
}
}
async set(path: string, value: any) {
path = getPath(path);
//fse.ensureDirSync(dirname(path));
fs.mkdirSync(dirname(path), {recursive: true});
value = Readable.from(value);
const cleaned_file = fs.createWriteStream(path);
return value.pipe(new ExifTransformer()).pipe(cleaned_file);
}
async delete(path: string) {
//TODO we should delete the parent directory if empty
fs.unlinkSync(getPath(path));
}
}
+60
View File
@@ -0,0 +1,60 @@
import { S3 } from "@aws-sdk/client-s3";
import { Readable } from "stream";
import { Storage } from "./Storage";
const readableToBuffer = (readable: Readable): Promise<Buffer> =>
new Promise((resolve, reject) => {
const chunks: Buffer[] = [];
readable.on("data", (chunk) => chunks.push(chunk));
readable.on("error", reject);
readable.on("end", () => resolve(Buffer.concat(chunks)));
});
export class S3Storage implements Storage {
public constructor(
private client: S3,
private bucket: string,
private basePath?: string
) {}
/**
* Always return a string, to ensure consistency.
*/
get bucketBasePath() {
return this.basePath ?? "";
}
async set(path: string, data: Buffer): Promise<void> {
await this.client.putObject({
Bucket: this.bucket,
Key: `${this.bucketBasePath}${path}`,
Body: data,
});
}
async get(path: string): Promise<Buffer | null> {
try {
const s3Object = await this.client.getObject({
Bucket: this.bucket,
Key: `${this.bucketBasePath ?? ""}${path}`,
});
if (!s3Object.Body) return null;
const body = s3Object.Body;
return await readableToBuffer(<Readable>body);
} catch (err) {
console.error(`[CDN] Unable to get S3 object at path ${path}.`);
console.error(err);
return null;
}
}
async delete(path: string): Promise<void> {
await this.client.deleteObject({
Bucket: this.bucket,
Key: `${this.bucketBasePath}${path}`,
});
}
}
+64
View File
@@ -0,0 +1,64 @@
import { FileStorage } from "./FileStorage";
import path from "path";
//import fse from "fs-extra";
import fs from "fs";
import { bgCyan, black } from "picocolors";
import { S3 } from "@aws-sdk/client-s3";
import { S3Storage } from "./S3Storage";
process.cwd();
export interface Storage {
set(path: string, data: Buffer): Promise<void>;
get(path: string): Promise<Buffer | null>;
delete(path: string): Promise<void>;
}
let storage: Storage;
if (process.env.STORAGE_PROVIDER === "file" || !process.env.STORAGE_PROVIDER) {
let location = process.env.STORAGE_LOCATION;
if (location) {
location = path.resolve(location);
} else {
location = path.join(process.cwd(), "files");
}
console.log(`[CDN] storage location: ${bgCyan(`${black(location)}`)}`);
//fse.ensureDirSync(location);
fs.mkdirSync(location, {recursive: true});
process.env.STORAGE_LOCATION = location;
storage = new FileStorage();
} else if (process.env.STORAGE_PROVIDER === "s3") {
const region = process.env.STORAGE_REGION,
bucket = process.env.STORAGE_BUCKET;
if (!region) {
console.error(
`[CDN] You must provide a region when using the S3 storage provider.`
);
process.exit(1);
}
if (!bucket) {
console.error(
`[CDN] You must provide a bucket when using the S3 storage provider.`
);
process.exit(1);
}
// in the S3 provider, this should be the root path in the bucket
let location = process.env.STORAGE_LOCATION;
if (!location) {
console.warn(
`[CDN] STORAGE_LOCATION unconfigured for S3 provider, defaulting to the bucket root...`
);
location = undefined;
}
const client = new S3({ region });
storage = new S3Storage(client, bucket, location);
}
export { storage };
+3
View File
@@ -0,0 +1,3 @@
export * from "./FileStorage";
export * from "./multer";
export * from "./Storage";
+10
View File
@@ -0,0 +1,10 @@
import multerConfig from "multer";
export const multer = multerConfig({
storage: multerConfig.memoryStorage(),
limits: {
fields: 10,
files: 10,
fileSize: 1024 * 1024 * 100, // 100 mb
},
});