Refactor to mono-repo + upgrade packages

src/bundle/Server.ts

@@ -0,0 +1,126 @@
+process.on("unhandledRejection", console.error);
+process.on("uncaughtException", console.error);
+
+import http from "http";
+import * as Api from "@fosscord/api";
+import * as Gateway from "@fosscord/gateway";
+import { CDNServer } from "@fosscord/cdn";
+import express from "express";
+import { green, bold, yellow } from "picocolors";
+import { Config, initDatabase, BannedWords } from "@fosscord/util";
+import * as Sentry from "@sentry/node";
+import * as Tracing from "@sentry/tracing";
+
+const app = express();
+const server = http.createServer();
+const port = Number(process.env.PORT) || 3001;
+const production = process.env.NODE_ENV == "development" ? false : true;
+server.on("request", app);
+
+const api = new Api.FosscordServer({ server, port, production, app });
+const cdn = new CDNServer({ server, port, production, app });
+const gateway = new Gateway.Server({ server, port, production });
+
+//this is what has been added for the /stop API route
+process.on('SIGTERM', () => {
+	server.close(() => {
+		console.log("Stop API has been successfully POSTed, SIGTERM sent");
+	});
+});
+//this is what has been added for the /stop API route
+
+async function main() {
+	await initDatabase();
+	await Config.init();
+	await BannedWords.init();
+	// only set endpointPublic, if not already set
+	await Config.set({
+		cdn: {
+			endpointClient: "${location.host}",
+			endpointPrivate: `http://localhost:${port}`,
+		},
+		gateway: {
+			endpointClient:
+				'${location.protocol === "https:" ? "wss://" : "ws://"}${location.host}',
+			endpointPrivate: `ws://localhost:${port}`,
+			...(!Config.get().gateway.endpointPublic && {
+				endpointPublic: `ws://localhost:${port}`,
+			}),
+		},
+		// regions: {
+		// 	default: "fosscord",
+		// 	useDefaultAsOptimal: true,
+		// 	available: [
+		// 		{
+		// 			id: "fosscord",
+		// 			name: "Fosscord",
+		// 			endpoint: "slowcord.maddy.k.vu:3004",
+		// 			vip: false,
+		// 			custom: false,
+		// 			deprecated: false,
+		// 		},
+		// 	],
+		// },
+	} as any);
+
+	//Sentry
+	if (Config.get().sentry.enabled) {
+		console.log(
+			`[Bundle] ${yellow("You are using Sentry! This may slightly impact performance on large loads!")}`
+		);
+		Sentry.init({
+			dsn: Config.get().sentry.endpoint,
+			integrations: [
+				new Sentry.Integrations.Http({ tracing: true }),
+				new Tracing.Integrations.Express({ app }),
+				new Tracing.Integrations.Mysql(),
+			],
+			tracesSampleRate: Config.get().sentry.traceSampleRate,
+			environment: Config.get().sentry.environment,
+		});
+
+		Sentry.addGlobalEventProcessor((event, hint) => {
+			if (event.transaction) {
+				event.transaction = event.transaction.split("/").map(x => !parseInt(x) ? x : ":id").join("/");
+			}
+
+			delete event.request?.cookies;
+			if (event.request?.headers) {
+				delete event.request.headers["X-Real-Ip"];
+				delete event.request.headers["X-Forwarded-For"];
+				delete event.request.headers["X-Forwarded-Host"];
+				delete event.request.headers["X-Super-Properties"];
+			}
+
+			if (event.breadcrumbs) {
+				event.breadcrumbs = event.breadcrumbs.filter(x => {
+					if (x.message?.includes("identified as")) return false;
+					if (x.message?.includes("[WebSocket] closed")) return false;
+					if (x.message?.includes("Got Resume -> cancel not implemented")) return false;
+					if (x.message?.includes("[Gateway] New connection from")) return false;
+
+					return true;
+				})
+			}
+
+			return event;
+		});
+
+		app.use(Sentry.Handlers.requestHandler());
+		app.use(Sentry.Handlers.tracingHandler());
+	}
+
+	await Promise.all([api.start(), cdn.start(), gateway.start()]);
+
+	if (Config.get().sentry.enabled) {
+		app.use(Sentry.Handlers.errorHandler());
+		app.use(function onError(err: any, req: any, res: any, next: any) {
+			res.statusCode = 500;
+			res.end(res.sentry + "\n");
+		});
+	}
+
+	console.log(`[Server] ${green(`listening on port ${bold(port)}`)}`);
+}
+
+main().catch(console.error);

src/bundle/index.ts

@@ -0,0 +1,4 @@
+export * from "@fosscord/api";
+export * from "@fosscord/util";
+export * from "@fosscord/gateway";
+export * from "@fosscord/cdn";

src/bundle/start.ts

@@ -0,0 +1,96 @@
+// process.env.MONGOMS_DEBUG = "true";
+require('module-alias/register');
+import "reflect-metadata";
+import cluster, { Worker } from "cluster";
+import os from "os";
+import { red, bold, yellow, cyan } from "picocolors";
+import { initStats } from "./stats";
+import { config } from "dotenv";
+config();
+import { execSync } from "child_process";
+
+// TODO: add socket event transmission
+var cores = 1;
+try {
+	cores = Number(process.env.THREADS) || os.cpus().length;
+} catch {
+	console.log("[API] Failed to get thread count! Using 1...");
+}
+
+if (cluster.isMaster) {
+	function getCommitOrFail() {
+		try {
+			return execSync("git rev-parse HEAD").toString().trim();
+		} catch (e) {
+			return null;
+		}
+	}
+	const commit = getCommitOrFail();
+
+	console.log(
+		bold(`
+███████╗ ██████╗ ███████╗███████╗ ██████╗ ██████╗ ██████╗ ██████╗
+██╔════╝██╔═══██╗██╔════╝██╔════╝██╔════╝██╔═══██╗██╔══██╗██╔══██╗
+█████╗  ██║   ██║███████╗███████╗██║     ██║   ██║██████╔╝██║  ██║
+██╔══╝  ██║   ██║╚════██║╚════██║██║     ██║   ██║██╔══██╗██║  ██║
+██║     ╚██████╔╝███████║███████║╚██████╗╚██████╔╝██║  ██║██████╔╝
+╚═╝      ╚═════╝ ╚══════╝╚══════╝ ╚═════╝ ╚═════╝ ╚═╝  ╚═╝╚═════╝
+
+		fosscord-server | ${yellow(
+			`Pre-release (${commit !== null
+				? commit.slice(0, 7)
+				: "Unknown (Git cannot be found)"
+			})`
+		)}
+
+Commit Hash: ${commit !== null
+				? `${cyan(commit)} (${yellow(commit.slice(0, 7))})`
+				: "Unknown (Git cannot be found)"
+			}
+Cores: ${cyan(os.cpus().length)} (Using ${cores} thread(s).)
+`)
+	);
+
+	if (commit == null) {
+		console.log(yellow(`Warning: Git is not installed or not in PATH.`));
+	}
+
+	initStats();
+
+	console.log(`[Process] starting with ${cores} threads`);
+
+	if (cores === 1) {
+		require("./Server");
+	} else {
+		process.env.EVENT_TRANSMISSION = "process";
+
+		// Fork workers.
+		for (let i = 0; i < cores; i++) {
+			// Delay each worker start if using sqlite database to prevent locking it
+			let delay = process.env.DATABASE?.includes("://") ? 0 : i * 1000;
+			setTimeout(() => {
+				cluster.fork();
+				console.log(`[Process] worker ${cyan(i)} started.`);
+			}, delay);
+		}
+
+		cluster.on("message", (sender: Worker, message: any) => {
+			for (const id in cluster.workers) {
+				const worker = cluster.workers[id];
+				if (worker === sender || !worker) continue;
+				worker.send(message);
+			}
+		});
+
+		cluster.on("exit", (worker: any, code: any, signal: any) => {
+			console.log(
+				`[Worker] ${red(
+					`died with PID: ${worker.process.pid} , restarting ...`
+				)}`
+			);
+			cluster.fork();
+		});
+	}
+} else {
+	require("./Server");
+}

src/bundle/stats.ts

@@ -0,0 +1,43 @@
+import os from "os";
+import osu from "node-os-utils";
+import { red } from "picocolors";
+
+export function initStats() {
+	console.log(`[Path] running in ${__dirname}`);
+	try {
+		console.log(`[CPU] ${osu.cpu.model()} Cores x${osu.cpu.count()}`);
+	}
+	catch {
+		console.log('[CPU] Failed to get cpu model!')
+	}
+	
+	console.log(`[System] ${os.platform()} ${os.arch()}`);
+	console.log(`[Process] running with PID: ${process.pid}`);
+	if (process.getuid && process.getuid() === 0) {
+		console.warn(
+			red(
+				`[Process] Warning fosscord is running as root, this highly discouraged and might expose your system vulnerable to attackers. Please run fosscord as a user without root privileges.`
+			)
+		);
+	}
+
+	// TODO: node-os-utils might have a memory leak, more investigation needed
+	// TODO: doesn't work if spawned with multiple threads
+	// setInterval(async () => {
+	// 	const [cpuUsed, memory, network] = await Promise.all([
+	// 		osu.cpu.usage(),
+	// 		osu.mem.info(),
+	// 		osu.netstat.inOut(),
+	// 	]);
+	// 	var networkUsage = "";
+	// 	if (typeof network === "object") {
+	// 		networkUsage = `| [Network]: in ${network.total.inputMb}mb | out ${network.total.outputMb}mb`;
+	// 	}
+
+	// 	console.log(
+	// 		`[CPU] ${cpuUsed.toPrecision(3)}% | [Memory] ${Math.round(
+	// 			process.memoryUsage().rss / 1024 / 1024
+	// 		)}mb/${memory.totalMemMb.toFixed(0)}mb ${networkUsage}`
+	// 	);
+	// }, 1000 * 60 * 5);
+}