Refactor to mono-repo + upgrade packages

scripts/benchmark/connections.js

@@ -0,0 +1,64 @@
+require("dotenv").config();
+const cluster = require("cluster");
+const WebSocket = require("ws");
+const endpoint = process.env.GATEWAY || "ws://localhost:3001";
+const connections = Number(process.env.CONNECTIONS) || 50;
+const token = process.env.TOKEN;
+var cores = 1;
+try {
+	cores = Number(process.env.THREADS) || os.cpus().length;
+} catch {
+	console.log("[Bundle] Failed to get thread count! Using 1...")
+}
+
+if (!token) {
+	console.error("TOKEN env var missing");
+	process.exit();
+}
+
+if (cluster.isMaster) {
+	for (let i = 0; i < threads; i++) {
+		cluster.fork();
+	}
+
+	cluster.on("exit", (worker, code, signal) => {
+		console.log(`worker ${worker.process.pid} died`);
+	});
+} else {
+	for (let i = 0; i < connections; i++) {
+		connect();
+	}
+}
+
+function connect() {
+	const client = new WebSocket(endpoint);
+	client.on("message", (data) => {
+		data = JSON.parse(data);
+
+		switch (data.op) {
+			case 10:
+				client.interval = setInterval(() => {
+					client.send(JSON.stringify({ op: 1 }));
+				}, data.d.heartbeat_interval);
+
+				client.send(
+					JSON.stringify({
+						op: 2,
+						d: {
+							token,
+							properties: {},
+						},
+					})
+				);
+
+				break;
+		}
+	});
+	client.once("close", (code, reason) => {
+		clearInterval(client.interval);
+		connect();
+	});
+	client.on("error", (err) => {
+		// console.log(err);
+	});
+}

scripts/benchmark/index.js

@@ -0,0 +1,4 @@
+require("dotenv").config();
+
+require("./connections");
+require("./messages");

scripts/benchmark/users.js

@@ -0,0 +1,25 @@
+require("dotenv").config();
+const fetch = require("node-fetch");
+const count = Number(process.env.COUNT) || 50;
+const endpoint = process.env.API || "http://localhost:3001";
+
+async function main() {
+	for (let i = 0; i < count; i++) {
+		fetch(`${endpoint}/api/auth/register`, {
+			method: "POST",
+			body: JSON.stringify({
+				fingerprint: `${i}.wR8vi8lGlFBJerErO9LG5NViJFw`,
+				username: `test${i}`,
+				invite: null,
+				consent: true,
+				date_of_birth: "2000-01-01",
+				gift_code_sku_id: null,
+				captcha_key: null,
+			}),
+			headers: { "content-type": "application/json" },
+		});
+		console.log(i);
+	}
+}
+
+main();

scripts/build.js

@@ -0,0 +1,72 @@
+const { execSync } = require("child_process");
+const path = require("path");
+const fs = require("fs");
+const { getSystemErrorMap } = require("util");
+const { argv } = require("process");
+
+var steps = 2, i = 0;
+if (argv.includes("clean")) steps++;
+if (argv.includes("copyonly")) steps--;
+const dirs = ["api", "util", "cdn", "gateway", "bundle"];
+const verbose = argv.includes("verbose") || argv.includes("v");
+
+var copyRecursiveSync = function(src, dest) {
+	if(verbose) console.log(`cpsync: ${src} -> ${dest}`);
+	var exists = fs.existsSync(src);
+	if(!exists){
+		console.log(src + " doesn't exist, not copying!");
+		return;
+	}
+	var stats = exists && fs.statSync(src);
+	var isDirectory = exists && stats.isDirectory();
+	if (isDirectory) {
+	  fs.mkdirSync(dest, {recursive: true});
+	  fs.readdirSync(src).forEach(function(childItemName) {
+		copyRecursiveSync(path.join(src, childItemName),
+						  path.join(dest, childItemName));
+	  });
+	} else {
+	  fs.copyFileSync(src, dest);
+	}
+  };
+
+if (argv.includes("clean")) {
+	console.log(`[${++i}/${steps}] Cleaning...`);
+	dirs.forEach((a) => {
+		var d = "../" + a + "/dist";
+		if (fs.existsSync(d)) {
+			fs.rmSync(d, { recursive: true });
+			if (verbose) console.log(`Deleted ${d}!`);
+		}
+	});
+}
+
+console.log(`[${++i}/${steps}] Copying src files...`);
+copyRecursiveSync(path.join(__dirname, "..", "..", "api", "assets"), path.join(__dirname, "..", "dist", "api", "assets"));
+copyRecursiveSync(path.join(__dirname, "..", "..", "api", "client_test"), path.join(__dirname, "..", "dist", "api", "client_test"));
+copyRecursiveSync(path.join(__dirname, "..", "..", "api", "locales"), path.join(__dirname, "..", "dist", "api", "locales"));
+dirs.forEach((a) => {
+	copyRecursiveSync("../" + a + "/src", "dist/" + a + "/src");
+	if (verbose) console.log(`Copied ${"../" + a + "/dist"} -> ${"dist/" + a + "/src"}!`);
+});
+
+if (!argv.includes("copyonly")) {
+	console.log(`[${++i}/${steps}] Compiling src files ...`);
+
+	console.log(
+		execSync(
+			'node "' +
+				path.join(__dirname, "..", "node_modules", "typescript", "lib", "tsc.js") +
+				'" -p "' +
+				path.join(__dirname, "..") +
+				'"',
+			{
+				cwd: path.join(__dirname, ".."),
+				shell: true,
+				env: process.env,
+				encoding: "utf8"
+			}
+		)
+	);
+}
+

scripts/droptables.sql

@@ -0,0 +1,31 @@
+DROP TABLE applications;
+DROP TABLE attachments;
+DROP TABLE audit_logs;
+DROP TABLE bans;
+DROP TABLE connected_accounts;
+DROP TABLE emojis;
+DROP TABLE invites;
+DROP TABLE member_roles;
+DROP TABLE message_channel_mentions;
+DROP TABLE message_role_mentions;
+DROP TABLE message_stickers;
+DROP TABLE message_user_mentions;
+DROP TABLE messages;
+DROP TABLE rate_limits;
+DROP TABLE read_states;
+DROP TABLE recipients;
+DROP TABLE relationships;
+DROP TABLE roles;
+DROP TABLE sessions;
+DROP TABLE stickers;
+DROP TABLE team_members;
+DROP TABLE teams;
+DROP TABLE templates;
+DROP TABLE voice_states;
+DROP TABLE webhooks;
+DROP TABLE channels;
+DROP TABLE members;
+DROP TABLE guilds;
+DROP TABLE client_release;
+-- DROP TABLE users;
+-- DROP TABLE config;

scripts/generate_openapi.js

@@ -0,0 +1,137 @@
+// https://mermade.github.io/openapi-gui/#
+// https://editor.swagger.io/
+const getRouteDescriptions = require("../jest/getRouteDescriptions");
+const path = require("path");
+const fs = require("fs");
+require("missing-native-js-functions");
+
+const openapiPath = path.join(__dirname, "..", "assets", "openapi.json");
+const SchemaPath = path.join(__dirname, "..", "assets", "schemas.json");
+const schemas = JSON.parse(fs.readFileSync(SchemaPath, { encoding: "utf8" }));
+const specification = JSON.parse(fs.readFileSync(openapiPath, { encoding: "utf8" }));
+
+function combineSchemas(schemas) {
+	var definitions = {};
+
+	for (const name in schemas) {
+		definitions = {
+			...definitions,
+			...schemas[name].definitions,
+			[name]: { ...schemas[name], definitions: undefined, $schema: undefined }
+		};
+	}
+
+	for (const key in definitions) {
+		specification.components.schemas[key] = definitions[key];
+		delete definitions[key].additionalProperties;
+		delete definitions[key].$schema;
+		const definition = definitions[key];
+
+		if (typeof definition.properties === "object") {
+			for (const property of Object.values(definition.properties)) {
+				if (Array.isArray(property.type)) {
+					if (property.type.includes("null")) {
+						property.type = property.type.find((x) => x !== "null");
+						property.nullable = true;
+					}
+				}
+			}
+		}
+	}
+
+	return definitions;
+}
+
+function getTag(key) {
+	return key.match(/\/([\w-]+)/)[1];
+}
+
+function apiRoutes() {
+	const routes = getRouteDescriptions();
+
+	const tags = Array.from(routes.keys()).map((x) => getTag(x));
+	specification.tags = [...specification.tags.map((x) => x.name), ...tags].unique().map((x) => ({ name: x }));
+
+	routes.forEach((route, pathAndMethod) => {
+		const [p, method] = pathAndMethod.split("|");
+		const path = p.replace(/:(\w+)/g, "{$1}");
+
+		let obj = specification.paths[path]?.[method] || {};
+		if (!obj.description) {
+			const permission = route.permission ? `##### Requires the \`\`${route.permission}\`\` permission\n` : "";
+			const event = route.test?.event ? `##### Fires a \`\`${route.test?.event}\`\` event\n` : "";
+			obj.description = permission + event;
+		}
+		if (route.body) {
+			obj.requestBody = {
+				required: true,
+				content: {
+					"application/json": {
+						schema: { $ref: `#/components/schemas/${route.body}` }
+					}
+				}
+			}.merge(obj.requestBody);
+		}
+		if (!obj.responses) {
+			obj.responses = {
+				default: {
+					description: "not documented"
+				}
+			};
+		}
+		if (route.test?.response) {
+			const status = route.test.response.status || 200;
+			let schema = {
+				allOf: [
+					{
+						$ref: `#/components/schemas/${route.test.response.body}`
+					},
+					{
+						example: route.test.body
+					}
+				]
+			};
+			if (!route.test.body) schema = schema.allOf[0];
+
+			obj.responses = {
+				[status]: {
+					...(route.test.response.body
+						? {
+								description: obj.responses[status].description || "",
+								content: {
+									"application/json": {
+										schema: schema
+									}
+								}
+						  }
+						: {})
+				}
+			}.merge(obj.responses);
+			delete obj.responses.default;
+		}
+		if (p.includes(":")) {
+			obj.parameters = p.match(/:\w+/g)?.map((x) => ({
+				name: x.replace(":", ""),
+				in: "path",
+				required: true,
+				schema: { type: "string" },
+				description: x.replace(":", "")
+			}));
+		}
+		obj.tags = [...(obj.tags || []), getTag(p)].unique();
+
+		specification.paths[path] = { ...specification.paths[path], [method]: obj };
+	});
+}
+
+function main() {
+	combineSchemas(schemas);
+	apiRoutes();
+
+	fs.writeFileSync(
+		openapiPath,
+		JSON.stringify(specification, null, 4).replaceAll("#/definitions", "#/components/schemas").replaceAll("bigint", "number")
+	);
+}
+
+main();

scripts/generate_schema.js

@@ -0,0 +1,96 @@
+// https://mermade.github.io/openapi-gui/#
+// https://editor.swagger.io/
+const path = require("path");
+const fs = require("fs");
+const TJS = require("typescript-json-schema");
+require("missing-native-js-functions");
+const schemaPath = path.join(__dirname, "..", "assets", "schemas.json");
+
+const settings = {
+	required: true,
+	ignoreErrors: true,
+	excludePrivate: true,
+	defaultNumberType: "integer",
+	noExtraProps: true,
+	defaultProps: false
+};
+const compilerOptions = {
+	strictNullChecks: true
+};
+const Excluded = [
+	"DefaultSchema",
+	"Schema",
+	"EntitySchema",
+	"ServerResponse",
+	"Http2ServerResponse",
+	"global.Express.Response",
+	"Response",
+	"e.Response",
+	"request.Response",
+	"supertest.Response",
+
+	// TODO: Figure out how to exclude schemas from node_modules?
+	"SomeJSONSchema",
+	"UncheckedPartialSchema",
+	"PartialSchema",
+	"UncheckedPropertiesSchema",
+	"PropertiesSchema",
+	"AsyncSchema",
+	"AnySchema",
+];
+
+function modify(obj) {
+	for (var k in obj) {
+		if (typeof obj[k] === "object" && obj[k] !== null) {
+			modify(obj[k]);
+		}
+	}
+}
+
+function main() {
+	const files = 		[
+		...walk(path.join(__dirname, "..", "src", "routes")),
+		...walk(path.join(__dirname, "..", "..", "util", "src")),
+	];
+	const program = TJS.getProgramFromFiles(
+		files,
+		compilerOptions
+	);
+	const generator = TJS.buildGenerator(program, settings);
+	if (!generator || !program) return;
+
+	let schemas = generator.getUserSymbols().filter((x) => (x.endsWith("Schema") || x.endsWith("Response")) && !Excluded.includes(x));
+	console.log(schemas);
+
+	var definitions = {};
+
+	for (const name of schemas) {
+		const part = TJS.generateSchema(program, name, settings, [], generator);
+		if (!part) continue;
+
+		definitions = { ...definitions, [name]: { ...part } };
+	}
+
+	modify(definitions);
+
+	fs.writeFileSync(schemaPath, JSON.stringify(definitions, null, 4));
+}
+
+main();
+
+function walk(dir) {
+	var results = [];
+	var list = fs.readdirSync(dir);
+	list.forEach(function (file) {
+		file = dir + "/" + file;
+		var stat = fs.statSync(file);
+		if (stat && stat.isDirectory()) {
+			/* Recurse into a subdirectory */
+			results = results.concat(walk(file));
+		} else {
+			if (!file.endsWith(".ts")) return;
+			results.push(file);
+		}
+	});
+	return results;
+}

scripts/install.js

@@ -0,0 +1,23 @@
+const path = require("path");
+const fs = require("fs");
+const parts = ["api", "util", "cdn", "gateway"];
+
+const bundle = require("../package.json");
+
+for (const part of parts) {
+	const { devDependencies, dependencies } = require(path.join(
+		"..",
+		"..",
+		part,
+		"package.json"
+	));
+	bundle.devDependencies = { ...bundle.devDependencies, ...devDependencies };
+	bundle.dependencies = { ...bundle.dependencies, ...dependencies };
+	delete bundle.dependencies["@fosscord/util"];
+}
+
+fs.writeFileSync(
+	path.join(__dirname, "..", "package.json"),
+	JSON.stringify(bundle, null, "\t"),
+	{ encoding: "utf8" }
+);

scripts/migrate_db_engine.js

@@ -0,0 +1,109 @@
+const { config } = require("dotenv");
+config();
+const { createConnection } = require("typeorm");
+const { initDatabase } = require("../../dist/util/Database");
+require("missing-native-js-functions");
+const {
+	Application,
+	Attachment,
+	Ban,
+	Channel,
+	ConfigEntity,
+	ConnectedAccount,
+	Emoji,
+	Guild,
+	Invite,
+	Member,
+	Message,
+	ReadState,
+	Recipient,
+	Relationship,
+	Role,
+	Sticker,
+	Team,
+	TeamMember,
+	Template,
+	User,
+	VoiceState,
+	Webhook,
+} = require("../../dist/entities/index");
+
+async function main() {
+	if (!process.env.TO) throw new Error("TO database env connection string not set");
+
+	// manually arrange them because of foreign keys
+	const entities = [
+		ConfigEntity,
+		User,
+		Guild,
+		Channel,
+		Invite,
+		Role,
+		Ban,
+		Application,
+		Emoji,
+		ConnectedAccount,
+		Member,
+		ReadState,
+		Recipient,
+		Relationship,
+		Sticker,
+		Team,
+		TeamMember,
+		Template,
+		VoiceState,
+		Webhook,
+		Message,
+		Attachment,
+	];
+
+	const oldDB = await initDatabase();
+
+	const type = process.env.TO.includes("://") ? process.env.TO.split(":")[0]?.replace("+srv", "") : "sqlite";
+	const isSqlite = type.includes("sqlite");
+
+	// @ts-ignore
+	const newDB = await createConnection({
+		type,
+		url: isSqlite ? undefined : process.env.TO,
+		database: isSqlite ? process.env.TO : undefined,
+		entities,
+		name: "new",
+		synchronize: true,
+	});
+	let i = 0;
+
+	try {
+		for (const entity of entities) {
+			const entries = await oldDB.manager.find(entity);
+
+			// @ts-ignore
+			console.log("migrating " + entries.length + " " + entity.name + " ...");
+
+			for (const entry of entries) {
+				console.log(i++);
+
+				try {
+					await newDB.manager.insert(entity, entry);
+				} catch (error) {
+					try {
+						if (!entry.id) throw new Error("object doesn't have a unique id: " + entry);
+						await newDB.manager.update(entity, { id: entry.id }, entry);
+					} catch (error) {
+						console.error("couldn't migrate " + i + " " + entity.name, error);
+					}
+				}
+			}
+
+			// @ts-ignore
+			console.log("migrated " + entries.length + " " + entity.name);
+		}
+	} catch (error) {
+		console.error(error.message);
+	}
+
+	console.log("SUCCESS migrated all data");
+	await newDB.close();
+}
+
+main().caught();

scripts/rights.js

@@ -0,0 +1,20 @@
+const { Rights } = require("..");
+
+const allRights = new Rights(1).bitfield;
+console.log(`All rights:`, allRights);
+
+var discordLike = allRights;
+discordLike -= Rights.FLAGS.OPERATOR;
+discordLike -= Rights.FLAGS.MANAGE_APPLICATIONS;
+discordLike -= Rights.FLAGS.MANAGE_GUILDS;
+discordLike -= Rights.FLAGS.MANAGE_MESSAGES;
+discordLike -= Rights.FLAGS.MANAGE_RATE_LIMITS;
+discordLike -= Rights.FLAGS.MANAGE_ROUTING;
+discordLike -= Rights.FLAGS.MANAGE_TICKETS;
+discordLike -= Rights.FLAGS.MANAGE_USERS;
+discordLike -= Rights.FLAGS.ADD_MEMBERS;
+discordLike -= Rights.FLAGS.BYPASS_RATE_LIMITS;
+discordLike -= Rights.FLAGS.CREDITABLE;
+discordLike -= Rights.FLAGS.MANAGE_GUILD_DIRECTORY;
+discordLike -= Rights.FLAGS.SEND_BACKDATED_EVENTS;
+console.log(`Discord.com-like rights:`, discordLike);

scripts/stresstest/.gitignore

@@ -0,0 +1,3 @@
+/node_modules
+config.json
+accounts.json

scripts/stresstest/config.json.example

@@ -0,0 +1,5 @@
+{
+	"url": "",
+	"text-channel": "",
+	"invite": ""
+}

scripts/stresstest/index.js

@@ -0,0 +1,38 @@
+const register = require("./src/register");
+const login = require("./src/login/index");
+const config = require("./config.json");
+const figlet = require("figlet");
+const sendMessage = require("./src/message/send");
+const fs = require("fs");
+figlet("Fosscord Stress Test :)", function (err, data) {
+	if (err) {
+		console.log("Something went wrong...");
+		console.dir(err);
+		return;
+	}
+	console.log("\x1b[32m", data);
+});
+setInterval(() => {
+	generate();
+}, 1000 * 5);
+setInterval(() => {
+	getUsers();
+}, 60 * 1000);
+async function generate() {
+	var accounts = await JSON.parse(fs.readFileSync("accounts.json"));
+	console.log(accounts);
+	var account = await register();
+	accounts.push(account);
+	fs.writeFileSync("accounts.json", JSON.stringify(accounts));
+	console.log(accounts.length);
+	var y = await login(account);
+	sendMessage(y);
+}
+async function getUsers() {
+	var accounts = await JSON.parse(fs.readFileSync("accounts.json"));
+	accounts.forEach(async (x) => {
+		var y = await login(x);
+		console.log(y);
+		sendMessage(y);
+	});
+}

scripts/stresstest/package.json

@@ -0,0 +1,17 @@
+{
+	"name": "stresstest",
+	"version": "1.0.0",
+	"description": "",
+	"main": "index.js",
+	"scripts": {
+		"test": "echo \"Error: no test specified\" && exit 1",
+		"start": "node ."
+	},
+	"author": "",
+	"license": "ISC",
+	"dependencies": {
+		"figlet": "^1.5.2",
+		"node-fetch": "^2.6.6",
+		"request": "^2.88.2"
+	}
+}

scripts/stresstest/src/login/index.js

@@ -0,0 +1,20 @@
+const fetch = require("node-fetch");
+const fs = require("fs");
+var config = require("../../config.json");
+module.exports = login;
+async function login(account) {
+	var body = {
+		fingerprint: "805826570869932034.wR8vi8lGlFBJerErO9LG5NViJFw",
+		login: account.email,
+		password: account.password
+	};
+	var x = await fetch(config.url + "/auth/login", {
+		method: "POST",
+		headers: { "Content-Type": "application/json" },
+		body: JSON.stringify(body)
+	});
+	console.log(x);
+	x = await x.json();
+	console.log(x);
+	return x;
+}

scripts/stresstest/src/message/send.js

@@ -0,0 +1,23 @@
+const fetch = require("node-fetch");
+const fs = require("fs");
+var config = require("../../config.json");
+module.exports = sendMessage;
+async function sendMessage(account) {
+	var body = {
+		fingerprint: "805826570869932034.wR8vi8lGlFBJerErO9LG5NViJFw",
+		content: "Test",
+		tts: false
+	};
+	var x = await fetch(config.url + "/channels/" + config["text-channel"] + "/messages", {
+		method: "POST",
+		headers: {
+			"Content-Type": "application/json",
+			Authorization: account.token
+		},
+		body: JSON.stringify(body)
+	});
+	console.log(x);
+	x = await x.json();
+	console.log(x);
+	return x;
+}

scripts/stresstest/src/register/index.js

@@ -0,0 +1,34 @@
+const fetch = require("node-fetch");
+const fs = require("fs");
+var config = require("../../config.json");
+module.exports = generate;
+async function generate() {
+	var mail = (Math.random() + 10).toString(36).substring(2);
+	mail = mail + "." + (Math.random() + 10).toString(36).substring(2) + "@stresstest.com";
+	var password =
+		(Math.random() * 69).toString(36).substring(-7) +
+		(Math.random() * 69).toString(36).substring(-7) +
+		(Math.random() * 69).toString(36).substring(-8);
+	console.log(mail);
+	console.log(password);
+	var body = {
+		fingerprint: "805826570869932034.wR8vi8lGlFBJerErO9LG5NViJFw",
+		email: mail,
+		username: "Fosscord Stress Test",
+		password: password,
+		invite: config.invite,
+		consent: true,
+		date_of_birth: "2000-04-04",
+		gift_code_sku_id: null,
+		captcha_key: null
+	};
+	var x = await fetch(config.url + "/auth/register", {
+		method: "POST",
+		headers: { "Content-Type": "application/json" },
+		body: JSON.stringify(body)
+	});
+	console.log(x);
+	x = await x.json();
+	console.log(x);
+	return { email: mail, password: password };
+}