dandri/server
1
0
Fork 0
mirror of https://github.com/spacebarchat/server.git synced 2026-08-29 03:18:40 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into pr/hbjydev/454

Browse Source
This commit is contained in:
Flam3rboy
2021-10-15 00:44:19 +02:00
parent 4009e9d2dd d4b6840fae
commit 288eeab6bb
91 changed files with 1180 additions and 638 deletions
Download Patch File Download Diff File Expand all files Collapse all files
bundle/.gitignore
-2
View File
@@ -1,2 +0,0 @@
files/
.env
bundle/.vscode/launch.json
Vendored
+4 -6
View File
@@ -8,13 +8,11 @@
"sourceMaps": true,
"type": "node",
"request": "launch",
"name": "Launch server bundle",
"program": "${workspaceFolder}/dist/start.js",
"runtimeArgs": ["-r", "./tsconfig-paths-bootstrap.js"],
"name": "Launch Server",
"program": "${workspaceFolder}/dist/bundle/src/start.js",
"preLaunchTask": "tsc: build - tsconfig.json",
"outFiles": ["${workspaceFolder}/dist/**/*.js", "${workspaceFolder}/node_modules/@fosscord/**/*.js"],
"envFile": "${workspaceFolder}/.env",
"outDir": "${workspaceFolder}/dist"
"outFiles": ["${workspaceFolder}/dist/**/*.js"],
"envFile": "${workspaceFolder}/.env"
}
]
}
bundle/package-lock.json
Generated
BIN
View File
Binary file not shown.
bundle/package.json
+61 -27
View File
@@ -4,12 +4,12 @@
"description": "",
"main": "src/start.js",
"scripts": {
"setup": "cd ../util && npm --production=false i && cd ../api && npm --production=false i && cd ../cdn && npm --production=false i && cd ../gateway && npm --production=false i && cd ../bundle/ && npm --production=false i && npm run build",
"setup": "node scripts/install.js && npm install && ts-patch install -s && patch-package --patch-dir ../api/patches/ && npm run build",
"build": "node scripts/build.js",
"build:bundle": "npx tsc -b .",
"start": "node scripts/build.js && node -r tsconfig-paths/register dist/start.js",
"start:bundle": "node -r tsconfig-paths/register dist/start.js",
"test": "echo \"Error: no test specified\" && exit 1"
"start": "node scripts/build.js && node dist/bundle/src/start.js",
"start:bundle": "node dist/bundle/src/start.js",
"test": "echo \"Error: no test specified\" && exit 1",
"migrate": "cd ../util/ && npm i && node --require ts-node/register node_modules/typeorm/cli.js -f ../util/ormconfig.json migration:run"
},
"repository": {
"type": "git",
@@ -23,42 +23,76 @@
},
"homepage": "https://fosscord.com",
"devDependencies": {
"@swc/cli": "^0.1.51",
"@swc/core": "^1.2.93",
"@babel/core": "^7.15.5",
"@babel/preset-env": "^7.15.6",
"@babel/preset-typescript": "^7.15.0",
"@types/amqplib": "^0.8.1",
"@types/async-exit-hook": "^2.0.0",
"@types/bcrypt": "^5.0.0",
"@types/express": "^4.17.9",
"@types/body-parser": "^1.19.0",
"@types/btoa": "^1.2.3",
"@types/dotenv": "^8.2.0",
"@types/express": "^4.17.12",
"@types/fs-extra": "^9.0.12",
"@types/i18next-node-fs-backend": "^2.1.0",
"@types/jest": "^27.0.1",
"@types/jest-expect-message": "^1.0.3",
"@types/jsonwebtoken": "^8.5.0",
"@types/mongodb": "^3.6.9",
"@types/mongoose-autopopulate": "^0.10.1",
"@types/mongoose-lean-virtuals": "^0.5.1",
"@types/multer": "^1.4.5",
"@types/node": "^14.17.20",
"@types/node-fetch": "^2.5.7",
"@types/multer": "^1.4.7",
"@types/node": "^14.17.9",
"@types/node-fetch": "^2.5.12",
"@types/node-os-utils": "^1.2.0",
"@types/uuid": "^8.3.0",
"@types/supertest": "^2.0.11",
"@types/ws": "^7.4.0",
"@zerollup/ts-transform-paths": "^1.7.18",
"esbuild": "^0.13.4",
"esbuild-plugin-tsc": "^0.3.0",
"ts-node": "^10.2.1",
"jest": "^27.0.6",
"jest-expect-message": "^1.0.2",
"jest-runtime": "^27.2.1",
"ts-node": "^9.1.1",
"ts-node-dev": "^1.1.6",
"ts-patch": "^1.4.4",
"tsconfig-paths": "^3.11.0",
"typescript": "^4.4.3"
"typescript": "^4.2.3",
"typescript-json-schema": "0.50.1",
"@types/morgan": "^1.9.3"
},
"dependencies": {
"@fosscord/api": "file:../api",
"@fosscord/cdn": "file:../cdn",
"@fosscord/gateway": "file:../gateway",
"@fosscord/util": "file:../util",
"ajv": "8.6.2",
"ajv-formats": "^2.1.1",
"amqplib": "^0.8.0",
"assert": "^1.5.0",
"async-exit-hook": "^2.0.1",
"dotenv": "^10.0.0",
"bcrypt": "^5.0.1",
"body-parser": "^1.19.0",
"btoa": "^1.2.1",
"dotenv": "^8.2.0",
"exif-be-gone": "^1.2.0",
"express": "^4.17.1",
"express-async-errors": "^3.1.1",
"file-type": "^16.5.0",
"form-data": "^3.0.0",
"fs-extra": "^10.0.0",
"i18next": "^19.9.2",
"i18next-http-middleware": "^3.1.3",
"i18next-node-fs-backend": "^2.1.3",
"image-size": "^1.0.0",
"jest": "^27.0.6",
"jsonwebtoken": "^8.5.1",
"lambert-db": "^1.2.3",
"lambert-server": "^1.2.11",
"missing-native-js-functions": "^1.2.17",
"morgan": "^1.10.0",
"multer": "^1.4.2",
"nanocolors": "^0.2.12",
"node-fetch": "^2.6.1",
"node-os-utils": "^1.3.5",
"reflect-metadata": "^0.1.13"
"patch-package": "^6.4.7",
"pg": "^8.7.1",
"reflect-metadata": "^0.1.13",
"sqlite3": "^5.0.2",
"supertest": "^6.1.6",
"typeorm": "^0.2.37",
"typescript": "^4.1.2",
"typescript-json-schema": "^0.50.1",
"ws": "^7.4.2",
"cheerio": "^1.0.0-rc.10"
}
}
bundle/scripts/benchmark/connections.js
+58
View File
@@ -0,0 +1,58 @@
const cluster = require("cluster");
const WebSocket = require("ws");
const endpoint = process.env.GATEWAY || "ws://localhost:3001";
const connections = Number(process.env.CONNECTIONS) || 50;
const threads = Number(process.env.THREADS) || require("os").cpus().length || 1;
const token = process.env.TOKEN;
if (!token) {
console.error("TOKEN env var missing");
process.exit();
}
if (cluster.isMaster) {
for (let i = 0; i < threads; i++) {
cluster.fork();
}
cluster.on("exit", (worker, code, signal) => {
console.log(`worker ${worker.process.pid} died`);
});
} else {
for (let i = 0; i < connections; i++) {
connect();
}
}
function connect() {
const client = new WebSocket(endpoint);
client.on("message", (data) => {
data = JSON.parse(data);
switch (data.op) {
case 10:
client.interval = setInterval(() => {
client.send(JSON.stringify({ op: 1 }));
}, data.d.heartbeat_interval);
client.send(
JSON.stringify({
op: 2,
d: {
token,
properties: {},
},
})
);
break;
}
});
client.once("close", (code, reason) => {
clearInterval(client.interval);
connect();
});
client.on("error", (err) => {
// console.log(err);
});
}
bundle/scripts/benchmark/index.js
+4
View File
@@ -0,0 +1,4 @@
require("dotenv").config();
require("./connections");
require("./messages");
bundle/scripts/benchmark/messages.js
+1
View File
@@ -0,0 +1 @@
// TODO
bundle/scripts/build.js
+38 -92
View File
@@ -1,103 +1,49 @@
const { spawn } = require("child_process");
const { execSync } = require("child_process");
const path = require("path");
const fs = require("fs");
const { performance } = require("perf_hooks");
const fse = require("fs-extra");
const { getSystemErrorMap } = require("util");
const { argv } = require("process");
let parts = "api,cdn,gateway,bundle".split(",");
const tscBin = path.join(__dirname, "..", "..", "util", "node_modules", "typescript", "bin", "tsc");
const swcBin = path.join(__dirname, "..", "..", "util", "node_modules", "@swc", "cli", "bin", "swc");
const dirs = ["api", "util", "cdn", "gateway", "bundle"];
// because npm run is slow we directly get the build script of the package.json script
const verbose = argv.includes("verbose") || argv.includes("v");
function buildPackage(dir) {
const element = path.basename(dir);
return require("esbuild").build({
entryPoints: walk(path.join(dir, "src")),
bundle: false,
outdir: path.join(dir, "dist"),
target: "es2021",
// plugins don't really work because bundle is false
keepNames: false,
tsconfig: path.join(dir, "tsconfig.json"),
});
}
const importPart = /import (\* as )?(({[^}]+})|(\w+)) from ("[.\w-/@q]+")/g;
const importMod = /import ("[\w-/@q.]+")/g;
const exportDefault = /export default/g;
const exportAllAs = /export \* from (".+")/g;
const exportMod = /export ({[\w, ]+})/g;
const exportConst = /export (const|var|let) (\w+)/g;
const exportPart = /export ((async )?\w+) (\w+)/g;
// resolves tsconfig paths + rewrites es6 imports/exports to require (because esbuild/swc doesn't work properly)
function transpileFiles() {
for (const part of ["gateway", "api", "cdn", "bundle"]) {
const files = walk(path.join(__dirname, "..", "..", part, "dist"));
for (const file of files) {
let content = fs.readFileSync(file, { encoding: "utf8" });
content = content
.replace(
new RegExp(`@fosscord/${part}`),
path.relative(file, path.join(__dirname, "..", "..", part, "dist")).slice(3)
)
.replace(importPart, `const $2 = require($5)`)
.replace(importMod, `require($1)`)
.replace(exportDefault, `module.exports =`)
.replace(exportAllAs, `module.exports = {...(module.exports)||{}, ...require($1)}`)
.replace(exportMod, "module.exports = $1")
.replace(exportConst, `let $2 = {};\nmodule.exports.$2 = $2`)
.replace(exportPart, `module.exports.$3 = $1 $3`);
fs.writeFileSync(file, content);
if (argv.includes("clean")) {
dirs.forEach((a) => {
var d = "../" + a + "/dist";
if (fse.existsSync(d)) {
fse.rmSync(d, { recursive: true });
if (verbose) console.log(`Deleted ${d}!`);
}
}
}
function util() {
// const child = spawn("node", `${swcBin} src --out-dir dist --sync`.split(" "), {
const child = spawn("node", `\"${tscBin}\" -b .`.split(" "), {
cwd: path.join(__dirname, "..", "..", "util"),
env: process.env,
shell: true,
});
function log(data) {
console.log(`[util] ` + data.toString().slice(0, -1));
}
child.stdout.on("data", log);
child.stderr.on("data", log);
child.on("error", (err) => console.error("util", err));
return child;
}
const start = performance.now();
async function main() {
console.log("[Build] starting ...");
util();
await Promise.all(parts.map((part) => buildPackage(path.join(__dirname, "..", "..", part))));
transpileFiles();
}
main();
process.on("exit", () => {
console.log("[Build] took " + Math.round(performance.now() - start) + "ms");
fse.copySync(path.join(__dirname, "..", "..", "api", "assets"), path.join(__dirname, "..", "dist", "api", "assets"));
fse.copySync(
path.join(__dirname, "..", "..", "api", "client_test"),
path.join(__dirname, "..", "dist", "api", "client_test")
);
fse.copySync(path.join(__dirname, "..", "..", "api", "locales"), path.join(__dirname, "..", "dist", "api", "locales"));
dirs.forEach((a) => {
fse.copySync("../" + a + "/src", "dist/" + a + "/src");
if (verbose) console.log(`Copied ${"../" + a + "/dist"} -> ${"dist/" + a + "/src"}!`);
});
function walk(dir) {
var results = [];
var list = fs.readdirSync(dir);
list.forEach(function (file) {
file = dir + "/" + file;
var stat = fs.statSync(file);
if (stat && stat.isDirectory()) {
/* Recurse into a subdirectory */
results = results.concat(walk(file));
} else if (file.endsWith(".ts") || file.endsWith(".js")) {
/* Is a file */
results.push(file);
console.log("Copying src files done");
console.log("Compiling src files ...");
console.log(
execSync(
'node "' +
path.join(__dirname, "..", "node_modules", "typescript", "lib", "tsc.js") +
'" -p "' +
path.join(__dirname, "..") +
'"',
{
cwd: path.join(__dirname, ".."),
shell: true,
env: process.env,
encoding: "utf8",
}
});
return results;
}
)
);
bundle/scripts/install.js
+14
View File
@@ -0,0 +1,14 @@
const path = require("path");
const fs = require("fs");
const parts = ["api", "util", "cdn", "gateway"];
const bundle = require("../package.json");
for (const part of parts) {
const { devDependencies, dependencies } = require(path.join("..", "..", part, "package.json"));
bundle.devDependencies = { ...bundle.devDependencies, ...devDependencies };
bundle.dependencies = { ...bundle.dependencies, ...dependencies };
delete bundle.dependencies["@fosscord/util"];
}
fs.writeFileSync(path.join(__dirname, "..", "package.json"), JSON.stringify(bundle, null, "\t"), { encoding: "utf8" });
bundle/src/Server.ts
+1 -4
View File
@@ -4,7 +4,7 @@ process.on("uncaughtException", console.error);
import http from "http";
import * as Api from "@fosscord/api";
import * as Gateway from "@fosscord/gateway";
import { CDNServer } from "@fosscord/cdn/";
import { CDNServer } from "@fosscord/cdn";
import express from "express";
import { green, bold } from "nanocolors";
import { Config, initDatabase } from "@fosscord/util";
@@ -30,9 +30,6 @@ async function main() {
cdn: {
endpointClient: "${location.host}",
endpointPrivate: `http://localhost:${port}`,
...(!Config.get().cdn.endpointPublic && {
endpointPublic: `http://localhost:${port}`,
}),
},
gateway: {
endpointClient:
bundle/src/start.ts
-16
View File
@@ -1,20 +1,4 @@
// process.env.MONGOMS_DEBUG = "true";
const tsConfigPaths = require("tsconfig-paths");
const path = require("path");
const baseUrl = path.join(__dirname, "..");
const cleanup = tsConfigPaths.register({
baseUrl,
paths: {
"@fosscord/api": ["../api/dist/index.js"],
"@fosscord/api/*": ["../api/dist/*"],
"@fosscord/gateway": ["../gateway/dist/index.js"],
"@fosscord/gateway/*": ["../gateway/dist/*"],
"@fosscord/cdn": ["../cdn/dist/index.js"],
"@fosscord/cdn/*": ["../cdn/dist/*"],
},
});
console.log(require("@fosscord/gateway"));
import "reflect-metadata";
import cluster from "cluster";
import os from "os";
bundle/src/stats.ts
+10 -1
View File
@@ -1,11 +1,19 @@
import os from "os";
import osu from "node-os-utils";
import { red } from "nanocolors";
export function initStats() {
console.log(`[Path] running in ${__dirname}`);
console.log(`[CPU] ${osu.cpu.model()} Cores x${osu.cpu.count()}`);
console.log(`[System] ${os.platform()} ${os.arch()}`);
console.log(`[Process] running with pid: ${process.pid}`);
if (process.getuid && process.getuid() === 0) {
console.warn(
red(
`[Process] Warning fosscord is running as root, this highly discouraged and might expose your system vulnerable to attackers. Please run fosscord as a user without root privileges.`
)
);
}
setInterval(async () => {
const [cpuUsed, memory, network] = await Promise.all([
@@ -23,5 +31,6 @@ export function initStats() {
process.memoryUsage().rss / 1024 / 1024
)}mb/${memory.totalMemMb.toFixed(0)}mb ${networkUsage}`
);
}, 1000 * 5);
// TODO: node-os-utils might have a memory leak, more investigation needed
}, 1000 * 60 * 5);
}
bundle/tsconfig.json
+16 -7
View File
@@ -1,22 +1,23 @@
{
"include": ["src/**/*.ts"],
"include": ["dist/**/*.ts"],
"exclude": [],
"compilerOptions": {
/* Visit https://aka.ms/tsconfig.json to read more about this file */
/* Basic Options */
"incremental": true /* Enable incremental compilation */,
"target": "ESNext" /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019', 'ES2020', or 'ESNEXT'. */,
"incremental": false /* Enable incremental compilation */,
"target": "ES6" /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019', 'ES2020', or 'ESNEXT'. */,
"module": "commonjs" /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', 'es2020', or 'ESNext'. */,
"lib": ["ES2021"] /* Specify library files to be included in the compilation. */,
"allowJs": true /* Allow javascript files to be compiled. */,
"checkJs": true /* Report errors in .js files. */,
// "jsx": "preserve", /* Specify JSX code generation: 'preserve', 'react-native', or 'react'. */
"declaration": true /* Generates corresponding '.d.ts' file. */,
"declaration": false /* Generates corresponding '.d.ts' file. */,
"declarationMap": false /* Generates a sourcemap for each corresponding '.d.ts' file. */,
"sourceMap": true /* Generates corresponding '.map' file. */,
"sourceMap": false /* Generates corresponding '.map' file. */,
// "outFile": "./", /* Concatenate and emit output to single file. */
"outDir": "./dist/" /* Redirect output structure to the directory. */,
"rootDir": "./src/" /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */,
"rootDir": "./dist/" /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */,
// "composite": true, /* Enable project compilation */
// "tsBuildInfoFile": "./", /* Specify file to store incremental compilation information */
// "removeComments": true, /* Do not emit comments to output. */
@@ -66,6 +67,14 @@
"forceConsistentCasingInFileNames": true /* Disallow inconsistently-cased references to the same file. */,
"emitDecoratorMetadata": true,
"experimentalDecorators": true,
"baseUrl": "."
"resolveJsonModule": true,
"baseUrl": "./dist/",
"paths": {
"@fosscord/api": ["api/src/index"],
"@fosscord/gateway": ["gateway/src/index"],
"@fosscord/cdn": ["cdn/src/index"],
"@fosscord/util": ["util/src/index"]
},
"plugins": [{ "transform": "@zerollup/ts-transform-paths" }]
}
}