Refactor to mono-repo + upgrade packages

This commit is contained in:
Madeline
2022-09-25 18:24:21 +10:00
parent 59d94b4894
commit f44f5d7ac2
583 changed files with 900 additions and 3898 deletions
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,154 @@
aol.com
att.net
comcast.net
facebook.com
gmail.com
gmx.com
googlemail.com
google.com
hotmail.com
hotmail.co.uk
mac.com
me.com
mail.com
msn.com
live.com
sbcglobal.net
verizon.net
yahoo.com
yahoo.co.uk
email.com
fastmail.fm
games.com
gmx.net
hush.com
hushmail.com
icloud.com
iname.com
inbox.com
lavabit.com
love.com
outlook.com
pobox.com
protonmail.ch
protonmail.com
tutanota.de
tutanota.com
tutamail.com
tuta.io
keemail.me
rocketmail.com
safe-mail.net
wow.com
ygm.com
ymail.com
zoho.com
yandex.com
bellsouth.net
charter.net
cox.net
earthlink.net
juno.com
btinternet.com
virginmedia.com
blueyonder.co.uk
freeserve.co.uk
live.co.uk
ntlworld.com
o2.co.uk
orange.net
sky.com
talktalk.co.uk
tiscali.co.uk
virgin.net
wanadoo.co.uk
bt.com
sina.com
sina.cn
qq.com
naver.com
hanmail.net
daum.net
nate.com
yahoo.co.jp
yahoo.co.kr
yahoo.co.id
yahoo.co.in
yahoo.com.sg
yahoo.com.ph
163.com
yeah.net
126.com
21cn.com
aliyun.com
foxmail.com
hotmail.fr
live.fr
laposte.net
yahoo.fr
wanadoo.fr
orange.fr
gmx.fr
sfr.fr
neuf.fr
free.fr
gmx.de
hotmail.de
live.de
online.de
t-online.de
web.de
yahoo.de
libero.it
virgilio.it
hotmail.it
aol.it
tiscali.it
alice.it
live.it
yahoo.it
email.it
tin.it
poste.it
teletu.it
mail.ru
rambler.ru
yandex.ru
ya.ru
list.ru
hotmail.be
live.be
skynet.be
voo.be
tvcablenet.be
telenet.be
hotmail.com.ar
live.com.ar
yahoo.com.ar
fibertel.com.ar
speedy.com.ar
arnet.com.ar
yahoo.com.mx
live.com.mx
hotmail.es
hotmail.com.mx
prodigy.net.mx
yahoo.ca
hotmail.ca
bell.net
shaw.ca
sympatico.ca
rogers.com
yahoo.com.br
hotmail.com.br
outlook.com.br
uol.com.br
bol.com.br
terra.com.br
ig.com.br
itelefonica.com.br
r7.com
zipmail.com.br
globo.com
globomail.com
oi.com.br
+21
View File
@@ -0,0 +1,21 @@
import { Config, Guild, Session } from "@fosscord/util";
export async function initInstance() {
// TODO: clean up database and delete tombstone data
// TODO: set first user as instance administrator/or generate one if none exists and output it in the terminal
// create default guild and add it to auto join
// TODO: check if any current user is not part of autoJoinGuilds
const { autoJoin } = Config.get().guild;
if (autoJoin.enabled && !autoJoin.guilds?.length) {
let guild = await Guild.findOne({ where: {}, select: ["id"] });
if (guild) {
// @ts-ignore
await Config.set({ guild: { autoJoin: { guilds: [guild.id] } } });
}
}
// TODO: do no clear sessions for instance cluster
await Session.delete({});
}
+287
View File
@@ -0,0 +1,287 @@
import {
Channel,
Embed,
emitEvent,
Guild,
Message,
MessageCreateEvent,
MessageUpdateEvent,
getPermission,
getRights,
CHANNEL_MENTION,
Snowflake,
USER_MENTION,
ROLE_MENTION,
Role,
EVERYONE_MENTION,
HERE_MENTION,
MessageType,
User,
Application,
Webhook,
Attachment,
Config,
Sticker,
} from "@fosscord/util";
import { HTTPError } from "lambert-server";
import fetch from "node-fetch";
import cheerio from "cheerio";
import { MessageCreateSchema } from "../../routes/channels/#channel_id/messages";
import { In } from "typeorm";
const allow_empty = false;
// TODO: check webhook, application, system author, stickers
// TODO: embed gifs/videos/images
const LINK_REGEX = /https?:\/\/(www\.)?[-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_\+.~#?&//=]*)/g;
const DEFAULT_FETCH_OPTIONS: any = {
redirect: "follow",
follow: 1,
headers: {
"user-agent": "Mozilla/5.0 (compatible; Fosscord/1.0; +https://github.com/fosscord/fosscord)"
},
// size: 1024 * 1024 * 5, // grabbed from config later
compress: true,
method: "GET"
};
export async function handleMessage(opts: MessageOptions): Promise<Message> {
const channel = await Channel.findOneOrFail({ where: { id: opts.channel_id }, relations: ["recipients"] });
if (!channel || !opts.channel_id) throw new HTTPError("Channel not found", 404);
const stickers = opts.sticker_ids ? await Sticker.find({ where: { id: In(opts.sticker_ids) } }) : undefined;
const message = Message.create({
...opts,
id: Snowflake.generate(),
sticker_items: stickers,
guild_id: channel.guild_id,
channel_id: opts.channel_id,
attachments: opts.attachments || [],
embeds: opts.embeds || [],
reactions: /*opts.reactions ||*/[],
type: opts.type ?? 0,
});
if (message.content && message.content.length > Config.get().limits.message.maxCharacters) {
throw new HTTPError("Content length over max character limit");
}
if (opts.author_id) {
message.author = await User.getPublicUser(opts.author_id);
const rights = await getRights(opts.author_id);
rights.hasThrow("SEND_MESSAGES");
}
if (opts.application_id) {
message.application = await Application.findOneOrFail({ where: { id: opts.application_id } });
}
if (opts.webhook_id) {
message.webhook = await Webhook.findOneOrFail({ where: { id: opts.webhook_id } });
}
const permission = await getPermission(opts.author_id, channel.guild_id, opts.channel_id);
permission.hasThrow("SEND_MESSAGES");
if (permission.cache.member) {
message.member = permission.cache.member;
}
if (opts.tts) permission.hasThrow("SEND_TTS_MESSAGES");
if (opts.message_reference) {
permission.hasThrow("READ_MESSAGE_HISTORY");
// code below has to be redone when we add custom message routing
if (message.guild_id !== null) {
const guild = await Guild.findOneOrFail({ where: { id: channel.guild_id } });
if (!guild.features.includes("CROSS_CHANNEL_REPLIES")) {
if (opts.message_reference.guild_id !== channel.guild_id) throw new HTTPError("You can only reference messages from this guild");
if (opts.message_reference.channel_id !== opts.channel_id) throw new HTTPError("You can only reference messages from this channel");
}
}
/** Q: should be checked if the referenced message exists? ANSWER: NO
otherwise backfilling won't work **/
// @ts-ignore
message.type = MessageType.REPLY;
}
// TODO: stickers/activity
if (!allow_empty && (!opts.content && !opts.embeds?.length && !opts.attachments?.length && !opts.sticker_ids?.length)) {
throw new HTTPError("Empty messages are not allowed", 50006);
}
var content = opts.content;
var mention_channel_ids = [] as string[];
var mention_role_ids = [] as string[];
var mention_user_ids = [] as string[];
var mention_everyone = false;
if (content) { // TODO: explicit-only mentions
message.content = content.trim();
for (const [_, mention] of content.matchAll(CHANNEL_MENTION)) {
if (!mention_channel_ids.includes(mention)) mention_channel_ids.push(mention);
}
for (const [_, mention] of content.matchAll(USER_MENTION)) {
if (!mention_user_ids.includes(mention)) mention_user_ids.push(mention);
}
await Promise.all(
Array.from(content.matchAll(ROLE_MENTION)).map(async ([_, mention]) => {
const role = await Role.findOneOrFail({ where: { id: mention, guild_id: channel.guild_id } });
if (role.mentionable || permission.has("MANAGE_ROLES")) {
mention_role_ids.push(mention);
}
})
);
if (permission.has("MENTION_EVERYONE")) {
mention_everyone = !!content.match(EVERYONE_MENTION) || !!content.match(HERE_MENTION);
}
}
message.mention_channels = mention_channel_ids.map((x) => Channel.create({ id: x }));
message.mention_roles = mention_role_ids.map((x) => Role.create({ id: x }));
message.mentions = mention_user_ids.map((x) => User.create({ id: x }));
message.mention_everyone = mention_everyone;
// TODO: check and put it all in the body
return message;
}
// TODO: cache link result in db
export async function postHandleMessage(message: Message) {
var links = message.content?.match(LINK_REGEX);
if (!links) return;
const data = { ...message };
data.embeds = data.embeds.filter((x) => x.type !== "link");
links = links.slice(0, 20) as RegExpMatchArray; // embed max 20 links — TODO: make this configurable with instance policies
const { endpointPublic, resizeWidthMax, resizeHeightMax } = Config.get().cdn;
for (const link of links) {
try {
const request = await fetch(link, {
...DEFAULT_FETCH_OPTIONS,
size: Config.get().limits.message.maxEmbedDownloadSize,
});
let embed: Embed;
const type = request.headers.get("content-type");
if (type?.indexOf("image") == 0) {
embed = {
provider: {
url: link,
name: new URL(link).hostname,
},
image: {
// can't be bothered rn
proxy_url: `${endpointPublic}/external/resize/${encodeURIComponent(link)}?width=500&height=400`,
url: link,
width: 500,
height: 400
}
};
data.embeds.push(embed);
}
else {
const text = await request.text();
const $ = cheerio.load(text);
const title = $('meta[property="og:title"]').attr("content");
const provider_name = $('meta[property="og:site_name"]').text();
const author_name = $('meta[property="article:author"]').attr("content");
const description = $('meta[property="og:description"]').attr("content") || $('meta[property="description"]').attr("content");
const image = $('meta[property="og:image"]').attr("content");
const width = parseInt($('meta[property="og:image:width"]').attr("content") || "") || undefined;
const height = parseInt($('meta[property="og:image:height"]').attr("content") || "") || undefined;
const url = $('meta[property="og:url"]').attr("content");
// TODO: color
embed = {
provider: {
url: link,
name: provider_name
}
};
const resizeWidth = Math.min(resizeWidthMax ?? 1, width ?? 100);
const resizeHeight = Math.min(resizeHeightMax ?? 1, height ?? 100);
if (author_name) embed.author = { name: author_name };
if (image) embed.thumbnail = {
proxy_url: `${endpointPublic}/external/resize/${encodeURIComponent(image)}?width=${resizeWidth}&height=${resizeHeight}`,
url: image,
width: width,
height: height
};
if (title) embed.title = title;
if (url) embed.url = url;
if (description) embed.description = description;
const approvedProviders = [
"media4.giphy.com",
"c.tenor.com",
// todo: make configurable? don't really care tho
];
// very bad code below
// don't care lol
if (embed?.thumbnail?.url && approvedProviders.indexOf(new URL(embed.thumbnail.url).hostname) !== -1) {
embed = {
provider: {
url: link,
name: new URL(link).hostname,
},
image: {
proxy_url: `${endpointPublic}/external/resize/${encodeURIComponent(image!)}?width=${resizeWidth}&height=${resizeHeight}`,
url: image,
width: width,
height: height
}
};
}
if (title || description) {
data.embeds.push(embed);
}
}
} catch (error) { }
}
await Promise.all([
emitEvent({
event: "MESSAGE_UPDATE",
channel_id: message.channel_id,
data
} as MessageUpdateEvent),
Message.update({ id: message.id, channel_id: message.channel_id }, { embeds: data.embeds })
]);
}
export async function sendMessage(opts: MessageOptions) {
const message = await handleMessage({ ...opts, timestamp: new Date() });
await Promise.all([
Message.insert(message),
emitEvent({ event: "MESSAGE_CREATE", channel_id: opts.channel_id, data: message.toJSON() } as MessageCreateEvent)
]);
postHandleMessage(message).catch((e) => { }); // no await as it should catch error non-blockingly
return message;
}
interface MessageOptions extends MessageCreateSchema {
id?: string;
type?: MessageType;
pinned?: boolean;
author_id?: string;
webhook_id?: string;
application_id?: string;
embeds?: Embed[];
channel_id?: string;
attachments?: Attachment[];
edited_timestamp?: Date;
timestamp?: Date;
}
+32
View File
@@ -0,0 +1,32 @@
import { Config } from "@fosscord/util";
import { distanceBetweenLocations, IPAnalysis } from "../utility/ipAddress";
export async function getVoiceRegions(ipAddress: string, vip: boolean) {
const regions = Config.get().regions;
const availableRegions = regions.available.filter((ar) => (vip ? true : !ar.vip));
let optimalId = regions.default;
if (!regions.useDefaultAsOptimal) {
const clientIpAnalysis = await IPAnalysis(ipAddress);
let min = Number.POSITIVE_INFINITY;
for (let ar of availableRegions) {
//TODO the endpoint location should be saved in the database if not already present to prevent IPAnalysis call
const dist = distanceBetweenLocations(clientIpAnalysis, ar.location || (await IPAnalysis(ar.endpoint)));
if (dist < min) {
min = dist;
optimalId = ar.id;
}
}
}
return availableRegions.map((ar) => ({
id: ar.id,
name: ar.name,
custom: ar.custom,
deprecated: ar.deprecated,
optimal: ar.id === optimalId
}));
}
+78
View File
@@ -0,0 +1,78 @@
import {
ajv,
DiscordApiErrors,
EVENT,
FieldErrors,
FosscordApiErrors,
getPermission,
getRights,
normalizeBody,
PermissionResolvable,
Permissions,
RightResolvable,
Rights
} from "@fosscord/util";
import { NextFunction, Request, Response } from "express";
import { AnyValidateFunction } from "ajv/dist/core";
declare global {
namespace Express {
interface Request {
permission?: Permissions;
}
}
}
export type RouteResponse = { status?: number; body?: `${string}Response`; headers?: Record<string, string> };
export interface RouteOptions {
permission?: PermissionResolvable;
right?: RightResolvable;
body?: `${string}Schema`; // typescript interface name
test?: {
response?: RouteResponse;
body?: any;
path?: string;
event?: EVENT | EVENT[];
headers?: Record<string, string>;
};
}
export function route(opts: RouteOptions) {
var validate: AnyValidateFunction<any> | undefined;
if (opts.body) {
validate = ajv.getSchema(opts.body);
if (!validate) throw new Error(`Body schema ${opts.body} not found`);
}
return async (req: Request, res: Response, next: NextFunction) => {
if (opts.permission) {
const required = new Permissions(opts.permission);
req.permission = await getPermission(req.user_id, req.params.guild_id, req.params.channel_id);
// bitfield comparison: check if user lacks certain permission
if (!req.permission.has(required)) {
throw DiscordApiErrors.MISSING_PERMISSIONS.withParams(opts.permission as string);
}
}
if (opts.right) {
const required = new Rights(opts.right);
req.rights = await getRights(req.user_id);
if (!req.rights || !req.rights.has(required)) {
throw FosscordApiErrors.MISSING_RIGHTS.withParams(opts.right as string);
}
}
if (validate) {
const valid = validate(normalizeBody(req.body));
if (!valid) {
const fields: Record<string, { code?: string; message: string }> = {};
validate.errors?.forEach((x) => (fields[x.instancePath.slice(1)] = { code: x.keyword, message: x.message || "" }));
throw FieldErrors(fields);
}
}
next();
};
}
+9
View File
@@ -0,0 +1,9 @@
export * from "./utility/Base64";
export * from "./utility/ipAddress";
export * from "./handlers/Message";
export * from "./utility/passwordStrength";
export * from "./utility/RandomInviteID";
export * from "./handlers/route";
export * from "./utility/String";
export * from "./handlers/Voice";
export * from "./utility/captcha";
+47
View File
@@ -0,0 +1,47 @@
const alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+";
// binary to string lookup table
const b2s = alphabet.split("");
// string to binary lookup table
// 123 == 'z'.charCodeAt(0) + 1
const s2b = new Array(123);
for (let i = 0; i < alphabet.length; i++) {
s2b[alphabet.charCodeAt(i)] = i;
}
// number to base64
export const ntob = (n: number): string => {
if (n < 0) return `-${ntob(-n)}`;
let lo = n >>> 0;
let hi = (n / 4294967296) >>> 0;
let right = "";
while (hi > 0) {
right = b2s[0x3f & lo] + right;
lo >>>= 6;
lo |= (0x3f & hi) << 26;
hi >>>= 6;
}
let left = "";
do {
left = b2s[0x3f & lo] + left;
lo >>>= 6;
} while (lo > 0);
return left + right;
};
// base64 to number
export const bton = (base64: string) => {
let number = 0;
const sign = base64.charAt(0) === "-" ? 1 : 0;
for (let i = sign; i < base64.length; i++) {
number = number * 64 + s2b[base64.charCodeAt(i)];
}
return sign ? -number : number;
};
+32
View File
@@ -0,0 +1,32 @@
import { Snowflake } from "@fosscord/util";
export function random(length = 6) {
// Declare all characters
let chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
// Pick characers randomly
let str = "";
for (let i = 0; i < length; i++) {
str += chars.charAt(Math.floor(Math.random() * chars.length));
}
return str;
}
export function snowflakeBasedInvite() {
// Declare all characters
let chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
let base = BigInt(chars.length);
let snowflake = Snowflake.generateWorkerProcess();
// snowflakes hold ~10.75 characters worth of entropy;
// safe to generate a 8-char invite out of them
let str = "";
for (let i=0; i < 10; i++) {
str.concat(chars.charAt(Number(snowflake % base)));
snowflake = snowflake / base;
}
return str.substr(3,8).split("").reverse().join("");
}
+18
View File
@@ -0,0 +1,18 @@
import { Request } from "express";
import { ntob } from "./Base64";
import { FieldErrors } from "@fosscord/util";
export function checkLength(str: string, min: number, max: number, key: string, req: Request) {
if (str.length < min || str.length > max) {
throw FieldErrors({
[key]: {
code: "BASE_TYPE_BAD_LENGTH",
message: req.t("common:field.BASE_TYPE_BAD_LENGTH", { length: `${min} - ${max}` })
}
});
}
}
export function generateCode() {
return ntob(Date.now() + Math.randomIntBetween(0, 10000));
}
+46
View File
@@ -0,0 +1,46 @@
import { Config } from "@fosscord/util";
import fetch from "node-fetch";
export interface hcaptchaResponse {
success: boolean;
challenge_ts: string;
hostname: string;
credit: boolean;
"error-codes": string[];
score: number; // enterprise only
score_reason: string[]; // enterprise only
}
export interface recaptchaResponse {
success: boolean;
score: number; // between 0 - 1
action: string;
challenge_ts: string;
hostname: string;
"error-codes"?: string[];
}
const verifyEndpoints = {
hcaptcha: "https://hcaptcha.com/siteverify",
recaptcha: "https://www.google.com/recaptcha/api/siteverify",
}
export async function verifyCaptcha(response: string, ip?: string) {
const { security } = Config.get();
const { service, secret, sitekey } = security.captcha;
if (!service) throw new Error("Cannot verify captcha without service");
const res = await fetch(verifyEndpoints[service], {
method: "POST",
headers: {
"Content-Type": "application/x-www-form-urlencoded",
},
body: `response=${encodeURIComponent(response)}`
+ `&secret=${encodeURIComponent(secret!)}`
+ `&sitekey=${encodeURIComponent(sitekey!)}`
+ (ip ? `&remoteip=${encodeURIComponent(ip!)}` : ""),
});
return await res.json() as hcaptchaResponse | recaptchaResponse;
}
+95
View File
@@ -0,0 +1,95 @@
import { Config } from "@fosscord/util";
import { Request } from "express";
// use ipdata package instead of simple fetch because of integrated caching
import fetch from "node-fetch";
const exampleData = {
ip: "",
is_eu: true,
city: "",
region: "",
region_code: "",
country_name: "",
country_code: "",
continent_name: "",
continent_code: "",
latitude: 0,
longitude: 0,
postal: "",
calling_code: "",
flag: "",
emoji_flag: "",
emoji_unicode: "",
asn: {
asn: "",
name: "",
domain: "",
route: "",
type: "isp"
},
languages: [
{
name: "",
native: ""
}
],
currency: {
name: "",
code: "",
symbol: "",
native: "",
plural: ""
},
time_zone: {
name: "",
abbr: "",
offset: "",
is_dst: true,
current_time: ""
},
threat: {
is_tor: false,
is_proxy: false,
is_anonymous: false,
is_known_attacker: false,
is_known_abuser: false,
is_threat: false,
is_bogon: false
},
count: 0,
status: 200
};
//TODO add function that support both ip and domain names
export async function IPAnalysis(ip: string): Promise<typeof exampleData> {
const { ipdataApiKey } = Config.get().security;
if (!ipdataApiKey) return { ...exampleData, ip };
return (await fetch(`https://api.ipdata.co/${ip}?api-key=${ipdataApiKey}`)).json() as any; // TODO: types
}
export function isProxy(data: typeof exampleData) {
if (!data || !data.asn || !data.threat) return false;
if (data.asn.type !== "isp") return true;
if (Object.values(data.threat).some((x) => x)) return true;
return false;
}
export function getIpAdress(req: Request): string {
// @ts-ignore
return req.headers[Config.get().security.forwadedFor] || req.socket.remoteAddress;
}
export function distanceBetweenLocations(loc1: any, loc2: any): number {
return distanceBetweenCoords(loc1.latitude, loc1.longitude, loc2.latitude, loc2.longitude);
}
//Haversine function
function distanceBetweenCoords(lat1: number, lon1: number, lat2: number, lon2: number) {
const p = 0.017453292519943295; // Math.PI / 180
const c = Math.cos;
const a = 0.5 - c((lat2 - lat1) * p) / 2 + (c(lat1 * p) * c(lat2 * p) * (1 - c((lon2 - lon1) * p))) / 2;
return 12742 * Math.asin(Math.sqrt(a)); // 2 * R; R = 6371 km
}
+60
View File
@@ -0,0 +1,60 @@
import { Config } from "@fosscord/util";
import "missing-native-js-functions";
const reNUMBER = /[0-9]/g;
const reUPPERCASELETTER = /[A-Z]/g;
const reSYMBOLS = /[A-Z,a-z,0-9]/g;
const blocklist: string[] = []; // TODO: update ones passwordblocklist is stored in db
/*
* https://en.wikipedia.org/wiki/Password_policy
* password must meet following criteria, to be perfect:
* - min <n> chars
* - min <n> numbers
* - min <n> symbols
* - min <n> uppercase chars
* - shannon entropy folded into [0, 1) interval
*
* Returns: 0 > pw > 1
*/
export function checkPassword(password: string): number {
const { minLength, minNumbers, minUpperCase, minSymbols } = Config.get().register.password;
var strength = 0;
// checks for total password len
if (password.length >= minLength - 1) {
strength += 0.05;
}
// checks for amount of Numbers
if (password.count(reNUMBER) >= minNumbers - 1) {
strength += 0.05;
}
// checks for amount of Uppercase Letters
if (password.count(reUPPERCASELETTER) >= minUpperCase - 1) {
strength += 0.05;
}
// checks for amount of symbols
if (password.replace(reSYMBOLS, "").length >= minSymbols - 1) {
strength += 0.05;
}
// checks if password only consists of numbers or only consists of chars
if (password.length == password.count(reNUMBER) || password.length === password.count(reUPPERCASELETTER)) {
strength = 0;
}
let entropyMap: { [key: string]: number } = {};
for (let i = 0; i < password.length; i++) {
if (entropyMap[password[i]]) entropyMap[password[i]]++;
else entropyMap[password[i]] = 1;
}
let entropies = Object.values(entropyMap);
entropies.map(x => (x / entropyMap.length));
strength += entropies.reduceRight((a: number, x: number) => a - (x * Math.log2(x))) / Math.log2(password.length);
return strength;
}