Merge branch 'master' into slowcord

api/src/middlewares/RateLimit.ts

@@ -1,4 +1,4 @@
-import { Config, listenEvent } from "@fosscord/util";
+import { Config, getRights, listenEvent, Rights } from "@fosscord/util";
 import { NextFunction, Request, Response, Router } from "express";
 import { getIpAdress } from "@fosscord/api";
 import { API_PREFIX_TRAILING_SLASH } from "./Authentication";
@@ -9,6 +9,7 @@ import { API_PREFIX_TRAILING_SLASH } from "./Authentication";
 
 /*
 ? bucket limit? Max actions/sec per bucket?
+(ANSWER: a small fosscord instance might not need a complex rate limiting system)
 
 TODO: delay database requests to include multiple queries
 TODO: different for methods (GET/POST)
@@ -44,21 +45,25 @@ export default function rateLimit(opts: {
 	onlyIp?: boolean;
 }): any {
 	return async (req: Request, res: Response, next: NextFunction): Promise<any> => {
+		// exempt user? if so, immediately short circuit
+		const rights = await getRights(req.user_id);
+		if (rights.has("BYPASS_RATE_LIMITS")) return;
+		
 		const bucket_id = opts.bucket || req.originalUrl.replace(API_PREFIX_TRAILING_SLASH, "");
 		var executor_id = getIpAdress(req);
-		if (!opts.onlyIp && req.user_id) executor_id = req.user_id;
+		if (!opts.onlyIp && req.user_id) executor_id = req.user_id;		
 
 		var max_hits = opts.count;
 		if (opts.bot && req.user_bot) max_hits = opts.bot;
 		if (opts.GET && ["GET", "OPTIONS", "HEAD"].includes(req.method)) max_hits = opts.GET;
 		else if (opts.MODIFY && ["POST", "DELETE", "PATCH", "PUT"].includes(req.method)) max_hits = opts.MODIFY;
 
-		const offender = Cache.get(executor_id + bucket_id);
+		let offender = Cache.get(executor_id + bucket_id);
 
 		if (offender) {
-			const reset = offender.expires_at.getTime();
-			const resetAfterMs = reset - Date.now();
-			const resetAfterSec = resetAfterMs / 1000;
+			let reset = offender.expires_at.getTime();
+			let resetAfterMs = reset - Date.now();
+			let resetAfterSec = Math.ceil(resetAfterMs / 1000);
 
 			if (resetAfterMs <= 0) {
 				offender.hits = 0;
@@ -70,6 +75,11 @@ export default function rateLimit(opts: {
 
 			if (offender.blocked) {
 				const global = bucket_id === "global";
+				// each block violation pushes the expiry one full window further
+				reset += opts.window * 1000;
+				offender.expires_at = new Date(offender.expires_at.getTime() + opts.window * 1000);
+				resetAfterMs = reset - Date.now();
+				resetAfterSec = Math.ceil(resetAfterMs / 1000);
 
 				console.log("blocked bucket: " + bucket_id, { resetAfterMs });
 				return (

api/src/routes/auth/register.ts

@@ -128,7 +128,7 @@ router.post("/", route({ body: "RegisterSchema" }), async (req: Request, res: Re
 		throw FieldErrors({
 			date_of_birth: { code: "BASE_TYPE_REQUIRED", message: req.t("common:field.BASE_TYPE_REQUIRED") }
 		});
-	} else if (register.dateOfBirth.minimum) {
+	} else if (register.dateOfBirth.required && register.dateOfBirth.minimum) {
 		const minimum = new Date();
 		minimum.setFullYear(minimum.getFullYear() - register.dateOfBirth.minimum);
 		body.date_of_birth = new Date(body.date_of_birth as Date);

api/src/routes/channels/#channel_id/messages/#message_id/index.ts

@@ -2,13 +2,16 @@ import {
 	Attachment,
 	Channel,
 	Embed,
+	DiscordApiErrors,
 	emitEvent,
+	FosscordApiErrors,
 	getPermission,
 	getRights,
  	Message,
 	MessageCreateEvent,
 	MessageDeleteEvent,
 	MessageUpdateEvent,
+	Snowflake,
 	uploadFile 
 } from "@fosscord/util";
 import { Router, Response, Request } from "express";
@@ -16,6 +19,7 @@ import multer from "multer";
 import { route } from "@fosscord/api";
 import { handleMessage, postHandleMessage } from "@fosscord/api";
 import { MessageCreateSchema } from "../index";
+import { HTTPError } from "lambert-server";
 
 const router = Router();
 // TODO: message content/embed string length limit
@@ -90,6 +94,25 @@ router.put(
 		const { channel_id, message_id } = req.params;
 		var body = req.body as MessageCreateSchema;
 		const attachments: Attachment[] = [];
+		
+		const rights = await getRights(req.user_id);
+		rights.hasThrow("SEND_MESSAGES");
+
+		// regex to check if message contains anything other than numerals ( also no decimals )
+		if (!message_id.match(/^\+?\d+$/)) {
+			throw new HTTPError("Message IDs must be positive integers", 400);
+		}
+
+		const snowflake = Snowflake.deconstruct(message_id)
+		if (Date.now() < snowflake.timestamp) {
+			// message is in the future
+			throw FosscordApiErrors.CANNOT_BACKFILL_TO_THE_FUTURE;
+		}
+
+		const exists = await Message.findOne({ where: { id: message_id, channel_id: channel_id }});
+		if (exists) {
+			throw FosscordApiErrors.CANNOT_REPLACE_BY_BACKFILL;
+		}
 
 		if (req.file) {
 			try {
@@ -100,8 +123,6 @@ router.put(
 			}
 		}
 		const channel = await Channel.findOneOrFail({ where: { id: channel_id }, relations: ["recipients", "recipients.user"] });
-		
-		// TODO: check the ID is not from the future, to prevent future-faking of channel histories
 
 		const embeds = body.embeds || [];
 		if (body.embed) embeds.push(body.embed);
@@ -115,11 +136,9 @@ router.put(
 			channel_id,
 			attachments,
 			edited_timestamp: undefined,
-			timestamp: undefined, // FIXME: calculate timestamp from snowflake
+			timestamp: new Date(snowflake.timestamp),
 		});
 
-		channel.last_message_id = message.id;
-
 		//Fix for the client bug
 		delete message.member
 		

api/src/routes/channels/#channel_id/messages/bulk-delete.ts

@@ -1,5 +1,5 @@
 import { Router, Response, Request } from "express";
-import { Channel, Config, emitEvent, getPermission, MessageDeleteBulkEvent, Message } from "@fosscord/util";
+import { Channel, Config, emitEvent, getPermission, getRights, MessageDeleteBulkEvent, Message } from "@fosscord/util";
 import { HTTPError } from "lambert-server";
 import { route } from "@fosscord/api";
 import { In } from "typeorm";
@@ -12,22 +12,28 @@ export interface BulkDeleteSchema {
 	messages: string[];
 }
 
-// TODO: should users be able to bulk delete messages or only bots?
-// TODO: should this request fail, if you provide messages older than 14 days/invalid ids?
+// should users be able to bulk delete messages or only bots? ANSWER: all users
+// should this request fail, if you provide messages older than 14 days/invalid ids? ANSWER: NO
 // https://discord.com/developers/docs/resources/channel#bulk-delete-messages
 router.post("/", route({ body: "BulkDeleteSchema" }), async (req: Request, res: Response) => {
 	const { channel_id } = req.params;
 	const channel = await Channel.findOneOrFail({ id: channel_id });
 	if (!channel.guild_id) throw new HTTPError("Can't bulk delete dm channel messages", 400);
 
+	const rights = await getRights(req.user_id);
+	rights.hasThrow("SELF_DELETE_MESSAGES");
+	
+	let superuser = rights.has("MANAGE_MESSAGES");
 	const permission = await getPermission(req.user_id, channel?.guild_id, channel_id);
-	permission.hasThrow("MANAGE_MESSAGES");
-
+		
 	const { maxBulkDelete } = Config.get().limits.message;
 
 	const { messages } = req.body as { messages: string[] };
-	if (messages.length < 2) throw new HTTPError("You must at least specify 2 messages to bulk delete");
-	if (messages.length > maxBulkDelete) throw new HTTPError(`You cannot delete more than ${maxBulkDelete} messages`);
+	if (messages.length === 0) throw new HTTPError("You must specify messages to bulk delete");
+	if (!superuser) {
+		permission.hasThrow("MANAGE_MESSAGES");
+		if (messages.length > maxBulkDelete) throw new HTTPError(`You cannot delete more than ${maxBulkDelete} messages`);
+	}
 
 	await Message.delete(messages.map((x) => ({ id: x })));
 

api/src/routes/channels/#channel_id/messages/index.ts

@@ -11,6 +11,7 @@ import {
 	getRights,
 	Message,
 	MessageCreateEvent,
+	Snowflake,
 	uploadFile,
 	Member
 } from "@fosscord/util";
@@ -30,6 +31,8 @@ export function isTextChannel(type: ChannelType): boolean {
 		case ChannelType.GUILD_VOICE:
 		case ChannelType.GUILD_STAGE_VOICE:
 		case ChannelType.GUILD_CATEGORY:
+		case ChannelType.GUILD_FORUM:
+		case ChannelType.DIRECTORY:
 			throw new HTTPError("not a text channel", 400);
 		case ChannelType.DM:
 		case ChannelType.GROUP_DM:
@@ -68,7 +71,11 @@ export interface MessageCreateSchema {
 	};
 	payload_json?: string;
 	file?: any;
-	attachments?: any[]; //TODO we should create an interface for attachments
+	/**
+	TODO: we should create an interface for attachments
+	TODO: OpenWAAO<-->attachment-style metadata conversion
+	**/
+	attachments?: any[];
 	sticker_ids?: string[];
 }
 
@@ -84,7 +91,7 @@ router.get("/", async (req: Request, res: Response) => {
 	const before = req.query.before ? `${req.query.before}` : undefined;
 	const after = req.query.after ? `${req.query.after}` : undefined;
 	const limit = Number(req.query.limit) || 50;
-	if (limit < 1 || limit > 100) throw new HTTPError("limit must be between 1 and 100");
+	if (limit < 1 || limit > 100) throw new HTTPError("limit must be between 1 and 100", 422);
 
 	var halfLimit = Math.floor(limit / 2);
 
@@ -98,9 +105,16 @@ router.get("/", async (req: Request, res: Response) => {
 		where: { channel_id },
 		relations: ["author", "webhook", "application", "mentions", "mention_roles", "mention_channels", "sticker_items", "attachments"]
 	};
+	
 
-	if (after) query.where.id = MoreThan(after);
-	else if (before) query.where.id = LessThan(before);
+	if (after) {
+		if (after > new Snowflake()) return res.status(422);
+		query.where.id = MoreThan(after);
+	}
+	else if (before) { 
+		if (before < req.params.channel_id) return res.status(422);
+		query.where.id = LessThan(before);
+	}
 	else if (around) {
 		query.where.id = [
 			MoreThan((BigInt(around) - BigInt(halfLimit)).toString()),
@@ -126,9 +140,12 @@ router.get("/", async (req: Request, res: Response) => {
 				const uri = y.proxy_url.startsWith("http") ? y.proxy_url : `https://example.org${y.proxy_url}`;
 				y.proxy_url = `${endpoint == null ? "" : endpoint}${new URL(uri).pathname}`;
 			});
-
-			//Some clients ( discord.js ) only check if a property exists within the response,
-			//which causes erorrs when, say, the `application` property is `null`.
+			
+			/**
+			Some clients ( discord.js ) only check if a property exists within the response,
+			which causes erorrs when, say, the `application` property is `null`.
+			**/
+			
 			for (var curr in x) {
 				if (x[curr] === null)
 					delete x[curr];
@@ -148,15 +165,14 @@ const messageUpload = multer({
 	},
 	storage: multer.memoryStorage()
 }); // max upload 50 mb
+/**
+ TODO: dynamically change limit of MessageCreateSchema with config
 
-// TODO: dynamically change limit of MessageCreateSchema with config
-// TODO: check: sum of all characters in an embed structure must not exceed instance limits
-
-// https://discord.com/developers/docs/resources/channel#create-message
-// TODO: text channel slowdown
-// TODO: trim and replace message content and every embed field
-// TODO: check allowed_mentions
-
+ https://discord.com/developers/docs/resources/channel#create-message
+ TODO: text channel slowdown (per-user and across-users)
+ Q: trim and replace message content and every embed field A: NO, given this cannot be implemented in E2EE channels
+ TODO: only dispatch notifications for mentions denoted in allowed_mentions
+**/
 // Send message
 router.post(
 	"/",
@@ -223,8 +239,6 @@ router.post(
 				})
 			);
 		}
-
-
 	
 		//Fix for the client bug
 		delete message.member
@@ -241,3 +255,4 @@ router.post(
 		return res.json(message);
 	}
 );
+

api/src/routes/channels/#channel_id/purge.ts

@@ -0,0 +1,84 @@
+import { HTTPError } from "lambert-server";
+import { route } from "@fosscord/api";
+import { isTextChannel } from "./messages";
+import { FindManyOptions, Between, Not } from "typeorm";
+import {
+	Attachment,
+	Channel,
+	Config,
+	Embed,
+	DiscordApiErrors,
+	emitEvent,
+	FosscordApiErrors,
+	getPermission,
+	getRights,
+ 	Message,
+	MessageDeleteBulkEvent,
+	Snowflake,
+	uploadFile 
+} from "@fosscord/util";
+import { Router, Response, Request } from "express";
+import multer from "multer";
+import { handleMessage, postHandleMessage } from "@fosscord/api";
+
+const router: Router = Router();
+
+export default router;
+
+export interface PurgeSchema {
+	before: string;
+	after: string
+}
+
+/**
+TODO: apply the delete bit by bit to prevent client and database stress
+**/
+router.post("/", route({ /*body: "PurgeSchema",*/ }), async (req: Request, res: Response) => {
+	const { channel_id } = req.params;
+	const channel = await Channel.findOneOrFail({ id: channel_id });
+	
+	if (!channel.guild_id) throw new HTTPError("Can't purge dm channels", 400);
+	isTextChannel(channel.type);
+
+	const rights = await getRights(req.user_id);
+	if (!rights.has("MANAGE_MESSAGES")) {
+		const permissions = await getPermission(req.user_id, channel.guild_id, channel_id);
+		permissions.hasThrow("MANAGE_MESSAGES");
+		permissions.hasThrow("MANAGE_CHANNELS");
+	}
+	
+	const { before, after } = req.body as PurgeSchema;
+
+	// TODO: send the deletion event bite-by-bite to prevent client stress
+
+	var query: FindManyOptions<Message> & { where: { id?: any; }; } = {
+		order: { id: "ASC" },
+		// take: limit,
+		where: {
+		 channel_id,
+		 id: Between(after, before), // the right way around
+		 author_id: rights.has("SELF_DELETE_MESSAGES") ? undefined : Not(req.user_id)
+		 // if you lack the right of self-deletion, you can't delete your own messages, even in purges
+		 },
+		relations: ["author", "webhook", "application", "mentions", "mention_roles", "mention_channels", "sticker_items", "attachments"]
+	};
+	
+
+	const messages = await Message.find(query);
+	const endpoint = Config.get().cdn.endpointPublic;
+			
+	if (messages.length == 0) { 
+		res.sendStatus(304);
+		return;
+	}
+
+	await Message.delete(messages.map((x) => ({ id: x })));
+	
+	await emitEvent({
+		event: "MESSAGE_DELETE_BULK",
+		channel_id,
+		data: { ids: messages.map(x => x.id), channel_id, guild_id: channel.guild_id }
+	} as MessageDeleteBulkEvent);
+
+	res.sendStatus(204);
+});

api/src/routes/guilds/#guild_id/prune.ts

@@ -11,6 +11,10 @@ export const inactiveMembers = async (guild_id: string, user_id: string, days: n
 	//Snowflake should have `generateFromTime` method? Or similar?
 	var minId = BigInt(date.valueOf() - Snowflake.EPOCH) << BigInt(22);
 
+	/**
+	idea: ability to customise the cutoff variable
+	possible candidates: public read receipt, last presence, last VC leave
+	**/
 	var members = await Member.find({
 		where: [
 			{
@@ -47,7 +51,7 @@ export const inactiveMembers = async (guild_id: string, user_id: string, days: n
 	return members;
 };
 
-router.get("/", route({ permission: "KICK_MEMBERS" }), async (req: Request, res: Response) => {
+router.get("/", route({}), async (req: Request, res: Response) => {
 	const days = parseInt(req.query.days as string);
 
 	var roles = req.query.include_roles;
@@ -65,7 +69,7 @@ export interface PruneSchema {
 	days: number;
 }
 
-router.post("/", route({ permission: "KICK_MEMBERS" }), async (req: Request, res: Response) => {
+router.post("/", route({ permission: "KICK_MEMBERS", right: "KICK_BAN_MEMBERS" }), async (req: Request, res: Response) => {
 	const days = parseInt(req.body.days);
 
 	var roles = req.query.include_roles;