From b7b9cc78c92a724cc9e296500990b2894ccd5675 Mon Sep 17 00:00:00 2001 From: gnuxie Date: Tue, 11 Mar 2025 17:46:16 +0000 Subject: [PATCH 01/48] Initial RoomTakedown "service". I want to change the APIs a little bit so that we can get a takedown reporter like the audit log but for message rendering and testing. I also want to make the BetterSqliteStore concrete not depend on where the database came from so that we can use in memory databases for unit testing stores. There are a couple other FIXME's littered around so be sure to check those before merging. --- .../better-sqlite3/BetterSqliteStore.ts | 2 +- src/backingstore/better-sqlite3/HashStore.ts | 175 ++++++++++++++++++ src/protections/RoomTakedown/RoomAuditLog.ts | 15 ++ src/protections/RoomTakedown/RoomTakedown.ts | 134 ++++++++++++++ .../RoomTakedown/SqliteRoomAuditLog.ts | 120 ++++++++++++ 5 files changed, 445 insertions(+), 1 deletion(-) create mode 100644 src/backingstore/better-sqlite3/HashStore.ts create mode 100644 src/protections/RoomTakedown/RoomAuditLog.ts create mode 100644 src/protections/RoomTakedown/RoomTakedown.ts create mode 100644 src/protections/RoomTakedown/SqliteRoomAuditLog.ts diff --git a/src/backingstore/better-sqlite3/BetterSqliteStore.ts b/src/backingstore/better-sqlite3/BetterSqliteStore.ts index 51b6ae47..4400e6e9 100644 --- a/src/backingstore/better-sqlite3/BetterSqliteStore.ts +++ b/src/backingstore/better-sqlite3/BetterSqliteStore.ts @@ -59,7 +59,7 @@ export type SchemaUpdateFunction = (db: Database) => void; * store.ensureSchema(); * const data = await store.getData(); */ -export abstract class BetterSqliteStore { +export class BetterSqliteStore { private hasEnded = false; public readonly db: Database; diff --git a/src/backingstore/better-sqlite3/HashStore.ts b/src/backingstore/better-sqlite3/HashStore.ts new file mode 100644 index 00000000..e700991b --- /dev/null +++ b/src/backingstore/better-sqlite3/HashStore.ts @@ -0,0 +1,175 @@ +// SPDX-FileCopyrightText: 2025 Gnuxie +// +// SPDX-License-Identifier: Apache-2.0 + +import { isError, Ok, Result } from "@gnuxie/typescript-result"; +import { StringRoomID } from "@the-draupnir-project/matrix-basic-types"; +import { + ActionException, + ActionExceptionKind, + HashedLiteralPolicyRule, + LiteralPolicyRule, + makeReversedHashedPolicy, + RoomHashRecord, + SHA256RoomHashStore, +} from "matrix-protection-suite"; +import { BetterSqliteOptions, BetterSqliteStore } from "./BetterSqliteStore"; +import { SHA256 } from "crypto-js"; + +// We will need some kind of audit database to store actions in against rooms + +const schema = [ + `CREATE TABLE room_sha256 ( + room_id TEXT PRIMARY KEY NOT NULL, + sha256 BLOB NOT NULL + ) STRICT;`, +]; + +type RoomSha256 = { + room_id: StringRoomID; + sha256: string; +}; + +export class SqliteHashReversalStore + extends BetterSqliteStore + implements Omit +{ + constructor(options: BetterSqliteOptions) { + super( + schema.map( + (text) => + function (db) { + db.prepare(text).run(); + } + ), + options + ); + this.db.pragma("journal_mode = WAL"); + this.db.pragma("foreign_keys = ON"); + this.db.pragma("temp_store = file"); // Avoid unnecessary memory usage. + this.ensureSchema(); + } + public async findRoomHash( + hash: string + ): Promise> { + try { + return Ok( + this.db + .prepare(`SELECT room_id FROM room_sha256 WHERE sha256 = base64(?)`) + .get(hash) as StringRoomID | undefined + ); + } catch (e) { + if (e instanceof Error) { + return ActionException.Result( + `Unexpected error while querying for a room hash`, + { + exception: e, + exceptionKind: ActionExceptionKind.Unknown, + } + ); + } else { + throw e; + } + } + } + + private async findMatchingRooms( + policies: HashedLiteralPolicyRule[] + ): Promise> { + const sha256s: string[] = []; + for (const policy of policies) { + if (policy.hashes["sha256"] !== undefined) { + sha256s.push(policy.hashes["sha256"]); + } + } + // This query looks sketch and I don't like it but we're just putting in question marks and nothing else + // I can't believe there's not a builtin for this fml. + try { + const statement = this.db.prepare( + `SELECT (room_id, base64(sha256)) FROM room_sha256 WHERE sha256 IN (${sha256s.map(() => "base64(?)").join(",")})` + ); + return Ok(statement.all(sha256s) as RoomSha256[]); + } catch (exception) { + if (exception instanceof Error) { + return ActionException.Result( + "Unexpected error while attempting to reverse hashed policies", + { + exception, + exceptionKind: ActionExceptionKind.Unknown, + } + ); + } else { + throw exception; + } + } + } + + public async reverseHashedRoomPolicies( + policies: HashedLiteralPolicyRule[] + ): Promise> { + const matches = await this.findMatchingRooms(policies); + if (isError(matches)) { + return matches; + } + const reversedPolicies: LiteralPolicyRule[] = []; + for (const match of matches.ok) { + const hashedLiteral = policies.find( + (policy) => policy.hashes["sha256"] === match.sha256 + ); + if (hashedLiteral == undefined) { + throw new TypeError( + "Something has gone badly wrong we should be able to find the policy we entered" + ); + } + reversedPolicies.push( + makeReversedHashedPolicy(match.room_id, hashedLiteral) + ); + } + return Ok(reversedPolicies); + } + + private findExistingRooms(roomIDs: StringRoomID[]): RoomHashRecord[] { + if (roomIDs.length === 0) { + return []; + } + const placeholders = roomIDs.map(() => "?").join(", "); + const statement = this.db.prepare( + `SELECT room_id, sha256 FROM room_sha256 WHERE room_id IN (${placeholders})` + ); + return statement.all(...roomIDs) as RoomHashRecord[]; + } + + public async storeUndiscoveredRooms( + roomIDs: StringRoomID[] + ): Promise> { + try { + const existingRooms = this.findExistingRooms(roomIDs); + const rowsToInsert: RoomHashRecord[] = []; + for (const roomID of roomIDs) { + if (!existingRooms.find((row) => row.room_id === roomID)) { + rowsToInsert.push({ + room_id: roomID, + sha256: SHA256(roomID).toString(), + }); + } + } + const statement = this.db.prepare( + `REPLACE INTO room_sha256 VALUES ${roomIDs.map(() => `(?, base64(?))`).join(", ")}` + ); + statement.run(rowsToInsert.map((row) => [row.room_id, row.sha256])); + return Ok(rowsToInsert); + } catch (exception) { + if (exception instanceof Error) { + return ActionException.Result( + `Error while trying to update known hashed rooms`, + { + exception, + exceptionKind: ActionExceptionKind.Unknown, + } + ); + } else { + throw exception; + } + } + } +} diff --git a/src/protections/RoomTakedown/RoomAuditLog.ts b/src/protections/RoomTakedown/RoomAuditLog.ts new file mode 100644 index 00000000..89100e9b --- /dev/null +++ b/src/protections/RoomTakedown/RoomAuditLog.ts @@ -0,0 +1,15 @@ +// SPDX-FileCopyrightText: 2025 Gnuxie +// +// SPDX-License-Identifier: Apache-2.0 + +import { Result } from "@gnuxie/typescript-result"; +import { StringRoomID } from "@the-draupnir-project/matrix-basic-types"; +import { LiteralPolicyRule } from "matrix-protection-suite"; + +// FIXME: Add a method for protection startup that takes all room ids from policies +// and only returns those not takendown +// i just want to save progress so far first. +export interface RoomAuditLog { + takedownRoom(policy: LiteralPolicyRule): Promise>; + isRoomTakendown(roomID: StringRoomID): boolean; +} diff --git a/src/protections/RoomTakedown/RoomTakedown.ts b/src/protections/RoomTakedown/RoomTakedown.ts new file mode 100644 index 00000000..d6d9504f --- /dev/null +++ b/src/protections/RoomTakedown/RoomTakedown.ts @@ -0,0 +1,134 @@ +// SPDX-FileCopyrightText: 2025 Gnuxie +// +// SPDX-License-Identifier: Apache-2.0 + +import { StringRoomID } from "@the-draupnir-project/matrix-basic-types"; +import { + LiteralPolicyRule, + Logger, + PolicyListRevision, + PolicyRuleChange, + PolicyRuleMatchType, + PolicyRuleType, + Recommendation, + SHA256RoomHashStore, + SimpleChangeType, +} from "matrix-protection-suite"; +import { RoomAuditLog } from "./RoomAuditLog"; +import { isError, Ok, Result, ResultError } from "@gnuxie/typescript-result"; + +const log = new Logger("RoomTakedown"); + +type RoomTakedown = { + handleDiscoveredRooms(rooms: StringRoomID[]): Promise>; + handlePolicyChange( + revision: PolicyListRevision, + changes: PolicyRuleChange[] + ): Promise>; + checkAllRooms(revision: PolicyListRevision): Promise>; +}; + +export type RoomTakedownCapability = { + isRoomTakendown(roomID: StringRoomID): Promise>; + takedownRoom(roomID: StringRoomID): Promise>; +}; + +/** + * This exists as the main handler for reacting to literal room policies + * (that have been reversed from hashed policies elsewhere) AND + * moving discovered rooms into the hashStore. Although it's not clear + * whether we need to do that? + */ +export class StandardRoomTakedown implements RoomTakedown { + public constructor( + private readonly hashStore: SHA256RoomHashStore, + private readonly auditLog: RoomAuditLog, + private readonly takedownCapability: RoomTakedownCapability + ) { + // nothing to do + } + public async handleDiscoveredRooms( + rooms: StringRoomID[] + ): Promise> { + return (await this.hashStore.storeUndiscoveredRooms(rooms)) as Result; + } + + private async takedownRoom( + roomID: StringRoomID, + rule: LiteralPolicyRule + ): Promise> { + const isRoomTakendownResult = + await this.takedownCapability.isRoomTakendown(roomID); + if (isError(isRoomTakendownResult)) { + return isRoomTakendownResult; + } + if (isRoomTakendownResult.ok) { + return ResultError.Result( + `The room ${roomID} has already been takendown according to your homeserver` + ); + } + const takedownResult = await this.takedownCapability.takedownRoom(roomID); + if (isError(takedownResult)) { + return takedownResult; + } else { + return await this.auditLog.takedownRoom(rule); + } + } + + public async handlePolicyChange( + revision: PolicyListRevision, + changes: PolicyRuleChange[] + ): Promise> { + const roomsToTakedown: LiteralPolicyRule[] = []; + for (const change of changes) { + if ( + change.changeType === SimpleChangeType.Added && + change.rule.kind === PolicyRuleType.Room && + change.rule.matchType === PolicyRuleMatchType.Literal && + change.rule.recommendation === Recommendation.Takedown + ) { + if ( + !this.auditLog.isRoomTakendown(change.rule.entity as StringRoomID) + ) { + roomsToTakedown.push(change.rule); + } + } + } + for (const policy of roomsToTakedown) { + const takedownResult = await this.takedownRoom( + policy.entity as StringRoomID, + policy + ); + if (isError(takedownResult)) { + log.error( + "Error while trying to takedown the room:", + policy.entity, + takedownResult.error + ); + } + } + return Ok(undefined); + } + + public async checkAllRooms( + revision: PolicyListRevision + ): Promise> { + const roomPolicies = revision + .allRulesOfType(PolicyRuleType.Room, Recommendation.Takedown) + .filter((policy) => policy.matchType === PolicyRuleMatchType.Literal); + for (const policy of roomPolicies) { + const takedownResult = await this.takedownRoom( + policy.entity as StringRoomID, + policy + ); + if (isError(takedownResult)) { + log.error( + "Error while trying to takedown the room:", + policy.entity, + takedownResult.error + ); + } + } + return Ok(undefined); + } +} diff --git a/src/protections/RoomTakedown/SqliteRoomAuditLog.ts b/src/protections/RoomTakedown/SqliteRoomAuditLog.ts new file mode 100644 index 00000000..5e400c6e --- /dev/null +++ b/src/protections/RoomTakedown/SqliteRoomAuditLog.ts @@ -0,0 +1,120 @@ +// SPDX-FileCopyrightText: 2025 Gnuxie +// +// SPDX-License-Identifier: Apache-2.0 + +import { Ok, Result } from "@gnuxie/typescript-result"; +import { StringRoomID } from "@the-draupnir-project/matrix-basic-types"; +import { RoomAuditLog } from "./RoomAuditLog"; +import { + ActionException, + ActionExceptionKind, + LiteralPolicyRule, + PolicyRuleType, +} from "matrix-protection-suite"; +import { BetterSqliteStore } from "../../backingstore/better-sqlite3/BetterSqliteStore"; + +// NOTE: This should only be used to check in bulk whether rooms are taken down +// upon getting a policy, you probably always want to try again or +// check with the server rather than check the audit log. +// Incase rooms were unbanned. + +const schema = [ + `CREATE TABLE policy_info ( + policy_id TEXT PRIMARY KEY NOT NULL, + sender_user_id TEXT NOT NULL, + room_id TEXT NOT NULL, + state_key TEXT NOT NULL, + type TEXT NOT NULL, + recommendation TEXT NOT NULL, + ) STRICT;`, + `CREATE TABLE room_takedown ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + policy_id TEXT NOT NULL, + target_room_id TEXT NOT NULL, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP NOT NULL, + FOREIGN KEY (policy_id) REFERENCES policy_info(policy_id) + ) STRICT;`, +]; + +export type RoomTakedown = { + id: number; + policy_id: string; + target_room_id: StringRoomID; + created_at: number; +}; + +export class SqliteRoomAuditLog + extends BetterSqliteStore + implements RoomAuditLog +{ + private readonly takedownRooms: Set; + public constructor(path: string) { + super( + schema.map( + (text) => + function (db) { + db.prepare(text).run(); + } + ), + { + path, + fileMustExist: false, + } + ); + // FIXME: We need a way to move these directives somewhere else... + // so that i can pass an in memory database to this function + // for testing.... + this.db.pragma("journal_mode = WAL"); + this.db.pragma("foreign_keys = ON"); + this.db.pragma("temp_store = file"); // Avoid unnecessary memory usage. + this.ensureSchema(); + this.takedownRooms = new Set(this.loadTakendownRooms()); + } + + public async takedownRoom(policy: LiteralPolicyRule): Promise> { + if (policy.kind !== PolicyRuleType.Room) { + throw new TypeError( + "You can only use takedownRoom on room policies what are you doing?" + ); + } + try { + const policyInfoStatement = this.db.prepare( + `REPLACE INTO policy_info (policy_id, sender_user_id, room_id, state_key, type, recommendation) + VALUES (?, ?, ?, ?, ?, ?)` + ); + const takedownStatement = this.db.prepare(` + INSERT INTO room_takedown (policy_id, target_room_id) VALUES (?, ?)`); + this.db.transaction(() => { + policyInfoStatement.run([ + policy.sourceEvent.event_id, + policy.sourceEvent.sender, + policy.sourceEvent.room_id, + policy.sourceEvent.state_key, + policy.sourceEvent.type, + policy.recommendation, + ]); + takedownStatement.run([policy.sourceEvent.event_id, policy.entity]); + })(); + this.takedownRooms.add(policy.entity as StringRoomID); + return Ok(undefined); + } catch (exception) { + if (exception instanceof Error) { + return ActionException.Result("Unable to log takedown", { + exception, + exceptionKind: ActionExceptionKind.Unknown, + }); + } else { + throw exception; + } + } + } + + private loadTakendownRooms(): StringRoomID[] { + return this.db + .prepare(`SELECT room_id FROM room_takedown`) + .all() as StringRoomID[]; + } + isRoomTakendown(roomID: StringRoomID): boolean { + return this.takedownRooms.has(roomID); + } +} From 7282374caf966d2f6e34c550c4ba553a26224899 Mon Sep 17 00:00:00 2001 From: gnuxie Date: Wed, 12 Mar 2025 11:51:48 +0000 Subject: [PATCH 02/48] Clean up better-sqlite store so that we can test stores with in memeory dbs. --- .../better-sqlite3/BetterSqliteStore.ts | 46 ++++++++++++++----- src/backingstore/better-sqlite3/HashStore.ts | 9 ++-- .../SqliteRoomStateBackingStore.ts | 29 ++++++++---- .../RoomTakedown/SqliteRoomAuditLog.ts | 20 ++++---- 4 files changed, 66 insertions(+), 38 deletions(-) diff --git a/src/backingstore/better-sqlite3/BetterSqliteStore.ts b/src/backingstore/better-sqlite3/BetterSqliteStore.ts index 4400e6e9..dc26f56c 100644 --- a/src/backingstore/better-sqlite3/BetterSqliteStore.ts +++ b/src/backingstore/better-sqlite3/BetterSqliteStore.ts @@ -34,6 +34,39 @@ export interface BetterSqliteOptions extends BetterSqlite3.Options { * Defaults to `true`. */ autocreateSchemaTable?: boolean; + + WALMode?: boolean; + foreignKeys?: boolean; +} + +export function makeBetterSqliteDB(options: BetterSqliteOptions): Database { + const db = new BetterSqlite3(options.path, options); + if (options.path !== ":memory:") { + db.pragma("temp_store = file"); // Avoid unnecessary memory usage. + } + if (options.WALMode) { + db.pragma("journal_mode = WAL"); + } + if (options.foreignKeys) { + db.pragma("foreign_keys = ON"); + } + let hasEnded = false; + process.once("beforeExit", () => { + // Ensure we clean up on exit + try { + log.info("Destroy called on db", options.path); + if (hasEnded) { + // No-op if end has already been called. + return; + } + hasEnded = true; + db.close(); + log.info("connection ended on db", options.path); + } catch (ex) { + log.warn("Failed to cleanly exit", ex); + } + }); + return db; } export type SchemaUpdateFunction = (db: Database) => void; @@ -61,7 +94,6 @@ export type SchemaUpdateFunction = (db: Database) => void; */ export class BetterSqliteStore { private hasEnded = false; - public readonly db: Database; public get latestSchema() { return this.schemas.length; @@ -75,18 +107,10 @@ export class BetterSqliteStore { */ constructor( private readonly schemas: SchemaUpdateFunction[], - private readonly opts: BetterSqliteOptions + private readonly opts: BetterSqliteOptions, + protected readonly db: Database ) { opts.autocreateSchemaTable = opts.autocreateSchemaTable ?? true; - this.db = new BetterSqlite3(opts.path, opts); - process.once("beforeExit", () => { - // Ensure we clean up on exit - try { - this.destroy(); - } catch (ex) { - log.warn("Failed to cleanly exit", ex); - } - }); } /** diff --git a/src/backingstore/better-sqlite3/HashStore.ts b/src/backingstore/better-sqlite3/HashStore.ts index e700991b..0c738d47 100644 --- a/src/backingstore/better-sqlite3/HashStore.ts +++ b/src/backingstore/better-sqlite3/HashStore.ts @@ -15,6 +15,7 @@ import { } from "matrix-protection-suite"; import { BetterSqliteOptions, BetterSqliteStore } from "./BetterSqliteStore"; import { SHA256 } from "crypto-js"; +import { Database } from "better-sqlite3"; // We will need some kind of audit database to store actions in against rooms @@ -34,7 +35,7 @@ export class SqliteHashReversalStore extends BetterSqliteStore implements Omit { - constructor(options: BetterSqliteOptions) { + constructor(options: BetterSqliteOptions, db: Database) { super( schema.map( (text) => @@ -42,11 +43,9 @@ export class SqliteHashReversalStore db.prepare(text).run(); } ), - options + options, + db ); - this.db.pragma("journal_mode = WAL"); - this.db.pragma("foreign_keys = ON"); - this.db.pragma("temp_store = file"); // Avoid unnecessary memory usage. this.ensureSchema(); } public async findRoomHash( diff --git a/src/backingstore/better-sqlite3/SqliteRoomStateBackingStore.ts b/src/backingstore/better-sqlite3/SqliteRoomStateBackingStore.ts index e1c624f6..20bd2e16 100644 --- a/src/backingstore/better-sqlite3/SqliteRoomStateBackingStore.ts +++ b/src/backingstore/better-sqlite3/SqliteRoomStateBackingStore.ts @@ -15,10 +15,16 @@ import { StateEvent, isError, } from "matrix-protection-suite"; -import { BetterSqliteStore, flatTransaction } from "./BetterSqliteStore"; +import { + BetterSqliteOptions, + BetterSqliteStore, + flatTransaction, + makeBetterSqliteDB, +} from "./BetterSqliteStore"; import { jsonReviver } from "../../utils"; import { StringRoomID } from "@the-draupnir-project/matrix-basic-types"; import path from "path"; +import { Database } from "better-sqlite3"; const log = new Logger("SqliteRoomStateBackingStore"); @@ -48,7 +54,8 @@ export class SqliteRoomStateBackingStore private readonly roomInfoMap = new Map(); public readonly revisionListener = this.handleRevision.bind(this); public constructor( - path: string, + options: BetterSqliteOptions, + db: Database, private readonly eventDecoder: EventDecoder ) { super( @@ -58,14 +65,9 @@ export class SqliteRoomStateBackingStore db.prepare(text).run(); } ), - { - path, - fileMustExist: false, - } + options, + db ); - this.db.pragma("journal_mode = WAL"); - this.db.pragma("foreign_keys = ON"); - this.db.pragma("temp_store = file"); // Avoid unnecessary memory usage. this.ensureSchema(); } @@ -73,8 +75,15 @@ export class SqliteRoomStateBackingStore storagePath: string, eventDecoder: EventDecoder ): SqliteRoomStateBackingStore { + const options = { + path: path.join(storagePath, "room-state-backing-store.db"), + WALMode: true, + foreignKeys: true, + fileMustExist: false, + }; return new SqliteRoomStateBackingStore( - path.join(storagePath, "room-state-backing-store.db"), + options, + makeBetterSqliteDB(options), eventDecoder ); } diff --git a/src/protections/RoomTakedown/SqliteRoomAuditLog.ts b/src/protections/RoomTakedown/SqliteRoomAuditLog.ts index 5e400c6e..8b161cc8 100644 --- a/src/protections/RoomTakedown/SqliteRoomAuditLog.ts +++ b/src/protections/RoomTakedown/SqliteRoomAuditLog.ts @@ -11,7 +11,11 @@ import { LiteralPolicyRule, PolicyRuleType, } from "matrix-protection-suite"; -import { BetterSqliteStore } from "../../backingstore/better-sqlite3/BetterSqliteStore"; +import { + BetterSqliteOptions, + BetterSqliteStore, +} from "../../backingstore/better-sqlite3/BetterSqliteStore"; +import { Database } from "better-sqlite3"; // NOTE: This should only be used to check in bulk whether rooms are taken down // upon getting a policy, you probably always want to try again or @@ -48,7 +52,7 @@ export class SqliteRoomAuditLog implements RoomAuditLog { private readonly takedownRooms: Set; - public constructor(path: string) { + public constructor(options: BetterSqliteOptions, db: Database) { super( schema.map( (text) => @@ -56,17 +60,9 @@ export class SqliteRoomAuditLog db.prepare(text).run(); } ), - { - path, - fileMustExist: false, - } + options, + db ); - // FIXME: We need a way to move these directives somewhere else... - // so that i can pass an in memory database to this function - // for testing.... - this.db.pragma("journal_mode = WAL"); - this.db.pragma("foreign_keys = ON"); - this.db.pragma("temp_store = file"); // Avoid unnecessary memory usage. this.ensureSchema(); this.takedownRooms = new Set(this.loadTakendownRooms()); } From b8bf6c56925ede10b4e6aa707c5cfa414c2c75b7 Mon Sep 17 00:00:00 2001 From: gnuxie Date: Wed, 12 Mar 2025 15:01:57 +0000 Subject: [PATCH 03/48] Add RoomTakedownCapability and SynapseAdmin implementation. --- src/capabilities/RoomTakedownCapability.tsx | 45 +++++++ .../RoomTakedownCapabilityRenderer.tsx | 124 ++++++++++++++++++ .../SynapseAdminRoomTakedown.ts | 64 +++++++++ src/capabilities/capabilityIndex.ts | 2 + src/protections/RoomTakedown/RoomTakedown.ts | 6 +- 5 files changed, 236 insertions(+), 5 deletions(-) create mode 100644 src/capabilities/RoomTakedownCapability.tsx create mode 100644 src/capabilities/RoomTakedownCapabilityRenderer.tsx create mode 100644 src/capabilities/SynapseAdminRoomTakedown/SynapseAdminRoomTakedown.ts diff --git a/src/capabilities/RoomTakedownCapability.tsx b/src/capabilities/RoomTakedownCapability.tsx new file mode 100644 index 00000000..55cb3dac --- /dev/null +++ b/src/capabilities/RoomTakedownCapability.tsx @@ -0,0 +1,45 @@ +// SPDX-FileCopyrightText: 2025 Gnuxie +// +// SPDX-License-Identifier: Apache-2.0 + +import { Result } from "@gnuxie/typescript-result"; +import { Type } from "@sinclair/typebox"; +import { + StringRoomID, + StringUserID, +} from "@the-draupnir-project/matrix-basic-types"; +import { + Capability, + CapabilityMethodSchema, + describeCapabilityInterface, +} from "matrix-protection-suite"; + +export type RoomTakedownDetails = Partial<{ + creator: StringUserID | undefined; + room_id: StringRoomID; + name: string | undefined; + topic: string | undefined; + avatar: string | undefined; +}>; + +export const RoomTakedownCapability = Type.Intersect([ + Type.Object({ + isRoomTakendown: CapabilityMethodSchema, + takedownRoom: CapabilityMethodSchema, + }), + Capability, +]); +// we'll probably want to include room details too so we can provide some +// rudinemtary information to takedown. +// I'd make the fields optional however because it may be impossible +// to get those details on conduwuit. +export type RoomTakedownCapability = { + isRoomTakendown(roomID: StringRoomID): Promise>; + takedownRoom(roomID: StringRoomID): Promise>; +} & Capability; + +describeCapabilityInterface({ + name: "RoomTakedownCapability", + description: "Capability that targets matrix rooms", + schema: RoomTakedownCapability, +}); diff --git a/src/capabilities/RoomTakedownCapabilityRenderer.tsx b/src/capabilities/RoomTakedownCapabilityRenderer.tsx new file mode 100644 index 00000000..11f53613 --- /dev/null +++ b/src/capabilities/RoomTakedownCapabilityRenderer.tsx @@ -0,0 +1,124 @@ +// SPDX-FileCopyrightText: 2025 Gnuxie +// +// SPDX-License-Identifier: Apache-2.0 + +import { + describeCapabilityRenderer, + DescriptionMeta, +} from "matrix-protection-suite"; +import { + RoomTakedownCapability, + RoomTakedownDetails, +} from "./RoomTakedownCapability"; +import { RendererMessageCollector } from "./RendererMessageCollector"; +import { + MatrixRoomReference, + StringRoomID, +} from "@the-draupnir-project/matrix-basic-types"; +import { isError, Result } from "@gnuxie/typescript-result"; +import { + DeadDocumentJSX, + DocumentNode, +} from "@the-draupnir-project/interface-manager"; +import { renderFailedSingularConsequence } from "./CommonRenderers"; +import { renderRoomPill } from "../commands/interface-manager/MatrixHelpRenderer"; +import { Draupnir } from "../Draupnir"; + +function renderCodeOrDefault( + item: string | undefined, + defaultText: string +): DocumentNode { + return item === undefined ? ( + {defaultText} + ) : ( + {item} + ); +} + +function renderTakedown( + roomID: StringRoomID, + details: RoomTakedownDetails +): DocumentNode { + return ( +
+ + Successfully takendown the room{" "} + {renderRoomPill(MatrixRoomReference.fromRoomID(roomID))} + +
    +
  • name: {renderCodeOrDefault(details.name, "no name available")}
    • +
  • + creator:{" "} + {renderCodeOrDefault( + details.creator, + "creator information unavailable" + )} +
    • +
  • + topic: {renderCodeOrDefault(details.topic, "topic unavailable")} +
    • +
+
+ ); +} + +class StandardRoomTakedownCapabilityRenderer implements RoomTakedownCapability { + public readonly requiredEventPermissions = + this.capability.requiredEventPermissions; + public readonly requiredPermissions = this.capability.requiredPermissions; + public readonly requiredStatePermissions = + this.capability.requiredStatePermissions; + constructor( + private readonly description: DescriptionMeta, + private readonly messageCollector: RendererMessageCollector, + private readonly capability: RoomTakedownCapability + ) { + // nothing to do. + } + + public async isRoomTakendown(roomID: StringRoomID): Promise> { + // pass this one through without rendering because it doesn't really have a consequence + return await this.capability.isRoomTakendown(roomID); + } + + public async takedownRoom( + roomID: StringRoomID + ): Promise> { + const capabilityResult = await this.capability.takedownRoom(roomID); + if (isError(capabilityResult)) { + this.messageCollector.addOneliner( + this.description, + this.capability, + renderFailedSingularConsequence( + this.description, + + Failed to takedown room{" "} + {renderRoomPill(MatrixRoomReference.fromRoomID(roomID))} + , + capabilityResult.error + ) + ); + return capabilityResult; + } + this.messageCollector.addMessage( + this.description, + this.capability, + renderTakedown(roomID, capabilityResult.ok) + ); + return capabilityResult; + } +} + +describeCapabilityRenderer({ + name: "StandardRoomTakedownCapabilityRenderer", + description: "Renders the standard room takedown capability result", + interface: "RoomTakedownCapability", + isDefaultForInterface: true, + factory(description, draupnir, capability) { + return new StandardRoomTakedownCapabilityRenderer( + description, + draupnir.capabilityMessageRenderer, + capability + ); + }, +}); diff --git a/src/capabilities/SynapseAdminRoomTakedown/SynapseAdminRoomTakedown.ts b/src/capabilities/SynapseAdminRoomTakedown/SynapseAdminRoomTakedown.ts new file mode 100644 index 00000000..88bb4490 --- /dev/null +++ b/src/capabilities/SynapseAdminRoomTakedown/SynapseAdminRoomTakedown.ts @@ -0,0 +1,64 @@ +// SPDX-FileCopyrightText: 2025 Gnuxie +// +// SPDX-License-Identifier: Apache-2.0 + +import { StringRoomID } from "@the-draupnir-project/matrix-basic-types"; +import { Logger } from "matrix-protection-suite"; +import { + RoomTakedownCapability, + RoomTakedownDetails, +} from "../RoomTakedownCapability"; +import { SynapseAdminClient } from "matrix-protection-suite-for-matrix-bot-sdk"; +import { isError, Ok, Result } from "@gnuxie/typescript-result"; + +const log = new Logger("SynapseAdminRoomTakedownCapability"); + +export class SynapseAdminRoomTakedownCapability + implements RoomTakedownCapability +{ + public readonly requiredPermissions = []; + public readonly requiredStatePermissions = []; + public readonly requiredEventPermissions = []; + public constructor(private readonly adminClient: SynapseAdminClient) { + // nothing to do mare. + } + public async isRoomTakendown(roomID: StringRoomID): Promise> { + const blockStatusResponse = await this.adminClient.getBlockStatus(roomID); + if (isError(blockStatusResponse)) { + return blockStatusResponse; + } else { + return Ok(blockStatusResponse.ok.block); + } + } + + public async takedownRoom( + roomID: StringRoomID + ): Promise> { + const detailsResponse = await this.adminClient.getRoomDetails(roomID); + let details: RoomTakedownDetails; + if (isError(detailsResponse)) { + log.warn( + "Unable to fetch details for a room being requested to shutdown", + detailsResponse.error + ); + details = { room_id: roomID }; + } else { + details = { + name: detailsResponse.ok?.name ?? undefined, + creator: detailsResponse.ok?.creator, + avatar: detailsResponse.ok?.avatar ?? undefined, + topic: detailsResponse.ok?.topic ?? undefined, + room_id: roomID, + }; + } + const takedownResult = await this.adminClient.deleteRoom(roomID, { + block: true, + purge: true, + }); + if (isError(takedownResult)) { + return takedownResult; + } else { + return Ok(details); + } + } +} diff --git a/src/capabilities/capabilityIndex.ts b/src/capabilities/capabilityIndex.ts index 8fe5f2eb..a11c57b6 100644 --- a/src/capabilities/capabilityIndex.ts +++ b/src/capabilities/capabilityIndex.ts @@ -11,3 +11,5 @@ import "./StandardEventConsequencesRenderer"; import "./ServerACLConsequencesRenderer"; import "./StandardUserConsequencesRenderer"; +import "./RoomTakedownCapability"; +import "./RoomTakedownCapabilityRenderer"; diff --git a/src/protections/RoomTakedown/RoomTakedown.ts b/src/protections/RoomTakedown/RoomTakedown.ts index d6d9504f..d63edabd 100644 --- a/src/protections/RoomTakedown/RoomTakedown.ts +++ b/src/protections/RoomTakedown/RoomTakedown.ts @@ -16,6 +16,7 @@ import { } from "matrix-protection-suite"; import { RoomAuditLog } from "./RoomAuditLog"; import { isError, Ok, Result, ResultError } from "@gnuxie/typescript-result"; +import { RoomTakedownCapability } from "../../capabilities/RoomTakedownCapability"; const log = new Logger("RoomTakedown"); @@ -28,11 +29,6 @@ type RoomTakedown = { checkAllRooms(revision: PolicyListRevision): Promise>; }; -export type RoomTakedownCapability = { - isRoomTakendown(roomID: StringRoomID): Promise>; - takedownRoom(roomID: StringRoomID): Promise>; -}; - /** * This exists as the main handler for reacting to literal room policies * (that have been reversed from hashed policies elsewhere) AND From 25f39e20d0b6c182ac773fbb94ce653d7299e596 Mon Sep 17 00:00:00 2001 From: gnuxie Date: Wed, 12 Mar 2025 15:46:24 +0000 Subject: [PATCH 04/48] Start plumbing up RoomTakedownProtection into Draupnir. There are a couple of FIXME's to address. Such as plumbing up the stores into draupnir and how to poll for new rooms. --- src/protections/RoomTakedown/RoomTakedown.ts | 2 + .../RoomTakedown/RoomTakedownProtection.ts | 94 +++++++++++++++++++ 2 files changed, 96 insertions(+) create mode 100644 src/protections/RoomTakedown/RoomTakedownProtection.ts diff --git a/src/protections/RoomTakedown/RoomTakedown.ts b/src/protections/RoomTakedown/RoomTakedown.ts index d63edabd..206bc27a 100644 --- a/src/protections/RoomTakedown/RoomTakedown.ts +++ b/src/protections/RoomTakedown/RoomTakedown.ts @@ -67,6 +67,8 @@ export class StandardRoomTakedown implements RoomTakedown { if (isError(takedownResult)) { return takedownResult; } else { + // FIXME: we should probably audit as simulated if the capability is simulated. + // or not audit at all because the protection preview might show things otherwise. return await this.auditLog.takedownRoom(rule); } } diff --git a/src/protections/RoomTakedown/RoomTakedownProtection.ts b/src/protections/RoomTakedown/RoomTakedownProtection.ts new file mode 100644 index 00000000..ae6b60c5 --- /dev/null +++ b/src/protections/RoomTakedown/RoomTakedownProtection.ts @@ -0,0 +1,94 @@ +// SPDX-FileCopyrightText: 2025 Gnuxie +// +// SPDX-License-Identifier: Apache-2.0 + +import { + AbstractProtection, + ActionResult, + describeProtection, + Ok, + PolicyListRevision, + PolicyRuleChange, + ProtectedRoomsSet, + Protection, + ProtectionDescription, + SHA256RoomHashStore, + Task, + UnknownConfig, +} from "matrix-protection-suite"; +import { RoomTakedownCapability } from "../../capabilities/RoomTakedownCapability"; +import { Draupnir } from "../../Draupnir"; +import { StandardRoomTakedown } from "./RoomTakedown"; +import { RoomAuditLog } from "./RoomAuditLog"; +import { SynapseAdminRoomTakedownCapability } from "../../capabilities/SynapseAdminRoomTakedown/SynapseAdminRoomTakedown"; + +// FIXME: We still haven't figured out how to poll for new rooms via the +// Synapse admin API. + +// FIXME: We need to add the stores to draupnir somehow. +// probably from the toplevel. + +type RoomTakedownProtectionCapabilities = { + roomTakedownCapability: RoomTakedownCapability; +}; + +type RoomTakedownProtectionDescription = ProtectionDescription< + Draupnir, + UnknownConfig, + RoomTakedownProtectionCapabilities +>; + +export class RoomTakedownProtection + extends AbstractProtection + implements Protection +{ + private readonly roomTakedown: StandardRoomTakedown; + constructor( + description: RoomTakedownProtectionDescription, + capabilities: RoomTakedownProtectionCapabilities, + protectedRoomsSet: ProtectedRoomsSet, + hashStore: SHA256RoomHashStore, + auditLog: RoomAuditLog + ) { + super(description, capabilities, protectedRoomsSet, {}); + this.roomTakedown = new StandardRoomTakedown( + hashStore, + auditLog, + capabilities.roomTakedownCapability + ); + void Task( + this.roomTakedown.checkAllRooms( + this.protectedRoomsSet.watchedPolicyRooms.currentRevision + ) + ); + } + + handlePolicyChange( + revision: PolicyListRevision, + changes: PolicyRuleChange[] + ): Promise> { + return this.roomTakedown.handlePolicyChange(revision, changes); + } +} + +describeProtection({ + name: RoomTakedownProtection.name, + description: `A protection to shutdown rooms matching policies from watched lists`, + capabilityInterfaces: { + roomTakedownCapability: "RoomTakedownCapability", + }, + defaultCapabilities: { + roomTakedownCapability: SynapseAdminRoomTakedownCapability.name, + }, + factory(description, protectedRoomsSet, draupnir, capabilitySet, _settings) { + return Ok( + new RoomTakedownProtection( + description, + capabilitySet, + protectedRoomsSet, + draupnir.hashStore, + draupnir.auditLog + ) + ); + }, +}); From b50033ba4af5fbd2c8addd174b6eabcde768ccc7 Mon Sep 17 00:00:00 2001 From: gnuxie Date: Wed, 12 Mar 2025 18:29:05 +0000 Subject: [PATCH 05/48] Fix manual launch script for changed room backing store interface. --- test/integration/manualLaunchScript.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/test/integration/manualLaunchScript.ts b/test/integration/manualLaunchScript.ts index f94a373d..2f412a9e 100644 --- a/test/integration/manualLaunchScript.ts +++ b/test/integration/manualLaunchScript.ts @@ -15,14 +15,13 @@ import { draupnirClient, makeBotModeToggle } from "./mjolnirSetupUtils"; import { configRead } from "../../src/config"; import { SqliteRoomStateBackingStore } from "../../src/backingstore/better-sqlite3/SqliteRoomStateBackingStore"; -import path from "path"; import { DefaultEventDecoder } from "matrix-protection-suite"; void (async () => { const config = configRead(); const toggle = await makeBotModeToggle(config, { - backingStore: new SqliteRoomStateBackingStore( - path.join(config.dataPath, "room-state-backing-store.db"), + backingStore: SqliteRoomStateBackingStore.create( + config.dataPath, DefaultEventDecoder ), allowSafeMode: true, From 4ae5e5f63abde5bfcedb6a31da2c0ccafafc499c Mon Sep 17 00:00:00 2001 From: gnuxie Date: Wed, 12 Mar 2025 18:30:43 +0000 Subject: [PATCH 06/48] Begin process of creating top level stores. We now need to move the roomStateBackingStore into this thingy. --- src/Draupnir.ts | 5 ++ src/DraupnirBotMode.ts | 5 +- src/appservice/AppService.ts | 6 +++ src/appservice/AppServiceDraupnirManager.ts | 7 ++- src/backingstore/DraupnirStores.ts | 46 +++++++++++++++++++ src/draupnirfactory/DraupnirFactory.ts | 8 +++- .../RoomTakedown/SqliteRoomAuditLog.ts | 12 +++++ test/integration/mjolnirSetupUtils.ts | 1 + 8 files changed, 87 insertions(+), 3 deletions(-) create mode 100644 src/backingstore/DraupnirStores.ts diff --git a/src/Draupnir.ts b/src/Draupnir.ts index 92b96fa3..28bb782a 100644 --- a/src/Draupnir.ts +++ b/src/Draupnir.ts @@ -82,6 +82,7 @@ import { makeConfirmationPromptListener, } from "./commands/interface-manager/MatrixPromptForConfirmation"; import { SynapseHttpAntispam } from "./webapis/SynapseHTTPAntispam/SynapseHttpAntispam"; +import { DraupnirStores } from "./backingstore/DraupnirStores"; const log = new Logger("Draupnir"); // webAPIS should not be included on the Draupnir class. @@ -145,6 +146,7 @@ export class Draupnir implements Client, MatrixAdaptorContext { public readonly acceptInvitesFromRoom: MatrixRoomID, public readonly acceptInvitesFromRoomIssuer: RoomMembershipRevisionIssuer, public readonly safeModeToggle: SafeModeToggle, + public readonly stores: DraupnirStores, public readonly synapseAdminClient: SynapseAdminClient | undefined, public readonly synapseHTTPAntispam: SynapseHttpAntispam | undefined ) { @@ -212,6 +214,7 @@ export class Draupnir implements Client, MatrixAdaptorContext { config: IConfig, loggableConfigTracker: LoggableConfigTracker, safeModeToggle: SafeModeToggle, + stores: DraupnirStores, synapseHTTPAntispam: SynapseHttpAntispam | undefined ): Promise> { const acceptInvitesFromRoom = await (async () => { @@ -270,6 +273,7 @@ export class Draupnir implements Client, MatrixAdaptorContext { acceptInvitesFromRoom.ok, acceptInvitesFromRoomIssuer.ok, safeModeToggle, + stores, new SynapseAdminClient(client, clientUserID), synapseHTTPAntispam ); @@ -388,6 +392,7 @@ export class Draupnir implements Client, MatrixAdaptorContext { this.clientRooms.off("timeline", this.timelineEventListener); this.reportPoller?.stop(); this.protectedRoomsSet.unregisterListeners(); + this.stores.dispose(); } public createRoomReference(roomID: StringRoomID): MatrixRoomID { diff --git a/src/DraupnirBotMode.ts b/src/DraupnirBotMode.ts index a08c8583..f0822f4e 100644 --- a/src/DraupnirBotMode.ts +++ b/src/DraupnirBotMode.ts @@ -49,6 +49,7 @@ import { ResultError } from "@gnuxie/typescript-result"; import { SafeModeCause, SafeModeReason } from "./safemode/SafeModeCause"; import { SafeModeBootOption } from "./safemode/BootOption"; import { SynapseHttpAntispam } from "./webapis/SynapseHTTPAntispam/SynapseHttpAntispam"; +import { TopLevelStores } from "./backingstore/DraupnirStores"; const log = new Logger("DraupnirBotMode"); @@ -113,6 +114,7 @@ export class DraupnirBotModeToggle implements BotModeTogle { client: MatrixSendClient, matrixEmitter: SafeMatrixEmitter, config: IConfig, + stores: TopLevelStores, backingStore?: RoomStateBackingStore ): Promise { const clientUserID = await client.getUserId(); @@ -169,7 +171,8 @@ export class DraupnirBotModeToggle implements BotModeTogle { clientsInRoomMap, clientCapabilityFactory, clientProvider, - roomStateManagerFactory + roomStateManagerFactory, + stores ); return new DraupnirBotModeToggle( clientUserID, diff --git a/src/appservice/AppService.ts b/src/appservice/AppService.ts index b10fe7d4..fe21cb6c 100644 --- a/src/appservice/AppService.ts +++ b/src/appservice/AppService.ts @@ -49,6 +49,9 @@ import { StringUserID, } from "@the-draupnir-project/matrix-basic-types"; import { SqliteRoomStateBackingStore } from "../backingstore/better-sqlite3/SqliteRoomStateBackingStore"; +import { TopLevelStores } from "../backingstore/DraupnirStores"; + +// FIXME: Move roomStateBacking store into top level stores. const log = new Logger("AppService"); /** @@ -100,6 +103,7 @@ export class MjolnirAppService { dataStore: DataStore, eventDecoder: EventDecoder, registrationFilePath: string, + stores: TopLevelStores, backingStore?: RoomStateBackingStore ) { const bridge = new Bridge({ @@ -190,6 +194,7 @@ export class MjolnirAppService { bridge, accessControl, roomStateManagerFactory, + stores, clientCapabilityFactory, clientProvider, instanceCountGauge @@ -239,6 +244,7 @@ export class MjolnirAppService { dataStore, DefaultEventDecoder, registrationFilePath, + {}, // we don't support any stores in appservice atm except backing store. backingStore ); // The call to `start` MUST happen last. As it needs the datastore, and the mjolnir manager to be initialized before it can process events from the homeserver. diff --git a/src/appservice/AppServiceDraupnirManager.ts b/src/appservice/AppServiceDraupnirManager.ts index 2ebe88b5..ad13d892 100644 --- a/src/appservice/AppServiceDraupnirManager.ts +++ b/src/appservice/AppServiceDraupnirManager.ts @@ -45,6 +45,7 @@ import { userLocalpart, isStringRoomID, } from "@the-draupnir-project/matrix-basic-types"; +import { TopLevelStores } from "../backingstore/DraupnirStores"; const log = new Logger("AppServiceDraupnirManager"); @@ -63,6 +64,7 @@ export class AppServiceDraupnirManager { private readonly bridge: Bridge, private readonly accessControl: AccessControl, private readonly roomStateManagerFactory: RoomStateManagerFactory, + stores: TopLevelStores, private readonly clientCapabilityFactory: ClientCapabilityFactory, clientProvider: ClientForUserID, private readonly instanceCountGauge: Gauge<"status" | "uuid"> @@ -71,7 +73,8 @@ export class AppServiceDraupnirManager { this.roomStateManagerFactory.clientsInRoomMap, this.clientCapabilityFactory, clientProvider, - this.roomStateManagerFactory + this.roomStateManagerFactory, + stores ); this.baseManager = new StandardDraupnirManager(draupnirFactory); } @@ -93,6 +96,7 @@ export class AppServiceDraupnirManager { bridge: Bridge, accessControl: AccessControl, roomStateManagerFactory: RoomStateManagerFactory, + stores: TopLevelStores, clientCapabilityFactory: ClientCapabilityFactory, clientProvider: ClientForUserID, instanceCountGauge: Gauge<"status" | "uuid"> @@ -103,6 +107,7 @@ export class AppServiceDraupnirManager { bridge, accessControl, roomStateManagerFactory, + stores, clientCapabilityFactory, clientProvider, instanceCountGauge diff --git a/src/backingstore/DraupnirStores.ts b/src/backingstore/DraupnirStores.ts new file mode 100644 index 00000000..8d5ef4fe --- /dev/null +++ b/src/backingstore/DraupnirStores.ts @@ -0,0 +1,46 @@ +// SPDX-FileCopyrightText: 2025 Gnuxie +// +// SPDX-License-Identifier: Apache-2.0 + +import { + SHA256RoomHashStore, + StandardSHA256RoomHashStore, +} from "matrix-protection-suite"; +import { RoomAuditLog } from "../protections/RoomTakedown/RoomAuditLog"; + +export type TopLevelStores = { + hashStore?: Omit; + roomAuditLog?: RoomAuditLog; +}; + +/** + * These stores will usually be created at the entrypoint of the draupnir + * application or attenuated for each draupnir. + * + * No i don't like it. Some of these stores ARE specific to the draupnir + * such as the hasStore's event emitter... that just can't be mixed. + * + * We could create a wrapper that disposes of only the stores that + * have been attenuated... but i don't know about it. + */ +export type DraupnirStores = { + hashStore?: SHA256RoomHashStore | undefined; + roomAuditLog?: RoomAuditLog | undefined; + /** + * Dispose of stores relevant to a specific draupnir instance. + * For example, the hash store is usually specific to a single draupnir. + */ + dispose(): void; +}; + +export function createDraupnirStores( + topLevelStores: TopLevelStores +): DraupnirStores { + return Object.freeze({ + roomAudditLog: topLevelStores.roomAuditLog, + hashStore: topLevelStores.hashStore + ? new StandardSHA256RoomHashStore(topLevelStores.hashStore) + : undefined, + dispose() {}, + }); +} diff --git a/src/draupnirfactory/DraupnirFactory.ts b/src/draupnirfactory/DraupnirFactory.ts index 85bb1af9..30d79e38 100644 --- a/src/draupnirfactory/DraupnirFactory.ts +++ b/src/draupnirfactory/DraupnirFactory.ts @@ -30,6 +30,10 @@ import { SafeModeCause } from "../safemode/SafeModeCause"; import { SafeModeToggle } from "../safemode/SafeModeToggle"; import { StandardManagementRoomDetail } from "../managementroom/ManagementRoomDetail"; import { DraupnirBotModeToggle } from "../DraupnirBotMode"; +import { + createDraupnirStores, + TopLevelStores, +} from "../backingstore/DraupnirStores"; const log = new Logger("DraupnirFactory"); @@ -59,7 +63,8 @@ export class DraupnirFactory { private readonly clientsInRoomMap: ClientsInRoomMap, private readonly clientCapabilityFactory: ClientCapabilityFactory, private readonly clientProvider: ClientForUserID, - private readonly roomStateManagerFactory: RoomStateManagerFactory + private readonly roomStateManagerFactory: RoomStateManagerFactory, + private readonly stores: TopLevelStores ) { // nothing to do. } @@ -143,6 +148,7 @@ export class DraupnirFactory { config, configLogTracker, toggle, + createDraupnirStores(this.stores), // synapseHTTPAntispam is only available in bot mode. toggle instanceof DraupnirBotModeToggle ? toggle.synapseHTTPAntispam diff --git a/src/protections/RoomTakedown/SqliteRoomAuditLog.ts b/src/protections/RoomTakedown/SqliteRoomAuditLog.ts index 8b161cc8..de0d3ecc 100644 --- a/src/protections/RoomTakedown/SqliteRoomAuditLog.ts +++ b/src/protections/RoomTakedown/SqliteRoomAuditLog.ts @@ -14,8 +14,10 @@ import { import { BetterSqliteOptions, BetterSqliteStore, + makeBetterSqliteDB, } from "../../backingstore/better-sqlite3/BetterSqliteStore"; import { Database } from "better-sqlite3"; +import path from "path"; // NOTE: This should only be used to check in bulk whether rooms are taken down // upon getting a policy, you probably always want to try again or @@ -67,6 +69,16 @@ export class SqliteRoomAuditLog this.takedownRooms = new Set(this.loadTakendownRooms()); } + public static createToplevel(storagePath: string): SqliteRoomAuditLog { + const options = { + path: path.join(storagePath, "room-audit-log.db"), + WALMode: true, + foreignKeys: true, + fileMustExist: false, + }; + return new SqliteRoomAuditLog(options, makeBetterSqliteDB(options)); + } + public async takedownRoom(policy: LiteralPolicyRule): Promise> { if (policy.kind !== PolicyRuleType.Room) { throw new TypeError( diff --git a/test/integration/mjolnirSetupUtils.ts b/test/integration/mjolnirSetupUtils.ts index 509226d4..adc91f15 100644 --- a/test/integration/mjolnirSetupUtils.ts +++ b/test/integration/mjolnirSetupUtils.ts @@ -165,6 +165,7 @@ export async function makeBotModeToggle( client, new SafeMatrixEmitterWrapper(client, DefaultEventDecoder), config, + {}, // we don't support top level stores for now. backingStore ); // we don't want to send status on startup incase we want to test e2ee from the manual launch script. From 62079457e9279088f184e3e5e394b810a5336e3a Mon Sep 17 00:00:00 2001 From: gnuxie Date: Wed, 12 Mar 2025 18:55:44 +0000 Subject: [PATCH 07/48] Move roomStateBackingStore into TopLevelStores abstraction. --- src/DraupnirBotMode.ts | 6 ++-- src/appservice/AppService.ts | 11 ++------ src/backingstore/DraupnirStores.ts | 29 +++++++++++++++++--- src/backingstore/better-sqlite3/HashStore.ts | 18 +++++++++++- src/index.ts | 11 ++++---- test/integration/manualLaunchScript.ts | 13 +++++---- test/integration/mjolnirSetupUtils.ts | 11 ++++---- 7 files changed, 65 insertions(+), 34 deletions(-) diff --git a/src/DraupnirBotMode.ts b/src/DraupnirBotMode.ts index f0822f4e..eeb849ff 100644 --- a/src/DraupnirBotMode.ts +++ b/src/DraupnirBotMode.ts @@ -11,7 +11,6 @@ import { StandardClientsInRoomMap, DefaultEventDecoder, - RoomStateBackingStore, ClientsInRoomMap, Task, Logger, @@ -114,8 +113,7 @@ export class DraupnirBotModeToggle implements BotModeTogle { client: MatrixSendClient, matrixEmitter: SafeMatrixEmitter, config: IConfig, - stores: TopLevelStores, - backingStore?: RoomStateBackingStore + stores: TopLevelStores ): Promise { const clientUserID = await client.getUserId(); if (!isStringUserID(clientUserID)) { @@ -165,7 +163,7 @@ export class DraupnirBotModeToggle implements BotModeTogle { clientsInRoomMap, clientProvider, DefaultEventDecoder, - backingStore + stores.roomStateBackingStore ); const draupnirFactory = new DraupnirFactory( clientsInRoomMap, diff --git a/src/appservice/AppService.ts b/src/appservice/AppService.ts index fe21cb6c..402e6b02 100644 --- a/src/appservice/AppService.ts +++ b/src/appservice/AppService.ts @@ -35,7 +35,6 @@ import { ClientsInRoomMap, DefaultEventDecoder, EventDecoder, - RoomStateBackingStore, StandardClientsInRoomMap, Task, isError, @@ -51,8 +50,6 @@ import { import { SqliteRoomStateBackingStore } from "../backingstore/better-sqlite3/SqliteRoomStateBackingStore"; import { TopLevelStores } from "../backingstore/DraupnirStores"; -// FIXME: Move roomStateBacking store into top level stores. - const log = new Logger("AppService"); /** * Responsible for setting up listeners and delegating functionality to a matrix-appservice-bridge `Bridge` for @@ -103,8 +100,7 @@ export class MjolnirAppService { dataStore: DataStore, eventDecoder: EventDecoder, registrationFilePath: string, - stores: TopLevelStores, - backingStore?: RoomStateBackingStore + stores: TopLevelStores ) { const bridge = new Bridge({ homeserverUrl: config.homeserver.url, @@ -132,7 +128,7 @@ export class MjolnirAppService { clientsInRoomMap, clientProvider, eventDecoder, - backingStore + stores.roomStateBackingStore ); const clientCapabilityFactory = new ClientCapabilityFactory( clientsInRoomMap, @@ -244,8 +240,7 @@ export class MjolnirAppService { dataStore, DefaultEventDecoder, registrationFilePath, - {}, // we don't support any stores in appservice atm except backing store. - backingStore + { roomStateBackingStore: backingStore } // we don't support any stores in appservice atm except backing store. ); // The call to `start` MUST happen last. As it needs the datastore, and the mjolnir manager to be initialized before it can process events from the homeserver. await service.start(port); diff --git a/src/backingstore/DraupnirStores.ts b/src/backingstore/DraupnirStores.ts index 8d5ef4fe..5863553a 100644 --- a/src/backingstore/DraupnirStores.ts +++ b/src/backingstore/DraupnirStores.ts @@ -3,14 +3,19 @@ // SPDX-License-Identifier: Apache-2.0 import { + EventDecoder, SHA256RoomHashStore, StandardSHA256RoomHashStore, } from "matrix-protection-suite"; import { RoomAuditLog } from "../protections/RoomTakedown/RoomAuditLog"; +import { SqliteRoomStateBackingStore } from "./better-sqlite3/SqliteRoomStateBackingStore"; +import { SqliteHashReversalStore } from "./better-sqlite3/HashStore"; +import { SqliteRoomAuditLog } from "../protections/RoomTakedown/SqliteRoomAuditLog"; export type TopLevelStores = { - hashStore?: Omit; - roomAuditLog?: RoomAuditLog; + hashStore?: Omit | undefined; + roomAuditLog?: RoomAuditLog | undefined; + roomStateBackingStore?: SqliteRoomStateBackingStore | undefined; }; /** @@ -37,10 +42,26 @@ export function createDraupnirStores( topLevelStores: TopLevelStores ): DraupnirStores { return Object.freeze({ - roomAudditLog: topLevelStores.roomAuditLog, + roomAuditLog: topLevelStores.roomAuditLog, hashStore: topLevelStores.hashStore ? new StandardSHA256RoomHashStore(topLevelStores.hashStore) : undefined, dispose() {}, - }); + } satisfies DraupnirStores); +} + +export function makeTopLevelStores( + storagePath: string, + eventDecoder: EventDecoder, + { + isRoomStateBackingStoreEnabled, + }: { isRoomStateBackingStoreEnabled: boolean } +): TopLevelStores { + return Object.freeze({ + roomStateBackingStore: isRoomStateBackingStoreEnabled + ? SqliteRoomStateBackingStore.create(storagePath, eventDecoder) + : undefined, + hashStore: SqliteHashReversalStore.createToplevel(storagePath), + roomAuditLog: SqliteRoomAuditLog.createToplevel(storagePath), + } satisfies TopLevelStores); } diff --git a/src/backingstore/better-sqlite3/HashStore.ts b/src/backingstore/better-sqlite3/HashStore.ts index 0c738d47..2c9316fa 100644 --- a/src/backingstore/better-sqlite3/HashStore.ts +++ b/src/backingstore/better-sqlite3/HashStore.ts @@ -13,9 +13,14 @@ import { RoomHashRecord, SHA256RoomHashStore, } from "matrix-protection-suite"; -import { BetterSqliteOptions, BetterSqliteStore } from "./BetterSqliteStore"; +import { + BetterSqliteOptions, + BetterSqliteStore, + makeBetterSqliteDB, +} from "./BetterSqliteStore"; import { SHA256 } from "crypto-js"; import { Database } from "better-sqlite3"; +import path from "path"; // We will need some kind of audit database to store actions in against rooms @@ -48,6 +53,17 @@ export class SqliteHashReversalStore ); this.ensureSchema(); } + + public static createToplevel(storagePath: string): SqliteHashReversalStore { + const options = { + path: path.join(storagePath, "hash-store.db"), + WALMode: true, + foreignKeys: true, + fileMustExist: false, + }; + return new SqliteHashReversalStore(options, makeBetterSqliteDB(options)); + } + public async findRoomHash( hash: string ): Promise> { diff --git a/src/index.ts b/src/index.ts index d29d7266..3c5eeb2f 100644 --- a/src/index.ts +++ b/src/index.ts @@ -24,7 +24,7 @@ import { initializeSentry, patchMatrixClient } from "./utils"; import { DraupnirBotModeToggle } from "./DraupnirBotMode"; import { SafeMatrixEmitterWrapper } from "matrix-protection-suite-for-matrix-bot-sdk"; import { DefaultEventDecoder } from "matrix-protection-suite"; -import { SqliteRoomStateBackingStore } from "./backingstore/better-sqlite3/SqliteRoomStateBackingStore"; +import { makeTopLevelStores } from "./backingstore/DraupnirStores"; void (async function () { const config = configRead(); @@ -83,14 +83,15 @@ void (async function () { } patchMatrixClient(); const eventDecoder = DefaultEventDecoder; - const store = config.roomStateBackingStore.enabled - ? SqliteRoomStateBackingStore.create(storagePath, eventDecoder) - : undefined; + const stores = makeTopLevelStores(storagePath, eventDecoder, { + isRoomStateBackingStoreEnabled: + config.roomStateBackingStore.enabled ?? false, + }); bot = await DraupnirBotModeToggle.create( client, new SafeMatrixEmitterWrapper(client, eventDecoder), config, - store + stores ); // We don't want to send the status on start, as we need to initialize e2ee first (using client.start); diff --git a/test/integration/manualLaunchScript.ts b/test/integration/manualLaunchScript.ts index 2f412a9e..14f17130 100644 --- a/test/integration/manualLaunchScript.ts +++ b/test/integration/manualLaunchScript.ts @@ -13,17 +13,18 @@ */ import { draupnirClient, makeBotModeToggle } from "./mjolnirSetupUtils"; -import { configRead } from "../../src/config"; -import { SqliteRoomStateBackingStore } from "../../src/backingstore/better-sqlite3/SqliteRoomStateBackingStore"; +import { configRead, getStoragePath } from "../../src/config"; import { DefaultEventDecoder } from "matrix-protection-suite"; +import { makeTopLevelStores } from "../../src/backingstore/DraupnirStores"; void (async () => { const config = configRead(); + const storagePath = getStoragePath(config.dataPath); const toggle = await makeBotModeToggle(config, { - backingStore: SqliteRoomStateBackingStore.create( - config.dataPath, - DefaultEventDecoder - ), + stores: makeTopLevelStores(storagePath, DefaultEventDecoder, { + isRoomStateBackingStoreEnabled: + config.roomStateBackingStore.enabled ?? false, + }), allowSafeMode: true, }); await draupnirClient()?.start(); diff --git a/test/integration/mjolnirSetupUtils.ts b/test/integration/mjolnirSetupUtils.ts index adc91f15..47592145 100644 --- a/test/integration/mjolnirSetupUtils.ts +++ b/test/integration/mjolnirSetupUtils.ts @@ -29,9 +29,9 @@ import { DefaultEventDecoder, MJOLNIR_PROTECTED_ROOMS_EVENT_TYPE, MJOLNIR_WATCHED_POLICY_ROOMS_EVENT_TYPE, - RoomStateBackingStore, } from "matrix-protection-suite"; import { SafeModeDraupnir } from "../../src/safemode/DraupnirSafeMode"; +import { TopLevelStores } from "../../src/backingstore/DraupnirStores"; patchMatrixClient(); @@ -127,14 +127,14 @@ let globalSafeEmitter: SafeMatrixEmitter | undefined; export async function makeBotModeToggle( config: IConfig, { - backingStore, + stores, eraseAccountData, allowSafeMode, }: { - backingStore?: RoomStateBackingStore; + stores: TopLevelStores; eraseAccountData?: boolean; allowSafeMode?: boolean; - } = {} + } = { stores: {} } ): Promise { await configureMjolnir(config); LogService.setLevel(LogLevel.fromString(config.logLevel, LogLevel.DEBUG)); @@ -165,8 +165,7 @@ export async function makeBotModeToggle( client, new SafeMatrixEmitterWrapper(client, DefaultEventDecoder), config, - {}, // we don't support top level stores for now. - backingStore + stores ); // we don't want to send status on startup incase we want to test e2ee from the manual launch script. const mj = ( From 15f94bc2b20a3e240555ef71d6fcb601286493fd Mon Sep 17 00:00:00 2001 From: gnuxie Date: Fri, 14 Mar 2025 19:15:28 +0000 Subject: [PATCH 08/48] Export request body types from SynapseHTTPAntispam. --- .../SynapseHTTPAntispam/CheckEventForSpamEndpoint.ts | 4 ++-- src/webapis/SynapseHTTPAntispam/UserMayInviteEndpoint.ts | 6 ++++-- src/webapis/SynapseHTTPAntispam/UserMayJoinRoomEndpoint.ts | 6 ++++-- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/webapis/SynapseHTTPAntispam/CheckEventForSpamEndpoint.ts b/src/webapis/SynapseHTTPAntispam/CheckEventForSpamEndpoint.ts index 9ab42682..9c69a56b 100644 --- a/src/webapis/SynapseHTTPAntispam/CheckEventForSpamEndpoint.ts +++ b/src/webapis/SynapseHTTPAntispam/CheckEventForSpamEndpoint.ts @@ -20,10 +20,10 @@ export type CheckEventForSpamListenerArguments = Parameters< (details: CheckEventForSpamRequestBody) => void >; -type CheckEventForSpamRequestBody = EDStatic< +export type CheckEventForSpamRequestBody = EDStatic< typeof CheckEventForSpamRequestBody >; -const CheckEventForSpamRequestBody = Type.Object({ +export const CheckEventForSpamRequestBody = Type.Object({ event: RoomEvent(Type.Unknown()), }); diff --git a/src/webapis/SynapseHTTPAntispam/UserMayInviteEndpoint.ts b/src/webapis/SynapseHTTPAntispam/UserMayInviteEndpoint.ts index 97016d64..3059ce35 100644 --- a/src/webapis/SynapseHTTPAntispam/UserMayInviteEndpoint.ts +++ b/src/webapis/SynapseHTTPAntispam/UserMayInviteEndpoint.ts @@ -23,8 +23,10 @@ export type UserMayInviteListenerArguments = Parameters< (details: UserMayInviteRequestBody) => void >; -type UserMayInviteRequestBody = EDStatic; -const UserMayInviteRequestBody = Type.Object({ +export type UserMayInviteRequestBody = EDStatic< + typeof UserMayInviteRequestBody +>; +export const UserMayInviteRequestBody = Type.Object({ inviter: StringUserIDSchema, invitee: StringUserIDSchema, room_id: StringRoomIDSchema, diff --git a/src/webapis/SynapseHTTPAntispam/UserMayJoinRoomEndpoint.ts b/src/webapis/SynapseHTTPAntispam/UserMayJoinRoomEndpoint.ts index be2cf007..135978e0 100644 --- a/src/webapis/SynapseHTTPAntispam/UserMayJoinRoomEndpoint.ts +++ b/src/webapis/SynapseHTTPAntispam/UserMayJoinRoomEndpoint.ts @@ -21,8 +21,10 @@ export type UserMayJoinRoomListenerArguments = Parameters< (details: UserMayJoinRoomRequestBody) => void >; -type UserMayJoinRoomRequestBody = EDStatic; -const UserMayJoinRoomRequestBody = Type.Object({ +export type UserMayJoinRoomRequestBody = EDStatic< + typeof UserMayJoinRoomRequestBody +>; +export const UserMayJoinRoomRequestBody = Type.Object({ user: StringUserIDSchema, room: StringRoomIDSchema, is_invited: Type.Boolean(), From 7b212d75aa67746a0ca0d4bba9e1da9d145f8e70 Mon Sep 17 00:00:00 2001 From: gnuxie Date: Fri, 14 Mar 2025 19:16:19 +0000 Subject: [PATCH 09/48] Fix typo in SynapseHTTPAntispam UserMayJoinRoom handle. --- src/webapis/SynapseHTTPAntispam/SynapseHttpAntispam.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/webapis/SynapseHTTPAntispam/SynapseHttpAntispam.ts b/src/webapis/SynapseHTTPAntispam/SynapseHttpAntispam.ts index c9e79b39..a6d9c729 100644 --- a/src/webapis/SynapseHTTPAntispam/SynapseHttpAntispam.ts +++ b/src/webapis/SynapseHTTPAntispam/SynapseHttpAntispam.ts @@ -45,10 +45,10 @@ export class SynapseHttpAntispam { private readonly userMayInviteEndpoint = new UserMayInviteEndpoint( this.userMayInviteHandles ); - public readonly userMayJoinRoomhandles = + public readonly userMayJoinRoomHandles = new SpamCheckEndpointPluginManager(); private readonly userMayJoinRoomEndpoint = new UserMayJoinRoomEndpoint( - this.userMayJoinRoomhandles + this.userMayJoinRoomHandles ); public readonly checkEventForSpamHandles = new SpamCheckEndpointPluginManager(); From 3f00346ae6e6ebc7569e79c825e0bbf693aaa32e Mon Sep 17 00:00:00 2001 From: gnuxie Date: Fri, 14 Mar 2025 19:16:57 +0000 Subject: [PATCH 10/48] Plumb in SynapseHTTPAntispam to RoomTakedownProtection room discovery. --- src/protections/RoomTakedown/RoomDiscovery.ts | 106 ++++++++++++++++++ .../RoomTakedown/RoomTakedownProtection.ts | 45 ++++++-- 2 files changed, 143 insertions(+), 8 deletions(-) create mode 100644 src/protections/RoomTakedown/RoomDiscovery.ts diff --git a/src/protections/RoomTakedown/RoomDiscovery.ts b/src/protections/RoomTakedown/RoomDiscovery.ts new file mode 100644 index 00000000..bddbdca6 --- /dev/null +++ b/src/protections/RoomTakedown/RoomDiscovery.ts @@ -0,0 +1,106 @@ +// SPDX-FileCopyrightText: 2025 Gnuxie +// +// SPDX-License-Identifier: Apache-2.0 + +import { + ConstantPeriodItemBatch, + isError, + Logger, + SHA256RoomHashStore, + StandardBatcher, +} from "matrix-protection-suite"; +import { CheckEventForSpamRequestBody } from "../../webapis/SynapseHTTPAntispam/CheckEventForSpamEndpoint"; +import { SynapseHttpAntispam } from "../../webapis/SynapseHTTPAntispam/SynapseHttpAntispam"; +import { StringRoomID } from "@the-draupnir-project/matrix-basic-types"; +import { UserMayInviteRequestBody } from "../../webapis/SynapseHTTPAntispam/UserMayInviteEndpoint"; +import { UserMayJoinRoomRequestBody } from "../../webapis/SynapseHTTPAntispam/UserMayJoinRoomEndpoint"; + +const log = new Logger("SynapseHTTPAntispamRoomDiscovery"); + +export interface RoomDiscovery { + unregisterListeners(): void; +} + +export class SynapseHTTPAntispamRoomDiscovery implements RoomDiscovery { + private readonly discoveredRooms = new Set(); + private readonly batcher = new StandardBatcher( + () => + new ConstantPeriodItemBatch( + this.forwardDiscoveredBatch, + { waitPeriodMS: 2500 } + ) + ); + constructor( + private readonly synapseHTTPAntispam: SynapseHttpAntispam, + private readonly hashStore: SHA256RoomHashStore + ) { + synapseHTTPAntispam.checkEventForSpamHandles.registerNonBlockingHandle( + this.handleCheckEventForSpam + ); + synapseHTTPAntispam.userMayInviteHandles.registerNonBlockingHandle( + this.handleUserMayInvite + ); + synapseHTTPAntispam.userMayJoinRoomHandles.registerNonBlockingHandle( + this.handleUserMayJoin + ); + } + + private readonly forwardDiscoveredBatch = async function ( + this: SynapseHTTPAntispamRoomDiscovery, + rawEntries: [StringRoomID][] + ): Promise { + const entries = rawEntries.flat(); + const storeResult = await this.hashStore.storeUndiscoveredRooms(entries); + if (isError(storeResult)) { + log.error( + "Unxpected error while trying to store undiscovered rooms", + storeResult.error + ); + return; + } + for (const roomID of entries) { + this.discoveredRooms.add(roomID); + } + }.bind(this); + + private readonly handleCheckEventForSpam = function ( + this: SynapseHTTPAntispamRoomDiscovery, + { event }: CheckEventForSpamRequestBody + ): void { + if (event.type === "m.room.member") { + if (!this.discoveredRooms.has(event.room_id)) { + this.batcher.add(event.room_id); + } + } + }.bind(this); + + private readonly handleUserMayInvite = function ( + this: SynapseHTTPAntispamRoomDiscovery, + { room_id }: UserMayInviteRequestBody + ): void { + if (!this.discoveredRooms.has(room_id)) { + this.batcher.add(room_id); + } + }.bind(this); + + private readonly handleUserMayJoin = function ( + this: SynapseHTTPAntispamRoomDiscovery, + { room }: UserMayJoinRoomRequestBody + ): void { + if (!this.discoveredRooms.has(room)) { + this.batcher.add(room); + } + }.bind(this); + + unregisterListeners(): void { + this.synapseHTTPAntispam.checkEventForSpamHandles.unregisterHandle( + this.handleCheckEventForSpam + ); + this.synapseHTTPAntispam.userMayInviteHandles.unregisterHandle( + this.handleUserMayInvite + ); + this.synapseHTTPAntispam.userMayJoinRoomHandles.unregisterHandle( + this.handleUserMayJoin + ); + } +} diff --git a/src/protections/RoomTakedown/RoomTakedownProtection.ts b/src/protections/RoomTakedown/RoomTakedownProtection.ts index ae6b60c5..6d3ff8a4 100644 --- a/src/protections/RoomTakedown/RoomTakedownProtection.ts +++ b/src/protections/RoomTakedown/RoomTakedownProtection.ts @@ -6,6 +6,7 @@ import { AbstractProtection, ActionResult, describeProtection, + Logger, Ok, PolicyListRevision, PolicyRuleChange, @@ -21,12 +22,13 @@ import { Draupnir } from "../../Draupnir"; import { StandardRoomTakedown } from "./RoomTakedown"; import { RoomAuditLog } from "./RoomAuditLog"; import { SynapseAdminRoomTakedownCapability } from "../../capabilities/SynapseAdminRoomTakedown/SynapseAdminRoomTakedown"; +import { ResultError } from "@gnuxie/typescript-result"; +import { + RoomDiscovery, + SynapseHTTPAntispamRoomDiscovery, +} from "./RoomDiscovery"; -// FIXME: We still haven't figured out how to poll for new rooms via the -// Synapse admin API. - -// FIXME: We need to add the stores to draupnir somehow. -// probably from the toplevel. +const log = new Logger("RoomTakedownProtection"); type RoomTakedownProtectionCapabilities = { roomTakedownCapability: RoomTakedownCapability; @@ -48,7 +50,8 @@ export class RoomTakedownProtection capabilities: RoomTakedownProtectionCapabilities, protectedRoomsSet: ProtectedRoomsSet, hashStore: SHA256RoomHashStore, - auditLog: RoomAuditLog + auditLog: RoomAuditLog, + private readonly roomDiscovery: RoomDiscovery | undefined ) { super(description, capabilities, protectedRoomsSet, {}); this.roomTakedown = new StandardRoomTakedown( @@ -69,6 +72,10 @@ export class RoomTakedownProtection ): Promise> { return this.roomTakedown.handlePolicyChange(revision, changes); } + + handleProtectionDisable(): void { + this.roomDiscovery?.unregisterListeners(); + } } describeProtection({ @@ -81,13 +88,35 @@ describeProtection({ roomTakedownCapability: SynapseAdminRoomTakedownCapability.name, }, factory(description, protectedRoomsSet, draupnir, capabilitySet, _settings) { + if ( + draupnir.stores.hashStore === undefined || + draupnir.stores.roomAuditLog === undefined + ) { + return ResultError.Result( + "This protection requires a hash store and audit log to be available to draupnir, and they are not in your configuration." + ); + } + const roomDiscovery = (() => { + if (draupnir.synapseHTTPAntispam !== undefined) { + return new SynapseHTTPAntispamRoomDiscovery( + draupnir.synapseHTTPAntispam, + draupnir.stores.hashStore + ); + } else { + log.warn( + "synapseHTTPAntispam is not configured for this draupnir, and will not be used for room discovery" + ); + return undefined; + } + })(); return Ok( new RoomTakedownProtection( description, capabilitySet, protectedRoomsSet, - draupnir.hashStore, - draupnir.auditLog + draupnir.stores.hashStore, + draupnir.stores.roomAuditLog, + roomDiscovery ) ); }, From 442b930d89a165ea231a24d096d38868ed71e4be Mon Sep 17 00:00:00 2001 From: gnuxie Date: Sat, 15 Mar 2025 14:39:48 +0000 Subject: [PATCH 11/48] Unit test and fix sqlite room audit log. --- .../RoomTakedown/SqliteRoomAuditLog.ts | 6 +-- test/unit/stores/roomAuditLogTest.ts | 45 +++++++++++++++++++ 2 files changed, 48 insertions(+), 3 deletions(-) create mode 100644 test/unit/stores/roomAuditLogTest.ts diff --git a/src/protections/RoomTakedown/SqliteRoomAuditLog.ts b/src/protections/RoomTakedown/SqliteRoomAuditLog.ts index de0d3ecc..46e955ca 100644 --- a/src/protections/RoomTakedown/SqliteRoomAuditLog.ts +++ b/src/protections/RoomTakedown/SqliteRoomAuditLog.ts @@ -31,13 +31,13 @@ const schema = [ room_id TEXT NOT NULL, state_key TEXT NOT NULL, type TEXT NOT NULL, - recommendation TEXT NOT NULL, + recommendation TEXT NOT NULL ) STRICT;`, `CREATE TABLE room_takedown ( id INTEGER PRIMARY KEY AUTOINCREMENT, policy_id TEXT NOT NULL, target_room_id TEXT NOT NULL, - created_at DATETIME DEFAULT CURRENT_TIMESTAMP NOT NULL, + created_at INTEGER DEFAULT (unixepoch()) NOT NULL, FOREIGN KEY (policy_id) REFERENCES policy_info(policy_id) ) STRICT;`, ]; @@ -119,7 +119,7 @@ export class SqliteRoomAuditLog private loadTakendownRooms(): StringRoomID[] { return this.db - .prepare(`SELECT room_id FROM room_takedown`) + .prepare(`SELECT target_room_id FROM room_takedown`) .all() as StringRoomID[]; } isRoomTakendown(roomID: StringRoomID): boolean { diff --git a/test/unit/stores/roomAuditLogTest.ts b/test/unit/stores/roomAuditLogTest.ts new file mode 100644 index 00000000..a3692971 --- /dev/null +++ b/test/unit/stores/roomAuditLogTest.ts @@ -0,0 +1,45 @@ +// SPDX-FileCopyrightText: 2025 Gnuxie +// +// SPDX-License-Identifier: AFL-3.0 + +import Database from "better-sqlite3"; +import { BetterSqliteOptions } from "../../../src/backingstore/better-sqlite3/BetterSqliteStore"; +import { SqliteRoomAuditLog } from "../../../src/protections/RoomTakedown/SqliteRoomAuditLog"; +import { + describePolicyRule, + LiteralPolicyRule, + parsePolicyRule, + PolicyRuleType, + randomRoomID, + Recommendation, +} from "matrix-protection-suite"; +import expect from "expect"; + +describe("RoomAuditLog test", function () { + const options = { path: ":memory:" } satisfies BetterSqliteOptions; + const db = new Database(options.path); + db.pragma("FOREIGN_KEYS = ON"); + const store = new SqliteRoomAuditLog(options, db); + it("Test that it doesn't return null or something", async function () { + expect(store.isRoomTakendown(randomRoomID([]).toRoomIDOrAlias())).toBe( + false + ); + }); + it("Can logged takendown rooms", async function () { + const bannedRoom = randomRoomID([]); + const policyRoom = randomRoomID([]); + const ban = parsePolicyRule( + describePolicyRule({ + room_id: policyRoom.toRoomIDOrAlias(), + entity: bannedRoom.toRoomIDOrAlias(), + reason: "spam", + recommendation: Recommendation.Takedown, + type: PolicyRuleType.Room, + }) as never + ).expect("Should bea ble to parse the policy rule."); + (await store.takedownRoom(ban as LiteralPolicyRule)).expect( + "Should be able to takedown a room" + ); + expect(store.isRoomTakendown(bannedRoom.toRoomIDOrAlias())).toBe(true); + }); +}); From d15322dff616556df8da846123124a63bcb7d73b Mon Sep 17 00:00:00 2001 From: gnuxie Date: Sat, 15 Mar 2025 17:48:02 +0000 Subject: [PATCH 12/48] Test and fix the hash store. --- src/backingstore/better-sqlite3/HashStore.ts | 22 +++--- test/unit/stores/hashStoreTest.ts | 77 ++++++++++++++++++++ 2 files changed, 90 insertions(+), 9 deletions(-) create mode 100644 test/unit/stores/hashStoreTest.ts diff --git a/src/backingstore/better-sqlite3/HashStore.ts b/src/backingstore/better-sqlite3/HashStore.ts index 2c9316fa..f149189e 100644 --- a/src/backingstore/better-sqlite3/HashStore.ts +++ b/src/backingstore/better-sqlite3/HashStore.ts @@ -18,7 +18,7 @@ import { BetterSqliteStore, makeBetterSqliteDB, } from "./BetterSqliteStore"; -import { SHA256 } from "crypto-js"; +import { enc, SHA256 } from "crypto-js"; import { Database } from "better-sqlite3"; import path from "path"; @@ -27,7 +27,7 @@ import path from "path"; const schema = [ `CREATE TABLE room_sha256 ( room_id TEXT PRIMARY KEY NOT NULL, - sha256 BLOB NOT NULL + sha256 TEXT NOT NULL ) STRICT;`, ]; @@ -69,9 +69,11 @@ export class SqliteHashReversalStore ): Promise> { try { return Ok( - this.db - .prepare(`SELECT room_id FROM room_sha256 WHERE sha256 = base64(?)`) - .get(hash) as StringRoomID | undefined + ( + this.db + .prepare(`SELECT room_id FROM room_sha256 WHERE sha256 = ?`) + .get(hash) as RoomSha256 | null + )?.room_id ?? undefined ); } catch (e) { if (e instanceof Error) { @@ -101,7 +103,7 @@ export class SqliteHashReversalStore // I can't believe there's not a builtin for this fml. try { const statement = this.db.prepare( - `SELECT (room_id, base64(sha256)) FROM room_sha256 WHERE sha256 IN (${sha256s.map(() => "base64(?)").join(",")})` + `SELECT room_id, sha256 FROM room_sha256 WHERE sha256 IN (${sha256s.map(() => "?").join(",")})` ); return Ok(statement.all(sha256s) as RoomSha256[]); } catch (exception) { @@ -164,14 +166,16 @@ export class SqliteHashReversalStore if (!existingRooms.find((row) => row.room_id === roomID)) { rowsToInsert.push({ room_id: roomID, - sha256: SHA256(roomID).toString(), + sha256: enc.Base64.stringify(SHA256(roomID)), }); } } const statement = this.db.prepare( - `REPLACE INTO room_sha256 VALUES ${roomIDs.map(() => `(?, base64(?))`).join(", ")}` + `REPLACE INTO room_sha256 VALUES ${roomIDs.map(() => `(?, ?)`).join(", ")}` + ); + statement.run( + ...rowsToInsert.flatMap((row) => [row.room_id, row.sha256]) ); - statement.run(rowsToInsert.map((row) => [row.room_id, row.sha256])); return Ok(rowsToInsert); } catch (exception) { if (exception instanceof Error) { diff --git a/test/unit/stores/hashStoreTest.ts b/test/unit/stores/hashStoreTest.ts new file mode 100644 index 00000000..7628babd --- /dev/null +++ b/test/unit/stores/hashStoreTest.ts @@ -0,0 +1,77 @@ +// SPDX-FileCopyrightText: 2025 Gnuxie +// +// SPDX-License-Identifier: AFL-3.0 + +import Database from "better-sqlite3"; +import { SqliteHashReversalStore } from "../../../src/backingstore/better-sqlite3/HashStore"; +import { BetterSqliteOptions } from "../../../src/backingstore/better-sqlite3/BetterSqliteStore"; +import { + describeStateEvent, + HashedLiteralPolicyRule, + parsePolicyRule, + PolicyRuleMatchType, + PolicyRuleType, + randomRoomID, + randomUserID, + Recommendation, +} from "matrix-protection-suite"; +import { SHA256, enc } from "crypto-js"; +import expect from "expect"; + +function makeStore(): SqliteHashReversalStore { + const options = { path: ":memory:" } satisfies BetterSqliteOptions; + const db = new Database(options.path); + db.pragma("FOREIGN_KEYS = ON"); + return new SqliteHashReversalStore(options, db); +} + +describe("meow", function () { + it("Doesn't return weird nulls if there is nothing in the database", async function () { + const store = makeStore(); + const bannedRoom = randomRoomID([]); + const policyRoom = randomRoomID([]); + const bannedRoomHash = enc.Base64.stringify( + SHA256(bannedRoom.toRoomIDOrAlias()) + ); + const findResult = (await store.findRoomHash(bannedRoomHash)).expect( + "Should be able to at least query this" + ); + expect(findResult).toBe(undefined); + // now for finding reversed hashed room policies: + const ban = parsePolicyRule( + describeStateEvent({ + room_id: policyRoom.toRoomIDOrAlias(), + content: { + recommendation: Recommendation.Takedown, + hashes: { + sha256: bannedRoomHash, + }, + }, + sender: randomUserID(), + state_key: bannedRoomHash, + type: PolicyRuleType.Room, + }) as never + ).expect("Should bea ble to parse the policy rule."); + expect(ban.matchType).toBe(PolicyRuleMatchType.HashedLiteral); + const reverserResult = ( + await store.reverseHashedRoomPolicies([ban as HashedLiteralPolicyRule]) + ).expect("Should be able to reverse policies that are unknown"); + expect(reverserResult.length).toBe(0); + // now try storing the room and testing that the policy is there + (await store.storeUndiscoveredRooms([bannedRoom.toRoomIDOrAlias()])).expect( + "Should be able to discover rooms jsut fine" + ); + const foundHash = (await store.findRoomHash(bannedRoomHash)).expect( + "Should be able to findRoomHash" + ); + expect(foundHash).toBe(bannedRoom.toRoomIDOrAlias()); + const reversedPolicies = ( + await store.reverseHashedRoomPolicies([ban as HashedLiteralPolicyRule]) + ).expect("Should be able to reverse meow"); + const reversedBan = reversedPolicies.at(0); + if (reversedBan === undefined) { + throw new TypeError("Reversed ban isn't here mare"); + } + expect(reversedBan.entity).toBe(bannedRoom.toRoomIDOrAlias()); + }); +}); From 1e581d7394af5606007cb52ddd1f26445d02c331 Mon Sep 17 00:00:00 2001 From: gnuxie Date: Mon, 17 Mar 2025 11:39:27 +0000 Subject: [PATCH 13/48] Test room takedown service and add thoughts to FIXME. --- src/protections/RoomTakedown/RoomTakedown.ts | 11 +- .../protections/RoomTakedownServiceTest.ts | 138 ++++++++++++++++++ 2 files changed, 147 insertions(+), 2 deletions(-) create mode 100644 test/unit/protections/RoomTakedownServiceTest.ts diff --git a/src/protections/RoomTakedown/RoomTakedown.ts b/src/protections/RoomTakedown/RoomTakedown.ts index 206bc27a..817210f2 100644 --- a/src/protections/RoomTakedown/RoomTakedown.ts +++ b/src/protections/RoomTakedown/RoomTakedown.ts @@ -20,7 +20,10 @@ import { RoomTakedownCapability } from "../../capabilities/RoomTakedownCapabilit const log = new Logger("RoomTakedown"); -type RoomTakedown = { +// FIXME: How can we segment this so that rooms are takendown on prompt in +// the abscence of policy approval? + +export type RoomTakedownService = { handleDiscoveredRooms(rooms: StringRoomID[]): Promise>; handlePolicyChange( revision: PolicyListRevision, @@ -35,7 +38,7 @@ type RoomTakedown = { * moving discovered rooms into the hashStore. Although it's not clear * whether we need to do that? */ -export class StandardRoomTakedown implements RoomTakedown { +export class StandardRoomTakedown implements RoomTakedownService { public constructor( private readonly hashStore: SHA256RoomHashStore, private readonly auditLog: RoomAuditLog, @@ -43,6 +46,7 @@ export class StandardRoomTakedown implements RoomTakedown { ) { // nothing to do } + // FIXME: We don't use this public async handleDiscoveredRooms( rooms: StringRoomID[] ): Promise> { @@ -85,6 +89,8 @@ export class StandardRoomTakedown implements RoomTakedown { change.rule.matchType === PolicyRuleMatchType.Literal && change.rule.recommendation === Recommendation.Takedown ) { + // FIXME: We probably do not want check the audit log for new policies + // and instead let the capability decide if ( !this.auditLog.isRoomTakendown(change.rule.entity as StringRoomID) ) { @@ -114,6 +120,7 @@ export class StandardRoomTakedown implements RoomTakedown { const roomPolicies = revision .allRulesOfType(PolicyRuleType.Room, Recommendation.Takedown) .filter((policy) => policy.matchType === PolicyRuleMatchType.Literal); + // FIXME: We don't seem to check the audit log for (const policy of roomPolicies) { const takedownResult = await this.takedownRoom( policy.entity as StringRoomID, diff --git a/test/unit/protections/RoomTakedownServiceTest.ts b/test/unit/protections/RoomTakedownServiceTest.ts new file mode 100644 index 00000000..9b9b820b --- /dev/null +++ b/test/unit/protections/RoomTakedownServiceTest.ts @@ -0,0 +1,138 @@ +// SPDX-FileCopyrightText: 2025 Gnuxie +// +// SPDX-License-Identifier: AFL-3.0 + +import { createMock } from "ts-auto-mock"; +import { StandardRoomTakedown } from "../../../src/protections/RoomTakedown/RoomTakedown"; +import { + describePolicyRule, + describeProtectedRoomsSet, + Ok, + parsePolicyRule, + PolicyRuleType, + randomRoomID, + Recommendation, + SHA256RoomHashStore, + SimpleChangeType, + StandardPolicyListRevision, +} from "matrix-protection-suite"; +import { RoomAuditLog } from "../../../src/protections/RoomTakedown/RoomAuditLog"; +import { RoomTakedownCapability } from "../../../src/capabilities/RoomTakedownCapability"; +import { StringRoomID } from "@the-draupnir-project/matrix-basic-types"; +import expect from "expect"; + +function makeServiceMocks(): { + auditLogItems: Parameters[]; + auditLog: RoomAuditLog; + takedownCapabilityItems: StringRoomID[]; + takedownCapability: RoomTakedownCapability; +} { + const auditLogItems: Parameters[] = []; + const takedownCapabilityItems: StringRoomID[] = []; + return { + auditLogItems, + auditLog: createMock({ + async takedownRoom(...args: Parameters) { + auditLogItems.push(args); + return Ok(undefined); + }, + }), + takedownCapabilityItems, + takedownCapability: createMock({ + async takedownRoom(roomID: StringRoomID) { + takedownCapabilityItems.push(roomID); + return Ok({ + room_id: roomID, + }); + }, + async isRoomTakendown(_roomID) { + return Ok(false); + }, + }), + }; +} + +describe("", function () { + const storeFake = createMock({}); + it("Test rooms are takendown when policies are added", async function () { + const policyRoom = randomRoomID([]); + const bannedRoom = randomRoomID([]); + const policy = parsePolicyRule( + describePolicyRule({ + room_id: policyRoom.toRoomIDOrAlias(), + entity: bannedRoom.toRoomIDOrAlias(), + reason: "spam", + recommendation: Recommendation.Takedown, + type: PolicyRuleType.Room, + }) as never + ).expect("Should be able to parse the policy rule."); + const { + auditLog, + takedownCapability, + auditLogItems, + takedownCapabilityItems, + } = makeServiceMocks(); + const takedownService = new StandardRoomTakedown( + storeFake, + auditLog, + takedownCapability + ); + // we give a blank revision because i haven't updated the describeProtectedRoomsSet code + // to use hashed policies... although thinking about it we don't use them here either lol + ( + await takedownService.handlePolicyChange( + StandardPolicyListRevision.blankRevision(), + [ + { + changeType: SimpleChangeType.Added, + rule: policy, + event: policy.sourceEvent, + sender: policy.sourceEvent.sender, + }, + ] + ) + ).expect("Should have run just fine"); + expect(auditLogItems.length).toBe(1); + expect(takedownCapabilityItems.length).toBe(1); + }); + it("Test rooms are takendown at startup", async function () { + const policyRoom = randomRoomID([]); + const { protectedRoomsSet } = await describeProtectedRoomsSet({ + lists: [ + { + room: policyRoom, + policyDescriptions: [ + { + entity: randomRoomID([]).toRoomIDOrAlias(), + recommendation: Recommendation.Takedown, + type: PolicyRuleType.Room, + }, + { + entity: randomRoomID([]).toRoomIDOrAlias(), + recommendation: Recommendation.Takedown, + type: PolicyRuleType.Room, + }, + ], + }, + ], + }); + const { + auditLog, + takedownCapability, + auditLogItems, + takedownCapabilityItems, + } = makeServiceMocks(); + const takedownService = new StandardRoomTakedown( + storeFake, + auditLog, + takedownCapability + ); + ( + await takedownService.checkAllRooms( + protectedRoomsSet.watchedPolicyRooms.currentRevision + ) + ).expect("Should be able to check all rooms"); + expect(auditLogItems.length).toBe(2); + expect(takedownCapabilityItems.length).toBe(2); + }); +}); From b80a503ea0a9383dc54d51f46fb72ddfe2745991 Mon Sep 17 00:00:00 2001 From: gnuxie Date: Mon, 17 Mar 2025 12:51:41 +0000 Subject: [PATCH 14/48] Improve hash store replace query a little. I'm after realising this whole thing is a mess but there's no time to fix it. --- src/backingstore/better-sqlite3/HashStore.ts | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/backingstore/better-sqlite3/HashStore.ts b/src/backingstore/better-sqlite3/HashStore.ts index f149189e..58de2c63 100644 --- a/src/backingstore/better-sqlite3/HashStore.ts +++ b/src/backingstore/better-sqlite3/HashStore.ts @@ -170,12 +170,18 @@ export class SqliteHashReversalStore }); } } - const statement = this.db.prepare( - `REPLACE INTO room_sha256 VALUES ${roomIDs.map(() => `(?, ?)`).join(", ")}` - ); - statement.run( - ...rowsToInsert.flatMap((row) => [row.room_id, row.sha256]) - ); + + if (rowsToInsert.length > 0) { + const serializedRows = JSON.stringify(rowsToInsert); + + const statement = this.db.prepare(` + REPLACE INTO room_sha256 + SELECT value ->> 'room_id', value ->> 'sha256' + FROM json_each(?) + `); + + statement.run(serializedRows); + } return Ok(rowsToInsert); } catch (exception) { if (exception instanceof Error) { From 003410d41a2f566a188d08ec257c3499dfa92d6e Mon Sep 17 00:00:00 2001 From: gnuxie Date: Mon, 17 Mar 2025 12:59:25 +0000 Subject: [PATCH 15/48] Change the order of checking the audit log. --- src/protections/RoomTakedown/RoomTakedown.ts | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/src/protections/RoomTakedown/RoomTakedown.ts b/src/protections/RoomTakedown/RoomTakedown.ts index 817210f2..b29bf7f7 100644 --- a/src/protections/RoomTakedown/RoomTakedown.ts +++ b/src/protections/RoomTakedown/RoomTakedown.ts @@ -89,13 +89,7 @@ export class StandardRoomTakedown implements RoomTakedownService { change.rule.matchType === PolicyRuleMatchType.Literal && change.rule.recommendation === Recommendation.Takedown ) { - // FIXME: We probably do not want check the audit log for new policies - // and instead let the capability decide - if ( - !this.auditLog.isRoomTakendown(change.rule.entity as StringRoomID) - ) { - roomsToTakedown.push(change.rule); - } + roomsToTakedown.push(change.rule); } } for (const policy of roomsToTakedown) { @@ -120,8 +114,10 @@ export class StandardRoomTakedown implements RoomTakedownService { const roomPolicies = revision .allRulesOfType(PolicyRuleType.Room, Recommendation.Takedown) .filter((policy) => policy.matchType === PolicyRuleMatchType.Literal); - // FIXME: We don't seem to check the audit log for (const policy of roomPolicies) { + if (this.auditLog.isRoomTakendown(policy.entity as StringRoomID)) { + continue; // room already takendown + } const takedownResult = await this.takedownRoom( policy.entity as StringRoomID, policy From 4fdb142ac25148e623d0390663bb3da1d8af2cf7 Mon Sep 17 00:00:00 2001 From: gnuxie Date: Mon, 17 Mar 2025 13:01:49 +0000 Subject: [PATCH 16/48] Only audit takedown when capabilities are not simulated. --- src/protections/RoomTakedown/RoomTakedown.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/protections/RoomTakedown/RoomTakedown.ts b/src/protections/RoomTakedown/RoomTakedown.ts index b29bf7f7..db4fb3a3 100644 --- a/src/protections/RoomTakedown/RoomTakedown.ts +++ b/src/protections/RoomTakedown/RoomTakedown.ts @@ -70,9 +70,11 @@ export class StandardRoomTakedown implements RoomTakedownService { const takedownResult = await this.takedownCapability.takedownRoom(roomID); if (isError(takedownResult)) { return takedownResult; + } + // Only audit the takedown if the capability is not simulated. + if (this.takedownCapability.isSimulated) { + return Ok(undefined); } else { - // FIXME: we should probably audit as simulated if the capability is simulated. - // or not audit at all because the protection preview might show things otherwise. return await this.auditLog.takedownRoom(rule); } } From 7f0a8da41f7001080299d93fd0b370be34ea9fd2 Mon Sep 17 00:00:00 2001 From: gnuxie Date: Mon, 17 Mar 2025 23:40:27 +0000 Subject: [PATCH 17/48] Flesh out RoomDiscovery some more. I really need to move the room details to the audit log, it's there so we can see what the rooms are that have been takendown. --- src/backingstore/better-sqlite3/HashStore.ts | 38 +++++++ src/capabilities/RoomTakedownCapability.tsx | 18 +-- .../RoomTakedownCapabilityRenderer.tsx | 10 +- .../SynapseAdminRoomTakedown.ts | 44 +++++-- src/protections/RoomTakedown/RoomDiscovery.ts | 51 ++++++++- .../RoomTakedown/RoomDiscoveryRenderer.tsx | 32 ++++++ src/protections/RoomTakedown/RoomTakedown.ts | 4 + .../RoomTakedown/RoomTakedownProtection.ts | 107 ++++++++++++++++-- .../RoomTakedown/SynapseAdminRoomDetails.ts | 3 + 9 files changed, 267 insertions(+), 40 deletions(-) create mode 100644 src/protections/RoomTakedown/RoomDiscoveryRenderer.tsx create mode 100644 src/protections/RoomTakedown/SynapseAdminRoomDetails.ts diff --git a/src/backingstore/better-sqlite3/HashStore.ts b/src/backingstore/better-sqlite3/HashStore.ts index 58de2c63..53eaeee5 100644 --- a/src/backingstore/better-sqlite3/HashStore.ts +++ b/src/backingstore/better-sqlite3/HashStore.ts @@ -10,6 +10,7 @@ import { HashedLiteralPolicyRule, LiteralPolicyRule, makeReversedHashedPolicy, + RoomBasicDetails, RoomHashRecord, SHA256RoomHashStore, } from "matrix-protection-suite"; @@ -29,6 +30,14 @@ const schema = [ room_id TEXT PRIMARY KEY NOT NULL, sha256 TEXT NOT NULL ) STRICT;`, + `CREATE TABLE room_detail ( + room_id TEXT PRIMARY KEY NOT NULL, + creator TEXT, + name TEXT, + topic TEXT, + joined_members INTEGER + ) STRICT; +`, ]; type RoomSha256 = { @@ -197,4 +206,33 @@ export class SqliteHashReversalStore } } } + + public async storeRoomDetails( + roomDetails: RoomBasicDetails + ): Promise> { + try { + const statement = this.db.prepare(` + REPLACE INTO room_detail SELECT + value ->> 'room_id', + value ->> 'creator', + value ->> 'name', + value ->> 'topic', + value ->> 'joined_members' + FROM json_each(?)`); + statement.run(JSON.stringify(roomDetails)); + return Ok(undefined); + } catch (exception) { + if (exception instanceof Error) { + return ActionException.Result( + `Error while trying to store details about a hashed room`, + { + exception, + exceptionKind: ActionExceptionKind.Unknown, + } + ); + } else { + throw exception; + } + } + } } diff --git a/src/capabilities/RoomTakedownCapability.tsx b/src/capabilities/RoomTakedownCapability.tsx index 55cb3dac..dfe060ab 100644 --- a/src/capabilities/RoomTakedownCapability.tsx +++ b/src/capabilities/RoomTakedownCapability.tsx @@ -4,23 +4,17 @@ import { Result } from "@gnuxie/typescript-result"; import { Type } from "@sinclair/typebox"; -import { - StringRoomID, - StringUserID, -} from "@the-draupnir-project/matrix-basic-types"; +import { StringRoomID } from "@the-draupnir-project/matrix-basic-types"; import { Capability, CapabilityMethodSchema, describeCapabilityInterface, + RoomBasicDetails, } from "matrix-protection-suite"; -export type RoomTakedownDetails = Partial<{ - creator: StringUserID | undefined; - room_id: StringRoomID; - name: string | undefined; - topic: string | undefined; - avatar: string | undefined; -}>; +export interface RoomDetailsProvider { + getRoomDetails(roomID: StringRoomID): Promise>; +} export const RoomTakedownCapability = Type.Intersect([ Type.Object({ @@ -35,7 +29,7 @@ export const RoomTakedownCapability = Type.Intersect([ // to get those details on conduwuit. export type RoomTakedownCapability = { isRoomTakendown(roomID: StringRoomID): Promise>; - takedownRoom(roomID: StringRoomID): Promise>; + takedownRoom(roomID: StringRoomID): Promise>; } & Capability; describeCapabilityInterface({ diff --git a/src/capabilities/RoomTakedownCapabilityRenderer.tsx b/src/capabilities/RoomTakedownCapabilityRenderer.tsx index 11f53613..0792dde0 100644 --- a/src/capabilities/RoomTakedownCapabilityRenderer.tsx +++ b/src/capabilities/RoomTakedownCapabilityRenderer.tsx @@ -5,11 +5,9 @@ import { describeCapabilityRenderer, DescriptionMeta, + RoomBasicDetails, } from "matrix-protection-suite"; -import { - RoomTakedownCapability, - RoomTakedownDetails, -} from "./RoomTakedownCapability"; +import { RoomTakedownCapability } from "./RoomTakedownCapability"; import { RendererMessageCollector } from "./RendererMessageCollector"; import { MatrixRoomReference, @@ -37,7 +35,7 @@ function renderCodeOrDefault( function renderTakedown( roomID: StringRoomID, - details: RoomTakedownDetails + details: RoomBasicDetails ): DocumentNode { return (
@@ -83,7 +81,7 @@ class StandardRoomTakedownCapabilityRenderer implements RoomTakedownCapability { public async takedownRoom( roomID: StringRoomID - ): Promise> { + ): Promise> { const capabilityResult = await this.capability.takedownRoom(roomID); if (isError(capabilityResult)) { this.messageCollector.addOneliner( diff --git a/src/capabilities/SynapseAdminRoomTakedown/SynapseAdminRoomTakedown.ts b/src/capabilities/SynapseAdminRoomTakedown/SynapseAdminRoomTakedown.ts index 88bb4490..e0db23e7 100644 --- a/src/capabilities/SynapseAdminRoomTakedown/SynapseAdminRoomTakedown.ts +++ b/src/capabilities/SynapseAdminRoomTakedown/SynapseAdminRoomTakedown.ts @@ -3,22 +3,47 @@ // SPDX-License-Identifier: Apache-2.0 import { StringRoomID } from "@the-draupnir-project/matrix-basic-types"; -import { Logger } from "matrix-protection-suite"; +import { Logger, RoomBasicDetails } from "matrix-protection-suite"; import { + RoomDetailsProvider, RoomTakedownCapability, - RoomTakedownDetails, } from "../RoomTakedownCapability"; import { SynapseAdminClient } from "matrix-protection-suite-for-matrix-bot-sdk"; import { isError, Ok, Result } from "@gnuxie/typescript-result"; const log = new Logger("SynapseAdminRoomTakedownCapability"); +export class SynapseAdminRoomDetailsProvider implements RoomDetailsProvider { + public constructor(private readonly adminClient: SynapseAdminClient) { + // nothing to do mare. + } + public async getRoomDetails( + roomID: StringRoomID + ): Promise> { + const detailsResponse = await this.adminClient.getRoomDetails(roomID); + if (isError(detailsResponse)) { + return detailsResponse; + } else { + return Ok({ + name: detailsResponse.ok?.name ?? undefined, + creator: detailsResponse.ok?.creator, + avatar: detailsResponse.ok?.avatar ?? undefined, + topic: detailsResponse.ok?.topic ?? undefined, + room_id: roomID, + }); + } + } +} + export class SynapseAdminRoomTakedownCapability implements RoomTakedownCapability { public readonly requiredPermissions = []; public readonly requiredStatePermissions = []; public readonly requiredEventPermissions = []; + private readonly roomDetailsProvider = new SynapseAdminRoomDetailsProvider( + this.adminClient + ); public constructor(private readonly adminClient: SynapseAdminClient) { // nothing to do mare. } @@ -33,9 +58,10 @@ export class SynapseAdminRoomTakedownCapability public async takedownRoom( roomID: StringRoomID - ): Promise> { - const detailsResponse = await this.adminClient.getRoomDetails(roomID); - let details: RoomTakedownDetails; + ): Promise> { + const detailsResponse = + await this.roomDetailsProvider.getRoomDetails(roomID); + let details: RoomBasicDetails; if (isError(detailsResponse)) { log.warn( "Unable to fetch details for a room being requested to shutdown", @@ -43,13 +69,7 @@ export class SynapseAdminRoomTakedownCapability ); details = { room_id: roomID }; } else { - details = { - name: detailsResponse.ok?.name ?? undefined, - creator: detailsResponse.ok?.creator, - avatar: detailsResponse.ok?.avatar ?? undefined, - topic: detailsResponse.ok?.topic ?? undefined, - room_id: roomID, - }; + details = detailsResponse.ok; } const takedownResult = await this.adminClient.deleteRoom(roomID, { block: true, diff --git a/src/protections/RoomTakedown/RoomDiscovery.ts b/src/protections/RoomTakedown/RoomDiscovery.ts index bddbdca6..6d1cd66d 100644 --- a/src/protections/RoomTakedown/RoomDiscovery.ts +++ b/src/protections/RoomTakedown/RoomDiscovery.ts @@ -6,22 +6,34 @@ import { ConstantPeriodItemBatch, isError, Logger, + RoomBasicDetails, + RoomHashRecord, SHA256RoomHashStore, StandardBatcher, + Task, } from "matrix-protection-suite"; import { CheckEventForSpamRequestBody } from "../../webapis/SynapseHTTPAntispam/CheckEventForSpamEndpoint"; import { SynapseHttpAntispam } from "../../webapis/SynapseHTTPAntispam/SynapseHttpAntispam"; import { StringRoomID } from "@the-draupnir-project/matrix-basic-types"; import { UserMayInviteRequestBody } from "../../webapis/SynapseHTTPAntispam/UserMayInviteEndpoint"; import { UserMayJoinRoomRequestBody } from "../../webapis/SynapseHTTPAntispam/UserMayJoinRoomEndpoint"; +import { EventEmitter } from "stream"; +import { RoomDetailsProvider } from "../../capabilities/RoomTakedownCapability"; const log = new Logger("SynapseHTTPAntispamRoomDiscovery"); +export type RoomDiscoveryListener = (details: RoomBasicDetails) => void; + export interface RoomDiscovery { unregisterListeners(): void; + on(event: "RoomDiscovery", listener: RoomDiscoveryListener): this; + off(event: "RoomDiscovery", listener: RoomDiscoveryListener): this; + emit(event: "RoomDiscovery", details: RoomBasicDetails): void; } - -export class SynapseHTTPAntispamRoomDiscovery implements RoomDiscovery { +export class SynapseHTTPAntispamRoomDiscovery + extends EventEmitter + implements RoomDiscovery +{ private readonly discoveredRooms = new Set(); private readonly batcher = new StandardBatcher( () => @@ -32,8 +44,10 @@ export class SynapseHTTPAntispamRoomDiscovery implements RoomDiscovery { ); constructor( private readonly synapseHTTPAntispam: SynapseHttpAntispam, - private readonly hashStore: SHA256RoomHashStore + private readonly hashStore: SHA256RoomHashStore, + private readonly roomDetailsProvider: RoomDetailsProvider ) { + super(); synapseHTTPAntispam.checkEventForSpamHandles.registerNonBlockingHandle( this.handleCheckEventForSpam ); @@ -45,6 +59,36 @@ export class SynapseHTTPAntispamRoomDiscovery implements RoomDiscovery { ); } + private addRoomDetails(discoveredRooms: RoomHashRecord[]): void { + void Task( + (async () => { + for (const { room_id: roomID } of discoveredRooms) { + const detailsResult = + await this.roomDetailsProvider.getRoomDetails(roomID); + if (isError(detailsResult)) { + log.error( + "Error fetching details for a discovered room", + roomID, + detailsResult.error + ); + continue; + } + this.emit("RoomDiscovery", detailsResult.ok); + const storeResult = await this.hashStore.storeRoomDetails( + detailsResult.ok + ); + if (isError(storeResult)) { + log.error( + "Error storing room details for a room", + roomID, + detailsResult.ok + ); + } + } + })() + ); + } + private readonly forwardDiscoveredBatch = async function ( this: SynapseHTTPAntispamRoomDiscovery, rawEntries: [StringRoomID][] @@ -61,6 +105,7 @@ export class SynapseHTTPAntispamRoomDiscovery implements RoomDiscovery { for (const roomID of entries) { this.discoveredRooms.add(roomID); } + this.addRoomDetails(storeResult.ok); }.bind(this); private readonly handleCheckEventForSpam = function ( diff --git a/src/protections/RoomTakedown/RoomDiscoveryRenderer.tsx b/src/protections/RoomTakedown/RoomDiscoveryRenderer.tsx new file mode 100644 index 00000000..b3af140d --- /dev/null +++ b/src/protections/RoomTakedown/RoomDiscoveryRenderer.tsx @@ -0,0 +1,32 @@ +// SPDX-FileCopyrightText: 2025 Gnuxie +// +// SPDX-License-Identifier: AFL-3.0 + +import { DocumentNode } from "@the-draupnir-project/interface-manager"; +import { RoomBasicDetails } from "matrix-protection-suite"; +import { DeadDocumentJSX } from "@the-draupnir-project/interface-manager"; + +export function renderDiscoveredRoom(details: RoomBasicDetails): DocumentNode { + return ( + +

Room Discovered

+
    +
  • + name: {details.name ?? "Unamed room"} +
    • +
  • + member count: {details.joined_members ?? "unknown"} +
    • +
  • + room ID: {details.room_id} +
    • +
  • + creator: {details.creator ?? "unknown"} +
    • +
  • + topic: 
    {details.topic ?? "unknown"}
    +
    • +
+ + ); +} diff --git a/src/protections/RoomTakedown/RoomTakedown.ts b/src/protections/RoomTakedown/RoomTakedown.ts index db4fb3a3..778734c7 100644 --- a/src/protections/RoomTakedown/RoomTakedown.ts +++ b/src/protections/RoomTakedown/RoomTakedown.ts @@ -22,6 +22,10 @@ const log = new Logger("RoomTakedown"); // FIXME: How can we segment this so that rooms are takendown on prompt in // the abscence of policy approval? +// Probably by using a simulated capability that asks to confirm and then +// does the takedown for real? +// Probably by using an updated version of the shutdown command... +// Ok no that all sucks we'll have to wait for policy approval... export type RoomTakedownService = { handleDiscoveredRooms(rooms: StringRoomID[]): Promise>; diff --git a/src/protections/RoomTakedown/RoomTakedownProtection.ts b/src/protections/RoomTakedown/RoomTakedownProtection.ts index 6d3ff8a4..1e70a7c6 100644 --- a/src/protections/RoomTakedown/RoomTakedownProtection.ts +++ b/src/protections/RoomTakedown/RoomTakedownProtection.ts @@ -13,30 +13,68 @@ import { ProtectedRoomsSet, Protection, ProtectionDescription, + RoomBasicDetails, + RoomMessageSender, SHA256RoomHashStore, + StringRoomIDSchema, Task, - UnknownConfig, } from "matrix-protection-suite"; import { RoomTakedownCapability } from "../../capabilities/RoomTakedownCapability"; import { Draupnir } from "../../Draupnir"; import { StandardRoomTakedown } from "./RoomTakedown"; import { RoomAuditLog } from "./RoomAuditLog"; -import { SynapseAdminRoomTakedownCapability } from "../../capabilities/SynapseAdminRoomTakedown/SynapseAdminRoomTakedown"; -import { ResultError } from "@gnuxie/typescript-result"; +import { + SynapseAdminRoomDetailsProvider, + SynapseAdminRoomTakedownCapability, +} from "../../capabilities/SynapseAdminRoomTakedown/SynapseAdminRoomTakedown"; +import { isError, ResultError } from "@gnuxie/typescript-result"; import { RoomDiscovery, + RoomDiscoveryListener, SynapseHTTPAntispamRoomDiscovery, } from "./RoomDiscovery"; +import { StringRoomID } from "@the-draupnir-project/matrix-basic-types"; +import { sendMatrixEventsFromDeadDocument } from "../../commands/interface-manager/MPSMatrixInterfaceAdaptor"; +import { wrapInRoot } from "../../commands/interface-manager/MatrixHelpRenderer"; +import { Type } from "@sinclair/typebox"; +import { EDStatic } from "matrix-protection-suite/dist/Interface/Static"; +import { renderDiscoveredRoom } from "./RoomDiscoveryRenderer"; const log = new Logger("RoomTakedownProtection"); +const RoomTakedownProtectionSettings = Type.Object({ + discoveryNotificationMembershipThreshold: Type.Integer({ + default: 20, + description: + "The number of members required in the room for it to appear in the notification. This is to prevent showing direct messages or small rooms that could be too much of an invasion of privacy. We don't have access to enough information to determine this a better way.", + }), + // There needs to be a transform for room references + discoveryNotificationRoom: Type.Union( + [StringRoomIDSchema, Type.Undefined()], + { + default: undefined, + description: + "The room where notifications should be sent. Currently broken and needs to be edited from a state event while we figure something out", + } + ), + discoveryNotificationEnabled: Type.Boolean({ + default: true, + description: + "Wether to send notifications for newly discovered rooms from the homerserver.", + }), +}); + +type RoomTakedownProtectionSettings = EDStatic< + typeof RoomTakedownProtectionSettings +>; + type RoomTakedownProtectionCapabilities = { roomTakedownCapability: RoomTakedownCapability; }; type RoomTakedownProtectionDescription = ProtectionDescription< Draupnir, - UnknownConfig, + typeof RoomTakedownProtectionSettings, RoomTakedownProtectionCapabilities >; @@ -51,6 +89,10 @@ export class RoomTakedownProtection protectedRoomsSet: ProtectedRoomsSet, hashStore: SHA256RoomHashStore, auditLog: RoomAuditLog, + private readonly roomMessageSender: RoomMessageSender, + private readonly discoveryNotificationEnabled: boolean, + private readonly discoveryNotificationMembershipThreshold: number, + private readonly discoveryNotificationRoom: StringRoomID, private readonly roomDiscovery: RoomDiscovery | undefined ) { super(description, capabilities, protectedRoomsSet, {}); @@ -64,8 +106,40 @@ export class RoomTakedownProtection this.protectedRoomsSet.watchedPolicyRooms.currentRevision ) ); + if (this.discoveryNotificationEnabled) { + this.roomDiscovery?.on("RoomDiscovery", this.roomDiscoveryListener); + } } + private readonly roomDiscoveryListener: RoomDiscoveryListener = function ( + this: RoomTakedownProtection, + details: RoomBasicDetails + ) { + if ( + (details.joined_members ?? 0) < + this.discoveryNotificationMembershipThreshold + ) { + return; + } + void Task( + (async () => { + const sendResult = await sendMatrixEventsFromDeadDocument( + this.roomMessageSender, + this.discoveryNotificationRoom, + wrapInRoot(renderDiscoveredRoom(details)), + {} + ); + if (isError(sendResult)) { + log.error( + "Error sending a notification about a discovered room", + details.room_id, + sendResult.error + ); + } + })() + ); + }.bind(this); + handlePolicyChange( revision: PolicyListRevision, changes: PolicyRuleChange[] @@ -75,10 +149,15 @@ export class RoomTakedownProtection handleProtectionDisable(): void { this.roomDiscovery?.unregisterListeners(); + this.roomDiscovery?.off("RoomDiscovery", this.roomDiscoveryListener); } } -describeProtection({ +describeProtection< + RoomTakedownProtectionCapabilities, + Draupnir, + typeof RoomTakedownProtectionSettings +>({ name: RoomTakedownProtection.name, description: `A protection to shutdown rooms matching policies from watched lists`, capabilityInterfaces: { @@ -87,7 +166,7 @@ describeProtection({ defaultCapabilities: { roomTakedownCapability: SynapseAdminRoomTakedownCapability.name, }, - factory(description, protectedRoomsSet, draupnir, capabilitySet, _settings) { + factory(description, protectedRoomsSet, draupnir, capabilitySet, settings) { if ( draupnir.stores.hashStore === undefined || draupnir.stores.roomAuditLog === undefined @@ -97,10 +176,20 @@ describeProtection({ ); } const roomDiscovery = (() => { + const roomDetailsProvider = draupnir.synapseAdminClient + ? new SynapseAdminRoomDetailsProvider(draupnir.synapseAdminClient) + : undefined; + if (roomDetailsProvider === undefined) { + log.warn( + "This protection currently requires synapse admin capability in order to fetch room details" + ); + return undefined; + } if (draupnir.synapseHTTPAntispam !== undefined) { return new SynapseHTTPAntispamRoomDiscovery( draupnir.synapseHTTPAntispam, - draupnir.stores.hashStore + draupnir.stores.hashStore, + roomDetailsProvider ); } else { log.warn( @@ -116,6 +205,10 @@ describeProtection({ protectedRoomsSet, draupnir.stores.hashStore, draupnir.stores.roomAuditLog, + draupnir.clientPlatform.toRoomMessageSender(), + settings.discoveryNotificationEnabled, + settings.discoveryNotificationMembershipThreshold, + settings.discoveryNotificationRoom ?? draupnir.managementRoomID, roomDiscovery ) ); diff --git a/src/protections/RoomTakedown/SynapseAdminRoomDetails.ts b/src/protections/RoomTakedown/SynapseAdminRoomDetails.ts new file mode 100644 index 00000000..b1678c30 --- /dev/null +++ b/src/protections/RoomTakedown/SynapseAdminRoomDetails.ts @@ -0,0 +1,3 @@ +// SPDX-FileCopyrightText: 2025 Gnuxie +// +// SPDX-License-Identifier: AFL-3.0 From 9d5e19514a98dba905da3c0aceab98009527c9cf Mon Sep 17 00:00:00 2001 From: gnuxie Date: Tue, 18 Mar 2025 17:03:37 +0000 Subject: [PATCH 18/48] Store details about the room at takedown in the audit log. We need to do this because they will be impossible to obtain once the room has been blocked on the server. We also need to make sure we now include the server the room was created on, and the creator in the room hash store. --- src/capabilities/RoomTakedownCapability.tsx | 3 +- .../RoomTakedownCapabilityRenderer.tsx | 6 ++ .../SynapseAdminRoomTakedown.ts | 8 ++ src/protections/RoomTakedown/RoomAuditLog.ts | 23 +++-- src/protections/RoomTakedown/RoomTakedown.ts | 16 +++- .../RoomTakedown/SqliteRoomAuditLog.ts | 92 ++++++++++++++++++- test/unit/stores/roomAuditLogTest.ts | 23 ++++- 7 files changed, 155 insertions(+), 16 deletions(-) diff --git a/src/capabilities/RoomTakedownCapability.tsx b/src/capabilities/RoomTakedownCapability.tsx index dfe060ab..ce3eb2af 100644 --- a/src/capabilities/RoomTakedownCapability.tsx +++ b/src/capabilities/RoomTakedownCapability.tsx @@ -30,7 +30,8 @@ export const RoomTakedownCapability = Type.Intersect([ export type RoomTakedownCapability = { isRoomTakendown(roomID: StringRoomID): Promise>; takedownRoom(roomID: StringRoomID): Promise>; -} & Capability; +} & RoomDetailsProvider & + Capability; describeCapabilityInterface({ name: "RoomTakedownCapability", diff --git a/src/capabilities/RoomTakedownCapabilityRenderer.tsx b/src/capabilities/RoomTakedownCapabilityRenderer.tsx index 0792dde0..4987e1aa 100644 --- a/src/capabilities/RoomTakedownCapabilityRenderer.tsx +++ b/src/capabilities/RoomTakedownCapabilityRenderer.tsx @@ -105,6 +105,12 @@ class StandardRoomTakedownCapabilityRenderer implements RoomTakedownCapability { ); return capabilityResult; } + + public async getRoomDetails( + roomID: StringRoomID + ): Promise> { + return await this.capability.getRoomDetails(roomID); + } } describeCapabilityRenderer({ diff --git a/src/capabilities/SynapseAdminRoomTakedown/SynapseAdminRoomTakedown.ts b/src/capabilities/SynapseAdminRoomTakedown/SynapseAdminRoomTakedown.ts index e0db23e7..18bfcaca 100644 --- a/src/capabilities/SynapseAdminRoomTakedown/SynapseAdminRoomTakedown.ts +++ b/src/capabilities/SynapseAdminRoomTakedown/SynapseAdminRoomTakedown.ts @@ -47,6 +47,8 @@ export class SynapseAdminRoomTakedownCapability public constructor(private readonly adminClient: SynapseAdminClient) { // nothing to do mare. } + + isSimulated?: true; public async isRoomTakendown(roomID: StringRoomID): Promise> { const blockStatusResponse = await this.adminClient.getBlockStatus(roomID); if (isError(blockStatusResponse)) { @@ -81,4 +83,10 @@ export class SynapseAdminRoomTakedownCapability return Ok(details); } } + + public async getRoomDetails( + roomID: StringRoomID + ): Promise> { + return await this.roomDetailsProvider.getRoomDetails(roomID); + } } diff --git a/src/protections/RoomTakedown/RoomAuditLog.ts b/src/protections/RoomTakedown/RoomAuditLog.ts index 89100e9b..6f169aa0 100644 --- a/src/protections/RoomTakedown/RoomAuditLog.ts +++ b/src/protections/RoomTakedown/RoomAuditLog.ts @@ -3,13 +3,24 @@ // SPDX-License-Identifier: Apache-2.0 import { Result } from "@gnuxie/typescript-result"; -import { StringRoomID } from "@the-draupnir-project/matrix-basic-types"; -import { LiteralPolicyRule } from "matrix-protection-suite"; +import { + StringEventID, + StringRoomID, +} from "@the-draupnir-project/matrix-basic-types"; +import { LiteralPolicyRule, RoomBasicDetails } from "matrix-protection-suite"; + +export type RoomTakedownDetails = Omit & { + policy_id: StringEventID; + created_at: number; +}; -// FIXME: Add a method for protection startup that takes all room ids from policies -// and only returns those not takendown -// i just want to save progress so far first. export interface RoomAuditLog { - takedownRoom(policy: LiteralPolicyRule): Promise>; + takedownRoom( + policy: LiteralPolicyRule, + details: RoomBasicDetails + ): Promise>; isRoomTakendown(roomID: StringRoomID): boolean; + getTakedownDetails( + roomID: StringRoomID + ): Promise>; } diff --git a/src/protections/RoomTakedown/RoomTakedown.ts b/src/protections/RoomTakedown/RoomTakedown.ts index 778734c7..0f0c4e45 100644 --- a/src/protections/RoomTakedown/RoomTakedown.ts +++ b/src/protections/RoomTakedown/RoomTakedown.ts @@ -71,6 +71,7 @@ export class StandardRoomTakedown implements RoomTakedownService { `The room ${roomID} has already been takendown according to your homeserver` ); } + const detailsResult = await this.takedownCapability.getRoomDetails(roomID); const takedownResult = await this.takedownCapability.takedownRoom(roomID); if (isError(takedownResult)) { return takedownResult; @@ -78,9 +79,20 @@ export class StandardRoomTakedown implements RoomTakedownService { // Only audit the takedown if the capability is not simulated. if (this.takedownCapability.isSimulated) { return Ok(undefined); - } else { - return await this.auditLog.takedownRoom(rule); } + const details = (() => { + if (isError(detailsResult)) { + log.error( + "Unable to fetch details for room before takedown", + roomID, + detailsResult.error + ); + return { room_id: roomID }; + } else { + return detailsResult.ok; + } + })(); + return await this.auditLog.takedownRoom(rule, details); } public async handlePolicyChange( diff --git a/src/protections/RoomTakedown/SqliteRoomAuditLog.ts b/src/protections/RoomTakedown/SqliteRoomAuditLog.ts index 46e955ca..f3f2df2c 100644 --- a/src/protections/RoomTakedown/SqliteRoomAuditLog.ts +++ b/src/protections/RoomTakedown/SqliteRoomAuditLog.ts @@ -3,13 +3,17 @@ // SPDX-License-Identifier: Apache-2.0 import { Ok, Result } from "@gnuxie/typescript-result"; -import { StringRoomID } from "@the-draupnir-project/matrix-basic-types"; -import { RoomAuditLog } from "./RoomAuditLog"; +import { + StringEventID, + StringRoomID, +} from "@the-draupnir-project/matrix-basic-types"; +import { RoomAuditLog, RoomTakedownDetails } from "./RoomAuditLog"; import { ActionException, ActionExceptionKind, LiteralPolicyRule, PolicyRuleType, + RoomBasicDetails, } from "matrix-protection-suite"; import { BetterSqliteOptions, @@ -24,6 +28,10 @@ import path from "path"; // check with the server rather than check the audit log. // Incase rooms were unbanned. +// FIXME: The entity should really be included in the policy info mare... +// This will be important when the rule targets users and servers and +// not just rooms. + const schema = [ `CREATE TABLE policy_info ( policy_id TEXT PRIMARY KEY NOT NULL, @@ -40,6 +48,16 @@ const schema = [ created_at INTEGER DEFAULT (unixepoch()) NOT NULL, FOREIGN KEY (policy_id) REFERENCES policy_info(policy_id) ) STRICT;`, + `CREATE TABLE room_detail_at_takedown ( + takedown_id INTEGER NOT NULL, + room_id TEXT NOT NULL, + creator TEXT, + name TEXT, + topic TEXT, + joined_members INTEGER, + FOREIGN KEY (takedown_id) REFERENCES room_takedown(id), + PRIMARY KEY (takedown_id, room_id) + ) STRICT;`, ]; export type RoomTakedown = { @@ -79,7 +97,10 @@ export class SqliteRoomAuditLog return new SqliteRoomAuditLog(options, makeBetterSqliteDB(options)); } - public async takedownRoom(policy: LiteralPolicyRule): Promise> { + public async takedownRoom( + policy: LiteralPolicyRule, + details: RoomBasicDetails + ): Promise> { if (policy.kind !== PolicyRuleType.Room) { throw new TypeError( "You can only use takedownRoom on room policies what are you doing?" @@ -92,6 +113,9 @@ export class SqliteRoomAuditLog ); const takedownStatement = this.db.prepare(` INSERT INTO room_takedown (policy_id, target_room_id) VALUES (?, ?)`); + const detailStatement = this.db.prepare(` + INSERT INTO room_detail_at_takedown (takedown_id, room_id, creator, name, topic, joined_members) + VALUES (?, ?, ?, ?, ?, ?)`); this.db.transaction(() => { policyInfoStatement.run([ policy.sourceEvent.event_id, @@ -101,7 +125,18 @@ export class SqliteRoomAuditLog policy.sourceEvent.type, policy.recommendation, ]); - takedownStatement.run([policy.sourceEvent.event_id, policy.entity]); + const takedownRow = takedownStatement.run([ + policy.sourceEvent.event_id, + policy.entity, + ]); + detailStatement.run([ + takedownRow.lastInsertRowid, + details.room_id, + details.creator ?? null, + details.name ?? null, + details.topic ?? null, + details.joined_members ?? null, + ]); })(); this.takedownRooms.add(policy.entity as StringRoomID); return Ok(undefined); @@ -117,6 +152,55 @@ export class SqliteRoomAuditLog } } + public async getTakedownDetails( + roomID: StringRoomID + ): Promise> { + try { + const query = this.db.prepare(` + SELECT + room_takedown.policy_id, + room_takedown.created_at, + room_detail_at_takedown.room_id, + room_detail_at_takedown.creator, + room_detail_at_takedown.name, + room_detail_at_takedown.topic, + room_detail_at_takedown.joined_members + FROM room_takedown + JOIN room_detail_at_takedown ON room_takedown.id = room_detail_at_takedown.takedown_id + WHERE room_takedown.target_room_id = ? + ORDER BY room_takedown.created_at DESC + LIMIT 1 + `); + type RowType = { + policy_id: StringEventID; + created_at: number; + room_id: StringEventID; + } & RoomTakedownDetails; + const row = query.get(roomID) as RowType | undefined; + if (row === undefined) { + return Ok(undefined); + } + return Ok({ + policy_id: row.policy_id, + created_at: row.created_at, + room_id: row.room_id, + creator: row.creator ?? undefined, + name: row.name ?? undefined, + topic: row.topic ?? undefined, + joined_members: row.joined_members ?? undefined, + }); + } catch (exception) { + if (exception instanceof Error) { + return ActionException.Result("Unable to fetch takedown details", { + exception, + exceptionKind: ActionExceptionKind.Unknown, + }); + } else { + throw exception; + } + } + } + private loadTakendownRooms(): StringRoomID[] { return this.db .prepare(`SELECT target_room_id FROM room_takedown`) diff --git a/test/unit/stores/roomAuditLogTest.ts b/test/unit/stores/roomAuditLogTest.ts index a3692971..5d9c039f 100644 --- a/test/unit/stores/roomAuditLogTest.ts +++ b/test/unit/stores/roomAuditLogTest.ts @@ -14,6 +14,7 @@ import { Recommendation, } from "matrix-protection-suite"; import expect from "expect"; +import { StringUserID } from "@the-draupnir-project/matrix-basic-types"; describe("RoomAuditLog test", function () { const options = { path: ":memory:" } satisfies BetterSqliteOptions; @@ -37,9 +38,25 @@ describe("RoomAuditLog test", function () { type: PolicyRuleType.Room, }) as never ).expect("Should bea ble to parse the policy rule."); - (await store.takedownRoom(ban as LiteralPolicyRule)).expect( - "Should be able to takedown a room" - ); + ( + await store.takedownRoom(ban as LiteralPolicyRule, { + room_id: bannedRoom.toRoomIDOrAlias(), + name: "Spam name", + topic: "Spam topic", + joined_members: 30, + creator: "@spam:example.com" as StringUserID, + }) + ).expect("Should be able to takedown a room"); expect(store.isRoomTakendown(bannedRoom.toRoomIDOrAlias())).toBe(true); + const takedownDetails = ( + await store.getTakedownDetails(bannedRoom.toRoomIDOrAlias()) + ).expect("Should be able to get takedown details"); + expect(takedownDetails?.created_at).toBeDefined(); + expect(takedownDetails?.creator).toBe("@spam:example.com"); + expect(takedownDetails?.joined_members).toBe(30); + expect(takedownDetails?.name).toBe("Spam name"); + expect(takedownDetails?.topic).toBe("Spam topic"); + expect(takedownDetails?.room_id).toBe(bannedRoom.toRoomIDOrAlias()); + expect(takedownDetails?.policy_id).toBeDefined(); }); }); From 2909464d36204e62fccc914d125366cf8cfab072 Mon Sep 17 00:00:00 2001 From: gnuxie Date: Wed, 19 Mar 2025 14:08:54 +0000 Subject: [PATCH 19/48] Store user and server in room hash store too. --- src/backingstore/better-sqlite3/HashStore.ts | 35 +++++++++---------- src/protections/RoomTakedown/RoomDiscovery.ts | 20 ++++++++--- src/protections/RoomTakedown/RoomTakedown.ts | 9 ----- .../RoomTakedown/RoomTakedownProtection.ts | 4 +-- .../protections/RoomTakedownServiceTest.ts | 4 --- 5 files changed, 34 insertions(+), 38 deletions(-) diff --git a/src/backingstore/better-sqlite3/HashStore.ts b/src/backingstore/better-sqlite3/HashStore.ts index 53eaeee5..deb54c10 100644 --- a/src/backingstore/better-sqlite3/HashStore.ts +++ b/src/backingstore/better-sqlite3/HashStore.ts @@ -10,8 +10,8 @@ import { HashedLiteralPolicyRule, LiteralPolicyRule, makeReversedHashedPolicy, - RoomBasicDetails, RoomHashRecord, + HashedRoomDetails, SHA256RoomHashStore, } from "matrix-protection-suite"; import { @@ -23,19 +23,16 @@ import { enc, SHA256 } from "crypto-js"; import { Database } from "better-sqlite3"; import path from "path"; -// We will need some kind of audit database to store actions in against rooms - const schema = [ `CREATE TABLE room_sha256 ( room_id TEXT PRIMARY KEY NOT NULL, sha256 TEXT NOT NULL ) STRICT;`, - `CREATE TABLE room_detail ( + `CREATE TABLE room_identification ( room_id TEXT PRIMARY KEY NOT NULL, - creator TEXT, - name TEXT, - topic TEXT, - joined_members INTEGER + creator TEXT NOT NULL, + server TEXT NOT NULL, + FOREIGN KEY (room_id) REFERENCES room_sha256(room_id) ) STRICT; `, ]; @@ -207,19 +204,19 @@ export class SqliteHashReversalStore } } - public async storeRoomDetails( - roomDetails: RoomBasicDetails + public async storeRoomIdentification( + roomDetails: HashedRoomDetails ): Promise> { try { - const statement = this.db.prepare(` - REPLACE INTO room_detail SELECT - value ->> 'room_id', - value ->> 'creator', - value ->> 'name', - value ->> 'topic', - value ->> 'joined_members' - FROM json_each(?)`); - statement.run(JSON.stringify(roomDetails)); + const statement = this.db.prepare( + `REPLACE INTO room_identification (room_id, creator, server) + VALUES (?, ?, ?)` + ); + statement.run([ + roomDetails.roomID, + roomDetails.creator, + roomDetails.server, + ]); return Ok(undefined); } catch (exception) { if (exception instanceof Error) { diff --git a/src/protections/RoomTakedown/RoomDiscovery.ts b/src/protections/RoomTakedown/RoomDiscovery.ts index 6d1cd66d..7dbc1076 100644 --- a/src/protections/RoomTakedown/RoomDiscovery.ts +++ b/src/protections/RoomTakedown/RoomDiscovery.ts @@ -14,7 +14,10 @@ import { } from "matrix-protection-suite"; import { CheckEventForSpamRequestBody } from "../../webapis/SynapseHTTPAntispam/CheckEventForSpamEndpoint"; import { SynapseHttpAntispam } from "../../webapis/SynapseHTTPAntispam/SynapseHttpAntispam"; -import { StringRoomID } from "@the-draupnir-project/matrix-basic-types"; +import { + StringRoomID, + userServerName, +} from "@the-draupnir-project/matrix-basic-types"; import { UserMayInviteRequestBody } from "../../webapis/SynapseHTTPAntispam/UserMayInviteEndpoint"; import { UserMayJoinRoomRequestBody } from "../../webapis/SynapseHTTPAntispam/UserMayJoinRoomEndpoint"; import { EventEmitter } from "stream"; @@ -74,9 +77,18 @@ export class SynapseHTTPAntispamRoomDiscovery continue; } this.emit("RoomDiscovery", detailsResult.ok); - const storeResult = await this.hashStore.storeRoomDetails( - detailsResult.ok - ); + if (detailsResult.ok.creator == undefined) { + log.error( + "Creator is missing from room details, which isn't great", + detailsResult.ok + ); + return; + } + const storeResult = await this.hashStore.storeRoomIdentification({ + creator: detailsResult.ok.creator, + roomID: detailsResult.ok.room_id, + server: userServerName(detailsResult.ok.creator), + }); if (isError(storeResult)) { log.error( "Error storing room details for a room", diff --git a/src/protections/RoomTakedown/RoomTakedown.ts b/src/protections/RoomTakedown/RoomTakedown.ts index 0f0c4e45..4a24673c 100644 --- a/src/protections/RoomTakedown/RoomTakedown.ts +++ b/src/protections/RoomTakedown/RoomTakedown.ts @@ -11,7 +11,6 @@ import { PolicyRuleMatchType, PolicyRuleType, Recommendation, - SHA256RoomHashStore, SimpleChangeType, } from "matrix-protection-suite"; import { RoomAuditLog } from "./RoomAuditLog"; @@ -28,7 +27,6 @@ const log = new Logger("RoomTakedown"); // Ok no that all sucks we'll have to wait for policy approval... export type RoomTakedownService = { - handleDiscoveredRooms(rooms: StringRoomID[]): Promise>; handlePolicyChange( revision: PolicyListRevision, changes: PolicyRuleChange[] @@ -44,18 +42,11 @@ export type RoomTakedownService = { */ export class StandardRoomTakedown implements RoomTakedownService { public constructor( - private readonly hashStore: SHA256RoomHashStore, private readonly auditLog: RoomAuditLog, private readonly takedownCapability: RoomTakedownCapability ) { // nothing to do } - // FIXME: We don't use this - public async handleDiscoveredRooms( - rooms: StringRoomID[] - ): Promise> { - return (await this.hashStore.storeUndiscoveredRooms(rooms)) as Result; - } private async takedownRoom( roomID: StringRoomID, diff --git a/src/protections/RoomTakedown/RoomTakedownProtection.ts b/src/protections/RoomTakedown/RoomTakedownProtection.ts index 1e70a7c6..9278a194 100644 --- a/src/protections/RoomTakedown/RoomTakedownProtection.ts +++ b/src/protections/RoomTakedown/RoomTakedownProtection.ts @@ -15,7 +15,7 @@ import { ProtectionDescription, RoomBasicDetails, RoomMessageSender, - SHA256RoomHashStore, + HashedRoomDetails, StringRoomIDSchema, Task, } from "matrix-protection-suite"; @@ -87,7 +87,7 @@ export class RoomTakedownProtection description: RoomTakedownProtectionDescription, capabilities: RoomTakedownProtectionCapabilities, protectedRoomsSet: ProtectedRoomsSet, - hashStore: SHA256RoomHashStore, + hashStore: HashedRoomDetails, auditLog: RoomAuditLog, private readonly roomMessageSender: RoomMessageSender, private readonly discoveryNotificationEnabled: boolean, diff --git a/test/unit/protections/RoomTakedownServiceTest.ts b/test/unit/protections/RoomTakedownServiceTest.ts index 9b9b820b..5bb9785d 100644 --- a/test/unit/protections/RoomTakedownServiceTest.ts +++ b/test/unit/protections/RoomTakedownServiceTest.ts @@ -12,7 +12,6 @@ import { PolicyRuleType, randomRoomID, Recommendation, - SHA256RoomHashStore, SimpleChangeType, StandardPolicyListRevision, } from "matrix-protection-suite"; @@ -53,7 +52,6 @@ function makeServiceMocks(): { } describe("", function () { - const storeFake = createMock({}); it("Test rooms are takendown when policies are added", async function () { const policyRoom = randomRoomID([]); const bannedRoom = randomRoomID([]); @@ -73,7 +71,6 @@ describe("", function () { takedownCapabilityItems, } = makeServiceMocks(); const takedownService = new StandardRoomTakedown( - storeFake, auditLog, takedownCapability ); @@ -123,7 +120,6 @@ describe("", function () { takedownCapabilityItems, } = makeServiceMocks(); const takedownService = new StandardRoomTakedown( - storeFake, auditLog, takedownCapability ); From 0de853b413675a6aba3a506fb49a7cbe919e23e5 Mon Sep 17 00:00:00 2001 From: gnuxie Date: Wed, 19 Mar 2025 14:36:07 +0000 Subject: [PATCH 20/48] Update Draupnir policy handling code for hashed entities. --- src/commands/Rules.tsx | 30 +++++++++++++++++++- src/commands/unban/UnbanUsers.tsx | 1 + src/protections/PolicyChangeNotification.tsx | 6 +++- src/protections/RedactionSynchronisation.ts | 10 +++++-- test/integration/banPropagationTest.ts | 3 +- 5 files changed, 44 insertions(+), 6 deletions(-) diff --git a/src/commands/Rules.tsx b/src/commands/Rules.tsx index 7ad80773..fe9006e3 100644 --- a/src/commands/Rules.tsx +++ b/src/commands/Rules.tsx @@ -9,10 +9,14 @@ // import { + GlobPolicyRule, + HashedLiteralPolicyRule, + LiteralPolicyRule, MemberPolicyMatches, Ok, PolicyRoomWatchProfile, PolicyRule, + PolicyRuleMatchType, isError, } from "matrix-protection-suite"; import { @@ -53,11 +57,35 @@ function renderListMatches( ); } +export function renderRuleHashes(rule: HashedLiteralPolicyRule): DocumentNode { + return ( +
    + {Object.entries(rule.hashes).map(([algorithm, hash]) => ( +
  • + {algorithm}: {hash} +
    • + ))} +
+ ); +} + +export function renderRuleClearText( + rule: LiteralPolicyRule | GlobPolicyRule +): DocumentNode { + return ( + + {rule.entity} ({rule.reason ?? ""}) + + ); +} + function renderRuleSummary(rule: PolicyRule) { return (
  • {rule.kind} ({rule.recommendation}):{" "} - {rule.entity} ({rule.reason}) + {rule.matchType === PolicyRuleMatchType.HashedLiteral + ? renderRuleHashes(rule) + : renderRuleClearText(rule)}
    • ); } diff --git a/src/commands/unban/UnbanUsers.tsx b/src/commands/unban/UnbanUsers.tsx index dd8059e6..436b5a0a 100644 --- a/src/commands/unban/UnbanUsers.tsx +++ b/src/commands/unban/UnbanUsers.tsx @@ -161,6 +161,7 @@ export async function unbanMembers( for (const policy of policyRoom.matches) { policiesRemoved.addResult( policyRoom.roomID, + // FIXME: Update editor to no longer match all matching entities. (await policyRoomEditor.ok.removePolicy( policy.kind, policy.recommendation, diff --git a/src/protections/PolicyChangeNotification.tsx b/src/protections/PolicyChangeNotification.tsx index fcba713f..3fca8b6c 100644 --- a/src/protections/PolicyChangeNotification.tsx +++ b/src/protections/PolicyChangeNotification.tsx @@ -17,6 +17,7 @@ import { PolicyRoomManager, PolicyRoomRevisionIssuer, PolicyRuleChange, + PolicyRuleMatchType, ProtectedRoomsSet, ProtectionDescription, UnknownConfig, @@ -38,6 +39,7 @@ import { DocumentNode, } from "@the-draupnir-project/interface-manager"; import { sendMatrixEventsFromDeadDocument } from "../commands/interface-manager/MPSMatrixInterfaceAdaptor"; +import { renderRuleHashes, renderRuleClearText } from "../commands/Rules"; const log = new Logger("PolicyChangeNotification"); @@ -134,7 +136,9 @@ function renderListChange(change: PolicyRuleChange): DocumentNode { {renderMentionPill(change.sender, change.sender)}{" "} {change.changeType} {change.rule.kind} ({change.rule.recommendation}) - {change.rule.entity} ({change.rule.reason}) + {change.rule.matchType === PolicyRuleMatchType.HashedLiteral + ? renderRuleHashes(change.rule) + : renderRuleClearText(change.rule)} ); diff --git a/src/protections/RedactionSynchronisation.ts b/src/protections/RedactionSynchronisation.ts index 31e9c8fc..c8d48992 100644 --- a/src/protections/RedactionSynchronisation.ts +++ b/src/protections/RedactionSynchronisation.ts @@ -17,6 +17,7 @@ import { PolicyListRevision, PolicyRule, PolicyRuleChange, + PolicyRuleMatchType, PolicyRuleType, PowerLevelPermission, ProtectedRoomsSet, @@ -59,7 +60,10 @@ export class RedactionSynchronisationProtection } public redactForNewUserPolicy(policy: PolicyRule): void { const rooms: StringRoomID[] = []; - if (policy.isGlob()) { + if (policy.matchType === PolicyRuleMatchType.HashedLiteral) { + return; // wait for them to be reversed and placed into the model as clear text. + } + if (policy.matchType === PolicyRuleMatchType.Glob) { this.protectedRoomsSet.allProtectedRooms.forEach((room) => rooms.push(room.toRoomIDOrAlias()) ); @@ -92,7 +96,7 @@ export class RedactionSynchronisationProtection change.changeType === SimpleChangeType.Added && change.rule.kind === PolicyRuleType.User && this.automaticRedactionReasons.some((reason) => - reason.test(change.rule.reason) + reason.test(change.rule.reason ?? "") ) ); // Can't see this fucking up at all when watching a new list :skull:. @@ -133,7 +137,7 @@ export class RedactionSynchronisationProtection return ( matchingPolicy !== undefined && this.automaticRedactionReasons.some((reason) => - reason.test(matchingPolicy.reason) + reason.test(matchingPolicy.reason ?? "") ) ); } else { diff --git a/test/integration/banPropagationTest.ts b/test/integration/banPropagationTest.ts index 5767d753..cb43fe31 100644 --- a/test/integration/banPropagationTest.ts +++ b/test/integration/banPropagationTest.ts @@ -7,6 +7,7 @@ import { newTestUser } from "./clientHelper"; import { getFirstEventMatching } from "./commands/commandUtils"; import { DraupnirTestContext, draupnirClient } from "./mjolnirSetupUtils"; import { + LiteralPolicyRule, MembershipChangeType, NoticeMessageContent, PolicyRuleType, @@ -122,7 +123,7 @@ describe("Ban propagation test", function () { PolicyRuleType.User ); expect(rules.length).toBe(1); - expect(rules[0]?.entity).toBe(spamUserID); + expect((rules[0] as LiteralPolicyRule).entity).toBe(spamUserID); expect(rules[0]?.reason).toBe("spam"); // now unban them >:3 From eaa21680cf77105991719f5cf6a36ed8e14f6868 Mon Sep 17 00:00:00 2001 From: gnuxie Date: Wed, 19 Mar 2025 14:38:28 +0000 Subject: [PATCH 21/48] Missing stores initialization code in test fixtures. --- test/integration/fixtures.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/integration/fixtures.ts b/test/integration/fixtures.ts index 2b6dd066..e7138a25 100644 --- a/test/integration/fixtures.ts +++ b/test/integration/fixtures.ts @@ -41,7 +41,10 @@ export const mochaHooks = { this.timeout(30000); const config = (this.config = configRead()); this.managementRoomAlias = config.managementRoom; - this.toggle = await makeBotModeToggle(config, { eraseAccountData: true }); + this.toggle = await makeBotModeToggle(config, { + eraseAccountData: true, + stores: {}, + }); this.draupnir = draupnir(); const draupnirMatrixClient = draupnirClient(); if (draupnirMatrixClient === null) { From a6b6508af1fb6be856377c3bb420d04e64f149f2 Mon Sep 17 00:00:00 2001 From: gnuxie Date: Wed, 19 Mar 2025 14:53:09 +0000 Subject: [PATCH 22/48] Update unban code for unmasked hashed entities. We will need to work harder to remove masked hashed entities... --- src/commands/unban/UnbanUsers.tsx | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/src/commands/unban/UnbanUsers.tsx b/src/commands/unban/UnbanUsers.tsx index 436b5a0a..425b4afa 100644 --- a/src/commands/unban/UnbanUsers.tsx +++ b/src/commands/unban/UnbanUsers.tsx @@ -27,6 +27,7 @@ import { isError, Ok, PolicyRuleType, + PolicyRuleMatchType, } from "matrix-protection-suite"; import { UnlistedUserRedactionQueue } from "../../queues/UnlistedUserRedactionQueue"; import { ListMatches } from "../Rules"; @@ -161,12 +162,10 @@ export async function unbanMembers( for (const policy of policyRoom.matches) { policiesRemoved.addResult( policyRoom.roomID, - // FIXME: Update editor to no longer match all matching entities. - (await policyRoomEditor.ok.removePolicy( + await policyRoomEditor.ok.removePolicyByStateKey( policy.kind, - policy.recommendation, - policy.entity - )) as Result + policy.sourceEvent.state_key + ) ); } } @@ -241,11 +240,16 @@ export function findUnbanInformationForMember( ]; for (const { matches } of policyMatchesToRemove) { for (const match of matches) { - if (match.isGlob()) { + if (match.matchType === PolicyRuleMatchType.Glob) { globsToScan.push(new MatrixGlob(match.entity)); - } else if (isStringUserID(match.entity)) { + } else if ( + match.matchType === PolicyRuleMatchType.Literal && + isStringUserID(match.entity) + ) { literalsToScan.push(match.entity); } + // HashedLiterals that match will be removed indriectly when their sourceEvent's + // get removed. } } const membersMatchingPoliciesAndEntity = matchMembers( From 19a78d670cff465667adbe93956530c11e4adc8d Mon Sep 17 00:00:00 2001 From: gnuxie Date: Wed, 19 Mar 2025 17:29:13 +0000 Subject: [PATCH 23/48] Some shite that didn't get comitted to do with messing up hash store. --- src/protections/RoomTakedown/RoomTakedownProtection.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/protections/RoomTakedown/RoomTakedownProtection.ts b/src/protections/RoomTakedown/RoomTakedownProtection.ts index 9278a194..d4e8f81d 100644 --- a/src/protections/RoomTakedown/RoomTakedownProtection.ts +++ b/src/protections/RoomTakedown/RoomTakedownProtection.ts @@ -15,7 +15,6 @@ import { ProtectionDescription, RoomBasicDetails, RoomMessageSender, - HashedRoomDetails, StringRoomIDSchema, Task, } from "matrix-protection-suite"; @@ -87,7 +86,6 @@ export class RoomTakedownProtection description: RoomTakedownProtectionDescription, capabilities: RoomTakedownProtectionCapabilities, protectedRoomsSet: ProtectedRoomsSet, - hashStore: HashedRoomDetails, auditLog: RoomAuditLog, private readonly roomMessageSender: RoomMessageSender, private readonly discoveryNotificationEnabled: boolean, @@ -97,7 +95,6 @@ export class RoomTakedownProtection ) { super(description, capabilities, protectedRoomsSet, {}); this.roomTakedown = new StandardRoomTakedown( - hashStore, auditLog, capabilities.roomTakedownCapability ); @@ -203,7 +200,6 @@ describeProtection< description, capabilitySet, protectedRoomsSet, - draupnir.stores.hashStore, draupnir.stores.roomAuditLog, draupnir.clientPlatform.toRoomMessageSender(), settings.discoveryNotificationEnabled, From 8cf7e4b638b0c73eb258811f8d4718e856129b70 Mon Sep 17 00:00:00 2001 From: gnuxie Date: Wed, 19 Mar 2025 17:34:24 +0000 Subject: [PATCH 24/48] Add initial draft of takedown command. Now we need to hook in discovery when the command is run. --- src/commands/Ban.tsx | 2 +- src/commands/DraupnirCommands.ts | 2 + src/commands/Takedown.tsx | 205 ++++++++++++++++++ src/protections/RoomTakedown/RoomDiscovery.ts | 2 + 4 files changed, 210 insertions(+), 1 deletion(-) create mode 100644 src/commands/Takedown.tsx diff --git a/src/commands/Ban.tsx b/src/commands/Ban.tsx index a8995845..4087c3aa 100644 --- a/src/commands/Ban.tsx +++ b/src/commands/Ban.tsx @@ -65,7 +65,7 @@ export const DraupnirBanCommand = describeCommand({ { name: "entity", description: - "The entity to ban. This can be a user ID, room ID, or server name.", + "The entity to ban. This can be a user ID, or server name. For rooms please see the takedown command", acceptor: union( MatrixUserIDPresentationType, MatrixRoomReferencePresentationSchema, diff --git a/src/commands/DraupnirCommands.ts b/src/commands/DraupnirCommands.ts index 24ad3abd..f1bad558 100644 --- a/src/commands/DraupnirCommands.ts +++ b/src/commands/DraupnirCommands.ts @@ -55,6 +55,7 @@ import { DraupnirSafeModeCommand } from "./SafeModeCommand"; import { DraupnirProtectionsShowCommand } from "./ProtectionsShowCommand"; import { DraupnirProtectionsCapabilityCommand } from "./ProtectionsCapabilitiesCommand"; import { JoinWaveCommandTable } from "../protections/JoinWaveShortCircuit"; +import { DraupnirTakedownCommand } from "./Takedown"; // TODO: These commands should all be moved to subdirectories tbh and this // should be split like an index file for each subdirectory. @@ -120,6 +121,7 @@ const DraupnirCommands = new StandardCommandTable("draupnir") .internCommand(DraupnirDisplaynameCommand, ["displayname"]) .internCommand(DraupnirSetPowerLevelCommand, ["powerlevel"]) .internCommand(DraupnirStatusCommand, ["status"]) + .internCommand(DraupnirTakedownCommand, ["takedown"]) .internCommand(DraupnirUnbanCommand, ["unban"]) .internCommand(DraupnirWatchPolicyRoomCommand, ["watch"]) .internCommand(DraupnirUnwatchPolicyRoomCommand, ["unwatch"]); diff --git a/src/commands/Takedown.tsx b/src/commands/Takedown.tsx new file mode 100644 index 00000000..bbc085f4 --- /dev/null +++ b/src/commands/Takedown.tsx @@ -0,0 +1,205 @@ +// SPDX-FileCopyrightText: 2025 Gnuxie +// +// SPDX-License-Identifier: AFL-3.0 + +import { + BasicInvocationInformation, + DeadDocumentJSX, + describeCommand, + DocumentNode, + MatrixRoomIDPresentationType, + MatrixRoomReferencePresentationSchema, + MatrixUserIDPresentationType, + StringPresentationType, + tuple, + union, +} from "@the-draupnir-project/interface-manager"; +import { + DraupnirBanCommandContext, + findPolicyRoomEditorFromRoomReference, +} from "./Ban"; +import { isError, Ok, PolicyRuleType } from "matrix-protection-suite"; +import { Result, ResultError } from "@gnuxie/typescript-result"; +import { + MatrixRoomID, + MatrixUserID, + StringEventID, +} from "@the-draupnir-project/matrix-basic-types"; +import { + DraupnirContextToCommandContextTranslator, + DraupnirInterfaceAdaptor, +} from "./DraupnirCommandPrerequisites"; +import { renderRoomPill } from "./interface-manager/MatrixHelpRenderer"; + +export type TakedownPolicyPreview = { + ruleType: PolicyRuleType; + entity: string; + policyRoom: MatrixRoomID; +}; + +export const DraupnirTakedownCommand = describeCommand({ + summary: + "Mark an entity for takedown. This command is used to mark illegal or intollerable content as takedown. This is the strongest of consequences and is usualy irreversable.\ + This can be used to block rooms on your homeserver and discard spam invitations.", + parameters: tuple( + { + name: "entity", + description: + "The entity to ban. This can be a user ID, room ID, or server name.", + acceptor: union( + MatrixUserIDPresentationType, + MatrixRoomReferencePresentationSchema, + StringPresentationType + ), + }, + { + name: "list", + acceptor: union( + MatrixRoomReferencePresentationSchema, + StringPresentationType + ), + prompt: async function ({ + policyRoomManager, + clientUserID, + }: DraupnirBanCommandContext) { + return Ok({ + suggestions: policyRoomManager + .getEditablePolicyRoomIDs(clientUserID, PolicyRuleType.User) + .map((room) => MatrixRoomIDPresentationType.wrap(room)), + }); + }, + } + ), + keywords: { + keywordDescriptions: { + "no-confirm": { + isFlag: true, + description: + "Runs the command without the preview of the unban and the confirmation prompt.", + }, + }, + }, + async executor( + { + watchedPolicyRooms, + policyRoomManager, + roomResolver, + }: DraupnirBanCommandContext, + _info: BasicInvocationInformation, + keywords, + _rest, + entity, + policyRoomDesignator + ): Promise> { + const policyRoomReference = + typeof policyRoomDesignator === "string" + ? Ok( + watchedPolicyRooms.findPolicyRoomFromShortcode(policyRoomDesignator) + ?.room + ) + : Ok(policyRoomDesignator); + if (isError(policyRoomReference)) { + return policyRoomReference; + } + if (policyRoomReference.ok === undefined) { + return ResultError.Result( + `Unable to find a policy room from the shortcode ${policyRoomDesignator.toString()}` + ); + } + const policyRoomEditorResult = await findPolicyRoomEditorFromRoomReference( + roomResolver, + policyRoomManager, + policyRoomReference.ok + ); + if (isError(policyRoomEditorResult)) { + return policyRoomEditorResult; + } + const policyRoomEditor = policyRoomEditorResult.ok; + const preview = await (async () => { + if (entity instanceof MatrixUserID) { + return Ok({ + ruleType: PolicyRuleType.User, + entity: entity.toString(), + policyRoom: policyRoomEditor.room, + }); + } else if (typeof entity === "string") { + return Ok({ + ruleType: PolicyRuleType.Server, + entity, + policyRoom: policyRoomEditor.room, + }); + } else { + const resolvedRoomReference = await roomResolver.resolveRoom(entity); + if (isError(resolvedRoomReference)) { + return resolvedRoomReference; + } + return Ok({ + ruleType: PolicyRuleType.Room, + entity: resolvedRoomReference.ok.toRoomIDOrAlias(), + policyRoom: policyRoomEditor.room, + }); + } + })(); + if (isError(preview)) { + return preview; + } + if (!keywords.getKeywordValue("no-confirm", false)) { + return preview; + } + // FIXME: we should inform the hash store about the entity right around here + return await policyRoomEditor.takedownEntity( + preview.ok.ruleType, + preview.ok.entity, + {} + ); + }, +}); + +function renderTakedownPreview(preview: TakedownPolicyPreview): DocumentNode { + return ( + + You are about to mark an entity to be takendown by any means necessary: +
      +
    • + entity: {preview.entity} +
      • +
    • + policy type: {preview.ruleType} +
      • +
    • policy room: {renderRoomPill(preview.policyRoom)}
      • +
    + Please consider that doing so may have irreversable effects. + + You MUST only use takedown policies to mark spam, illegal, or otherwise + intolerable content. DO NOT takedown users who have committed a code of + conduct violation. + +     + ); +} + +DraupnirInterfaceAdaptor.describeRenderer(DraupnirTakedownCommand, { + isAlwaysSupposedToUseDefaultRenderer: true, + confirmationPromptJSXRenderer(commandResult) { + if (isError(commandResult)) { + return Ok(undefined); + } else if (typeof commandResult.ok === "string") { + return Ok(undefined); // it's an event id that's got nothing going for it. + } else { + return Ok({renderTakedownPreview(commandResult.ok)}); + } + }, +}); + +DraupnirContextToCommandContextTranslator.registerTranslation( + DraupnirTakedownCommand, + function (draupnir) { + return { + policyRoomManager: draupnir.policyRoomManager, + watchedPolicyRooms: draupnir.protectedRoomsSet.watchedPolicyRooms, + defaultReasons: draupnir.config.commands.ban.defaultReasons, + roomResolver: draupnir.clientPlatform.toRoomResolver(), + clientUserID: draupnir.clientUserID, + }; + } +); diff --git a/src/protections/RoomTakedown/RoomDiscovery.ts b/src/protections/RoomTakedown/RoomDiscovery.ts index 7dbc1076..754c61ff 100644 --- a/src/protections/RoomTakedown/RoomDiscovery.ts +++ b/src/protections/RoomTakedown/RoomDiscovery.ts @@ -62,6 +62,8 @@ export class SynapseHTTPAntispamRoomDiscovery ); } + // FIXME: Factor this out so that we can add room details and hash + // when the takedown command is run. private addRoomDetails(discoveredRooms: RoomHashRecord[]): void { void Task( (async () => { From ef592a95e4a0bcfcbd2058fd81d760ffcbed1697 Mon Sep 17 00:00:00 2001 From: gnuxie Date: Wed, 19 Mar 2025 18:02:32 +0000 Subject: [PATCH 25/48] Plumb takedown command into room discovery. --- src/commands/Takedown.tsx | 99 +++++++++++++++++-- src/protections/RoomTakedown/RoomDiscovery.ts | 2 - 2 files changed, 92 insertions(+), 9 deletions(-) diff --git a/src/commands/Takedown.tsx b/src/commands/Takedown.tsx index bbc085f4..c85d9482 100644 --- a/src/commands/Takedown.tsx +++ b/src/commands/Takedown.tsx @@ -14,22 +14,85 @@ import { tuple, union, } from "@the-draupnir-project/interface-manager"; +import { findPolicyRoomEditorFromRoomReference } from "./Ban"; import { - DraupnirBanCommandContext, - findPolicyRoomEditorFromRoomReference, -} from "./Ban"; -import { isError, Ok, PolicyRuleType } from "matrix-protection-suite"; + isError, + Logger, + Ok, + PolicyRoomManager, + PolicyRuleType, + RoomResolver, + SHA256RoomHashStore, + Task, + WatchedPolicyRooms, +} from "matrix-protection-suite"; import { Result, ResultError } from "@gnuxie/typescript-result"; import { MatrixRoomID, MatrixUserID, StringEventID, + StringRoomID, + StringUserID, + userServerName, } from "@the-draupnir-project/matrix-basic-types"; import { DraupnirContextToCommandContextTranslator, DraupnirInterfaceAdaptor, } from "./DraupnirCommandPrerequisites"; import { renderRoomPill } from "./interface-manager/MatrixHelpRenderer"; +import { RoomDetailsProvider } from "../capabilities/RoomTakedownCapability"; +import { SynapseAdminRoomDetailsProvider } from "../capabilities/SynapseAdminRoomTakedown/SynapseAdminRoomTakedown"; + +const log = new Logger("DraupnirTakedownCommand"); + +/** + * Make sure that the hash store is up to date with the entity + */ +function handleRoomDiscovery( + roomID: StringRoomID, + store: SHA256RoomHashStore | undefined, + detailsProvider: RoomDetailsProvider | undefined +): void { + if (store === undefined) { + log.warn( + "Unable to discover a room provided by the takedown command because the hash store has not been configured" + ); + return; + } + if (detailsProvider === undefined) { + log.warn( + "Unable to discover a room provided by the takedown command because the details provider has not been configured" + ); + return; + } + void Task( + (async () => { + const storeResult = await store.storeUndiscoveredRooms([roomID]); + if (isError(storeResult)) { + return storeResult.elaborate( + "Unable to store the room from takedown command into the hash store" + ); + } + const detailsResult = await detailsProvider.getRoomDetails(roomID); + if (isError(detailsResult)) { + return detailsResult.elaborate( + "Failed to fetch details for a room discovered via the takedown command" + ); + } + if (detailsResult.ok.creator === undefined) { + log.debug("Broken details", detailsResult.ok); + return ResultError.Result( + `No creator was provided so we cannot store the details for this room ${roomID}` + ); + } + return await store.storeRoomIdentification({ + roomID, + creator: detailsResult.ok.creator, + server: userServerName(detailsResult.ok.creator), + }); + })() + ); +} export type TakedownPolicyPreview = { ruleType: PolicyRuleType; @@ -61,7 +124,7 @@ export const DraupnirTakedownCommand = describeCommand({ prompt: async function ({ policyRoomManager, clientUserID, - }: DraupnirBanCommandContext) { + }: DraupnirTakedownCommandContext) { return Ok({ suggestions: policyRoomManager .getEditablePolicyRoomIDs(clientUserID, PolicyRuleType.User) @@ -84,7 +147,9 @@ export const DraupnirTakedownCommand = describeCommand({ watchedPolicyRooms, policyRoomManager, roomResolver, - }: DraupnirBanCommandContext, + hashStore, + detailsProvider, + }: DraupnirTakedownCommandContext, _info: BasicInvocationInformation, keywords, _rest, @@ -146,7 +211,13 @@ export const DraupnirTakedownCommand = describeCommand({ if (!keywords.getKeywordValue("no-confirm", false)) { return preview; } - // FIXME: we should inform the hash store about the entity right around here + if (preview.ok.ruleType === PolicyRuleType.Room) { + handleRoomDiscovery( + preview.ok.entity as StringRoomID, + hashStore, + detailsProvider + ); + } return await policyRoomEditor.takedownEntity( preview.ok.ruleType, preview.ok.entity, @@ -191,6 +262,16 @@ DraupnirInterfaceAdaptor.describeRenderer(DraupnirTakedownCommand, { }, }); +export type DraupnirTakedownCommandContext = { + policyRoomManager: PolicyRoomManager; + watchedPolicyRooms: WatchedPolicyRooms; + defaultReasons: string[]; + roomResolver: RoomResolver; + clientUserID: StringUserID; + hashStore: SHA256RoomHashStore | undefined; + detailsProvider: RoomDetailsProvider | undefined; +}; + DraupnirContextToCommandContextTranslator.registerTranslation( DraupnirTakedownCommand, function (draupnir) { @@ -200,6 +281,10 @@ DraupnirContextToCommandContextTranslator.registerTranslation( defaultReasons: draupnir.config.commands.ban.defaultReasons, roomResolver: draupnir.clientPlatform.toRoomResolver(), clientUserID: draupnir.clientUserID, + hashStore: draupnir.stores.hashStore, + detailsProvider: draupnir.synapseAdminClient + ? new SynapseAdminRoomDetailsProvider(draupnir.synapseAdminClient) + : undefined, }; } ); diff --git a/src/protections/RoomTakedown/RoomDiscovery.ts b/src/protections/RoomTakedown/RoomDiscovery.ts index 754c61ff..7dbc1076 100644 --- a/src/protections/RoomTakedown/RoomDiscovery.ts +++ b/src/protections/RoomTakedown/RoomDiscovery.ts @@ -62,8 +62,6 @@ export class SynapseHTTPAntispamRoomDiscovery ); } - // FIXME: Factor this out so that we can add room details and hash - // when the takedown command is run. private addRoomDetails(discoveredRooms: RoomHashRecord[]): void { void Task( (async () => { From 377917f6f403eb7263d3bc18b857285023629022 Mon Sep 17 00:00:00 2001 From: gnuxie Date: Wed, 19 Mar 2025 20:15:20 +0000 Subject: [PATCH 26/48] Improve Sqlite wrapper Schema migration. This cleans up all of the code surrounding schema migration and allows the migration procedure to be transactional so that people can downgrade draupnir if something explodes. Co-authored-by: Bea <20361868+enbea@users.noreply.github.com> --- .../better-sqlite3/BetterSqliteStore.ts | 117 ++---------------- src/backingstore/better-sqlite3/HashStore.ts | 48 ++++--- .../SqliteRoomStateBackingStore.ts | 64 ++++++---- .../better-sqlite3/SqliteSchema.ts | 102 +++++++++++++++ .../RoomTakedown/SqliteRoomAuditLog.ts | 62 ++++++---- 5 files changed, 220 insertions(+), 173 deletions(-) create mode 100644 src/backingstore/better-sqlite3/SqliteSchema.ts diff --git a/src/backingstore/better-sqlite3/BetterSqliteStore.ts b/src/backingstore/better-sqlite3/BetterSqliteStore.ts index dc26f56c..b9b07663 100644 --- a/src/backingstore/better-sqlite3/BetterSqliteStore.ts +++ b/src/backingstore/better-sqlite3/BetterSqliteStore.ts @@ -10,36 +10,19 @@ import BetterSqlite3, { Database } from "better-sqlite3"; import { Logger } from "matrix-protection-suite"; - -const log = new Logger("BetterSqliteStore"); - -export function sqliteV0Schema(db: Database) { - // we have to prepare and run them separately because `prepare` checks if the - // table exists. - const createTable = db.transaction(() => { - db.prepare( - `CREATE TABLE schema ( - version INTEGER UNIQUE NOT NULL - ) STRICT;` - ).run(); - db.prepare("INSERT INTO schema VALUES (0);").run(); - }); - createTable(); -} +import { ensureSqliteSchema, SqliteSchemaOptions } from "./SqliteSchema"; export interface BetterSqliteOptions extends BetterSqlite3.Options { path: string; - /** - * Should the schema table be automatically created (the v0 schema effectively). - * Defaults to `true`. - */ - autocreateSchemaTable?: boolean; - WALMode?: boolean; foreignKeys?: boolean; } -export function makeBetterSqliteDB(options: BetterSqliteOptions): Database { +export function makeBetterSqliteDB( + options: BetterSqliteOptions, + log: Logger +): Database { + log.info("Opening db: ", options.path); const db = new BetterSqlite3(options.path, options); if (options.path !== ":memory:") { db.pragma("temp_store = file"); // Avoid unnecessary memory usage. @@ -95,10 +78,6 @@ export type SchemaUpdateFunction = (db: Database) => void; export class BetterSqliteStore { private hasEnded = false; - public get latestSchema() { - return this.schemas.length; - } - /** * Construct a new store. * @param schemas The set of schema functions to apply to a database. The ordering of this array determines the @@ -106,54 +85,11 @@ export class BetterSqliteStore { * @param opts Options to supply to the BetterSqliteStore client, such as `path`. */ constructor( - private readonly schemas: SchemaUpdateFunction[], - private readonly opts: BetterSqliteOptions, - protected readonly db: Database + private readonly schema: SqliteSchemaOptions, + protected readonly db: Database, + private readonly log: Logger ) { - opts.autocreateSchemaTable = opts.autocreateSchemaTable ?? true; - } - - /** - * Ensure the database schema is up to date. If you supplied - * `autocreateSchemaTable` to `opts` in the constructor, a fresh database - * will have a `schema` table created for it. - * - * @throws If a schema could not be applied cleanly. - */ - public ensureSchema(): void { - log.info("Starting database engine"); - let currentVersion = this.getSchemaVersion(); - - if (currentVersion === -1) { - if (this.opts.autocreateSchemaTable) { - log.info(`Applying v0 schema (schema table)`); - sqliteV0Schema(this.db); - currentVersion = 0; - } else { - // We aren't autocreating the schema table, so assume schema 0. - currentVersion = 0; - } - } - - // Zero-indexed, so schema 1 would be in slot 0. - while (this.schemas[currentVersion]) { - log.info(`Updating schema to v${currentVersion + 1}`); - const runSchema = this.schemas[currentVersion]; - if (runSchema === undefined) { - throw new TypeError( - `there is no schema defined for the schema version: ${currentVersion}` - ); - } - try { - runSchema(this.db); - currentVersion++; - this.updateSchemaVersion(currentVersion); - } catch (ex) { - log.warn(`Failed to run schema v${currentVersion + 1}:`, ex); - throw Error("Failed to update database schema"); - } - } - log.info(`Database schema is at version v${currentVersion}`); + ensureSqliteSchema(this.db, this.log, this.schema); } /** @@ -161,43 +97,14 @@ export class BetterSqliteStore { * called before the process exits. */ public destroy(): void { - log.info("Destroy called"); + this.log.info("Destroy called"); if (this.hasEnded) { // No-op if end has already been called. return; } this.hasEnded = true; this.db.close(); - log.info("connection ended"); - } - - /** - * Update the current schema version. - * @param version - */ - protected updateSchemaVersion(version: number): void { - log.debug(`updateSchemaVersion: ${version}`); - this.db.prepare(`UPDATE schema SET version = ?;`).run(version); - } - - /** - * Get the current schema version. - * @returns The current schema version, or `-1` if no schema table is found. - */ - protected getSchemaVersion(): number { - try { - const result = this.db.prepare(`SELECT version FROM SCHEMA;`).get() as { - version: number; - }; - return result.version; - } catch (ex) { - if (ex instanceof Error && ex.message === "no such table: SCHEMA") { - return -1; - } else { - log.error("Failed to get schema version", ex); - } - } - throw Error("Couldn't fetch schema version"); + this.log.info("connection ended"); } } diff --git a/src/backingstore/better-sqlite3/HashStore.ts b/src/backingstore/better-sqlite3/HashStore.ts index deb54c10..7d8c7407 100644 --- a/src/backingstore/better-sqlite3/HashStore.ts +++ b/src/backingstore/better-sqlite3/HashStore.ts @@ -13,6 +13,7 @@ import { RoomHashRecord, HashedRoomDetails, SHA256RoomHashStore, + Logger, } from "matrix-protection-suite"; import { BetterSqliteOptions, @@ -22,21 +23,37 @@ import { import { enc, SHA256 } from "crypto-js"; import { Database } from "better-sqlite3"; import path from "path"; +import { checkKnownTables, SqliteSchemaOptions } from "./SqliteSchema"; -const schema = [ - `CREATE TABLE room_sha256 ( - room_id TEXT PRIMARY KEY NOT NULL, - sha256 TEXT NOT NULL - ) STRICT;`, - `CREATE TABLE room_identification ( +const log = new Logger("SqliteHashReversalStore"); + +const SchemaText = [ + ` + CREATE TABLE room_sha256 ( + room_id TEXT PRIMARY KEY NOT NULL, + sha256 TEXT NOT NULL + ) STRICT; + CREATE TABLE room_identification ( room_id TEXT PRIMARY KEY NOT NULL, creator TEXT NOT NULL, server TEXT NOT NULL, FOREIGN KEY (room_id) REFERENCES room_sha256(room_id) ) STRICT; -`, + `, ]; +const SchemaOptions = { + upgradeSteps: SchemaText.map( + (text) => + function (db) { + db.exec(text); + } + ), + consistencyCheck(db) { + return checkKnownTables(db, ["room_sha256", "room_identification"]); + }, +} satisfies SqliteSchemaOptions; + type RoomSha256 = { room_id: StringRoomID; sha256: string; @@ -47,17 +64,7 @@ export class SqliteHashReversalStore implements Omit { constructor(options: BetterSqliteOptions, db: Database) { - super( - schema.map( - (text) => - function (db) { - db.prepare(text).run(); - } - ), - options, - db - ); - this.ensureSchema(); + super(SchemaOptions, db, log); } public static createToplevel(storagePath: string): SqliteHashReversalStore { @@ -67,7 +74,10 @@ export class SqliteHashReversalStore foreignKeys: true, fileMustExist: false, }; - return new SqliteHashReversalStore(options, makeBetterSqliteDB(options)); + return new SqliteHashReversalStore( + options, + makeBetterSqliteDB(options, log) + ); } public async findRoomHash( diff --git a/src/backingstore/better-sqlite3/SqliteRoomStateBackingStore.ts b/src/backingstore/better-sqlite3/SqliteRoomStateBackingStore.ts index 20bd2e16..b1e1a32f 100644 --- a/src/backingstore/better-sqlite3/SqliteRoomStateBackingStore.ts +++ b/src/backingstore/better-sqlite3/SqliteRoomStateBackingStore.ts @@ -25,24 +25,48 @@ import { jsonReviver } from "../../utils"; import { StringRoomID } from "@the-draupnir-project/matrix-basic-types"; import path from "path"; import { Database } from "better-sqlite3"; +import { checkKnownTables, SqliteSchemaOptions } from "./SqliteSchema"; const log = new Logger("SqliteRoomStateBackingStore"); -const schema = [ - `CREATE TABLE room_info ( - room_id TEXT PRIMARY KEY NOT NULL, - last_complete_writeback INTEGER NOT NULL - ) STRICT;`, - `CREATE TABLE room_state_event ( - room_id TEXT NOT NULL, - event_type TEXT NOT NULL, - state_key TEXT NOT NULL, - event BLOB NOT NULL, - PRIMARY KEY (room_id, event_type, state_key), - FOREIGN KEY (room_id) REFERENCES room_info(room_id) - ) STRICT;`, +const SchemaText = [ + ` + CREATE TABLE room_info ( + room_id TEXT PRIMARY KEY NOT NULL, + last_complete_writeback INTEGER NOT NULL + ) STRICT, WITHOUT ROWID; + CREATE TABLE room_state_event ( + room_id TEXT NOT NULL, + event_type TEXT NOT NULL, + state_key TEXT NOT NULL, + event BLOB NOT NULL, + PRIMARY KEY (room_id, event_type, state_key), + FOREIGN KEY (room_id) REFERENCES room_info(room_id) + ) STRICT; + `, ]; +const SchemaOptions = { + upgradeSteps: SchemaText.map( + (text) => + function (db) { + db.exec(text); + } + ), + legacyUpgrade(db) { + // An older version of this store used a different schema management system + // fortunatley the data is just a psersistent cache so we don't need to worry about removing it. + db.exec(` + DELETE TABLE room_state_event; + DELETE TABLE room_info; + DELETE TABLE schema; + `); + }, + consistencyCheck(db) { + return checkKnownTables(db, ["room_info", "room_state_event"]); + }, +} satisfies SqliteSchemaOptions; + type RoomStateEventReplaceValue = [StringRoomID, string, string, string]; type RoomInfo = { last_complete_writeback: number; room_id: StringRoomID }; @@ -58,17 +82,7 @@ export class SqliteRoomStateBackingStore db: Database, private readonly eventDecoder: EventDecoder ) { - super( - schema.map( - (text) => - function (db) { - db.prepare(text).run(); - } - ), - options, - db - ); - this.ensureSchema(); + super(SchemaOptions, db, log); } public static create( @@ -83,7 +97,7 @@ export class SqliteRoomStateBackingStore }; return new SqliteRoomStateBackingStore( options, - makeBetterSqliteDB(options), + makeBetterSqliteDB(options, log), eventDecoder ); } diff --git a/src/backingstore/better-sqlite3/SqliteSchema.ts b/src/backingstore/better-sqlite3/SqliteSchema.ts new file mode 100644 index 00000000..37847476 --- /dev/null +++ b/src/backingstore/better-sqlite3/SqliteSchema.ts @@ -0,0 +1,102 @@ +// Copyright 2025 Bea <20361868+enbea@users.noreply.github.com> +// Copyright 2025 Gnuxie +// +// SPDX-License-Identifier: Apache-2.0 + +import { Database } from "better-sqlite3"; +import { Logger } from "matrix-protection-suite"; + +export type SqliteSchemaOptions = { + upgradeSteps: ((db: Database) => void)[]; + legacyUpgrade?: (db: Database) => void; + consistencyCheck?: (db: Database) => boolean; +}; + +export type SqliteSchemaMigrationResult = { + previousVersion: number; + version: number; +}; + +// NOTE: Must be run in a `db.transaction` to rollback on thrown errors +// and prevent partial upgrades. (allows user to revert to a last +// known working version of the software in case of errors) +function ensureSchemaPragma( + db: Database, + log: Logger, + { upgradeSteps, consistencyCheck, legacyUpgrade }: SqliteSchemaOptions +): SqliteSchemaMigrationResult { + // remove this check to allow empty (v0) databases + if (upgradeSteps.length === 0) { + throw new TypeError("Missing version definitions."); + } + + // https://www.sqlite.org/pragma.html#pragma_user_version + const previousVersion = db.pragma("user_version", { simple: true }) as number; + + if (previousVersion > upgradeSteps.length) { + throw new RangeError("Database is newer than defined `upgradeSteps`."); + } + + const tableCount = db + .prepare('SELECT count(*) FROM "sqlite_master";') + .pluck() + .get() as number; + if (previousVersion === 0 && tableCount > 0) { + // Database isn't empty, try to migrate it + log.info("Running legacy schema upgrade"); + if (legacyUpgrade) { + legacyUpgrade(db); + } else { + log.error("Incompatible database"); // log so we can tell which database this is. + throw new TypeError("Incompatible database"); + } + } + + // read the version again in case of legacy upgrade changing the number. + const startAt = db.pragma("user_version", { simple: true }) as number; + upgradeSteps.slice(startAt).forEach((step) => { + step(db); + }); + db.pragma(`user_version = ${upgradeSteps.length}`); + if (consistencyCheck && !consistencyCheck(db)) { + throw new Error("Database failed the consistency check."); + } + const version = db.pragma("user_version", { simple: true }) as number; + if (previousVersion !== version) { + log.info("Migrated database version from", previousVersion, "to", version); + } + return { + previousVersion, + version, + }; +} + +export function ensureSqliteSchema( + db: Database, + log: Logger, + options: SqliteSchemaOptions +): SqliteSchemaMigrationResult { + const migrationVersion = db.transaction(ensureSchemaPragma)(db, log, options); + db.exec("VACUUM;"); // why not + return migrationVersion; +} + +export function checkKnownTables( + db: Database, + unsortedKnownTables: string[] +): boolean { + const knownTables = unsortedKnownTables.slice().sort(), + currentTables = db + .prepare(`SELECT name FROM "sqlite_master" WHERE type = 'table';`) + .pluck() + .all() + .sort() + .filter((table: string) => !/^sqlite_/.test(table)); + if (knownTables.length !== currentTables.length) { + return false; + } + if (!knownTables.every((name, i) => currentTables[i] === name)) { + return false; + } + return true; +} diff --git a/src/protections/RoomTakedown/SqliteRoomAuditLog.ts b/src/protections/RoomTakedown/SqliteRoomAuditLog.ts index f3f2df2c..4c00f85d 100644 --- a/src/protections/RoomTakedown/SqliteRoomAuditLog.ts +++ b/src/protections/RoomTakedown/SqliteRoomAuditLog.ts @@ -12,6 +12,7 @@ import { ActionException, ActionExceptionKind, LiteralPolicyRule, + Logger, PolicyRuleType, RoomBasicDetails, } from "matrix-protection-suite"; @@ -22,6 +23,12 @@ import { } from "../../backingstore/better-sqlite3/BetterSqliteStore"; import { Database } from "better-sqlite3"; import path from "path"; +import { + checkKnownTables, + SqliteSchemaOptions, +} from "../../backingstore/better-sqlite3/SqliteSchema"; + +const log = new Logger("SqliteRoomAuditLog"); // NOTE: This should only be used to check in bulk whether rooms are taken down // upon getting a policy, you probably always want to try again or @@ -32,23 +39,24 @@ import path from "path"; // This will be important when the rule targets users and servers and // not just rooms. -const schema = [ - `CREATE TABLE policy_info ( - policy_id TEXT PRIMARY KEY NOT NULL, - sender_user_id TEXT NOT NULL, - room_id TEXT NOT NULL, - state_key TEXT NOT NULL, - type TEXT NOT NULL, - recommendation TEXT NOT NULL - ) STRICT;`, - `CREATE TABLE room_takedown ( +const SchemaText = [ + ` + CREATE TABLE policy_info ( + policy_id TEXT PRIMARY KEY NOT NULL, + sender_user_id TEXT NOT NULL, + room_id TEXT NOT NULL, + state_key TEXT NOT NULL, + type TEXT NOT NULL, + recommendation TEXT NOT NULL + ) STRICT; + CREATE TABLE room_takedown ( id INTEGER PRIMARY KEY AUTOINCREMENT, policy_id TEXT NOT NULL, target_room_id TEXT NOT NULL, created_at INTEGER DEFAULT (unixepoch()) NOT NULL, FOREIGN KEY (policy_id) REFERENCES policy_info(policy_id) - ) STRICT;`, - `CREATE TABLE room_detail_at_takedown ( + ) STRICT; + CREATE TABLE room_detail_at_takedown ( takedown_id INTEGER NOT NULL, room_id TEXT NOT NULL, creator TEXT, @@ -60,6 +68,22 @@ const schema = [ ) STRICT;`, ]; +const SchemaOptions = { + upgradeSteps: SchemaText.map( + (text) => + function (db) { + db.exec(text); + } + ), + consistencyCheck(db) { + return checkKnownTables(db, [ + "policy_info", + "room_takedown", + "room_detail_at_takedown", + ]); + }, +} satisfies SqliteSchemaOptions; + export type RoomTakedown = { id: number; policy_id: string; @@ -73,17 +97,7 @@ export class SqliteRoomAuditLog { private readonly takedownRooms: Set; public constructor(options: BetterSqliteOptions, db: Database) { - super( - schema.map( - (text) => - function (db) { - db.prepare(text).run(); - } - ), - options, - db - ); - this.ensureSchema(); + super(SchemaOptions, db, log); this.takedownRooms = new Set(this.loadTakendownRooms()); } @@ -94,7 +108,7 @@ export class SqliteRoomAuditLog foreignKeys: true, fileMustExist: false, }; - return new SqliteRoomAuditLog(options, makeBetterSqliteDB(options)); + return new SqliteRoomAuditLog(options, makeBetterSqliteDB(options, log)); } public async takedownRoom( From 6203171c71404a9f5ea366a13c0a7d88f4d341ae Mon Sep 17 00:00:00 2001 From: gnuxie Date: Wed, 19 Mar 2025 22:27:35 +0000 Subject: [PATCH 27/48] Fix delete room/all rooms in RoomStateBackingStore. Co-authored-by: Bea <20361868+enbea@users.noreply.github.com> --- .../SqliteRoomStateBackingStore.ts | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/src/backingstore/better-sqlite3/SqliteRoomStateBackingStore.ts b/src/backingstore/better-sqlite3/SqliteRoomStateBackingStore.ts index b1e1a32f..bb62472d 100644 --- a/src/backingstore/better-sqlite3/SqliteRoomStateBackingStore.ts +++ b/src/backingstore/better-sqlite3/SqliteRoomStateBackingStore.ts @@ -216,15 +216,15 @@ export class SqliteRoomStateBackingStore } public forgetRoom(roomID: StringRoomID): Promise> { - const deleteMetaStatement = this.db.prepare( - `DELETE FROM room_info WHERE room_id = ?` - ); const deleteStateStatement = this.db.prepare( `DELETE FROM room_state_event WHERE room_id = ?` ); + const deleteMetaStatement = this.db.prepare( + `DELETE FROM room_info WHERE room_id = ?` + ); const deleteRoom = this.db.transaction(() => { - deleteMetaStatement.run(roomID); deleteStateStatement.run(roomID); + deleteMetaStatement.run(roomID); // needs to be last to avoid violating foriegn key cnostraint }); try { deleteRoom(); @@ -240,12 +240,13 @@ export class SqliteRoomStateBackingStore } public async forgetAllRooms(): Promise> { - for (const roomID of this.roomInfoMap.keys()) { - const result = await this.forgetRoom(roomID); - if (isError(result)) { - return result; - } - } + this.db.transaction(() => { + this.db.exec(` + DELETE FROM room_state_event; + DELETE FROM room_info; + `); + this.roomInfoMap.clear(); + })(); return Ok(undefined); } } From a17cd517635a9a768d40ae641a7412fb66977be9 Mon Sep 17 00:00:00 2001 From: gnuxie Date: Wed, 19 Mar 2025 22:33:01 +0000 Subject: [PATCH 28/48] Clean up queries in hash store. Co-authored-by: Bea <20361868+enbea@users.noreply.github.com> --- src/backingstore/better-sqlite3/HashStore.ts | 36 +++++++++----------- 1 file changed, 16 insertions(+), 20 deletions(-) diff --git a/src/backingstore/better-sqlite3/HashStore.ts b/src/backingstore/better-sqlite3/HashStore.ts index 7d8c7407..717521f6 100644 --- a/src/backingstore/better-sqlite3/HashStore.ts +++ b/src/backingstore/better-sqlite3/HashStore.ts @@ -32,7 +32,8 @@ const SchemaText = [ CREATE TABLE room_sha256 ( room_id TEXT PRIMARY KEY NOT NULL, sha256 TEXT NOT NULL - ) STRICT; + ) STRICT, WITHOUT ROWID; + CREATE INDEX idx_sha256 ON room_sha256 (sha256, room_id); CREATE TABLE room_identification ( room_id TEXT PRIMARY KEY NOT NULL, creator TEXT NOT NULL, @@ -85,11 +86,10 @@ export class SqliteHashReversalStore ): Promise> { try { return Ok( - ( - this.db - .prepare(`SELECT room_id FROM room_sha256 WHERE sha256 = ?`) - .get(hash) as RoomSha256 | null - )?.room_id ?? undefined + this.db + .prepare(`SELECT room_id FROM room_sha256 WHERE sha256 = ?`) + .pluck() + .get(hash) as StringRoomID | undefined ); } catch (e) { if (e instanceof Error) { @@ -115,13 +115,12 @@ export class SqliteHashReversalStore sha256s.push(policy.hashes["sha256"]); } } - // This query looks sketch and I don't like it but we're just putting in question marks and nothing else - // I can't believe there's not a builtin for this fml. try { - const statement = this.db.prepare( - `SELECT room_id, sha256 FROM room_sha256 WHERE sha256 IN (${sha256s.map(() => "?").join(",")})` - ); - return Ok(statement.all(sha256s) as RoomSha256[]); + const statement = this.db.prepare(` + SELECT room_id, sha256 + FROM room_sha256 + WHERE sha256 IN (SELECT value FROM json_each(?))`); + return Ok(statement.all(JSON.stringify(sha256s)) as RoomSha256[]); } catch (exception) { if (exception instanceof Error) { return ActionException.Result( @@ -162,14 +161,11 @@ export class SqliteHashReversalStore } private findExistingRooms(roomIDs: StringRoomID[]): RoomHashRecord[] { - if (roomIDs.length === 0) { - return []; - } - const placeholders = roomIDs.map(() => "?").join(", "); - const statement = this.db.prepare( - `SELECT room_id, sha256 FROM room_sha256 WHERE room_id IN (${placeholders})` - ); - return statement.all(...roomIDs) as RoomHashRecord[]; + const statement = this.db.prepare(` + SELECT room_id, sha256 + FROM room_sha256 + WHERE room_id IN (SELECT value FROM json_each(?))`); + return statement.all(JSON.stringify(roomIDs)) as RoomHashRecord[]; } public async storeUndiscoveredRooms( From 522eae6ce26e5fb5413a94a303a1fdda161d38c1 Mon Sep 17 00:00:00 2001 From: gnuxie Date: Wed, 19 Mar 2025 23:35:22 +0000 Subject: [PATCH 29/48] Be explicit when returning ROWID after INSERT in audit log. Co-authored-by: Bea <20361868+enbea@users.noreply.github.com> --- src/protections/RoomTakedown/SqliteRoomAuditLog.ts | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/protections/RoomTakedown/SqliteRoomAuditLog.ts b/src/protections/RoomTakedown/SqliteRoomAuditLog.ts index 4c00f85d..4b12f263 100644 --- a/src/protections/RoomTakedown/SqliteRoomAuditLog.ts +++ b/src/protections/RoomTakedown/SqliteRoomAuditLog.ts @@ -125,8 +125,11 @@ export class SqliteRoomAuditLog `REPLACE INTO policy_info (policy_id, sender_user_id, room_id, state_key, type, recommendation) VALUES (?, ?, ?, ?, ?, ?)` ); - const takedownStatement = this.db.prepare(` - INSERT INTO room_takedown (policy_id, target_room_id) VALUES (?, ?)`); + const takedownStatement = this.db + .prepare( + `INSERT INTO room_takedown (policy_id, target_room_id) VALUES (?, ?) RETURNING id` + ) + .pluck(); const detailStatement = this.db.prepare(` INSERT INTO room_detail_at_takedown (takedown_id, room_id, creator, name, topic, joined_members) VALUES (?, ?, ?, ?, ?, ?)`); @@ -139,12 +142,12 @@ export class SqliteRoomAuditLog policy.sourceEvent.type, policy.recommendation, ]); - const takedownRow = takedownStatement.run([ + const takedownID = takedownStatement.get([ policy.sourceEvent.event_id, policy.entity, - ]); + ]) as number; detailStatement.run([ - takedownRow.lastInsertRowid, + takedownID, details.room_id, details.creator ?? null, details.name ?? null, From b5e3ebb53aaf36cbf06bdb569215c689ee82fc40 Mon Sep 17 00:00:00 2001 From: gnuxie Date: Thu, 20 Mar 2025 11:43:52 +0000 Subject: [PATCH 30/48] Explicitly choose the setting of temp_store in better sqlite. Memory isn't the default. Co-authored-by: Bea <20361868+enbea@users.noreply.github.com> --- src/backingstore/better-sqlite3/BetterSqliteStore.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/backingstore/better-sqlite3/BetterSqliteStore.ts b/src/backingstore/better-sqlite3/BetterSqliteStore.ts index b9b07663..fe93f63d 100644 --- a/src/backingstore/better-sqlite3/BetterSqliteStore.ts +++ b/src/backingstore/better-sqlite3/BetterSqliteStore.ts @@ -24,7 +24,9 @@ export function makeBetterSqliteDB( ): Database { log.info("Opening db: ", options.path); const db = new BetterSqlite3(options.path, options); - if (options.path !== ":memory:") { + if (options.path === ":memory:") { + db.pragma("temp_store = memory"); + } else { db.pragma("temp_store = file"); // Avoid unnecessary memory usage. } if (options.WALMode) { From 862ed5c5003b913b2b68e0fa360c8ac7dbdd0e6e Mon Sep 17 00:00:00 2001 From: gnuxie Date: Thu, 20 Mar 2025 11:51:36 +0000 Subject: [PATCH 31/48] Remove duplicated `hasEnded` variables in BetterSqliteStore. `db.open` has the same purpose. Co-authored-by: Bea <20361868+enbea@users.noreply.github.com> --- .../better-sqlite3/BetterSqliteStore.ts | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/src/backingstore/better-sqlite3/BetterSqliteStore.ts b/src/backingstore/better-sqlite3/BetterSqliteStore.ts index fe93f63d..b8148d25 100644 --- a/src/backingstore/better-sqlite3/BetterSqliteStore.ts +++ b/src/backingstore/better-sqlite3/BetterSqliteStore.ts @@ -1,4 +1,4 @@ -// Copyright 2024 Gnuxie +// Copyright 2024 - 2025 Gnuxie // Copyright 2022 The Matrix.org Foundation C.I.C. // // SPDX-License-Identifier: Apache-2.0 @@ -35,16 +35,14 @@ export function makeBetterSqliteDB( if (options.foreignKeys) { db.pragma("foreign_keys = ON"); } - let hasEnded = false; process.once("beforeExit", () => { // Ensure we clean up on exit try { log.info("Destroy called on db", options.path); - if (hasEnded) { + if (db.open) { // No-op if end has already been called. return; } - hasEnded = true; db.close(); log.info("connection ended on db", options.path); } catch (ex) { @@ -58,8 +56,7 @@ export type SchemaUpdateFunction = (db: Database) => void; /** * BetterSqliteStore datastore abstraction which can be inherited by a specialised bridge class. - * Please note, that the client library provides synchronous access to sqlite, due to the nature of - * node.js FFI to C I imagine. + * Please note, that the client library provides synchronous access to sqlite. * * @example * class MyBridgeStore extends BetterSqliteStore { @@ -78,8 +75,6 @@ export type SchemaUpdateFunction = (db: Database) => void; * const data = await store.getData(); */ export class BetterSqliteStore { - private hasEnded = false; - /** * Construct a new store. * @param schemas The set of schema functions to apply to a database. The ordering of this array determines the @@ -95,16 +90,14 @@ export class BetterSqliteStore { } /** - * Clean away any resources used by the database. This is automatically - * called before the process exits. + * Clean away any resources used by the database */ public destroy(): void { this.log.info("Destroy called"); - if (this.hasEnded) { + if (this.db.open) { // No-op if end has already been called. return; } - this.hasEnded = true; this.db.close(); this.log.info("connection ended"); } From f92d18662cebece724beff7a1af3ba84063b1910 Mon Sep 17 00:00:00 2001 From: gnuxie Date: Thu, 20 Mar 2025 12:03:39 +0000 Subject: [PATCH 32/48] getTakedownDetails query improvements. Co-authored-by: Bea <20361868+enbea@users.noreply.github.com> --- .../RoomTakedown/SqliteRoomAuditLog.ts | 27 ++++++++----------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/src/protections/RoomTakedown/SqliteRoomAuditLog.ts b/src/protections/RoomTakedown/SqliteRoomAuditLog.ts index 4b12f263..a8a682d4 100644 --- a/src/protections/RoomTakedown/SqliteRoomAuditLog.ts +++ b/src/protections/RoomTakedown/SqliteRoomAuditLog.ts @@ -173,27 +173,22 @@ export class SqliteRoomAuditLog roomID: StringRoomID ): Promise> { try { - const query = this.db.prepare(` - SELECT - room_takedown.policy_id, - room_takedown.created_at, - room_detail_at_takedown.room_id, - room_detail_at_takedown.creator, - room_detail_at_takedown.name, - room_detail_at_takedown.topic, - room_detail_at_takedown.joined_members - FROM room_takedown - JOIN room_detail_at_takedown ON room_takedown.id = room_detail_at_takedown.takedown_id - WHERE room_takedown.target_room_id = ? - ORDER BY room_takedown.created_at DESC - LIMIT 1 - `); type RowType = { policy_id: StringEventID; created_at: number; room_id: StringEventID; } & RoomTakedownDetails; - const row = query.get(roomID) as RowType | undefined; + const row = this.db + .prepare( + ` + SELECT room_takedown.policy_id, room_takedown.created_at, room_detail_at_takedown.* + FROM room_takedown + JOIN room_detail_at_takedown ON room_takedown.id = room_detail_at_takedown.takedown_id + WHERE room_takedown.target_room_id = ? + ORDER BY room_takedown.created_at DESC + LIMIT 1` + ) + .get(roomID) as RowType | undefined; if (row === undefined) { return Ok(undefined); } From fa3bce12d6052c390a9a95b0b070c858984e1807 Mon Sep 17 00:00:00 2001 From: gnuxie Date: Thu, 20 Mar 2025 12:25:30 +0000 Subject: [PATCH 33/48] Include policy entity in the takedown audit log policy info. --- src/protections/RoomTakedown/SqliteRoomAuditLog.ts | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/protections/RoomTakedown/SqliteRoomAuditLog.ts b/src/protections/RoomTakedown/SqliteRoomAuditLog.ts index a8a682d4..7b8a12e1 100644 --- a/src/protections/RoomTakedown/SqliteRoomAuditLog.ts +++ b/src/protections/RoomTakedown/SqliteRoomAuditLog.ts @@ -35,16 +35,13 @@ const log = new Logger("SqliteRoomAuditLog"); // check with the server rather than check the audit log. // Incase rooms were unbanned. -// FIXME: The entity should really be included in the policy info mare... -// This will be important when the rule targets users and servers and -// not just rooms. - const SchemaText = [ ` CREATE TABLE policy_info ( policy_id TEXT PRIMARY KEY NOT NULL, sender_user_id TEXT NOT NULL, - room_id TEXT NOT NULL, + entity TEXT NOT NULL, + policy_room_id TEXT NOT NULL, state_key TEXT NOT NULL, type TEXT NOT NULL, recommendation TEXT NOT NULL @@ -122,8 +119,8 @@ export class SqliteRoomAuditLog } try { const policyInfoStatement = this.db.prepare( - `REPLACE INTO policy_info (policy_id, sender_user_id, room_id, state_key, type, recommendation) - VALUES (?, ?, ?, ?, ?, ?)` + `REPLACE INTO policy_info (policy_id, sender_user_id, policy_room_id, entity, state_key, type, recommendation) + VALUES (?, ?, ?, ?, ?, ?, ?)` ); const takedownStatement = this.db .prepare( @@ -138,6 +135,7 @@ export class SqliteRoomAuditLog policy.sourceEvent.event_id, policy.sourceEvent.sender, policy.sourceEvent.room_id, + policy.entity, policy.sourceEvent.state_key, policy.sourceEvent.type, policy.recommendation, From 1d0feb724af940053b90609fc1da6c0e043fd4d8 Mon Sep 17 00:00:00 2001 From: gnuxie Date: Thu, 20 Mar 2025 17:56:34 +0000 Subject: [PATCH 34/48] Update Draupnir for new PolicyListRevision API. We should probably have just accepted multiple recommendations in all these methods. --- src/commands/Rules.tsx | 5 +- src/commands/unban/UnbanEntity.tsx | 11 ++-- src/commands/unban/UnbanUsers.tsx | 71 ++++++++++++++------- src/protections/BanPropagation.tsx | 11 ++-- src/protections/RedactionSynchronisation.ts | 16 +++-- test/integration/banPropagationTest.ts | 11 ++-- 6 files changed, 82 insertions(+), 43 deletions(-) diff --git a/src/commands/Rules.tsx b/src/commands/Rules.tsx index fe9006e3..4b0df6dc 100644 --- a/src/commands/Rules.tsx +++ b/src/commands/Rules.tsx @@ -158,7 +158,10 @@ export const DraupnirRulesMatchingCommand = describeCommand({ return { room: profile.revision.room, roomID: profile.revision.room.toRoomIDOrAlias(), - matches: profile.revision.allRulesMatchingEntity(entity.toString()), + matches: profile.revision.allRulesMatchingEntity( + entity.toString(), + {} + ), profile: profile, }; }) diff --git a/src/commands/unban/UnbanEntity.tsx b/src/commands/unban/UnbanEntity.tsx index 3ee0105f..d6ec8318 100644 --- a/src/commands/unban/UnbanEntity.tsx +++ b/src/commands/unban/UnbanEntity.tsx @@ -36,10 +36,13 @@ export function findPoliciesToRemove( room: profile.room, roomID: profile.room.toRoomIDOrAlias(), profile, - matches: profile.revision.allRulesMatchingEntity( - entity.toString(), - entityType, - Recommendation.Ban + matches: [Recommendation.Ban, Recommendation.Takedown].flatMap( + (recommendation) => + profile.revision.allRulesMatchingEntity(entity.toString(), { + type: entityType, + searchHashedRules: true, + recommendation, + }) ), }); } diff --git a/src/commands/unban/UnbanUsers.tsx b/src/commands/unban/UnbanUsers.tsx index 425b4afa..6778a152 100644 --- a/src/commands/unban/UnbanUsers.tsx +++ b/src/commands/unban/UnbanUsers.tsx @@ -28,6 +28,7 @@ import { Ok, PolicyRuleType, PolicyRuleMatchType, + PolicyListRevision, } from "matrix-protection-suite"; import { UnlistedUserRedactionQueue } from "../../queues/UnlistedUserRedactionQueue"; import { ListMatches } from "../Rules"; @@ -91,6 +92,42 @@ export function matchMembers( return [...map.values()]; } +export function revisionRulesMatchingEntity( + entity: string, + ruleType: PolicyRuleType, + recommendations: Recommendation[], + revision: PolicyListRevision +): PolicyRule[] { + return recommendations.flatMap((recommendation) => + revision.allRulesMatchingEntity(entity, { + type: ruleType, + recommendation, + searchHashedRules: true, + }) + ); +} + +export function revisionRulesMatchingUser( + userID: StringUserID, + recommendations: Recommendation[], + revision: PolicyListRevision +): PolicyRule[] { + return [ + ...revisionRulesMatchingEntity( + userID, + PolicyRuleType.User, + recommendations, + revision + ), + ...revisionRulesMatchingEntity( + userServerName(userID), + PolicyRuleType.Server, + recommendations, + revision + ), + ]; +} + export function findBanPoliciesMatchingUsers( watchedPolicyRooms: WatchedPolicyRooms, users: StringUserID[] @@ -105,18 +142,11 @@ export function findBanPoliciesMatchingUsers( } }; for (const user of users) { - const memberPolicies = [ - ...watchedPolicyRooms.currentRevision.allRulesMatchingEntity( - user, - PolicyRuleType.User, - Recommendation.Ban - ), - ...watchedPolicyRooms.currentRevision.allRulesMatchingEntity( - userServerName(user), - PolicyRuleType.Server, - Recommendation.Ban - ), - ]; + const memberPolicies = revisionRulesMatchingUser( + user, + [Recommendation.Ban, Recommendation.Takedown], + watchedPolicyRooms.currentRevision + ); for (const policy of memberPolicies) { addPolicy(policy); } @@ -258,18 +288,11 @@ export function findUnbanInformationForMember( { inviteMembers } ); const isMemberStillBannedAfterPolicyRemoval = (member: MemberRooms) => { - const policiesMatchingMember = [ - ...watchedPolicyRooms.currentRevision.allRulesMatchingEntity( - member.member, - PolicyRuleType.User, - Recommendation.Ban - ), - ...watchedPolicyRooms.currentRevision.allRulesMatchingEntity( - userServerName(member.member), - PolicyRuleType.Server, - Recommendation.Ban - ), - ]; + const policiesMatchingMember = revisionRulesMatchingUser( + member.member, + [Recommendation.Takedown, Recommendation.Ban], + watchedPolicyRooms.currentRevision + ); return ( policiesMatchingMember.filter( (policy) => diff --git a/src/protections/BanPropagation.tsx b/src/protections/BanPropagation.tsx index fa35ee6a..88823f68 100644 --- a/src/protections/BanPropagation.tsx +++ b/src/protections/BanPropagation.tsx @@ -50,7 +50,10 @@ import { UnbanMembersPreview, renderUnbanMembersPreview, } from "../commands/unban/Unban"; -import { findUnbanInformationForMember } from "../commands/unban/UnbanUsers"; +import { + findUnbanInformationForMember, + revisionRulesMatchingUser, +} from "../commands/unban/UnbanUsers"; const log = new Logger("BanPropagationProtection"); @@ -254,10 +257,10 @@ export class BanPropagationProtection private handleBan(change: MembershipChange): void { const policyRevision = this.protectedRoomsSet.watchedPolicyRooms.currentRevision; - const rulesMatchingUser = policyRevision.allRulesMatchingEntity( + const rulesMatchingUser = revisionRulesMatchingUser( change.userID, - PolicyRuleType.User, - Recommendation.Ban + [Recommendation.Ban, Recommendation.Takedown], + policyRevision ); if (rulesMatchingUser.length > 0) { return; // user is already banned. diff --git a/src/protections/RedactionSynchronisation.ts b/src/protections/RedactionSynchronisation.ts index c8d48992..d8fb4580 100644 --- a/src/protections/RedactionSynchronisation.ts +++ b/src/protections/RedactionSynchronisation.ts @@ -129,11 +129,17 @@ export class RedactionSynchronisationProtection ) { const policyRevision = this.protectedRoomsSet.watchedPolicyRooms.currentRevision; - const matchingPolicy = policyRevision.findRuleMatchingEntity( - change.userID, - PolicyRuleType.User, - Recommendation.Ban - ); + const matchingPolicy = + policyRevision.findRuleMatchingEntity(change.userID, { + type: PolicyRuleType.User, + recommendation: Recommendation.Ban, + searchHashedRules: false, + }) ?? + policyRevision.findRuleMatchingEntity(change.userID, { + type: PolicyRuleType.User, + recommendation: Recommendation.Takedown, + searchHashedRules: false, + }); return ( matchingPolicy !== undefined && this.automaticRedactionReasons.some((reason) => diff --git a/test/integration/banPropagationTest.ts b/test/integration/banPropagationTest.ts index cb43fe31..66462ea9 100644 --- a/test/integration/banPropagationTest.ts +++ b/test/integration/banPropagationTest.ts @@ -120,7 +120,9 @@ describe("Ban propagation test", function () { draupnir.protectedRoomsSet.watchedPolicyRooms.currentRevision; const rules = policyListRevisionAfterBan.allRulesMatchingEntity( spamUserID, - PolicyRuleType.User + { + type: PolicyRuleType.User, + } ); expect(rules.length).toBe(1); expect((rules[0] as LiteralPolicyRule).entity).toBe(spamUserID); @@ -154,10 +156,9 @@ describe("Ban propagation test", function () { draupnir.protectedRoomsSet.watchedPolicyRooms.currentRevision; const rulesAfterUnban = - policyListRevisionAfterUnBan.allRulesMatchingEntity( - spamUserID, - PolicyRuleType.User - ); + policyListRevisionAfterUnBan.allRulesMatchingEntity(spamUserID, { + type: PolicyRuleType.User, + }); expect(rulesAfterUnban.length).toBe(0); for (const room of protectedRooms) { const membershipRevision = From 15c1bf23a9f0d6e05aa4d32f913f8011b0ae3089 Mon Sep 17 00:00:00 2001 From: gnuxie Date: Fri, 21 Mar 2025 16:11:57 +0000 Subject: [PATCH 35/48] Rework hash store to support storing details about all entities. --- src/backingstore/DraupnirStores.ts | 14 +- src/backingstore/better-sqlite3/HashStore.ts | 365 +++++++++++++------ src/commands/Takedown.tsx | 6 +- test/unit/stores/hashStoreTest.ts | 156 +++++++- 4 files changed, 416 insertions(+), 125 deletions(-) diff --git a/src/backingstore/DraupnirStores.ts b/src/backingstore/DraupnirStores.ts index 5863553a..d2a97feb 100644 --- a/src/backingstore/DraupnirStores.ts +++ b/src/backingstore/DraupnirStores.ts @@ -2,18 +2,14 @@ // // SPDX-License-Identifier: Apache-2.0 -import { - EventDecoder, - SHA256RoomHashStore, - StandardSHA256RoomHashStore, -} from "matrix-protection-suite"; +import { EventDecoder, SHA256HashStore } from "matrix-protection-suite"; import { RoomAuditLog } from "../protections/RoomTakedown/RoomAuditLog"; import { SqliteRoomStateBackingStore } from "./better-sqlite3/SqliteRoomStateBackingStore"; import { SqliteHashReversalStore } from "./better-sqlite3/HashStore"; import { SqliteRoomAuditLog } from "../protections/RoomTakedown/SqliteRoomAuditLog"; export type TopLevelStores = { - hashStore?: Omit | undefined; + hashStore?: SHA256HashStore; roomAuditLog?: RoomAuditLog | undefined; roomStateBackingStore?: SqliteRoomStateBackingStore | undefined; }; @@ -29,7 +25,7 @@ export type TopLevelStores = { * have been attenuated... but i don't know about it. */ export type DraupnirStores = { - hashStore?: SHA256RoomHashStore | undefined; + hashStore?: SHA256HashStore | undefined; roomAuditLog?: RoomAuditLog | undefined; /** * Dispose of stores relevant to a specific draupnir instance. @@ -43,9 +39,7 @@ export function createDraupnirStores( ): DraupnirStores { return Object.freeze({ roomAuditLog: topLevelStores.roomAuditLog, - hashStore: topLevelStores.hashStore - ? new StandardSHA256RoomHashStore(topLevelStores.hashStore) - : undefined, + hashStore: topLevelStores.hashStore, dispose() {}, } satisfies DraupnirStores); } diff --git a/src/backingstore/better-sqlite3/HashStore.ts b/src/backingstore/better-sqlite3/HashStore.ts index 717521f6..c31d5e54 100644 --- a/src/backingstore/better-sqlite3/HashStore.ts +++ b/src/backingstore/better-sqlite3/HashStore.ts @@ -1,9 +1,20 @@ // SPDX-FileCopyrightText: 2025 Gnuxie // // SPDX-License-Identifier: Apache-2.0 +// +// SPDX-FileAttributionText: +// This modified file incorporates work from Draupnir +// https://github.com/the-draupnir-project/Draupnir +// import { isError, Ok, Result } from "@gnuxie/typescript-result"; -import { StringRoomID } from "@the-draupnir-project/matrix-basic-types"; +import { + roomIDServerName, + StringRoomID, + StringServerName, + StringUserID, + userServerName, +} from "@the-draupnir-project/matrix-basic-types"; import { ActionException, ActionExceptionKind, @@ -12,34 +23,46 @@ import { makeReversedHashedPolicy, RoomHashRecord, HashedRoomDetails, - SHA256RoomHashStore, + SHA256HashStore, Logger, + UserHashRecord, + PolicyRuleType, } from "matrix-protection-suite"; -import { - BetterSqliteOptions, - BetterSqliteStore, - makeBetterSqliteDB, -} from "./BetterSqliteStore"; -import { enc, SHA256 } from "crypto-js"; -import { Database } from "better-sqlite3"; +import { BetterSqliteStore, makeBetterSqliteDB } from "./BetterSqliteStore"; +import { Database, Statement } from "better-sqlite3"; import path from "path"; import { checkKnownTables, SqliteSchemaOptions } from "./SqliteSchema"; +import EventEmitter from "events"; +import { createHash } from "crypto"; const log = new Logger("SqliteHashReversalStore"); +// The reason why room is split into room_identification is because the creator +// is usually not readily available when discovering rooms from the anti-spam apis. const SchemaText = [ ` CREATE TABLE room_sha256 ( room_id TEXT PRIMARY KEY NOT NULL, sha256 TEXT NOT NULL ) STRICT, WITHOUT ROWID; - CREATE INDEX idx_sha256 ON room_sha256 (sha256, room_id); + CREATE INDEX idx_room_sha256 ON room_sha256 (sha256, room_id); CREATE TABLE room_identification ( room_id TEXT PRIMARY KEY NOT NULL, creator TEXT NOT NULL, server TEXT NOT NULL, FOREIGN KEY (room_id) REFERENCES room_sha256(room_id) ) STRICT; + CREATE TABLE user_sha256 ( + user_id TEXT PRIMARY KEY NOT NULL, + server_name TEXT NOT NULL, + sha256 TEXT NOT NULL + ) STRICT, WITHOUT ROWID; + CREATE INDEX idx_user_sha256 ON user_sha256 (sha256, user_id); + CREATE TABLE server_sha256 ( + server_name TEXT PRIMARY KEY NOT NULL, + sha256 TEXT NOT NULL + ) STRICT, WITHOUT ROWID; + CREATE INDEX idx_server_sha256 ON server_sha256 (sha256, server_name); `, ]; @@ -51,21 +74,66 @@ const SchemaOptions = { } ), consistencyCheck(db) { - return checkKnownTables(db, ["room_sha256", "room_identification"]); + return checkKnownTables(db, [ + "room_sha256", + "room_identification", + "user_sha256", + "server_sha256", + ]); }, } satisfies SqliteSchemaOptions; -type RoomSha256 = { - room_id: StringRoomID; - sha256: string; -}; +function wrapInTryCatch(cb: () => Result, message: string): Result { + try { + return cb(); + } catch (e) { + if (e instanceof Error) { + return ActionException.Result(message, { + exception: e, + exceptionKind: ActionExceptionKind.Unknown, + }); + } else { + throw e; + } + } +} export class SqliteHashReversalStore - extends BetterSqliteStore - implements Omit + extends EventEmitter + implements SHA256HashStore { - constructor(options: BetterSqliteOptions, db: Database) { - super(SchemaOptions, db, log); + private readonly baseStore: BetterSqliteStore; + constructor(private readonly db: Database) { + super(); + this.baseStore = new BetterSqliteStore(SchemaOptions, db, log); + } + public async findUserHash( + hash: string + ): Promise> { + return wrapInTryCatch( + () => + Ok( + this.db + .prepare(`SELECT user_id FROM user_sha256 WHERE sha256 = ?`) + .pluck() + .get(hash) as StringUserID | undefined + ), + "error while querying hash for a user" + ); + } + public async findServerHash( + hash: string + ): Promise> { + return wrapInTryCatch( + () => + Ok( + this.db + .prepare(`SELECT server_name FROM server_sha256 WHERE sha256 = ?`) + .pluck() + .get(hash) as string | undefined + ), + "error while querying hash for a server name" + ); } public static createToplevel(storagePath: string): SqliteHashReversalStore { @@ -75,10 +143,7 @@ export class SqliteHashReversalStore foreignKeys: true, fileMustExist: false, }; - return new SqliteHashReversalStore( - options, - makeBetterSqliteDB(options, log) - ); + return new SqliteHashReversalStore(makeBetterSqliteDB(options, log)); } public async findRoomHash( @@ -106,108 +171,186 @@ export class SqliteHashReversalStore } } - private async findMatchingRooms( - policies: HashedLiteralPolicyRule[] - ): Promise> { - const sha256s: string[] = []; + private reversePolicies( + policies: HashedLiteralPolicyRule[], + selectEntityFromSha256: Statement + ): LiteralPolicyRule[] { + const reversedPolicies: LiteralPolicyRule[] = []; for (const policy of policies) { - if (policy.hashes["sha256"] !== undefined) { - sha256s.push(policy.hashes["sha256"]); + const sha256 = policy.hashes["sha256"]; + if (sha256 === undefined) { + continue; } - } - try { - const statement = this.db.prepare(` - SELECT room_id, sha256 - FROM room_sha256 - WHERE sha256 IN (SELECT value FROM json_each(?))`); - return Ok(statement.all(JSON.stringify(sha256s)) as RoomSha256[]); - } catch (exception) { - if (exception instanceof Error) { - return ActionException.Result( - "Unexpected error while attempting to reverse hashed policies", - { - exception, - exceptionKind: ActionExceptionKind.Unknown, - } - ); - } else { - throw exception; + const entity = selectEntityFromSha256.get(sha256) as string | undefined; + if (entity === undefined) { + continue; } + reversedPolicies.push(makeReversedHashedPolicy(entity, policy)); } + return reversedPolicies; } - public async reverseHashedRoomPolicies( + private reverseRoomPolicies( + policies: HashedLiteralPolicyRule[] + ): LiteralPolicyRule[] { + return this.reversePolicies( + policies, + this.db + .prepare( + ` + SELECT room_id + FROM room_sha256 + WHERE sha256 = ?` + ) + .pluck() + ); + } + + private reverseUserPolicies( + policies: HashedLiteralPolicyRule[] + ): LiteralPolicyRule[] { + return this.reversePolicies( + policies, + this.db + .prepare( + ` + SELECT user_id + FROM user_sha256 + WHERE sha256 = ?` + ) + .pluck() + ); + } + + private reverseServerPolicies( + policies: HashedLiteralPolicyRule[] + ): LiteralPolicyRule[] { + return this.reversePolicies( + policies, + this.db + .prepare( + ` + SELECT server_name + FROM server_sha256 + WHERE sha256 = ?` + ) + .pluck() + ); + } + + public async reverseHashedPolicies( policies: HashedLiteralPolicyRule[] ): Promise> { - const matches = await this.findMatchingRooms(policies); - if (isError(matches)) { - return matches; - } - const reversedPolicies: LiteralPolicyRule[] = []; - for (const match of matches.ok) { - const hashedLiteral = policies.find( - (policy) => policy.hashes["sha256"] === match.sha256 - ); - if (hashedLiteral == undefined) { - throw new TypeError( - "Something has gone badly wrong we should be able to find the policy we entered" - ); - } - reversedPolicies.push( - makeReversedHashedPolicy(match.room_id, hashedLiteral) - ); - } - return Ok(reversedPolicies); + const hashedRoomPolicies = policies.filter( + (policy) => policy.kind === PolicyRuleType.Room + ); + const hashedUserPolicies = policies.filter( + (policy) => policy.kind === PolicyRuleType.User + ); + const hashedServerPolicies = policies.filter( + (policy) => policy.kind === PolicyRuleType.Server + ); + return wrapInTryCatch(() => { + return Ok([ + ...this.reverseRoomPolicies(hashedRoomPolicies), + ...this.reverseUserPolicies(hashedUserPolicies), + ...this.reverseServerPolicies(hashedServerPolicies), + ]); + }, "Error trying to reverse hashed policies"); } - private findExistingRooms(roomIDs: StringRoomID[]): RoomHashRecord[] { - const statement = this.db.prepare(` - SELECT room_id, sha256 - FROM room_sha256 - WHERE room_id IN (SELECT value FROM json_each(?))`); - return statement.all(JSON.stringify(roomIDs)) as RoomHashRecord[]; + private storeUndiscoveredEntities( + entities: string[], + filterStatement: Statement, + insertOrIgnore: Statement, + recordFactoryFn: (entity: string, hash: string) => Record, + eventEmitFn: (newRecords: Record[]) => void, + serverExtractFn?: (record: Record) => StringServerName + ): Result { + return wrapInTryCatch(() => { + const undiscoveredEntities = entities.filter((entity) => + filterStatement.get(entity) + ); + const rowsToInsert: Record[] = undiscoveredEntities.map((entity) => + recordFactoryFn( + entity, + createHash("sha256").update(entity, "utf8").digest("base64") + ) + ); + insertOrIgnore.run(JSON.stringify(rowsToInsert)); + // try to discover servers too while we're here + if (serverExtractFn) { + const storeResult = this.storeUndiscoveredEntities( + rowsToInsert.map(serverExtractFn), + this.db + .prepare( + `SELECT NOT EXISTS (SELECT 1 FROM server_sha256 where server_name = ?)` + ) + .pluck(), + this.db.prepare(` + INSERT OR IGNORE INTO server_sha256 (server_name, sha256) + SELECT value ->> 'server_name', value ->> 'sha256' + FROM json_each(?)`), + (server_name: StringServerName, sha256) => ({ server_name, sha256 }), + (serverRecords) => { + this.emit("ReversedHashes", [], [], serverRecords); + } + ); + if (isError(storeResult)) { + return storeResult.elaborate("Failed to update discovered servers"); + } + } + eventEmitFn(rowsToInsert); + return Ok(rowsToInsert); + }, "Failed to insert undiscovered entities into hash store"); } public async storeUndiscoveredRooms( roomIDs: StringRoomID[] ): Promise> { - try { - const existingRooms = this.findExistingRooms(roomIDs); - const rowsToInsert: RoomHashRecord[] = []; - for (const roomID of roomIDs) { - if (!existingRooms.find((row) => row.room_id === roomID)) { - rowsToInsert.push({ - room_id: roomID, - sha256: enc.Base64.stringify(SHA256(roomID)), - }); - } - } + return this.storeUndiscoveredEntities( + roomIDs, + this.db + .prepare( + `SELECT NOT EXISTS (SELECT 1 FROM room_sha256 where room_id = ?)` + ) + .pluck(), + this.db.prepare(` + INSERT OR IGNORE INTO room_sha256 (room_id, sha256) + SELECT value ->> 'room_id', value ->> 'sha256' + FROM json_each(?)`), + (room_id: StringRoomID, sha256) => ({ room_id, sha256 }), + (roomRecords) => { + this.emit("ReversedHashes", roomRecords, [], []); + }, + (roomRecord) => roomIDServerName(roomRecord.room_id) + ); + } - if (rowsToInsert.length > 0) { - const serializedRows = JSON.stringify(rowsToInsert); - - const statement = this.db.prepare(` - REPLACE INTO room_sha256 - SELECT value ->> 'room_id', value ->> 'sha256' - FROM json_each(?) - `); - - statement.run(serializedRows); - } - return Ok(rowsToInsert); - } catch (exception) { - if (exception instanceof Error) { - return ActionException.Result( - `Error while trying to update known hashed rooms`, - { - exception, - exceptionKind: ActionExceptionKind.Unknown, - } - ); - } else { - throw exception; - } - } + public async storeUndiscoveredUsers( + userIDs: StringUserID[] + ): Promise> { + return this.storeUndiscoveredEntities( + userIDs, + this.db + .prepare( + `SELECT NOT EXISTS (SELECT 1 FROM user_sha256 WHERE user_id = ?)` + ) + .pluck(), + this.db.prepare(` + INSERT OR IGNORE INTO user_sha256 (user_id, server_name, sha256) + SELECT value ->> 'user_id', value ->> 'server_name', value ->> 'sha256' + FROM json_each(?)`), + (user_id: StringUserID, sha256) => ({ + user_id, + server_name: userServerName(user_id), + sha256, + }), + (userRecords) => { + this.emit("ReversedHashes", [], userRecords, []); + }, + (userRecord) => userRecord.server_name + ); } public async storeRoomIdentification( @@ -238,4 +381,8 @@ export class SqliteHashReversalStore } } } + + public destroy(): void { + this.baseStore.destroy(); + } } diff --git a/src/commands/Takedown.tsx b/src/commands/Takedown.tsx index c85d9482..3f6c9ef8 100644 --- a/src/commands/Takedown.tsx +++ b/src/commands/Takedown.tsx @@ -22,7 +22,7 @@ import { PolicyRoomManager, PolicyRuleType, RoomResolver, - SHA256RoomHashStore, + SHA256HashStore, Task, WatchedPolicyRooms, } from "matrix-protection-suite"; @@ -50,7 +50,7 @@ const log = new Logger("DraupnirTakedownCommand"); */ function handleRoomDiscovery( roomID: StringRoomID, - store: SHA256RoomHashStore | undefined, + store: SHA256HashStore | undefined, detailsProvider: RoomDetailsProvider | undefined ): void { if (store === undefined) { @@ -268,7 +268,7 @@ export type DraupnirTakedownCommandContext = { defaultReasons: string[]; roomResolver: RoomResolver; clientUserID: StringUserID; - hashStore: SHA256RoomHashStore | undefined; + hashStore: SHA256HashStore | undefined; detailsProvider: RoomDetailsProvider | undefined; }; diff --git a/test/unit/stores/hashStoreTest.ts b/test/unit/stores/hashStoreTest.ts index 7628babd..0eb2a4e6 100644 --- a/test/unit/stores/hashStoreTest.ts +++ b/test/unit/stores/hashStoreTest.ts @@ -17,12 +17,22 @@ import { } from "matrix-protection-suite"; import { SHA256, enc } from "crypto-js"; import expect from "expect"; +import { + StringRoomID, + StringServerName, + StringUserID, +} from "@the-draupnir-project/matrix-basic-types"; +import { createHash } from "crypto"; function makeStore(): SqliteHashReversalStore { const options = { path: ":memory:" } satisfies BetterSqliteOptions; const db = new Database(options.path); db.pragma("FOREIGN_KEYS = ON"); - return new SqliteHashReversalStore(options, db); + return new SqliteHashReversalStore(db); +} + +function base64sha256(entity: string): string { + return createHash("sha256").update(entity, "utf8").digest("base64"); } describe("meow", function () { @@ -54,7 +64,7 @@ describe("meow", function () { ).expect("Should bea ble to parse the policy rule."); expect(ban.matchType).toBe(PolicyRuleMatchType.HashedLiteral); const reverserResult = ( - await store.reverseHashedRoomPolicies([ban as HashedLiteralPolicyRule]) + await store.reverseHashedPolicies([ban as HashedLiteralPolicyRule]) ).expect("Should be able to reverse policies that are unknown"); expect(reverserResult.length).toBe(0); // now try storing the room and testing that the policy is there @@ -66,7 +76,7 @@ describe("meow", function () { ); expect(foundHash).toBe(bannedRoom.toRoomIDOrAlias()); const reversedPolicies = ( - await store.reverseHashedRoomPolicies([ban as HashedLiteralPolicyRule]) + await store.reverseHashedPolicies([ban as HashedLiteralPolicyRule]) ).expect("Should be able to reverse meow"); const reversedBan = reversedPolicies.at(0); if (reversedBan === undefined) { @@ -74,4 +84,144 @@ describe("meow", function () { } expect(reversedBan.entity).toBe(bannedRoom.toRoomIDOrAlias()); }); + it("Can discover servers from stored rooms", async function () { + const store = makeStore(); + const bannedServer = StringServerName("banned.example.com"); + const roomBannedViaServer = StringRoomID("!banned:banned.example.com"); + const policyRoom = randomRoomID([]); + const bannedServerHash = base64sha256(bannedServer); + const findResult = (await store.findServerHash(bannedServerHash)).expect( + "Should be able to at least query this" + ); + expect(findResult).toBe(undefined); + // now for finding reversed hashed room policies: + const ban = parsePolicyRule( + describeStateEvent({ + room_id: policyRoom.toRoomIDOrAlias(), + content: { + recommendation: Recommendation.Takedown, + hashes: { + sha256: bannedServerHash, + }, + }, + sender: randomUserID(), + state_key: bannedServerHash, + type: PolicyRuleType.Server, + }) as never + ).expect("Should bea ble to parse the policy rule."); + expect(ban.matchType).toBe(PolicyRuleMatchType.HashedLiteral); + const reverserResult = ( + await store.reverseHashedPolicies([ban as HashedLiteralPolicyRule]) + ).expect("Should be able to reverse policies that are unknown"); + expect(reverserResult.length).toBe(0); + // now try storing the room and testing that the policy is there + (await store.storeUndiscoveredRooms([roomBannedViaServer])).expect( + "Should be able to discover rooms jsut fine" + ); + const foundHash = (await store.findServerHash(bannedServerHash)).expect( + "Should be able to now find the server hash from the room we discovered" + ); + expect(foundHash).toBe(bannedServer); + const reversedPolicies = ( + await store.reverseHashedPolicies([ban as HashedLiteralPolicyRule]) + ).expect("Should be able to reverse meow"); + const reversedBan = reversedPolicies.at(0); + if (reversedBan === undefined) { + throw new TypeError("Reversed ban isn't here mare"); + } + expect(reversedBan.entity).toBe(bannedServer); + }); + it("Can reverse user policies", async function () { + const store = makeStore(); + const bannedUser = StringUserID("@banned:banned.example.com"); + const policyRoom = randomRoomID([]); + const bannedUserHash = base64sha256(bannedUser); + const findResult = (await store.findUserHash(bannedUserHash)).expect( + "Should be able to at least query this" + ); + expect(findResult).toBe(undefined); + // now for finding reversed hashed room policies: + const ban = parsePolicyRule( + describeStateEvent({ + room_id: policyRoom.toRoomIDOrAlias(), + content: { + recommendation: Recommendation.Takedown, + hashes: { + sha256: bannedUserHash, + }, + }, + sender: randomUserID(), + state_key: bannedUserHash, + type: PolicyRuleType.User, + }) as never + ).expect("Should bea ble to parse the policy rule."); + expect(ban.matchType).toBe(PolicyRuleMatchType.HashedLiteral); + const reverserResult = ( + await store.reverseHashedPolicies([ban as HashedLiteralPolicyRule]) + ).expect("Should be able to reverse policies that are unknown"); + expect(reverserResult.length).toBe(0); + // now try storing the room and testing that the policy is there + (await store.storeUndiscoveredUsers([bannedUser])).expect( + "Should be able to discover banned users just fine" + ); + const foundHash = (await store.findUserHash(bannedUserHash)).expect( + "Should be able to now find the server hash from the room we discovered" + ); + expect(foundHash).toBe(bannedUser); + const reversedPolicies = ( + await store.reverseHashedPolicies([ban as HashedLiteralPolicyRule]) + ).expect("Should be able to reverse meow"); + const reversedBan = reversedPolicies.at(0); + if (reversedBan === undefined) { + throw new TypeError("Reversed ban isn't here mare"); + } + expect(reversedBan.entity).toBe(bannedUser); + }); + it("Can discover servers from stored users", async function () { + const store = makeStore(); + const bannedServer = StringServerName("banned.example.com"); + const bannedUser = StringUserID("@banned:banned.example.com"); + const policyRoom = randomRoomID([]); + const bannedServerHash = base64sha256(bannedServer); + const findResult = (await store.findServerHash(bannedServerHash)).expect( + "Should be able to at least query this" + ); + expect(findResult).toBe(undefined); + // now for finding reversed hashed room policies: + const ban = parsePolicyRule( + describeStateEvent({ + room_id: policyRoom.toRoomIDOrAlias(), + content: { + recommendation: Recommendation.Takedown, + hashes: { + sha256: bannedServerHash, + }, + }, + sender: randomUserID(), + state_key: bannedServerHash, + type: PolicyRuleType.Server, + }) as never + ).expect("Should bea ble to parse the policy rule."); + expect(ban.matchType).toBe(PolicyRuleMatchType.HashedLiteral); + const reverserResult = ( + await store.reverseHashedPolicies([ban as HashedLiteralPolicyRule]) + ).expect("Should be able to reverse policies that are unknown"); + expect(reverserResult.length).toBe(0); + // now try storing the room and testing that the policy is there + (await store.storeUndiscoveredUsers([bannedUser])).expect( + "Should be able to discover banned users just fine" + ); + const foundHash = (await store.findServerHash(bannedServerHash)).expect( + "Should be able to now find the server hash from the room we discovered" + ); + expect(foundHash).toBe(bannedServer); + const reversedPolicies = ( + await store.reverseHashedPolicies([ban as HashedLiteralPolicyRule]) + ).expect("Should be able to reverse meow"); + const reversedBan = reversedPolicies.at(0); + if (reversedBan === undefined) { + throw new TypeError("Reversed ban isn't here mare"); + } + expect(reversedBan.entity).toBe(bannedServer); + }); }); From 6168669fffb2ea960aedc5ae63bd579a09f3587e Mon Sep 17 00:00:00 2001 From: gnuxie Date: Sat, 22 Mar 2025 11:35:57 +0000 Subject: [PATCH 36/48] Update Draupnir for reworked hash reversal. We introduced a new PolicyRuleChangeType enum with a variant for revealed literals. And we changed our mind about where reversal will happen, it'll happen in the room state manager factory. --- src/DraupnirBotMode.ts | 3 ++- src/appservice/AppService.ts | 3 ++- src/protections/RedactionSynchronisation.ts | 4 ++-- src/protections/RoomTakedown/RoomDiscovery.ts | 4 ++-- src/protections/RoomTakedown/RoomTakedown.ts | 4 ++-- test/unit/protections/RoomTakedownServiceTest.ts | 4 ++-- 6 files changed, 12 insertions(+), 10 deletions(-) diff --git a/src/DraupnirBotMode.ts b/src/DraupnirBotMode.ts index eeb849ff..11014981 100644 --- a/src/DraupnirBotMode.ts +++ b/src/DraupnirBotMode.ts @@ -163,7 +163,8 @@ export class DraupnirBotModeToggle implements BotModeTogle { clientsInRoomMap, clientProvider, DefaultEventDecoder, - stores.roomStateBackingStore + stores.roomStateBackingStore, + stores.hashStore ); const draupnirFactory = new DraupnirFactory( clientsInRoomMap, diff --git a/src/appservice/AppService.ts b/src/appservice/AppService.ts index 402e6b02..daeaa48d 100644 --- a/src/appservice/AppService.ts +++ b/src/appservice/AppService.ts @@ -128,7 +128,8 @@ export class MjolnirAppService { clientsInRoomMap, clientProvider, eventDecoder, - stores.roomStateBackingStore + stores.roomStateBackingStore, + stores.hashStore ); const clientCapabilityFactory = new ClientCapabilityFactory( clientsInRoomMap, diff --git a/src/protections/RedactionSynchronisation.ts b/src/protections/RedactionSynchronisation.ts index d8fb4580..b85e1401 100644 --- a/src/protections/RedactionSynchronisation.ts +++ b/src/protections/RedactionSynchronisation.ts @@ -17,6 +17,7 @@ import { PolicyListRevision, PolicyRule, PolicyRuleChange, + PolicyRuleChangeType, PolicyRuleMatchType, PolicyRuleType, PowerLevelPermission, @@ -25,7 +26,6 @@ import { ProtectionDescription, Recommendation, RoomMembershipRevision, - SimpleChangeType, Task, describeProtection, } from "matrix-protection-suite"; @@ -93,7 +93,7 @@ export class RedactionSynchronisationProtection ): Promise> { const relevantChanges = changes.filter( (change) => - change.changeType === SimpleChangeType.Added && + change.changeType !== PolicyRuleChangeType.Removed && change.rule.kind === PolicyRuleType.User && this.automaticRedactionReasons.some((reason) => reason.test(change.rule.reason ?? "") diff --git a/src/protections/RoomTakedown/RoomDiscovery.ts b/src/protections/RoomTakedown/RoomDiscovery.ts index 7dbc1076..173f307c 100644 --- a/src/protections/RoomTakedown/RoomDiscovery.ts +++ b/src/protections/RoomTakedown/RoomDiscovery.ts @@ -8,7 +8,7 @@ import { Logger, RoomBasicDetails, RoomHashRecord, - SHA256RoomHashStore, + SHA256HashStore, StandardBatcher, Task, } from "matrix-protection-suite"; @@ -47,7 +47,7 @@ export class SynapseHTTPAntispamRoomDiscovery ); constructor( private readonly synapseHTTPAntispam: SynapseHttpAntispam, - private readonly hashStore: SHA256RoomHashStore, + private readonly hashStore: SHA256HashStore, private readonly roomDetailsProvider: RoomDetailsProvider ) { super(); diff --git a/src/protections/RoomTakedown/RoomTakedown.ts b/src/protections/RoomTakedown/RoomTakedown.ts index 4a24673c..9e12948a 100644 --- a/src/protections/RoomTakedown/RoomTakedown.ts +++ b/src/protections/RoomTakedown/RoomTakedown.ts @@ -8,10 +8,10 @@ import { Logger, PolicyListRevision, PolicyRuleChange, + PolicyRuleChangeType, PolicyRuleMatchType, PolicyRuleType, Recommendation, - SimpleChangeType, } from "matrix-protection-suite"; import { RoomAuditLog } from "./RoomAuditLog"; import { isError, Ok, Result, ResultError } from "@gnuxie/typescript-result"; @@ -93,7 +93,7 @@ export class StandardRoomTakedown implements RoomTakedownService { const roomsToTakedown: LiteralPolicyRule[] = []; for (const change of changes) { if ( - change.changeType === SimpleChangeType.Added && + change.changeType === PolicyRuleChangeType.RevealedLiteral && change.rule.kind === PolicyRuleType.Room && change.rule.matchType === PolicyRuleMatchType.Literal && change.rule.recommendation === Recommendation.Takedown diff --git a/test/unit/protections/RoomTakedownServiceTest.ts b/test/unit/protections/RoomTakedownServiceTest.ts index 5bb9785d..97018ef0 100644 --- a/test/unit/protections/RoomTakedownServiceTest.ts +++ b/test/unit/protections/RoomTakedownServiceTest.ts @@ -9,10 +9,10 @@ import { describeProtectedRoomsSet, Ok, parsePolicyRule, + PolicyRuleChangeType, PolicyRuleType, randomRoomID, Recommendation, - SimpleChangeType, StandardPolicyListRevision, } from "matrix-protection-suite"; import { RoomAuditLog } from "../../../src/protections/RoomTakedown/RoomAuditLog"; @@ -81,7 +81,7 @@ describe("", function () { StandardPolicyListRevision.blankRevision(), [ { - changeType: SimpleChangeType.Added, + changeType: PolicyRuleChangeType.Added, rule: policy, event: policy.sourceEvent, sender: policy.sourceEvent.sender, From aea1d1d36e7bc0dad70b147616e3705a3e3560ab Mon Sep 17 00:00:00 2001 From: gnuxie Date: Sat, 22 Mar 2025 11:37:10 +0000 Subject: [PATCH 37/48] Plumb in SynapseAdminRoomTakedownCapability. Was missing registration. It sucks that the factory for capability providers doesn't let us return a Result type?? --- .../SynapseAdminRoomTakedown.ts | 23 +++++++++++++++++-- src/capabilities/capabilityIndex.ts | 1 + 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/src/capabilities/SynapseAdminRoomTakedown/SynapseAdminRoomTakedown.ts b/src/capabilities/SynapseAdminRoomTakedown/SynapseAdminRoomTakedown.ts index 18bfcaca..75c540ac 100644 --- a/src/capabilities/SynapseAdminRoomTakedown/SynapseAdminRoomTakedown.ts +++ b/src/capabilities/SynapseAdminRoomTakedown/SynapseAdminRoomTakedown.ts @@ -3,14 +3,19 @@ // SPDX-License-Identifier: Apache-2.0 import { StringRoomID } from "@the-draupnir-project/matrix-basic-types"; -import { Logger, RoomBasicDetails } from "matrix-protection-suite"; +import { + describeCapabilityProvider, + Logger, + RoomBasicDetails, +} from "matrix-protection-suite"; import { RoomDetailsProvider, RoomTakedownCapability, } from "../RoomTakedownCapability"; import { SynapseAdminClient } from "matrix-protection-suite-for-matrix-bot-sdk"; import { isError, Ok, Result } from "@gnuxie/typescript-result"; - +import { Draupnir } from "../../Draupnir"; +import "../RoomTakedownCapability"; // needed for the interface to load. const log = new Logger("SynapseAdminRoomTakedownCapability"); export class SynapseAdminRoomDetailsProvider implements RoomDetailsProvider { @@ -90,3 +95,17 @@ export class SynapseAdminRoomTakedownCapability return await this.roomDetailsProvider.getRoomDetails(roomID); } } + +describeCapabilityProvider({ + name: SynapseAdminRoomTakedownCapability.name, + interface: "RoomTakedownCapability", + description: "Takesdown rooms using the synapse admin API", + factory(description, draupnir) { + if (draupnir.synapseAdminClient === undefined) { + throw new TypeError( + "Synapse admin client is not available on this draupnir instance" + ); + } + return new SynapseAdminRoomTakedownCapability(draupnir.synapseAdminClient); + }, +}); diff --git a/src/capabilities/capabilityIndex.ts b/src/capabilities/capabilityIndex.ts index a11c57b6..066b4d61 100644 --- a/src/capabilities/capabilityIndex.ts +++ b/src/capabilities/capabilityIndex.ts @@ -13,3 +13,4 @@ import "./ServerACLConsequencesRenderer"; import "./StandardUserConsequencesRenderer"; import "./RoomTakedownCapability"; import "./RoomTakedownCapabilityRenderer"; +import "./SynapseAdminRoomTakedown/SynapseAdminRoomTakedown"; From 27142cf722adb2f6aabee4bff870f41e33aba321 Mon Sep 17 00:00:00 2001 From: gnuxie Date: Sat, 22 Mar 2025 11:38:21 +0000 Subject: [PATCH 38/48] Create top level stores in integration test fixtures. --- test/integration/fixtures.ts | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/test/integration/fixtures.ts b/test/integration/fixtures.ts index e7138a25..c4ce249d 100644 --- a/test/integration/fixtures.ts +++ b/test/integration/fixtures.ts @@ -9,10 +9,11 @@ // import { + DefaultEventDecoder, MJOLNIR_PROTECTED_ROOMS_EVENT_TYPE, MJOLNIR_WATCHED_POLICY_ROOMS_EVENT_TYPE, } from "matrix-protection-suite"; -import { configRead } from "../../src/config"; +import { configRead, getStoragePath } from "../../src/config"; import { patchMatrixClient } from "../../src/utils"; import { DraupnirTestContext, @@ -21,6 +22,7 @@ import { makeBotModeToggle, teardownManagementRoom, } from "./mjolnirSetupUtils"; +import { makeTopLevelStores } from "../../src/backingstore/DraupnirStores"; patchMatrixClient(); @@ -41,9 +43,14 @@ export const mochaHooks = { this.timeout(30000); const config = (this.config = configRead()); this.managementRoomAlias = config.managementRoom; + const storagePath = getStoragePath(config.dataPath); this.toggle = await makeBotModeToggle(config, { eraseAccountData: true, - stores: {}, + stores: makeTopLevelStores(storagePath, DefaultEventDecoder, { + isRoomStateBackingStoreEnabled: Boolean( + config.roomStateBackingStore.enabled + ), + }), }); this.draupnir = draupnir(); const draupnirMatrixClient = draupnirClient(); From d8dc0facdffaca0c57fefeffa37309191fd32b7e Mon Sep 17 00:00:00 2001 From: gnuxie Date: Sat, 22 Mar 2025 13:46:05 +0000 Subject: [PATCH 39/48] Clarify error in takedown command while fetching room details. --- src/commands/Takedown.tsx | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/commands/Takedown.tsx b/src/commands/Takedown.tsx index 3f6c9ef8..1c897aa4 100644 --- a/src/commands/Takedown.tsx +++ b/src/commands/Takedown.tsx @@ -80,10 +80,12 @@ function handleRoomDiscovery( ); } if (detailsResult.ok.creator === undefined) { - log.debug("Broken details", detailsResult.ok); - return ResultError.Result( - `No creator was provided so we cannot store the details for this room ${roomID}` + log.warn( + "No creator was provided in the details for the room, so we cannot store them", + roomID, + detailsResult.ok ); + return Ok(undefined); } return await store.storeRoomIdentification({ roomID, From 45c8189dbcf9d4638b669eaea8bc0c1901cdaf3a Mon Sep 17 00:00:00 2001 From: gnuxie Date: Sat, 22 Mar 2025 13:46:29 +0000 Subject: [PATCH 40/48] Add option to send takedown for plain-text entities. --- src/commands/Takedown.tsx | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/commands/Takedown.tsx b/src/commands/Takedown.tsx index 1c897aa4..eada0010 100644 --- a/src/commands/Takedown.tsx +++ b/src/commands/Takedown.tsx @@ -142,6 +142,11 @@ export const DraupnirTakedownCommand = describeCommand({ description: "Runs the command without the preview of the unban and the confirmation prompt.", }, + "plain-text": { + isFlag: true, + description: + "Creates a plain-text version of the policy rather than masking the entity with SHA256. There are not many reason to do this other than compatibility with other tools.", + }, }, }, async executor( @@ -220,10 +225,11 @@ export const DraupnirTakedownCommand = describeCommand({ detailsProvider ); } + const plainText = keywords.getKeywordValue("plain-text", false); return await policyRoomEditor.takedownEntity( preview.ok.ruleType, preview.ok.entity, - {} + { shouldHash: !plainText } ); }, }); From 2928b2ffd7af97ee42f4b517a216bcecbdaf5e9e Mon Sep 17 00:00:00 2001 From: gnuxie Date: Sat, 22 Mar 2025 13:46:53 +0000 Subject: [PATCH 41/48] RoomTakedownProtection was missing from DraupnirProtectionsIndex.ts --- src/protections/DraupnirProtectionsIndex.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/protections/DraupnirProtectionsIndex.ts b/src/protections/DraupnirProtectionsIndex.ts index 8d8a7aef..074939dc 100644 --- a/src/protections/DraupnirProtectionsIndex.ts +++ b/src/protections/DraupnirProtectionsIndex.ts @@ -9,6 +9,7 @@ */ // keep alphabetical please. +import "./RoomTakedown/RoomTakedownProtection"; import "./BanPropagation"; import "./BasicFlooding"; import "./FirstMessageIsImage"; From 0bce47d487f2f49ed9430aeb40c2056e472b681f Mon Sep 17 00:00:00 2001 From: gnuxie Date: Sat, 22 Mar 2025 16:21:38 +0000 Subject: [PATCH 42/48] Room Takedown needs to be aware of all rules that are literal. --- .../SynapseAdminRoomTakedown/SynapseAdminRoomTakedown.ts | 1 + src/protections/RoomTakedown/RoomTakedown.ts | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/capabilities/SynapseAdminRoomTakedown/SynapseAdminRoomTakedown.ts b/src/capabilities/SynapseAdminRoomTakedown/SynapseAdminRoomTakedown.ts index 75c540ac..84a41c32 100644 --- a/src/capabilities/SynapseAdminRoomTakedown/SynapseAdminRoomTakedown.ts +++ b/src/capabilities/SynapseAdminRoomTakedown/SynapseAdminRoomTakedown.ts @@ -78,6 +78,7 @@ export class SynapseAdminRoomTakedownCapability } else { details = detailsResponse.ok; } + log.debug("Taking down room", roomID); const takedownResult = await this.adminClient.deleteRoom(roomID, { block: true, purge: true, diff --git a/src/protections/RoomTakedown/RoomTakedown.ts b/src/protections/RoomTakedown/RoomTakedown.ts index 9e12948a..12fc5835 100644 --- a/src/protections/RoomTakedown/RoomTakedown.ts +++ b/src/protections/RoomTakedown/RoomTakedown.ts @@ -93,8 +93,8 @@ export class StandardRoomTakedown implements RoomTakedownService { const roomsToTakedown: LiteralPolicyRule[] = []; for (const change of changes) { if ( - change.changeType === PolicyRuleChangeType.RevealedLiteral && change.rule.kind === PolicyRuleType.Room && + change.changeType !== PolicyRuleChangeType.Removed && change.rule.matchType === PolicyRuleMatchType.Literal && change.rule.recommendation === Recommendation.Takedown ) { From 99cb3b317b649d9360632f92e5fb6cca5bfa34f2 Mon Sep 17 00:00:00 2001 From: gnuxie Date: Sat, 22 Mar 2025 21:45:22 +0000 Subject: [PATCH 43/48] Split SynapseHTTPAntispam from webapis. Webapis need to depend on the callback handler and the callback handler needs to be created per draupnir instance. --- src/DraupnirBotMode.ts | 14 +++++--------- src/draupnirfactory/DraupnirFactory.ts | 11 +++++++---- .../SynapseHTTPAntispam/SynapseHttpAntispam.ts | 13 +++++-------- src/webapis/WebAPIs.ts | 13 ++++--------- test/integration/httpAntispamTest.ts | 2 +- 5 files changed, 22 insertions(+), 31 deletions(-) diff --git a/src/DraupnirBotMode.ts b/src/DraupnirBotMode.ts index 11014981..6c40f4e7 100644 --- a/src/DraupnirBotMode.ts +++ b/src/DraupnirBotMode.ts @@ -47,13 +47,16 @@ import { SafeModeDraupnir } from "./safemode/DraupnirSafeMode"; import { ResultError } from "@gnuxie/typescript-result"; import { SafeModeCause, SafeModeReason } from "./safemode/SafeModeCause"; import { SafeModeBootOption } from "./safemode/BootOption"; -import { SynapseHttpAntispam } from "./webapis/SynapseHTTPAntispam/SynapseHttpAntispam"; import { TopLevelStores } from "./backingstore/DraupnirStores"; const log = new Logger("DraupnirBotMode"); export function constructWebAPIs(draupnir: Draupnir): WebAPIs { - return new WebAPIs(draupnir.reportManager, draupnir.config); + return new WebAPIs( + draupnir.reportManager, + draupnir.config, + draupnir.synapseHTTPAntispam + ); } /** @@ -74,9 +77,6 @@ interface BotModeTogle extends SafeModeToggle { error: ResultError, options?: SafeModeToggleOptions ): Promise>; - // The SynapseHTTPAntispam listeners, if available. - // Which they won't be for some bot mode and all application service users. - readonly synapseHTTPAntispam: SynapseHttpAntispam | undefined; } export class DraupnirBotModeToggle implements BotModeTogle { @@ -84,10 +84,6 @@ export class DraupnirBotModeToggle implements BotModeTogle { private safeModeDraupnir: SafeModeDraupnir | null = null; private webAPIs: WebAPIs | null = null; - public get synapseHTTPAntispam() { - return this.webAPIs?.synapseHTTPAntispam ?? undefined; - } - private constructor( private readonly clientUserID: StringUserID, private readonly managementRoom: MatrixRoomID, diff --git a/src/draupnirfactory/DraupnirFactory.ts b/src/draupnirfactory/DraupnirFactory.ts index 30d79e38..784d0c7c 100644 --- a/src/draupnirfactory/DraupnirFactory.ts +++ b/src/draupnirfactory/DraupnirFactory.ts @@ -29,11 +29,11 @@ import { SafeModeDraupnir } from "../safemode/DraupnirSafeMode"; import { SafeModeCause } from "../safemode/SafeModeCause"; import { SafeModeToggle } from "../safemode/SafeModeToggle"; import { StandardManagementRoomDetail } from "../managementroom/ManagementRoomDetail"; -import { DraupnirBotModeToggle } from "../DraupnirBotMode"; import { createDraupnirStores, TopLevelStores, } from "../backingstore/DraupnirStores"; +import { SynapseHttpAntispam } from "../webapis/SynapseHTTPAntispam/SynapseHttpAntispam"; const log = new Logger("DraupnirFactory"); @@ -133,6 +133,11 @@ export class DraupnirFactory { managementRoomMembership.ok, managementRoomState.ok ); + // synapesHTTPAntispam is made here so that the listeners get destroyed + // when draupnir is destroyed. + const synapseHTTPAntispam = config.web.synapseHTTPAntispam.enabled + ? new SynapseHttpAntispam(config.web.synapseHTTPAntispam.authorization) + : undefined; const clientProfile = await safelyFetchProfile(client, clientUserID); return await Draupnir.makeDraupnirBot( client, @@ -150,9 +155,7 @@ export class DraupnirFactory { toggle, createDraupnirStores(this.stores), // synapseHTTPAntispam is only available in bot mode. - toggle instanceof DraupnirBotModeToggle - ? toggle.synapseHTTPAntispam - : undefined + synapseHTTPAntispam ); } diff --git a/src/webapis/SynapseHTTPAntispam/SynapseHttpAntispam.ts b/src/webapis/SynapseHTTPAntispam/SynapseHttpAntispam.ts index a6d9c729..7392d5d5 100644 --- a/src/webapis/SynapseHTTPAntispam/SynapseHttpAntispam.ts +++ b/src/webapis/SynapseHTTPAntispam/SynapseHttpAntispam.ts @@ -55,27 +55,24 @@ export class SynapseHttpAntispam { private readonly checkEventForSpamEndpoint = new CheckEventForSpamEndpoint( this.checkEventForSpamHandles ); - public constructor( - private readonly webController: Express, - private readonly secret: string - ) { + public constructor(private readonly secret: string) { // nothing to do } - public register(): void { - this.webController.post( + public register(webController: Express): void { + webController.post( `${SPAM_CHECK_PREFIX}/user_may_invite`, makeAuthenticatedEndpointHandler(this.secret, (request, response) => { this.userMayInviteEndpoint.handleUserMayInvite(request, response); }) ); - this.webController.post( + webController.post( `${SPAM_CHECK_PREFIX}/user_may_join_room`, makeAuthenticatedEndpointHandler(this.secret, (request, response) => { this.userMayJoinRoomEndpoint.handleUserMayJoinRoom(request, response); }) ); - this.webController.post( + webController.post( `${SPAM_CHECK_PREFIX}/check_event_for_spam`, makeAuthenticatedEndpointHandler(this.secret, (request, response) => { this.checkEventForSpamEndpoint.handleCheckEventForSpam( diff --git a/src/webapis/WebAPIs.ts b/src/webapis/WebAPIs.ts index 4569bad3..6a9e5f0c 100644 --- a/src/webapis/WebAPIs.ts +++ b/src/webapis/WebAPIs.ts @@ -34,17 +34,15 @@ const AUTHORIZATION = new RegExp("Bearer (.*)"); export class WebAPIs { private webController: express.Express = express(); private httpServer?: Server | undefined; - public readonly synapseHTTPAntispam = new SynapseHttpAntispam( - this.webController, - this.config.web.synapseHTTPAntispam.authorization - ); constructor( private reportManager: StandardReportManager, - private readonly config: IConfig + private readonly config: IConfig, + private readonly synapseHTTPAntispam: SynapseHttpAntispam | undefined ) { // Setup JSON parsing. this.webController.use(express.json()); + this.synapseHTTPAntispam?.register(this.webController); } /** @@ -63,10 +61,7 @@ export class WebAPIs { } ); }); - // enable synapse http antispam - if (this.config.web.synapseHTTPAntispam.enabled) { - this.synapseHTTPAntispam.register(); - } + // configure /report API. if (this.config.web.abuseReporting.enabled) { log.info(`configuring ${API_PREFIX}/report/:room_id/:event_id...`); diff --git a/test/integration/httpAntispamTest.ts b/test/integration/httpAntispamTest.ts index 011c12fc..616e6ef5 100644 --- a/test/integration/httpAntispamTest.ts +++ b/test/integration/httpAntispamTest.ts @@ -19,7 +19,7 @@ describe("Test for http antispam callbacks", function () { if (draupnir === undefined) { throw new TypeError(`setup code is wrong`); } - const synapseHTTPAntispam = this.toggle?.synapseHTTPAntispam; + const synapseHTTPAntispam = draupnir.synapseHTTPAntispam; if (synapseHTTPAntispam === undefined) { throw new TypeError("Setup code is wrong"); } From 9f3afce02c41d05155cc3c816d5cdbb8daaccc92 Mon Sep 17 00:00:00 2001 From: gnuxie Date: Sat, 22 Mar 2025 22:25:36 +0000 Subject: [PATCH 44/48] Fix RoomDiscovery. Two problems: 1. the batcher was being initialised before the slot for the listener was. 2. That item batch interface with void is sketch and using flat would make an array of [roomID, undefined, roomID, undefined...], not good. --- src/protections/RoomTakedown/RoomDiscovery.ts | 27 ++++++++++++------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/src/protections/RoomTakedown/RoomDiscovery.ts b/src/protections/RoomTakedown/RoomDiscovery.ts index 173f307c..6c924016 100644 --- a/src/protections/RoomTakedown/RoomDiscovery.ts +++ b/src/protections/RoomTakedown/RoomDiscovery.ts @@ -23,6 +23,10 @@ import { UserMayJoinRoomRequestBody } from "../../webapis/SynapseHTTPAntispam/Us import { EventEmitter } from "stream"; import { RoomDetailsProvider } from "../../capabilities/RoomTakedownCapability"; +// FIXME: This should really have its own "Room acknowledged" table in the audit log... +// which only sends when a notification has been sent to the admin room. +// This would allow for complete coverage... + const log = new Logger("SynapseHTTPAntispamRoomDiscovery"); export type RoomDiscoveryListener = (details: RoomBasicDetails) => void; @@ -38,17 +42,13 @@ export class SynapseHTTPAntispamRoomDiscovery implements RoomDiscovery { private readonly discoveredRooms = new Set(); - private readonly batcher = new StandardBatcher( - () => - new ConstantPeriodItemBatch( - this.forwardDiscoveredBatch, - { waitPeriodMS: 2500 } - ) - ); + private readonly batcher: StandardBatcher; + constructor( private readonly synapseHTTPAntispam: SynapseHttpAntispam, private readonly hashStore: SHA256HashStore, - private readonly roomDetailsProvider: RoomDetailsProvider + private readonly roomDetailsProvider: RoomDetailsProvider, + private readonly batchWaitPeriodMS = 500 ) { super(); synapseHTTPAntispam.checkEventForSpamHandles.registerNonBlockingHandle( @@ -60,6 +60,13 @@ export class SynapseHTTPAntispamRoomDiscovery synapseHTTPAntispam.userMayJoinRoomHandles.registerNonBlockingHandle( this.handleUserMayJoin ); + this.batcher = new StandardBatcher( + () => + new ConstantPeriodItemBatch( + this.forwardDiscoveredBatch, + { waitPeriodMS: this.batchWaitPeriodMS } + ) + ); } private addRoomDetails(discoveredRooms: RoomHashRecord[]): void { @@ -103,9 +110,9 @@ export class SynapseHTTPAntispamRoomDiscovery private readonly forwardDiscoveredBatch = async function ( this: SynapseHTTPAntispamRoomDiscovery, - rawEntries: [StringRoomID][] + rawEntries: [StringRoomID, undefined][] ): Promise { - const entries = rawEntries.flat(); + const entries = rawEntries.flatMap((entry) => entry[0]); const storeResult = await this.hashStore.storeUndiscoveredRooms(entries); if (isError(storeResult)) { log.error( From ec45086560844d7a0d2858322535b34d857c5f13 Mon Sep 17 00:00:00 2001 From: gnuxie Date: Sat, 22 Mar 2025 22:27:06 +0000 Subject: [PATCH 45/48] Test room takedown. --- .../protections/RoomTakedownProtectionTest.ts | 146 ++++++++++++++++++ 1 file changed, 146 insertions(+) create mode 100644 test/integration/protections/RoomTakedownProtectionTest.ts diff --git a/test/integration/protections/RoomTakedownProtectionTest.ts b/test/integration/protections/RoomTakedownProtectionTest.ts new file mode 100644 index 00000000..3af8c1d8 --- /dev/null +++ b/test/integration/protections/RoomTakedownProtectionTest.ts @@ -0,0 +1,146 @@ +// SPDX-FileCopyrightText: 2025 Gnuxie +// +// SPDX-License-Identifier: AFL-3.0 + +import { + MatrixRoomReference, + StringRoomID, + StringUserID, +} from "@the-draupnir-project/matrix-basic-types"; +import { newTestUser } from "../clientHelper"; +import { DraupnirTestContext } from "../mjolnirSetupUtils"; +import { RoomTakedownProtection } from "../../../src/protections/RoomTakedown/RoomTakedownProtection"; +import expect from "expect"; +import { Draupnir } from "../../../src/Draupnir"; +import { PolicyRuleType } from "matrix-protection-suite"; + +async function createWatchedPolicyRoom( + draupnir: Draupnir +): Promise { + const policyRoomID = (await draupnir.client.createRoom({ + preset: "public_chat", + })) as StringRoomID; + ( + await draupnir.protectedRoomsSet.watchedPolicyRooms.watchPolicyRoomDirectly( + MatrixRoomReference.fromRoomID(policyRoomID) + ) + ).expect("Should be able to watch the new policy room"); + return policyRoomID; +} + +// FIXME: We need the fixtures store to either be deleted in setup (not teardown) or reside in memory +// FIXME: Should we split discovery from takedown protection? I think so... +// FIXME: We need to check whether discovery runs even when the protection is not enabled... + +describe("RoomTakedownProtectionTest", function () { + it("Will takedown a room that is added to the policy list", async function ( + this: DraupnirTestContext + ) { + const draupnir = this.draupnir; + if (draupnir === undefined) { + throw new TypeError(`setup didn't run properly`); + } + const moderator = await newTestUser(this.config.homeserverUrl, { + name: { contains: "moderator" }, + }); + const moderatorUserID = StringUserID(await moderator.getUserId()); + const takedownTarget = await newTestUser(this.config.homeserverUrl, { + name: { contains: "takedown-target" }, + }); + const takedownTargetRoomID = StringRoomID( + await takedownTarget.createRoom({ + preset: "public_chat", + }) + ); + await moderator.joinRoom(draupnir.managementRoomID); + const policyRoom = await createWatchedPolicyRoom(draupnir); + ( + await draupnir.sendTextCommand( + moderatorUserID, + `!draupnir protections enable ${RoomTakedownProtection.name}` + ) + ).expect("Should be able to enable the protection"); + ( + await draupnir.sendTextCommand( + moderatorUserID, + `!draupnir takedown ${takedownTargetRoomID} ${policyRoom} --no-confirm` + ) + ).expect("Should be able to create the policy targetting the room"); + + // give some time for the room to be takendown, synapse can be quite slow at this... + await new Promise((resolve) => setTimeout(resolve, 1000)); + expect( + draupnir.stores.roomAuditLog?.isRoomTakendown(takedownTargetRoomID) + ).toBe(true); + expect( + ( + await draupnir.clientPlatform + .toRoomJoiner() + .joinRoom(takedownTargetRoomID) + ).isOkay + ).toBe(false); + } as unknown as Mocha.AsyncFunc); + it( + "Takedown a room through discovery and a revealed Literal policy change", + async function (this: DraupnirTestContext) { + const draupnir = this.draupnir; + if (draupnir === undefined) { + throw new TypeError(`setup didn't run properly`); + } + const synapseHTTPAntispam = draupnir.synapseHTTPAntispam; + if (synapseHTTPAntispam === undefined) { + throw new TypeError("Setup code is wrong"); + } + const moderator = await newTestUser(this.config.homeserverUrl, { + name: { contains: "moderator" }, + }); + const moderatorUserID = StringUserID(await moderator.getUserId()); + const takedownTarget = await newTestUser(this.config.homeserverUrl, { + name: { contains: "takedown-target" }, + }); + const takedownTargetRoomID = StringRoomID( + await takedownTarget.createRoom({ + preset: "public_chat", + }) + ); + await moderator.joinRoom(draupnir.managementRoomID); + const policyRoom = await createWatchedPolicyRoom(draupnir); + ( + await draupnir.sendTextCommand( + moderatorUserID, + `!draupnir protections enable ${RoomTakedownProtection.name}` + ) + ).expect("Should be able to enable the protection"); + + const policyRoomEditor = ( + await draupnir.policyRoomManager.getPolicyRoomEditor( + MatrixRoomReference.fromRoomID(policyRoom) + ) + ).expect("Should be able to get the policy room editor"); + ( + await policyRoomEditor.takedownEntity( + PolicyRuleType.Room, + takedownTargetRoomID, + { shouldHash: true } + ) + ).expect("Should be able to takedown the room via a policy list editor"); + // give some time for the room to be takendown, synapse can be quite slow at this... + await new Promise((resolve) => setTimeout(resolve, 1000)); + const bystander = await newTestUser(this.config.homeserverUrl, { + name: { contains: "bystander" }, + }); + await bystander.joinRoom(takedownTargetRoomID); + await new Promise((resolve) => setTimeout(resolve, 1000)); + expect( + draupnir.stores.roomAuditLog?.isRoomTakendown(takedownTargetRoomID) + ).toBe(true); + expect( + ( + await draupnir.clientPlatform + .toRoomJoiner() + .joinRoom(takedownTargetRoomID) + ).isOkay + ).toBe(false); + } as unknown as Mocha.AsyncFunc + ); +}); From b474e24c5ba12ba1610cb374491b8f3b5dc4a840 Mon Sep 17 00:00:00 2001 From: gnuxie Date: Sun, 23 Mar 2025 10:20:15 +0000 Subject: [PATCH 46/48] Update MPS and matrix-basic-types. --- package.json | 6 +++--- yarn.lock | 24 ++++++++++++------------ 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/package.json b/package.json index ac159d04..0fbeaa97 100644 --- a/package.json +++ b/package.json @@ -53,7 +53,7 @@ "@sentry/node": "^7.17.2", "@sinclair/typebox": "0.34.13", "@the-draupnir-project/interface-manager": "4.0.2", - "@the-draupnir-project/matrix-basic-types": "1.2.0", + "@the-draupnir-project/matrix-basic-types": "1.3.0", "better-sqlite3": "^9.4.3", "body-parser": "^1.20.2", "config": "^3.3.9", @@ -63,8 +63,8 @@ "jsdom": "^24.0.0", "matrix-appservice-bridge": "^10.3.1", "matrix-bot-sdk": "npm:@vector-im/matrix-bot-sdk@^0.7.1-element.6", - "matrix-protection-suite": "npm:@gnuxie/matrix-protection-suite@2.10.0", - "matrix-protection-suite-for-matrix-bot-sdk": "npm:@gnuxie/matrix-protection-suite-for-matrix-bot-sdk@2.10.1", + "matrix-protection-suite": "npm:@gnuxie/matrix-protection-suite@3.0.0", + "matrix-protection-suite-for-matrix-bot-sdk": "npm:@gnuxie/matrix-protection-suite-for-matrix-bot-sdk@3.0.0", "pg": "^8.8.0", "yaml": "^2.3.2" }, diff --git a/yarn.lock b/yarn.lock index f54426ca..ca891002 100644 --- a/yarn.lock +++ b/yarn.lock @@ -319,10 +319,10 @@ "@gnuxie/super-cool-stream" "^0.2.1" "@gnuxie/typescript-result" "^1.0.0" -"@the-draupnir-project/matrix-basic-types@1.2.0": - version "1.2.0" - resolved "https://registry.yarnpkg.com/@the-draupnir-project/matrix-basic-types/-/matrix-basic-types-1.2.0.tgz#94d2a895aafd824a768942c34dec4683ca9449b5" - integrity sha512-1v1VCezE17hR5s12ob05uywZKxPxr1FXC8G1RG9TQXB7BBOcZPnVkebOtv/+qLjSwwOyqeHp2nHx9NNQrioXEQ== +"@the-draupnir-project/matrix-basic-types@1.3.0": + version "1.3.0" + resolved "https://registry.yarnpkg.com/@the-draupnir-project/matrix-basic-types/-/matrix-basic-types-1.3.0.tgz#02fa9bd75eeed778a57da0844447feca3a4c663c" + integrity sha512-WkUD7cqs9qFr1NQZiJWxa+/1trWrhQqv8SbPxrMLRKv1LMm93ELeguROP470kHNWHrCkVcKmYQVc4uC2Yup+EQ== dependencies: "@gnuxie/typescript-result" "^1.0.0" glob-to-regexp "^0.4.1" @@ -2593,18 +2593,18 @@ matrix-appservice@^2.0.0: request-promise "^4.2.6" sanitize-html "^2.11.0" -"matrix-protection-suite-for-matrix-bot-sdk@npm:@gnuxie/matrix-protection-suite-for-matrix-bot-sdk@2.10.1": - version "2.10.1" - resolved "https://registry.yarnpkg.com/@gnuxie/matrix-protection-suite-for-matrix-bot-sdk/-/matrix-protection-suite-for-matrix-bot-sdk-2.10.1.tgz#63ee16db0bcbadd8f52e472a349dae0253662783" - integrity sha512-vB0zhdZv8KTZ0FdyqG93CIW9TJHlyHUFCTBsfYsNFFtlVce5fzihL01GqPiJT1BMjLxwVZ84OAvGQa7I5a/DKw== +"matrix-protection-suite-for-matrix-bot-sdk@npm:@gnuxie/matrix-protection-suite-for-matrix-bot-sdk@3.0.0": + version "3.0.0" + resolved "https://registry.yarnpkg.com/@gnuxie/matrix-protection-suite-for-matrix-bot-sdk/-/matrix-protection-suite-for-matrix-bot-sdk-3.0.0.tgz#eb4eaf07b040d223d18e949dfbe8d6712563974e" + integrity sha512-Rv7Ouxc11z08RvJ6d+W1v8pTxWUsQoJztMR53YQaRV1Q2YBkRAPJxIm/xIAfAf15j32LYJR+2TKrs1HxwFMybQ== dependencies: "@gnuxie/typescript-result" "^1.0.0" await-lock "^2.2.2" -"matrix-protection-suite@npm:@gnuxie/matrix-protection-suite@2.10.0": - version "2.10.0" - resolved "https://registry.yarnpkg.com/@gnuxie/matrix-protection-suite/-/matrix-protection-suite-2.10.0.tgz#23016419cbed28b8a645fe9c82b5df2fe5cf4751" - integrity sha512-Zwrm9RryTgJEqZ6Y9uPa39YZZwsRLUqfS5I5X0fXrre3M68VX7BNIYKQ9xKvjTBKsTI/jSSCJImARYt5fBT0Zg== +"matrix-protection-suite@npm:@gnuxie/matrix-protection-suite@3.0.0": + version "3.0.0" + resolved "https://registry.yarnpkg.com/@gnuxie/matrix-protection-suite/-/matrix-protection-suite-3.0.0.tgz#6e679f1f0e3b9a16f046c095d1ed1416a078e376" + integrity sha512-X06uzOKN/fwOW2Vm3AKL2HKLbjU1PrqhUsYTzZGGQnEQYkAvoFnuK/TyKF8QaVoMJ/PF4Ybkkj6t5GbZGsGajg== dependencies: "@gnuxie/typescript-result" "^1.0.0" await-lock "^2.2.2" From e033d531270b9b18441604c60604850dd46c4c79 Mon Sep 17 00:00:00 2001 From: gnuxie Date: Mon, 24 Mar 2025 19:16:44 +0000 Subject: [PATCH 47/48] Add store management to integration test fixtures. --- src/backingstore/better-sqlite3/HashStore.ts | 3 +- .../SqliteRoomStateBackingStore.ts | 3 +- src/protections/RoomTakedown/RoomAuditLog.ts | 1 + .../RoomTakedown/SqliteRoomAuditLog.ts | 3 +- test/integration/fixtures.ts | 56 +++++++++++++++++-- test/integration/mjolnirSetupUtils.ts | 1 + .../protections/RoomTakedownProtectionTest.ts | 4 -- 7 files changed, 58 insertions(+), 13 deletions(-) diff --git a/src/backingstore/better-sqlite3/HashStore.ts b/src/backingstore/better-sqlite3/HashStore.ts index c31d5e54..3b14b3a3 100644 --- a/src/backingstore/better-sqlite3/HashStore.ts +++ b/src/backingstore/better-sqlite3/HashStore.ts @@ -136,9 +136,10 @@ export class SqliteHashReversalStore ); } + public static readonly StoreName = "hash-store.db"; public static createToplevel(storagePath: string): SqliteHashReversalStore { const options = { - path: path.join(storagePath, "hash-store.db"), + path: path.join(storagePath, SqliteHashReversalStore.StoreName), WALMode: true, foreignKeys: true, fileMustExist: false, diff --git a/src/backingstore/better-sqlite3/SqliteRoomStateBackingStore.ts b/src/backingstore/better-sqlite3/SqliteRoomStateBackingStore.ts index bb62472d..22c60ca4 100644 --- a/src/backingstore/better-sqlite3/SqliteRoomStateBackingStore.ts +++ b/src/backingstore/better-sqlite3/SqliteRoomStateBackingStore.ts @@ -85,12 +85,13 @@ export class SqliteRoomStateBackingStore super(SchemaOptions, db, log); } + public static readonly StoreName = "room-state-backing-store.db"; public static create( storagePath: string, eventDecoder: EventDecoder ): SqliteRoomStateBackingStore { const options = { - path: path.join(storagePath, "room-state-backing-store.db"), + path: path.join(storagePath, SqliteRoomStateBackingStore.StoreName), WALMode: true, foreignKeys: true, fileMustExist: false, diff --git a/src/protections/RoomTakedown/RoomAuditLog.ts b/src/protections/RoomTakedown/RoomAuditLog.ts index 6f169aa0..24d0295b 100644 --- a/src/protections/RoomTakedown/RoomAuditLog.ts +++ b/src/protections/RoomTakedown/RoomAuditLog.ts @@ -23,4 +23,5 @@ export interface RoomAuditLog { getTakedownDetails( roomID: StringRoomID ): Promise>; + destroy(): void; } diff --git a/src/protections/RoomTakedown/SqliteRoomAuditLog.ts b/src/protections/RoomTakedown/SqliteRoomAuditLog.ts index 7b8a12e1..6ebe2601 100644 --- a/src/protections/RoomTakedown/SqliteRoomAuditLog.ts +++ b/src/protections/RoomTakedown/SqliteRoomAuditLog.ts @@ -98,9 +98,10 @@ export class SqliteRoomAuditLog this.takedownRooms = new Set(this.loadTakendownRooms()); } + public static readonly StoreName = "room-audit-log.db"; public static createToplevel(storagePath: string): SqliteRoomAuditLog { const options = { - path: path.join(storagePath, "room-audit-log.db"), + path: path.join(storagePath, SqliteRoomAuditLog.StoreName), WALMode: true, foreignKeys: true, fileMustExist: false, diff --git a/test/integration/fixtures.ts b/test/integration/fixtures.ts index c4ce249d..68ebda3e 100644 --- a/test/integration/fixtures.ts +++ b/test/integration/fixtures.ts @@ -23,9 +23,45 @@ import { teardownManagementRoom, } from "./mjolnirSetupUtils"; import { makeTopLevelStores } from "../../src/backingstore/DraupnirStores"; +import { SqliteRoomStateBackingStore } from "../../src/backingstore/better-sqlite3/SqliteRoomStateBackingStore"; +import { SqliteHashReversalStore } from "../../src/backingstore/better-sqlite3/HashStore"; +import { SqliteRoomAuditLog } from "../../src/protections/RoomTakedown/SqliteRoomAuditLog"; +import path from "path"; +import { promises as fs } from "fs"; patchMatrixClient(); +/** + * We cleanup the stores at the start of the tests, so that they can be + * inspected if a test fails. + */ +async function cleanUpTopLevelStores(storagePath: string) { + await fs.mkdir(storagePath, { recursive: true }); // Ensure storagePath exists + const storePaths = [ + SqliteRoomStateBackingStore.StoreName, + SqliteHashReversalStore.StoreName, + SqliteRoomAuditLog.StoreName, + ].map((name) => path.join(storagePath, name)); + await Promise.all( + storePaths.map(async (filePath) => { + try { + await fs.unlink(filePath); + } catch (error: unknown) { + if ( + typeof error === "object" && + error !== null && + "code" in error && + error.code === "ENOENT" + ) { + // do nothing + } else { + throw error; // Re-throw other errors + } + } + }) + ); +} + // When Draupnir starts (src/index.ts) it clobbers the config by resolving the management room // alias specified in the config (config.managementRoom) and overwriting that with the room ID. // Unfortunately every piece of code importing that config imports the same instance, including @@ -44,13 +80,15 @@ export const mochaHooks = { const config = (this.config = configRead()); this.managementRoomAlias = config.managementRoom; const storagePath = getStoragePath(config.dataPath); + await cleanUpTopLevelStores(storagePath); + this.stores = makeTopLevelStores(storagePath, DefaultEventDecoder, { + isRoomStateBackingStoreEnabled: Boolean( + config.roomStateBackingStore.enabled + ), + }); this.toggle = await makeBotModeToggle(config, { eraseAccountData: true, - stores: makeTopLevelStores(storagePath, DefaultEventDecoder, { - isRoomStateBackingStoreEnabled: Boolean( - config.roomStateBackingStore.enabled - ), - }), + stores: this.stores, }); this.draupnir = draupnir(); const draupnirMatrixClient = draupnirClient(); @@ -67,7 +105,13 @@ export const mochaHooks = { this.timeout(10000); await this.toggle?.stopEverything(); draupnirClient()?.stop(); - + this.stores?.roomStateBackingStore?.destroy(); + this.stores?.roomAuditLog?.destroy(); + // We forgot to add a destroy method in the interface in MPS + // and we'll probably need to wait until the next release to remove this. + if (this.stores?.hashStore instanceof SqliteHashReversalStore) { + this.stores.hashStore.destroy(); + } // remove alias from management room and leave it. if (this.draupnir !== undefined) { await Promise.all([ diff --git a/test/integration/mjolnirSetupUtils.ts b/test/integration/mjolnirSetupUtils.ts index 47592145..c6a70aa5 100644 --- a/test/integration/mjolnirSetupUtils.ts +++ b/test/integration/mjolnirSetupUtils.ts @@ -53,6 +53,7 @@ export interface DraupnirTestContext extends SafeMochaContext { managementRoomAlias?: string; toggle?: DraupnirBotModeToggle; config: IConfig; + stores?: TopLevelStores; } /** diff --git a/test/integration/protections/RoomTakedownProtectionTest.ts b/test/integration/protections/RoomTakedownProtectionTest.ts index 3af8c1d8..d4b52aa9 100644 --- a/test/integration/protections/RoomTakedownProtectionTest.ts +++ b/test/integration/protections/RoomTakedownProtectionTest.ts @@ -28,10 +28,6 @@ async function createWatchedPolicyRoom( return policyRoomID; } -// FIXME: We need the fixtures store to either be deleted in setup (not teardown) or reside in memory -// FIXME: Should we split discovery from takedown protection? I think so... -// FIXME: We need to check whether discovery runs even when the protection is not enabled... - describe("RoomTakedownProtectionTest", function () { it("Will takedown a room that is added to the policy list", async function ( this: DraupnirTestContext From 903a7b6dbc9dd2f91c49ab9d7a2a100e9f8c1c1e Mon Sep 17 00:00:00 2001 From: gnuxie Date: Mon, 24 Mar 2025 19:18:42 +0000 Subject: [PATCH 48/48] Passthrough all events for check_event_for_spam. We realise the PDU rate for most homeservers will not be significant... and if it is they can just disable the callback. --- src/protections/RoomTakedown/RoomDiscovery.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/protections/RoomTakedown/RoomDiscovery.ts b/src/protections/RoomTakedown/RoomDiscovery.ts index 6c924016..98e2ed15 100644 --- a/src/protections/RoomTakedown/RoomDiscovery.ts +++ b/src/protections/RoomTakedown/RoomDiscovery.ts @@ -131,10 +131,8 @@ export class SynapseHTTPAntispamRoomDiscovery this: SynapseHTTPAntispamRoomDiscovery, { event }: CheckEventForSpamRequestBody ): void { - if (event.type === "m.room.member") { - if (!this.discoveredRooms.has(event.room_id)) { - this.batcher.add(event.room_id); - } + if (!this.discoveredRooms.has(event.room_id)) { + this.batcher.add(event.room_id); } }.bind(this);