From 5446156680a4d0ce65b61531476deeb3d2dec9bf Mon Sep 17 00:00:00 2001 From: Rory& Date: Thu, 23 Oct 2025 19:04:46 +0200 Subject: [PATCH] Refactor config --- scripts/configTemplates/bundle.json | 15 ++ scripts/configTemplates/split.json | 15 ++ src/applyConfig.ts | 19 ++ src/util/util/Config.ts | 333 +++++++++++++++++----------- 4 files changed, 248 insertions(+), 134 deletions(-) create mode 100644 scripts/configTemplates/bundle.json create mode 100644 scripts/configTemplates/split.json create mode 100755 src/applyConfig.ts diff --git a/scripts/configTemplates/bundle.json b/scripts/configTemplates/bundle.json new file mode 100644 index 000000000..e975a1d9c --- /dev/null +++ b/scripts/configTemplates/bundle.json @@ -0,0 +1,15 @@ +{ + "gateway": { + "endpointPublic": "ws://localhost:3001" + }, + "cdn": { + "endpointPrivate": "http://localhost:3001", + "endpointPublic": "http://localhost:3001" + }, + "api": { + "endpointPublic": "http://localhost:3001/api/v9/" + }, + "rabbitmq": { + "host": null + } +} \ No newline at end of file diff --git a/scripts/configTemplates/split.json b/scripts/configTemplates/split.json new file mode 100644 index 000000000..f58adc8df --- /dev/null +++ b/scripts/configTemplates/split.json @@ -0,0 +1,15 @@ +{ + "api": { + "endpointPublic": "http://localhost:3001/api/v9/" + }, + "cdn": { + "endpointPublic": "http://localhost:3003", + "endpointPrivate": "http://localhost:3003" + }, + "gateway": { + "endpointPublic": "ws://localhost:3002" + }, + "rabbitmq": { + "host": "amqp://guest:guest@localhost/" + } +} \ No newline at end of file diff --git a/src/applyConfig.ts b/src/applyConfig.ts new file mode 100755 index 000000000..e414b2a2a --- /dev/null +++ b/src/applyConfig.ts @@ -0,0 +1,19 @@ +import { bgRedBright } from "picocolors"; + +require("dotenv").config({ quiet: true }); +import moduleAlias from "module-alias"; +moduleAlias(); + +process.env.CONFIG_PATH = process.argv[2] || ""; +process.env.CONFIG_MODE = "overwrite"; +process.env.CONFIG_WRITEBACK = "false"; + +import { initDatabase } from "@spacebar/util"; +import { Config } from "@spacebar/util"; +import { EnvConfig } from "@spacebar/util"; + +(async () => { + console.log("Env config:", JSON.stringify(EnvConfig.get(), null, 2)); + await initDatabase(); + await Config.init(); +})(); diff --git a/src/util/util/Config.ts b/src/util/util/Config.ts index 1ca455ff7..c0e1e09ce 100644 --- a/src/util/util/Config.ts +++ b/src/util/util/Config.ts @@ -18,180 +18,245 @@ import { existsSync } from "fs"; import fs from "fs/promises"; -import { OrmUtils } from ".."; +import { EnvConfig, OrmUtils } from ".."; import { ConfigValue } from "../config"; import { ConfigEntity } from "../entities/Config"; import { JsonValue } from "@protobuf-ts/runtime"; +import { yellow, yellowBright } from "picocolors"; -// TODO: yaml instead of json -const overridePath = process.env.CONFIG_PATH ?? ""; - -let config: ConfigValue; -let pairs: ConfigEntity[]; +let cachedConfig: ConfigValue; +let cachedConfigWithOverrides: ConfigValue | undefined; +let cachedPairs: ConfigEntity[]; // TODO: use events to inform about config updates // Config keys are separated with _ export class Config { - public static async init(force: boolean = false) { - if (config && !force) return config; - console.log("[Config] Loading configuration..."); - if (!process.env.CONFIG_PATH) { - pairs = await validateConfig(); - config = pairsToConfig(pairs); - } else { - console.log(`[Config] Using CONFIG_PATH rather than database`); - if (existsSync(process.env.CONFIG_PATH)) { - const file = JSON.parse((await fs.readFile(process.env.CONFIG_PATH)).toString()); - config = file; - } else config = new ConfigValue(); - pairs = generatePairs(config); - } + public static async init(force: boolean = false) { + if ((cachedConfigWithOverrides || cachedConfig) && !force) return cachedConfigWithOverrides ?? cachedConfig; + console.log("[Config] Loading configuration..."); - // If a config doesn't exist, create it. - if (Object.keys(config).length == 0) config = new ConfigValue(); + const { path: jsonPath, enabled: jsonEnabled, mode: jsonMode, writebackEnabled: jsonWritebackEnabled } = EnvConfig.get().configuration; + let jsonConfig: ConfigValue; - config = OrmUtils.mergeDeep({}, { ...new ConfigValue() }, config); + if (jsonEnabled) console.log(`[Config] Using JSON configuration file at ${jsonPath} in '${jsonMode}' mode.`); - await this.set(config); - validateFinalConfig(config); - return config; - } - public static get() { - if (!config) { - // If we haven't initialised the config yet, return default config. - // Typeorm instantiates each entity once when initialising database, - // which means when we use config values as default values in entity classes, - // the config isn't initialised yet and would throw an error about the config being undefined. + if (jsonEnabled && jsonMode === "single") { + if (!existsSync(jsonPath)) throw new Error(`[Config] CONFIG_PATH does not exist, but CONFIG_MODE is set to 'single'. Please ensure the file at ${jsonPath} exists.`); + jsonConfig = JSON.parse((await fs.readFile(jsonPath)).toString()); - return new ConfigValue(); - } + // merge with defaults to allow partial configs + cachedConfig = OrmUtils.mergeDeep({}, { ...new ConfigValue() }, jsonConfig); + return await saveConfig(cachedConfig); // handle writeback if enabled + } - return config; - } - public static set(val: Partial) { - if (!config || !val) return; - config = OrmUtils.mergeDeep(config); + if (jsonEnabled && existsSync(jsonPath) && jsonMode === "overwrite") { + console.log(`[Config] Loading configuration from JSON file at ${jsonPath}...`); + jsonConfig = JSON.parse((await fs.readFile(jsonPath)).toString()); + console.log("[Config] Overwriting database configuration with JSON configuration..."); + await saveConfigToDatabaseAtomic(jsonConfig); + } - return applyConfig(config); - } + console.log("[Config] Loading configuration from database..."); + cachedPairs = await validateConfig(); + cachedConfig = pairsToConfig(cachedPairs); + + // If a config doesn't exist, create it. + if (Object.keys(cachedConfig).length == 0) cachedConfig = new ConfigValue(); + + cachedConfig = OrmUtils.mergeDeep({}, { ...new ConfigValue() }, cachedConfig); + + let ret = await this.set(cachedConfig); + + if (jsonEnabled && existsSync(jsonPath) && jsonMode === "override") { + console.log(`[Config] Loading configuration from JSON file at ${jsonPath}...`); + jsonConfig = JSON.parse((await fs.readFile(jsonPath)).toString()); + console.log("[Config] Overriding database configuration values with JSON configuration..."); + ret = cachedConfigWithOverrides = OrmUtils.mergeDeep({}, cachedConfig, jsonConfig); + } + + const changes = getChanges(ret!); + for (const [key, change] of Object.entries(changes)) { + if (change.type === "changed") { + console.log(yellowBright(`[Config] Setting '${key}' has been changed from '${JSON.stringify(change.old)}' to '${JSON.stringify(change.new)}'`)); + } else if (change.type === "unknown") { + console.log(yellow(`[Config] Unknown setting '${key}' with value '${JSON.stringify(change.new)}'`)); + } + } + + validateFinalConfig(ret!); + return ret; + } + + public static get() { + if (!cachedConfig) { + // If we haven't initialised the config yet, return default config. + // Typeorm instantiates each entity once when initialising database, + // which means when we use config values as default values in entity classes, + // the config isn't initialised yet and would throw an error about the config being undefined. + + return new ConfigValue(); + } + + return cachedConfigWithOverrides ?? cachedConfig; + } + public static set(val: Partial) { + if (!cachedConfig || !val) return; + cachedConfig = OrmUtils.mergeDeep(cachedConfig, val); + + return saveConfig(val); + } } // TODO: better types -const generatePairs = (obj: object | null, key = ""): ConfigEntity[] => { - if (typeof obj == "object" && obj != null) { - return Object.keys(obj) - .map((k) => - // eslint-disable-next-line @typescript-eslint/no-explicit-any - generatePairs((obj as any)[k], key ? `${key}_${k}` : k), - ) - .flat(); - } +function generatePairs(obj: object | null, key = ""): ConfigEntity[] { + if (typeof obj == "object" && obj != null) { + return Object.keys(obj) + .map((k) => + // eslint-disable-next-line @typescript-eslint/no-explicit-any + generatePairs((obj as any)[k], key ? `${key}_${k}` : k), + ) + .flat(); + } - const ret = new ConfigEntity(); - ret.key = key; - ret.value = obj; - return [ret]; -}; + const ret = new ConfigEntity(); + ret.key = key; + ret.value = obj; + return [ret]; +} -async function applyConfig(val: ConfigValue) { - if (process.env.CONFIG_PATH) - if (!process.env.CONFIG_READONLY) await fs.writeFile(overridePath, JSON.stringify(val, null, 4)); - else console.log("[WARNING] JSON config file in use, and writing is disabled! Programmatic config changes will not be persisted, and your config will not get updated!"); - else { - const pairs = generatePairs(val); - // keys are sorted to try to influence database order... - await Promise.all(pairs.sort((x, y) => (x.key > y.key ? 1 : -1)).map((pair) => pair.save())); - } - return val; +async function saveConfig(val: Partial) { + const merged = OrmUtils.mergeDeep({}, cachedConfig, val) as ConfigValue; + if (EnvConfig.get().configuration.enabled) { + if (EnvConfig.get().configuration.writebackEnabled) { + await fs.writeFile(EnvConfig.get().configuration.path, JSON.stringify(merged, null, 4)); + } else { + console.log("[Config/WARN] JSON config file in use, and writeback is disabled!"); + console.log("[Config/WARN] Programmatic config changes will not be persisted, and your config will not get updated!"); + console.log("[Config/WARN] Please check regularly to make adjustments as necessary!"); + } + + if (EnvConfig.get().configuration.mode == "overwrite") await saveConfigToDatabaseAtomic(val); + } else await saveConfigToDatabaseAtomic(val); // not using a JSON file + + return merged; +} + +// Atomically save changes to database +async function saveConfigToDatabaseAtomic(val: Partial) { + const pairs = generatePairs(val); + const pairsToUpdate = + cachedPairs === undefined ? pairs : pairs.filter((p) => cachedPairs.some((cp) => cp.key === p.key && JSON.stringify(cp.value) !== JSON.stringify(p.value))); + + if (pairsToUpdate.length > 0) console.log("[Config] Atomic update:", pairsToUpdate); + + // keys are sorted to try to influence database order... + await Promise.all(pairsToUpdate.sort((x, y) => (x.key > y.key ? 1 : -1)).map((pair) => pair.save())); } function pairsToConfig(pairs: ConfigEntity[]) { - // TODO: typings - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const value: any = {}; + // TODO: typings + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const value: any = {}; - pairs.forEach((p) => { - const keys = p.key.split("_"); - let obj = value; - let prev = ""; - let prevObj = obj; - let i = 0; + pairs.forEach((p) => { + const keys = p.key.split("_"); + let obj = value; + let prev = ""; + let prevObj = obj; + let i = 0; - for (const key of keys) { - if (!isNaN(Number(key)) && !prevObj[prev]?.length) prevObj[prev] = obj = []; - if (i++ === keys.length - 1) obj[key] = p.value; - else if (!obj[key]) obj[key] = {}; + for (const key of keys) { + if (!isNaN(Number(key)) && !prevObj[prev]?.length) prevObj[prev] = obj = []; + if (i++ === keys.length - 1) obj[key] = p.value; + else if (!obj[key]) obj[key] = {}; - prev = key; - prevObj = obj; - obj = obj[key]; - } - }); + prev = key; + prevObj = obj; + obj = obj[key]; + } + }); - return value as ConfigValue; + return value as ConfigValue; } const validateConfig = async () => { - let hasErrored = false; - const totalStartTime = new Date(); - const config = await ConfigEntity.find({ select: { key: true } }); + let hasErrored = false; + const totalStartTime = new Date(); + const config = await ConfigEntity.find({ select: { key: true }, order: { key: "ASC" } }); - for (const row in config) { - // extension methods... - if (typeof config[row] === "function") continue; + for (const row in config) { + // extension methods... + if (typeof config[row] === "function") continue; - try { - const found = await ConfigEntity.findOne({ - where: { key: config[row].key }, - }); - if (!found) continue; - config[row] = found; - } catch (e) { - console.error(`Config key '${config[row].key}' has invalid JSON value : ${(e as Error)?.message}`); - hasErrored = true; - } - } + try { + const found = await ConfigEntity.findOne({ + where: { key: config[row].key }, + }); + if (!found) continue; + config[row] = found; + } catch (e) { + console.error(`Config key '${config[row].key}' has invalid JSON value: ${(e as Error)?.message}`); + hasErrored = true; + } + } - console.log("[Config] Total config load time:", new Date().getTime() - totalStartTime.getTime(), "ms"); + console.log("[Config] Total config load time:", new Date().getTime() - totalStartTime.getTime(), "ms"); - if (hasErrored) { - console.error("[Config] Your config has invalid values. Fix them first https://docs.spacebar.chat/setup/server/configuration"); - process.exit(1); - } + if (hasErrored) { + console.error("[Config] Your config has invalid values. Fix them first https://docs.spacebar.chat/setup/server/configuration"); + process.exit(1); + } - return config; + return config; }; function validateFinalConfig(config: ConfigValue) { - let hasErrors = false; - function assertConfig(path: string, condition: (val: JsonValue) => boolean, recommendedValue: string) { - // _ to separate keys - const keys = path.split("_"); - let obj: never = config as never; + let hasErrors = false; + function assertConfig(path: string, condition: (val: JsonValue) => boolean, recommendedValue: string) { + // _ to separate keys + const keys = path.split("_"); + let obj: never = config as never; - for (const key of keys) { - if (obj == null || !(key in obj)) { - console.warn(`[Config] Missing config value for '${path}'. Recommended value: ${recommendedValue}`); - return; - } - obj = obj[key]; - } + for (const key of keys) { + if (obj == null || !(key in obj)) { + console.warn(`[Config] Missing config value for '${path}'. Recommended value: ${recommendedValue}`); + return; + } + obj = obj[key]; + } - if (!condition(obj)) { - console.warn(`[Config] Invalid config value for '${path}': ${obj}. Recommended value: ${recommendedValue}`); - hasErrors = true; - } - } + if (!condition(obj)) { + console.warn(`[Config] Invalid config value for '${path}': ${obj}. Recommended value: ${recommendedValue}`); + hasErrors = true; + } + } - assertConfig("api_endpointPublic", (v) => v != null, 'A valid public API endpoint URL, ex. "http://localhost:3001/api/v9"'); - assertConfig("cdn_endpointPublic", (v) => v != null, 'A valid public CDN endpoint URL, ex. "http://localhost:3003/"'); - assertConfig("cdn_endpointPrivate", (v) => v != null, 'A valid private CDN endpoint URL, ex. "http://localhost:3003/" - must be routable from the API server!'); - assertConfig("gateway_endpointPublic", (v) => v != null, 'A valid public gateway endpoint URL, ex. "ws://localhost:3002/"'); + assertConfig("api_endpointPublic", (v) => v != null, 'A valid public API endpoint URL, ex. "http://localhost:3001/api/v9"'); + assertConfig("cdn_endpointPublic", (v) => v != null, 'A valid public CDN endpoint URL, ex. "http://localhost:3003/"'); + assertConfig("cdn_endpointPrivate", (v) => v != null, 'A valid private CDN endpoint URL, ex. "http://localhost:3003/" - must be routable from the API server!'); + assertConfig("gateway_endpointPublic", (v) => v != null, 'A valid public gateway endpoint URL, ex. "ws://localhost:3002/"'); - if (hasErrors) { - console.error("[Config] Your config has invalid values. Fix them first https://docs.spacebar.chat/setup/server/configuration"); - console.error("[Config] Hint: if you're just testing with bundle (`npm run start`), you can set all endpoint URLs to [proto]://localhost:3001"); - process.exit(1); - } else console.log("[Config] Configuration validated successfully."); + if (hasErrors) { + console.error("[Config] Your config has invalid values. Fix them first https://docs.spacebar.chat/setup/server/configuration"); + console.error("[Config] Hint: if you're just testing with bundle (`npm run start`), you can set all endpoint URLs to [proto]://localhost:3001"); + process.exit(1); + } else console.log("[Config] Configuration validated successfully."); +} + +function getChanges(config: ConfigValue) { + const defaultPairs = generatePairs(new ConfigValue()); + const newPairs = generatePairs(cachedConfig); + const ignoredKeys = ["general_instanceId", "security_requestSignature", "security_jwtSecret", "security_cdnSignatureKey"]; + const changes: { [key: string]: { type: "changed" | "unknown"; old?: string | number | boolean | null | undefined; new: string | number | boolean | null | undefined } } = {}; + + for (const newPair of newPairs) { + const defaultPair = defaultPairs.find((p) => p.key === newPair.key); + if (defaultPair && JSON.stringify(defaultPair.value) !== JSON.stringify(newPair.value) && !ignoredKeys.includes(newPair.key)) { + changes[newPair.key] = { type: "changed", old: defaultPair.value, new: newPair.value }; + } else if (!defaultPair) { + changes[newPair.key] = { type: "unknown", new: newPair.value }; + } + } + return changes; }