diff --git a/extra/admin-api/Tests/Spacebar.Tests/Tests/MessageTests.cs b/extra/admin-api/Tests/Spacebar.Tests/Tests/MessageTests.cs index d563b244a..e87d7f8fc 100644 --- a/extra/admin-api/Tests/Spacebar.Tests/Tests/MessageTests.cs +++ b/extra/admin-api/Tests/Spacebar.Tests/Tests/MessageTests.cs @@ -181,7 +181,7 @@ public class MessageTests(ITestOutputHelper testOutputHelper, TestFixture fixtur } public static IEnumerable WebhookExecuteCombinations() { - string[] contents = ["meow", "# hi!!!", "https://spacebar.chat/favicon.ico", "@everyone", "@here"]; + string[] contents = ["meow", "# hi!!!", Client.ClientWellKnown.Api.BaseUrl, "@everyone", "@here"]; bool?[] ttsEnabled = [null, true, false]; int?[] messageFlags = [ null, diff --git a/extra/admin-api/Tests/Spacebar.Tests/Tests/WebhookTests.cs b/extra/admin-api/Tests/Spacebar.Tests/Tests/WebhookTests.cs index 682192169..965f2f05f 100644 --- a/extra/admin-api/Tests/Spacebar.Tests/Tests/WebhookTests.cs +++ b/extra/admin-api/Tests/Spacebar.Tests/Tests/WebhookTests.cs @@ -119,9 +119,9 @@ public class WebhookTests(ITestOutputHelper testOutputHelper, TestFixture fixtur } public static IEnumerable WebhookExecuteCombinations() { - string[] contents = ["meow", "# hi!!!", "https://spacebar.chat/favicon.ico", "@everyone", "@here"]; + string[] contents = ["meow", "# hi!!!", Client.ClientWellKnown.Api.BaseUrl, "@everyone", "@here"]; string?[] usernames = [null, "meow"]; - string?[] avatarUrls = [null, "https://spacebar.chat/favicon.ico"]; + string?[] avatarUrls = [null, Client.ClientWellKnown.Api.BaseUrl + "/static/logo.png"]; bool?[] ttsEnabled = [null, true, false]; int?[] messageFlags = [ null, @@ -157,7 +157,7 @@ public class WebhookTests(ITestOutputHelper testOutputHelper, TestFixture fixtur public async Task SendWebhookMessageWithAvatarUrl() { await Assert.SuccessfullyHttpPostAsJsonAsync(Webhook.Url + "?wait=true", new JsonObject() { { "content", "meow" }, - { "avatar_url", "https://spacebar.chat/favicon.ico" } + { "avatar_url", Client.ClientWellKnown.Api.BaseUrl + "/static/logo.png" } }); } } \ No newline at end of file diff --git a/nix/modules/default/default.nix b/nix/modules/default/default.nix index 6c6437a85..12abfe777 100644 --- a/nix/modules/default/default.nix +++ b/nix/modules/default/default.nix @@ -127,6 +127,7 @@ in ); serviceConfig = { ExecStart = "${cfg.package}/bin/start-api"; + Type = "notify"; }; }; @@ -151,6 +152,7 @@ in ); serviceConfig = { ExecStart = "${cfg.package}/bin/start-gateway"; + Type = "notify"; }; }; @@ -174,6 +176,7 @@ in ); serviceConfig = { ExecStart = "${cfg.package}/bin/start-cdn"; + Type = "notify"; }; }); }; diff --git a/nix/modules/default/pion-sfu.nix b/nix/modules/default/pion-sfu.nix index 92bcf5330..cd8c88cb1 100644 --- a/nix/modules/default/pion-sfu.nix +++ b/nix/modules/default/pion-sfu.nix @@ -69,6 +69,7 @@ in ); serviceConfig = { ExecStart = "${cfg.package}/bin/start-webrtc"; + Type = "notify"; }; }; diff --git a/nix/tests/test-bundle-starts.nix b/nix/tests/test-bundle-starts.nix index 8fd6b98e9..b2be0bb86 100644 --- a/nix/tests/test-bundle-starts.nix +++ b/nix/tests/test-bundle-starts.nix @@ -12,6 +12,16 @@ let sb = import ../lib/mkEndpoint.nix; isRabbitMqTest = lib.strings.hasPrefix "rabbitmq" withIpc; + + testBin = lib.getExe self.outputs.packages.${pkgs.stdenv.system}.Spacebar-Tests; + testConfigPath = pkgs.writeText "Spacebar-Tests-appsettings.json" ( + builtins.toJSON { + Configuration = { + TestInstance = "http://localhost:3001"; + RegisterConcurrentCount = 150; + }; + } + ); in { name = "test-bundle-starts" + lib.optionalString (withIpc != "unix") ("_ipc=" + withIpc); @@ -60,6 +70,45 @@ in lib.trace ("Testing with config: " + builtins.toJSON cfg) cfg; services.nginx.enable = true; services.rabbitmq.enable = isRabbitMqTest; + + # ...fix startup ordering + systemd.services = + let + services = [ "postgresql.service" ] ++ lib.optional (isRabbitMqTest) [ "rabbitmq.service" ]; + serviceDef = { + after = services; + wants = services; + }; + in + { + "spacebar-api" = serviceDef; + "spacebar-cdn" = serviceDef; + "spacebar-gateway" = serviceDef; + "spacebar-webrtc" = serviceDef; + + "spacebar-tests" = { + documentation = [ "https://docs.spacebar.chat/" ]; + wantedBy = [ "multi-user.target" ]; + wants = [ "network-online.target" ]; + after = [ + "network-online.target" + "spacebar-api.service" + ]; + requires = [ "spacebar-api.service" ]; + environment = { + TEST_APPSETTINGS_PATH=testConfigPath; + }; + serviceConfig = { + ExecStart = "${testBin} -reporter verbose -parallelAlgorithm aggressive -maxThreads unlimited -preEnumerateTheories"; + DynamicUser = true; + RestrictSUIDSGID = true; + + Restart = "no"; + UMask = "077"; + }; + }; + }; + services.postgresql = { enable = true; initdbArgs = [ @@ -86,37 +135,28 @@ in # https://nixos.org/manual/nixos/stable/index.html#sec-nixos-tests # https://nixos.org/manual/nixpkgs/unstable/#tester-runNixOSTest - testScript = - let - testBin = lib.getExe self.outputs.packages.${pkgs.stdenv.system}.Spacebar-Tests; - testConfigPath = pkgs.writeText "Spacebar-Tests-appsettings.json" ( - builtins.toJSON { - Configuration = { - TestInstance = "http://localhost:3001"; - RegisterConcurrentCount = 150; - }; - } - ); - in - '' - machine.wait_for_unit("spacebar-api") - machine.wait_for_unit("spacebar-cdn") - machine.wait_for_unit("spacebar-gateway") - # Wait for unit doesn't mean the service is actually ready to accept connections - machine.wait_for_open_port(80) - machine.wait_for_open_port(3001) - machine.wait_for_open_port(3002) - machine.wait_for_open_port(3003) + testScript = '' + machine.wait_for_unit("spacebar-api") + machine.wait_for_unit("spacebar-cdn") + machine.wait_for_unit("spacebar-gateway") + # Wait for unit doesn't mean the service is actually ready to accept connections + machine.wait_for_open_port(80) + machine.wait_for_open_port(3001) + machine.wait_for_open_port(3002) + machine.wait_for_open_port(3003) - # this should be working - machine.succeed("curl -f http://api.sb.localhost/.well-known/spacebar/client") + # this should be working + machine.succeed("curl -f http://api.sb.localhost/.well-known/spacebar/client") - # check if metrics endpoint works on all services - machine.succeed("curl -f http://api.sb.localhost/metrics") - machine.succeed("curl -f http://gateway.sb.localhost/metrics") - machine.succeed("curl -f http://cdn.sb.localhost/metrics") + # check if metrics endpoint works on all services + machine.succeed("curl -f http://api.sb.localhost/metrics") + machine.succeed("curl -f http://gateway.sb.localhost/metrics") + machine.succeed("curl -f http://cdn.sb.localhost/metrics") - # run integration tests - machine.succeed("/usr/bin/env TEST_APPSETTINGS_PATH=${testConfigPath} ${testBin} -reporter verbose -parallelAlgorithm aggressive -maxThreads unlimited -preEnumerateTheories") - ''; + machine.wait_for_unit("spacebar-tests") + machine.wait_until_fails("systemctl show spacebar-tests.service | grep 'SubState=running' -q") + # run integration tests + # machine.succeed("/usr/bin/env TEST_APPSETTINGS_PATH=${testConfigPath} ${testBin} -reporter verbose -parallelAlgorithm aggressive -maxThreads unlimited -preEnumerateTheories") + + ''; } diff --git a/package-lock.json b/package-lock.json index 64c4ebc7c..627f267fb 100644 Binary files a/package-lock.json and b/package-lock.json differ diff --git a/package.json b/package.json index 8610aa66e..cfaf821d7 100644 --- a/package.json +++ b/package.json @@ -111,6 +111,7 @@ "multer": "^2.1.1", "murmurhash-js": "^1.0.0", "node-2fa": "^2.0.3", + "node-unix-socket": "^0.2.7", "pg": "^8.21.0", "pg-query-stream": "^4.15.0", "picocolors": "^1.1.1", diff --git a/src/api/Server.ts b/src/api/Server.ts index 83f73367b..84503d6eb 100644 --- a/src/api/Server.ts +++ b/src/api/Server.ts @@ -35,7 +35,7 @@ import { pendingPolls, JwtKeypairManager, } from "@spacebar/util"; -import { ProcessLifecycle } from "../util/util/ProcessLifecycle"; +import { ProcessLifecycle, SystemdLifecycle } from "../util/util/ProcessLifecycle"; import { Monitoring } from "../util/monitoring/Monitoring"; import { BcryptWorkerPool } from "../util/util/workers/bcrypt/BcryptWorkerPool"; import { Authentication, CORS, ImageProxy, BodyParser, ErrorHandler, initRateLimits, initTranslation } from "./middlewares"; @@ -238,7 +238,8 @@ export class SpacebarServer extends Server { if (logRequests) console.log(red(`Warning: Request logging is enabled! This will spam your console!\nTo disable this, unset the 'LOG_REQUESTS' environment variable!`)); + await super.start(); + await SystemdLifecycle.setStatus(`Listening on ${this.options.host}:${this.options.port}...`); await ProcessLifecycle.Ready(); - return super.start(); } } diff --git a/src/cdn/Server.ts b/src/cdn/Server.ts index a6a4330e0..622e0911b 100644 --- a/src/cdn/Server.ts +++ b/src/cdn/Server.ts @@ -22,7 +22,7 @@ import { Server, ServerOptions } from "lambert-server/Server"; import { CORS, BodyParser } from "@spacebar/api/middlewares"; import { Attachment, initDatabase } from "@spacebar/database"; import { Config, registerRoutes } from "@spacebar/util"; -import { ProcessLifecycle } from "../util/util/ProcessLifecycle"; +import { ProcessLifecycle, SystemdLifecycle } from "../util/util/ProcessLifecycle"; import { Monitoring } from "../util/monitoring/Monitoring"; import guildProfilesRoute from "./routes/guild-profiles"; import { storage } from "./util"; @@ -76,8 +76,9 @@ export class CDNServer extends Server { this.app.use("/guilds/:guild_id/users/:user_id/banners", guildProfilesRoute); if (process.env.LOG_ROUTES !== "false") console.log("[Server] Route /guilds/:guild_id/users/:user_id/banners registered"); + await super.start(); + await SystemdLifecycle.setStatus(`Listening on ${this.options.host}:${this.options.port}...`); await ProcessLifecycle.Ready(); - return super.start(); } async migrateAttachments() { diff --git a/src/gateway/Server.ts b/src/gateway/Server.ts index 56014f265..89ecc8eb9 100644 --- a/src/gateway/Server.ts +++ b/src/gateway/Server.ts @@ -22,7 +22,7 @@ import ws from "ws"; import { initDatabase } from "@spacebar/database"; import { Random } from "@spacebar/extensions"; import { checkToken, Config, initEvent, JwtKeypairManager, Rights } from "@spacebar/util"; -import { ProcessLifecycle } from "../util/util/ProcessLifecycle"; +import { ProcessLifecycle, SystemdLifecycle } from "../util/util/ProcessLifecycle"; import { Monitoring } from "../util/monitoring/Monitoring"; import { Connection, openConnections } from "./events/Connection"; import { cleanupOnStartup } from "./util"; @@ -187,6 +187,7 @@ export class Server { if (!this.server.listening) { this.server.listen(this.port); console.log(`[Gateway] online on 0.0.0.0:${this.port}`); + await SystemdLifecycle.setStatus(`Listening on 0.0.0.0:${this.port}...`); } await ProcessLifecycle.Ready(); diff --git a/src/util/util/ProcessLifecycle.ts b/src/util/util/ProcessLifecycle.ts index 6b4a6d778..f625ae77d 100644 --- a/src/util/util/ProcessLifecycle.ts +++ b/src/util/util/ProcessLifecycle.ts @@ -18,6 +18,9 @@ import EventEmitter from "node:events"; import whyIsNodeRunning from "why-is-node-running"; +import net from "node:net"; +import * as dgram from "node:dgram"; +import { DgramSocket } from "node-unix-socket"; interface ProcessLifecycleEvents { starting: unknown[]; @@ -33,10 +36,12 @@ export class ProcessLifecycle { // to be ran after startup is finished static async Ready() { await this.emitAsync((this.state = "running")); + await SystemdLifecycle.sendReady(); } // to be ran at the start of shutdown static async Shutdown() { + await SystemdLifecycle.sendStopping(); await this.emitAsync((this.state = "stopping")); } @@ -60,3 +65,33 @@ process.on("SIGUSR1", () => { whyIsNodeRunning(); console.log("\nProcess state:", ProcessLifecycle.state); }); + +export class SystemdLifecycle { + private static writeData(data: string): Promise { + const socketPath = process.env.NOTIFY_SOCKET; + if (!socketPath) return Promise.resolve(); + + const buf = Buffer.from(data); + console.log("Systemd notify socket path:", socketPath, "-", buf.length, "bytes"); + + return new Promise((res, rej) => { + new DgramSocket().sendTo(buf, 0, buf.length, socketPath, (err) => { + if (err) rej(err); + res(); + }); + }); + } + static async sendReady() { + await this.writeData("READY=1"); + } + + static async sendStopping() { + await this.writeData("STOPPING=1"); + } + + static async setStatus(status: string) { + await this.writeData("STATUS=" + status); + } + + // TODO: do we want to support the watchdog? +} diff --git a/src/webrtc/Server.ts b/src/webrtc/Server.ts index d86d8eaa9..a46d1293d 100644 --- a/src/webrtc/Server.ts +++ b/src/webrtc/Server.ts @@ -21,7 +21,7 @@ import ws from "ws"; import { green, yellow } from "picocolors"; import { initDatabase } from "@spacebar/database"; import { Config, initEvent, JwtKeypairManager } from "@spacebar/util"; -import { ProcessLifecycle } from "../util/util/ProcessLifecycle"; +import { ProcessLifecycle, SystemdLifecycle } from "../util/util/ProcessLifecycle"; import { Monitoring } from "../util/monitoring/Monitoring"; import { Connection } from "./events/Connection"; import { loadWebRtcLibrary, mediaServer, WRTC_PORT_MAX, WRTC_PORT_MIN, WRTC_PUBLIC_IP } from "./util"; @@ -82,6 +82,7 @@ export class Server { if (!this.server.listening) { this.server.listen(this.port); console.log(`[WebRTC] ${green(`online on 0.0.0.0:${this.port}`)}`); + await SystemdLifecycle.setStatus(`Listening on 0.0.0.0:${this.port}...`); } await ProcessLifecycle.Ready();