Systemd notify hooks, move nixos integration tests to a systemd service

extra/admin-api/Tests/Spacebar.Tests/Tests/MessageTests.cs

@@ -181,7 +181,7 @@ public class MessageTests(ITestOutputHelper testOutputHelper, TestFixture fixtur
     }
 
     public static IEnumerable<object?[]> WebhookExecuteCombinations() {
-        string[] contents = ["meow", "# hi!!!", "https://spacebar.chat/favicon.ico", "@everyone", "@here"];
+        string[] contents = ["meow", "# hi!!!", Client.ClientWellKnown.Api.BaseUrl, "@everyone", "@here"];
         bool?[] ttsEnabled = [null, true, false];
         int?[] messageFlags = [
             null,

extra/admin-api/Tests/Spacebar.Tests/Tests/WebhookTests.cs

@@ -119,9 +119,9 @@ public class WebhookTests(ITestOutputHelper testOutputHelper, TestFixture fixtur
     }
 
     public static IEnumerable<object?[]> WebhookExecuteCombinations() {
-        string[] contents = ["meow", "# hi!!!", "https://spacebar.chat/favicon.ico", "@everyone", "@here"];
+        string[] contents = ["meow", "# hi!!!", Client.ClientWellKnown.Api.BaseUrl, "@everyone", "@here"];
         string?[] usernames = [null, "meow"];
-        string?[] avatarUrls = [null, "https://spacebar.chat/favicon.ico"];
+        string?[] avatarUrls = [null, Client.ClientWellKnown.Api.BaseUrl + "/static/logo.png"];
         bool?[] ttsEnabled = [null, true, false];
         int?[] messageFlags = [
             null,
@@ -157,7 +157,7 @@ public class WebhookTests(ITestOutputHelper testOutputHelper, TestFixture fixtur
     public async Task SendWebhookMessageWithAvatarUrl() {
         await Assert.SuccessfullyHttpPostAsJsonAsync(Webhook.Url + "?wait=true", new JsonObject() {
             { "content", "meow" },
-            { "avatar_url", "https://spacebar.chat/favicon.ico" }
+            { "avatar_url", Client.ClientWellKnown.Api.BaseUrl + "/static/logo.png" }
         });
     }
 }

nix/modules/default/default.nix

@@ -127,6 +127,7 @@ in
       );
       serviceConfig = {
         ExecStart = "${cfg.package}/bin/start-api";
+        Type = "notify";
       };
     };
 
@@ -151,6 +152,7 @@ in
       );
       serviceConfig = {
         ExecStart = "${cfg.package}/bin/start-gateway";
+        Type = "notify";
       };
     };
 
@@ -174,6 +176,7 @@ in
       );
       serviceConfig = {
         ExecStart = "${cfg.package}/bin/start-cdn";
+        Type = "notify";
       };
     });
   };

nix/modules/default/pion-sfu.nix

@@ -69,6 +69,7 @@ in
       );
       serviceConfig = {
         ExecStart = "${cfg.package}/bin/start-webrtc";
+        Type = "notify";
       };
     };
 

nix/tests/test-bundle-starts.nix

@@ -12,6 +12,16 @@
 let
   sb = import ../lib/mkEndpoint.nix;
   isRabbitMqTest = lib.strings.hasPrefix "rabbitmq" withIpc;
+
+  testBin = lib.getExe self.outputs.packages.${pkgs.stdenv.system}.Spacebar-Tests;
+  testConfigPath = pkgs.writeText "Spacebar-Tests-appsettings.json" (
+    builtins.toJSON {
+      Configuration = {
+        TestInstance = "http://localhost:3001";
+        RegisterConcurrentCount = 150;
+      };
+    }
+  );
 in
 {
   name = "test-bundle-starts" + lib.optionalString (withIpc != "unix") ("_ipc=" + withIpc);
@@ -60,6 +70,45 @@ in
       lib.trace ("Testing with config: " + builtins.toJSON cfg) cfg;
     services.nginx.enable = true;
     services.rabbitmq.enable = isRabbitMqTest;
+
+    # ...fix startup ordering
+    systemd.services =
+      let
+        services = [ "postgresql.service" ] ++ lib.optional (isRabbitMqTest) [ "rabbitmq.service" ];
+        serviceDef = {
+          after = services;
+          wants = services;
+        };
+      in
+      {
+        "spacebar-api" = serviceDef;
+        "spacebar-cdn" = serviceDef;
+        "spacebar-gateway" = serviceDef;
+        "spacebar-webrtc" = serviceDef;
+
+        "spacebar-tests" = {
+          documentation = [ "https://docs.spacebar.chat/" ];
+          wantedBy = [ "multi-user.target" ];
+          wants = [ "network-online.target" ];
+          after = [
+            "network-online.target"
+            "spacebar-api.service"
+          ];
+          requires = [ "spacebar-api.service" ];
+          environment = {
+            TEST_APPSETTINGS_PATH=testConfigPath;
+          };
+          serviceConfig = {
+            ExecStart = "${testBin} -reporter verbose -parallelAlgorithm aggressive -maxThreads unlimited -preEnumerateTheories";
+            DynamicUser = true;
+            RestrictSUIDSGID = true;
+
+            Restart = "no";
+            UMask = "077";
+          };
+        };
+      };
+
     services.postgresql = {
       enable = true;
       initdbArgs = [
@@ -86,37 +135,28 @@ in
 
   # https://nixos.org/manual/nixos/stable/index.html#sec-nixos-tests
   # https://nixos.org/manual/nixpkgs/unstable/#tester-runNixOSTest
-  testScript =
-    let
-      testBin = lib.getExe self.outputs.packages.${pkgs.stdenv.system}.Spacebar-Tests;
-      testConfigPath = pkgs.writeText "Spacebar-Tests-appsettings.json" (
-        builtins.toJSON {
-          Configuration = {
-            TestInstance = "http://localhost:3001";
-            RegisterConcurrentCount = 150;
-          };
-        }
-      );
-    in
-    ''
-      machine.wait_for_unit("spacebar-api")
-      machine.wait_for_unit("spacebar-cdn")
-      machine.wait_for_unit("spacebar-gateway")
-      # Wait for unit doesn't mean the service is actually ready to accept connections
-      machine.wait_for_open_port(80)
-      machine.wait_for_open_port(3001)
-      machine.wait_for_open_port(3002)
-      machine.wait_for_open_port(3003)
+  testScript = ''
+    machine.wait_for_unit("spacebar-api")
+    machine.wait_for_unit("spacebar-cdn")
+    machine.wait_for_unit("spacebar-gateway")
+    # Wait for unit doesn't mean the service is actually ready to accept connections
+    machine.wait_for_open_port(80)
+    machine.wait_for_open_port(3001)
+    machine.wait_for_open_port(3002)
+    machine.wait_for_open_port(3003)
 
-      # this should be working
-      machine.succeed("curl -f http://api.sb.localhost/.well-known/spacebar/client")
+    # this should be working
+    machine.succeed("curl -f http://api.sb.localhost/.well-known/spacebar/client")
 
-      # check if metrics endpoint works on all services
-      machine.succeed("curl -f http://api.sb.localhost/metrics")
-      machine.succeed("curl -f http://gateway.sb.localhost/metrics")
-      machine.succeed("curl -f http://cdn.sb.localhost/metrics")
+    # check if metrics endpoint works on all services
+    machine.succeed("curl -f http://api.sb.localhost/metrics")
+    machine.succeed("curl -f http://gateway.sb.localhost/metrics")
+    machine.succeed("curl -f http://cdn.sb.localhost/metrics")
 
-      # run integration tests
-      machine.succeed("/usr/bin/env TEST_APPSETTINGS_PATH=${testConfigPath} ${testBin} -reporter verbose -parallelAlgorithm aggressive -maxThreads unlimited -preEnumerateTheories")
-    '';
+    machine.wait_for_unit("spacebar-tests")
+    machine.wait_until_fails("systemctl show spacebar-tests.service | grep 'SubState=running' -q")
+    # run integration tests
+    # machine.succeed("/usr/bin/env TEST_APPSETTINGS_PATH=${testConfigPath} ${testBin} -reporter verbose -parallelAlgorithm aggressive -maxThreads unlimited -preEnumerateTheories")
+
+  '';
 }

package.json

@@ -111,6 +111,7 @@
         "multer": "^2.1.1",
         "murmurhash-js": "^1.0.0",
         "node-2fa": "^2.0.3",
+        "node-unix-socket": "^0.2.7",
         "pg": "^8.21.0",
         "pg-query-stream": "^4.15.0",
         "picocolors": "^1.1.1",

src/api/Server.ts

@@ -35,7 +35,7 @@ import {
     pendingPolls,
     JwtKeypairManager,
 } from "@spacebar/util";
-import { ProcessLifecycle } from "../util/util/ProcessLifecycle";
+import { ProcessLifecycle, SystemdLifecycle } from "../util/util/ProcessLifecycle";
 import { Monitoring } from "../util/monitoring/Monitoring";
 import { BcryptWorkerPool } from "../util/util/workers/bcrypt/BcryptWorkerPool";
 import { Authentication, CORS, ImageProxy, BodyParser, ErrorHandler, initRateLimits, initTranslation } from "./middlewares";
@@ -238,7 +238,8 @@ export class SpacebarServer extends Server {
 
         if (logRequests) console.log(red(`Warning: Request logging is enabled! This will spam your console!\nTo disable this, unset the 'LOG_REQUESTS' environment variable!`));
 
+        await super.start();
+        await SystemdLifecycle.setStatus(`Listening on ${this.options.host}:${this.options.port}...`);
         await ProcessLifecycle.Ready();
-        return super.start();
     }
 }

src/cdn/Server.ts

@@ -22,7 +22,7 @@ import { Server, ServerOptions } from "lambert-server/Server";
 import { CORS, BodyParser } from "@spacebar/api/middlewares";
 import { Attachment, initDatabase } from "@spacebar/database";
 import { Config, registerRoutes } from "@spacebar/util";
-import { ProcessLifecycle } from "../util/util/ProcessLifecycle";
+import { ProcessLifecycle, SystemdLifecycle } from "../util/util/ProcessLifecycle";
 import { Monitoring } from "../util/monitoring/Monitoring";
 import guildProfilesRoute from "./routes/guild-profiles";
 import { storage } from "./util";
@@ -76,8 +76,9 @@ export class CDNServer extends Server {
         this.app.use("/guilds/:guild_id/users/:user_id/banners", guildProfilesRoute);
         if (process.env.LOG_ROUTES !== "false") console.log("[Server] Route /guilds/:guild_id/users/:user_id/banners registered");
 
+        await super.start();
+        await SystemdLifecycle.setStatus(`Listening on ${this.options.host}:${this.options.port}...`);
         await ProcessLifecycle.Ready();
-        return super.start();
     }
 
     async migrateAttachments() {

src/gateway/Server.ts

@@ -22,7 +22,7 @@ import ws from "ws";
 import { initDatabase } from "@spacebar/database";
 import { Random } from "@spacebar/extensions";
 import { checkToken, Config, initEvent, JwtKeypairManager, Rights } from "@spacebar/util";
-import { ProcessLifecycle } from "../util/util/ProcessLifecycle";
+import { ProcessLifecycle, SystemdLifecycle } from "../util/util/ProcessLifecycle";
 import { Monitoring } from "../util/monitoring/Monitoring";
 import { Connection, openConnections } from "./events/Connection";
 import { cleanupOnStartup } from "./util";
@@ -187,6 +187,7 @@ export class Server {
         if (!this.server.listening) {
             this.server.listen(this.port);
             console.log(`[Gateway] online on 0.0.0.0:${this.port}`);
+            await SystemdLifecycle.setStatus(`Listening on 0.0.0.0:${this.port}...`);
         }
 
         await ProcessLifecycle.Ready();

src/util/util/ProcessLifecycle.ts

@@ -18,6 +18,9 @@
 
 import EventEmitter from "node:events";
 import whyIsNodeRunning from "why-is-node-running";
+import net from "node:net";
+import * as dgram from "node:dgram";
+import { DgramSocket } from "node-unix-socket";
 
 interface ProcessLifecycleEvents {
     starting: unknown[];
@@ -33,10 +36,12 @@ export class ProcessLifecycle {
     // to be ran after startup is finished
     static async Ready() {
         await this.emitAsync((this.state = "running"));
+        await SystemdLifecycle.sendReady();
     }
 
     // to be ran at the start of shutdown
     static async Shutdown() {
+        await SystemdLifecycle.sendStopping();
         await this.emitAsync((this.state = "stopping"));
     }
 
@@ -60,3 +65,33 @@ process.on("SIGUSR1", () => {
     whyIsNodeRunning();
     console.log("\nProcess state:", ProcessLifecycle.state);
 });
+
+export class SystemdLifecycle {
+    private static writeData(data: string): Promise<void> {
+        const socketPath = process.env.NOTIFY_SOCKET;
+        if (!socketPath) return Promise.resolve();
+
+        const buf = Buffer.from(data);
+        console.log("Systemd notify socket path:", socketPath, "-", buf.length, "bytes");
+
+        return new Promise((res, rej) => {
+            new DgramSocket().sendTo(buf, 0, buf.length, socketPath, (err) => {
+                if (err) rej(err);
+                res();
+            });
+        });
+    }
+    static async sendReady() {
+        await this.writeData("READY=1");
+    }
+
+    static async sendStopping() {
+        await this.writeData("STOPPING=1");
+    }
+
+    static async setStatus(status: string) {
+        await this.writeData("STATUS=" + status);
+    }
+
+    // TODO: do we want to support the watchdog?
+}

src/webrtc/Server.ts

@@ -21,7 +21,7 @@ import ws from "ws";
 import { green, yellow } from "picocolors";
 import { initDatabase } from "@spacebar/database";
 import { Config, initEvent, JwtKeypairManager } from "@spacebar/util";
-import { ProcessLifecycle } from "../util/util/ProcessLifecycle";
+import { ProcessLifecycle, SystemdLifecycle } from "../util/util/ProcessLifecycle";
 import { Monitoring } from "../util/monitoring/Monitoring";
 import { Connection } from "./events/Connection";
 import { loadWebRtcLibrary, mediaServer, WRTC_PORT_MAX, WRTC_PORT_MIN, WRTC_PUBLIC_IP } from "./util";
@@ -82,6 +82,7 @@ export class Server {
         if (!this.server.listening) {
             this.server.listen(this.port);
             console.log(`[WebRTC] ${green(`online on 0.0.0.0:${this.port}`)}`);
+            await SystemdLifecycle.setStatus(`Listening on 0.0.0.0:${this.port}...`);
         }
 
         await ProcessLifecycle.Ready();