nix: add gateway sharding options

nix/modules/default/default.nix

@@ -17,6 +17,7 @@ in
   imports = [
     ./integration-nginx.nix
     ./users.nix
+    (import ./gw-sharding.nix self)
     (import ./pion-sfu.nix self)
     (import ./cs/cdn-cs.nix self)
     (import ./cs/gateway-offload-cs.nix self)

nix/modules/default/gw-sharding.nix

@@ -0,0 +1,59 @@
+self:
+{
+  config,
+  lib,
+  pkgs,
+  spacebar,
+  ...
+}:
+
+let
+  secrets = import ./secrets.nix { inherit lib config; };
+  cfg = config.services.spacebarchat-server;
+  jsonFormat = pkgs.formats.json { };
+  configFile = (import ./config-file.nix { inherit config lib pkgs; });
+in
+{
+  options.services.spacebarchat-server = {
+    extraGatewayPorts = lib.mkOption {
+      type = lib.types.listOf lib.types.port;
+      description = "Extra gateway ports";
+    };
+  };
+
+  config = lib.mkIf cfg.enable (
+    let
+      makeServerTsService = import ./makeServerTsService.nix { inherit cfg lib secrets; };
+    in
+    {
+      systemd.services = builtins.listToAttrs (
+        map (port: {
+          name = "spacebar-gateway-${toString port}";
+          value = makeServerTsService {
+            description = "Spacebar Server - Gateway (extra port ${toString port})";
+            # after = [ "spacebar-api.service" ];
+            environment = builtins.mapAttrs (_: val: builtins.toString val) (
+              {
+                # things we set by default...
+                EVENT_TRANSMISSION = "unix";
+                EVENT_SOCKET_PATH = "/run/spacebar/";
+              }
+              // cfg.extraEnvironment
+              // {
+                # things we force...
+                CONFIG_PATH = configFile;
+                CONFIG_READONLY = 1;
+                PORT = toString port;
+                STORAGE_LOCATION = cfg.cdnPath;
+                APPLY_DB_MIGRATIONS = "false";
+              }
+            );
+            serviceConfig = {
+              ExecStart = "${cfg.package}/bin/start-gateway";
+            };
+          };
+        }) cfg.extraGatewayPorts
+      );
+    }
+  );
+}

nix/modules/default/integration-nginx.nix

@@ -16,12 +16,40 @@ in
   config = lib.mkIf (cfg.enable && cfg.nginx.enable) {
     services.nginx = {
       recommendedProxySettings = true;
+      upstreams = {
+        "spacebar-api" = {
+          servers = {
+            "127.0.0.1:${if cfg.uApi.enable then toString cfg.uApi.listenPort else toString cfg.apiEndpoint.localPort}" = { };
+          };
+        };
+        "spacebar-gateway" = {
+          servers = {
+            "127.0.0.1:${toString cfg.gatewayEndpoint.localPort}" = { };
+          }
+          // builtins.listToAttrs (
+            map (port: {
+              name = "127.0.0.1:${toString port}";
+              value = { };
+            }) cfg.extraGatewayPorts
+          );
+        };
+        "spacebar-cdn" = {
+          servers = {
+            "127.0.0.1:${toString cfg.cdnEndpoint.localPort}" = { };
+          };
+        };
+        "spacebar-admin" = lib.mkIf cfg.adminApi.enable {
+          servers = {
+            "127.0.0.1:${toString cfg.adminApiEndpoint.localPort}" = { };
+          };
+        };
+      };
       virtualHosts = lib.mkIf cfg.enable {
         "${cfg.apiEndpoint.host}" = {
           enableACME = cfg.apiEndpoint.useSsl;
           forceSSL = cfg.apiEndpoint.useSsl;
           locations."/" = {
-            proxyPass = if cfg.uApi.enable then "http://127.0.0.1:${toString cfg.uApi.listenPort}/" else "http://127.0.0.1:${toString cfg.apiEndpoint.localPort}/";
+            proxyPass = "http://spacebar-api/";
           };
         };
         "${cfg.gatewayEndpoint.host}" = {
@@ -29,21 +57,21 @@ in
           forceSSL = cfg.gatewayEndpoint.useSsl;
           locations."/" = {
             proxyWebsockets = true;
-            proxyPass = "http://127.0.0.1:${toString cfg.gatewayEndpoint.localPort}/";
+            proxyPass = "http://spacebar-gateway/";
           };
         };
         "${cfg.cdnEndpoint.host}" = {
           enableACME = cfg.cdnEndpoint.useSsl;
           forceSSL = cfg.cdnEndpoint.useSsl;
           locations."/" = {
-            proxyPass = "http://127.0.0.1:${toString cfg.cdnEndpoint.localPort}/";
+            proxyPass = "http://spacebar-cdn/";
           };
         };
         "${cfg.adminApiEndpoint.host}" = lib.mkIf cfg.adminApi.enable {
           enableACME = cfg.adminApiEndpoint.useSsl;
           forceSSL = cfg.adminApiEndpoint.useSsl;
           locations."/" = {
-            proxyPass = "http://127.0.0.1:${toString cfg.adminApiEndpoint.localPort}/";
+            proxyPass = "http://spacebar-admin/";
           };
         };
       };

nix/testVm/configuration.nix

@@ -31,6 +31,7 @@ in
         enable = true;
         apiEndpoint = sbLib.mkEndpointRaw "api.sb.localhost" 3001 8080 false;
         gatewayEndpoint = sbLib.mkEndpointRaw "gw.sb.localhost" 3002 8080 false;
+        extraGatewayPorts = lib.range 3100 3116;
         cdnEndpoint = sbLib.mkEndpointRaw "cdn.sb.localhost" 3003 8080 false;
         adminApiEndpoint = sbLib.mkEndpointRaw "admin.sb.localhost" 3004 8080 false;
         webrtcEndpoint = sbLib.mkEndpointRaw "voice.sb.localhost" 3005 8080 false;

nix/testVm/vm.nix

@@ -17,8 +17,8 @@
       "-display gtk,zoom-to-fit=off,show-cursor=on"
       "-device virtio-balloon"
     ];
-    virtualisation.memorySize = 4096;
-    virtualisation.cores = 6;
+    virtualisation.memorySize = 8192;
+    virtualisation.cores = 10;
     virtualisation.forwardPorts = [
         # { hostPort = 2222; guestPort = 22; } # Probably shouldn't do this with root:root lol
         { from = "host"; host.port = 8080; guest.port = 80; }