Demo of overlay-able runtime configuration

application.nix

@@ -27,6 +27,7 @@ rec {
         ./application/playos-status.nix
         ./application/power-management/default.nix
         ./application/limit-vtes.nix
+        (import ./application/runtime-config.nix { inherit pkgs; }).module
       ];
 
       # Kiosk runs as a non-privileged user
@@ -45,6 +46,20 @@ rec {
         group = "users";
       };
 
+      playos.runtimeConfig.config = {
+          kiosk = {
+              # TODO: config.playos.kioskUrl should be removed in favour of this
+              # but not dealing with this now
+              url = config.playos.kioskUrl;
+              remote_debug_listen = "127.0.0.1:3355";
+          };
+          controller = {
+              port = 3333;
+          };
+      };
+
+
+
       # Limit virtual terminals that can be switched to
       # Virtual terminal 7 is the kiosk, 8 is the status screen
       playos.xserver.activeVirtualTerminals = [ 7 8 ];
@@ -53,7 +68,9 @@ rec {
       environment.systemPackages = with pkgs; [ breeze-contrast-cursor-theme ];
 
       # Kiosk session
-      services.xserver = let sessionName = "kiosk-browser";
+      services.xserver = let
+        sessionName = "kiosk-browser";
+        getCfgVal = cfgPath: "$(${config.playos.runtimeConfig.getValCmd cfgPath})";
       in {
         enable = true;
 
@@ -89,11 +106,13 @@ rec {
               esac
 
               # Enable Qt WebEngine Developer Tools (https://doc.qt.io/qt-6/qtwebengine-debugging.html)
-              export QTWEBENGINE_REMOTE_DEBUGGING="127.0.0.1:3355"
+              export QTWEBENGINE_REMOTE_DEBUGGING="${getCfgVal "kiosk.remote_debug_listen"}"
+
+              controller_port="${getCfgVal "controller.port"}"
 
               ${pkgs.playos-kiosk-browser}/bin/kiosk-browser \
-                ${config.playos.kioskUrl} \
-                http://localhost:3333/
+                "${getCfgVal "kiosk.url"}" \
+                "http://localhost:$controller_port"
 
               waitPID=$!
             '';

application/runtime-config.nix

@@ -0,0 +1,87 @@
+{ pkgs }:
+with pkgs.lib;
+let
+    # "private" function
+    generateTOML = fullConfig:
+        let
+            toml = (pkgs.formats.toml {});
+        in
+            toml.generate "config.toml" fullConfig;
+
+    runtimeConfigType = types.submodule { options = {
+        kiosk = {
+            url = mkOption {
+                type = types.nonEmptyStr;
+                example = "http://play.dividat.com";
+                description = "URL served by the Kiosk browser";
+            };
+            remote_debug_listen = mkOption {
+                type = types.str;
+                default = "127.0.0.1:3355";
+                example = "0.0.0.0:3355";
+                description = "TCP address for Qt WebEngine debug console";
+            };
+        };
+        controller = {
+            port = mkOption {
+                type = types.port;
+                default = 3333;
+                description = "Listen port of playos-controller";
+            };
+        };
+    }; };
+in
+rec {
+    # Produces a TOML file from a partial runtime config by merging it with
+    # defaults. Checks types via the module system.
+    # Used in tests to produce an overlay config outside of the normal NixOS
+    # module workflow.
+    mergeAndGenTOML = partialConfig:
+        let
+            # see https://github.com/NixOS/nixpkgs/pull/42838 for a similar
+            # approach to (ab)usage of modules for default merging and type checking
+            fullConfig = (evalModules { modules = [ {
+                config = { overlayCfg = partialConfig; };
+                options = { overlayCfg = mkOption { type = runtimeConfigType; }; };
+                _file = "overlay-config"; # only needed for err logs
+            } ]; }).config.overlayCfg;
+        in
+            generateTOML fullConfig;
+
+    module = { config, pkgs, lib, ... }: {
+        options = {
+            playos.runtimeConfig = {
+                config = mkOption {
+                    description = "PlayOS runtime configuration";
+                    type = runtimeConfigType;
+                };
+
+                getValCmd = mkOption {
+                    description = ''
+                        Helper function that given an attrset path returns a
+                        bash-snippet, which upon evaluation returns the provided
+                        runtime config value. Validates that the provided path
+                        corrends to an existing config key.
+                    '';
+                    readOnly = true;
+                    type = types.functionTo types.str;
+                };
+            };
+        };
+
+        config = {
+            environment.etc."playos-config.toml".source =
+                generateTOML config.playos.runtimeConfig.config;
+
+            playos.runtimeConfig.getValCmd = path: let
+                containsPath = attrsets.hasAttrByPath
+                    (strings.splitString "." path)
+                    config.playos.runtimeConfig.config;
+                in
+                # dynamic type checking
+                assert asserts.assertMsg containsPath
+                   "playos.runtimeConfig.config config does not specify a value at '${path}', check names";
+                "${pkgs.yq-go}/bin/yq eval -o toml -p toml .${path} /etc/playos-config.toml";
+        };
+    };
+}

build

@@ -116,8 +116,9 @@ elif [ "$TARGET" == "shed-key" ]; then
 
 elif [ "$TARGET" == "test-e2e" ]; then
 
+  # Note: updateUrl is still here because controller PR needs
+  # be merged to allow passing runtime configuration to it
   test_flags="
-    --arg kioskUrl http://10.0.2.99:8989/ \
     --arg updateUrl http://update-server.local/ \
     --arg buildVm false \
     --arg buildInstaller false \

testing/end-to-end/default.nix

@@ -2,12 +2,13 @@ args@{pkgs, disk, safeProductName, updateUrl, kioskUrl, ...}:
 with builtins;
 with pkgs.lib;
 let
+    runtimeConfig = pkgs.callPackage ../../application/runtime-config.nix {};
     overlayPath = "/tmp/playos-test-disk-overlay.qcow2";
     # fileFilter is recursive, so tests can in theory be in subfolders
     testFiles = fileset.fileFilter (file: file.hasExt "nix") ./tests;
     testPackages = map
         (file: pkgs.callPackage file
-            (args // { inherit overlayPath; })
+            (args // { inherit overlayPath runtimeConfig; })
         )
         (fileset.toList testFiles);
     testDeriv = pkgs.linkFarmFromDrvs "out" testPackages;

testing/end-to-end/profile.nix

@@ -9,6 +9,19 @@
         # don't need opengl for running tests, reduces image size vastly
         hardware.opengl.enable = false;
 
+        # Enable runtime configuration overrides without rebuilding the disk
+        fileSystems = {
+            "/tmp/extra-test-files" = {
+                device = "extra-test-files";
+                fsType = "9p";
+                options = [ "nofail" "trans=virtio" "version=9p2000.L" "cache=loose" ];
+            };
+            "/etc/playos-config.toml" = {
+                device = "/tmp/extra-test-files/playos-config.toml";
+                options = [ "bind" "nofail" ];
+            };
+        };
+
         # test-instrumentation.nix sets this in the boot as kernel param,
         # but since we are booting with a custom GRUB config it has no effect,
         # so instead we set this directly in journald

testing/end-to-end/tests/application/kiosk-persistence-helpers.py

@@ -14,13 +14,6 @@ def expose_local_port(vm, port):
         print(f"Port {port} already exposed")
         return
 
-    # enable NAT on loopback
-    vm.succeed("sysctl net.ipv4.conf.all.route_localnet=1")
-
-    # forward the port
-    vm.succeed("iptables -t nat -A PREROUTING -p tcp " + \
-                f"--dport {port} -j DNAT --to-destination 127.0.0.1:{port}")
-
     # open the port in the firewall
     vm.succeed(f"iptables -A INPUT -p tcp --dport {port} -j ACCEPT")
     vm.succeed("systemctl reload firewall")

testing/end-to-end/tests/application/kiosk-persistence.nix

@@ -1,13 +1,14 @@
-{pkgs, disk, overlayPath, kioskUrl, ...}:
+{pkgs, disk, overlayPath, runtimeConfig, ...}:
 let
     # currently hard-coded in application.nix
     guestCDPport = 3355;
     hostCDPport = 13355;
 
-    kioskParts = builtins.match "http://(.*):([0-9]+).*" kioskUrl;
-    guestKioskIP = builtins.elemAt kioskParts 0;
-    guestKioskURLport = pkgs.lib.strings.toInt (builtins.elemAt kioskParts 1);
+    guestKioskIP = "10.0.2.99";
+    guestKioskURLport = 8989;
     hostKioskURLport = 18989;
+
+    kioskUrl = "http://${guestKioskIP}:${toString guestKioskURLport}/";
 in
 pkgs.testers.runNixOSTest {
   name = "Kiosk's web storage persistence";
@@ -16,9 +17,17 @@ pkgs.testers.runNixOSTest {
     playos = { config, lib, pkgs, ... }:
     {
       imports = [
-        (import ../../virtualisation-config.nix { inherit overlayPath; })
+        ../../virtualisation-config.nix
       ];
       config = {
+        playos.e2e-tests.overlayPath = overlayPath;
+        playos.e2e-tests.overlayConfig = runtimeConfig.mergeAndGenTOML {
+            kiosk = {
+                url = kioskUrl;
+                remote_debug_listen = "0.0.0.0:3355";
+            };
+        };
+
         virtualisation.forwardPorts = [
             # CDP access inside of PlayOS VM from test driver
             {   from = "host";

testing/end-to-end/tests/base/factory-reset.nix

@@ -6,8 +6,11 @@ pkgs.testers.runNixOSTest {
     playos = { config, lib, pkgs, ... }:
     {
       imports = [
-        (import ../../virtualisation-config.nix { inherit overlayPath; })
+        ../../virtualisation-config.nix
       ];
+      config = {
+        playos.e2e-tests.overlayPath = overlayPath;
+      };
     };
   };
 

testing/end-to-end/tests/base/proxy-and-update.nix

@@ -56,9 +56,12 @@ pkgs.testers.runNixOSTest {
     playos = { config, lib, pkgs, ... }:
     {
       imports = [
-        (import ../../virtualisation-config.nix { inherit overlayPath; })
+        ../../virtualisation-config.nix
       ];
-      virtualisation.vlans = [ 1 ];
+      config = {
+        playos.e2e-tests.overlayPath = overlayPath;
+        virtualisation.vlans = [ 1 ];
+      };
     };
     # runs an HTTP proxy and a mock HTTP update/bundle server
     sidekick = { config, nodes, lib, pkgs, ... }:

testing/end-to-end/virtualisation-config.nix

@@ -1,5 +1,29 @@
-{ overlayPath, ... }:
+{ config, pkgs, lib, ... }:
+with lib;
+let
+    cfg = config.playos.e2e-tests;
+    # Note: cannot use symlinks (e.g. `linkFarm`) here
+    virtfsDir = pkgs.runCommand "extra-test-files" {} ''
+        mkdir -p $out
+        cp ${cfg.overlayConfig} $out/playos-config.toml
+    '';
+in
 {
+    options = {
+        playos.e2e-tests.overlayPath = mkOption {
+            type = types.path;
+            description = "Path to a qcow overlay disk from which to boot";
+        };
+        playos.e2e-tests.overlayConfig = mkOption {
+            type = types.nullOr types.path;
+            description = ''
+                If provided, will bind-mount the config file at
+                /etc/playos-config.toml as a way to override configuration
+                parameters without rebuilding the PlayOS disk.
+            '';
+            default = null;
+        };
+    };
     config = {
         # Kinda abusing the NixOS testing infra here, because
         # there is no other interface for creating test VMs/nodes.
@@ -32,7 +56,11 @@
             # prior to launching a VM. Since it is not configurable to our
             # needs, we create the overlay image instead in the `testScript`,
             # so this path is a "forward reference" that does not exist.
-            "-hda ${overlayPath}"
+            "-hda ${cfg.overlayPath}"
+        ] ++ lib.lists.optionals (cfg.overlayConfig != null) [
+            "--virtfs"
+            # Note: extra-test-files is hardcoded in profile.nix
+            "local,path=${virtfsDir},mount_tag=extra-test-files,readonly=on,security_model=none,multidevs=remap"
         ];
     };
 }