Start of modernization of skyflake.

.gitignore

@@ -0,0 +1,6 @@
+example1-ceph.img
+example1-persist.img
+example2-ceph.img
+example2-persist.img
+example3-ceph.img
+example3-persist.img

default-customization.nix

@@ -37,7 +37,7 @@ in
       mountPoint = "/nix/.ro-store";
     } ];
     volumes = [ {
-      image = config.skyflake.deploy.rbds.root.path;
+      image = config.skyflake.deploy.ceph.rbds.root.path;
       mountPoint = "/";
       # don't let microvm.nix create an image file
       autoCreate = false;
@@ -55,7 +55,7 @@ in
     } ];
   };
 
-  config.skyflake.deploy.rbds.root = {
+  config.skyflake.deploy.ceph.rbds.root = {
     pool = "microvms";
     namespace = user;
     name = "${repo}-${vmName}-root";
@@ -78,5 +78,5 @@ in
     value = "yes";
   } ];
 
-  config.fileSystems."/".fsType = lib.mkForce "ext4";
+  config.fileSystems."/".fsType = lib.mkForce "btrfs";
 }

doc/host/intro.md

@@ -34,7 +34,7 @@ always be able to access its storage, regardless of the server they
 are started on.
 
 We solve this problem by moving the VM filesystems to a network
-filesystem: Ceph.
+filesystem: Ceph and also seaweedfs.
 
 
 ### Network Setup

doc/user/flake.md

@@ -22,7 +22,7 @@ A sample `flake.nix`:
     nixosConfigurations = {
       my-microvm = nixpkgs.lib.nixosSystem {
         modules = [ {
-          system.stateVersion = "22.11";
+          system.stateVersion = "24.11";
           networking.hostName = "my-microvm";
           services.openssh = {
             enable = true;

example-server.nix

@@ -7,21 +7,27 @@
     vcpu = 2;
     mem = 4096;
 
-    shares = [ {
-      tag = "ro-store";
-      source = "/nix/store";
-      mountPoint = "/nix/.ro-store";
-    } ];
-    volumes = [ {
-      image = "example${toString instance}-persist.img";
-      mountPoint = "/";
-      size = 20 * 1024;
-    } {
-      image = "example${toString instance}-ceph.img";
-      mountPoint = null;
-      size = 20 * 1024;
-    } ];
-    writableStoreOverlay = "/nix/.rw-store";
+    shares = [
+      {
+        tag = "ro-store";
+        source = "/nix/store";
+        mountPoint = "/nix/.ro-store";
+      } 
+    ];
+    volumes = [
+      {
+        image = "example${toString instance}-persist.img";
+        mountPoint = "/";
+        size = 20 * 1024;
+        fsType = "btrfs"; # needed for some seaweedfs optimizations.
+      }
+      {
+        image = "example${toString instance}-ceph.img";
+        mountPoint = null;
+        size = 20 * 1024;
+      }
+   ];
+   writableStoreOverlay = "/nix/.rw-store";
 
     interfaces = [ {
       id = "eth0";
@@ -34,8 +40,7 @@
   networking.hostName = "example${toString instance}";
   users.users.root.password = "";
 
-  # TODO:
-  networking.firewall.enable = false;
+  networking.firewall.enable = true;
 
   networking.useDHCP = false;
   networking.useNetworkd = true;
@@ -64,7 +69,7 @@
           IPv6AcceptRA = true;
         };
         addresses = [ {
-          addressConfig.Address = "fec0::${toString instance}/64";
+          Address = "fec0::${toString instance}/64";
         } ];
       };
     };
@@ -74,11 +79,25 @@
     nodes = builtins.listToAttrs (
       map (instance: {
         name = "example${toString instance}";
-        value.address = "fec0::${toString instance}";
+        value.address = "[fec0::${toString instance}]";
       }) [ 1 2 3 ]
     );
 
-    storage.ceph = rec {
+
+    storage.seaweedfs = {
+      enable = false;
+      volumeStorage.encrypt = true;
+      # example mount below.
+      # mounts."/mnt".mountSource = "/filesystems/1a32bfd9-0cbc-430a-a28a-d9fd862e9ebc";
+      filer.db.etcd = {
+        enable = true;
+        certFile = example/certs/default.pem; 
+        keyFile = example/certs/default-key.pem;
+        trustedCaFile = example/certs/ca.pem;
+      };
+    };
+    storage.ceph = {
+      enable = true;
       fsid = "8364da79-5e03-49ae-82ea-7d936278cb0f";
       monKeyring = example/ceph.mon.keyring;
       adminKeyring = example/ceph.client.admin.keyring;
@@ -96,7 +115,7 @@
     };
 
     nomad = {
-      servers = [ "example1" "example2" "example3" ];
+      servers = builtins.attrNames config.skyflake.nodes;
       client.meta = {
         example-deployment = "yes";
       };
@@ -107,12 +126,14 @@
         uid = 1000;
         sshKeys = [
           "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGJJTSJdpDh82486uPiMhhyhnci4tScp5uUe7156MBC8 astro"
+          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPRRdToCDUupkkwI+crB3fGDwdBIFkDsBHjOImn+qsjg openpgp:0xE8D3D833"
         ];
       };
     };
   };
 
   environment.systemPackages = with pkgs; [
     tcpdump
+    nmap
   ];
-}
+}

example/certs/ca-key.pem

@@ -0,0 +1,3 @@
+-----BEGIN Ed25519 PRIVATE KEY-----
+MC4CAQAwBQYDK2VwBCIEIKPAHuclX3Tz8jwZE0hb9Kdjx5Kxg3p+FqJv0O9SCx00
+-----END Ed25519 PRIVATE KEY-----

example/certs/ca.csr

@@ -0,0 +1,8 @@
+-----BEGIN CERTIFICATE REQUEST-----
+MIHvMIGiAgEAMG8xFTATBgNVBAcTDHRoZSBpbnRlcm5ldDEWMBQGA1UEChMNYXV0
+b2dlbmVyYXRlZDEeMBwGA1UECxMVZXRjZCBza3lmbGFrZSBjbHVzdGVyMR4wHAYD
+VQQDExVza3lmbGFrZS1ldGNkLWV4YW1wbGUwKjAFBgMrZXADIQAgcVYcKr8yQnKz
+dTPmiUyRWgFWafPsYoMVg4znKaBBGqAAMAUGAytlcANBABK3ArwpSnK4Azv9vDSa
+sdy+lAiy1xNmOFKN0pV0nKBdjyxjBadKDWVlSWBpoZWt1CSHu9rVLBPrXTooilL3
+/gc=
+-----END CERTIFICATE REQUEST-----

example/certs/ca.pem

@@ -0,0 +1,13 @@
+-----BEGIN CERTIFICATE-----
+MIIB4jCCAZSgAwIBAgIUHzKbs9uzVilNYp5EYDs00/C2H2wwBQYDK2VwMG8xFTAT
+BgNVBAcTDHRoZSBpbnRlcm5ldDEWMBQGA1UEChMNYXV0b2dlbmVyYXRlZDEeMBwG
+A1UECxMVZXRjZCBza3lmbGFrZSBjbHVzdGVyMR4wHAYDVQQDExVza3lmbGFrZS1l
+dGNkLWV4YW1wbGUwHhcNMjQxMjAyMTE1OTAwWhcNMjkxMjAxMTE1OTAwWjBvMRUw
+EwYDVQQHEwx0aGUgaW50ZXJuZXQxFjAUBgNVBAoTDWF1dG9nZW5lcmF0ZWQxHjAc
+BgNVBAsTFWV0Y2Qgc2t5Zmxha2UgY2x1c3RlcjEeMBwGA1UEAxMVc2t5Zmxha2Ut
+ZXRjZC1leGFtcGxlMCowBQYDK2VwAyEAIHFWHCq/MkJys3Uz5olMkVoBVmnz7GKD
+FYOM5ymgQRqjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0G
+A1UdDgQWBBRlGT1d+LgpnUisEaXKURLm+42OiTAFBgMrZXADQQA2DVOglAcuK3Z6
+chnNl64vxWGhKyJymvt9WeX3+PUZb2iD5XIuBAE7YlD+7ppp4CPHz1/ou7A6Qrzt
+ow5S9t8B
+-----END CERTIFICATE-----

example/certs/config/ca-config.json

@@ -0,0 +1,13 @@
+{
+  "signing": {
+    "default": {
+        "usages": [
+          "signing",
+          "key encipherment",
+          "server auth",
+          "client auth"
+        ],
+        "expiry": "8760h"
+    }
+  }
+}

example/certs/config/ca-csr.json

@@ -0,0 +1,14 @@
+{
+  "CN": "skyflake-etcd-example",
+  "key": {
+    "algo": "ed25519",
+    "size": 512
+  },
+  "names": [
+    {
+      "O": "autogenerated",
+      "OU": "etcd skyflake cluster",
+      "L": "the internet"
+    }
+  ]
+}

example/certs/config/default.json

@@ -0,0 +1,21 @@
+{
+    "CN": "example.net",
+    "hosts": [
+        "fec0::1",
+        "fec0::2",
+        "fec0::3",
+        "::1",
+        "127.0.0.1"
+        ],
+    "key": {
+        "algo": "ed25519"
+    },
+    "names": [
+        {
+            "O": "autogenerated",
+            "OU": "etcd skyflake cluster",
+            "L": "the internet"
+        }
+    ]
+}
+

example/certs/config/req-csr.json

@@ -0,0 +1,20 @@
+{
+  "CN": "skyflake-etcd-example",
+  "hosts": [
+    "fec0::1",
+    "fec0::2",
+    "fec0::3",
+    "::1",
+    "127.0.0.1"
+  ],
+  "key": {
+    "algo": "ed25519"
+  },
+  "names": [
+    {
+      "O": "autogenerated",
+      "OU": "etcd skyflake cluster",
+      "L": "the internet"
+    }
+  ]
+}

example/certs/default-key.pem

@@ -0,0 +1,3 @@
+-----BEGIN Ed25519 PRIVATE KEY-----
+MC4CAQAwBQYDK2VwBCIEILHkEluUia6mFJwDVvkw25Py2Qje0XwdmTeOhK+SWMGi
+-----END Ed25519 PRIVATE KEY-----

example/certs/default.csr

@@ -0,0 +1,10 @@
+-----BEGIN CERTIFICATE REQUEST-----
+MIIBUDCCAQICAQAwZTEVMBMGA1UEBxMMdGhlIGludGVybmV0MRYwFAYDVQQKEw1h
+dXRvZ2VuZXJhdGVkMR4wHAYDVQQLExVldGNkIHNreWZsYWtlIGNsdXN0ZXIxFDAS
+BgNVBAMTC2V4YW1wbGUubmV0MCowBQYDK2VwAyEAmcDRG4K1OwMpjeBrNgzMQS4J
+4vnvWo2Ktj43Mww0Y+GgajBoBgkqhkiG9w0BCQ4xWzBZMFcGA1UdEQRQME6HEP7A
+AAAAAAAAAAAAAAAAAAGHEP7AAAAAAAAAAAAAAAAAAAKHEP7AAAAAAAAAAAAAAAAA
+AAOHEAAAAAAAAAAAAAAAAAAAAAGHBH8AAAEwBQYDK2VwA0EAKis22c0zZ9GqQYAI
+YGhz+R/k00VIpWzlXsSsnbJpsfj18FWOMGne2F0FrGyTIMgwLxqjFrvUIX1jt7No
+vnYWCg==
+-----END CERTIFICATE REQUEST-----

example/certs/default.pem

@@ -0,0 +1,16 @@
+-----BEGIN CERTIFICATE-----
+MIICcDCCAiKgAwIBAgIUEqxQwbyAuUbFloz2Af1r1a/CcjkwBQYDK2VwMG8xFTAT
+BgNVBAcTDHRoZSBpbnRlcm5ldDEWMBQGA1UEChMNYXV0b2dlbmVyYXRlZDEeMBwG
+A1UECxMVZXRjZCBza3lmbGFrZSBjbHVzdGVyMR4wHAYDVQQDExVza3lmbGFrZS1l
+dGNkLWV4YW1wbGUwHhcNMjQxMjAyMTIwMDAwWhcNMjUxMjAyMTIwMDAwWjBlMRUw
+EwYDVQQHEwx0aGUgaW50ZXJuZXQxFjAUBgNVBAoTDWF1dG9nZW5lcmF0ZWQxHjAc
+BgNVBAsTFWV0Y2Qgc2t5Zmxha2UgY2x1c3RlcjEUMBIGA1UEAxMLZXhhbXBsZS5u
+ZXQwKjAFBgMrZXADIQCZwNEbgrU7AymN4Gs2DMxBLgni+e9ajYq2PjczDDRj4aOB
+2TCB1jAOBgNVHQ8BAf8EBAMCBaAwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUF
+BwMCMAwGA1UdEwEB/wQCMAAwHQYDVR0OBBYEFM193VfeS/PgCTJdre/3qQTxAX0U
+MB8GA1UdIwQYMBaAFGUZPV34uCmdSKwRpcpREub7jY6JMFcGA1UdEQRQME6HEP7A
+AAAAAAAAAAAAAAAAAAGHEP7AAAAAAAAAAAAAAAAAAAKHEP7AAAAAAAAAAAAAAAAA
+AAOHEAAAAAAAAAAAAAAAAAAAAAGHBH8AAAEwBQYDK2VwA0EAP0zSSpejQs8qwXeC
+jrUZasY4YWRUtZIS+nqo+4/Uaqxq+U+zDT+SHd3vjosHsl4k6KXcY2j7r9rDv1+h
+o6q0Bw==
+-----END CERTIFICATE-----