diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ff32b0b --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +example1-ceph.img +example1-persist.img +example2-ceph.img +example2-persist.img +example3-ceph.img +example3-persist.img diff --git a/default-customization.nix b/default-customization.nix index 471965e..16e508f 100644 --- a/default-customization.nix +++ b/default-customization.nix @@ -37,7 +37,7 @@ in mountPoint = "/nix/.ro-store"; } ]; volumes = [ { - image = config.skyflake.deploy.rbds.root.path; + image = config.skyflake.deploy.ceph.rbds.root.path; mountPoint = "/"; # don't let microvm.nix create an image file autoCreate = false; @@ -55,7 +55,7 @@ in } ]; }; - config.skyflake.deploy.rbds.root = { + config.skyflake.deploy.ceph.rbds.root = { pool = "microvms"; namespace = user; name = "${repo}-${vmName}-root"; @@ -78,5 +78,5 @@ in value = "yes"; } ]; - config.fileSystems."/".fsType = lib.mkForce "ext4"; + config.fileSystems."/".fsType = lib.mkForce "btrfs"; } diff --git a/doc/host/intro.md b/doc/host/intro.md index 92c406a..9f9b9b5 100644 --- a/doc/host/intro.md +++ b/doc/host/intro.md @@ -34,7 +34,7 @@ always be able to access its storage, regardless of the server they are started on. We solve this problem by moving the VM filesystems to a network -filesystem: Ceph. +filesystem: Ceph and also seaweedfs. ### Network Setup diff --git a/doc/user/flake.md b/doc/user/flake.md index f0bc4f3..37d7a84 100644 --- a/doc/user/flake.md +++ b/doc/user/flake.md @@ -22,7 +22,7 @@ A sample `flake.nix`: nixosConfigurations = { my-microvm = nixpkgs.lib.nixosSystem { modules = [ { - system.stateVersion = "22.11"; + system.stateVersion = "24.11"; networking.hostName = "my-microvm"; services.openssh = { enable = true; diff --git a/example-server.nix b/example-server.nix index 72d06b9..677acd9 100644 --- a/example-server.nix +++ b/example-server.nix @@ -7,21 +7,27 @@ vcpu = 2; mem = 4096; - shares = [ { - tag = "ro-store"; - source = "/nix/store"; - mountPoint = "/nix/.ro-store"; - } ]; - volumes = [ { - image = "example${toString instance}-persist.img"; - mountPoint = "/"; - size = 20 * 1024; - } { - image = "example${toString instance}-ceph.img"; - mountPoint = null; - size = 20 * 1024; - } ]; - writableStoreOverlay = "/nix/.rw-store"; + shares = [ + { + tag = "ro-store"; + source = "/nix/store"; + mountPoint = "/nix/.ro-store"; + } + ]; + volumes = [ + { + image = "example${toString instance}-persist.img"; + mountPoint = "/"; + size = 20 * 1024; + fsType = "btrfs"; # needed for some seaweedfs optimizations. + } + { + image = "example${toString instance}-ceph.img"; + mountPoint = null; + size = 20 * 1024; + } + ]; + writableStoreOverlay = "/nix/.rw-store"; interfaces = [ { id = "eth0"; @@ -34,8 +40,7 @@ networking.hostName = "example${toString instance}"; users.users.root.password = ""; - # TODO: - networking.firewall.enable = false; + networking.firewall.enable = true; networking.useDHCP = false; networking.useNetworkd = true; @@ -64,7 +69,7 @@ IPv6AcceptRA = true; }; addresses = [ { - addressConfig.Address = "fec0::${toString instance}/64"; + Address = "fec0::${toString instance}/64"; } ]; }; }; @@ -74,11 +79,25 @@ nodes = builtins.listToAttrs ( map (instance: { name = "example${toString instance}"; - value.address = "fec0::${toString instance}"; + value.address = "[fec0::${toString instance}]"; }) [ 1 2 3 ] ); - storage.ceph = rec { + + storage.seaweedfs = { + enable = false; + volumeStorage.encrypt = true; + # example mount below. + # mounts."/mnt".mountSource = "/filesystems/1a32bfd9-0cbc-430a-a28a-d9fd862e9ebc"; + filer.db.etcd = { + enable = true; + certFile = example/certs/default.pem; + keyFile = example/certs/default-key.pem; + trustedCaFile = example/certs/ca.pem; + }; + }; + storage.ceph = { + enable = true; fsid = "8364da79-5e03-49ae-82ea-7d936278cb0f"; monKeyring = example/ceph.mon.keyring; adminKeyring = example/ceph.client.admin.keyring; @@ -96,7 +115,7 @@ }; nomad = { - servers = [ "example1" "example2" "example3" ]; + servers = builtins.attrNames config.skyflake.nodes; client.meta = { example-deployment = "yes"; }; @@ -107,6 +126,7 @@ uid = 1000; sshKeys = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGJJTSJdpDh82486uPiMhhyhnci4tScp5uUe7156MBC8 astro" + "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPRRdToCDUupkkwI+crB3fGDwdBIFkDsBHjOImn+qsjg openpgp:0xE8D3D833" ]; }; }; @@ -114,5 +134,6 @@ environment.systemPackages = with pkgs; [ tcpdump + nmap ]; -} +} \ No newline at end of file diff --git a/example/certs/ca-key.pem b/example/certs/ca-key.pem new file mode 100644 index 0000000..963dd0f --- /dev/null +++ b/example/certs/ca-key.pem @@ -0,0 +1,3 @@ +-----BEGIN Ed25519 PRIVATE KEY----- +MC4CAQAwBQYDK2VwBCIEIKPAHuclX3Tz8jwZE0hb9Kdjx5Kxg3p+FqJv0O9SCx00 +-----END Ed25519 PRIVATE KEY----- diff --git a/example/certs/ca.csr b/example/certs/ca.csr new file mode 100644 index 0000000..f58c787 --- /dev/null +++ b/example/certs/ca.csr @@ -0,0 +1,8 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIHvMIGiAgEAMG8xFTATBgNVBAcTDHRoZSBpbnRlcm5ldDEWMBQGA1UEChMNYXV0 +b2dlbmVyYXRlZDEeMBwGA1UECxMVZXRjZCBza3lmbGFrZSBjbHVzdGVyMR4wHAYD +VQQDExVza3lmbGFrZS1ldGNkLWV4YW1wbGUwKjAFBgMrZXADIQAgcVYcKr8yQnKz +dTPmiUyRWgFWafPsYoMVg4znKaBBGqAAMAUGAytlcANBABK3ArwpSnK4Azv9vDSa +sdy+lAiy1xNmOFKN0pV0nKBdjyxjBadKDWVlSWBpoZWt1CSHu9rVLBPrXTooilL3 +/gc= +-----END CERTIFICATE REQUEST----- diff --git a/example/certs/ca.pem b/example/certs/ca.pem new file mode 100644 index 0000000..557e1b8 --- /dev/null +++ b/example/certs/ca.pem @@ -0,0 +1,13 @@ +-----BEGIN CERTIFICATE----- +MIIB4jCCAZSgAwIBAgIUHzKbs9uzVilNYp5EYDs00/C2H2wwBQYDK2VwMG8xFTAT +BgNVBAcTDHRoZSBpbnRlcm5ldDEWMBQGA1UEChMNYXV0b2dlbmVyYXRlZDEeMBwG +A1UECxMVZXRjZCBza3lmbGFrZSBjbHVzdGVyMR4wHAYDVQQDExVza3lmbGFrZS1l +dGNkLWV4YW1wbGUwHhcNMjQxMjAyMTE1OTAwWhcNMjkxMjAxMTE1OTAwWjBvMRUw +EwYDVQQHEwx0aGUgaW50ZXJuZXQxFjAUBgNVBAoTDWF1dG9nZW5lcmF0ZWQxHjAc +BgNVBAsTFWV0Y2Qgc2t5Zmxha2UgY2x1c3RlcjEeMBwGA1UEAxMVc2t5Zmxha2Ut +ZXRjZC1leGFtcGxlMCowBQYDK2VwAyEAIHFWHCq/MkJys3Uz5olMkVoBVmnz7GKD +FYOM5ymgQRqjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0G +A1UdDgQWBBRlGT1d+LgpnUisEaXKURLm+42OiTAFBgMrZXADQQA2DVOglAcuK3Z6 +chnNl64vxWGhKyJymvt9WeX3+PUZb2iD5XIuBAE7YlD+7ppp4CPHz1/ou7A6Qrzt +ow5S9t8B +-----END CERTIFICATE----- diff --git a/example/certs/config/ca-config.json b/example/certs/config/ca-config.json new file mode 100644 index 0000000..424ea37 --- /dev/null +++ b/example/certs/config/ca-config.json @@ -0,0 +1,13 @@ +{ + "signing": { + "default": { + "usages": [ + "signing", + "key encipherment", + "server auth", + "client auth" + ], + "expiry": "8760h" + } + } +} \ No newline at end of file diff --git a/example/certs/config/ca-csr.json b/example/certs/config/ca-csr.json new file mode 100644 index 0000000..eec5b9c --- /dev/null +++ b/example/certs/config/ca-csr.json @@ -0,0 +1,14 @@ +{ + "CN": "skyflake-etcd-example", + "key": { + "algo": "ed25519", + "size": 512 + }, + "names": [ + { + "O": "autogenerated", + "OU": "etcd skyflake cluster", + "L": "the internet" + } + ] +} diff --git a/example/certs/config/default.json b/example/certs/config/default.json new file mode 100644 index 0000000..1463f48 --- /dev/null +++ b/example/certs/config/default.json @@ -0,0 +1,21 @@ +{ + "CN": "example.net", + "hosts": [ + "fec0::1", + "fec0::2", + "fec0::3", + "::1", + "127.0.0.1" + ], + "key": { + "algo": "ed25519" + }, + "names": [ + { + "O": "autogenerated", + "OU": "etcd skyflake cluster", + "L": "the internet" + } + ] +} + diff --git a/example/certs/config/req-csr.json b/example/certs/config/req-csr.json new file mode 100644 index 0000000..4886099 --- /dev/null +++ b/example/certs/config/req-csr.json @@ -0,0 +1,20 @@ +{ + "CN": "skyflake-etcd-example", + "hosts": [ + "fec0::1", + "fec0::2", + "fec0::3", + "::1", + "127.0.0.1" + ], + "key": { + "algo": "ed25519" + }, + "names": [ + { + "O": "autogenerated", + "OU": "etcd skyflake cluster", + "L": "the internet" + } + ] +} \ No newline at end of file diff --git a/example/certs/default-key.pem b/example/certs/default-key.pem new file mode 100644 index 0000000..fa3507b --- /dev/null +++ b/example/certs/default-key.pem @@ -0,0 +1,3 @@ +-----BEGIN Ed25519 PRIVATE KEY----- +MC4CAQAwBQYDK2VwBCIEILHkEluUia6mFJwDVvkw25Py2Qje0XwdmTeOhK+SWMGi +-----END Ed25519 PRIVATE KEY----- diff --git a/example/certs/default.csr b/example/certs/default.csr new file mode 100644 index 0000000..cb1913a --- /dev/null +++ b/example/certs/default.csr @@ -0,0 +1,10 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIIBUDCCAQICAQAwZTEVMBMGA1UEBxMMdGhlIGludGVybmV0MRYwFAYDVQQKEw1h +dXRvZ2VuZXJhdGVkMR4wHAYDVQQLExVldGNkIHNreWZsYWtlIGNsdXN0ZXIxFDAS +BgNVBAMTC2V4YW1wbGUubmV0MCowBQYDK2VwAyEAmcDRG4K1OwMpjeBrNgzMQS4J +4vnvWo2Ktj43Mww0Y+GgajBoBgkqhkiG9w0BCQ4xWzBZMFcGA1UdEQRQME6HEP7A +AAAAAAAAAAAAAAAAAAGHEP7AAAAAAAAAAAAAAAAAAAKHEP7AAAAAAAAAAAAAAAAA +AAOHEAAAAAAAAAAAAAAAAAAAAAGHBH8AAAEwBQYDK2VwA0EAKis22c0zZ9GqQYAI +YGhz+R/k00VIpWzlXsSsnbJpsfj18FWOMGne2F0FrGyTIMgwLxqjFrvUIX1jt7No +vnYWCg== +-----END CERTIFICATE REQUEST----- diff --git a/example/certs/default.pem b/example/certs/default.pem new file mode 100644 index 0000000..4de6126 --- /dev/null +++ b/example/certs/default.pem @@ -0,0 +1,16 @@ +-----BEGIN CERTIFICATE----- +MIICcDCCAiKgAwIBAgIUEqxQwbyAuUbFloz2Af1r1a/CcjkwBQYDK2VwMG8xFTAT +BgNVBAcTDHRoZSBpbnRlcm5ldDEWMBQGA1UEChMNYXV0b2dlbmVyYXRlZDEeMBwG +A1UECxMVZXRjZCBza3lmbGFrZSBjbHVzdGVyMR4wHAYDVQQDExVza3lmbGFrZS1l +dGNkLWV4YW1wbGUwHhcNMjQxMjAyMTIwMDAwWhcNMjUxMjAyMTIwMDAwWjBlMRUw +EwYDVQQHEwx0aGUgaW50ZXJuZXQxFjAUBgNVBAoTDWF1dG9nZW5lcmF0ZWQxHjAc +BgNVBAsTFWV0Y2Qgc2t5Zmxha2UgY2x1c3RlcjEUMBIGA1UEAxMLZXhhbXBsZS5u +ZXQwKjAFBgMrZXADIQCZwNEbgrU7AymN4Gs2DMxBLgni+e9ajYq2PjczDDRj4aOB +2TCB1jAOBgNVHQ8BAf8EBAMCBaAwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUF +BwMCMAwGA1UdEwEB/wQCMAAwHQYDVR0OBBYEFM193VfeS/PgCTJdre/3qQTxAX0U +MB8GA1UdIwQYMBaAFGUZPV34uCmdSKwRpcpREub7jY6JMFcGA1UdEQRQME6HEP7A +AAAAAAAAAAAAAAAAAAGHEP7AAAAAAAAAAAAAAAAAAAKHEP7AAAAAAAAAAAAAAAAA +AAOHEAAAAAAAAAAAAAAAAAAAAAGHBH8AAAEwBQYDK2VwA0EAP0zSSpejQs8qwXeC +jrUZasY4YWRUtZIS+nqo+4/Uaqxq+U+zDT+SHd3vjosHsl4k6KXcY2j7r9rDv1+h +o6q0Bw== +-----END CERTIFICATE----- diff --git a/flake.lock b/flake.lock index d4894c9..b9f7d53 100644 --- a/flake.lock +++ b/flake.lock @@ -5,11 +5,11 @@ "systems": "systems" }, "locked": { - "lastModified": 1692799911, - "narHash": "sha256-3eihraek4qL744EvQXsK1Ha6C3CR7nnT8X2qWap4RNk=", + "lastModified": 1726560853, + "narHash": "sha256-X6rJYSESBVr3hBoH0WbKE5KvhPU5bloyZ2L4K60/fPQ=", "owner": "numtide", "repo": "flake-utils", - "rev": "f9e7cf818399d17d347f847525c5a5a8032e4e44", + "rev": "c1dfcf08411b08f6b8615f7d8971a2bfa81d5e8a", "type": "github" }, "original": { @@ -23,14 +23,15 @@ "flake-utils": "flake-utils", "nixpkgs": [ "nixpkgs" - ] + ], + "spectrum": "spectrum" }, "locked": { - "lastModified": 1693776909, - "narHash": "sha256-9JRxyhGS0vnciGgiwdzpupN88/K6seBSf9PqEVTL280=", + "lastModified": 1731240174, + "narHash": "sha256-HYu+bPoV3UILhwc4Ar5iQ7aF+DuQWHXl4mljN6Bwq6A=", "owner": "astro", "repo": "microvm.nix", - "rev": "85790506c0d131181805ffbd40617580be23c67e", + "rev": "dd89404e1885b8d7033106f3898eaef8db660cb2", "type": "github" }, "original": { @@ -64,7 +65,9 @@ "nix-cache-cut": { "inputs": { "naersk": "naersk", - "nixpkgs": "nixpkgs", + "nixpkgs": [ + "nixpkgs" + ], "utils": "utils" }, "locked": { @@ -83,38 +86,58 @@ }, "nixpkgs": { "locked": { - "lastModified": 1686089707, - "narHash": "sha256-LTNlJcru2qJ0XhlhG9Acp5KyjB774Pza3tRH0pKIb3o=", + "lastModified": 1731676054, + "narHash": "sha256-OZiZ3m8SCMfh3B6bfGC/Bm4x3qc1m2SVEAlkV6iY7Yg=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "af21c31b2a1ec5d361ed8050edd0303c31306397", + "rev": "5e4fbfb6b3de1aa2872b76d49fafc942626e2add", "type": "github" }, "original": { "owner": "NixOS", - "ref": "nixpkgs-unstable", + "ref": "nixos-unstable", "repo": "nixpkgs", "type": "github" } }, - "nixpkgs_2": { + "nomad-nixpkgs": { "locked": { - "lastModified": 1693250523, - "narHash": "sha256-y3up5gXMTbnCsXrNEB5j+7TVantDLUYyQLu/ueiXuyg=", - "path": "/nix/store/nvdrwx25ril8dys6yryrp39d14b2p742-source", - "rev": "3efb0f6f404ec8dae31bdb1a9b17705ce0d6986e", - "type": "path" + "lastModified": 1727331237, + "narHash": "sha256-nkhXMPuxbqrgdOrT5Ggy0iTNOkL7g5o2NN2KUDRnjck=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "1457235a9eee6e05916cd543d3143360e6fd1080", + "type": "github" }, "original": { - "id": "nixpkgs", - "type": "indirect" + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "1457235a9eee6e05916cd543d3143360e6fd1080", + "type": "github" } }, "root": { "inputs": { "microvm": "microvm", "nix-cache-cut": "nix-cache-cut", - "nixpkgs": "nixpkgs_2" + "nixpkgs": "nixpkgs", + "nomad-nixpkgs": "nomad-nixpkgs" + } + }, + "spectrum": { + "flake": false, + "locked": { + "lastModified": 1729945407, + "narHash": "sha256-iGNMamNOAnVTETnIVqDWd6fl74J8fLEi1ejdZiNjEtY=", + "ref": "refs/heads/main", + "rev": "f1d94ee7029af18637dbd5fdf4749621533693fa", + "revCount": 764, + "type": "git", + "url": "https://spectrum-os.org/git/spectrum" + }, + "original": { + "type": "git", + "url": "https://spectrum-os.org/git/spectrum" } }, "systems": { diff --git a/flake.nix b/flake.nix index 42ce46d..93d24ab 100644 --- a/flake.nix +++ b/flake.nix @@ -2,18 +2,30 @@ description = "Hyperconverged Infratructure for NixOS"; inputs = { - microvm.url = "github:astro/microvm.nix"; - microvm.inputs.nixpkgs.follows = "nixpkgs"; - nix-cache-cut.url = "github:astro/nix-cache-cut"; + nixpkgs = { + url = "github:NixOS/nixpkgs/nixos-unstable"; + }; + nomad-nixpkgs = { + url = "github:NixOS/nixpkgs/1457235a9eee6e05916cd543d3143360e6fd1080"; # Last version of NixOS unstable that supports a foss version of nomad. + }; + microvm = { + url = "github:astro/microvm.nix"; + inputs.nixpkgs.follows = "nixpkgs"; + }; + nix-cache-cut = { + url = "github:astro/nix-cache-cut"; + inputs.nixpkgs.follows = "nixpkgs"; + }; }; - outputs = { self, nixpkgs, microvm, nix-cache-cut }: + outputs = { self, nixpkgs, nomad-nixpkgs, microvm, nix-cache-cut }: let system = "x86_64-linux"; pkgs = nixpkgs.legacyPackages.${system}; in { + # formatter.${system} = pkgs.alejandra; packages.${system} = import ./pkgs/doc.nix { inherit pkgs self; }; @@ -21,14 +33,28 @@ nixosModules = { default = { imports = [ + {nixpkgs.overlays = [ (final: prev: { nomadPin = nomad-nixpkgs.legacyPackages.${prev.system}; }) ];} + ./nixos-modules/storage/seaweedfs/options.nix + ./nixos-modules/storage/seaweedfs/server.nix + ./nixos-modules/storage/seaweedfs/db-backend/etcd/default.nix + ./nixos-modules/storage/seaweedfs/db-backend/etcd/options.nix ./nixos-modules/storage/ceph/server.nix ./nixos-modules/defaults.nix + ./nixos-modules/firewall.nix ./nixos-modules/nodes.nix ./nixos-modules/nomad.nix ./nixos-modules/users.nix - (import ./nixos-modules/ssh-deploy.nix { - inherit microvm nixpkgs; - }) + ./nixos-modules/ssh-deployOptions.nix + (import + ./nixos-modules/storage/seaweedfs/ssh-deploy.nix { + inherit microvm nixpkgs; + } + ) + (import + ./nixos-modules/storage/ceph/ssh-deploy.nix { + inherit microvm nixpkgs; + } + ) { nixpkgs.overlays = [ nix-cache-cut.overlays.default @@ -48,7 +74,7 @@ microvm.nixosModules.microvm self.nixosModules.default (import ./example-server.nix { inherit instance; }) - ]; + ]; }; in { diff --git a/nixos-modules/cache-cut.nix b/nixos-modules/cache-cut.nix index 1476f78..492ed57 100644 --- a/nixos-modules/cache-cut.nix +++ b/nixos-modules/cache-cut.nix @@ -41,7 +41,7 @@ in systemd.services.skyflake-install-cache-gc = { wantedBy = [ "multi-user.target" ]; requires = [ "nomad.service" ]; - path = with pkgs; [ nomad ]; + path = [ config.services.nomad.package ]; script = '' nomad run -detach ${jobFile} ''; diff --git a/nixos-modules/defaults.nix b/nixos-modules/defaults.nix index c979dd3..bcd3134 100644 --- a/nixos-modules/defaults.nix +++ b/nixos-modules/defaults.nix @@ -15,5 +15,4 @@ builders-use-substitutes = true ''; }; - } diff --git a/nixos-modules/firewall.nix b/nixos-modules/firewall.nix new file mode 100644 index 0000000..0e75bb5 --- /dev/null +++ b/nixos-modules/firewall.nix @@ -0,0 +1,7 @@ +{ ... }: +{ + networking.nftables = { + enable = true; + flushRuleset = true; + }; +} diff --git a/nixos-modules/nodes.nix b/nixos-modules/nodes.nix index 88d6fd5..c66ab6d 100644 --- a/nixos-modules/nodes.nix +++ b/nixos-modules/nodes.nix @@ -9,7 +9,11 @@ in description = '' All cluster nodes with their addresses, or at least those who run coordination servers (eg. nomad servers, ceph server, - ...). + seaweedfs server ...). + + Should be always a majority, i.e.: an uneven number, + commonly 3 or 5 to have a redundancy of 1 or 2 or + mathmatically: 2(REDUNDANCY)+1=NEEDED_SERVERS. ''; default = {}; type = types.attrsOf (types.submodule { diff --git a/nixos-modules/nomad.nix b/nixos-modules/nomad.nix index d0defc9..05a6d74 100644 --- a/nixos-modules/nomad.nix +++ b/nixos-modules/nomad.nix @@ -1,7 +1,6 @@ { config, lib, pkgs, ... }: let cfg = config.skyflake.nomad; - in { options.skyflake.nomad = with lib; { @@ -10,6 +9,15 @@ in default = "sky0"; }; + # https://developer.hashicorp.com/nomad/docs/install/production/requirements#ports-used + networking.firewall.allowedUDPPorts = [ + 4648 #Serf WAN + ]; + networking.firewall.allowedTCPPorts = [ + 4646 # TODO Fix firewall with option HTTP API + 4648 # Serf WAN + ]; + server.enable = mkOption { type = types.bool; default = builtins.elem config.networking.hostName cfg.servers; @@ -42,7 +50,7 @@ in config = { services.nomad = { enable = true; - package = pkgs.nomad_1_4; + package = pkgs.nomadPin.nomad_1_6; # nomad 1.6 is the newest version under an foss license. dropPrivileges = false; enableDocker = false; @@ -56,9 +64,14 @@ in server = { enabled = cfg.server.enable; - bootstrap_expect = (builtins.length cfg.servers + 2) / 2; + bootstrap_expect = builtins.length cfg.servers; # why not this? Why this weird formular? (${NOMAD_SERVERS} + 2) / 2 ? server_join.retry_join = cfg.servers; }; + advertise = let + address = config.skyflake.nodes.${config.networking.hostName}.address; + in { + serf = "${address}:4648"; + }; client = { enabled = cfg.client.enable; inherit (cfg.client) meta; @@ -75,7 +88,7 @@ in # alternatives to the nomad web ui wander damon # needed for microvms - virtiofsd ceph + virtiofsd jq kmod e2fsprogs ]; }; diff --git a/nixos-modules/ssh-deployOptions.nix b/nixos-modules/ssh-deployOptions.nix new file mode 100644 index 0000000..482e955 --- /dev/null +++ b/nixos-modules/ssh-deployOptions.nix @@ -0,0 +1,68 @@ +{ config, lib, ... }: +{ + options.skyflake = with lib; { + deploy = { + datacenters = mkOption { + type = with types; listOf str; + default = [ config.skyflake.nomad.datacenter ]; + description = '' + List of datacenters to deploy to. + ''; + }; + + binaryCachePath = mkOption { + type = types.str; + default = "/var/lib/skyflake/binary-cache"; + description = '' + Directory which is mounted on all nodes that will be used to + share the /nix/store with MicroVMs. + ''; + }; + + sharedGcrootsPath = mkOption { + type = types.str; + default = "/nix/var/nix/gcroots/skyflake"; + description = '' + Directory which is mounted on all nodes, is linked from + /nix/var/nix/gcroots/, and contains links to all currently + required microvms. + ''; + }; + + customizationModule = mkOption { + type = types.path; + default = ../default-customization.nix; + description = '' + NixOS module to add when extending a guest NixOS configuration + with MicroVM settings. + ''; + }; + }; + + gc = { + cron = mkOption { + type = types.str; + default = "@hourly"; + description = lib.mdDoc '' + See `cron` in https://developer.hashicorp.com/nomad/docs/job-specification/periodic#periodic-parameters + ''; + }; + }; + + microvmUid = mkOption { + type = types.int; + default = 999; + description = '' + A fixed UID for MicroVM files makes sense for the whole cluster. + ''; + }; + + debug = mkOption { + type = types.bool; + default = false; + description = '' + Enable debug output. Do not use in production! + ''; + }; + }; +} \ No newline at end of file diff --git a/nixos-modules/storage/ceph/server.nix b/nixos-modules/storage/ceph/server.nix index 5c1779f..cde3ee1 100644 --- a/nixos-modules/storage/ceph/server.nix +++ b/nixos-modules/storage/ceph/server.nix @@ -51,6 +51,11 @@ let in { options.skyflake.storage.ceph = { + enable = lib.mkOption { + type = lib.types.bool; + default = false; + example = true; + }; package = lib.mkPackageOption pkgs "ceph" { }; fsid = lib.mkOption { type = lib.types.str; @@ -100,7 +105,7 @@ in { }; rbdPools = lib.mkOption { default = {}; - type = with lib.types; attrsOf (submodule ({ name, ... }: { + type = with lib.types; attrsOf (submodule ({ ... }: { options = { params = poolParamsOpts; }; @@ -126,7 +131,7 @@ in { }; }; - config = { + config = lib.mkIf config.skyflake.storage.ceph.enable { boot.kernelModules = [ "ceph" ]; environment.systemPackages = [ cfg.package ]; @@ -143,10 +148,12 @@ in { enable = true; global = rec { inherit (cfg) fsid; - publicNetwork = clusterNetwork; #"0.0.0.0/0, ::/0"; + publicNetwork = clusterNetwork; clusterNetwork = lib.concatStringsSep ", " ( lib.concatMap ({ addresses ? [], ... }: - lib.concatMap ({ addressConfig ? {}, ... }: + # Needs to be changed because of + # trace: warning: Using 'addressConfig' is deprecated! Move all attributes inside one level up and remove it. + lib.concatMap ({ addressConfig ? {}, ... }: if addressConfig ? Address then [ addressConfig.Address ] else [] @@ -387,4 +394,4 @@ in { ) cfg.cephfs )); }; -} +} \ No newline at end of file diff --git a/nixos-modules/ssh-deploy.nix b/nixos-modules/storage/ceph/ssh-deploy.nix similarity index 72% rename from nixos-modules/ssh-deploy.nix rename to nixos-modules/storage/ceph/ssh-deploy.nix index 97672d9..c102605 100644 --- a/nixos-modules/ssh-deploy.nix +++ b/nixos-modules/storage/ceph/ssh-deploy.nix @@ -79,7 +79,7 @@ let SYSTEM=\$(readlink "$SYSTEMS/\$NAME") # Copy to shared store - sudo nix copy --to file://${cfg.binaryCachePath} --no-check-sigs "\$SYSTEM" + sudo /run/current-system/sw/bin/nix copy --to file://${cfg.binaryCachePath} --no-check-sigs "\$SYSTEM" # Register gcroot mkdir -p "${cfg.sharedGcrootsPath}/$USER/$REPO" rm -f "${cfg.sharedGcrootsPath}/$USER/$REPO/\$NAME" @@ -141,84 +141,19 @@ let "no-user-rc" "restrict" ]; - cfg = config.skyflake.deploy; - gcCfg = config.skyflake.gc; in { - options.skyflake = with lib; { - deploy = { - datacenters = mkOption { - type = with types; listOf str; - default = [ config.skyflake.nomad.datacenter ]; - description = '' - List of datacenters to deploy to. - ''; - }; - - binaryCachePath = mkOption { - type = types.str; - default = cephfs.skyflake-binary-cache.mountPoint; - description = '' - Directory which is mounted on all nodes that will be used to - share the /nix/store with MicroVMs. - ''; - }; - - sharedGcrootsPath = mkOption { - type = types.str; - default = cephfs.skyflake-gcroots.mountPoint; - description = '' - Directory which is mounted on all nodes, is linked from - /nix/var/nix/gcroots/, and contains links to all currently - required microvms. - ''; - }; - - customizationModule = mkOption { - type = types.path; - default = ../default-customization.nix; - description = '' - NixOS module to add when extending a guest NixOS configuration - with MicroVM settings. - ''; - }; - }; - - gc = { - cron = mkOption { - type = types.str; - default = "@hourly"; - description = lib.mdDoc '' - See `cron` in https://developer.hashicorp.com/nomad/docs/job-specification/periodic#periodic-parameters - ''; - }; - }; - - microvmUid = mkOption { - type = types.int; - default = 999; - description = '' - A fixed UID for MicroVM files makes sense for the whole cluster. - ''; - }; - - debug = mkOption { - type = types.bool; - default = false; - description = '' - Enable debug output. Do not use in production! - ''; - }; - }; - - config = { + config = lib.mkIf config.skyflake.storage.ceph.enable { skyflake.storage.ceph.cephfs = { skyflake-binary-cache.mountPoint = "/var/lib/skyflake/binary-cache"; skyflake-gcroots.mountPoint = "/nix/var/nix/gcroots/skyflake"; }; - services.openssh.enable = true; + services.openssh = { + enable = true; + openFirewall = true; + }; users.users = builtins.mapAttrs (_: userConfig: { openssh.authorizedKeys.keys = map (sshKey: @@ -230,7 +165,7 @@ in { }; environment.etc."skyflake/vm".source = pkgs.substituteAllFiles { - src = ../vm; + src = ../../../vm; files = [ "." ]; inherit (config.skyflake.deploy) binaryCachePath customizationModule; }; @@ -241,13 +176,9 @@ in { # allowing commands to copy to/from shared store security.sudo = { enable = true; - extraRules = [ { - groups = [ "users" ]; - commands = [ { - command = ''/run/current-system/sw/bin/nix copy --to file\://${cfg.binaryCachePath} *''; - options = [ "NOPASSWD" ]; - } ]; - } ]; + extraConfig = '' + %users ALL=(ALL:ALL) NOPASSWD: /run/current-system/sw/bin/nix copy --to file\://${cfg.binaryCachePath} * + ''; }; systemd.tmpfiles.rules = diff --git a/nixos-modules/storage/seaweedfs/db-backend/etcd/default.nix b/nixos-modules/storage/seaweedfs/db-backend/etcd/default.nix new file mode 100644 index 0000000..2ddb9e0 --- /dev/null +++ b/nixos-modules/storage/seaweedfs/db-backend/etcd/default.nix @@ -0,0 +1,119 @@ +{ + pkgs, + config, + lib, + ... +}: +{ + config = + lib.mkIf + (builtins.all (x: x == true) [ + config.skyflake.storage.seaweedfs.filer.db.etcd.enable + config.skyflake.storage.seaweedfs.enable + ]) + { + systemd.tmpfiles.settings."10-etcd"."/var/lib/etcd".d = { + user = "etcd"; + mode = "0700"; + }; + + networking.firewall.allowedTCPPorts = [ 2380 ]; + + systemd.services."etcd" = { + description = "etcd key-value store"; + wantedBy = [ "multi-user.target" ]; + after = [ + "network-online.target" + "network.target" + ] ++ lib.optionals config.networking.firewall.enable [ "firewall.service" ]; + wants = + [ "network-online.target" ] + ++ lib.optionals config.networking.firewall.enable [ + "firewall.service" + ]; + environment = + # (nixpgs.filterAttrs (n: v: v != null) + let + address = config.skyflake.nodes.${config.networking.hostName}.address; + in + { + ETCD_NAME = config.networking.hostName; + ETCD_DATA_DIR = "/var/lib/etcd"; + ETCD_ADVERTISE_CLIENT_URLS = "https://${address}:2379"; + ETCD_LISTEN_CLIENT_URLS = "https://${address}:2379"; + ETCD_LISTEN_PEER_URLS = "https://${address}:2380"; + ETCD_INITIAL_ADVERTISE_PEER_URLS = "https://${address}:2380"; + ETCD_CLIENT_CERT_AUTH = "true"; + ETCD_CIPHER_SUITES = "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384"; # allow only strong ciphers. + ETCD_TRUSTED_CA_FILE = "${config.skyflake.storage.seaweedfs.filer.db.etcd.trustedCaFile}"; + ETCD_CERT_FILE = "${config.skyflake.storage.seaweedfs.filer.db.etcd.certFile}"; + ETCD_KEY_FILE = "${config.skyflake.storage.seaweedfs.filer.db.etcd.keyFile}"; + ETCD_PEER_CLIENT_CERT_AUTH = "true"; + ETCD_PEER_TRUSTED_CA_FILE = "${config.skyflake.storage.seaweedfs.filer.db.etcd.peerTrustedCaFile}"; + ETCD_PEER_CERT_FILE = "${config.skyflake.storage.seaweedfs.filer.db.etcd.peerCertFile}"; + ETCD_PEER_KEY_FILE = "${config.skyflake.storage.seaweedfs.filer.db.etcd.peerKeyFile}"; + ETCD_INITIAL_CLUSTER = "${lib.concatMapStringsSep "," ( + node: "${node}=https://" + (config.skyflake.nodes."${node}").address + ":2380" + ) (builtins.attrNames config.skyflake.nodes)}"; + ETCD_INITIAL_CLUSTER_STATE = "new"; + ETCD_INITIAL_CLUSTER_TOKEN = "etcd-cluster"; + }; + unitConfig = { + Documentation = "https://etcd.io/docs/v3.5/"; + }; + + serviceConfig = { + Type = "notify"; + Restart = "always"; + RestartSec = "5s"; + ExecStartPre = "${pkgs.coreutils}/bin/sleep 2"; # TODO fix workaround, so that it doesnt stop on first start because it cant bind. + ExecStart = "${pkgs.etcd}/bin/etcd"; + User = "etcd"; + LimitNOFILE = 40000; + }; + }; + + environment.etc.seaweedfs-filer = { + text = '' + [etcd] + enabled = true + servers = "${ + lib.concatMapStringsSep "," (node: (config.skyflake.nodes."${node}").address + ":2379") ( + builtins.attrNames config.skyflake.nodes + ) + }" + # username = "seaweedfs" + # password = "" + key_prefix = "seaweedfs." + timeout = "3s" + # Set the CA certificate path + tls_ca_file = "${config.skyflake.storage.seaweedfs.filer.db.etcd.peerTrustedCaFile}" + # Set the client certificate path + tls_client_crt_file = "${config.skyflake.storage.seaweedfs.filer.db.etcd.peerCertFile}" + # Set the client private key path + tls_client_key_file = "${config.skyflake.storage.seaweedfs.filer.db.etcd.peerKeyFile}" + ''; + target = "./seaweedfs/filer.toml"; + user = "seaweedfs"; + mode = "0440"; + }; + + environment.systemPackages = [ pkgs.etcd ]; + /* + TODO: add firewall to skyflake. + networking.firewall = lib.mkIf config.services.etcd.openFirewall { + allowedTCPPorts = [ + 2379 # for client requests + 2380 # for peer communication + ]; + }; + */ + users.users.etcd = { + isSystemUser = true; + group = "etcd"; + description = "Etcd daemon user"; + home = "/var/lib/etcd"; # TODO bring it under a single setting, the state path. + }; + users.groups.etcd = { }; + }; +} diff --git a/nixos-modules/storage/seaweedfs/db-backend/etcd/options.nix b/nixos-modules/storage/seaweedfs/db-backend/etcd/options.nix new file mode 100644 index 0000000..15b9cb5 --- /dev/null +++ b/nixos-modules/storage/seaweedfs/db-backend/etcd/options.nix @@ -0,0 +1,50 @@ +{ + lib, + config, + options, + ... +}: +{ + options.skyflake.storage.seaweedfs.filer.db.etcd = { + enable = lib.mkOption { + type = lib.types.bool; + default = true; + description = '' + Use to enable Kubernetes etcd database as a backend for seaweedfs. + ''; + }; + trustedCaFile = lib.mkOption { + description = "Certificate authority file to use for clients"; + default = null; + type = lib.types.nullOr lib.types.path; + }; + certFile = lib.mkOption { + description = "Cert file to use for clients"; + default = null; + type = lib.types.nullOr lib.types.path; + }; + keyFile = lib.mkOption { + description = "Key file to use for clients"; + default = null; + type = lib.types.nullOr lib.types.path; + }; + peerCertFile = lib.mkOption { + description = "Cert file to use for peer to peer communication"; + default = config.skyflake.storage.seaweedfs.filer.db.etcd.certFile; + defaultText = lib.literalExpression "config.${options.skyflake.storage.seaweedfs.filer.db.etcd.certFile}"; + type = lib.types.nullOr lib.types.path; + }; + peerKeyFile = lib.mkOption { + description = "Key file to use for peer to peer communication"; + default = config.skyflake.storage.seaweedfs.filer.db.etcd.keyFile; + defaultText = lib.literalExpression "config.${options.skyflake.storage.seaweedfs.filer.db.etcd.keyFile}"; + type = lib.types.nullOr lib.types.path; + }; + peerTrustedCaFile = lib.mkOption { + description = "Certificate authority file to use for peer to peer communication"; + default = config.skyflake.storage.seaweedfs.filer.db.etcd.trustedCaFile; + defaultText = lib.literalExpression "config.${options.skyflake.storage.seaweedfs.filer.db.etcd.trustedCaFile}"; + type = lib.types.nullOr lib.types.path; + }; + }; +} diff --git a/nixos-modules/storage/seaweedfs/options.nix b/nixos-modules/storage/seaweedfs/options.nix new file mode 100644 index 0000000..1be0943 --- /dev/null +++ b/nixos-modules/storage/seaweedfs/options.nix @@ -0,0 +1,112 @@ +{ + lib, + ... +}: +{ + options.skyflake.storage.seaweedfs = { + enable = lib.mkOption { + type = lib.types.bool; + default = false; + description = '' + enable seaweedfs as the storage backend. + ''; + }; + volumeStorage = { + encrypt = lib.mkOption { + type = lib.types.bool; + default = false; + description = '' + enable encryption on volume store. + ''; + }; + #datacenter = { + # type = lib.str; + # description = '' + # The datacenter location of the node. + # ''; + #}; + #rack = { + # type = lib.str; + # description = '' + # The rack location of the node. + # ''; + #}; + }; + s3 = { + enable = lib.mkOption { + type = lib.types.bool; + default = false; + description = '' + If you want to expose an S3 compatible bucket. + ''; + }; + port = lib.mkOption { + type = lib.types.port; + default = 8333; + description = '' + The port the S3 API should listen to. + ''; + }; + }; + mounts = lib.mkOption { + type = lib.types.attrsOf ( + lib.types.submodule { + options = { + mountSource = lib.mkOption { + type = lib.types.str; + default = null; + example = "/filesystems/1a32bfd9-0cbc-430a-a28a-d9fd862e9ebc"; + description = '' + Place where the filesystem is saved in seaweedfs. + ''; + }; + replication = lib.mkOption { + type = lib.types.nullOr lib.types.ints.u8; + default = null; + description = '' + Is the replication level for each file. + It overwrites replication settings on both filer and master. + ''; + }; + cacheCapacity = lib.mkOption { + type = lib.types.ints.unsigned; + default = 0; + description = '' + Means file chunk read cache capacity in MB with tiered cache(memory + disk), + default 0 which means chunk cache for read is disabled. + ''; + }; + chunkSizeLimit = lib.mkOption { + type = lib.types.ints.positive; + default = 2; + description = '' + Local write buffer size, also chunk large file, default 2 MB. + ''; + }; + }; + } + ); + }; + filer = { + #TODO + size = lib.mkOption { + type = lib.types.ints.unsigned; + default = 30000; + description = '' + TODO + ''; + }; + #TODO + deviceClass = lib.mkOption { + type = lib.types.str; + default = "unset"; + example = '' + `NVME` `SSD` `HDD` + ''; + description = '' + hard drive or solid state drive or any tag. + ''; + }; + }; + }; +} diff --git a/nixos-modules/storage/seaweedfs/server.nix b/nixos-modules/storage/seaweedfs/server.nix new file mode 100644 index 0000000..ea73f2d --- /dev/null +++ b/nixos-modules/storage/seaweedfs/server.nix @@ -0,0 +1,188 @@ +{ + config, + lib, + pkgs, + ... +}: +{ + config = lib.mkIf config.skyflake.storage.seaweedfs.enable { + + networking.firewall = { + allowedTCPPorts = + [ + 9333 # seaweedfs Master + 19333 # seaweedfs Master + # TODO Make option to block web UI + 8080 # seaweedfs Volume + 18080 # seaweedfs Volume + 8888 # seaweedfs Filer + 18888 # seaweedfs Filer + ] + ++ lib.optionals config.skyflake.storage.seaweedfs.s3.enable [ + config.skyflake.storage.seaweedfs.s3.port + ]; + }; + + users.users.seaweedfs = { + isSystemUser = true; + group = "seaweedfs"; + description = "seaweedfs daemon user"; + home = "/var/lib/seaweedfs"; # TODO bring it under a single setting, the state path. + createHome = true; + }; + users.groups.seaweedfs = { }; + + # config for the volume deamon of seaweedfs + systemd.tmpfiles.settings."10-seaweedfs-volume"."/var/lib/seaweedfs/volume".d = { + user = "seaweedfs"; + group = "seaweedfs"; + mode = "0700"; + }; + + # config for the master deamon of seaweedfs + systemd.tmpfiles.settings."10-seaweedfs-master"."/var/lib/seaweedfs/master".d = { + user = "seaweedfs"; + group = "seaweedfs"; + mode = "0700"; + }; + + systemd.services = lib.mkMerge [ + { + seaweedfs-master = { + description = "seaweedfs master service"; + wantedBy = [ "multi-user.target" ]; + after = [ + "network-online.target" + "etcd.service" + ] ++ lib.optionals config.networking.firewall.enable [ "firewall.service" ]; + wants = [ + "network-online.target" + "etcd.service" + ] ++ lib.optionals config.networking.firewall.enable [ "firewall.service" ]; + unitConfig = { + Documentation = "https://github.com/seaweedfs/seaweedfs/wiki"; + }; + serviceConfig = + let + address = config.skyflake.nodes.${config.networking.hostName}.address; + peers = "${lib.concatMapStrings (x: x + ":9333,") ( + builtins.catAttrs "address" (builtins.attrValues config.skyflake.nodes) + )}"; + in + { + Type = "simple"; + Restart = "always"; + RestartSec = "5s"; + ExecStart = ''${pkgs.seaweedfs}/bin/weed master -ip=${address} -peers=${peers} -mdir=/var/lib/seaweedfs/master''; + User = "seaweedfs"; + LimitNOFILE = 40000; + }; + }; + + # config for the filer deamon of seaweedfs + seaweedfs-filer = { + description = "seaweedfs filer service"; + wantedBy = [ "multi-user.target" ]; + after = [ + "network-online.target" + "etcd.service" + "seaweedfs-master.service" + ] ++ lib.optionals config.networking.firewall.enable [ "firewall.service" ]; + wants = [ + "network-online.target" + "etcd.service" + "seaweedfs-master.service" + ] ++ lib.optionals config.networking.firewall.enable [ "firewall.service" ]; + unitConfig = { + Documentation = "https://github.com/seaweedfs/seaweedfs/wiki"; + }; + serviceConfig = + let + address = config.skyflake.nodes.${config.networking.hostName}.address; + in + { + Type = "simple"; + Restart = "always"; + RestartSec = "5s"; + # TODO make userdefinable port + ExecStart = ''${pkgs.seaweedfs}/bin/weed filer ${lib.optionalString config.skyflake.storage.seaweedfs.volumeStorage.encrypt "-encryptVolumeData"} -master=${address}:9333 -port=8888''; + User = "seaweedfs"; + LimitNOFILE = 40000; + }; + }; + + seaweedfs-volume = { + description = "seaweedfs volume service"; + wantedBy = [ "multi-user.target" ]; + after = [ + "network-online.target" + "etcd.service" + "seaweedfs-master.service" + ] ++ lib.optionals config.networking.firewall.enable [ "firewall.service" ]; + wants = [ + "network-online.target" + "etcd.service" + "seaweedfs-master.service" + ] ++ lib.optionals config.networking.firewall.enable [ "firewall.service" ]; + unitConfig = { + Documentation = "https://github.com/seaweedfs/seaweedfs/wiki"; + }; + serviceConfig = + let + address = config.skyflake.nodes.${config.networking.hostName}.address; + in + { + Type = "simple"; + Restart = "always"; + RestartSec = "5s"; + # TODO add S3 bucket support in here: https://github.com/seaweedfs/seaweedfs/wiki/Production-Setup#setup-s3-api + ExecStart = ''${pkgs.seaweedfs}/bin/weed volume -port=8080 -max=5 -ip=${address} -mserver=localhost:9333 -dir=/var/lib/seaweedfs/volume''; + User = "seaweedfs"; + LimitNOFILE = 40000; + }; + }; + } + + # config for the mount deamon of seaweedfs + (lib.mapAttrs' ( + name: value: + lib.nameValuePair ("seaweedfs-mount" + (lib.replaceStrings [ "/" ] [ "-" ] name)) { + description = "seaweedfs mount service"; + wantedBy = [ "multi-user.target" ]; + after = [ + "network-online.target" + "etcd.service" + "seaweedfs-filer.service" + ] ++ lib.optionals config.networking.firewall.enable [ "firewall.service" ]; + wants = [ + "network-online.target" + "etcd.service" + "seaweedfs-filer.service" + ] ++ lib.optionals config.networking.firewall.enable [ "firewall.service" ]; + unitConfig = { + Documentation = "https://github.com/seaweedfs/seaweedfs/wiki"; + }; + path = with pkgs; [ fuse3 ]; + serviceConfig = { + Type = "simple"; + Restart = "always"; + RestartSec = "5s"; + #TODO FIX hardcoding of port https://github.com/seaweedfs/seaweedfs/issues/877 + ExecStart = ''${pkgs.seaweedfs}/bin/weed mount -nonempty -filer=localhost:8888 ${ + lib.optionalString ( + !builtins.isNull value.replication + ) "-replication=${builtins.toString value.replication}" + } -cacheCapacityMB=${builtins.toString value.cacheCapacity} -chunkSizeLimitMB=${builtins.toString value.chunkSizeLimit} -dirAutoCreate -dir=${builtins.toString name} -filer.path=${value.mountSource}''; + # TODO FIX mount with root!!! + User = "root"; + LimitNOFILE = 40000; + }; + } + ) config.skyflake.storage.seaweedfs.mounts) + ]; + + environment.systemPackages = with pkgs; [ + seaweedfs # install seaweedfs utils + ]; + }; +} diff --git a/nixos-modules/storage/seaweedfs/ssh-deploy.nix b/nixos-modules/storage/seaweedfs/ssh-deploy.nix new file mode 100644 index 0000000..c8eb2b2 --- /dev/null +++ b/nixos-modules/storage/seaweedfs/ssh-deploy.nix @@ -0,0 +1,211 @@ +{ microvm, nixpkgs }: + +{ config, lib, pkgs, ... }: + +let + inherit (config.skyflake.storage.seaweedfs) mounts; + + debugShell = lib.optionalString config.skyflake.debug '' + set -x + ''; + + deployCommand = with pkgs; writeScript "skyflake-ssh-deploy" '' + #! ${runtimeShell} -e + ${debugShell} + + PATH=${lib.makeBinPath ([ + git + ])}:$PATH + + if [[ "$SSH_ORIGINAL_COMMAND" =~ ^git-receive-pack\ \'([\\-_a-zA-Z0-9]+)\'$ ]]; then + REPO="''${BASH_REMATCH[1]}" + if ! [ -e $REPO ]; then + echo "Creating $REPO anew..." >&2 + mkdir $REPO + cd $REPO + ${git}/bin/git init --bare -b main >/dev/null + else + echo "Updating existing $REPO" >&2 + cd $REPO + fi + + SYSTEMS=$(mktemp --tmpdir -d deploy-systems-XXXXXXXX) + + cat > hooks/update <&2 + nomad namespace apply "$USER-$REPO" >/dev/null + + for NAME in * ; do + SYSTEM=$(readlink $NAME) + echo $SYSTEM >&2 + nomad run -detach "$SYSTEM" >&2 + + # Register gcroot + mkdir -p "${cfg.sharedGcrootsPath}/$USER/$REPO" + rm -f "${cfg.sharedGcrootsPath}/$USER/$REPO/$NAME" + ln -s "$SYSTEM" "${cfg.sharedGcrootsPath}/$USER/$REPO/$NAME" + done + cd - + rm -r $SYSTEMS + echo All done >&2 + + elif [[ "$SSH_ORIGINAL_COMMAND" =~ ^git-upload-pack\ \'([\\-_a-zA-Z0-9]+)\'$ ]]; then + REPO="''${BASH_REMATCH[1]}" + exec git-upload-pack "$REPO" + + elif [[ "$SSH_ORIGINAL_COMMAND" = status ]]; then + NAMESPACES=$(nomad namespace list -t "{{ range . }}{{ .Name }} + {{ end }}"|grep -e "^$USER-") + for NAMESPACE in $NAMESPACES ; do + nomad job status -namespace "$NAMESPACE" + done + + else + echo "Invalid SSH command: $SSH_ORIGINAL_COMMAND" >&2 + exit 1 + fi + ''; + + sshKeyOpts = [ + "command=\"${deployCommand}\"" + "no-port-forwarding" + "no-X11-forwarding" + "no-agent-forwarding" + "no-pty" + "no-user-rc" + "restrict" + ]; + cfg = config.skyflake.deploy; + +in { + config = lib.mkIf config.skyflake.storage.seaweedfs.enable { + skyflake.storage.seaweedfs.mounts = { + ${config.skyflake.deploy.binaryCachePath}.mountSource = "/skyflake-internals${config.skyflake.deploy.binaryCachePath}"; + ${config.skyflake.deploy.sharedGcrootsPath}.mountSource = "/skyflake-internals${config.skyflake.deploy.sharedGcrootsPath}"; + }; + + services.openssh = { + enable = true; + openFirewall = true; + }; + + users.users = builtins.mapAttrs (_: userConfig: { + openssh.authorizedKeys.keys = map (sshKey: + "${lib.concatStringsSep "," sshKeyOpts} ${sshKey}" + ) userConfig.sshKeys; + }) config.skyflake.users // { + # stable uid is useful across network filesystems + microvm.uid = config.skyflake.microvmUid; + }; + + environment.etc."skyflake/vm".source = pkgs.substituteAllFiles { + src = ../../../vm; + files = [ "." ]; + inherit (config.skyflake.deploy) binaryCachePath customizationModule; + }; + + # lets the hook use $binaryCachePath + nix.settings.trusted-users = builtins.attrNames config.skyflake.users; + + # allowing commands to copy to/from shared store + security.sudo = { + enable = true; + extraConfig = '' + %users ALL=(ALL:ALL) NOPASSWD: /run/current-system/sw/bin/nix copy --to file\://${cfg.binaryCachePath} * + ''; + + }; + + systemd.tmpfiles.rules = + [ + # workDir for nomad jobs + "d /run/microvms 0700 microvm kvm - -" + "d ${cfg.binaryCachePath} 0777 root root - -" + ] + ++ + map (userName: + "d ${config.skyflake.deploy.sharedGcrootsPath}/${userName} 0750 ${userName} root - -" + ) (builtins.attrNames config.skyflake.users); + + systemd.services.skyflake-permissions = { + wantedBy = [ "multi-user.target" ]; + after = [ "remote-fs.target" ]; + script = '' + mkdir -p ${cfg.binaryCachePath} + chmod 0777 ${cfg.binaryCachePath} + + ${lib.concatMapStrings (userName: '' + D="${config.skyflake.deploy.sharedGcrootsPath}/${userName}" + mkdir -p "$D" + chown "${userName}" "$D" + '') (builtins.attrNames config.skyflake.users)} + ''; + }; + }; +} diff --git a/vm/build-vm.nix b/vm/build-vm.nix index 8836844..437fca7 100644 --- a/vm/build-vm.nix +++ b/vm/build-vm.nix @@ -11,7 +11,6 @@ let nixpkgs = builtins.getFlake nixpkgsRef; pkgs = nixpkgs.legacyPackages.${system}; - inherit (pkgs) lib; microvm = builtins.getFlake microvmFlake; flake = builtins.getFlake flakeRef; @@ -46,7 +45,6 @@ let }; } # From the host's skyflake.deploy.customizationModule - @customizationModule@ ]; }; @@ -64,4 +62,4 @@ import ./nomad-job.nix { inherit pkgs runner; inherit (extended) config; -} +} \ No newline at end of file diff --git a/vm/customization-options.nix b/vm/customization-options.nix index 535a243..9aba435 100644 --- a/vm/customization-options.nix +++ b/vm/customization-options.nix @@ -20,7 +20,7 @@ ''; }; - deploy.rbds = mkOption { + deploy.ceph.rbds = mkOption { default = {}; description = '' Ceph RBDs used by this MicroVM @@ -46,7 +46,7 @@ }; fsType = mkOption { type = str; - default = "ext4"; + default = "btrfs"; description = '' Which mkfs to use when `autoCreate = true` ''; diff --git a/vm/nomad-job.nix b/vm/nomad-job.nix index 5f2a51d..06ee0df 100644 --- a/vm/nomad-job.nix +++ b/vm/nomad-job.nix @@ -64,7 +64,7 @@ let } '') config.skyflake.nomadJob.affinities} - ${lib.concatMapStrings (interface@{ id, ... }: '' + ${lib.concatMapStrings ({ id, ... }: '' task "add-interface-${id}" { lifecycle { hook = "prestart" @@ -120,7 +120,7 @@ ${'' } '') config.microvm.interfaces} - ${lib.concatMapStrings (share@{ tag, source, socket, proto, ... }: + ${lib.concatMapStrings ({ tag, source, socket, proto, ... }: lib.optionalString (proto == "virtiofs") '' task "virtiofsd-${tag}" { lifecycle { @@ -207,7 +207,7 @@ ${'' } } - ${lib.concatMapStrings (id: with config.skyflake.deploy.rbds.${id}; '' + ${lib.concatMapStrings (id: with config.skyflake.deploy.ceph.rbds.${id}; '' task "rbd-map-${id}" { driver = "raw_exec" lifecycle { @@ -270,7 +270,7 @@ ${'' EOD } } - '') (builtins.attrNames config.skyflake.deploy.rbds)} + '') (builtins.attrNames config.skyflake.deploy.ceph.rbds)} task "hypervisor" { driver = "raw_exec" @@ -347,4 +347,4 @@ pkgs.stdenv.mkDerivation rec { installPhase = '' cp $NAME $out ''; -} +} \ No newline at end of file