From cebf26e1877e16cf19239b7ddf3213e09f129b92 Mon Sep 17 00:00:00 2001 From: Makuru Date: Sat, 8 Jun 2024 17:21:07 +0200 Subject: [PATCH 01/41] Start of modernization of skyflake. --- example-server.nix | 48 ++++++----- flake.lock | 48 +++++++---- flake.nix | 18 ++-- nixos-modules/cache-cut.nix | 2 +- nixos-modules/nodes.nix | 2 +- nixos-modules/nomad.nix | 4 +- nixos-modules/ssh-deploy.nix | 12 ++- nixos-modules/storage/ceph/server.nix | 13 ++- .../storage/seaweedfs/dbBackend/etcd.nix | 70 ++++++++++++++++ .../storage/seaweedfs/dbBackend/sqlite.nix | 59 +++++++++++++ nixos-modules/storage/seaweedfs/options.nix | 83 +++++++++++++++++++ vm/build-vm.nix | 4 +- vm/customization-options.nix | 1 + vm/nomad-job.nix | 4 +- 14 files changed, 311 insertions(+), 57 deletions(-) create mode 100644 nixos-modules/storage/seaweedfs/dbBackend/etcd.nix create mode 100644 nixos-modules/storage/seaweedfs/dbBackend/sqlite.nix create mode 100644 nixos-modules/storage/seaweedfs/options.nix diff --git a/example-server.nix b/example-server.nix index 72d06b9..2b1fb13 100644 --- a/example-server.nix +++ b/example-server.nix @@ -1,10 +1,10 @@ { instance }: -{ config, lib, pkgs, ... }: +{ config, pkgs, ... }: { microvm = { - vcpu = 2; + vcpu = 4; mem = 4096; shares = [ { @@ -34,7 +34,7 @@ networking.hostName = "example${toString instance}"; users.users.root.password = ""; - # TODO: + # TODO:? Shoulld firewall be fixed? networking.firewall.enable = false; networking.useDHCP = false; @@ -64,7 +64,9 @@ IPv6AcceptRA = true; }; addresses = [ { - addressConfig.Address = "fec0::${toString instance}/64"; + # TODO: addressConfig needs to be removed. + # trace: warning: Using 'addressConfig' is deprecated! Move all attributes inside one level up and remove it. + addressConfig.Address = "fec0::${toString instance}/64"; # } ]; }; }; @@ -78,22 +80,29 @@ }) [ 1 2 3 ] ); - storage.ceph = rec { - fsid = "8364da79-5e03-49ae-82ea-7d936278cb0f"; - monKeyring = example/ceph.mon.keyring; - adminKeyring = example/ceph.client.admin.keyring; - osds = [ { - id = instance; - fsid = "8e4ae689-5c15-4381-bd75-19de743378e${toString instance}"; - path = "/dev/vdb"; - deviceClass = "ssd"; - keyfile = toString (./example + "/osd.${toString instance}.keyring"); - } ]; - rbdPools.microvms = { - params = { size = 2; class = "ssd"; }; + + storage.seaweedfs = { + enable = true; + filer.db.etcd = { + enable = true; }; - cephfs.skyflake.metaParams = { size = 2; class = "ssd"; }; }; + #storage.ceph = { + # fsid = "8364da79-5e03-49ae-82ea-7d936278cb0f"; + # monKeyring = example/ceph.mon.keyring; + # adminKeyring = example/ceph.client.admin.keyring; + # osds = [ { + # id = instance; + # fsid = "8e4ae689-5c15-4381-bd75-19de743378e${toString instance}"; + # path = "/dev/vdb"; + # deviceClass = "ssd"; + # keyfile = toString (./example + "/osd.${toString instance}.keyring"); + # } ]; + # rbdPools.microvms = { + # params = { size = 2; class = "ssd"; }; + # }; + # cephfs.skyflake.metaParams = { size = 2; class = "ssd"; }; + #}; nomad = { servers = [ "example1" "example2" "example3" ]; @@ -107,6 +116,7 @@ uid = 1000; sshKeys = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGJJTSJdpDh82486uPiMhhyhnci4tScp5uUe7156MBC8 astro" + "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPRRdToCDUupkkwI+crB3fGDwdBIFkDsBHjOImn+qsjg openpgp:0xE8D3D833" ]; }; }; @@ -115,4 +125,4 @@ environment.systemPackages = with pkgs; [ tcpdump ]; -} +} \ No newline at end of file diff --git a/flake.lock b/flake.lock index d4894c9..2ea9f5d 100644 --- a/flake.lock +++ b/flake.lock @@ -5,11 +5,11 @@ "systems": "systems" }, "locked": { - "lastModified": 1692799911, - "narHash": "sha256-3eihraek4qL744EvQXsK1Ha6C3CR7nnT8X2qWap4RNk=", + "lastModified": 1705309234, + "narHash": "sha256-uNRRNRKmJyCRC/8y1RqBkqWBLM034y4qN7EprSdmgyA=", "owner": "numtide", "repo": "flake-utils", - "rev": "f9e7cf818399d17d347f847525c5a5a8032e4e44", + "rev": "1ef2e671c3b0c19053962c07dbda38332dcebf26", "type": "github" }, "original": { @@ -23,14 +23,15 @@ "flake-utils": "flake-utils", "nixpkgs": [ "nixpkgs" - ] + ], + "spectrum": "spectrum" }, "locked": { - "lastModified": 1693776909, - "narHash": "sha256-9JRxyhGS0vnciGgiwdzpupN88/K6seBSf9PqEVTL280=", + "lastModified": 1717441449, + "narHash": "sha256-juxjgmLnFbl+/hhIO2cVtIa6caCO4pLKlZWUMwAOznM=", "owner": "astro", "repo": "microvm.nix", - "rev": "85790506c0d131181805ffbd40617580be23c67e", + "rev": "e3a4dd5b381fb580804105594cc9c71dc45abdb5", "type": "github" }, "original": { @@ -99,15 +100,18 @@ }, "nixpkgs_2": { "locked": { - "lastModified": 1693250523, - "narHash": "sha256-y3up5gXMTbnCsXrNEB5j+7TVantDLUYyQLu/ueiXuyg=", - "path": "/nix/store/nvdrwx25ril8dys6yryrp39d14b2p742-source", - "rev": "3efb0f6f404ec8dae31bdb1a9b17705ce0d6986e", - "type": "path" + "lastModified": 1717786204, + "narHash": "sha256-4q0s6m0GUcN7q+Y2DqD27iLvbcd1G50T2lv08kKxkSI=", + "owner": "nixos", + "repo": "nixpkgs", + "rev": "051f920625ab5aabe37c920346e3e69d7d34400e", + "type": "github" }, "original": { - "id": "nixpkgs", - "type": "indirect" + "owner": "nixos", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" } }, "root": { @@ -117,6 +121,22 @@ "nixpkgs": "nixpkgs_2" } }, + "spectrum": { + "flake": false, + "locked": { + "lastModified": 1708358594, + "narHash": "sha256-e71YOotu2FYA67HoC/voJDTFsiPpZNRwmiQb4f94OxQ=", + "ref": "refs/heads/main", + "rev": "6d0e73864d28794cdbd26ab7b37259ab0e1e044c", + "revCount": 614, + "type": "git", + "url": "https://spectrum-os.org/git/spectrum" + }, + "original": { + "type": "git", + "url": "https://spectrum-os.org/git/spectrum" + } + }, "systems": { "locked": { "lastModified": 1681028828, diff --git a/flake.nix b/flake.nix index 42ce46d..97e1370 100644 --- a/flake.nix +++ b/flake.nix @@ -2,8 +2,12 @@ description = "Hyperconverged Infratructure for NixOS"; inputs = { - microvm.url = "github:astro/microvm.nix"; - microvm.inputs.nixpkgs.follows = "nixpkgs"; + nixpkgs.url = "github:nixos/nixpkgs/nixos-unstable"; + microvm = { + url = "github:astro/microvm.nix"; + inputs.nixpkgs.follows = "nixpkgs"; + }; + nix-cache-cut.url = "github:astro/nix-cache-cut"; }; @@ -14,6 +18,7 @@ pkgs = nixpkgs.legacyPackages.${system}; in { + formatter.${system} = pkgs.alejandra; packages.${system} = import ./pkgs/doc.nix { inherit pkgs self; }; @@ -21,7 +26,10 @@ nixosModules = { default = { imports = [ - ./nixos-modules/storage/ceph/server.nix + ./nixos-modules/storage/seaweedfs/options.nix + # ./nixos-modules/storage/seaweedfs/dbBackend/sqlite.nix + ./nixos-modules/storage/seaweedfs/dbBackend/etcd.nix + # ./nixos-modules/storage/ceph/server.nix ./nixos-modules/defaults.nix ./nixos-modules/nodes.nix ./nixos-modules/nomad.nix @@ -71,9 +79,9 @@ make-ceph = { type = "app"; - program = toString (pkgs.callPackage ./pkgs/make-ceph.nix {}); + #program = toString (pkgs.callPackage ./pkgs/make-ceph.nix {}); }; }; }; -} +} \ No newline at end of file diff --git a/nixos-modules/cache-cut.nix b/nixos-modules/cache-cut.nix index 1476f78..492ed57 100644 --- a/nixos-modules/cache-cut.nix +++ b/nixos-modules/cache-cut.nix @@ -41,7 +41,7 @@ in systemd.services.skyflake-install-cache-gc = { wantedBy = [ "multi-user.target" ]; requires = [ "nomad.service" ]; - path = with pkgs; [ nomad ]; + path = [ config.services.nomad.package ]; script = '' nomad run -detach ${jobFile} ''; diff --git a/nixos-modules/nodes.nix b/nixos-modules/nodes.nix index 88d6fd5..e2acd13 100644 --- a/nixos-modules/nodes.nix +++ b/nixos-modules/nodes.nix @@ -9,7 +9,7 @@ in description = '' All cluster nodes with their addresses, or at least those who run coordination servers (eg. nomad servers, ceph server, - ...). + seaweedfs server ...). ''; default = {}; type = types.attrsOf (types.submodule { diff --git a/nixos-modules/nomad.nix b/nixos-modules/nomad.nix index d0defc9..d60b1d4 100644 --- a/nixos-modules/nomad.nix +++ b/nixos-modules/nomad.nix @@ -42,7 +42,7 @@ in config = { services.nomad = { enable = true; - package = pkgs.nomad_1_4; + package = pkgs.nomad_1_6; # nomad 1.6 is the newest version under an foss license. dropPrivileges = false; enableDocker = false; @@ -75,7 +75,7 @@ in # alternatives to the nomad web ui wander damon # needed for microvms - virtiofsd ceph + virtiofsd ceph seaweedfs jq kmod e2fsprogs ]; }; diff --git a/nixos-modules/ssh-deploy.nix b/nixos-modules/ssh-deploy.nix index 97672d9..d2c5362 100644 --- a/nixos-modules/ssh-deploy.nix +++ b/nixos-modules/ssh-deploy.nix @@ -3,7 +3,7 @@ { config, lib, pkgs, ... }: let - inherit (config.skyflake.storage.ceph) cephfs; + inherit (config.skyflake.storage.seaweedfs) seaweedfsMount; debugShell = lib.optionalString config.skyflake.debug '' set -x @@ -143,8 +143,6 @@ let ]; cfg = config.skyflake.deploy; - gcCfg = config.skyflake.gc; - in { options.skyflake = with lib; { deploy = { @@ -158,7 +156,7 @@ in { binaryCachePath = mkOption { type = types.str; - default = cephfs.skyflake-binary-cache.mountPoint; + default = seaweedfsMount.skyflake-binary-cache.mountPoint; description = '' Directory which is mounted on all nodes that will be used to share the /nix/store with MicroVMs. @@ -167,7 +165,7 @@ in { sharedGcrootsPath = mkOption { type = types.str; - default = cephfs.skyflake-gcroots.mountPoint; + default = seaweedfsMount.skyflake-gcroots.mountPoint; description = '' Directory which is mounted on all nodes, is linked from /nix/var/nix/gcroots/, and contains links to all currently @@ -211,9 +209,9 @@ in { ''; }; }; - + config = { - skyflake.storage.ceph.cephfs = { + skyflake.storage.seaweedfs.seaweedfsMount = { skyflake-binary-cache.mountPoint = "/var/lib/skyflake/binary-cache"; skyflake-gcroots.mountPoint = "/nix/var/nix/gcroots/skyflake"; }; diff --git a/nixos-modules/storage/ceph/server.nix b/nixos-modules/storage/ceph/server.nix index 5c1779f..3ce023a 100644 --- a/nixos-modules/storage/ceph/server.nix +++ b/nixos-modules/storage/ceph/server.nix @@ -51,6 +51,11 @@ let in { options.skyflake.storage.ceph = { + enable = lib.mkOption { + type = lib.types.bool; + default = false; + example = true; + }; package = lib.mkPackageOption pkgs "ceph" { }; fsid = lib.mkOption { type = lib.types.str; @@ -100,7 +105,7 @@ in { }; rbdPools = lib.mkOption { default = {}; - type = with lib.types; attrsOf (submodule ({ name, ... }: { + type = with lib.types; attrsOf (submodule ({ ... }: { options = { params = poolParamsOpts; }; @@ -146,7 +151,9 @@ in { publicNetwork = clusterNetwork; #"0.0.0.0/0, ::/0"; clusterNetwork = lib.concatStringsSep ", " ( lib.concatMap ({ addresses ? [], ... }: - lib.concatMap ({ addressConfig ? {}, ... }: + # Needs to be changed because of + # trace: warning: Using 'addressConfig' is deprecated! Move all attributes inside one level up and remove it. + lib.concatMap ({ addressConfig ? {}, ... }: if addressConfig ? Address then [ addressConfig.Address ] else [] @@ -387,4 +394,4 @@ in { ) cfg.cephfs )); }; -} +} \ No newline at end of file diff --git a/nixos-modules/storage/seaweedfs/dbBackend/etcd.nix b/nixos-modules/storage/seaweedfs/dbBackend/etcd.nix new file mode 100644 index 0000000..f70d9c2 --- /dev/null +++ b/nixos-modules/storage/seaweedfs/dbBackend/etcd.nix @@ -0,0 +1,70 @@ +{ pkgs, config, lib, ... }: +{ + config = lib.mkIf config.skyflake.storage.seaweedfs.filer.db.etcd.enable { + systemd.tmpfiles.settings."10-etcd"."/var/lib/etcd".d = { + user = "etcd"; + mode = "0700"; + }; + + systemd.services.etcd = { + description = "etcd key-value store"; + wantedBy = [ "multi-user.target" ]; + after = [ "network-online.target" ]; + # ++ lib.optional config.networking.firewall.enable "firewall.service"; + wants = [ "network-online.target" ]; + # ++ lib.optional config.networking.firewall.enable "firewall.service"; + + environment = /* (nixpgs.filterAttrs (n: v: v != null) */ { + ETCD_NAME = config.networking.hostName; + #ETCD_DISCOVERY = "true"; + ETCD_DATA_DIR = "/var/lib/etcd"; + ETCD_ADVERTISE_CLIENT_URLS = lib.concatMapStringsSep ", " (node: "http://[" + (config.skyflake.nodes."${node}").address + "]:2379") (lib.attrNames config.skyflake.nodes); + ETCD_LISTEN_CLIENT_URLS = "http://[0::]:2379"; + ETCD_LISTEN_PEER_URLS = "http://[0::]:2380"; + #ETCD_INITIAL_ADVERTISE_PEER_URLS = nixpgs.concatStringsSep "," config.skyflake.nodes.address; + # ETCD_PEER_CLIENT_CERT_AUTH = toString config.services.etcd.peerClientCertAuth; + # ETCD_PEER_TRUSTED_CA_FILE = config.services.etcd.peerTrustedCaFile; + # ETCD_PEER_CERT_FILE = config.services.etcd.peerCertFile; + # ETCD_PEER_KEY_FILE = config.services.etcd.peerKeyFile; + # ETCD_CLIENT_CERT_AUTH = toString config.services.etcd.clientCertAuth; + # ETCD_TRUSTED_CA_FILE = config.services.etcd.trustedCaFile; + # ETCD_CERT_FILE = config.services.etcd.certFile; + # ETCD_KEY_FILE = config.services.etcd.keyFile; + #}) // (nixpgs.optionalAttrs (config.services.etcd.discovery == ""){ + ETCD_INITIAL_CLUSTER = lib.concatMapStringsSep ", " (node: "http://[" + (config.skyflake.nodes."${node}").address + "]:2380") (lib.attrNames config.skyflake.nodes); + ETCD_INITIAL_CLUSTER_STATE = "new"; + ETCD_INITIAL_CLUSTER_TOKEN = "etcd-cluster"; + #}) // (nixpgs.mapAttrs' (n: v: nixpgs.nameValuePair "ETCD_${n}" v) config.services.etcd.extraConf); + }; + unitConfig = { + Documentation = "https://github.com/coreos/etcd"; + }; + + serviceConfig = { + Type = "notify"; + Restart = "always"; + RestartSec = "30s"; + ExecStart = "${pkgs.etcd}/bin/etcd"; + User = "etcd"; + LimitNOFILE = 40000; + }; + }; + + environment.systemPackages = [ pkgs.etcd ]; + /* TODO: add firewall to skyflake. + networking.firewall = lib.mkIf config.services.etcd.openFirewall { + allowedTCPPorts = [ + 2379 # for client requests + 2380 # for peer communication + ]; + }; + */ + users.users.etcd = { + isSystemUser = true; + group = "etcd"; + description = "Etcd daemon user"; + home = "/var/lib/etcd"; # TODO bring it under a single setting, the state path. + }; + users.groups.etcd = {}; + }; +} \ No newline at end of file diff --git a/nixos-modules/storage/seaweedfs/dbBackend/sqlite.nix b/nixos-modules/storage/seaweedfs/dbBackend/sqlite.nix new file mode 100644 index 0000000..812c237 --- /dev/null +++ b/nixos-modules/storage/seaweedfs/dbBackend/sqlite.nix @@ -0,0 +1,59 @@ +{ pkgs, lib, config, ... }: + +let + seaweedFS-filer-file = pkgs.formats.toml '' + [sqlite] + # local on disk. + enabled = false + dbFile = "./filer.db" # sqlite db file + ''; + +in { + config = lib.mkIf config.skyflake.storage.seaweedfs.filer.db.sqlite.enable { + systemd.services.seaweedfs-filer = { + description = "seaweedFS filer storage node"; + wantedBy = [ "multi-user.target" ]; + # after = [ "network-online.target" ] # TODO add firewall + # ++ nixpgs.lib.optional config.networking.firewall.enable "firewall.service"; + # wants = [ "network-online.target" ] + # ++ nixpgs.lib.optional config.networking.firewall.enable "firewall.service"; + + #environment = { + #}; + + + unitConfig = { + Documentation = "https://github.com/seaweedfs/seaweedfs/wiki"; + }; + + serviceConfig = { + Type = "notify"; + Restart = "always"; + RestartSec = "30s"; + ExecStart = "${pkgs.seaweedfs}/bin/weed filer"; + User = "seaweedfs-filer"; + Group = "seaweedfs-filer"; + LimitNOFILE = 40000; + WorkingDirectory = "/var/lib/seaweedfs-filer"; + SyslogIdentifier = "seaweedfs-filer"; + }; + }; + users.users.seaweedfs-filer = { + isSystemUser = true; + group = "seaweedfs-filer"; + description = "seaweedfs filer store user"; + home = config.systemd.services.seaweedfs-filer.WorkingDirectory; + }; + users.groups.seaweedfs-filer = {}; + + environment.etc = { + "seaweedFS-filer" = { + source = "${seaweedFS-filer-file}"; + target = "seaweedfs/filer.toml"; + mode = "0440"; + }; + }; + + environment.systemPackages = [ pkgs.seaweedfs ]; + }; +} \ No newline at end of file diff --git a/nixos-modules/storage/seaweedfs/options.nix b/nixos-modules/storage/seaweedfs/options.nix new file mode 100644 index 0000000..a3fdb61 --- /dev/null +++ b/nixos-modules/storage/seaweedfs/options.nix @@ -0,0 +1,83 @@ +{ lib, ... }: +{ + options.skyflake.storage.seaweedfs = { + enable = lib.mkOption { + type = lib.types.bool; + default = false; + description = '' + enable seaweedfs as the storage backend. + ''; + }; + seaweedfsMount = lib.mkOption { + type = lib.types.attrsOf (lib.types.submodule { + options = { + mountPoint = lib.mkOption { + type = with lib.types; nullOr str; + default = null; + }; + }; + }); + }; + #master = { + # serverIP = lib.mkOption { + # type = lib.str; + # description = '' + # IP of this node. + # ''; + # }; + # listenIPs = lib.mkOption { + # type = lib.listOf lib.str; + # description = '' + # IP of all the master servers. + # Can be the same as storage nodes. + # ''; + # }; + #}; + volumeStorage = { + encrypt = lib.mkOption { + type = lib.types.bool; + default = false; + description = '' + enable encryption on volume store. + ''; + }; + #datacenter = { + # type = lib.str; + # description = '' + # The datacenter location of the node. + # ''; + #}; + #rack = { + # type = lib.str; + # description = '' + # The rack location of the node. + # ''; + #}; + #serverIP = lib.mkOption { + # type = lib.listOf lib.str; + # description = '' + # IP of this node. + # ''; + #}; + #listenIPs = lib.mkOption { + # type = lib.listOf lib.str; + # description = '' + # IPs of all the nodes that should store the actual data but not metadata. + # Can be the same as DB nodes. + # ''; + #}; + }; + filer.db = { + etcd = { + enable = lib.mkOption { + type = lib.types.bool; + default = true; + description = '' + Use to enable Kubernetes etcd database as a backend for seaweedfs. + ''; + }; + + }; + }; + }; +} \ No newline at end of file diff --git a/vm/build-vm.nix b/vm/build-vm.nix index 8836844..437fca7 100644 --- a/vm/build-vm.nix +++ b/vm/build-vm.nix @@ -11,7 +11,6 @@ let nixpkgs = builtins.getFlake nixpkgsRef; pkgs = nixpkgs.legacyPackages.${system}; - inherit (pkgs) lib; microvm = builtins.getFlake microvmFlake; flake = builtins.getFlake flakeRef; @@ -46,7 +45,6 @@ let }; } # From the host's skyflake.deploy.customizationModule - @customizationModule@ ]; }; @@ -64,4 +62,4 @@ import ./nomad-job.nix { inherit pkgs runner; inherit (extended) config; -} +} \ No newline at end of file diff --git a/vm/customization-options.nix b/vm/customization-options.nix index 535a243..1aa5581 100644 --- a/vm/customization-options.nix +++ b/vm/customization-options.nix @@ -20,6 +20,7 @@ ''; }; + # TODO: rename it to something that allows more than ceph. deploy.rbds = mkOption { default = {}; description = '' diff --git a/vm/nomad-job.nix b/vm/nomad-job.nix index 5f2a51d..e8616cc 100644 --- a/vm/nomad-job.nix +++ b/vm/nomad-job.nix @@ -64,7 +64,7 @@ let } '') config.skyflake.nomadJob.affinities} - ${lib.concatMapStrings (interface@{ id, ... }: '' + ${lib.concatMapStrings ({ id, ... }: '' task "add-interface-${id}" { lifecycle { hook = "prestart" @@ -120,7 +120,7 @@ ${'' } '') config.microvm.interfaces} - ${lib.concatMapStrings (share@{ tag, source, socket, proto, ... }: + ${lib.concatMapStrings ({ tag, source, socket, proto, ... }: lib.optionalString (proto == "virtiofs") '' task "virtiofsd-${tag}" { lifecycle { From 80e4e6221f1a4d9e683821a6cb7d97be69d2dfed Mon Sep 17 00:00:00 2001 From: Makuru Date: Mon, 7 Oct 2024 18:59:26 +0200 Subject: [PATCH 02/41] Working file upload. --- .gitignore | 6 + example-server.nix | 37 +++--- example/ca-csr.json | 16 +++ example/ceph.client.admin.keyring | 6 - example/ceph.mon.keyring | 3 - example/certs/ca-key.pem | 3 + example/certs/ca.csr | 9 ++ example/certs/ca.pem | 14 ++ example/certs/example1-key.pem | 3 + example/certs/example1.csr | 9 ++ example/certs/example1.pem | 14 ++ example/certs/example2-key.pem | 3 + example/certs/example2.csr | 9 ++ example/certs/example2.pem | 14 ++ example/certs/example3-key.pem | 3 + example/certs/example3.csr | 9 ++ example/certs/example3.pem | 14 ++ example/certs/peer-example1-key.pem | 3 + example/certs/peer-example1.csr | 9 ++ example/certs/peer-example1.pem | 14 ++ example/certs/peer-example2-key.pem | 3 + example/certs/peer-example2.csr | 9 ++ example/certs/peer-example2.pem | 14 ++ example/certs/peer-example3-key.pem | 3 + example/certs/peer-example3.csr | 9 ++ example/certs/peer-example3.pem | 14 ++ example/config/ca-config.json | 13 ++ example/config/req-csr.json | 20 +++ flake.lock | 26 ++-- flake.nix | 1 + .../storage/seaweedfs/dbBackend/etcd.nix | 33 ++--- .../storage/seaweedfs/dbBackend/sqlite.nix | 59 --------- nixos-modules/storage/seaweedfs/options.nix | 14 +- nixos-modules/storage/seaweedfs/server.nix | 121 ++++++++++++++++++ 34 files changed, 418 insertions(+), 119 deletions(-) create mode 100644 .gitignore create mode 100644 example/ca-csr.json delete mode 100644 example/ceph.client.admin.keyring delete mode 100644 example/ceph.mon.keyring create mode 100644 example/certs/ca-key.pem create mode 100644 example/certs/ca.csr create mode 100644 example/certs/ca.pem create mode 100644 example/certs/example1-key.pem create mode 100644 example/certs/example1.csr create mode 100644 example/certs/example1.pem create mode 100644 example/certs/example2-key.pem create mode 100644 example/certs/example2.csr create mode 100644 example/certs/example2.pem create mode 100644 example/certs/example3-key.pem create mode 100644 example/certs/example3.csr create mode 100644 example/certs/example3.pem create mode 100644 example/certs/peer-example1-key.pem create mode 100644 example/certs/peer-example1.csr create mode 100644 example/certs/peer-example1.pem create mode 100644 example/certs/peer-example2-key.pem create mode 100644 example/certs/peer-example2.csr create mode 100644 example/certs/peer-example2.pem create mode 100644 example/certs/peer-example3-key.pem create mode 100644 example/certs/peer-example3.csr create mode 100644 example/certs/peer-example3.pem create mode 100644 example/config/ca-config.json create mode 100644 example/config/req-csr.json delete mode 100644 nixos-modules/storage/seaweedfs/dbBackend/sqlite.nix create mode 100644 nixos-modules/storage/seaweedfs/server.nix diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ff32b0b --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +example1-ceph.img +example1-persist.img +example2-ceph.img +example2-persist.img +example3-ceph.img +example3-persist.img diff --git a/example-server.nix b/example-server.nix index 2b1fb13..736ac6a 100644 --- a/example-server.nix +++ b/example-server.nix @@ -7,21 +7,26 @@ vcpu = 4; mem = 4096; - shares = [ { - tag = "ro-store"; - source = "/nix/store"; - mountPoint = "/nix/.ro-store"; - } ]; - volumes = [ { - image = "example${toString instance}-persist.img"; - mountPoint = "/"; - size = 20 * 1024; - } { - image = "example${toString instance}-ceph.img"; - mountPoint = null; - size = 20 * 1024; - } ]; - writableStoreOverlay = "/nix/.rw-store"; + shares = [ + { + tag = "ro-store"; + source = "/nix/store"; + mountPoint = "/nix/.ro-store"; + } + ]; + volumes = [ + { + image = "example${toString instance}-persist.img"; + mountPoint = "/"; + size = 20 * 1024; + } + # { + # image = "example${toString instance}-ceph.img"; + # mountPoint = null; + # size = 20 * 1024; + # } + ]; + writableStoreOverlay = "/nix/.rw-store"; interfaces = [ { id = "eth0"; @@ -66,7 +71,7 @@ addresses = [ { # TODO: addressConfig needs to be removed. # trace: warning: Using 'addressConfig' is deprecated! Move all attributes inside one level up and remove it. - addressConfig.Address = "fec0::${toString instance}/64"; # + Address = "fec0::${toString instance}/64"; # } ]; }; }; diff --git a/example/ca-csr.json b/example/ca-csr.json new file mode 100644 index 0000000..c7aef0f --- /dev/null +++ b/example/ca-csr.json @@ -0,0 +1,16 @@ +{ + "CN": "Autogenerated CA", + "key": { + "algo": "ed25519", + "size": 512 + }, + "names": [ + { + "O": "Honest Achmed's Used Certificates", + "OU": "Hastily-Generated Values Divison", + "L": "San Francisco", + "ST": "California", + "C": "US" + } + ] +} diff --git a/example/ceph.client.admin.keyring b/example/ceph.client.admin.keyring deleted file mode 100644 index 93ca576..0000000 --- a/example/ceph.client.admin.keyring +++ /dev/null @@ -1,6 +0,0 @@ -[client.admin] - key = AQDECbJjxEw2DRAAHq4gG9hq7md9AWR8194/Yg== - caps mds = "allow *" - caps mgr = "allow *" - caps mon = "allow *" - caps osd = "allow *" diff --git a/example/ceph.mon.keyring b/example/ceph.mon.keyring deleted file mode 100644 index 11b91c0..0000000 --- a/example/ceph.mon.keyring +++ /dev/null @@ -1,3 +0,0 @@ -[mon.] - key = AQDECbJjf1z7CxAA8ENCZ+VYhLmgtLKO7JkGcg== - caps mon = "allow *" diff --git a/example/certs/ca-key.pem b/example/certs/ca-key.pem new file mode 100644 index 0000000..f583c0a --- /dev/null +++ b/example/certs/ca-key.pem @@ -0,0 +1,3 @@ +-----BEGIN Ed25519 PRIVATE KEY----- +MC4CAQAwBQYDK2VwBCIEIEMTEFvz1RJeAp22hTB1qmPFcSXaHzMJlKVP70EWzTHN +-----END Ed25519 PRIVATE KEY----- diff --git a/example/certs/ca.csr b/example/certs/ca.csr new file mode 100644 index 0000000..a4a5b15 --- /dev/null +++ b/example/certs/ca.csr @@ -0,0 +1,9 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIIBRzCB+gIBADBdMRUwEwYDVQQHEwx0aGUgaW50ZXJuZXQxFjAUBgNVBAoTDWF1 +dG9nZW5lcmF0ZWQxFTATBgNVBAsTDGV0Y2QgY2x1c3RlcjEVMBMGA1UEAxMMZXRj +ZC1leGFtcGxlMCowBQYDK2VwAyEAWvy5UAfHdxYpGq81CFm3Z2e3ChGoKsTH/7hc +5zL/ZSygajBoBgkqhkiG9w0BCQ4xWzBZMFcGA1UdEQRQME6HEP7AAAAAAAAAAAAA +AAAAAAGHEP7AAAAAAAAAAAAAAAAAAAKHEP7AAAAAAAAAAAAAAAAAAAOHEAAAAAAA +AAAAAAAAAAAAAAGHBH8AAAEwBQYDK2VwA0EAlDTB4Rd/PpNC01C7TxI8u/4019Oj +0lRBgHzTJpX8HqZdQuEYBt94v0KqQKQa7/oxmgBvAxJUrLTos7QBAlYbCw== +-----END CERTIFICATE REQUEST----- diff --git a/example/certs/ca.pem b/example/certs/ca.pem new file mode 100644 index 0000000..66e3ace --- /dev/null +++ b/example/certs/ca.pem @@ -0,0 +1,14 @@ +-----BEGIN CERTIFICATE----- +MIICGTCCAcugAwIBAgIUCNkLEEqavh/CRNNDoM+rnN2JSOYwBQYDK2VwMF0xFTAT +BgNVBAcTDHRoZSBpbnRlcm5ldDEWMBQGA1UEChMNYXV0b2dlbmVyYXRlZDEVMBMG +A1UECxMMZXRjZCBjbHVzdGVyMRUwEwYDVQQDEwxldGNkLWV4YW1wbGUwHhcNMjQw +NzE2MTk0MjAwWhcNMjkwNzE1MTk0MjAwWjBdMRUwEwYDVQQHEwx0aGUgaW50ZXJu +ZXQxFjAUBgNVBAoTDWF1dG9nZW5lcmF0ZWQxFTATBgNVBAsTDGV0Y2QgY2x1c3Rl +cjEVMBMGA1UEAxMMZXRjZC1leGFtcGxlMCowBQYDK2VwAyEAWvy5UAfHdxYpGq81 +CFm3Z2e3ChGoKsTH/7hc5zL/ZSyjgZwwgZkwDgYDVR0PAQH/BAQDAgEGMA8GA1Ud +EwEB/wQFMAMBAf8wHQYDVR0OBBYEFFQ8tbzoTdud2KZhxAl9bvTbqnrUMFcGA1Ud +EQRQME6HEP7AAAAAAAAAAAAAAAAAAAGHEP7AAAAAAAAAAAAAAAAAAAKHEP7AAAAA +AAAAAAAAAAAAAAOHEAAAAAAAAAAAAAAAAAAAAAGHBH8AAAEwBQYDK2VwA0EAEtgh +QPnf/sW6sOTt1nAyHwhiHaZF2RTGyHCZcx5jVVe1U/z8ON0hntLlRDzHV62YwsFB +aaI44gU7H9f4b25WAQ== +-----END CERTIFICATE----- diff --git a/example/certs/example1-key.pem b/example/certs/example1-key.pem new file mode 100644 index 0000000..564ae45 --- /dev/null +++ b/example/certs/example1-key.pem @@ -0,0 +1,3 @@ +-----BEGIN Ed25519 PRIVATE KEY----- +MC4CAQAwBQYDK2VwBCIEIBGEyCEiUp84bElnRioTt3CBv/LkgxdvZd2oxzZ/mdOw +-----END Ed25519 PRIVATE KEY----- diff --git a/example/certs/example1.csr b/example/certs/example1.csr new file mode 100644 index 0000000..d66e35d --- /dev/null +++ b/example/certs/example1.csr @@ -0,0 +1,9 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIIBRzCB+gIBADBdMRUwEwYDVQQHEwx0aGUgaW50ZXJuZXQxFjAUBgNVBAoTDWF1 +dG9nZW5lcmF0ZWQxFTATBgNVBAsTDGV0Y2QgY2x1c3RlcjEVMBMGA1UEAxMMZXRj +ZC1leGFtcGxlMCowBQYDK2VwAyEAX0ISCrwOyEQ64fPNRadnpiwjuHH+PhoYt4g3 +BD5VRACgajBoBgkqhkiG9w0BCQ4xWzBZMFcGA1UdEQRQME6HEP7AAAAAAAAAAAAA +AAAAAAGHEP7AAAAAAAAAAAAAAAAAAAKHEP7AAAAAAAAAAAAAAAAAAAOHEAAAAAAA +AAAAAAAAAAAAAAGHBH8AAAEwBQYDK2VwA0EAERkiHl8ftX6tbeaXZ3NcwJaWVknS +VobentHaevSap5l8NIarOt8J04LXbR48HNQIo1FFNkvbfcPvqQuHuKGJCw== +-----END CERTIFICATE REQUEST----- diff --git a/example/certs/example1.pem b/example/certs/example1.pem new file mode 100644 index 0000000..3a02513 --- /dev/null +++ b/example/certs/example1.pem @@ -0,0 +1,14 @@ +-----BEGIN CERTIFICATE----- +MIICNzCCAemgAwIBAgIUYgHAOcH6pLLfVtGVUx4TK/sX5dAwBQYDK2VwMF0xFTAT +BgNVBAcTDHRoZSBpbnRlcm5ldDEWMBQGA1UEChMNYXV0b2dlbmVyYXRlZDEVMBMG +A1UECxMMZXRjZCBjbHVzdGVyMRUwEwYDVQQDEwxldGNkLWV4YW1wbGUwIBcNMjQw +NzE2MTk0MjAwWhgPMjEyNDA2MjIxOTQyMDBaMF0xFTATBgNVBAcTDHRoZSBpbnRl +cm5ldDEWMBQGA1UEChMNYXV0b2dlbmVyYXRlZDEVMBMGA1UECxMMZXRjZCBjbHVz +dGVyMRUwEwYDVQQDEwxldGNkLWV4YW1wbGUwKjAFBgMrZXADIQBfQhIKvA7IRDrh +881Fp2emLCO4cf4+Ghi3iDcEPlVEAKOBuDCBtTAOBgNVHQ8BAf8EBAMCBaAwHQYD +VR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMAwGA1UdEwEB/wQCMAAwHQYDVR0O +BBYEFB2kJrQDhUGYGpHcVGD4KbkRk+ByMFcGA1UdEQRQME6HEP7AAAAAAAAAAAAA +AAAAAAGHEP7AAAAAAAAAAAAAAAAAAAKHEP7AAAAAAAAAAAAAAAAAAAOHEAAAAAAA +AAAAAAAAAAAAAAGHBH8AAAEwBQYDK2VwA0EANadWFqPd7H6soA/zUPHPSbOKpij1 +uA2f673WZw9zzPWMzIX12wjfAxsegIpmghL5cMPPsFDgC4MtY0BJPUeMCA== +-----END CERTIFICATE----- diff --git a/example/certs/example2-key.pem b/example/certs/example2-key.pem new file mode 100644 index 0000000..a257b22 --- /dev/null +++ b/example/certs/example2-key.pem @@ -0,0 +1,3 @@ +-----BEGIN Ed25519 PRIVATE KEY----- +MC4CAQAwBQYDK2VwBCIEINj1txt2IO3S4958/Rny1hungjcAqzqhYnT1odiKRWwT +-----END Ed25519 PRIVATE KEY----- diff --git a/example/certs/example2.csr b/example/certs/example2.csr new file mode 100644 index 0000000..d38286e --- /dev/null +++ b/example/certs/example2.csr @@ -0,0 +1,9 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIIBRzCB+gIBADBdMRUwEwYDVQQHEwx0aGUgaW50ZXJuZXQxFjAUBgNVBAoTDWF1 +dG9nZW5lcmF0ZWQxFTATBgNVBAsTDGV0Y2QgY2x1c3RlcjEVMBMGA1UEAxMMZXRj +ZC1leGFtcGxlMCowBQYDK2VwAyEApjYRMG9MSYJd5iJKmP8vP/HrgFoiPf2y0mE6 +gh4m8ZWgajBoBgkqhkiG9w0BCQ4xWzBZMFcGA1UdEQRQME6HEP7AAAAAAAAAAAAA +AAAAAAGHEP7AAAAAAAAAAAAAAAAAAAKHEP7AAAAAAAAAAAAAAAAAAAOHEAAAAAAA +AAAAAAAAAAAAAAGHBH8AAAEwBQYDK2VwA0EA0flUnLg77p20YEh/wMgRQWxWTx+c +g31yRdemq3vM+8xL5Gc8GZFGTPgtA2Lx32E637MgGUdUo4dhLPcy4VY2AA== +-----END CERTIFICATE REQUEST----- diff --git a/example/certs/example2.pem b/example/certs/example2.pem new file mode 100644 index 0000000..10d04b9 --- /dev/null +++ b/example/certs/example2.pem @@ -0,0 +1,14 @@ +-----BEGIN CERTIFICATE----- +MIICNzCCAemgAwIBAgIUe7XBGPB38YhFDMx2G0KC8KkmY8UwBQYDK2VwMF0xFTAT +BgNVBAcTDHRoZSBpbnRlcm5ldDEWMBQGA1UEChMNYXV0b2dlbmVyYXRlZDEVMBMG +A1UECxMMZXRjZCBjbHVzdGVyMRUwEwYDVQQDEwxldGNkLWV4YW1wbGUwIBcNMjQw +NzE2MTk0MjAwWhgPMjEyNDA2MjIxOTQyMDBaMF0xFTATBgNVBAcTDHRoZSBpbnRl +cm5ldDEWMBQGA1UEChMNYXV0b2dlbmVyYXRlZDEVMBMGA1UECxMMZXRjZCBjbHVz +dGVyMRUwEwYDVQQDEwxldGNkLWV4YW1wbGUwKjAFBgMrZXADIQCmNhEwb0xJgl3m +IkqY/y8/8euAWiI9/bLSYTqCHibxlaOBuDCBtTAOBgNVHQ8BAf8EBAMCBaAwHQYD +VR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMAwGA1UdEwEB/wQCMAAwHQYDVR0O +BBYEFIbRMrkmOEWTHBPsudvMtuCFMhn2MFcGA1UdEQRQME6HEP7AAAAAAAAAAAAA +AAAAAAGHEP7AAAAAAAAAAAAAAAAAAAKHEP7AAAAAAAAAAAAAAAAAAAOHEAAAAAAA +AAAAAAAAAAAAAAGHBH8AAAEwBQYDK2VwA0EAOtbGD5c5rrrbrRZzxuP7aEyVIIJQ +XNpkqSZgXXKOn0Eyi1GB5eeTsfZQlskSQ9ChbGZxp7PAlJaI1qUzZ5ybCg== +-----END CERTIFICATE----- diff --git a/example/certs/example3-key.pem b/example/certs/example3-key.pem new file mode 100644 index 0000000..5316f89 --- /dev/null +++ b/example/certs/example3-key.pem @@ -0,0 +1,3 @@ +-----BEGIN Ed25519 PRIVATE KEY----- +MC4CAQAwBQYDK2VwBCIEIIQ9xVv63Y7Jo9rNtjwTtxouIJUgdEo65HPiLjqjq+uZ +-----END Ed25519 PRIVATE KEY----- diff --git a/example/certs/example3.csr b/example/certs/example3.csr new file mode 100644 index 0000000..96d995c --- /dev/null +++ b/example/certs/example3.csr @@ -0,0 +1,9 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIIBRzCB+gIBADBdMRUwEwYDVQQHEwx0aGUgaW50ZXJuZXQxFjAUBgNVBAoTDWF1 +dG9nZW5lcmF0ZWQxFTATBgNVBAsTDGV0Y2QgY2x1c3RlcjEVMBMGA1UEAxMMZXRj +ZC1leGFtcGxlMCowBQYDK2VwAyEArInudki4shX2NRzMwXIdKfgv8QL3z39j/YNX +LU3Mw1qgajBoBgkqhkiG9w0BCQ4xWzBZMFcGA1UdEQRQME6HEP7AAAAAAAAAAAAA +AAAAAAGHEP7AAAAAAAAAAAAAAAAAAAKHEP7AAAAAAAAAAAAAAAAAAAOHEAAAAAAA +AAAAAAAAAAAAAAGHBH8AAAEwBQYDK2VwA0EA1u8gPSVRrSgonzEiyZINcloLtZCr +F2wG1jXg2V4xcafzHe2HeD94XOp70rl8R53rShCPDiuf7redQGIbCHseAQ== +-----END CERTIFICATE REQUEST----- diff --git a/example/certs/example3.pem b/example/certs/example3.pem new file mode 100644 index 0000000..e0182da --- /dev/null +++ b/example/certs/example3.pem @@ -0,0 +1,14 @@ +-----BEGIN CERTIFICATE----- +MIICNzCCAemgAwIBAgIURpq8lJcnm/XnCxsXpCnpy1Jf7GQwBQYDK2VwMF0xFTAT +BgNVBAcTDHRoZSBpbnRlcm5ldDEWMBQGA1UEChMNYXV0b2dlbmVyYXRlZDEVMBMG +A1UECxMMZXRjZCBjbHVzdGVyMRUwEwYDVQQDEwxldGNkLWV4YW1wbGUwIBcNMjQw +NzE2MTk0MjAwWhgPMjEyNDA2MjIxOTQyMDBaMF0xFTATBgNVBAcTDHRoZSBpbnRl +cm5ldDEWMBQGA1UEChMNYXV0b2dlbmVyYXRlZDEVMBMGA1UECxMMZXRjZCBjbHVz +dGVyMRUwEwYDVQQDEwxldGNkLWV4YW1wbGUwKjAFBgMrZXADIQCsie52SLiyFfY1 +HMzBch0p+C/xAvfPf2P9g1ctTczDWqOBuDCBtTAOBgNVHQ8BAf8EBAMCBaAwHQYD +VR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMAwGA1UdEwEB/wQCMAAwHQYDVR0O +BBYEFKTHiiohxrYsb8MJT/Y41J23rdXIMFcGA1UdEQRQME6HEP7AAAAAAAAAAAAA +AAAAAAGHEP7AAAAAAAAAAAAAAAAAAAKHEP7AAAAAAAAAAAAAAAAAAAOHEAAAAAAA +AAAAAAAAAAAAAAGHBH8AAAEwBQYDK2VwA0EAALA2oCPaZV12mlTWRqvKr48s12uj +htFgHDoQAYMW70lBrmvEw0dPlh+xioegtmFoy+NNBn7E5V6MMIzZZbYeBg== +-----END CERTIFICATE----- diff --git a/example/certs/peer-example1-key.pem b/example/certs/peer-example1-key.pem new file mode 100644 index 0000000..390e9d8 --- /dev/null +++ b/example/certs/peer-example1-key.pem @@ -0,0 +1,3 @@ +-----BEGIN Ed25519 PRIVATE KEY----- +MC4CAQAwBQYDK2VwBCIEIJ/zC5RpEvSvt4ZU4VeXQ4Bn4So8Sdkre1/FpHZW1CDo +-----END Ed25519 PRIVATE KEY----- diff --git a/example/certs/peer-example1.csr b/example/certs/peer-example1.csr new file mode 100644 index 0000000..3309bed --- /dev/null +++ b/example/certs/peer-example1.csr @@ -0,0 +1,9 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIIBRzCB+gIBADBdMRUwEwYDVQQHEwx0aGUgaW50ZXJuZXQxFjAUBgNVBAoTDWF1 +dG9nZW5lcmF0ZWQxFTATBgNVBAsTDGV0Y2QgY2x1c3RlcjEVMBMGA1UEAxMMZXRj +ZC1leGFtcGxlMCowBQYDK2VwAyEA2tMIR3z+Wp6PL8TN4JMpgUyPj1n2CpOayV2J +ih/8/8GgajBoBgkqhkiG9w0BCQ4xWzBZMFcGA1UdEQRQME6HEP7AAAAAAAAAAAAA +AAAAAAGHEP7AAAAAAAAAAAAAAAAAAAKHEP7AAAAAAAAAAAAAAAAAAAOHEAAAAAAA +AAAAAAAAAAAAAAGHBH8AAAEwBQYDK2VwA0EAOCzFzWw0+u5eLVbBOhQI7e6t0XsK +or1tBCVErs16w8khlN8gx8y1uuKQEUsQYgtvOiubJy8cu8JTTzg9cybmAQ== +-----END CERTIFICATE REQUEST----- diff --git a/example/certs/peer-example1.pem b/example/certs/peer-example1.pem new file mode 100644 index 0000000..a2247c9 --- /dev/null +++ b/example/certs/peer-example1.pem @@ -0,0 +1,14 @@ +-----BEGIN CERTIFICATE----- +MIICNzCCAemgAwIBAgIUAwC6lT0eYpLDSXLsq0B21aG7EqgwBQYDK2VwMF0xFTAT +BgNVBAcTDHRoZSBpbnRlcm5ldDEWMBQGA1UEChMNYXV0b2dlbmVyYXRlZDEVMBMG +A1UECxMMZXRjZCBjbHVzdGVyMRUwEwYDVQQDEwxldGNkLWV4YW1wbGUwIBcNMjQw +NzE2MTk0MjAwWhgPMjEyNDA2MjIxOTQyMDBaMF0xFTATBgNVBAcTDHRoZSBpbnRl +cm5ldDEWMBQGA1UEChMNYXV0b2dlbmVyYXRlZDEVMBMGA1UECxMMZXRjZCBjbHVz +dGVyMRUwEwYDVQQDEwxldGNkLWV4YW1wbGUwKjAFBgMrZXADIQDa0whHfP5ano8v +xM3gkymBTI+PWfYKk5rJXYmKH/z/waOBuDCBtTAOBgNVHQ8BAf8EBAMCBaAwHQYD +VR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMAwGA1UdEwEB/wQCMAAwHQYDVR0O +BBYEFH4DS96JIOdG2zrfJoL2ojxfTqXdMFcGA1UdEQRQME6HEP7AAAAAAAAAAAAA +AAAAAAGHEP7AAAAAAAAAAAAAAAAAAAKHEP7AAAAAAAAAAAAAAAAAAAOHEAAAAAAA +AAAAAAAAAAAAAAGHBH8AAAEwBQYDK2VwA0EA3tR2EgvN9g8Z0gcWFsERuoOUnmYT +nwE7p5nQcaflbjEcYxCkAKvExhR0NlurMCflMh+dVbhldIyIDyzeK/tnDA== +-----END CERTIFICATE----- diff --git a/example/certs/peer-example2-key.pem b/example/certs/peer-example2-key.pem new file mode 100644 index 0000000..0be5f04 --- /dev/null +++ b/example/certs/peer-example2-key.pem @@ -0,0 +1,3 @@ +-----BEGIN Ed25519 PRIVATE KEY----- +MC4CAQAwBQYDK2VwBCIEIPp2+itRObBvM5NTJgS9/5Y9oAxy1KAcfQZaTxgTmCP6 +-----END Ed25519 PRIVATE KEY----- diff --git a/example/certs/peer-example2.csr b/example/certs/peer-example2.csr new file mode 100644 index 0000000..68cb476 --- /dev/null +++ b/example/certs/peer-example2.csr @@ -0,0 +1,9 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIIBRzCB+gIBADBdMRUwEwYDVQQHEwx0aGUgaW50ZXJuZXQxFjAUBgNVBAoTDWF1 +dG9nZW5lcmF0ZWQxFTATBgNVBAsTDGV0Y2QgY2x1c3RlcjEVMBMGA1UEAxMMZXRj +ZC1leGFtcGxlMCowBQYDK2VwAyEAwe/vnjdOAvLdJbMvH22FKWtYIL81VCNwuZkD +OhMHh/6gajBoBgkqhkiG9w0BCQ4xWzBZMFcGA1UdEQRQME6HEP7AAAAAAAAAAAAA +AAAAAAGHEP7AAAAAAAAAAAAAAAAAAAKHEP7AAAAAAAAAAAAAAAAAAAOHEAAAAAAA +AAAAAAAAAAAAAAGHBH8AAAEwBQYDK2VwA0EAfhKGZe3rZABx1OXxLy/NPS/t4qoe +jy65mUgUo9TysyaoxSMM1cLzGOkWW8BOtP0KX4EUhJYXlukC0IviPXQ7AQ== +-----END CERTIFICATE REQUEST----- diff --git a/example/certs/peer-example2.pem b/example/certs/peer-example2.pem new file mode 100644 index 0000000..bdd00d1 --- /dev/null +++ b/example/certs/peer-example2.pem @@ -0,0 +1,14 @@ +-----BEGIN CERTIFICATE----- +MIICNzCCAemgAwIBAgIUUjaY6MBwV/m6LIu0lqFHm3ruFXMwBQYDK2VwMF0xFTAT +BgNVBAcTDHRoZSBpbnRlcm5ldDEWMBQGA1UEChMNYXV0b2dlbmVyYXRlZDEVMBMG +A1UECxMMZXRjZCBjbHVzdGVyMRUwEwYDVQQDEwxldGNkLWV4YW1wbGUwIBcNMjQw +NzE2MTk0MjAwWhgPMjEyNDA2MjIxOTQyMDBaMF0xFTATBgNVBAcTDHRoZSBpbnRl +cm5ldDEWMBQGA1UEChMNYXV0b2dlbmVyYXRlZDEVMBMGA1UECxMMZXRjZCBjbHVz +dGVyMRUwEwYDVQQDEwxldGNkLWV4YW1wbGUwKjAFBgMrZXADIQDB7++eN04C8t0l +sy8fbYUpa1ggvzVUI3C5mQM6EweH/qOBuDCBtTAOBgNVHQ8BAf8EBAMCBaAwHQYD +VR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMAwGA1UdEwEB/wQCMAAwHQYDVR0O +BBYEFI63W5D655IjJGj0tWCqUGrNsjLAMFcGA1UdEQRQME6HEP7AAAAAAAAAAAAA +AAAAAAGHEP7AAAAAAAAAAAAAAAAAAAKHEP7AAAAAAAAAAAAAAAAAAAOHEAAAAAAA +AAAAAAAAAAAAAAGHBH8AAAEwBQYDK2VwA0EAQcr6QipYgI3IoQigptc0Inj1YeXf +S/rxnuMsyUs8uvqqg4BwaoMZX6WYTOhhZAMM4v3NFjhz3mmKu0Af4n7ABw== +-----END CERTIFICATE----- diff --git a/example/certs/peer-example3-key.pem b/example/certs/peer-example3-key.pem new file mode 100644 index 0000000..d9cc782 --- /dev/null +++ b/example/certs/peer-example3-key.pem @@ -0,0 +1,3 @@ +-----BEGIN Ed25519 PRIVATE KEY----- +MC4CAQAwBQYDK2VwBCIEIKNmdhBTKZL0CMwT3yDSDGNI310ueSsLWmDf01A3nDYJ +-----END Ed25519 PRIVATE KEY----- diff --git a/example/certs/peer-example3.csr b/example/certs/peer-example3.csr new file mode 100644 index 0000000..deb3301 --- /dev/null +++ b/example/certs/peer-example3.csr @@ -0,0 +1,9 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIIBRzCB+gIBADBdMRUwEwYDVQQHEwx0aGUgaW50ZXJuZXQxFjAUBgNVBAoTDWF1 +dG9nZW5lcmF0ZWQxFTATBgNVBAsTDGV0Y2QgY2x1c3RlcjEVMBMGA1UEAxMMZXRj +ZC1leGFtcGxlMCowBQYDK2VwAyEAggNboIQYKrmxO0809jRk4ly/FTA2ppf0DVoL +rEb/aNqgajBoBgkqhkiG9w0BCQ4xWzBZMFcGA1UdEQRQME6HEP7AAAAAAAAAAAAA +AAAAAAGHEP7AAAAAAAAAAAAAAAAAAAKHEP7AAAAAAAAAAAAAAAAAAAOHEAAAAAAA +AAAAAAAAAAAAAAGHBH8AAAEwBQYDK2VwA0EAjmfEW1tKrm+1AUIk1zkEN/XLJYre +uMTrlnGQxRZstfLTI6h95CrV1V2vKu99eSFl0idr3TPTvwaVf8MmWdCrDA== +-----END CERTIFICATE REQUEST----- diff --git a/example/certs/peer-example3.pem b/example/certs/peer-example3.pem new file mode 100644 index 0000000..9221d55 --- /dev/null +++ b/example/certs/peer-example3.pem @@ -0,0 +1,14 @@ +-----BEGIN CERTIFICATE----- +MIICNzCCAemgAwIBAgIUQwLFi+8HL1Up3TaGB59peaSgeQ8wBQYDK2VwMF0xFTAT +BgNVBAcTDHRoZSBpbnRlcm5ldDEWMBQGA1UEChMNYXV0b2dlbmVyYXRlZDEVMBMG +A1UECxMMZXRjZCBjbHVzdGVyMRUwEwYDVQQDEwxldGNkLWV4YW1wbGUwIBcNMjQw +NzE2MTk0MjAwWhgPMjEyNDA2MjIxOTQyMDBaMF0xFTATBgNVBAcTDHRoZSBpbnRl +cm5ldDEWMBQGA1UEChMNYXV0b2dlbmVyYXRlZDEVMBMGA1UECxMMZXRjZCBjbHVz +dGVyMRUwEwYDVQQDEwxldGNkLWV4YW1wbGUwKjAFBgMrZXADIQCCA1ughBgqubE7 +TzT2NGTiXL8VMDaml/QNWgusRv9o2qOBuDCBtTAOBgNVHQ8BAf8EBAMCBaAwHQYD +VR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMAwGA1UdEwEB/wQCMAAwHQYDVR0O +BBYEFOJNRyO0Z1wXC8ed5URpXewiC59oMFcGA1UdEQRQME6HEP7AAAAAAAAAAAAA +AAAAAAGHEP7AAAAAAAAAAAAAAAAAAAKHEP7AAAAAAAAAAAAAAAAAAAOHEAAAAAAA +AAAAAAAAAAAAAAGHBH8AAAEwBQYDK2VwA0EAp7ZkuHYzWiD0kVBj+qajxZolQ5MW +9M16/RPCkdatPkCAkPmGVujZo6h4HIqPSgD6zF2jHk4rAt/nXPf6u5r7Aw== +-----END CERTIFICATE----- diff --git a/example/config/ca-config.json b/example/config/ca-config.json new file mode 100644 index 0000000..edd0c07 --- /dev/null +++ b/example/config/ca-config.json @@ -0,0 +1,13 @@ +{ + "signing": { + "default": { + "usages": [ + "signing", + "key encipherment", + "server auth", + "client auth" + ], + "expiry": "876000h" + } + } +} diff --git a/example/config/req-csr.json b/example/config/req-csr.json new file mode 100644 index 0000000..e9c6b0b --- /dev/null +++ b/example/config/req-csr.json @@ -0,0 +1,20 @@ +{ + "CN": "etcd-example", + "hosts": [ + "fec0::1", + "fec0::2", + "fec0::3", + "::1", + "127.0.0.1" + ], + "key": { + "algo": "ed25519" + }, + "names": [ + { + "O": "autogenerated", + "OU": "etcd cluster", + "L": "the internet" + } + ] +} diff --git a/flake.lock b/flake.lock index 2ea9f5d..483aae4 100644 --- a/flake.lock +++ b/flake.lock @@ -5,11 +5,11 @@ "systems": "systems" }, "locked": { - "lastModified": 1705309234, - "narHash": "sha256-uNRRNRKmJyCRC/8y1RqBkqWBLM034y4qN7EprSdmgyA=", + "lastModified": 1710146030, + "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=", "owner": "numtide", "repo": "flake-utils", - "rev": "1ef2e671c3b0c19053962c07dbda38332dcebf26", + "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a", "type": "github" }, "original": { @@ -27,11 +27,11 @@ "spectrum": "spectrum" }, "locked": { - "lastModified": 1717441449, - "narHash": "sha256-juxjgmLnFbl+/hhIO2cVtIa6caCO4pLKlZWUMwAOznM=", + "lastModified": 1727872811, + "narHash": "sha256-78hOotCZ/G8l4nyWPdMWSOojdJ5NwHTR0usBVVBNTtU=", "owner": "astro", "repo": "microvm.nix", - "rev": "e3a4dd5b381fb580804105594cc9c71dc45abdb5", + "rev": "e832ffc16b09b1b5c7c1224532d03ed3ce68afd0", "type": "github" }, "original": { @@ -100,11 +100,11 @@ }, "nixpkgs_2": { "locked": { - "lastModified": 1717786204, - "narHash": "sha256-4q0s6m0GUcN7q+Y2DqD27iLvbcd1G50T2lv08kKxkSI=", + "lastModified": 1728018373, + "narHash": "sha256-NOiTvBbRLIOe5F6RbHaAh6++BNjsb149fGZd1T4+KBg=", "owner": "nixos", "repo": "nixpkgs", - "rev": "051f920625ab5aabe37c920346e3e69d7d34400e", + "rev": "bc947f541ae55e999ffdb4013441347d83b00feb", "type": "github" }, "original": { @@ -124,11 +124,11 @@ "spectrum": { "flake": false, "locked": { - "lastModified": 1708358594, - "narHash": "sha256-e71YOotu2FYA67HoC/voJDTFsiPpZNRwmiQb4f94OxQ=", + "lastModified": 1720264467, + "narHash": "sha256-xzM92n3Q9L90faJIJrkrTtTx+JqCGRHMkHWztkV4PuY=", "ref": "refs/heads/main", - "rev": "6d0e73864d28794cdbd26ab7b37259ab0e1e044c", - "revCount": 614, + "rev": "fb59d42542049f586c84b0f8bb86ff3be338e9d3", + "revCount": 674, "type": "git", "url": "https://spectrum-os.org/git/spectrum" }, diff --git a/flake.nix b/flake.nix index 97e1370..1e301c8 100644 --- a/flake.nix +++ b/flake.nix @@ -27,6 +27,7 @@ default = { imports = [ ./nixos-modules/storage/seaweedfs/options.nix + ./nixos-modules/storage/seaweedfs/server.nix # ./nixos-modules/storage/seaweedfs/dbBackend/sqlite.nix ./nixos-modules/storage/seaweedfs/dbBackend/etcd.nix # ./nixos-modules/storage/ceph/server.nix diff --git a/nixos-modules/storage/seaweedfs/dbBackend/etcd.nix b/nixos-modules/storage/seaweedfs/dbBackend/etcd.nix index f70d9c2..f164c6f 100644 --- a/nixos-modules/storage/seaweedfs/dbBackend/etcd.nix +++ b/nixos-modules/storage/seaweedfs/dbBackend/etcd.nix @@ -14,24 +14,27 @@ wants = [ "network-online.target" ]; # ++ lib.optional config.networking.firewall.enable "firewall.service"; - environment = /* (nixpgs.filterAttrs (n: v: v != null) */ { + environment = /* (nixpgs.filterAttrs (n: v: v != null) */ let + address = builtins.elemAt (lib.splitString "/" (lib.head config.systemd.network.networks."01-br0".addresses).Address) 0; + in { ETCD_NAME = config.networking.hostName; #ETCD_DISCOVERY = "true"; ETCD_DATA_DIR = "/var/lib/etcd"; - ETCD_ADVERTISE_CLIENT_URLS = lib.concatMapStringsSep ", " (node: "http://[" + (config.skyflake.nodes."${node}").address + "]:2379") (lib.attrNames config.skyflake.nodes); - ETCD_LISTEN_CLIENT_URLS = "http://[0::]:2379"; - ETCD_LISTEN_PEER_URLS = "http://[0::]:2380"; - #ETCD_INITIAL_ADVERTISE_PEER_URLS = nixpgs.concatStringsSep "," config.skyflake.nodes.address; - # ETCD_PEER_CLIENT_CERT_AUTH = toString config.services.etcd.peerClientCertAuth; - # ETCD_PEER_TRUSTED_CA_FILE = config.services.etcd.peerTrustedCaFile; - # ETCD_PEER_CERT_FILE = config.services.etcd.peerCertFile; - # ETCD_PEER_KEY_FILE = config.services.etcd.peerKeyFile; - # ETCD_CLIENT_CERT_AUTH = toString config.services.etcd.clientCertAuth; - # ETCD_TRUSTED_CA_FILE = config.services.etcd.trustedCaFile; - # ETCD_CERT_FILE = config.services.etcd.certFile; - # ETCD_KEY_FILE = config.services.etcd.keyFile; + ETCD_ADVERTISE_CLIENT_URLS = "https://[${address}]:2379"; # TODO Make it choose the IP of the device declaratively + ETCD_LISTEN_CLIENT_URLS = "https://[${address}]:2379"; # TODO Make it choose the IP of the device declaratively + ETCD_LISTEN_PEER_URLS = "https://[${address}]:2380"; # TODO Make it choose the IP of the device declaratively + ETCD_INITIAL_ADVERTISE_PEER_URLS = "https://[${address}]:2380"; # TODO Make it choose the IP of the device declaratively + ETCD_CLIENT_CERT_AUTH = "true"; + ETCD_TRUSTED_CA_FILE = ../../../../example/certs/ca.pem; + ETCD_CERT_FILE = ../../../../example/certs/${config.networking.hostName}.pem; + ETCD_KEY_FILE = ../../../../example/certs/${config.networking.hostName}-key.pem; + ETCD_PEER_CLIENT_CERT_AUTH = "true"; + ETCD_PEER_TRUSTED_CA_FILE = ../../../../example/certs/ca.pem; + ETCD_PEER_CERT_FILE = ../../../../example/certs/${config.networking.hostName}.pem; + ETCD_PEER_KEY_FILE = ../../../../example/certs/${config.networking.hostName}-key.pem; #}) // (nixpgs.optionalAttrs (config.services.etcd.discovery == ""){ - ETCD_INITIAL_CLUSTER = lib.concatMapStringsSep ", " (node: "http://[" + (config.skyflake.nodes."${node}").address + "]:2380") (lib.attrNames config.skyflake.nodes); + #ETCD_INITIAL_CLUSTER = lib.concatMapStringsSep ", " (node: "http://[" + (config.skyflake.nodes."${node}").address + "]:2380") (lib.attrNames config.skyflake.nodes); + ETCD_INITIAL_CLUSTER = "example1=https://[fec0::1]:2380,example2=https://[fec0::2]:2380,example3=https://[fec0::3]:2380"; ETCD_INITIAL_CLUSTER_STATE = "new"; ETCD_INITIAL_CLUSTER_TOKEN = "etcd-cluster"; #}) // (nixpgs.mapAttrs' (n: v: nixpgs.nameValuePair "ETCD_${n}" v) config.services.etcd.extraConf); @@ -43,7 +46,7 @@ serviceConfig = { Type = "notify"; Restart = "always"; - RestartSec = "30s"; + RestartSec = "5s"; ExecStart = "${pkgs.etcd}/bin/etcd"; User = "etcd"; LimitNOFILE = 40000; diff --git a/nixos-modules/storage/seaweedfs/dbBackend/sqlite.nix b/nixos-modules/storage/seaweedfs/dbBackend/sqlite.nix deleted file mode 100644 index 812c237..0000000 --- a/nixos-modules/storage/seaweedfs/dbBackend/sqlite.nix +++ /dev/null @@ -1,59 +0,0 @@ -{ pkgs, lib, config, ... }: - -let - seaweedFS-filer-file = pkgs.formats.toml '' - [sqlite] - # local on disk. - enabled = false - dbFile = "./filer.db" # sqlite db file - ''; - -in { - config = lib.mkIf config.skyflake.storage.seaweedfs.filer.db.sqlite.enable { - systemd.services.seaweedfs-filer = { - description = "seaweedFS filer storage node"; - wantedBy = [ "multi-user.target" ]; - # after = [ "network-online.target" ] # TODO add firewall - # ++ nixpgs.lib.optional config.networking.firewall.enable "firewall.service"; - # wants = [ "network-online.target" ] - # ++ nixpgs.lib.optional config.networking.firewall.enable "firewall.service"; - - #environment = { - #}; - - - unitConfig = { - Documentation = "https://github.com/seaweedfs/seaweedfs/wiki"; - }; - - serviceConfig = { - Type = "notify"; - Restart = "always"; - RestartSec = "30s"; - ExecStart = "${pkgs.seaweedfs}/bin/weed filer"; - User = "seaweedfs-filer"; - Group = "seaweedfs-filer"; - LimitNOFILE = 40000; - WorkingDirectory = "/var/lib/seaweedfs-filer"; - SyslogIdentifier = "seaweedfs-filer"; - }; - }; - users.users.seaweedfs-filer = { - isSystemUser = true; - group = "seaweedfs-filer"; - description = "seaweedfs filer store user"; - home = config.systemd.services.seaweedfs-filer.WorkingDirectory; - }; - users.groups.seaweedfs-filer = {}; - - environment.etc = { - "seaweedFS-filer" = { - source = "${seaweedFS-filer-file}"; - target = "seaweedfs/filer.toml"; - mode = "0440"; - }; - }; - - environment.systemPackages = [ pkgs.seaweedfs ]; - }; -} \ No newline at end of file diff --git a/nixos-modules/storage/seaweedfs/options.nix b/nixos-modules/storage/seaweedfs/options.nix index a3fdb61..84398d0 100644 --- a/nixos-modules/storage/seaweedfs/options.nix +++ b/nixos-modules/storage/seaweedfs/options.nix @@ -34,13 +34,13 @@ # }; #}; volumeStorage = { - encrypt = lib.mkOption { - type = lib.types.bool; - default = false; - description = '' - enable encryption on volume store. - ''; - }; + #encrypt = lib.mkOption { + # type = lib.types.bool; + # default = false; + # description = '' + # enable encryption on volume store. + # ''; + #}; #datacenter = { # type = lib.str; # description = '' diff --git a/nixos-modules/storage/seaweedfs/server.nix b/nixos-modules/storage/seaweedfs/server.nix new file mode 100644 index 0000000..30ed972 --- /dev/null +++ b/nixos-modules/storage/seaweedfs/server.nix @@ -0,0 +1,121 @@ +{ config, lib, pkgs, ... }: +{ + config = lib.mkIf config.skyflake.storage.seaweedfs.filer.db.etcd.enable { + + users.users.seaweedfs = { + isSystemUser = true; + group = "seaweedfs"; + description = "seaweedfs daemon user"; + home = "/var/lib/seaweedfs"; # TODO bring it under a single setting, the state path. + createHome = true; + }; + users.groups.seaweedfs = {}; + + # config for the master deamon of seaweedfs + systemd.tmpfiles.settings."10-seaweedfs-master"."/var/lib/seaweedfs/master".d = { + user = "seaweedfs"; + mode = "0700"; + }; + systemd.services.seaweedfs-master = { + description = "seaweedfs master service"; + wantedBy = [ "multi-user.target" ]; + after = [ "network-online.target" "etcd.service" ]; + # ++ lib.optional config.networking.firewall.enable "firewall.service"; + wants = [ "network-online.target" "etcd.service" ]; + # ++ lib.optional config.networking.firewall.enable "firewall.service"; + unitConfig = { + Documentation = "https://github.com/seaweedfs/seaweedfs/wiki"; + }; + serviceConfig = let + address = builtins.elemAt (lib.splitString "/" (lib.head config.systemd.network.networks."01-br0".addresses).Address) 0; + in { + Type = "notify"; + Restart = "always"; + RestartSec = "5s"; + ExecStart = ''${pkgs.seaweedfs}/bin/weed master -ip=[${address}] -mdir=/var/lib/seaweedfs/master''; + User = "seaweedfs"; + LimitNOFILE = 40000; + }; + }; + + # config for the filer deamon of seaweedfs + systemd.tmpfiles.settings."10-seaweedfs-filer"."/var/lib/seaweedfs/filer".d = { + user = "seaweedfs"; + mode = "0700"; + }; + systemd.services.seaweedfs-filer = { + description = "seaweedfs filer service"; + wantedBy = [ "multi-user.target" ]; + after = [ "network-online.target" "etcd.service" "seaweedfs-master.service" ]; + # ++ lib.optional config.networking.firewall.enable "firewall.service"; + wants = [ "network-online.target" "etcd.service" "seaweedfs-master.service" ]; + # ++ lib.optional config.networking.firewall.enable "firewall.service"; + unitConfig = { + Documentation = "https://github.com/seaweedfs/seaweedfs/wiki"; + }; + serviceConfig = { + Type = "notify"; + Restart = "always"; + RestartSec = "5s"; + ExecStart = ''${pkgs.seaweedfs}/bin/weed filer -master.port=9333 -volume.port=8080 -dir=/var/lib/seaweedfs/filer''; + User = "seaweedfs"; + LimitNOFILE = 40000; + }; + }; + environment.etc.seaweedfs-filer = { + text = '' + [etcd] + enabled = true + servers = "example1=https://[fec0::1]:2380,example2=https://[fec0::2]:2380,example3=https://[fec0::3]:2380" + # username = "seaweedfs" + # password = "" + key_prefix = "seaweedfs." + timeout = "3s" + # Set the CA certificate path + tls_ca_file="" + # Set the client certificate path + tls_client_crt_file="" + # Set the client private key path + tls_client_key_file="" ''; + target = "./seaweedfs/filer.toml"; + mode = "0440"; + }; + + # config for the volume deamon of seaweedfs + systemd.tmpfiles.settings."10-seaweedfs-volume"."/var/lib/seaweedfs/volume".d = { + user = "seaweedfs"; + mode = "0700"; + }; + systemd.services.seaweedfs-volume = { + description = "seaweedfs volume service"; + wantedBy = [ "multi-user.target" ]; + after = [ "network-online.target" "etcd.service" "seaweedfs-master.service" "seaweedfs-filer.service" ]; + # ++ lib.optional config.networking.firewall.enable "firewall.service"; + wants = [ "network-online.target" "etcd.service" "seaweedfs-master.service" "seaweedfs-filer.service" ]; + # ++ lib.optional config.networking.firewall.enable "firewall.service"; + unitConfig = { + Documentation = "https://github.com/seaweedfs/seaweedfs/wiki"; + }; + serviceConfig = let + address = builtins.elemAt (lib.splitString "/" (lib.head config.systemd.network.networks."01-br0".addresses).Address) 0; + in { + Type = "notify"; + Restart = "always"; + RestartSec = "5s"; + ExecStart = ''${pkgs.seaweedfs}/bin/weed volume -port=8080 -max=5 -ip=${address} -mserver=localhost:9333 -dir=/var/lib/seaweedfs/volume''; + User = "seaweedfs"; + LimitNOFILE = 40000; + }; + }; + + environment.systemPackages = [ pkgs.seaweedfs ]; + /* TODO: add firewall to skyflake. + networking.firewall = lib.mkIf config.services.etcd.openFirewall { + allowedTCPPorts = [ + 2379 # for client requests + 2380 # for peer communication + ]; + }; + */ + }; +} \ No newline at end of file From b7291d23d35d8092b7cb7a8cd3a13b3f2af07ea1 Mon Sep 17 00:00:00 2001 From: Makuru Date: Mon, 7 Oct 2024 21:13:09 +0200 Subject: [PATCH 03/41] fixes etcd communication. --- nixos-modules/storage/seaweedfs/server.nix | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/nixos-modules/storage/seaweedfs/server.nix b/nixos-modules/storage/seaweedfs/server.nix index 30ed972..7435c1d 100644 --- a/nixos-modules/storage/seaweedfs/server.nix +++ b/nixos-modules/storage/seaweedfs/server.nix @@ -14,6 +14,7 @@ # config for the master deamon of seaweedfs systemd.tmpfiles.settings."10-seaweedfs-master"."/var/lib/seaweedfs/master".d = { user = "seaweedfs"; + group = "seaweedfs"; mode = "0700"; }; systemd.services.seaweedfs-master = { @@ -32,17 +33,13 @@ Type = "notify"; Restart = "always"; RestartSec = "5s"; - ExecStart = ''${pkgs.seaweedfs}/bin/weed master -ip=[${address}] -mdir=/var/lib/seaweedfs/master''; + ExecStart = ''${pkgs.seaweedfs}/bin/weed master -ip=[${address}] -peers=[fec0::1]:9333,[fec0::2]:9333,[fec0::3]:9333 -mdir=/var/lib/seaweedfs/master''; User = "seaweedfs"; LimitNOFILE = 40000; }; }; # config for the filer deamon of seaweedfs - systemd.tmpfiles.settings."10-seaweedfs-filer"."/var/lib/seaweedfs/filer".d = { - user = "seaweedfs"; - mode = "0700"; - }; systemd.services.seaweedfs-filer = { description = "seaweedfs filer service"; wantedBy = [ "multi-user.target" ]; @@ -53,11 +50,13 @@ unitConfig = { Documentation = "https://github.com/seaweedfs/seaweedfs/wiki"; }; - serviceConfig = { + serviceConfig = let + address = builtins.elemAt (lib.splitString "/" (lib.head config.systemd.network.networks."01-br0".addresses).Address) 0; + in { Type = "notify"; Restart = "always"; RestartSec = "5s"; - ExecStart = ''${pkgs.seaweedfs}/bin/weed filer -master.port=9333 -volume.port=8080 -dir=/var/lib/seaweedfs/filer''; + ExecStart = ''${pkgs.seaweedfs}/bin/weed filer -master=${address}:9333 -port=8888''; User = "seaweedfs"; LimitNOFILE = 40000; }; @@ -72,18 +71,21 @@ key_prefix = "seaweedfs." timeout = "3s" # Set the CA certificate path - tls_ca_file="" + tls_ca_file = "${../../../example/certs/ca.pem}" # Set the client certificate path - tls_client_crt_file="" + tls_client_crt_file = "${../../../example/certs/${config.networking.hostName}.pem}" # Set the client private key path - tls_client_key_file="" ''; + tls_client_key_file = "${../../../example/certs/${config.networking.hostName}-key.pem}" + ''; target = "./seaweedfs/filer.toml"; + user = "seaweedfs"; mode = "0440"; }; # config for the volume deamon of seaweedfs systemd.tmpfiles.settings."10-seaweedfs-volume"."/var/lib/seaweedfs/volume".d = { user = "seaweedfs"; + group = "seaweedfs"; mode = "0700"; }; systemd.services.seaweedfs-volume = { From fa9fc37673ff64eceee44d55e8a9deae714cd75a Mon Sep 17 00:00:00 2001 From: Makuru Date: Mon, 7 Oct 2024 21:54:29 +0200 Subject: [PATCH 04/41] allows "weed benchmark", to start. --- nixos-modules/storage/seaweedfs/server.nix | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nixos-modules/storage/seaweedfs/server.nix b/nixos-modules/storage/seaweedfs/server.nix index 7435c1d..b6b0ded 100644 --- a/nixos-modules/storage/seaweedfs/server.nix +++ b/nixos-modules/storage/seaweedfs/server.nix @@ -56,7 +56,7 @@ Type = "notify"; Restart = "always"; RestartSec = "5s"; - ExecStart = ''${pkgs.seaweedfs}/bin/weed filer -master=${address}:9333 -port=8888''; + ExecStart = ''${pkgs.seaweedfs}/bin/weed filer -master=[${address}]:9333 -port=8888''; User = "seaweedfs"; LimitNOFILE = 40000; }; @@ -91,9 +91,9 @@ systemd.services.seaweedfs-volume = { description = "seaweedfs volume service"; wantedBy = [ "multi-user.target" ]; - after = [ "network-online.target" "etcd.service" "seaweedfs-master.service" "seaweedfs-filer.service" ]; + after = [ "network-online.target" "etcd.service" "seaweedfs-master.service" ]; # ++ lib.optional config.networking.firewall.enable "firewall.service"; - wants = [ "network-online.target" "etcd.service" "seaweedfs-master.service" "seaweedfs-filer.service" ]; + wants = [ "network-online.target" "etcd.service" "seaweedfs-master.service" ]; # ++ lib.optional config.networking.firewall.enable "firewall.service"; unitConfig = { Documentation = "https://github.com/seaweedfs/seaweedfs/wiki"; From 6bd9b02f5737fef0903de4cd2c4b4ef73297565e Mon Sep 17 00:00:00 2001 From: Makuru Date: Tue, 8 Oct 2024 13:09:29 +0200 Subject: [PATCH 05/41] Fixed random timouts. --- nixos-modules/storage/seaweedfs/dbBackend/etcd.nix | 2 +- nixos-modules/storage/seaweedfs/server.nix | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/nixos-modules/storage/seaweedfs/dbBackend/etcd.nix b/nixos-modules/storage/seaweedfs/dbBackend/etcd.nix index f164c6f..44a6ba6 100644 --- a/nixos-modules/storage/seaweedfs/dbBackend/etcd.nix +++ b/nixos-modules/storage/seaweedfs/dbBackend/etcd.nix @@ -44,7 +44,7 @@ }; serviceConfig = { - Type = "notify"; + Type = "simple"; Restart = "always"; RestartSec = "5s"; ExecStart = "${pkgs.etcd}/bin/etcd"; diff --git a/nixos-modules/storage/seaweedfs/server.nix b/nixos-modules/storage/seaweedfs/server.nix index b6b0ded..f8c769f 100644 --- a/nixos-modules/storage/seaweedfs/server.nix +++ b/nixos-modules/storage/seaweedfs/server.nix @@ -30,7 +30,7 @@ serviceConfig = let address = builtins.elemAt (lib.splitString "/" (lib.head config.systemd.network.networks."01-br0".addresses).Address) 0; in { - Type = "notify"; + Type = "simple"; Restart = "always"; RestartSec = "5s"; ExecStart = ''${pkgs.seaweedfs}/bin/weed master -ip=[${address}] -peers=[fec0::1]:9333,[fec0::2]:9333,[fec0::3]:9333 -mdir=/var/lib/seaweedfs/master''; @@ -53,10 +53,10 @@ serviceConfig = let address = builtins.elemAt (lib.splitString "/" (lib.head config.systemd.network.networks."01-br0".addresses).Address) 0; in { - Type = "notify"; + Type = "simple"; Restart = "always"; RestartSec = "5s"; - ExecStart = ''${pkgs.seaweedfs}/bin/weed filer -master=[${address}]:9333 -port=8888''; + ExecStart = ''${pkgs.seaweedfs}/bin/weed filer -master=[ ${address}]:9333 -port=8888''; User = "seaweedfs"; LimitNOFILE = 40000; }; @@ -101,7 +101,7 @@ serviceConfig = let address = builtins.elemAt (lib.splitString "/" (lib.head config.systemd.network.networks."01-br0".addresses).Address) 0; in { - Type = "notify"; + Type = "simple"; Restart = "always"; RestartSec = "5s"; ExecStart = ''${pkgs.seaweedfs}/bin/weed volume -port=8080 -max=5 -ip=${address} -mserver=localhost:9333 -dir=/var/lib/seaweedfs/volume''; From 9ce14ed9683e62578033a36622ed1b5ac4fdd807 Mon Sep 17 00:00:00 2001 From: Makuru Date: Wed, 16 Oct 2024 18:36:53 +0200 Subject: [PATCH 06/41] added encryption and various bug bugfixes. --- example-server.nix | 3 +- flake.nix | 1 - nixos-modules/nomad.nix | 7 ++++- .../storage/seaweedfs/dbBackend/etcd.nix | 8 ++--- nixos-modules/storage/seaweedfs/options.nix | 29 +++++-------------- nixos-modules/storage/seaweedfs/server.nix | 8 ++--- 6 files changed, 23 insertions(+), 33 deletions(-) diff --git a/example-server.nix b/example-server.nix index 736ac6a..74df6ec 100644 --- a/example-server.nix +++ b/example-server.nix @@ -88,7 +88,8 @@ storage.seaweedfs = { enable = true; - filer.db.etcd = { + volumeStorage.encrypt = true; + filer.db.etcd = { enable = true; }; }; diff --git a/flake.nix b/flake.nix index 1e301c8..d78f874 100644 --- a/flake.nix +++ b/flake.nix @@ -28,7 +28,6 @@ imports = [ ./nixos-modules/storage/seaweedfs/options.nix ./nixos-modules/storage/seaweedfs/server.nix - # ./nixos-modules/storage/seaweedfs/dbBackend/sqlite.nix ./nixos-modules/storage/seaweedfs/dbBackend/etcd.nix # ./nixos-modules/storage/ceph/server.nix ./nixos-modules/defaults.nix diff --git a/nixos-modules/nomad.nix b/nixos-modules/nomad.nix index d60b1d4..b44de7d 100644 --- a/nixos-modules/nomad.nix +++ b/nixos-modules/nomad.nix @@ -59,6 +59,11 @@ in bootstrap_expect = (builtins.length cfg.servers + 2) / 2; server_join.retry_join = cfg.servers; }; + advertise = let + address = config.skyflake.nodes.${config.networking.hostName}.address; + in { + serf = "[${address}]:4648"; + }; client = { enabled = cfg.client.enable; inherit (cfg.client) meta; @@ -75,7 +80,7 @@ in # alternatives to the nomad web ui wander damon # needed for microvms - virtiofsd ceph seaweedfs + virtiofsd ceph seaweedfs # TODO fix that both need to be installed. jq kmod e2fsprogs ]; }; diff --git a/nixos-modules/storage/seaweedfs/dbBackend/etcd.nix b/nixos-modules/storage/seaweedfs/dbBackend/etcd.nix index 44a6ba6..7acba8c 100644 --- a/nixos-modules/storage/seaweedfs/dbBackend/etcd.nix +++ b/nixos-modules/storage/seaweedfs/dbBackend/etcd.nix @@ -15,7 +15,7 @@ # ++ lib.optional config.networking.firewall.enable "firewall.service"; environment = /* (nixpgs.filterAttrs (n: v: v != null) */ let - address = builtins.elemAt (lib.splitString "/" (lib.head config.systemd.network.networks."01-br0".addresses).Address) 0; + address = config.skyflake.nodes.${config.networking.hostName}.address; in { ETCD_NAME = config.networking.hostName; #ETCD_DISCOVERY = "true"; @@ -33,8 +33,8 @@ ETCD_PEER_CERT_FILE = ../../../../example/certs/${config.networking.hostName}.pem; ETCD_PEER_KEY_FILE = ../../../../example/certs/${config.networking.hostName}-key.pem; #}) // (nixpgs.optionalAttrs (config.services.etcd.discovery == ""){ - #ETCD_INITIAL_CLUSTER = lib.concatMapStringsSep ", " (node: "http://[" + (config.skyflake.nodes."${node}").address + "]:2380") (lib.attrNames config.skyflake.nodes); - ETCD_INITIAL_CLUSTER = "example1=https://[fec0::1]:2380,example2=https://[fec0::2]:2380,example3=https://[fec0::3]:2380"; + ETCD_INITIAL_CLUSTER = lib.concatMapStringsSep ", " (node: "http://[" + (config.skyflake.nodes."${node}").address + "]:2380") (lib.attrNames config.skyflake.nodes); + # ETCD_INITIAL_CLUSTER = "example1=https://[fec0::1]:2380,example2=https://[fec0::2]:2380,example3=https://[fec0::3]:2380"; ETCD_INITIAL_CLUSTER_STATE = "new"; ETCD_INITIAL_CLUSTER_TOKEN = "etcd-cluster"; #}) // (nixpgs.mapAttrs' (n: v: nixpgs.nameValuePair "ETCD_${n}" v) config.services.etcd.extraConf); @@ -44,7 +44,7 @@ }; serviceConfig = { - Type = "simple"; + Type = "notify"; Restart = "always"; RestartSec = "5s"; ExecStart = "${pkgs.etcd}/bin/etcd"; diff --git a/nixos-modules/storage/seaweedfs/options.nix b/nixos-modules/storage/seaweedfs/options.nix index 84398d0..143c600 100644 --- a/nixos-modules/storage/seaweedfs/options.nix +++ b/nixos-modules/storage/seaweedfs/options.nix @@ -18,29 +18,14 @@ }; }); }; - #master = { - # serverIP = lib.mkOption { - # type = lib.str; - # description = '' - # IP of this node. - # ''; - # }; - # listenIPs = lib.mkOption { - # type = lib.listOf lib.str; - # description = '' - # IP of all the master servers. - # Can be the same as storage nodes. - # ''; - # }; - #}; volumeStorage = { - #encrypt = lib.mkOption { - # type = lib.types.bool; - # default = false; - # description = '' - # enable encryption on volume store. - # ''; - #}; + encrypt = lib.mkOption { + type = lib.types.bool; + default = false; + description = '' + enable encryption on volume store. + ''; + }; #datacenter = { # type = lib.str; # description = '' diff --git a/nixos-modules/storage/seaweedfs/server.nix b/nixos-modules/storage/seaweedfs/server.nix index f8c769f..272acef 100644 --- a/nixos-modules/storage/seaweedfs/server.nix +++ b/nixos-modules/storage/seaweedfs/server.nix @@ -28,7 +28,7 @@ Documentation = "https://github.com/seaweedfs/seaweedfs/wiki"; }; serviceConfig = let - address = builtins.elemAt (lib.splitString "/" (lib.head config.systemd.network.networks."01-br0".addresses).Address) 0; + address = config.skyflake.nodes.${config.networking.hostName}.address; in { Type = "simple"; Restart = "always"; @@ -51,12 +51,12 @@ Documentation = "https://github.com/seaweedfs/seaweedfs/wiki"; }; serviceConfig = let - address = builtins.elemAt (lib.splitString "/" (lib.head config.systemd.network.networks."01-br0".addresses).Address) 0; + address = config.skyflake.nodes.${config.networking.hostName}.address; in { Type = "simple"; Restart = "always"; RestartSec = "5s"; - ExecStart = ''${pkgs.seaweedfs}/bin/weed filer -master=[ ${address}]:9333 -port=8888''; + ExecStart = ''${pkgs.seaweedfs}/bin/weed filer ${lib.optionalString config.skyflake.storage.seaweedfs.volumeStorage.encrypt "-encryptVolumeData"} -master=[${address}]:9333 -port=8888''; User = "seaweedfs"; LimitNOFILE = 40000; }; @@ -99,7 +99,7 @@ Documentation = "https://github.com/seaweedfs/seaweedfs/wiki"; }; serviceConfig = let - address = builtins.elemAt (lib.splitString "/" (lib.head config.systemd.network.networks."01-br0".addresses).Address) 0; + address = config.skyflake.nodes.${config.networking.hostName}.address; in { Type = "simple"; Restart = "always"; From 70fa73cd39964bf3172213ff1f8b7a2cfbfe2320 Mon Sep 17 00:00:00 2001 From: Makuru Date: Wed, 23 Oct 2024 19:16:54 +0200 Subject: [PATCH 07/41] Pinned to latest version in nixpkgs unstable that works with nomad_1_6 and added back ceph. --- example/ceph.client.admin.keyring | 6 ++++++ example/ceph.mon.keyring | 3 +++ flake.lock | 18 +++++++++--------- flake.nix | 8 +++++--- 4 files changed, 23 insertions(+), 12 deletions(-) create mode 100644 example/ceph.client.admin.keyring create mode 100644 example/ceph.mon.keyring diff --git a/example/ceph.client.admin.keyring b/example/ceph.client.admin.keyring new file mode 100644 index 0000000..93ca576 --- /dev/null +++ b/example/ceph.client.admin.keyring @@ -0,0 +1,6 @@ +[client.admin] + key = AQDECbJjxEw2DRAAHq4gG9hq7md9AWR8194/Yg== + caps mds = "allow *" + caps mgr = "allow *" + caps mon = "allow *" + caps osd = "allow *" diff --git a/example/ceph.mon.keyring b/example/ceph.mon.keyring new file mode 100644 index 0000000..11b91c0 --- /dev/null +++ b/example/ceph.mon.keyring @@ -0,0 +1,3 @@ +[mon.] + key = AQDECbJjf1z7CxAA8ENCZ+VYhLmgtLKO7JkGcg== + caps mon = "allow *" diff --git a/flake.lock b/flake.lock index 483aae4..3b89e79 100644 --- a/flake.lock +++ b/flake.lock @@ -27,11 +27,11 @@ "spectrum": "spectrum" }, "locked": { - "lastModified": 1727872811, - "narHash": "sha256-78hOotCZ/G8l4nyWPdMWSOojdJ5NwHTR0usBVVBNTtU=", + "lastModified": 1729646324, + "narHash": "sha256-tQOYDfaoetPZSi56dej5zCsSsxPhmV+vWxuW3vWeg60=", "owner": "astro", "repo": "microvm.nix", - "rev": "e832ffc16b09b1b5c7c1224532d03ed3ce68afd0", + "rev": "21c36d162ba8127377ce7dee234bdc99b9af0dde", "type": "github" }, "original": { @@ -100,17 +100,17 @@ }, "nixpkgs_2": { "locked": { - "lastModified": 1728018373, - "narHash": "sha256-NOiTvBbRLIOe5F6RbHaAh6++BNjsb149fGZd1T4+KBg=", - "owner": "nixos", + "lastModified": 1727331237, + "narHash": "sha256-nkhXMPuxbqrgdOrT5Ggy0iTNOkL7g5o2NN2KUDRnjck=", + "owner": "NixOS", "repo": "nixpkgs", - "rev": "bc947f541ae55e999ffdb4013441347d83b00feb", + "rev": "1457235a9eee6e05916cd543d3143360e6fd1080", "type": "github" }, "original": { - "owner": "nixos", - "ref": "nixos-unstable", + "owner": "NixOS", "repo": "nixpkgs", + "rev": "1457235a9eee6e05916cd543d3143360e6fd1080", "type": "github" } }, diff --git a/flake.nix b/flake.nix index d78f874..dbe41c7 100644 --- a/flake.nix +++ b/flake.nix @@ -2,7 +2,9 @@ description = "Hyperconverged Infratructure for NixOS"; inputs = { - nixpkgs.url = "github:nixos/nixpkgs/nixos-unstable"; + nixpkgs = { + url = "github:NixOS/nixpkgs/1457235a9eee6e05916cd543d3143360e6fd1080"; + }; microvm = { url = "github:astro/microvm.nix"; inputs.nixpkgs.follows = "nixpkgs"; @@ -29,7 +31,7 @@ ./nixos-modules/storage/seaweedfs/options.nix ./nixos-modules/storage/seaweedfs/server.nix ./nixos-modules/storage/seaweedfs/dbBackend/etcd.nix - # ./nixos-modules/storage/ceph/server.nix + ./nixos-modules/storage/ceph/server.nix ./nixos-modules/defaults.nix ./nixos-modules/nodes.nix ./nixos-modules/nomad.nix @@ -84,4 +86,4 @@ }; }; -} \ No newline at end of file +} From 88daecee57af6bc900721faac2a6afdcf3aa1567 Mon Sep 17 00:00:00 2001 From: Makuru Date: Wed, 23 Oct 2024 19:18:32 +0200 Subject: [PATCH 08/41] Removed redundant input from user. --- example-server.nix | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/example-server.nix b/example-server.nix index 74df6ec..bbab9b1 100644 --- a/example-server.nix +++ b/example-server.nix @@ -1,6 +1,6 @@ { instance }: -{ config, pkgs, ... }: +{ config, pkgs, lib, ... }: { microvm = { @@ -93,25 +93,25 @@ enable = true; }; }; - #storage.ceph = { - # fsid = "8364da79-5e03-49ae-82ea-7d936278cb0f"; - # monKeyring = example/ceph.mon.keyring; - # adminKeyring = example/ceph.client.admin.keyring; - # osds = [ { - # id = instance; - # fsid = "8e4ae689-5c15-4381-bd75-19de743378e${toString instance}"; - # path = "/dev/vdb"; - # deviceClass = "ssd"; - # keyfile = toString (./example + "/osd.${toString instance}.keyring"); - # } ]; - # rbdPools.microvms = { - # params = { size = 2; class = "ssd"; }; - # }; - # cephfs.skyflake.metaParams = { size = 2; class = "ssd"; }; - #}; + storage.ceph = { + fsid = "8364da79-5e03-49ae-82ea-7d936278cb0f"; + monKeyring = example/ceph.mon.keyring; + adminKeyring = example/ceph.client.admin.keyring; + osds = [ { + id = instance; + fsid = "8e4ae689-5c15-4381-bd75-19de743378e${toString instance}"; + path = "/dev/vdb"; + deviceClass = "ssd"; + keyfile = toString (./example + "/osd.${toString instance}.keyring"); + } ]; + rbdPools.microvms = { + params = { size = 2; class = "ssd"; }; + }; + cephfs.skyflake.metaParams = { size = 2; class = "ssd"; }; + }; nomad = { - servers = [ "example1" "example2" "example3" ]; + servers = [ (builtins.concatStringsSep " " (lib.attrNames config.skyflake.nodes)) ]; client.meta = { example-deployment = "yes"; }; From 64c915ed7094b792a1c642d0a43539820084ed8b Mon Sep 17 00:00:00 2001 From: Makuru Date: Wed, 23 Oct 2024 19:19:26 +0200 Subject: [PATCH 09/41] Removed duplicate ceph and seaweedfs input in nomad.nix . --- nixos-modules/nomad.nix | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/nixos-modules/nomad.nix b/nixos-modules/nomad.nix index b44de7d..9962f57 100644 --- a/nixos-modules/nomad.nix +++ b/nixos-modules/nomad.nix @@ -1,7 +1,6 @@ { config, lib, pkgs, ... }: let cfg = config.skyflake.nomad; - in { options.skyflake.nomad = with lib; { @@ -80,7 +79,7 @@ in # alternatives to the nomad web ui wander damon # needed for microvms - virtiofsd ceph seaweedfs # TODO fix that both need to be installed. + virtiofsd jq kmod e2fsprogs ]; }; From 9333d5c4f0407c2a0cf8fedf9356db2b8e78b351 Mon Sep 17 00:00:00 2001 From: Makuru Date: Wed, 23 Oct 2024 19:20:48 +0200 Subject: [PATCH 10/41] Made ceph config, activate conditionally. --- nixos-modules/storage/ceph/server.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nixos-modules/storage/ceph/server.nix b/nixos-modules/storage/ceph/server.nix index 3ce023a..cc7de5d 100644 --- a/nixos-modules/storage/ceph/server.nix +++ b/nixos-modules/storage/ceph/server.nix @@ -131,7 +131,7 @@ in { }; }; - config = { + config = lib.mkIf config.skyflake.storage.ceph.enable { boot.kernelModules = [ "ceph" ]; environment.systemPackages = [ cfg.package ]; From 4123801d3c58748d5735412c87131cad9d985642 Mon Sep 17 00:00:00 2001 From: Makuru Date: Wed, 23 Oct 2024 19:23:21 +0200 Subject: [PATCH 11/41] Moved /etc/seaweedfs/filer.toml to etcd.nix . --- .../storage/seaweedfs/dbBackend/etcd.nix | 21 +++++++++++++++++++ nixos-modules/storage/seaweedfs/server.nix | 5 +++-- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/nixos-modules/storage/seaweedfs/dbBackend/etcd.nix b/nixos-modules/storage/seaweedfs/dbBackend/etcd.nix index 7acba8c..12707f3 100644 --- a/nixos-modules/storage/seaweedfs/dbBackend/etcd.nix +++ b/nixos-modules/storage/seaweedfs/dbBackend/etcd.nix @@ -53,6 +53,27 @@ }; }; + environment.etc.seaweedfs-filer = { + text = '' + [etcd] + enabled = true + servers = "example1=https://[fec0::1]:2380,example2=https://[fec0::2]:2380,example3=https://[fec0::3]:2380" + # username = "seaweedfs" + # password = "" + key_prefix = "seaweedfs." + timeout = "3s" + # Set the CA certificate path + tls_ca_file = "${../../../../example/certs/ca.pem}" + # Set the client certificate path + tls_client_crt_file = "${../../../../example/certs/${config.networking.hostName}.pem}" + # Set the client private key path + tls_client_key_file = "${../../../../example/certs/${config.networking.hostName}-key.pem}" + ''; + target = "./seaweedfs/filer.toml"; + user = "seaweedfs"; + mode = "0440"; + }; + environment.systemPackages = [ pkgs.etcd ]; /* TODO: add firewall to skyflake. networking.firewall = lib.mkIf config.services.etcd.openFirewall { diff --git a/nixos-modules/storage/seaweedfs/server.nix b/nixos-modules/storage/seaweedfs/server.nix index 272acef..2618ba1 100644 --- a/nixos-modules/storage/seaweedfs/server.nix +++ b/nixos-modules/storage/seaweedfs/server.nix @@ -1,6 +1,6 @@ { config, lib, pkgs, ... }: { - config = lib.mkIf config.skyflake.storage.seaweedfs.filer.db.etcd.enable { + config = lib.mkIf config.skyflake.storage.seaweedfs.enable { users.users.seaweedfs = { isSystemUser = true; @@ -29,11 +29,12 @@ }; serviceConfig = let address = config.skyflake.nodes.${config.networking.hostName}.address; + peers = "[fec0::1]:9333,[fec0::2]:9333,[fec0::3]:9333"; in { Type = "simple"; Restart = "always"; RestartSec = "5s"; - ExecStart = ''${pkgs.seaweedfs}/bin/weed master -ip=[${address}] -peers=[fec0::1]:9333,[fec0::2]:9333,[fec0::3]:9333 -mdir=/var/lib/seaweedfs/master''; + ExecStart = ''${pkgs.seaweedfs}/bin/weed master -ip=[${address}] -peers=${peers} -mdir=/var/lib/seaweedfs/master''; User = "seaweedfs"; LimitNOFILE = 40000; }; From 1ed5a4885ff3b4a291738c8700a9a649b0f3c8d1 Mon Sep 17 00:00:00 2001 From: Makuru Date: Wed, 23 Oct 2024 19:47:51 +0200 Subject: [PATCH 12/41] Made etcd activate only if seaweedfs and etcd backend is active. --- nixos-modules/storage/seaweedfs/dbBackend/etcd.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nixos-modules/storage/seaweedfs/dbBackend/etcd.nix b/nixos-modules/storage/seaweedfs/dbBackend/etcd.nix index 12707f3..dfde585 100644 --- a/nixos-modules/storage/seaweedfs/dbBackend/etcd.nix +++ b/nixos-modules/storage/seaweedfs/dbBackend/etcd.nix @@ -1,6 +1,6 @@ { pkgs, config, lib, ... }: { - config = lib.mkIf config.skyflake.storage.seaweedfs.filer.db.etcd.enable { + config = lib.mkIf (builtins.all (x: x == true) [ config.skyflake.storage.seaweedfs.filer.db.etcd.enable config.skyflake.storage.seaweedfs.enable ]) { systemd.tmpfiles.settings."10-etcd"."/var/lib/etcd".d = { user = "etcd"; mode = "0700"; From f7db11aa1b4a0cfff708a7e7622b28ab04f7ef38 Mon Sep 17 00:00:00 2001 From: Makuru Date: Wed, 23 Oct 2024 19:48:27 +0200 Subject: [PATCH 13/41] changed back to ceph. --- example-server.nix | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/example-server.nix b/example-server.nix index bbab9b1..21f8bad 100644 --- a/example-server.nix +++ b/example-server.nix @@ -87,13 +87,14 @@ storage.seaweedfs = { - enable = true; + enable = false; volumeStorage.encrypt = true; filer.db.etcd = { enable = true; }; }; storage.ceph = { + enable = true; fsid = "8364da79-5e03-49ae-82ea-7d936278cb0f"; monKeyring = example/ceph.mon.keyring; adminKeyring = example/ceph.client.admin.keyring; From 026415d5a8ac0d2a2c42525174e62c95f06bcd6c Mon Sep 17 00:00:00 2001 From: Makuru Date: Wed, 23 Oct 2024 20:07:33 +0200 Subject: [PATCH 14/41] Removed etc connect config from seaweed config. --- nixos-modules/storage/seaweedfs/server.nix | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/nixos-modules/storage/seaweedfs/server.nix b/nixos-modules/storage/seaweedfs/server.nix index 2618ba1..0d037b6 100644 --- a/nixos-modules/storage/seaweedfs/server.nix +++ b/nixos-modules/storage/seaweedfs/server.nix @@ -62,26 +62,6 @@ LimitNOFILE = 40000; }; }; - environment.etc.seaweedfs-filer = { - text = '' - [etcd] - enabled = true - servers = "example1=https://[fec0::1]:2380,example2=https://[fec0::2]:2380,example3=https://[fec0::3]:2380" - # username = "seaweedfs" - # password = "" - key_prefix = "seaweedfs." - timeout = "3s" - # Set the CA certificate path - tls_ca_file = "${../../../example/certs/ca.pem}" - # Set the client certificate path - tls_client_crt_file = "${../../../example/certs/${config.networking.hostName}.pem}" - # Set the client private key path - tls_client_key_file = "${../../../example/certs/${config.networking.hostName}-key.pem}" - ''; - target = "./seaweedfs/filer.toml"; - user = "seaweedfs"; - mode = "0440"; - }; # config for the volume deamon of seaweedfs systemd.tmpfiles.settings."10-seaweedfs-volume"."/var/lib/seaweedfs/volume".d = { From 2bfef5f355a56a76bd1cfe05ed3d928b2efa1407 Mon Sep 17 00:00:00 2001 From: Makuru Date: Wed, 23 Oct 2024 21:31:26 +0200 Subject: [PATCH 15/41] Fixed an hardcoded option. --- nixos-modules/storage/seaweedfs/server.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nixos-modules/storage/seaweedfs/server.nix b/nixos-modules/storage/seaweedfs/server.nix index 0d037b6..7d85810 100644 --- a/nixos-modules/storage/seaweedfs/server.nix +++ b/nixos-modules/storage/seaweedfs/server.nix @@ -29,12 +29,12 @@ }; serviceConfig = let address = config.skyflake.nodes.${config.networking.hostName}.address; - peers = "[fec0::1]:9333,[fec0::2]:9333,[fec0::3]:9333"; + peers = "${lib.concatMapStrings (x: x + ":9333,") (builtins.catAttrs "address" (builtins.attrValues config.skyflake.nodes))}"; in { Type = "simple"; Restart = "always"; RestartSec = "5s"; - ExecStart = ''${pkgs.seaweedfs}/bin/weed master -ip=[${address}] -peers=${peers} -mdir=/var/lib/seaweedfs/master''; + ExecStart = ''${pkgs.seaweedfs}/bin/weed master -ip=${address} -peers=${peers} -mdir=/var/lib/seaweedfs/master''; User = "seaweedfs"; LimitNOFILE = 40000; }; From d05119c3f4959790a07f7683d736908f524e64bf Mon Sep 17 00:00:00 2001 From: Makuru Date: Wed, 23 Oct 2024 21:31:47 +0200 Subject: [PATCH 16/41] Made the IP valid. --- example-server.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/example-server.nix b/example-server.nix index 21f8bad..cf68fe0 100644 --- a/example-server.nix +++ b/example-server.nix @@ -81,7 +81,7 @@ nodes = builtins.listToAttrs ( map (instance: { name = "example${toString instance}"; - value.address = "fec0::${toString instance}"; + value.address = "[fec0::${toString instance}]"; }) [ 1 2 3 ] ); From 070a2d6939dc0a2a79e9004b01b2af92e8842483 Mon Sep 17 00:00:00 2001 From: Makuru Date: Wed, 23 Oct 2024 21:39:16 +0200 Subject: [PATCH 17/41] Fixed a nother hard coded value in etcd.nix . --- .../storage/seaweedfs/dbBackend/etcd.nix | 23 ++++++++----------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/nixos-modules/storage/seaweedfs/dbBackend/etcd.nix b/nixos-modules/storage/seaweedfs/dbBackend/etcd.nix index dfde585..3525b80 100644 --- a/nixos-modules/storage/seaweedfs/dbBackend/etcd.nix +++ b/nixos-modules/storage/seaweedfs/dbBackend/etcd.nix @@ -6,10 +6,10 @@ mode = "0700"; }; - systemd.services.etcd = { + systemd.services."etcd" = { description = "etcd key-value store"; wantedBy = [ "multi-user.target" ]; - after = [ "network-online.target" ]; + after = [ "network-online.target" "network.target" ]; # ++ lib.optional config.networking.firewall.enable "firewall.service"; wants = [ "network-online.target" ]; # ++ lib.optional config.networking.firewall.enable "firewall.service"; @@ -18,12 +18,11 @@ address = config.skyflake.nodes.${config.networking.hostName}.address; in { ETCD_NAME = config.networking.hostName; - #ETCD_DISCOVERY = "true"; ETCD_DATA_DIR = "/var/lib/etcd"; - ETCD_ADVERTISE_CLIENT_URLS = "https://[${address}]:2379"; # TODO Make it choose the IP of the device declaratively - ETCD_LISTEN_CLIENT_URLS = "https://[${address}]:2379"; # TODO Make it choose the IP of the device declaratively - ETCD_LISTEN_PEER_URLS = "https://[${address}]:2380"; # TODO Make it choose the IP of the device declaratively - ETCD_INITIAL_ADVERTISE_PEER_URLS = "https://[${address}]:2380"; # TODO Make it choose the IP of the device declaratively + ETCD_ADVERTISE_CLIENT_URLS = "https://${address}:2379"; + ETCD_LISTEN_CLIENT_URLS = "https://${address}:2379"; + ETCD_LISTEN_PEER_URLS = "https://${address}:2380"; + ETCD_INITIAL_ADVERTISE_PEER_URLS = "https://${address}:2380"; ETCD_CLIENT_CERT_AUTH = "true"; ETCD_TRUSTED_CA_FILE = ../../../../example/certs/ca.pem; ETCD_CERT_FILE = ../../../../example/certs/${config.networking.hostName}.pem; @@ -32,21 +31,19 @@ ETCD_PEER_TRUSTED_CA_FILE = ../../../../example/certs/ca.pem; ETCD_PEER_CERT_FILE = ../../../../example/certs/${config.networking.hostName}.pem; ETCD_PEER_KEY_FILE = ../../../../example/certs/${config.networking.hostName}-key.pem; - #}) // (nixpgs.optionalAttrs (config.services.etcd.discovery == ""){ - ETCD_INITIAL_CLUSTER = lib.concatMapStringsSep ", " (node: "http://[" + (config.skyflake.nodes."${node}").address + "]:2380") (lib.attrNames config.skyflake.nodes); - # ETCD_INITIAL_CLUSTER = "example1=https://[fec0::1]:2380,example2=https://[fec0::2]:2380,example3=https://[fec0::3]:2380"; + ETCD_INITIAL_CLUSTER = "${lib.concatMapStringsSep "," (node: "${node}=https://" + (config.skyflake.nodes."${node}").address + ":2380") (builtins.attrNames config.skyflake.nodes)}"; ETCD_INITIAL_CLUSTER_STATE = "new"; ETCD_INITIAL_CLUSTER_TOKEN = "etcd-cluster"; - #}) // (nixpgs.mapAttrs' (n: v: nixpgs.nameValuePair "ETCD_${n}" v) config.services.etcd.extraConf); }; unitConfig = { - Documentation = "https://github.com/coreos/etcd"; + Documentation = "https://etcd.io/docs/v3.5/"; }; serviceConfig = { Type = "notify"; Restart = "always"; RestartSec = "5s"; + ExecStartPre = "${pkgs.coreutils}/bin/sleep 2"; # TODO fix workaround, so that it doesnt stop on first start because it cant bind. ExecStart = "${pkgs.etcd}/bin/etcd"; User = "etcd"; LimitNOFILE = 40000; @@ -57,7 +54,7 @@ text = '' [etcd] enabled = true - servers = "example1=https://[fec0::1]:2380,example2=https://[fec0::2]:2380,example3=https://[fec0::3]:2380" + servers = "${lib.concatMapStringsSep "," (node: "${node}=https://" + (config.skyflake.nodes."${node}").address + ":2380") (builtins.attrNames config.skyflake.nodes)}" # username = "seaweedfs" # password = "" key_prefix = "seaweedfs." From bd858de3cf33670db425b847a4f3b74dad85edef Mon Sep 17 00:00:00 2001 From: Makuru Date: Sun, 17 Nov 2024 02:46:52 +0100 Subject: [PATCH 18/41] Fixed seaweedfs filer service. --- default-customization.nix | 4 +- nixos-modules/ssh-deploy.nix | 8 +- nixos-modules/storage/ceph/server.nix | 2 +- .../storage/seaweedfs/dbBackend/etcd.nix | 2 +- nixos-modules/storage/seaweedfs/options.nix | 109 +++++++++++++----- nixos-modules/storage/seaweedfs/server.nix | 50 ++++++-- vm/customization-options.nix | 3 +- vm/nomad-job.nix | 6 +- 8 files changed, 131 insertions(+), 53 deletions(-) diff --git a/default-customization.nix b/default-customization.nix index 471965e..8c3be28 100644 --- a/default-customization.nix +++ b/default-customization.nix @@ -37,7 +37,7 @@ in mountPoint = "/nix/.ro-store"; } ]; volumes = [ { - image = config.skyflake.deploy.rbds.root.path; + image = config.skyflake.deploy.ceph.rbds.root.path; mountPoint = "/"; # don't let microvm.nix create an image file autoCreate = false; @@ -55,7 +55,7 @@ in } ]; }; - config.skyflake.deploy.rbds.root = { + config.skyflake.deploy.ceph.rbds.root = { pool = "microvms"; namespace = user; name = "${repo}-${vmName}-root"; diff --git a/nixos-modules/ssh-deploy.nix b/nixos-modules/ssh-deploy.nix index d2c5362..99daa44 100644 --- a/nixos-modules/ssh-deploy.nix +++ b/nixos-modules/ssh-deploy.nix @@ -3,7 +3,7 @@ { config, lib, pkgs, ... }: let - inherit (config.skyflake.storage.seaweedfs) seaweedfsMount; + inherit (config.skyflake.storage.seaweedfs) mounts; debugShell = lib.optionalString config.skyflake.debug '' set -x @@ -156,7 +156,7 @@ in { binaryCachePath = mkOption { type = types.str; - default = seaweedfsMount.skyflake-binary-cache.mountPoint; + default = mounts.skyflake-binary-cache.mountPoint; description = '' Directory which is mounted on all nodes that will be used to share the /nix/store with MicroVMs. @@ -165,7 +165,7 @@ in { sharedGcrootsPath = mkOption { type = types.str; - default = seaweedfsMount.skyflake-gcroots.mountPoint; + default = mounts.skyflake-gcroots.mountPoint; description = '' Directory which is mounted on all nodes, is linked from /nix/var/nix/gcroots/, and contains links to all currently @@ -211,7 +211,7 @@ in { }; config = { - skyflake.storage.seaweedfs.seaweedfsMount = { + skyflake.storage.seaweedfs.mounts = { skyflake-binary-cache.mountPoint = "/var/lib/skyflake/binary-cache"; skyflake-gcroots.mountPoint = "/nix/var/nix/gcroots/skyflake"; }; diff --git a/nixos-modules/storage/ceph/server.nix b/nixos-modules/storage/ceph/server.nix index cc7de5d..cde3ee1 100644 --- a/nixos-modules/storage/ceph/server.nix +++ b/nixos-modules/storage/ceph/server.nix @@ -148,7 +148,7 @@ in { enable = true; global = rec { inherit (cfg) fsid; - publicNetwork = clusterNetwork; #"0.0.0.0/0, ::/0"; + publicNetwork = clusterNetwork; clusterNetwork = lib.concatStringsSep ", " ( lib.concatMap ({ addresses ? [], ... }: # Needs to be changed because of diff --git a/nixos-modules/storage/seaweedfs/dbBackend/etcd.nix b/nixos-modules/storage/seaweedfs/dbBackend/etcd.nix index 3525b80..3ba7dde 100644 --- a/nixos-modules/storage/seaweedfs/dbBackend/etcd.nix +++ b/nixos-modules/storage/seaweedfs/dbBackend/etcd.nix @@ -54,7 +54,7 @@ text = '' [etcd] enabled = true - servers = "${lib.concatMapStringsSep "," (node: "${node}=https://" + (config.skyflake.nodes."${node}").address + ":2380") (builtins.attrNames config.skyflake.nodes)}" + servers = "${lib.concatMapStringsSep "," (node: (config.skyflake.nodes."${node}").address + ":2379") (builtins.attrNames config.skyflake.nodes)}" # username = "seaweedfs" # password = "" key_prefix = "seaweedfs." diff --git a/nixos-modules/storage/seaweedfs/options.nix b/nixos-modules/storage/seaweedfs/options.nix index 143c600..372f1a8 100644 --- a/nixos-modules/storage/seaweedfs/options.nix +++ b/nixos-modules/storage/seaweedfs/options.nix @@ -8,16 +8,6 @@ enable seaweedfs as the storage backend. ''; }; - seaweedfsMount = lib.mkOption { - type = lib.types.attrsOf (lib.types.submodule { - options = { - mountPoint = lib.mkOption { - type = with lib.types; nullOr str; - default = null; - }; - }; - }); - }; volumeStorage = { encrypt = lib.mkOption { type = lib.types.bool; @@ -38,30 +28,91 @@ # The rack location of the node. # ''; #}; - #serverIP = lib.mkOption { - # type = lib.listOf lib.str; - # description = '' - # IP of this node. - # ''; - #}; - #listenIPs = lib.mkOption { - # type = lib.listOf lib.str; - # description = '' - # IPs of all the nodes that should store the actual data but not metadata. - # Can be the same as DB nodes. - # ''; - #}; }; - filer.db = { - etcd = { + S3 = lib.mkOption { + type = lib.types { enable = lib.mkOption { - type = lib.types.bool; - default = true; + type = lib.bool; + default = false; description = '' - Use to enable Kubernetes etcd database as a backend for seaweedfs. + If you want to expose an S3 compatible bucket. ''; }; - + port = lib.mkOption { + type = lib.types.port; # FIXME + default = 8333; + description = '' + The port the S3 API should listen to. + ''; + }; + }; + }; + mounts = lib.mkOption { + type = lib.types.attrsOf (lib.types.submodule { + options = { + mountPoint = lib.mkOption { + type = lib.types.str; + default = null; + description = '' + Place where the filesystem is Mounted. + ''; + }; + replication = lib.mkOption { + type = lib.types.nullOr lib.types.ints.u8; + default = null; + description = '' + Is the replication level for each file. + It overwrites replication settings on both filer and master. + ''; + }; + cacheCapacity = lib.mkOption { + type = lib.types.ints.unsigned; + default = 0; + description = '' + Means file chunk read cache capacity in MB with tiered cache(memory + disk), + default 0 which means chunk cache for read is disabled. + ''; + }; + chunkSizeLimit = lib.mkOption { + type = lib.types.ints.positive; + default = 2; + description = '' + Local write buffer size, also chunk large file, default 2 MB. + ''; + }; + }; + }); + }; + filer = { + #TODO + size = lib.mkOption { + type = lib.types.ints.unsigned; + default = 30000; + description = '' + TODO + ''; + }; + #TODO + deviceClass = lib.mkOption { + type = lib.types.str; + default = "unset"; + example = '' + `NVME` `SSD` `HDD` + ''; + description = '' + hard drive or solid state drive or any tag. + ''; + }; + db = { + etcd = { + enable = lib.mkOption { + type = lib.types.bool; + default = true; + description = '' + Use to enable Kubernetes etcd database as a backend for seaweedfs. + ''; + }; + }; }; }; }; diff --git a/nixos-modules/storage/seaweedfs/server.nix b/nixos-modules/storage/seaweedfs/server.nix index 7d85810..474a57c 100644 --- a/nixos-modules/storage/seaweedfs/server.nix +++ b/nixos-modules/storage/seaweedfs/server.nix @@ -11,13 +11,22 @@ }; users.groups.seaweedfs = {}; + # config for the volume deamon of seaweedfs + systemd.tmpfiles.settings."10-seaweedfs-volume"."/var/lib/seaweedfs/volume".d = { + user = "seaweedfs"; + group = "seaweedfs"; + mode = "0700"; + }; + # config for the master deamon of seaweedfs systemd.tmpfiles.settings."10-seaweedfs-master"."/var/lib/seaweedfs/master".d = { user = "seaweedfs"; group = "seaweedfs"; mode = "0700"; }; - systemd.services.seaweedfs-master = { + + systemd.services = lib.mkMerge [ { + seaweedfs-master = { description = "seaweedfs master service"; wantedBy = [ "multi-user.target" ]; after = [ "network-online.target" "etcd.service" ]; @@ -41,7 +50,7 @@ }; # config for the filer deamon of seaweedfs - systemd.services.seaweedfs-filer = { + seaweedfs-filer = { description = "seaweedfs filer service"; wantedBy = [ "multi-user.target" ]; after = [ "network-online.target" "etcd.service" "seaweedfs-master.service" ]; @@ -57,19 +66,14 @@ Type = "simple"; Restart = "always"; RestartSec = "5s"; - ExecStart = ''${pkgs.seaweedfs}/bin/weed filer ${lib.optionalString config.skyflake.storage.seaweedfs.volumeStorage.encrypt "-encryptVolumeData"} -master=[${address}]:9333 -port=8888''; + # TODO make userdefinable port + ExecStart = ''${pkgs.seaweedfs}/bin/weed filer ${lib.optionalString config.skyflake.storage.seaweedfs.volumeStorage.encrypt "-encryptVolumeData"} -master=${address}:9333 -port=8888''; User = "seaweedfs"; LimitNOFILE = 40000; }; }; - # config for the volume deamon of seaweedfs - systemd.tmpfiles.settings."10-seaweedfs-volume"."/var/lib/seaweedfs/volume".d = { - user = "seaweedfs"; - group = "seaweedfs"; - mode = "0700"; - }; - systemd.services.seaweedfs-volume = { + seaweedfs-volume = { description = "seaweedfs volume service"; wantedBy = [ "multi-user.target" ]; after = [ "network-online.target" "etcd.service" "seaweedfs-master.service" ]; @@ -85,11 +89,35 @@ Type = "simple"; Restart = "always"; RestartSec = "5s"; + # TODO add S3 bucket support in here: https://github.com/seaweedfs/seaweedfs/wiki/Production-Setup#setup-s3-api ExecStart = ''${pkgs.seaweedfs}/bin/weed volume -port=8080 -max=5 -ip=${address} -mserver=localhost:9333 -dir=/var/lib/seaweedfs/volume''; User = "seaweedfs"; LimitNOFILE = 40000; }; - }; + }; } + + # config for the mount deamon of seaweedfs + (lib.mapAttrs' (name: value: lib.nameValuePair ("seaweedfs-mount-" + name) { + description = "seaweedfs mount service"; + wantedBy = [ "multi-user.target" ]; + after = [ "network-online.target" "etcd.service" "seaweedfs-filer.service" ]; + # ++ lib.optional config.networking.firewall.enable "firewall.service"; + wants = [ "network-online.target" "etcd.service" "seaweedfs-filer.service" ]; + # ++ lib.optional config.networking.firewall.enable "firewall.service"; + unitConfig = { + Documentation = "https://github.com/seaweedfs/seaweedfs/wiki"; + }; + serviceConfig = { + Type = "simple"; + Restart = "always"; + RestartSec = "5s"; + #TODO FIX hardcoding of port + ExecStart = ''${pkgs.seaweedfs}/bin/weed mount -filer=localhost:8888 -volumeServerAccess=filerProxy ${lib.optionalString (!builtins.isNull value.replication) "-replication=${builtins.toString value.replication}"} -cacheCapacityMB=${builtins.toString value.cacheCapacity} -chunkSizeLimitMB=${builtins.toString value.chunkSizeLimit} -dirAutoCreate -dir=${value.mountPoint} -filer.path=${value.mountPoint}''; + User = "seaweedfs"; + LimitNOFILE = 40000; + }; + }) config.skyflake.storage.seaweedfs.mounts) + ]; environment.systemPackages = [ pkgs.seaweedfs ]; /* TODO: add firewall to skyflake. diff --git a/vm/customization-options.nix b/vm/customization-options.nix index 1aa5581..fc93906 100644 --- a/vm/customization-options.nix +++ b/vm/customization-options.nix @@ -20,8 +20,7 @@ ''; }; - # TODO: rename it to something that allows more than ceph. - deploy.rbds = mkOption { + deploy.ceph.rbds = mkOption { default = {}; description = '' Ceph RBDs used by this MicroVM diff --git a/vm/nomad-job.nix b/vm/nomad-job.nix index e8616cc..06ee0df 100644 --- a/vm/nomad-job.nix +++ b/vm/nomad-job.nix @@ -207,7 +207,7 @@ ${'' } } - ${lib.concatMapStrings (id: with config.skyflake.deploy.rbds.${id}; '' + ${lib.concatMapStrings (id: with config.skyflake.deploy.ceph.rbds.${id}; '' task "rbd-map-${id}" { driver = "raw_exec" lifecycle { @@ -270,7 +270,7 @@ ${'' EOD } } - '') (builtins.attrNames config.skyflake.deploy.rbds)} + '') (builtins.attrNames config.skyflake.deploy.ceph.rbds)} task "hypervisor" { driver = "raw_exec" @@ -347,4 +347,4 @@ pkgs.stdenv.mkDerivation rec { installPhase = '' cp $NAME $out ''; -} +} \ No newline at end of file From 836c2c15c829898fd05787454a1820a07a760c38 Mon Sep 17 00:00:00 2001 From: Makuru Date: Sun, 17 Nov 2024 03:19:46 +0100 Subject: [PATCH 19/41] Working `fuse` mounts with seaweedfs. --- nixos-modules/storage/seaweedfs/server.nix | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/nixos-modules/storage/seaweedfs/server.nix b/nixos-modules/storage/seaweedfs/server.nix index 474a57c..fd17dfe 100644 --- a/nixos-modules/storage/seaweedfs/server.nix +++ b/nixos-modules/storage/seaweedfs/server.nix @@ -107,19 +107,23 @@ unitConfig = { Documentation = "https://github.com/seaweedfs/seaweedfs/wiki"; }; + path = with pkgs; [ fuse3 ]; serviceConfig = { Type = "simple"; Restart = "always"; RestartSec = "5s"; - #TODO FIX hardcoding of port + #TODO FIX hardcoding of port https://github.com/seaweedfs/seaweedfs/issues/877 ExecStart = ''${pkgs.seaweedfs}/bin/weed mount -filer=localhost:8888 -volumeServerAccess=filerProxy ${lib.optionalString (!builtins.isNull value.replication) "-replication=${builtins.toString value.replication}"} -cacheCapacityMB=${builtins.toString value.cacheCapacity} -chunkSizeLimitMB=${builtins.toString value.chunkSizeLimit} -dirAutoCreate -dir=${value.mountPoint} -filer.path=${value.mountPoint}''; - User = "seaweedfs"; + # TODO FIX mount with root!!! + User = "root"; LimitNOFILE = 40000; }; }) config.skyflake.storage.seaweedfs.mounts) ]; - environment.systemPackages = [ pkgs.seaweedfs ]; + environment.systemPackages = with pkgs; [ + seaweedfs # install seaweedfs utils + ]; /* TODO: add firewall to skyflake. networking.firewall = lib.mkIf config.services.etcd.openFirewall { allowedTCPPorts = [ From b074234143639352a03a60ea2348d647eaf6bf7f Mon Sep 17 00:00:00 2001 From: Makuru Date: Sun, 17 Nov 2024 03:26:13 +0100 Subject: [PATCH 20/41] Cleaned up for merge and added documentation. --- example-server.nix | 17 ++++++++--------- flake.nix | 4 ++-- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/example-server.nix b/example-server.nix index cf68fe0..ab4dae0 100644 --- a/example-server.nix +++ b/example-server.nix @@ -1,10 +1,10 @@ { instance }: -{ config, pkgs, lib, ... }: +{ config, lib, pkgs, ... }: { microvm = { - vcpu = 4; + vcpu = 2; mem = 4096; shares = [ @@ -20,11 +20,11 @@ mountPoint = "/"; size = 20 * 1024; } - # { - # image = "example${toString instance}-ceph.img"; - # mountPoint = null; - # size = 20 * 1024; - # } + { + image = "example${toString instance}-ceph.img"; + mountPoint = null; + size = 20 * 1024; + } ]; writableStoreOverlay = "/nix/.rw-store"; @@ -69,8 +69,6 @@ IPv6AcceptRA = true; }; addresses = [ { - # TODO: addressConfig needs to be removed. - # trace: warning: Using 'addressConfig' is deprecated! Move all attributes inside one level up and remove it. Address = "fec0::${toString instance}/64"; # } ]; }; @@ -131,5 +129,6 @@ environment.systemPackages = with pkgs; [ tcpdump + nmap ]; } \ No newline at end of file diff --git a/flake.nix b/flake.nix index dbe41c7..130a6cb 100644 --- a/flake.nix +++ b/flake.nix @@ -3,7 +3,7 @@ inputs = { nixpkgs = { - url = "github:NixOS/nixpkgs/1457235a9eee6e05916cd543d3143360e6fd1080"; + url = "github:NixOS/nixpkgs/1457235a9eee6e05916cd543d3143360e6fd1080"; # Last version of NixOS unstable that supports a foss version of nomad. }; microvm = { url = "github:astro/microvm.nix"; @@ -81,7 +81,7 @@ make-ceph = { type = "app"; - #program = toString (pkgs.callPackage ./pkgs/make-ceph.nix {}); + program = toString (pkgs.callPackage ./pkgs/make-ceph.nix {}); }; }; From b4316bf4c10eca27cbdc759e8994dc3bdc39b0dd Mon Sep 17 00:00:00 2001 From: Makuru Date: Sun, 17 Nov 2024 03:47:52 +0100 Subject: [PATCH 21/41] Switched to btrfs for VMs for futureproofing. --- example-server.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/example-server.nix b/example-server.nix index ab4dae0..78d2539 100644 --- a/example-server.nix +++ b/example-server.nix @@ -19,6 +19,7 @@ image = "example${toString instance}-persist.img"; mountPoint = "/"; size = 20 * 1024; + fsType = "btrfs"; # needed for some seaweedfs optimizations. } { image = "example${toString instance}-ceph.img"; From ae210aea682db1245084ed457a487b8945ce0f0f Mon Sep 17 00:00:00 2001 From: Makuru Date: Sun, 17 Nov 2024 04:47:19 +0100 Subject: [PATCH 22/41] Updated flake.lock by pinning only nomad to its latest compatible nixpkgs release. --- flake.lock | 37 ++++++++++++++++++++----------------- flake.nix | 14 ++++++++++---- nixos-modules/nomad.nix | 2 +- 3 files changed, 31 insertions(+), 22 deletions(-) diff --git a/flake.lock b/flake.lock index 3b89e79..b9f7d53 100644 --- a/flake.lock +++ b/flake.lock @@ -5,11 +5,11 @@ "systems": "systems" }, "locked": { - "lastModified": 1710146030, - "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=", + "lastModified": 1726560853, + "narHash": "sha256-X6rJYSESBVr3hBoH0WbKE5KvhPU5bloyZ2L4K60/fPQ=", "owner": "numtide", "repo": "flake-utils", - "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a", + "rev": "c1dfcf08411b08f6b8615f7d8971a2bfa81d5e8a", "type": "github" }, "original": { @@ -27,11 +27,11 @@ "spectrum": "spectrum" }, "locked": { - "lastModified": 1729646324, - "narHash": "sha256-tQOYDfaoetPZSi56dej5zCsSsxPhmV+vWxuW3vWeg60=", + "lastModified": 1731240174, + "narHash": "sha256-HYu+bPoV3UILhwc4Ar5iQ7aF+DuQWHXl4mljN6Bwq6A=", "owner": "astro", "repo": "microvm.nix", - "rev": "21c36d162ba8127377ce7dee234bdc99b9af0dde", + "rev": "dd89404e1885b8d7033106f3898eaef8db660cb2", "type": "github" }, "original": { @@ -65,7 +65,9 @@ "nix-cache-cut": { "inputs": { "naersk": "naersk", - "nixpkgs": "nixpkgs", + "nixpkgs": [ + "nixpkgs" + ], "utils": "utils" }, "locked": { @@ -84,21 +86,21 @@ }, "nixpkgs": { "locked": { - "lastModified": 1686089707, - "narHash": "sha256-LTNlJcru2qJ0XhlhG9Acp5KyjB774Pza3tRH0pKIb3o=", + "lastModified": 1731676054, + "narHash": "sha256-OZiZ3m8SCMfh3B6bfGC/Bm4x3qc1m2SVEAlkV6iY7Yg=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "af21c31b2a1ec5d361ed8050edd0303c31306397", + "rev": "5e4fbfb6b3de1aa2872b76d49fafc942626e2add", "type": "github" }, "original": { "owner": "NixOS", - "ref": "nixpkgs-unstable", + "ref": "nixos-unstable", "repo": "nixpkgs", "type": "github" } }, - "nixpkgs_2": { + "nomad-nixpkgs": { "locked": { "lastModified": 1727331237, "narHash": "sha256-nkhXMPuxbqrgdOrT5Ggy0iTNOkL7g5o2NN2KUDRnjck=", @@ -118,17 +120,18 @@ "inputs": { "microvm": "microvm", "nix-cache-cut": "nix-cache-cut", - "nixpkgs": "nixpkgs_2" + "nixpkgs": "nixpkgs", + "nomad-nixpkgs": "nomad-nixpkgs" } }, "spectrum": { "flake": false, "locked": { - "lastModified": 1720264467, - "narHash": "sha256-xzM92n3Q9L90faJIJrkrTtTx+JqCGRHMkHWztkV4PuY=", + "lastModified": 1729945407, + "narHash": "sha256-iGNMamNOAnVTETnIVqDWd6fl74J8fLEi1ejdZiNjEtY=", "ref": "refs/heads/main", - "rev": "fb59d42542049f586c84b0f8bb86ff3be338e9d3", - "revCount": 674, + "rev": "f1d94ee7029af18637dbd5fdf4749621533693fa", + "revCount": 764, "type": "git", "url": "https://spectrum-os.org/git/spectrum" }, diff --git a/flake.nix b/flake.nix index 130a6cb..ace4a5c 100644 --- a/flake.nix +++ b/flake.nix @@ -3,17 +3,22 @@ inputs = { nixpkgs = { + url = "github:NixOS/nixpkgs/nixos-unstable"; + }; + nomad-nixpkgs = { url = "github:NixOS/nixpkgs/1457235a9eee6e05916cd543d3143360e6fd1080"; # Last version of NixOS unstable that supports a foss version of nomad. }; microvm = { url = "github:astro/microvm.nix"; inputs.nixpkgs.follows = "nixpkgs"; }; - - nix-cache-cut.url = "github:astro/nix-cache-cut"; + nix-cache-cut = { + url = "github:astro/nix-cache-cut"; + inputs.nixpkgs.follows = "nixpkgs"; + }; }; - outputs = { self, nixpkgs, microvm, nix-cache-cut }: + outputs = { self, nixpkgs, nomad-nixpkgs, microvm, nix-cache-cut }: let system = "x86_64-linux"; @@ -55,10 +60,11 @@ nixpkgs.lib.nixosSystem { inherit system; modules = [ + {nixpkgs.overlays = [ (final: prev: { nomadPin = nomad-nixpkgs.legacyPackages.${prev.system}; }) ];} microvm.nixosModules.microvm self.nixosModules.default (import ./example-server.nix { inherit instance; }) - ]; + ]; }; in { diff --git a/nixos-modules/nomad.nix b/nixos-modules/nomad.nix index 9962f57..e2aa7e5 100644 --- a/nixos-modules/nomad.nix +++ b/nixos-modules/nomad.nix @@ -41,7 +41,7 @@ in config = { services.nomad = { enable = true; - package = pkgs.nomad_1_6; # nomad 1.6 is the newest version under an foss license. + package = pkgs.nomadPin.nomad_1_6; # nomad 1.6 is the newest version under an foss license. dropPrivileges = false; enableDocker = false; From e2946a5c9aaf91a67b47414a9497277017463ddd Mon Sep 17 00:00:00 2001 From: Makuru Date: Sun, 17 Nov 2024 04:48:18 +0100 Subject: [PATCH 23/41] Force seaweedfs to mount over an existing files, if needed. --- nixos-modules/storage/seaweedfs/server.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nixos-modules/storage/seaweedfs/server.nix b/nixos-modules/storage/seaweedfs/server.nix index fd17dfe..b132d4b 100644 --- a/nixos-modules/storage/seaweedfs/server.nix +++ b/nixos-modules/storage/seaweedfs/server.nix @@ -113,7 +113,7 @@ Restart = "always"; RestartSec = "5s"; #TODO FIX hardcoding of port https://github.com/seaweedfs/seaweedfs/issues/877 - ExecStart = ''${pkgs.seaweedfs}/bin/weed mount -filer=localhost:8888 -volumeServerAccess=filerProxy ${lib.optionalString (!builtins.isNull value.replication) "-replication=${builtins.toString value.replication}"} -cacheCapacityMB=${builtins.toString value.cacheCapacity} -chunkSizeLimitMB=${builtins.toString value.chunkSizeLimit} -dirAutoCreate -dir=${value.mountPoint} -filer.path=${value.mountPoint}''; + ExecStart = ''${pkgs.seaweedfs}/bin/weed mount -nonempty -filer=localhost:8888 ${lib.optionalString (!builtins.isNull value.replication) "-replication=${builtins.toString value.replication}"} -cacheCapacityMB=${builtins.toString value.cacheCapacity} -chunkSizeLimitMB=${builtins.toString value.chunkSizeLimit} -dirAutoCreate -dir=${value.mountPoint} -filer.path=${value.mountPoint}''; # TODO FIX mount with root!!! User = "root"; LimitNOFILE = 40000; From a29a0fb9610cb5c0e39019240c1926ac3a6c5cb4 Mon Sep 17 00:00:00 2001 From: Makuru Date: Sun, 17 Nov 2024 16:46:30 +0100 Subject: [PATCH 24/41] Made `seaweedfs.mounts` behave more like NixOS `fileSystems` option. --- nixos-modules/ssh-deploy.nix | 8 ++++---- nixos-modules/storage/seaweedfs/options.nix | 5 +++-- nixos-modules/storage/seaweedfs/server.nix | 2 +- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/nixos-modules/ssh-deploy.nix b/nixos-modules/ssh-deploy.nix index 99daa44..09adcbd 100644 --- a/nixos-modules/ssh-deploy.nix +++ b/nixos-modules/ssh-deploy.nix @@ -156,7 +156,7 @@ in { binaryCachePath = mkOption { type = types.str; - default = mounts.skyflake-binary-cache.mountPoint; + default = "/var/lib/skyflake/binary-cache"; description = '' Directory which is mounted on all nodes that will be used to share the /nix/store with MicroVMs. @@ -165,7 +165,7 @@ in { sharedGcrootsPath = mkOption { type = types.str; - default = mounts.skyflake-gcroots.mountPoint; + default = "/nix/var/nix/gcroots/skyflake"; description = '' Directory which is mounted on all nodes, is linked from /nix/var/nix/gcroots/, and contains links to all currently @@ -212,8 +212,8 @@ in { config = { skyflake.storage.seaweedfs.mounts = { - skyflake-binary-cache.mountPoint = "/var/lib/skyflake/binary-cache"; - skyflake-gcroots.mountPoint = "/nix/var/nix/gcroots/skyflake"; + ${config.skyflake.deploy.binaryCachePath}.mountSource = "/skyflake-internals${config.skyflake.deploy.binaryCachePath}"; + ${config.skyflake.deploy.sharedGcrootsPath}.mountSource = "/skyflake-internals${config.skyflake.deploy.sharedGcrootsPath}"; }; services.openssh.enable = true; diff --git a/nixos-modules/storage/seaweedfs/options.nix b/nixos-modules/storage/seaweedfs/options.nix index 372f1a8..961047c 100644 --- a/nixos-modules/storage/seaweedfs/options.nix +++ b/nixos-modules/storage/seaweedfs/options.nix @@ -50,11 +50,12 @@ mounts = lib.mkOption { type = lib.types.attrsOf (lib.types.submodule { options = { - mountPoint = lib.mkOption { + mountSource = lib.mkOption { type = lib.types.str; default = null; + example = "/filesystems/1a32bfd9-0cbc-430a-a28a-d9fd862e9ebc"; description = '' - Place where the filesystem is Mounted. + Place where the filesystem is saved in seaweedfs. ''; }; replication = lib.mkOption { diff --git a/nixos-modules/storage/seaweedfs/server.nix b/nixos-modules/storage/seaweedfs/server.nix index b132d4b..d82fd5d 100644 --- a/nixos-modules/storage/seaweedfs/server.nix +++ b/nixos-modules/storage/seaweedfs/server.nix @@ -113,7 +113,7 @@ Restart = "always"; RestartSec = "5s"; #TODO FIX hardcoding of port https://github.com/seaweedfs/seaweedfs/issues/877 - ExecStart = ''${pkgs.seaweedfs}/bin/weed mount -nonempty -filer=localhost:8888 ${lib.optionalString (!builtins.isNull value.replication) "-replication=${builtins.toString value.replication}"} -cacheCapacityMB=${builtins.toString value.cacheCapacity} -chunkSizeLimitMB=${builtins.toString value.chunkSizeLimit} -dirAutoCreate -dir=${value.mountPoint} -filer.path=${value.mountPoint}''; + ExecStart = ''${pkgs.seaweedfs}/bin/weed mount -nonempty -filer=localhost:8888 ${lib.optionalString (!builtins.isNull value.replication) "-replication=${builtins.toString value.replication}"} -cacheCapacityMB=${builtins.toString value.cacheCapacity} -chunkSizeLimitMB=${builtins.toString value.chunkSizeLimit} -dirAutoCreate -dir=${builtins.toString name } -filer.path=${value.mountSource}''; # TODO FIX mount with root!!! User = "root"; LimitNOFILE = 40000; From 0c0ed5dea51bc2fdb7173da08fc5a7a5edec7fd6 Mon Sep 17 00:00:00 2001 From: Makuru Date: Sun, 17 Nov 2024 21:36:19 +0100 Subject: [PATCH 25/41] Fixed weird `failed to create symbolic link` error. --- nixos-modules/storage/seaweedfs/server.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nixos-modules/storage/seaweedfs/server.nix b/nixos-modules/storage/seaweedfs/server.nix index d82fd5d..b9ce211 100644 --- a/nixos-modules/storage/seaweedfs/server.nix +++ b/nixos-modules/storage/seaweedfs/server.nix @@ -97,7 +97,7 @@ }; } # config for the mount deamon of seaweedfs - (lib.mapAttrs' (name: value: lib.nameValuePair ("seaweedfs-mount-" + name) { + (lib.mapAttrs' (name: value: lib.nameValuePair ("seaweedfs-mount" + (lib.replaceStrings [ "/" ] [ "-" ] name)) { description = "seaweedfs mount service"; wantedBy = [ "multi-user.target" ]; after = [ "network-online.target" "etcd.service" "seaweedfs-filer.service" ]; From 176764ecbb902ed3eeaf4d5875f4a98e35d7d419 Mon Sep 17 00:00:00 2001 From: Makuru Date: Tue, 26 Nov 2024 15:34:19 +0100 Subject: [PATCH 26/41] renamed `ssh-deploy.nix` and removed anything that is not a option. --- flake.nix | 4 +- nixos-modules/ssh-deploy.nix | 277 ---------------------------- nixos-modules/ssh-deployOptions.nix | 68 +++++++ 3 files changed, 69 insertions(+), 280 deletions(-) delete mode 100644 nixos-modules/ssh-deploy.nix create mode 100644 nixos-modules/ssh-deployOptions.nix diff --git a/flake.nix b/flake.nix index ace4a5c..9685527 100644 --- a/flake.nix +++ b/flake.nix @@ -41,9 +41,7 @@ ./nixos-modules/nodes.nix ./nixos-modules/nomad.nix ./nixos-modules/users.nix - (import ./nixos-modules/ssh-deploy.nix { - inherit microvm nixpkgs; - }) + ./nixos-modules/ssh-deployOptions.nix { nixpkgs.overlays = [ nix-cache-cut.overlays.default diff --git a/nixos-modules/ssh-deploy.nix b/nixos-modules/ssh-deploy.nix deleted file mode 100644 index 09adcbd..0000000 --- a/nixos-modules/ssh-deploy.nix +++ /dev/null @@ -1,277 +0,0 @@ -{ microvm, nixpkgs }: - -{ config, lib, pkgs, ... }: - -let - inherit (config.skyflake.storage.seaweedfs) mounts; - - debugShell = lib.optionalString config.skyflake.debug '' - set -x - ''; - - deployCommand = with pkgs; writeScript "skyflake-ssh-deploy" '' - #! ${runtimeShell} -e - ${debugShell} - - PATH=${lib.makeBinPath ([ - git - ])}:$PATH - - if [[ "$SSH_ORIGINAL_COMMAND" =~ ^git-receive-pack\ \'([\\-_a-zA-Z0-9]+)\'$ ]]; then - REPO="''${BASH_REMATCH[1]}" - if ! [ -e $REPO ]; then - echo "Creating $REPO anew..." >&2 - mkdir $REPO - cd $REPO - ${git}/bin/git init --bare -b main >/dev/null - else - echo "Updating existing $REPO" >&2 - cd $REPO - fi - - SYSTEMS=$(mktemp --tmpdir -d deploy-systems-XXXXXXXX) - - cat > hooks/update <&2 - nomad namespace apply "$USER-$REPO" >/dev/null - - for NAME in * ; do - SYSTEM=$(readlink $NAME) - echo $SYSTEM >&2 - nomad run -detach "$SYSTEM" >&2 - - # Register gcroot - mkdir -p "${cfg.sharedGcrootsPath}/$USER/$REPO" - rm -f "${cfg.sharedGcrootsPath}/$USER/$REPO/$NAME" - ln -s "$SYSTEM" "${cfg.sharedGcrootsPath}/$USER/$REPO/$NAME" - done - cd - - rm -r $SYSTEMS - echo All done >&2 - - elif [[ "$SSH_ORIGINAL_COMMAND" =~ ^git-upload-pack\ \'([\\-_a-zA-Z0-9]+)\'$ ]]; then - REPO="''${BASH_REMATCH[1]}" - exec git-upload-pack "$REPO" - - elif [[ "$SSH_ORIGINAL_COMMAND" = status ]]; then - NAMESPACES=$(nomad namespace list -t "{{ range . }}{{ .Name }} - {{ end }}"|grep -e "^$USER-") - for NAMESPACE in $NAMESPACES ; do - nomad job status -namespace "$NAMESPACE" - done - - else - echo "Invalid SSH command: $SSH_ORIGINAL_COMMAND" >&2 - exit 1 - fi - ''; - - sshKeyOpts = [ - "command=\"${deployCommand}\"" - "no-port-forwarding" - "no-X11-forwarding" - "no-agent-forwarding" - "no-pty" - "no-user-rc" - "restrict" - ]; - - cfg = config.skyflake.deploy; -in { - options.skyflake = with lib; { - deploy = { - datacenters = mkOption { - type = with types; listOf str; - default = [ config.skyflake.nomad.datacenter ]; - description = '' - List of datacenters to deploy to. - ''; - }; - - binaryCachePath = mkOption { - type = types.str; - default = "/var/lib/skyflake/binary-cache"; - description = '' - Directory which is mounted on all nodes that will be used to - share the /nix/store with MicroVMs. - ''; - }; - - sharedGcrootsPath = mkOption { - type = types.str; - default = "/nix/var/nix/gcroots/skyflake"; - description = '' - Directory which is mounted on all nodes, is linked from - /nix/var/nix/gcroots/, and contains links to all currently - required microvms. - ''; - }; - - customizationModule = mkOption { - type = types.path; - default = ../default-customization.nix; - description = '' - NixOS module to add when extending a guest NixOS configuration - with MicroVM settings. - ''; - }; - }; - - gc = { - cron = mkOption { - type = types.str; - default = "@hourly"; - description = lib.mdDoc '' - See `cron` in https://developer.hashicorp.com/nomad/docs/job-specification/periodic#periodic-parameters - ''; - }; - }; - - microvmUid = mkOption { - type = types.int; - default = 999; - description = '' - A fixed UID for MicroVM files makes sense for the whole cluster. - ''; - }; - - debug = mkOption { - type = types.bool; - default = false; - description = '' - Enable debug output. Do not use in production! - ''; - }; - }; - - config = { - skyflake.storage.seaweedfs.mounts = { - ${config.skyflake.deploy.binaryCachePath}.mountSource = "/skyflake-internals${config.skyflake.deploy.binaryCachePath}"; - ${config.skyflake.deploy.sharedGcrootsPath}.mountSource = "/skyflake-internals${config.skyflake.deploy.sharedGcrootsPath}"; - }; - - services.openssh.enable = true; - - users.users = builtins.mapAttrs (_: userConfig: { - openssh.authorizedKeys.keys = map (sshKey: - "${lib.concatStringsSep "," sshKeyOpts} ${sshKey}" - ) userConfig.sshKeys; - }) config.skyflake.users // { - # stable uid is useful across network filesystems - microvm.uid = config.skyflake.microvmUid; - }; - - environment.etc."skyflake/vm".source = pkgs.substituteAllFiles { - src = ../vm; - files = [ "." ]; - inherit (config.skyflake.deploy) binaryCachePath customizationModule; - }; - - # lets the hook use $binaryCachePath - nix.settings.trusted-users = builtins.attrNames config.skyflake.users; - - # allowing commands to copy to/from shared store - security.sudo = { - enable = true; - extraRules = [ { - groups = [ "users" ]; - commands = [ { - command = ''/run/current-system/sw/bin/nix copy --to file\://${cfg.binaryCachePath} *''; - options = [ "NOPASSWD" ]; - } ]; - } ]; - }; - - systemd.tmpfiles.rules = - [ - # workDir for nomad jobs - "d /run/microvms 0700 microvm kvm - -" - "d ${cfg.binaryCachePath} 0777 root root - -" - ] - ++ - map (userName: - "d ${config.skyflake.deploy.sharedGcrootsPath}/${userName} 0750 ${userName} root - -" - ) (builtins.attrNames config.skyflake.users); - - systemd.services.skyflake-permissions = { - wantedBy = [ "multi-user.target" ]; - after = [ "remote-fs.target" ]; - script = '' - mkdir -p ${cfg.binaryCachePath} - chmod 0777 ${cfg.binaryCachePath} - - ${lib.concatMapStrings (userName: '' - D="${config.skyflake.deploy.sharedGcrootsPath}/${userName}" - mkdir -p "$D" - chown "${userName}" "$D" - '') (builtins.attrNames config.skyflake.users)} - ''; - }; - }; -} diff --git a/nixos-modules/ssh-deployOptions.nix b/nixos-modules/ssh-deployOptions.nix new file mode 100644 index 0000000..482e955 --- /dev/null +++ b/nixos-modules/ssh-deployOptions.nix @@ -0,0 +1,68 @@ +{ config, lib, ... }: +{ + options.skyflake = with lib; { + deploy = { + datacenters = mkOption { + type = with types; listOf str; + default = [ config.skyflake.nomad.datacenter ]; + description = '' + List of datacenters to deploy to. + ''; + }; + + binaryCachePath = mkOption { + type = types.str; + default = "/var/lib/skyflake/binary-cache"; + description = '' + Directory which is mounted on all nodes that will be used to + share the /nix/store with MicroVMs. + ''; + }; + + sharedGcrootsPath = mkOption { + type = types.str; + default = "/nix/var/nix/gcroots/skyflake"; + description = '' + Directory which is mounted on all nodes, is linked from + /nix/var/nix/gcroots/, and contains links to all currently + required microvms. + ''; + }; + + customizationModule = mkOption { + type = types.path; + default = ../default-customization.nix; + description = '' + NixOS module to add when extending a guest NixOS configuration + with MicroVM settings. + ''; + }; + }; + + gc = { + cron = mkOption { + type = types.str; + default = "@hourly"; + description = lib.mdDoc '' + See `cron` in https://developer.hashicorp.com/nomad/docs/job-specification/periodic#periodic-parameters + ''; + }; + }; + + microvmUid = mkOption { + type = types.int; + default = 999; + description = '' + A fixed UID for MicroVM files makes sense for the whole cluster. + ''; + }; + + debug = mkOption { + type = types.bool; + default = false; + description = '' + Enable debug output. Do not use in production! + ''; + }; + }; +} \ No newline at end of file From bd70c1a9205b942ff5da45dbaf1cc8c7e537ff0b Mon Sep 17 00:00:00 2001 From: Makuru Date: Tue, 26 Nov 2024 15:36:38 +0100 Subject: [PATCH 27/41] Moved nixos config part of `ssh-deploy.nix` to the corresponding storage folder. --- flake.nix | 6 + nixos-modules/storage/ceph/ssh-deploy.nix | 211 ++++++++++++++++++ .../storage/seaweedfs/ssh-deploy.nix | 211 ++++++++++++++++++ 3 files changed, 428 insertions(+) create mode 100644 nixos-modules/storage/ceph/ssh-deploy.nix create mode 100644 nixos-modules/storage/seaweedfs/ssh-deploy.nix diff --git a/flake.nix b/flake.nix index 9685527..3c6c472 100644 --- a/flake.nix +++ b/flake.nix @@ -42,6 +42,12 @@ ./nixos-modules/nomad.nix ./nixos-modules/users.nix ./nixos-modules/ssh-deployOptions.nix + (import ./nixos-modules/seaweedf/ssh-deploy.nix { + inherit microvm nixpkgs; + }) + (import ./nixos-modules/ceph/ssh-deploy.nix { + inherit microvm nixpkgs; + }) { nixpkgs.overlays = [ nix-cache-cut.overlays.default diff --git a/nixos-modules/storage/ceph/ssh-deploy.nix b/nixos-modules/storage/ceph/ssh-deploy.nix new file mode 100644 index 0000000..8f369ba --- /dev/null +++ b/nixos-modules/storage/ceph/ssh-deploy.nix @@ -0,0 +1,211 @@ +{ microvm, nixpkgs }: + +{ config, lib, pkgs, ... }: + +let + inherit (config.skyflake.storage.ceph) cephfs; + + debugShell = lib.optionalString config.skyflake.debug '' + set -x + ''; + + deployCommand = with pkgs; writeScript "skyflake-ssh-deploy" '' + #! ${runtimeShell} -e + ${debugShell} + + PATH=${lib.makeBinPath ([ + git + ])}:$PATH + + if [[ "$SSH_ORIGINAL_COMMAND" =~ ^git-receive-pack\ \'([\\-_a-zA-Z0-9]+)\'$ ]]; then + REPO="''${BASH_REMATCH[1]}" + if ! [ -e $REPO ]; then + echo "Creating $REPO anew..." >&2 + mkdir $REPO + cd $REPO + ${git}/bin/git init --bare -b main >/dev/null + else + echo "Updating existing $REPO" >&2 + cd $REPO + fi + + SYSTEMS=$(mktemp --tmpdir -d deploy-systems-XXXXXXXX) + + cat > hooks/update <&2 + nomad namespace apply "$USER-$REPO" >/dev/null + + for NAME in * ; do + SYSTEM=$(readlink $NAME) + echo $SYSTEM >&2 + nomad run -detach "$SYSTEM" >&2 + + # Register gcroot + mkdir -p "${cfg.sharedGcrootsPath}/$USER/$REPO" + rm -f "${cfg.sharedGcrootsPath}/$USER/$REPO/$NAME" + ln -s "$SYSTEM" "${cfg.sharedGcrootsPath}/$USER/$REPO/$NAME" + done + cd - + rm -r $SYSTEMS + echo All done >&2 + + elif [[ "$SSH_ORIGINAL_COMMAND" =~ ^git-upload-pack\ \'([\\-_a-zA-Z0-9]+)\'$ ]]; then + REPO="''${BASH_REMATCH[1]}" + exec git-upload-pack "$REPO" + + elif [[ "$SSH_ORIGINAL_COMMAND" = status ]]; then + NAMESPACES=$(nomad namespace list -t "{{ range . }}{{ .Name }} + {{ end }}"|grep -e "^$USER-") + for NAMESPACE in $NAMESPACES ; do + nomad job status -namespace "$NAMESPACE" + done + + else + echo "Invalid SSH command: $SSH_ORIGINAL_COMMAND" >&2 + exit 1 + fi + ''; + + sshKeyOpts = [ + "command=\"${deployCommand}\"" + "no-port-forwarding" + "no-X11-forwarding" + "no-agent-forwarding" + "no-pty" + "no-user-rc" + "restrict" + ]; + cfg = config.skyflake.deploy; + +in { + config = { + skyflake.storage.ceph.cephfs = lib.mkIf config.skyflake.storage.ceph.enable { + ${config.skyflake.deploy.binaryCachePath}.mountSource = "/skyflake-internals${config.skyflake.deploy.binaryCachePath}"; + ${config.skyflake.deploy.sharedGcrootsPath}.mountSource = "/skyflake-internals${config.skyflake.deploy.sharedGcrootsPath}"; + }; + + services.openssh.enable = true; + + users.users = builtins.mapAttrs (_: userConfig: { + openssh.authorizedKeys.keys = map (sshKey: + "${lib.concatStringsSep "," sshKeyOpts} ${sshKey}" + ) userConfig.sshKeys; + }) config.skyflake.users // { + # stable uid is useful across network filesystems + microvm.uid = config.skyflake.microvmUid; + }; + + environment.etc."skyflake/vm".source = pkgs.substituteAllFiles { + src = ../vm; + files = [ "." ]; + inherit (config.skyflake.deploy) binaryCachePath customizationModule; + }; + + # lets the hook use $binaryCachePath + nix.settings.trusted-users = builtins.attrNames config.skyflake.users; + + # allowing commands to copy to/from shared store + security.sudo = { + enable = true; + extraRules = [ { + groups = [ "users" ]; + commands = [ { + command = ''/run/current-system/sw/bin/nix copy --to file\://${cfg.binaryCachePath} *''; + options = [ "NOPASSWD" ]; + } ]; + } ]; + }; + + systemd.tmpfiles.rules = + [ + # workDir for nomad jobs + "d /run/microvms 0700 microvm kvm - -" + "d ${cfg.binaryCachePath} 0777 root root - -" + ] + ++ + map (userName: + "d ${config.skyflake.deploy.sharedGcrootsPath}/${userName} 0750 ${userName} root - -" + ) (builtins.attrNames config.skyflake.users); + + systemd.services.skyflake-permissions = { + wantedBy = [ "multi-user.target" ]; + after = [ "remote-fs.target" ]; + script = '' + mkdir -p ${cfg.binaryCachePath} + chmod 0777 ${cfg.binaryCachePath} + + ${lib.concatMapStrings (userName: '' + D="${config.skyflake.deploy.sharedGcrootsPath}/${userName}" + mkdir -p "$D" + chown "${userName}" "$D" + '') (builtins.attrNames config.skyflake.users)} + ''; + }; + }; +} diff --git a/nixos-modules/storage/seaweedfs/ssh-deploy.nix b/nixos-modules/storage/seaweedfs/ssh-deploy.nix new file mode 100644 index 0000000..456135f --- /dev/null +++ b/nixos-modules/storage/seaweedfs/ssh-deploy.nix @@ -0,0 +1,211 @@ +{ microvm, nixpkgs }: + +{ config, lib, pkgs, ... }: + +let + inherit (config.skyflake.storage.seaweedfs) mounts; + + debugShell = lib.optionalString config.skyflake.debug '' + set -x + ''; + + deployCommand = with pkgs; writeScript "skyflake-ssh-deploy" '' + #! ${runtimeShell} -e + ${debugShell} + + PATH=${lib.makeBinPath ([ + git + ])}:$PATH + + if [[ "$SSH_ORIGINAL_COMMAND" =~ ^git-receive-pack\ \'([\\-_a-zA-Z0-9]+)\'$ ]]; then + REPO="''${BASH_REMATCH[1]}" + if ! [ -e $REPO ]; then + echo "Creating $REPO anew..." >&2 + mkdir $REPO + cd $REPO + ${git}/bin/git init --bare -b main >/dev/null + else + echo "Updating existing $REPO" >&2 + cd $REPO + fi + + SYSTEMS=$(mktemp --tmpdir -d deploy-systems-XXXXXXXX) + + cat > hooks/update <&2 + nomad namespace apply "$USER-$REPO" >/dev/null + + for NAME in * ; do + SYSTEM=$(readlink $NAME) + echo $SYSTEM >&2 + nomad run -detach "$SYSTEM" >&2 + + # Register gcroot + mkdir -p "${cfg.sharedGcrootsPath}/$USER/$REPO" + rm -f "${cfg.sharedGcrootsPath}/$USER/$REPO/$NAME" + ln -s "$SYSTEM" "${cfg.sharedGcrootsPath}/$USER/$REPO/$NAME" + done + cd - + rm -r $SYSTEMS + echo All done >&2 + + elif [[ "$SSH_ORIGINAL_COMMAND" =~ ^git-upload-pack\ \'([\\-_a-zA-Z0-9]+)\'$ ]]; then + REPO="''${BASH_REMATCH[1]}" + exec git-upload-pack "$REPO" + + elif [[ "$SSH_ORIGINAL_COMMAND" = status ]]; then + NAMESPACES=$(nomad namespace list -t "{{ range . }}{{ .Name }} + {{ end }}"|grep -e "^$USER-") + for NAMESPACE in $NAMESPACES ; do + nomad job status -namespace "$NAMESPACE" + done + + else + echo "Invalid SSH command: $SSH_ORIGINAL_COMMAND" >&2 + exit 1 + fi + ''; + + sshKeyOpts = [ + "command=\"${deployCommand}\"" + "no-port-forwarding" + "no-X11-forwarding" + "no-agent-forwarding" + "no-pty" + "no-user-rc" + "restrict" + ]; + cfg = config.skyflake.deploy; + +in { + config = { + skyflake.storage.seaweedfs.mounts = lib.mkIf config.skyflake.storage.seaweedfs.enable { + skyflake-binary-cache.mountPoint = "/var/lib/skyflake/binary-cache"; + skyflake-gcroots.mountPoint = "/nix/var/nix/gcroots/skyflake"; + }; + + services.openssh.enable = true; + + users.users = builtins.mapAttrs (_: userConfig: { + openssh.authorizedKeys.keys = map (sshKey: + "${lib.concatStringsSep "," sshKeyOpts} ${sshKey}" + ) userConfig.sshKeys; + }) config.skyflake.users // { + # stable uid is useful across network filesystems + microvm.uid = config.skyflake.microvmUid; + }; + + environment.etc."skyflake/vm".source = pkgs.substituteAllFiles { + src = ../vm; + files = [ "." ]; + inherit (config.skyflake.deploy) binaryCachePath customizationModule; + }; + + # lets the hook use $binaryCachePath + nix.settings.trusted-users = builtins.attrNames config.skyflake.users; + + # allowing commands to copy to/from shared store + security.sudo = { + enable = true; + extraRules = [ { + groups = [ "users" ]; + commands = [ { + command = ''/run/current-system/sw/bin/nix copy --to file\://${cfg.binaryCachePath} *''; + options = [ "NOPASSWD" ]; + } ]; + } ]; + }; + + systemd.tmpfiles.rules = + [ + # workDir for nomad jobs + "d /run/microvms 0700 microvm kvm - -" + "d ${cfg.binaryCachePath} 0777 root root - -" + ] + ++ + map (userName: + "d ${config.skyflake.deploy.sharedGcrootsPath}/${userName} 0750 ${userName} root - -" + ) (builtins.attrNames config.skyflake.users); + + systemd.services.skyflake-permissions = { + wantedBy = [ "multi-user.target" ]; + after = [ "remote-fs.target" ]; + script = '' + mkdir -p ${cfg.binaryCachePath} + chmod 0777 ${cfg.binaryCachePath} + + ${lib.concatMapStrings (userName: '' + D="${config.skyflake.deploy.sharedGcrootsPath}/${userName}" + mkdir -p "$D" + chown "${userName}" "$D" + '') (builtins.attrNames config.skyflake.users)} + ''; + }; + }; +} From 9b9e2d73186cc9c94fb55938152255d2c5770ea7 Mon Sep 17 00:00:00 2001 From: Makuru Date: Tue, 26 Nov 2024 16:12:40 +0100 Subject: [PATCH 28/41] Fixed miscellaneous errors. --- flake.nix | 16 ++++++++++------ nixos-modules/storage/ceph/ssh-deploy.nix | 6 +++--- nixos-modules/storage/seaweedfs/ssh-deploy.nix | 6 +++--- 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/flake.nix b/flake.nix index 3c6c472..2681da6 100644 --- a/flake.nix +++ b/flake.nix @@ -42,12 +42,16 @@ ./nixos-modules/nomad.nix ./nixos-modules/users.nix ./nixos-modules/ssh-deployOptions.nix - (import ./nixos-modules/seaweedf/ssh-deploy.nix { - inherit microvm nixpkgs; - }) - (import ./nixos-modules/ceph/ssh-deploy.nix { - inherit microvm nixpkgs; - }) + (import + ./nixos-modules/storage/seaweedfs/ssh-deploy.nix { + inherit microvm nixpkgs; + } + ) + (import + ./nixos-modules/storage/ceph/ssh-deploy.nix { + inherit microvm nixpkgs; + } + ) { nixpkgs.overlays = [ nix-cache-cut.overlays.default diff --git a/nixos-modules/storage/ceph/ssh-deploy.nix b/nixos-modules/storage/ceph/ssh-deploy.nix index 8f369ba..426763a 100644 --- a/nixos-modules/storage/ceph/ssh-deploy.nix +++ b/nixos-modules/storage/ceph/ssh-deploy.nix @@ -146,8 +146,8 @@ let in { config = { skyflake.storage.ceph.cephfs = lib.mkIf config.skyflake.storage.ceph.enable { - ${config.skyflake.deploy.binaryCachePath}.mountSource = "/skyflake-internals${config.skyflake.deploy.binaryCachePath}"; - ${config.skyflake.deploy.sharedGcrootsPath}.mountSource = "/skyflake-internals${config.skyflake.deploy.sharedGcrootsPath}"; + skyflake-binary-cache.mountPoint = "/var/lib/skyflake/binary-cache"; + skyflake-gcroots.mountPoint = "/nix/var/nix/gcroots/skyflake"; }; services.openssh.enable = true; @@ -162,7 +162,7 @@ in { }; environment.etc."skyflake/vm".source = pkgs.substituteAllFiles { - src = ../vm; + src = ../../../vm; files = [ "." ]; inherit (config.skyflake.deploy) binaryCachePath customizationModule; }; diff --git a/nixos-modules/storage/seaweedfs/ssh-deploy.nix b/nixos-modules/storage/seaweedfs/ssh-deploy.nix index 456135f..aad9f10 100644 --- a/nixos-modules/storage/seaweedfs/ssh-deploy.nix +++ b/nixos-modules/storage/seaweedfs/ssh-deploy.nix @@ -146,8 +146,8 @@ let in { config = { skyflake.storage.seaweedfs.mounts = lib.mkIf config.skyflake.storage.seaweedfs.enable { - skyflake-binary-cache.mountPoint = "/var/lib/skyflake/binary-cache"; - skyflake-gcroots.mountPoint = "/nix/var/nix/gcroots/skyflake"; + ${config.skyflake.deploy.binaryCachePath}.mountSource = "/skyflake-internals${config.skyflake.deploy.binaryCachePath}"; + ${config.skyflake.deploy.sharedGcrootsPath}.mountSource = "/skyflake-internals${config.skyflake.deploy.sharedGcrootsPath}"; }; services.openssh.enable = true; @@ -162,7 +162,7 @@ in { }; environment.etc."skyflake/vm".source = pkgs.substituteAllFiles { - src = ../vm; + src = ../../../vm; files = [ "." ]; inherit (config.skyflake.deploy) binaryCachePath customizationModule; }; From e0f7ff9f2176eb78fff488e7b3ee59ee7a948843 Mon Sep 17 00:00:00 2001 From: Makuru Date: Tue, 26 Nov 2024 19:06:33 +0100 Subject: [PATCH 29/41] Updated doc. --- doc/host/intro.md | 2 +- doc/user/flake.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/host/intro.md b/doc/host/intro.md index 92c406a..9f9b9b5 100644 --- a/doc/host/intro.md +++ b/doc/host/intro.md @@ -34,7 +34,7 @@ always be able to access its storage, regardless of the server they are started on. We solve this problem by moving the VM filesystems to a network -filesystem: Ceph. +filesystem: Ceph and also seaweedfs. ### Network Setup diff --git a/doc/user/flake.md b/doc/user/flake.md index f0bc4f3..37d7a84 100644 --- a/doc/user/flake.md +++ b/doc/user/flake.md @@ -22,7 +22,7 @@ A sample `flake.nix`: nixosConfigurations = { my-microvm = nixpkgs.lib.nixosSystem { modules = [ { - system.stateVersion = "22.11"; + system.stateVersion = "24.11"; networking.hostName = "my-microvm"; services.openssh = { enable = true; From ae3f28350b62e5c9500dc6e7faf561a68f19f2fb Mon Sep 17 00:00:00 2001 From: Makuru Date: Tue, 26 Nov 2024 19:31:15 +0100 Subject: [PATCH 30/41] Fixed the new IP formad, for `nomad`. --- nixos-modules/nomad.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nixos-modules/nomad.nix b/nixos-modules/nomad.nix index e2aa7e5..b4ceeaf 100644 --- a/nixos-modules/nomad.nix +++ b/nixos-modules/nomad.nix @@ -61,7 +61,7 @@ in advertise = let address = config.skyflake.nodes.${config.networking.hostName}.address; in { - serf = "[${address}]:4648"; + serf = "${address}:4648"; }; client = { enabled = cfg.client.enable; From 1ae848e9c1446e7f5836e47b59240a92b82277c2 Mon Sep 17 00:00:00 2001 From: Makuru Date: Tue, 26 Nov 2024 22:23:02 +0100 Subject: [PATCH 31/41] Fixed bug, in my `example-server.nix` config file. --- example-server.nix | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/example-server.nix b/example-server.nix index 78d2539..51a5c0f 100644 --- a/example-server.nix +++ b/example-server.nix @@ -70,7 +70,7 @@ IPv6AcceptRA = true; }; addresses = [ { - Address = "fec0::${toString instance}/64"; # + Address = "fec0::${toString instance}/64"; } ]; }; }; @@ -88,6 +88,8 @@ storage.seaweedfs = { enable = false; volumeStorage.encrypt = true; + # example mount below. + # mounts."/mnt".mountSource = "/filesystems/1a32bfd9-0cbc-430a-a28a-d9fd862e9ebc"; filer.db.etcd = { enable = true; }; @@ -111,7 +113,7 @@ }; nomad = { - servers = [ (builtins.concatStringsSep " " (lib.attrNames config.skyflake.nodes)) ]; + servers = builtins.attrNames config.skyflake.nodes; client.meta = { example-deployment = "yes"; }; From 03044f8cedcbf07a3795503b38568cf6b0591b6b Mon Sep 17 00:00:00 2001 From: Makuru Date: Tue, 26 Nov 2024 22:24:17 +0100 Subject: [PATCH 32/41] changed weirdness in `nixos-modules/nomad.nix`, by default it should be an uneven number greater 1, typically 3 or 5. --- nixos-modules/nomad.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nixos-modules/nomad.nix b/nixos-modules/nomad.nix index b4ceeaf..561e9b5 100644 --- a/nixos-modules/nomad.nix +++ b/nixos-modules/nomad.nix @@ -55,7 +55,7 @@ in server = { enabled = cfg.server.enable; - bootstrap_expect = (builtins.length cfg.servers + 2) / 2; + bootstrap_expect = builtins.length cfg.servers; # why not this? Why this weird formular? (${NOMAD_SERVERS} + 2) / 2 ? server_join.retry_join = cfg.servers; }; advertise = let From 24ab3a18223f032ef84f4c4ce79c6adc37cac9eb Mon Sep 17 00:00:00 2001 From: Makuru Date: Fri, 29 Nov 2024 20:15:14 +0100 Subject: [PATCH 33/41] Switched VM storage to btrfs. --- default-customization.nix | 2 +- vm/customization-options.nix | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/default-customization.nix b/default-customization.nix index 8c3be28..16e508f 100644 --- a/default-customization.nix +++ b/default-customization.nix @@ -78,5 +78,5 @@ in value = "yes"; } ]; - config.fileSystems."/".fsType = lib.mkForce "ext4"; + config.fileSystems."/".fsType = lib.mkForce "btrfs"; } diff --git a/vm/customization-options.nix b/vm/customization-options.nix index fc93906..9aba435 100644 --- a/vm/customization-options.nix +++ b/vm/customization-options.nix @@ -46,7 +46,7 @@ }; fsType = mkOption { type = str; - default = "ext4"; + default = "btrfs"; description = '' Which mkfs to use when `autoCreate = true` ''; From 8a2e1be7dba96fb132938c083cd82107c802f813 Mon Sep 17 00:00:00 2001 From: Makuru Date: Sat, 30 Nov 2024 16:12:06 +0100 Subject: [PATCH 34/41] Fixed nixosModule subflake. --- flake.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index 2681da6..1414f23 100644 --- a/flake.nix +++ b/flake.nix @@ -33,6 +33,7 @@ nixosModules = { default = { imports = [ + {nixpkgs.overlays = [ (final: prev: { nomadPin = nomad-nixpkgs.legacyPackages.${prev.system}; }) ];} ./nixos-modules/storage/seaweedfs/options.nix ./nixos-modules/storage/seaweedfs/server.nix ./nixos-modules/storage/seaweedfs/dbBackend/etcd.nix @@ -68,7 +69,6 @@ nixpkgs.lib.nixosSystem { inherit system; modules = [ - {nixpkgs.overlays = [ (final: prev: { nomadPin = nomad-nixpkgs.legacyPackages.${prev.system}; }) ];} microvm.nixosModules.microvm self.nixosModules.default (import ./example-server.nix { inherit instance; }) From ccc45b919ffdb2c82147dce71719ab9278ca2f82 Mon Sep 17 00:00:00 2001 From: Makuru Date: Mon, 2 Dec 2024 13:02:34 +0100 Subject: [PATCH 35/41] Refractured how `seaweedfs filer` works. and rotated etcd keys. --- example-server.nix | 3 + example/ca-csr.json | 16 -- example/certs/ca-key.pem | 2 +- example/certs/ca.csr | 13 +- example/certs/ca.pem | 23 +- example/{ => certs}/config/ca-config.json | 4 +- example/certs/config/ca-csr.json | 14 + example/certs/config/default.json | 21 ++ example/{ => certs}/config/req-csr.json | 6 +- .../{example2-key.pem => default-key.pem} | 2 +- example/certs/default.csr | 10 + example/certs/default.pem | 16 ++ example/certs/example1-key.pem | 3 - example/certs/example1.csr | 9 - example/certs/example1.pem | 14 - example/certs/example2.csr | 9 - example/certs/example2.pem | 14 - example/certs/example3-key.pem | 3 - example/certs/example3.csr | 9 - example/certs/example3.pem | 14 - example/certs/peer-example1-key.pem | 3 - example/certs/peer-example1.csr | 9 - example/certs/peer-example1.pem | 14 - example/certs/peer-example2-key.pem | 3 - example/certs/peer-example2.csr | 9 - example/certs/peer-example2.pem | 14 - example/certs/peer-example3-key.pem | 3 - example/certs/peer-example3.csr | 9 - example/certs/peer-example3.pem | 14 - flake.nix | 5 +- nixos-modules/defaults.nix | 1 - .../seaweedfs/db-backend/etcd/default.nix | 115 ++++++++ .../seaweedfs/db-backend/etcd/options.nix | 50 ++++ .../storage/seaweedfs/dbBackend/etcd.nix | 91 ------- nixos-modules/storage/seaweedfs/options.nix | 90 +++--- nixos-modules/storage/seaweedfs/server.nix | 257 +++++++++++------- .../storage/seaweedfs/ssh-deploy.nix | 2 +- 37 files changed, 454 insertions(+), 440 deletions(-) delete mode 100644 example/ca-csr.json rename example/{ => certs}/config/ca-config.json (85%) create mode 100644 example/certs/config/ca-csr.json create mode 100644 example/certs/config/default.json rename example/{ => certs}/config/req-csr.json (75%) rename example/certs/{example2-key.pem => default-key.pem} (52%) create mode 100644 example/certs/default.csr create mode 100644 example/certs/default.pem delete mode 100644 example/certs/example1-key.pem delete mode 100644 example/certs/example1.csr delete mode 100644 example/certs/example1.pem delete mode 100644 example/certs/example2.csr delete mode 100644 example/certs/example2.pem delete mode 100644 example/certs/example3-key.pem delete mode 100644 example/certs/example3.csr delete mode 100644 example/certs/example3.pem delete mode 100644 example/certs/peer-example1-key.pem delete mode 100644 example/certs/peer-example1.csr delete mode 100644 example/certs/peer-example1.pem delete mode 100644 example/certs/peer-example2-key.pem delete mode 100644 example/certs/peer-example2.csr delete mode 100644 example/certs/peer-example2.pem delete mode 100644 example/certs/peer-example3-key.pem delete mode 100644 example/certs/peer-example3.csr delete mode 100644 example/certs/peer-example3.pem create mode 100644 nixos-modules/storage/seaweedfs/db-backend/etcd/default.nix create mode 100644 nixos-modules/storage/seaweedfs/db-backend/etcd/options.nix delete mode 100644 nixos-modules/storage/seaweedfs/dbBackend/etcd.nix diff --git a/example-server.nix b/example-server.nix index 51a5c0f..4efbfb7 100644 --- a/example-server.nix +++ b/example-server.nix @@ -92,6 +92,9 @@ # mounts."/mnt".mountSource = "/filesystems/1a32bfd9-0cbc-430a-a28a-d9fd862e9ebc"; filer.db.etcd = { enable = true; + certFile = example/certs/default.pem; + keyFile = example/certs/default-key.pem; + trustedCaFile = example/certs/ca.pem; }; }; storage.ceph = { diff --git a/example/ca-csr.json b/example/ca-csr.json deleted file mode 100644 index c7aef0f..0000000 --- a/example/ca-csr.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "CN": "Autogenerated CA", - "key": { - "algo": "ed25519", - "size": 512 - }, - "names": [ - { - "O": "Honest Achmed's Used Certificates", - "OU": "Hastily-Generated Values Divison", - "L": "San Francisco", - "ST": "California", - "C": "US" - } - ] -} diff --git a/example/certs/ca-key.pem b/example/certs/ca-key.pem index f583c0a..963dd0f 100644 --- a/example/certs/ca-key.pem +++ b/example/certs/ca-key.pem @@ -1,3 +1,3 @@ -----BEGIN Ed25519 PRIVATE KEY----- -MC4CAQAwBQYDK2VwBCIEIEMTEFvz1RJeAp22hTB1qmPFcSXaHzMJlKVP70EWzTHN +MC4CAQAwBQYDK2VwBCIEIKPAHuclX3Tz8jwZE0hb9Kdjx5Kxg3p+FqJv0O9SCx00 -----END Ed25519 PRIVATE KEY----- diff --git a/example/certs/ca.csr b/example/certs/ca.csr index a4a5b15..f58c787 100644 --- a/example/certs/ca.csr +++ b/example/certs/ca.csr @@ -1,9 +1,8 @@ -----BEGIN CERTIFICATE REQUEST----- -MIIBRzCB+gIBADBdMRUwEwYDVQQHEwx0aGUgaW50ZXJuZXQxFjAUBgNVBAoTDWF1 -dG9nZW5lcmF0ZWQxFTATBgNVBAsTDGV0Y2QgY2x1c3RlcjEVMBMGA1UEAxMMZXRj -ZC1leGFtcGxlMCowBQYDK2VwAyEAWvy5UAfHdxYpGq81CFm3Z2e3ChGoKsTH/7hc -5zL/ZSygajBoBgkqhkiG9w0BCQ4xWzBZMFcGA1UdEQRQME6HEP7AAAAAAAAAAAAA -AAAAAAGHEP7AAAAAAAAAAAAAAAAAAAKHEP7AAAAAAAAAAAAAAAAAAAOHEAAAAAAA -AAAAAAAAAAAAAAGHBH8AAAEwBQYDK2VwA0EAlDTB4Rd/PpNC01C7TxI8u/4019Oj -0lRBgHzTJpX8HqZdQuEYBt94v0KqQKQa7/oxmgBvAxJUrLTos7QBAlYbCw== +MIHvMIGiAgEAMG8xFTATBgNVBAcTDHRoZSBpbnRlcm5ldDEWMBQGA1UEChMNYXV0 +b2dlbmVyYXRlZDEeMBwGA1UECxMVZXRjZCBza3lmbGFrZSBjbHVzdGVyMR4wHAYD +VQQDExVza3lmbGFrZS1ldGNkLWV4YW1wbGUwKjAFBgMrZXADIQAgcVYcKr8yQnKz +dTPmiUyRWgFWafPsYoMVg4znKaBBGqAAMAUGAytlcANBABK3ArwpSnK4Azv9vDSa +sdy+lAiy1xNmOFKN0pV0nKBdjyxjBadKDWVlSWBpoZWt1CSHu9rVLBPrXTooilL3 +/gc= -----END CERTIFICATE REQUEST----- diff --git a/example/certs/ca.pem b/example/certs/ca.pem index 66e3ace..557e1b8 100644 --- a/example/certs/ca.pem +++ b/example/certs/ca.pem @@ -1,14 +1,13 @@ -----BEGIN CERTIFICATE----- -MIICGTCCAcugAwIBAgIUCNkLEEqavh/CRNNDoM+rnN2JSOYwBQYDK2VwMF0xFTAT -BgNVBAcTDHRoZSBpbnRlcm5ldDEWMBQGA1UEChMNYXV0b2dlbmVyYXRlZDEVMBMG -A1UECxMMZXRjZCBjbHVzdGVyMRUwEwYDVQQDEwxldGNkLWV4YW1wbGUwHhcNMjQw -NzE2MTk0MjAwWhcNMjkwNzE1MTk0MjAwWjBdMRUwEwYDVQQHEwx0aGUgaW50ZXJu -ZXQxFjAUBgNVBAoTDWF1dG9nZW5lcmF0ZWQxFTATBgNVBAsTDGV0Y2QgY2x1c3Rl -cjEVMBMGA1UEAxMMZXRjZC1leGFtcGxlMCowBQYDK2VwAyEAWvy5UAfHdxYpGq81 -CFm3Z2e3ChGoKsTH/7hc5zL/ZSyjgZwwgZkwDgYDVR0PAQH/BAQDAgEGMA8GA1Ud -EwEB/wQFMAMBAf8wHQYDVR0OBBYEFFQ8tbzoTdud2KZhxAl9bvTbqnrUMFcGA1Ud -EQRQME6HEP7AAAAAAAAAAAAAAAAAAAGHEP7AAAAAAAAAAAAAAAAAAAKHEP7AAAAA -AAAAAAAAAAAAAAOHEAAAAAAAAAAAAAAAAAAAAAGHBH8AAAEwBQYDK2VwA0EAEtgh -QPnf/sW6sOTt1nAyHwhiHaZF2RTGyHCZcx5jVVe1U/z8ON0hntLlRDzHV62YwsFB -aaI44gU7H9f4b25WAQ== +MIIB4jCCAZSgAwIBAgIUHzKbs9uzVilNYp5EYDs00/C2H2wwBQYDK2VwMG8xFTAT +BgNVBAcTDHRoZSBpbnRlcm5ldDEWMBQGA1UEChMNYXV0b2dlbmVyYXRlZDEeMBwG +A1UECxMVZXRjZCBza3lmbGFrZSBjbHVzdGVyMR4wHAYDVQQDExVza3lmbGFrZS1l +dGNkLWV4YW1wbGUwHhcNMjQxMjAyMTE1OTAwWhcNMjkxMjAxMTE1OTAwWjBvMRUw +EwYDVQQHEwx0aGUgaW50ZXJuZXQxFjAUBgNVBAoTDWF1dG9nZW5lcmF0ZWQxHjAc +BgNVBAsTFWV0Y2Qgc2t5Zmxha2UgY2x1c3RlcjEeMBwGA1UEAxMVc2t5Zmxha2Ut +ZXRjZC1leGFtcGxlMCowBQYDK2VwAyEAIHFWHCq/MkJys3Uz5olMkVoBVmnz7GKD +FYOM5ymgQRqjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0G +A1UdDgQWBBRlGT1d+LgpnUisEaXKURLm+42OiTAFBgMrZXADQQA2DVOglAcuK3Z6 +chnNl64vxWGhKyJymvt9WeX3+PUZb2iD5XIuBAE7YlD+7ppp4CPHz1/ou7A6Qrzt +ow5S9t8B -----END CERTIFICATE----- diff --git a/example/config/ca-config.json b/example/certs/config/ca-config.json similarity index 85% rename from example/config/ca-config.json rename to example/certs/config/ca-config.json index edd0c07..424ea37 100644 --- a/example/config/ca-config.json +++ b/example/certs/config/ca-config.json @@ -7,7 +7,7 @@ "server auth", "client auth" ], - "expiry": "876000h" + "expiry": "8760h" } } -} +} \ No newline at end of file diff --git a/example/certs/config/ca-csr.json b/example/certs/config/ca-csr.json new file mode 100644 index 0000000..eec5b9c --- /dev/null +++ b/example/certs/config/ca-csr.json @@ -0,0 +1,14 @@ +{ + "CN": "skyflake-etcd-example", + "key": { + "algo": "ed25519", + "size": 512 + }, + "names": [ + { + "O": "autogenerated", + "OU": "etcd skyflake cluster", + "L": "the internet" + } + ] +} diff --git a/example/certs/config/default.json b/example/certs/config/default.json new file mode 100644 index 0000000..1463f48 --- /dev/null +++ b/example/certs/config/default.json @@ -0,0 +1,21 @@ +{ + "CN": "example.net", + "hosts": [ + "fec0::1", + "fec0::2", + "fec0::3", + "::1", + "127.0.0.1" + ], + "key": { + "algo": "ed25519" + }, + "names": [ + { + "O": "autogenerated", + "OU": "etcd skyflake cluster", + "L": "the internet" + } + ] +} + diff --git a/example/config/req-csr.json b/example/certs/config/req-csr.json similarity index 75% rename from example/config/req-csr.json rename to example/certs/config/req-csr.json index e9c6b0b..4886099 100644 --- a/example/config/req-csr.json +++ b/example/certs/config/req-csr.json @@ -1,5 +1,5 @@ { - "CN": "etcd-example", + "CN": "skyflake-etcd-example", "hosts": [ "fec0::1", "fec0::2", @@ -13,8 +13,8 @@ "names": [ { "O": "autogenerated", - "OU": "etcd cluster", + "OU": "etcd skyflake cluster", "L": "the internet" } ] -} +} \ No newline at end of file diff --git a/example/certs/example2-key.pem b/example/certs/default-key.pem similarity index 52% rename from example/certs/example2-key.pem rename to example/certs/default-key.pem index a257b22..fa3507b 100644 --- a/example/certs/example2-key.pem +++ b/example/certs/default-key.pem @@ -1,3 +1,3 @@ -----BEGIN Ed25519 PRIVATE KEY----- -MC4CAQAwBQYDK2VwBCIEINj1txt2IO3S4958/Rny1hungjcAqzqhYnT1odiKRWwT +MC4CAQAwBQYDK2VwBCIEILHkEluUia6mFJwDVvkw25Py2Qje0XwdmTeOhK+SWMGi -----END Ed25519 PRIVATE KEY----- diff --git a/example/certs/default.csr b/example/certs/default.csr new file mode 100644 index 0000000..cb1913a --- /dev/null +++ b/example/certs/default.csr @@ -0,0 +1,10 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIIBUDCCAQICAQAwZTEVMBMGA1UEBxMMdGhlIGludGVybmV0MRYwFAYDVQQKEw1h +dXRvZ2VuZXJhdGVkMR4wHAYDVQQLExVldGNkIHNreWZsYWtlIGNsdXN0ZXIxFDAS +BgNVBAMTC2V4YW1wbGUubmV0MCowBQYDK2VwAyEAmcDRG4K1OwMpjeBrNgzMQS4J +4vnvWo2Ktj43Mww0Y+GgajBoBgkqhkiG9w0BCQ4xWzBZMFcGA1UdEQRQME6HEP7A +AAAAAAAAAAAAAAAAAAGHEP7AAAAAAAAAAAAAAAAAAAKHEP7AAAAAAAAAAAAAAAAA +AAOHEAAAAAAAAAAAAAAAAAAAAAGHBH8AAAEwBQYDK2VwA0EAKis22c0zZ9GqQYAI +YGhz+R/k00VIpWzlXsSsnbJpsfj18FWOMGne2F0FrGyTIMgwLxqjFrvUIX1jt7No +vnYWCg== +-----END CERTIFICATE REQUEST----- diff --git a/example/certs/default.pem b/example/certs/default.pem new file mode 100644 index 0000000..4de6126 --- /dev/null +++ b/example/certs/default.pem @@ -0,0 +1,16 @@ +-----BEGIN CERTIFICATE----- +MIICcDCCAiKgAwIBAgIUEqxQwbyAuUbFloz2Af1r1a/CcjkwBQYDK2VwMG8xFTAT +BgNVBAcTDHRoZSBpbnRlcm5ldDEWMBQGA1UEChMNYXV0b2dlbmVyYXRlZDEeMBwG +A1UECxMVZXRjZCBza3lmbGFrZSBjbHVzdGVyMR4wHAYDVQQDExVza3lmbGFrZS1l +dGNkLWV4YW1wbGUwHhcNMjQxMjAyMTIwMDAwWhcNMjUxMjAyMTIwMDAwWjBlMRUw +EwYDVQQHEwx0aGUgaW50ZXJuZXQxFjAUBgNVBAoTDWF1dG9nZW5lcmF0ZWQxHjAc +BgNVBAsTFWV0Y2Qgc2t5Zmxha2UgY2x1c3RlcjEUMBIGA1UEAxMLZXhhbXBsZS5u +ZXQwKjAFBgMrZXADIQCZwNEbgrU7AymN4Gs2DMxBLgni+e9ajYq2PjczDDRj4aOB +2TCB1jAOBgNVHQ8BAf8EBAMCBaAwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUF +BwMCMAwGA1UdEwEB/wQCMAAwHQYDVR0OBBYEFM193VfeS/PgCTJdre/3qQTxAX0U +MB8GA1UdIwQYMBaAFGUZPV34uCmdSKwRpcpREub7jY6JMFcGA1UdEQRQME6HEP7A +AAAAAAAAAAAAAAAAAAGHEP7AAAAAAAAAAAAAAAAAAAKHEP7AAAAAAAAAAAAAAAAA +AAOHEAAAAAAAAAAAAAAAAAAAAAGHBH8AAAEwBQYDK2VwA0EAP0zSSpejQs8qwXeC +jrUZasY4YWRUtZIS+nqo+4/Uaqxq+U+zDT+SHd3vjosHsl4k6KXcY2j7r9rDv1+h +o6q0Bw== +-----END CERTIFICATE----- diff --git a/example/certs/example1-key.pem b/example/certs/example1-key.pem deleted file mode 100644 index 564ae45..0000000 --- a/example/certs/example1-key.pem +++ /dev/null @@ -1,3 +0,0 @@ ------BEGIN Ed25519 PRIVATE KEY----- -MC4CAQAwBQYDK2VwBCIEIBGEyCEiUp84bElnRioTt3CBv/LkgxdvZd2oxzZ/mdOw ------END Ed25519 PRIVATE KEY----- diff --git a/example/certs/example1.csr b/example/certs/example1.csr deleted file mode 100644 index d66e35d..0000000 --- a/example/certs/example1.csr +++ /dev/null @@ -1,9 +0,0 @@ ------BEGIN CERTIFICATE REQUEST----- -MIIBRzCB+gIBADBdMRUwEwYDVQQHEwx0aGUgaW50ZXJuZXQxFjAUBgNVBAoTDWF1 -dG9nZW5lcmF0ZWQxFTATBgNVBAsTDGV0Y2QgY2x1c3RlcjEVMBMGA1UEAxMMZXRj -ZC1leGFtcGxlMCowBQYDK2VwAyEAX0ISCrwOyEQ64fPNRadnpiwjuHH+PhoYt4g3 -BD5VRACgajBoBgkqhkiG9w0BCQ4xWzBZMFcGA1UdEQRQME6HEP7AAAAAAAAAAAAA -AAAAAAGHEP7AAAAAAAAAAAAAAAAAAAKHEP7AAAAAAAAAAAAAAAAAAAOHEAAAAAAA -AAAAAAAAAAAAAAGHBH8AAAEwBQYDK2VwA0EAERkiHl8ftX6tbeaXZ3NcwJaWVknS -VobentHaevSap5l8NIarOt8J04LXbR48HNQIo1FFNkvbfcPvqQuHuKGJCw== ------END CERTIFICATE REQUEST----- diff --git a/example/certs/example1.pem b/example/certs/example1.pem deleted file mode 100644 index 3a02513..0000000 --- a/example/certs/example1.pem +++ /dev/null @@ -1,14 +0,0 @@ ------BEGIN CERTIFICATE----- -MIICNzCCAemgAwIBAgIUYgHAOcH6pLLfVtGVUx4TK/sX5dAwBQYDK2VwMF0xFTAT -BgNVBAcTDHRoZSBpbnRlcm5ldDEWMBQGA1UEChMNYXV0b2dlbmVyYXRlZDEVMBMG -A1UECxMMZXRjZCBjbHVzdGVyMRUwEwYDVQQDEwxldGNkLWV4YW1wbGUwIBcNMjQw -NzE2MTk0MjAwWhgPMjEyNDA2MjIxOTQyMDBaMF0xFTATBgNVBAcTDHRoZSBpbnRl -cm5ldDEWMBQGA1UEChMNYXV0b2dlbmVyYXRlZDEVMBMGA1UECxMMZXRjZCBjbHVz -dGVyMRUwEwYDVQQDEwxldGNkLWV4YW1wbGUwKjAFBgMrZXADIQBfQhIKvA7IRDrh -881Fp2emLCO4cf4+Ghi3iDcEPlVEAKOBuDCBtTAOBgNVHQ8BAf8EBAMCBaAwHQYD -VR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMAwGA1UdEwEB/wQCMAAwHQYDVR0O -BBYEFB2kJrQDhUGYGpHcVGD4KbkRk+ByMFcGA1UdEQRQME6HEP7AAAAAAAAAAAAA -AAAAAAGHEP7AAAAAAAAAAAAAAAAAAAKHEP7AAAAAAAAAAAAAAAAAAAOHEAAAAAAA -AAAAAAAAAAAAAAGHBH8AAAEwBQYDK2VwA0EANadWFqPd7H6soA/zUPHPSbOKpij1 -uA2f673WZw9zzPWMzIX12wjfAxsegIpmghL5cMPPsFDgC4MtY0BJPUeMCA== ------END CERTIFICATE----- diff --git a/example/certs/example2.csr b/example/certs/example2.csr deleted file mode 100644 index d38286e..0000000 --- a/example/certs/example2.csr +++ /dev/null @@ -1,9 +0,0 @@ ------BEGIN CERTIFICATE REQUEST----- -MIIBRzCB+gIBADBdMRUwEwYDVQQHEwx0aGUgaW50ZXJuZXQxFjAUBgNVBAoTDWF1 -dG9nZW5lcmF0ZWQxFTATBgNVBAsTDGV0Y2QgY2x1c3RlcjEVMBMGA1UEAxMMZXRj -ZC1leGFtcGxlMCowBQYDK2VwAyEApjYRMG9MSYJd5iJKmP8vP/HrgFoiPf2y0mE6 -gh4m8ZWgajBoBgkqhkiG9w0BCQ4xWzBZMFcGA1UdEQRQME6HEP7AAAAAAAAAAAAA -AAAAAAGHEP7AAAAAAAAAAAAAAAAAAAKHEP7AAAAAAAAAAAAAAAAAAAOHEAAAAAAA -AAAAAAAAAAAAAAGHBH8AAAEwBQYDK2VwA0EA0flUnLg77p20YEh/wMgRQWxWTx+c -g31yRdemq3vM+8xL5Gc8GZFGTPgtA2Lx32E637MgGUdUo4dhLPcy4VY2AA== ------END CERTIFICATE REQUEST----- diff --git a/example/certs/example2.pem b/example/certs/example2.pem deleted file mode 100644 index 10d04b9..0000000 --- a/example/certs/example2.pem +++ /dev/null @@ -1,14 +0,0 @@ ------BEGIN CERTIFICATE----- -MIICNzCCAemgAwIBAgIUe7XBGPB38YhFDMx2G0KC8KkmY8UwBQYDK2VwMF0xFTAT -BgNVBAcTDHRoZSBpbnRlcm5ldDEWMBQGA1UEChMNYXV0b2dlbmVyYXRlZDEVMBMG -A1UECxMMZXRjZCBjbHVzdGVyMRUwEwYDVQQDEwxldGNkLWV4YW1wbGUwIBcNMjQw -NzE2MTk0MjAwWhgPMjEyNDA2MjIxOTQyMDBaMF0xFTATBgNVBAcTDHRoZSBpbnRl -cm5ldDEWMBQGA1UEChMNYXV0b2dlbmVyYXRlZDEVMBMGA1UECxMMZXRjZCBjbHVz -dGVyMRUwEwYDVQQDEwxldGNkLWV4YW1wbGUwKjAFBgMrZXADIQCmNhEwb0xJgl3m -IkqY/y8/8euAWiI9/bLSYTqCHibxlaOBuDCBtTAOBgNVHQ8BAf8EBAMCBaAwHQYD -VR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMAwGA1UdEwEB/wQCMAAwHQYDVR0O -BBYEFIbRMrkmOEWTHBPsudvMtuCFMhn2MFcGA1UdEQRQME6HEP7AAAAAAAAAAAAA -AAAAAAGHEP7AAAAAAAAAAAAAAAAAAAKHEP7AAAAAAAAAAAAAAAAAAAOHEAAAAAAA -AAAAAAAAAAAAAAGHBH8AAAEwBQYDK2VwA0EAOtbGD5c5rrrbrRZzxuP7aEyVIIJQ -XNpkqSZgXXKOn0Eyi1GB5eeTsfZQlskSQ9ChbGZxp7PAlJaI1qUzZ5ybCg== ------END CERTIFICATE----- diff --git a/example/certs/example3-key.pem b/example/certs/example3-key.pem deleted file mode 100644 index 5316f89..0000000 --- a/example/certs/example3-key.pem +++ /dev/null @@ -1,3 +0,0 @@ ------BEGIN Ed25519 PRIVATE KEY----- -MC4CAQAwBQYDK2VwBCIEIIQ9xVv63Y7Jo9rNtjwTtxouIJUgdEo65HPiLjqjq+uZ ------END Ed25519 PRIVATE KEY----- diff --git a/example/certs/example3.csr b/example/certs/example3.csr deleted file mode 100644 index 96d995c..0000000 --- a/example/certs/example3.csr +++ /dev/null @@ -1,9 +0,0 @@ ------BEGIN CERTIFICATE REQUEST----- -MIIBRzCB+gIBADBdMRUwEwYDVQQHEwx0aGUgaW50ZXJuZXQxFjAUBgNVBAoTDWF1 -dG9nZW5lcmF0ZWQxFTATBgNVBAsTDGV0Y2QgY2x1c3RlcjEVMBMGA1UEAxMMZXRj -ZC1leGFtcGxlMCowBQYDK2VwAyEArInudki4shX2NRzMwXIdKfgv8QL3z39j/YNX -LU3Mw1qgajBoBgkqhkiG9w0BCQ4xWzBZMFcGA1UdEQRQME6HEP7AAAAAAAAAAAAA -AAAAAAGHEP7AAAAAAAAAAAAAAAAAAAKHEP7AAAAAAAAAAAAAAAAAAAOHEAAAAAAA -AAAAAAAAAAAAAAGHBH8AAAEwBQYDK2VwA0EA1u8gPSVRrSgonzEiyZINcloLtZCr -F2wG1jXg2V4xcafzHe2HeD94XOp70rl8R53rShCPDiuf7redQGIbCHseAQ== ------END CERTIFICATE REQUEST----- diff --git a/example/certs/example3.pem b/example/certs/example3.pem deleted file mode 100644 index e0182da..0000000 --- a/example/certs/example3.pem +++ /dev/null @@ -1,14 +0,0 @@ ------BEGIN CERTIFICATE----- -MIICNzCCAemgAwIBAgIURpq8lJcnm/XnCxsXpCnpy1Jf7GQwBQYDK2VwMF0xFTAT -BgNVBAcTDHRoZSBpbnRlcm5ldDEWMBQGA1UEChMNYXV0b2dlbmVyYXRlZDEVMBMG -A1UECxMMZXRjZCBjbHVzdGVyMRUwEwYDVQQDEwxldGNkLWV4YW1wbGUwIBcNMjQw -NzE2MTk0MjAwWhgPMjEyNDA2MjIxOTQyMDBaMF0xFTATBgNVBAcTDHRoZSBpbnRl -cm5ldDEWMBQGA1UEChMNYXV0b2dlbmVyYXRlZDEVMBMGA1UECxMMZXRjZCBjbHVz -dGVyMRUwEwYDVQQDEwxldGNkLWV4YW1wbGUwKjAFBgMrZXADIQCsie52SLiyFfY1 -HMzBch0p+C/xAvfPf2P9g1ctTczDWqOBuDCBtTAOBgNVHQ8BAf8EBAMCBaAwHQYD -VR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMAwGA1UdEwEB/wQCMAAwHQYDVR0O -BBYEFKTHiiohxrYsb8MJT/Y41J23rdXIMFcGA1UdEQRQME6HEP7AAAAAAAAAAAAA -AAAAAAGHEP7AAAAAAAAAAAAAAAAAAAKHEP7AAAAAAAAAAAAAAAAAAAOHEAAAAAAA -AAAAAAAAAAAAAAGHBH8AAAEwBQYDK2VwA0EAALA2oCPaZV12mlTWRqvKr48s12uj -htFgHDoQAYMW70lBrmvEw0dPlh+xioegtmFoy+NNBn7E5V6MMIzZZbYeBg== ------END CERTIFICATE----- diff --git a/example/certs/peer-example1-key.pem b/example/certs/peer-example1-key.pem deleted file mode 100644 index 390e9d8..0000000 --- a/example/certs/peer-example1-key.pem +++ /dev/null @@ -1,3 +0,0 @@ ------BEGIN Ed25519 PRIVATE KEY----- -MC4CAQAwBQYDK2VwBCIEIJ/zC5RpEvSvt4ZU4VeXQ4Bn4So8Sdkre1/FpHZW1CDo ------END Ed25519 PRIVATE KEY----- diff --git a/example/certs/peer-example1.csr b/example/certs/peer-example1.csr deleted file mode 100644 index 3309bed..0000000 --- a/example/certs/peer-example1.csr +++ /dev/null @@ -1,9 +0,0 @@ ------BEGIN CERTIFICATE REQUEST----- -MIIBRzCB+gIBADBdMRUwEwYDVQQHEwx0aGUgaW50ZXJuZXQxFjAUBgNVBAoTDWF1 -dG9nZW5lcmF0ZWQxFTATBgNVBAsTDGV0Y2QgY2x1c3RlcjEVMBMGA1UEAxMMZXRj -ZC1leGFtcGxlMCowBQYDK2VwAyEA2tMIR3z+Wp6PL8TN4JMpgUyPj1n2CpOayV2J -ih/8/8GgajBoBgkqhkiG9w0BCQ4xWzBZMFcGA1UdEQRQME6HEP7AAAAAAAAAAAAA -AAAAAAGHEP7AAAAAAAAAAAAAAAAAAAKHEP7AAAAAAAAAAAAAAAAAAAOHEAAAAAAA -AAAAAAAAAAAAAAGHBH8AAAEwBQYDK2VwA0EAOCzFzWw0+u5eLVbBOhQI7e6t0XsK -or1tBCVErs16w8khlN8gx8y1uuKQEUsQYgtvOiubJy8cu8JTTzg9cybmAQ== ------END CERTIFICATE REQUEST----- diff --git a/example/certs/peer-example1.pem b/example/certs/peer-example1.pem deleted file mode 100644 index a2247c9..0000000 --- a/example/certs/peer-example1.pem +++ /dev/null @@ -1,14 +0,0 @@ ------BEGIN CERTIFICATE----- -MIICNzCCAemgAwIBAgIUAwC6lT0eYpLDSXLsq0B21aG7EqgwBQYDK2VwMF0xFTAT -BgNVBAcTDHRoZSBpbnRlcm5ldDEWMBQGA1UEChMNYXV0b2dlbmVyYXRlZDEVMBMG -A1UECxMMZXRjZCBjbHVzdGVyMRUwEwYDVQQDEwxldGNkLWV4YW1wbGUwIBcNMjQw -NzE2MTk0MjAwWhgPMjEyNDA2MjIxOTQyMDBaMF0xFTATBgNVBAcTDHRoZSBpbnRl -cm5ldDEWMBQGA1UEChMNYXV0b2dlbmVyYXRlZDEVMBMGA1UECxMMZXRjZCBjbHVz -dGVyMRUwEwYDVQQDEwxldGNkLWV4YW1wbGUwKjAFBgMrZXADIQDa0whHfP5ano8v -xM3gkymBTI+PWfYKk5rJXYmKH/z/waOBuDCBtTAOBgNVHQ8BAf8EBAMCBaAwHQYD -VR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMAwGA1UdEwEB/wQCMAAwHQYDVR0O -BBYEFH4DS96JIOdG2zrfJoL2ojxfTqXdMFcGA1UdEQRQME6HEP7AAAAAAAAAAAAA -AAAAAAGHEP7AAAAAAAAAAAAAAAAAAAKHEP7AAAAAAAAAAAAAAAAAAAOHEAAAAAAA -AAAAAAAAAAAAAAGHBH8AAAEwBQYDK2VwA0EA3tR2EgvN9g8Z0gcWFsERuoOUnmYT -nwE7p5nQcaflbjEcYxCkAKvExhR0NlurMCflMh+dVbhldIyIDyzeK/tnDA== ------END CERTIFICATE----- diff --git a/example/certs/peer-example2-key.pem b/example/certs/peer-example2-key.pem deleted file mode 100644 index 0be5f04..0000000 --- a/example/certs/peer-example2-key.pem +++ /dev/null @@ -1,3 +0,0 @@ ------BEGIN Ed25519 PRIVATE KEY----- -MC4CAQAwBQYDK2VwBCIEIPp2+itRObBvM5NTJgS9/5Y9oAxy1KAcfQZaTxgTmCP6 ------END Ed25519 PRIVATE KEY----- diff --git a/example/certs/peer-example2.csr b/example/certs/peer-example2.csr deleted file mode 100644 index 68cb476..0000000 --- a/example/certs/peer-example2.csr +++ /dev/null @@ -1,9 +0,0 @@ ------BEGIN CERTIFICATE REQUEST----- -MIIBRzCB+gIBADBdMRUwEwYDVQQHEwx0aGUgaW50ZXJuZXQxFjAUBgNVBAoTDWF1 -dG9nZW5lcmF0ZWQxFTATBgNVBAsTDGV0Y2QgY2x1c3RlcjEVMBMGA1UEAxMMZXRj -ZC1leGFtcGxlMCowBQYDK2VwAyEAwe/vnjdOAvLdJbMvH22FKWtYIL81VCNwuZkD -OhMHh/6gajBoBgkqhkiG9w0BCQ4xWzBZMFcGA1UdEQRQME6HEP7AAAAAAAAAAAAA -AAAAAAGHEP7AAAAAAAAAAAAAAAAAAAKHEP7AAAAAAAAAAAAAAAAAAAOHEAAAAAAA -AAAAAAAAAAAAAAGHBH8AAAEwBQYDK2VwA0EAfhKGZe3rZABx1OXxLy/NPS/t4qoe -jy65mUgUo9TysyaoxSMM1cLzGOkWW8BOtP0KX4EUhJYXlukC0IviPXQ7AQ== ------END CERTIFICATE REQUEST----- diff --git a/example/certs/peer-example2.pem b/example/certs/peer-example2.pem deleted file mode 100644 index bdd00d1..0000000 --- a/example/certs/peer-example2.pem +++ /dev/null @@ -1,14 +0,0 @@ ------BEGIN CERTIFICATE----- -MIICNzCCAemgAwIBAgIUUjaY6MBwV/m6LIu0lqFHm3ruFXMwBQYDK2VwMF0xFTAT -BgNVBAcTDHRoZSBpbnRlcm5ldDEWMBQGA1UEChMNYXV0b2dlbmVyYXRlZDEVMBMG -A1UECxMMZXRjZCBjbHVzdGVyMRUwEwYDVQQDEwxldGNkLWV4YW1wbGUwIBcNMjQw -NzE2MTk0MjAwWhgPMjEyNDA2MjIxOTQyMDBaMF0xFTATBgNVBAcTDHRoZSBpbnRl -cm5ldDEWMBQGA1UEChMNYXV0b2dlbmVyYXRlZDEVMBMGA1UECxMMZXRjZCBjbHVz -dGVyMRUwEwYDVQQDEwxldGNkLWV4YW1wbGUwKjAFBgMrZXADIQDB7++eN04C8t0l -sy8fbYUpa1ggvzVUI3C5mQM6EweH/qOBuDCBtTAOBgNVHQ8BAf8EBAMCBaAwHQYD -VR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMAwGA1UdEwEB/wQCMAAwHQYDVR0O -BBYEFI63W5D655IjJGj0tWCqUGrNsjLAMFcGA1UdEQRQME6HEP7AAAAAAAAAAAAA -AAAAAAGHEP7AAAAAAAAAAAAAAAAAAAKHEP7AAAAAAAAAAAAAAAAAAAOHEAAAAAAA -AAAAAAAAAAAAAAGHBH8AAAEwBQYDK2VwA0EAQcr6QipYgI3IoQigptc0Inj1YeXf -S/rxnuMsyUs8uvqqg4BwaoMZX6WYTOhhZAMM4v3NFjhz3mmKu0Af4n7ABw== ------END CERTIFICATE----- diff --git a/example/certs/peer-example3-key.pem b/example/certs/peer-example3-key.pem deleted file mode 100644 index d9cc782..0000000 --- a/example/certs/peer-example3-key.pem +++ /dev/null @@ -1,3 +0,0 @@ ------BEGIN Ed25519 PRIVATE KEY----- -MC4CAQAwBQYDK2VwBCIEIKNmdhBTKZL0CMwT3yDSDGNI310ueSsLWmDf01A3nDYJ ------END Ed25519 PRIVATE KEY----- diff --git a/example/certs/peer-example3.csr b/example/certs/peer-example3.csr deleted file mode 100644 index deb3301..0000000 --- a/example/certs/peer-example3.csr +++ /dev/null @@ -1,9 +0,0 @@ ------BEGIN CERTIFICATE REQUEST----- -MIIBRzCB+gIBADBdMRUwEwYDVQQHEwx0aGUgaW50ZXJuZXQxFjAUBgNVBAoTDWF1 -dG9nZW5lcmF0ZWQxFTATBgNVBAsTDGV0Y2QgY2x1c3RlcjEVMBMGA1UEAxMMZXRj -ZC1leGFtcGxlMCowBQYDK2VwAyEAggNboIQYKrmxO0809jRk4ly/FTA2ppf0DVoL -rEb/aNqgajBoBgkqhkiG9w0BCQ4xWzBZMFcGA1UdEQRQME6HEP7AAAAAAAAAAAAA -AAAAAAGHEP7AAAAAAAAAAAAAAAAAAAKHEP7AAAAAAAAAAAAAAAAAAAOHEAAAAAAA -AAAAAAAAAAAAAAGHBH8AAAEwBQYDK2VwA0EAjmfEW1tKrm+1AUIk1zkEN/XLJYre -uMTrlnGQxRZstfLTI6h95CrV1V2vKu99eSFl0idr3TPTvwaVf8MmWdCrDA== ------END CERTIFICATE REQUEST----- diff --git a/example/certs/peer-example3.pem b/example/certs/peer-example3.pem deleted file mode 100644 index 9221d55..0000000 --- a/example/certs/peer-example3.pem +++ /dev/null @@ -1,14 +0,0 @@ ------BEGIN CERTIFICATE----- -MIICNzCCAemgAwIBAgIUQwLFi+8HL1Up3TaGB59peaSgeQ8wBQYDK2VwMF0xFTAT -BgNVBAcTDHRoZSBpbnRlcm5ldDEWMBQGA1UEChMNYXV0b2dlbmVyYXRlZDEVMBMG -A1UECxMMZXRjZCBjbHVzdGVyMRUwEwYDVQQDEwxldGNkLWV4YW1wbGUwIBcNMjQw -NzE2MTk0MjAwWhgPMjEyNDA2MjIxOTQyMDBaMF0xFTATBgNVBAcTDHRoZSBpbnRl -cm5ldDEWMBQGA1UEChMNYXV0b2dlbmVyYXRlZDEVMBMGA1UECxMMZXRjZCBjbHVz -dGVyMRUwEwYDVQQDEwxldGNkLWV4YW1wbGUwKjAFBgMrZXADIQCCA1ughBgqubE7 -TzT2NGTiXL8VMDaml/QNWgusRv9o2qOBuDCBtTAOBgNVHQ8BAf8EBAMCBaAwHQYD -VR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMAwGA1UdEwEB/wQCMAAwHQYDVR0O -BBYEFOJNRyO0Z1wXC8ed5URpXewiC59oMFcGA1UdEQRQME6HEP7AAAAAAAAAAAAA -AAAAAAGHEP7AAAAAAAAAAAAAAAAAAAKHEP7AAAAAAAAAAAAAAAAAAAOHEAAAAAAA -AAAAAAAAAAAAAAGHBH8AAAEwBQYDK2VwA0EAp7ZkuHYzWiD0kVBj+qajxZolQ5MW -9M16/RPCkdatPkCAkPmGVujZo6h4HIqPSgD6zF2jHk4rAt/nXPf6u5r7Aw== ------END CERTIFICATE----- diff --git a/flake.nix b/flake.nix index 1414f23..3bd32ea 100644 --- a/flake.nix +++ b/flake.nix @@ -25,7 +25,7 @@ pkgs = nixpkgs.legacyPackages.${system}; in { - formatter.${system} = pkgs.alejandra; + # formatter.${system} = pkgs.alejandra; packages.${system} = import ./pkgs/doc.nix { inherit pkgs self; }; @@ -36,7 +36,8 @@ {nixpkgs.overlays = [ (final: prev: { nomadPin = nomad-nixpkgs.legacyPackages.${prev.system}; }) ];} ./nixos-modules/storage/seaweedfs/options.nix ./nixos-modules/storage/seaweedfs/server.nix - ./nixos-modules/storage/seaweedfs/dbBackend/etcd.nix + ./nixos-modules/storage/seaweedfs/db-backend/etcd/default.nix + ./nixos-modules/storage/seaweedfs/db-backend/etcd/options.nix ./nixos-modules/storage/ceph/server.nix ./nixos-modules/defaults.nix ./nixos-modules/nodes.nix diff --git a/nixos-modules/defaults.nix b/nixos-modules/defaults.nix index c979dd3..bcd3134 100644 --- a/nixos-modules/defaults.nix +++ b/nixos-modules/defaults.nix @@ -15,5 +15,4 @@ builders-use-substitutes = true ''; }; - } diff --git a/nixos-modules/storage/seaweedfs/db-backend/etcd/default.nix b/nixos-modules/storage/seaweedfs/db-backend/etcd/default.nix new file mode 100644 index 0000000..1b6d371 --- /dev/null +++ b/nixos-modules/storage/seaweedfs/db-backend/etcd/default.nix @@ -0,0 +1,115 @@ +{ + pkgs, + config, + lib, + ... +}: +{ + config = + lib.mkIf + (builtins.all (x: x == true) [ + config.skyflake.storage.seaweedfs.filer.db.etcd.enable + config.skyflake.storage.seaweedfs.enable + ]) + { + systemd.tmpfiles.settings."10-etcd"."/var/lib/etcd".d = { + user = "etcd"; + mode = "0700"; + }; + + systemd.services."etcd" = { + description = "etcd key-value store"; + wantedBy = [ "multi-user.target" ]; + after = [ + "network-online.target" + "network.target" + ]; + # ++ lib.optional config.networking.firewall.enable "firewall.service"; + wants = [ "network-online.target" ]; + # ++ lib.optional config.networking.firewall.enable "firewall.service"; + + environment = + # (nixpgs.filterAttrs (n: v: v != null) + let + address = config.skyflake.nodes.${config.networking.hostName}.address; + in + { + ETCD_NAME = config.networking.hostName; + ETCD_DATA_DIR = "/var/lib/etcd"; + ETCD_ADVERTISE_CLIENT_URLS = "https://${address}:2379"; + ETCD_LISTEN_CLIENT_URLS = "https://${address}:2379"; + ETCD_LISTEN_PEER_URLS = "https://${address}:2380"; + ETCD_INITIAL_ADVERTISE_PEER_URLS = "https://${address}:2380"; + ETCD_CLIENT_CERT_AUTH = "true"; + ETCD_TRUSTED_CA_FILE = "${config.skyflake.storage.seaweedfs.filer.db.etcd.trustedCaFile}"; + ETCD_CERT_FILE = "${config.skyflake.storage.seaweedfs.filer.db.etcd.certFile}"; + ETCD_KEY_FILE = "${config.skyflake.storage.seaweedfs.filer.db.etcd.keyFile}"; + ETCD_PEER_CLIENT_CERT_AUTH = "true"; + ETCD_PEER_TRUSTED_CA_FILE = "${config.skyflake.storage.seaweedfs.filer.db.etcd.peerTrustedCaFile}"; + ETCD_PEER_CERT_FILE = "${config.skyflake.storage.seaweedfs.filer.db.etcd.peerCertFile}"; + ETCD_PEER_KEY_FILE = "${config.skyflake.storage.seaweedfs.filer.db.etcd.peerKeyFile}"; + ETCD_INITIAL_CLUSTER = "${lib.concatMapStringsSep "," ( + node: "${node}=https://" + (config.skyflake.nodes."${node}").address + ":2380" + ) (builtins.attrNames config.skyflake.nodes)}"; + ETCD_INITIAL_CLUSTER_STATE = "new"; + ETCD_INITIAL_CLUSTER_TOKEN = "etcd-cluster"; + }; + unitConfig = { + Documentation = "https://etcd.io/docs/v3.5/"; + }; + + serviceConfig = { + Type = "notify"; + Restart = "always"; + RestartSec = "5s"; + ExecStartPre = "${pkgs.coreutils}/bin/sleep 2"; # TODO fix workaround, so that it doesnt stop on first start because it cant bind. + ExecStart = "${pkgs.etcd}/bin/etcd"; + User = "etcd"; + LimitNOFILE = 40000; + }; + }; + + environment.etc.seaweedfs-filer = { + text = '' + [etcd] + enabled = true + servers = "${ + lib.concatMapStringsSep "," (node: (config.skyflake.nodes."${node}").address + ":2379") ( + builtins.attrNames config.skyflake.nodes + ) + }" + # username = "seaweedfs" + # password = "" + key_prefix = "seaweedfs." + timeout = "3s" + # Set the CA certificate path + tls_ca_file = "${config.skyflake.storage.seaweedfs.filer.db.etcd.peerTrustedCaFile}" + # Set the client certificate path + tls_client_crt_file = "${config.skyflake.storage.seaweedfs.filer.db.etcd.peerCertFile}" + # Set the client private key path + tls_client_key_file = "${config.skyflake.storage.seaweedfs.filer.db.etcd.peerKeyFile}" + ''; + target = "./seaweedfs/filer.toml"; + user = "seaweedfs"; + mode = "0440"; + }; + + environment.systemPackages = [ pkgs.etcd ]; + /* + TODO: add firewall to skyflake. + networking.firewall = lib.mkIf config.services.etcd.openFirewall { + allowedTCPPorts = [ + 2379 # for client requests + 2380 # for peer communication + ]; + }; + */ + users.users.etcd = { + isSystemUser = true; + group = "etcd"; + description = "Etcd daemon user"; + home = "/var/lib/etcd"; # TODO bring it under a single setting, the state path. + }; + users.groups.etcd = { }; + }; +} diff --git a/nixos-modules/storage/seaweedfs/db-backend/etcd/options.nix b/nixos-modules/storage/seaweedfs/db-backend/etcd/options.nix new file mode 100644 index 0000000..15b9cb5 --- /dev/null +++ b/nixos-modules/storage/seaweedfs/db-backend/etcd/options.nix @@ -0,0 +1,50 @@ +{ + lib, + config, + options, + ... +}: +{ + options.skyflake.storage.seaweedfs.filer.db.etcd = { + enable = lib.mkOption { + type = lib.types.bool; + default = true; + description = '' + Use to enable Kubernetes etcd database as a backend for seaweedfs. + ''; + }; + trustedCaFile = lib.mkOption { + description = "Certificate authority file to use for clients"; + default = null; + type = lib.types.nullOr lib.types.path; + }; + certFile = lib.mkOption { + description = "Cert file to use for clients"; + default = null; + type = lib.types.nullOr lib.types.path; + }; + keyFile = lib.mkOption { + description = "Key file to use for clients"; + default = null; + type = lib.types.nullOr lib.types.path; + }; + peerCertFile = lib.mkOption { + description = "Cert file to use for peer to peer communication"; + default = config.skyflake.storage.seaweedfs.filer.db.etcd.certFile; + defaultText = lib.literalExpression "config.${options.skyflake.storage.seaweedfs.filer.db.etcd.certFile}"; + type = lib.types.nullOr lib.types.path; + }; + peerKeyFile = lib.mkOption { + description = "Key file to use for peer to peer communication"; + default = config.skyflake.storage.seaweedfs.filer.db.etcd.keyFile; + defaultText = lib.literalExpression "config.${options.skyflake.storage.seaweedfs.filer.db.etcd.keyFile}"; + type = lib.types.nullOr lib.types.path; + }; + peerTrustedCaFile = lib.mkOption { + description = "Certificate authority file to use for peer to peer communication"; + default = config.skyflake.storage.seaweedfs.filer.db.etcd.trustedCaFile; + defaultText = lib.literalExpression "config.${options.skyflake.storage.seaweedfs.filer.db.etcd.trustedCaFile}"; + type = lib.types.nullOr lib.types.path; + }; + }; +} diff --git a/nixos-modules/storage/seaweedfs/dbBackend/etcd.nix b/nixos-modules/storage/seaweedfs/dbBackend/etcd.nix deleted file mode 100644 index 3ba7dde..0000000 --- a/nixos-modules/storage/seaweedfs/dbBackend/etcd.nix +++ /dev/null @@ -1,91 +0,0 @@ -{ pkgs, config, lib, ... }: -{ - config = lib.mkIf (builtins.all (x: x == true) [ config.skyflake.storage.seaweedfs.filer.db.etcd.enable config.skyflake.storage.seaweedfs.enable ]) { - systemd.tmpfiles.settings."10-etcd"."/var/lib/etcd".d = { - user = "etcd"; - mode = "0700"; - }; - - systemd.services."etcd" = { - description = "etcd key-value store"; - wantedBy = [ "multi-user.target" ]; - after = [ "network-online.target" "network.target" ]; - # ++ lib.optional config.networking.firewall.enable "firewall.service"; - wants = [ "network-online.target" ]; - # ++ lib.optional config.networking.firewall.enable "firewall.service"; - - environment = /* (nixpgs.filterAttrs (n: v: v != null) */ let - address = config.skyflake.nodes.${config.networking.hostName}.address; - in { - ETCD_NAME = config.networking.hostName; - ETCD_DATA_DIR = "/var/lib/etcd"; - ETCD_ADVERTISE_CLIENT_URLS = "https://${address}:2379"; - ETCD_LISTEN_CLIENT_URLS = "https://${address}:2379"; - ETCD_LISTEN_PEER_URLS = "https://${address}:2380"; - ETCD_INITIAL_ADVERTISE_PEER_URLS = "https://${address}:2380"; - ETCD_CLIENT_CERT_AUTH = "true"; - ETCD_TRUSTED_CA_FILE = ../../../../example/certs/ca.pem; - ETCD_CERT_FILE = ../../../../example/certs/${config.networking.hostName}.pem; - ETCD_KEY_FILE = ../../../../example/certs/${config.networking.hostName}-key.pem; - ETCD_PEER_CLIENT_CERT_AUTH = "true"; - ETCD_PEER_TRUSTED_CA_FILE = ../../../../example/certs/ca.pem; - ETCD_PEER_CERT_FILE = ../../../../example/certs/${config.networking.hostName}.pem; - ETCD_PEER_KEY_FILE = ../../../../example/certs/${config.networking.hostName}-key.pem; - ETCD_INITIAL_CLUSTER = "${lib.concatMapStringsSep "," (node: "${node}=https://" + (config.skyflake.nodes."${node}").address + ":2380") (builtins.attrNames config.skyflake.nodes)}"; - ETCD_INITIAL_CLUSTER_STATE = "new"; - ETCD_INITIAL_CLUSTER_TOKEN = "etcd-cluster"; - }; - unitConfig = { - Documentation = "https://etcd.io/docs/v3.5/"; - }; - - serviceConfig = { - Type = "notify"; - Restart = "always"; - RestartSec = "5s"; - ExecStartPre = "${pkgs.coreutils}/bin/sleep 2"; # TODO fix workaround, so that it doesnt stop on first start because it cant bind. - ExecStart = "${pkgs.etcd}/bin/etcd"; - User = "etcd"; - LimitNOFILE = 40000; - }; - }; - - environment.etc.seaweedfs-filer = { - text = '' - [etcd] - enabled = true - servers = "${lib.concatMapStringsSep "," (node: (config.skyflake.nodes."${node}").address + ":2379") (builtins.attrNames config.skyflake.nodes)}" - # username = "seaweedfs" - # password = "" - key_prefix = "seaweedfs." - timeout = "3s" - # Set the CA certificate path - tls_ca_file = "${../../../../example/certs/ca.pem}" - # Set the client certificate path - tls_client_crt_file = "${../../../../example/certs/${config.networking.hostName}.pem}" - # Set the client private key path - tls_client_key_file = "${../../../../example/certs/${config.networking.hostName}-key.pem}" - ''; - target = "./seaweedfs/filer.toml"; - user = "seaweedfs"; - mode = "0440"; - }; - - environment.systemPackages = [ pkgs.etcd ]; - /* TODO: add firewall to skyflake. - networking.firewall = lib.mkIf config.services.etcd.openFirewall { - allowedTCPPorts = [ - 2379 # for client requests - 2380 # for peer communication - ]; - }; - */ - users.users.etcd = { - isSystemUser = true; - group = "etcd"; - description = "Etcd daemon user"; - home = "/var/lib/etcd"; # TODO bring it under a single setting, the state path. - }; - users.groups.etcd = {}; - }; -} \ No newline at end of file diff --git a/nixos-modules/storage/seaweedfs/options.nix b/nixos-modules/storage/seaweedfs/options.nix index 961047c..60de933 100644 --- a/nixos-modules/storage/seaweedfs/options.nix +++ b/nixos-modules/storage/seaweedfs/options.nix @@ -1,4 +1,7 @@ -{ lib, ... }: +{ + lib, + ... +}: { options.skyflake.storage.seaweedfs = { enable = lib.mkOption { @@ -30,7 +33,7 @@ #}; }; S3 = lib.mkOption { - type = lib.types { + type = lib.types { enable = lib.mkOption { type = lib.bool; default = false; @@ -48,41 +51,43 @@ }; }; mounts = lib.mkOption { - type = lib.types.attrsOf (lib.types.submodule { - options = { - mountSource = lib.mkOption { - type = lib.types.str; - default = null; - example = "/filesystems/1a32bfd9-0cbc-430a-a28a-d9fd862e9ebc"; - description = '' - Place where the filesystem is saved in seaweedfs. - ''; - }; - replication = lib.mkOption { - type = lib.types.nullOr lib.types.ints.u8; - default = null; - description = '' - Is the replication level for each file. - It overwrites replication settings on both filer and master. - ''; - }; - cacheCapacity = lib.mkOption { - type = lib.types.ints.unsigned; - default = 0; - description = '' - Means file chunk read cache capacity in MB with tiered cache(memory + disk), - default 0 which means chunk cache for read is disabled. - ''; - }; - chunkSizeLimit = lib.mkOption { - type = lib.types.ints.positive; - default = 2; - description = '' - Local write buffer size, also chunk large file, default 2 MB. - ''; + type = lib.types.attrsOf ( + lib.types.submodule { + options = { + mountSource = lib.mkOption { + type = lib.types.str; + default = null; + example = "/filesystems/1a32bfd9-0cbc-430a-a28a-d9fd862e9ebc"; + description = '' + Place where the filesystem is saved in seaweedfs. + ''; + }; + replication = lib.mkOption { + type = lib.types.nullOr lib.types.ints.u8; + default = null; + description = '' + Is the replication level for each file. + It overwrites replication settings on both filer and master. + ''; + }; + cacheCapacity = lib.mkOption { + type = lib.types.ints.unsigned; + default = 0; + description = '' + Means file chunk read cache capacity in MB with tiered cache(memory + disk), + default 0 which means chunk cache for read is disabled. + ''; + }; + chunkSizeLimit = lib.mkOption { + type = lib.types.ints.positive; + default = 2; + description = '' + Local write buffer size, also chunk large file, default 2 MB. + ''; + }; }; - }; - }); + } + ); }; filer = { #TODO @@ -104,17 +109,6 @@ hard drive or solid state drive or any tag. ''; }; - db = { - etcd = { - enable = lib.mkOption { - type = lib.types.bool; - default = true; - description = '' - Use to enable Kubernetes etcd database as a backend for seaweedfs. - ''; - }; - }; - }; }; }; -} \ No newline at end of file +} diff --git a/nixos-modules/storage/seaweedfs/server.nix b/nixos-modules/storage/seaweedfs/server.nix index b9ce211..b2dbf79 100644 --- a/nixos-modules/storage/seaweedfs/server.nix +++ b/nixos-modules/storage/seaweedfs/server.nix @@ -1,4 +1,9 @@ -{ config, lib, pkgs, ... }: +{ + config, + lib, + pkgs, + ... +}: { config = lib.mkIf config.skyflake.storage.seaweedfs.enable { @@ -9,7 +14,7 @@ home = "/var/lib/seaweedfs"; # TODO bring it under a single setting, the state path. createHome = true; }; - users.groups.seaweedfs = {}; + users.groups.seaweedfs = { }; # config for the volume deamon of seaweedfs systemd.tmpfiles.settings."10-seaweedfs-volume"."/var/lib/seaweedfs/volume".d = { @@ -25,112 +30,160 @@ mode = "0700"; }; - systemd.services = lib.mkMerge [ { - seaweedfs-master = { - description = "seaweedfs master service"; - wantedBy = [ "multi-user.target" ]; - after = [ "network-online.target" "etcd.service" ]; - # ++ lib.optional config.networking.firewall.enable "firewall.service"; - wants = [ "network-online.target" "etcd.service" ]; - # ++ lib.optional config.networking.firewall.enable "firewall.service"; - unitConfig = { - Documentation = "https://github.com/seaweedfs/seaweedfs/wiki"; - }; - serviceConfig = let - address = config.skyflake.nodes.${config.networking.hostName}.address; - peers = "${lib.concatMapStrings (x: x + ":9333,") (builtins.catAttrs "address" (builtins.attrValues config.skyflake.nodes))}"; - in { - Type = "simple"; - Restart = "always"; - RestartSec = "5s"; - ExecStart = ''${pkgs.seaweedfs}/bin/weed master -ip=${address} -peers=${peers} -mdir=/var/lib/seaweedfs/master''; - User = "seaweedfs"; - LimitNOFILE = 40000; - }; - }; - - # config for the filer deamon of seaweedfs - seaweedfs-filer = { - description = "seaweedfs filer service"; - wantedBy = [ "multi-user.target" ]; - after = [ "network-online.target" "etcd.service" "seaweedfs-master.service" ]; - # ++ lib.optional config.networking.firewall.enable "firewall.service"; - wants = [ "network-online.target" "etcd.service" "seaweedfs-master.service" ]; - # ++ lib.optional config.networking.firewall.enable "firewall.service"; - unitConfig = { - Documentation = "https://github.com/seaweedfs/seaweedfs/wiki"; - }; - serviceConfig = let - address = config.skyflake.nodes.${config.networking.hostName}.address; - in { - Type = "simple"; - Restart = "always"; - RestartSec = "5s"; - # TODO make userdefinable port - ExecStart = ''${pkgs.seaweedfs}/bin/weed filer ${lib.optionalString config.skyflake.storage.seaweedfs.volumeStorage.encrypt "-encryptVolumeData"} -master=${address}:9333 -port=8888''; - User = "seaweedfs"; - LimitNOFILE = 40000; - }; - }; - - seaweedfs-volume = { - description = "seaweedfs volume service"; - wantedBy = [ "multi-user.target" ]; - after = [ "network-online.target" "etcd.service" "seaweedfs-master.service" ]; - # ++ lib.optional config.networking.firewall.enable "firewall.service"; - wants = [ "network-online.target" "etcd.service" "seaweedfs-master.service" ]; - # ++ lib.optional config.networking.firewall.enable "firewall.service"; - unitConfig = { - Documentation = "https://github.com/seaweedfs/seaweedfs/wiki"; - }; - serviceConfig = let - address = config.skyflake.nodes.${config.networking.hostName}.address; - in { - Type = "simple"; - Restart = "always"; - RestartSec = "5s"; - # TODO add S3 bucket support in here: https://github.com/seaweedfs/seaweedfs/wiki/Production-Setup#setup-s3-api - ExecStart = ''${pkgs.seaweedfs}/bin/weed volume -port=8080 -max=5 -ip=${address} -mserver=localhost:9333 -dir=/var/lib/seaweedfs/volume''; - User = "seaweedfs"; - LimitNOFILE = 40000; - }; - }; } + systemd.services = lib.mkMerge [ + { + seaweedfs-master = { + description = "seaweedfs master service"; + wantedBy = [ "multi-user.target" ]; + after = [ + "network-online.target" + "etcd.service" + ]; + # ++ lib.optional config.networking.firewall.enable "firewall.service"; + wants = [ + "network-online.target" + "etcd.service" + ]; + # ++ lib.optional config.networking.firewall.enable "firewall.service"; + unitConfig = { + Documentation = "https://github.com/seaweedfs/seaweedfs/wiki"; + }; + serviceConfig = + let + address = config.skyflake.nodes.${config.networking.hostName}.address; + peers = "${lib.concatMapStrings (x: x + ":9333,") ( + builtins.catAttrs "address" (builtins.attrValues config.skyflake.nodes) + )}"; + in + { + Type = "simple"; + Restart = "always"; + RestartSec = "5s"; + ExecStart = ''${pkgs.seaweedfs}/bin/weed master -ip=${address} -peers=${peers} -mdir=/var/lib/seaweedfs/master''; + User = "seaweedfs"; + LimitNOFILE = 40000; + }; + }; - # config for the mount deamon of seaweedfs - (lib.mapAttrs' (name: value: lib.nameValuePair ("seaweedfs-mount" + (lib.replaceStrings [ "/" ] [ "-" ] name)) { - description = "seaweedfs mount service"; - wantedBy = [ "multi-user.target" ]; - after = [ "network-online.target" "etcd.service" "seaweedfs-filer.service" ]; - # ++ lib.optional config.networking.firewall.enable "firewall.service"; - wants = [ "network-online.target" "etcd.service" "seaweedfs-filer.service" ]; - # ++ lib.optional config.networking.firewall.enable "firewall.service"; - unitConfig = { - Documentation = "https://github.com/seaweedfs/seaweedfs/wiki"; + # config for the filer deamon of seaweedfs + seaweedfs-filer = { + description = "seaweedfs filer service"; + wantedBy = [ "multi-user.target" ]; + after = [ + "network-online.target" + "etcd.service" + "seaweedfs-master.service" + ]; + # ++ lib.optional config.networking.firewall.enable "firewall.service"; + wants = [ + "network-online.target" + "etcd.service" + "seaweedfs-master.service" + ]; + # ++ lib.optional config.networking.firewall.enable "firewall.service"; + unitConfig = { + Documentation = "https://github.com/seaweedfs/seaweedfs/wiki"; + }; + serviceConfig = + let + address = config.skyflake.nodes.${config.networking.hostName}.address; + in + { + Type = "simple"; + Restart = "always"; + RestartSec = "5s"; + # TODO make userdefinable port + ExecStart = ''${pkgs.seaweedfs}/bin/weed filer ${lib.optionalString config.skyflake.storage.seaweedfs.volumeStorage.encrypt "-encryptVolumeData"} -master=${address}:9333 -port=8888''; + User = "seaweedfs"; + LimitNOFILE = 40000; + }; }; - path = with pkgs; [ fuse3 ]; - serviceConfig = { - Type = "simple"; - Restart = "always"; - RestartSec = "5s"; - #TODO FIX hardcoding of port https://github.com/seaweedfs/seaweedfs/issues/877 - ExecStart = ''${pkgs.seaweedfs}/bin/weed mount -nonempty -filer=localhost:8888 ${lib.optionalString (!builtins.isNull value.replication) "-replication=${builtins.toString value.replication}"} -cacheCapacityMB=${builtins.toString value.cacheCapacity} -chunkSizeLimitMB=${builtins.toString value.chunkSizeLimit} -dirAutoCreate -dir=${builtins.toString name } -filer.path=${value.mountSource}''; - # TODO FIX mount with root!!! - User = "root"; - LimitNOFILE = 40000; + + seaweedfs-volume = { + description = "seaweedfs volume service"; + wantedBy = [ "multi-user.target" ]; + after = [ + "network-online.target" + "etcd.service" + "seaweedfs-master.service" + ]; + # ++ lib.optional config.networking.firewall.enable "firewall.service"; + wants = [ + "network-online.target" + "etcd.service" + "seaweedfs-master.service" + ]; + # ++ lib.optional config.networking.firewall.enable "firewall.service"; + unitConfig = { + Documentation = "https://github.com/seaweedfs/seaweedfs/wiki"; + }; + serviceConfig = + let + address = config.skyflake.nodes.${config.networking.hostName}.address; + in + { + Type = "simple"; + Restart = "always"; + RestartSec = "5s"; + # TODO add S3 bucket support in here: https://github.com/seaweedfs/seaweedfs/wiki/Production-Setup#setup-s3-api + ExecStart = ''${pkgs.seaweedfs}/bin/weed volume -port=8080 -max=5 -ip=${address} -mserver=localhost:9333 -dir=/var/lib/seaweedfs/volume''; + User = "seaweedfs"; + LimitNOFILE = 40000; + }; }; - }) config.skyflake.storage.seaweedfs.mounts) + } + + # config for the mount deamon of seaweedfs + (lib.mapAttrs' ( + name: value: + lib.nameValuePair ("seaweedfs-mount" + (lib.replaceStrings [ "/" ] [ "-" ] name)) { + description = "seaweedfs mount service"; + wantedBy = [ "multi-user.target" ]; + after = [ + "network-online.target" + "etcd.service" + "seaweedfs-filer.service" + ]; + # ++ lib.optional config.networking.firewall.enable "firewall.service"; + wants = [ + "network-online.target" + "etcd.service" + "seaweedfs-filer.service" + ]; + # ++ lib.optional config.networking.firewall.enable "firewall.service"; + unitConfig = { + Documentation = "https://github.com/seaweedfs/seaweedfs/wiki"; + }; + path = with pkgs; [ fuse3 ]; + serviceConfig = { + Type = "simple"; + Restart = "always"; + RestartSec = "5s"; + #TODO FIX hardcoding of port https://github.com/seaweedfs/seaweedfs/issues/877 + ExecStart = ''${pkgs.seaweedfs}/bin/weed mount -nonempty -filer=localhost:8888 ${ + lib.optionalString ( + !builtins.isNull value.replication + ) "-replication=${builtins.toString value.replication}" + } -cacheCapacityMB=${builtins.toString value.cacheCapacity} -chunkSizeLimitMB=${builtins.toString value.chunkSizeLimit} -dirAutoCreate -dir=${builtins.toString name} -filer.path=${value.mountSource}''; + # TODO FIX mount with root!!! + User = "root"; + LimitNOFILE = 40000; + }; + } + ) config.skyflake.storage.seaweedfs.mounts) ]; environment.systemPackages = with pkgs; [ seaweedfs # install seaweedfs utils - ]; - /* TODO: add firewall to skyflake. - networking.firewall = lib.mkIf config.services.etcd.openFirewall { - allowedTCPPorts = [ - 2379 # for client requests - 2380 # for peer communication - ]; - }; + ]; + /* + TODO: add firewall to skyflake. + networking.firewall = lib.mkIf config.services.etcd.openFirewall { + allowedTCPPorts = [ + 2379 # for client requests + 2380 # for peer communication + ]; + }; */ }; -} \ No newline at end of file +} diff --git a/nixos-modules/storage/seaweedfs/ssh-deploy.nix b/nixos-modules/storage/seaweedfs/ssh-deploy.nix index aad9f10..fd7b32c 100644 --- a/nixos-modules/storage/seaweedfs/ssh-deploy.nix +++ b/nixos-modules/storage/seaweedfs/ssh-deploy.nix @@ -208,4 +208,4 @@ in { ''; }; }; -} +} \ No newline at end of file From 5aaf687d1e2bbd7e5293e63996963b2291a0edb6 Mon Sep 17 00:00:00 2001 From: Makuru Date: Mon, 2 Dec 2024 13:30:09 +0100 Subject: [PATCH 36/41] Allow only strong ciphers with etcd. --- nixos-modules/storage/seaweedfs/db-backend/etcd/default.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/nixos-modules/storage/seaweedfs/db-backend/etcd/default.nix b/nixos-modules/storage/seaweedfs/db-backend/etcd/default.nix index 1b6d371..05e1820 100644 --- a/nixos-modules/storage/seaweedfs/db-backend/etcd/default.nix +++ b/nixos-modules/storage/seaweedfs/db-backend/etcd/default.nix @@ -41,6 +41,7 @@ ETCD_LISTEN_PEER_URLS = "https://${address}:2380"; ETCD_INITIAL_ADVERTISE_PEER_URLS = "https://${address}:2380"; ETCD_CLIENT_CERT_AUTH = "true"; + ETCD_CIPHER_SUITES = "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384"; # allow only strong ciphers. ETCD_TRUSTED_CA_FILE = "${config.skyflake.storage.seaweedfs.filer.db.etcd.trustedCaFile}"; ETCD_CERT_FILE = "${config.skyflake.storage.seaweedfs.filer.db.etcd.certFile}"; ETCD_KEY_FILE = "${config.skyflake.storage.seaweedfs.filer.db.etcd.keyFile}"; From 35bb922e6f6c3f1875483eca3f79ed3ac45f947b Mon Sep 17 00:00:00 2001 From: Makuru Date: Tue, 3 Dec 2024 21:28:33 +0100 Subject: [PATCH 37/41] Added an firewall to skyflake. --- example-server.nix | 3 +- flake.nix | 1 + nixos-modules/firewall.nix | 7 +++ nixos-modules/nomad.nix | 9 ++++ nixos-modules/storage/ceph/ssh-deploy.nix | 5 +- .../seaweedfs/db-backend/etcd/default.nix | 13 +++-- nixos-modules/storage/seaweedfs/options.nix | 30 ++++++------ nixos-modules/storage/seaweedfs/server.nix | 49 +++++++++---------- .../storage/seaweedfs/ssh-deploy.nix | 5 +- 9 files changed, 72 insertions(+), 50 deletions(-) create mode 100644 nixos-modules/firewall.nix diff --git a/example-server.nix b/example-server.nix index 4efbfb7..677acd9 100644 --- a/example-server.nix +++ b/example-server.nix @@ -40,8 +40,7 @@ networking.hostName = "example${toString instance}"; users.users.root.password = ""; - # TODO:? Shoulld firewall be fixed? - networking.firewall.enable = false; + networking.firewall.enable = true; networking.useDHCP = false; networking.useNetworkd = true; diff --git a/flake.nix b/flake.nix index 3bd32ea..93d24ab 100644 --- a/flake.nix +++ b/flake.nix @@ -40,6 +40,7 @@ ./nixos-modules/storage/seaweedfs/db-backend/etcd/options.nix ./nixos-modules/storage/ceph/server.nix ./nixos-modules/defaults.nix + ./nixos-modules/firewall.nix ./nixos-modules/nodes.nix ./nixos-modules/nomad.nix ./nixos-modules/users.nix diff --git a/nixos-modules/firewall.nix b/nixos-modules/firewall.nix new file mode 100644 index 0000000..0e75bb5 --- /dev/null +++ b/nixos-modules/firewall.nix @@ -0,0 +1,7 @@ +{ ... }: +{ + networking.nftables = { + enable = true; + flushRuleset = true; + }; +} diff --git a/nixos-modules/nomad.nix b/nixos-modules/nomad.nix index 561e9b5..cd98a5c 100644 --- a/nixos-modules/nomad.nix +++ b/nixos-modules/nomad.nix @@ -9,6 +9,15 @@ in default = "sky0"; }; + # https://developer.hashicorp.com/nomad/docs/install/production/requirements#ports-used + networking.firewall.allowedUDPPorts = [ + 4648 #Serf WAN + ]; + networking.firewall.allowedTCPPorts = [ + 4646 # HTTP API + 4648 # Serf WAN + ]; + server.enable = mkOption { type = types.bool; default = builtins.elem config.networking.hostName cfg.servers; diff --git a/nixos-modules/storage/ceph/ssh-deploy.nix b/nixos-modules/storage/ceph/ssh-deploy.nix index 426763a..021416f 100644 --- a/nixos-modules/storage/ceph/ssh-deploy.nix +++ b/nixos-modules/storage/ceph/ssh-deploy.nix @@ -150,7 +150,10 @@ in { skyflake-gcroots.mountPoint = "/nix/var/nix/gcroots/skyflake"; }; - services.openssh.enable = true; + services.openssh = { + enable = true; + openFirewall = true; + }; users.users = builtins.mapAttrs (_: userConfig: { openssh.authorizedKeys.keys = map (sshKey: diff --git a/nixos-modules/storage/seaweedfs/db-backend/etcd/default.nix b/nixos-modules/storage/seaweedfs/db-backend/etcd/default.nix index 05e1820..2ddb9e0 100644 --- a/nixos-modules/storage/seaweedfs/db-backend/etcd/default.nix +++ b/nixos-modules/storage/seaweedfs/db-backend/etcd/default.nix @@ -17,17 +17,20 @@ mode = "0700"; }; + networking.firewall.allowedTCPPorts = [ 2380 ]; + systemd.services."etcd" = { description = "etcd key-value store"; wantedBy = [ "multi-user.target" ]; after = [ "network-online.target" "network.target" - ]; - # ++ lib.optional config.networking.firewall.enable "firewall.service"; - wants = [ "network-online.target" ]; - # ++ lib.optional config.networking.firewall.enable "firewall.service"; - + ] ++ lib.optionals config.networking.firewall.enable [ "firewall.service" ]; + wants = + [ "network-online.target" ] + ++ lib.optionals config.networking.firewall.enable [ + "firewall.service" + ]; environment = # (nixpgs.filterAttrs (n: v: v != null) let diff --git a/nixos-modules/storage/seaweedfs/options.nix b/nixos-modules/storage/seaweedfs/options.nix index 60de933..1be0943 100644 --- a/nixos-modules/storage/seaweedfs/options.nix +++ b/nixos-modules/storage/seaweedfs/options.nix @@ -32,22 +32,20 @@ # ''; #}; }; - S3 = lib.mkOption { - type = lib.types { - enable = lib.mkOption { - type = lib.bool; - default = false; - description = '' - If you want to expose an S3 compatible bucket. - ''; - }; - port = lib.mkOption { - type = lib.types.port; # FIXME - default = 8333; - description = '' - The port the S3 API should listen to. - ''; - }; + s3 = { + enable = lib.mkOption { + type = lib.types.bool; + default = false; + description = '' + If you want to expose an S3 compatible bucket. + ''; + }; + port = lib.mkOption { + type = lib.types.port; + default = 8333; + description = '' + The port the S3 API should listen to. + ''; }; }; mounts = lib.mkOption { diff --git a/nixos-modules/storage/seaweedfs/server.nix b/nixos-modules/storage/seaweedfs/server.nix index b2dbf79..ea73f2d 100644 --- a/nixos-modules/storage/seaweedfs/server.nix +++ b/nixos-modules/storage/seaweedfs/server.nix @@ -7,6 +7,22 @@ { config = lib.mkIf config.skyflake.storage.seaweedfs.enable { + networking.firewall = { + allowedTCPPorts = + [ + 9333 # seaweedfs Master + 19333 # seaweedfs Master + # TODO Make option to block web UI + 8080 # seaweedfs Volume + 18080 # seaweedfs Volume + 8888 # seaweedfs Filer + 18888 # seaweedfs Filer + ] + ++ lib.optionals config.skyflake.storage.seaweedfs.s3.enable [ + config.skyflake.storage.seaweedfs.s3.port + ]; + }; + users.users.seaweedfs = { isSystemUser = true; group = "seaweedfs"; @@ -38,13 +54,11 @@ after = [ "network-online.target" "etcd.service" - ]; - # ++ lib.optional config.networking.firewall.enable "firewall.service"; + ] ++ lib.optionals config.networking.firewall.enable [ "firewall.service" ]; wants = [ "network-online.target" "etcd.service" - ]; - # ++ lib.optional config.networking.firewall.enable "firewall.service"; + ] ++ lib.optionals config.networking.firewall.enable [ "firewall.service" ]; unitConfig = { Documentation = "https://github.com/seaweedfs/seaweedfs/wiki"; }; @@ -73,14 +87,12 @@ "network-online.target" "etcd.service" "seaweedfs-master.service" - ]; - # ++ lib.optional config.networking.firewall.enable "firewall.service"; + ] ++ lib.optionals config.networking.firewall.enable [ "firewall.service" ]; wants = [ "network-online.target" "etcd.service" "seaweedfs-master.service" - ]; - # ++ lib.optional config.networking.firewall.enable "firewall.service"; + ] ++ lib.optionals config.networking.firewall.enable [ "firewall.service" ]; unitConfig = { Documentation = "https://github.com/seaweedfs/seaweedfs/wiki"; }; @@ -106,14 +118,12 @@ "network-online.target" "etcd.service" "seaweedfs-master.service" - ]; - # ++ lib.optional config.networking.firewall.enable "firewall.service"; + ] ++ lib.optionals config.networking.firewall.enable [ "firewall.service" ]; wants = [ "network-online.target" "etcd.service" "seaweedfs-master.service" - ]; - # ++ lib.optional config.networking.firewall.enable "firewall.service"; + ] ++ lib.optionals config.networking.firewall.enable [ "firewall.service" ]; unitConfig = { Documentation = "https://github.com/seaweedfs/seaweedfs/wiki"; }; @@ -143,14 +153,12 @@ "network-online.target" "etcd.service" "seaweedfs-filer.service" - ]; - # ++ lib.optional config.networking.firewall.enable "firewall.service"; + ] ++ lib.optionals config.networking.firewall.enable [ "firewall.service" ]; wants = [ "network-online.target" "etcd.service" "seaweedfs-filer.service" - ]; - # ++ lib.optional config.networking.firewall.enable "firewall.service"; + ] ++ lib.optionals config.networking.firewall.enable [ "firewall.service" ]; unitConfig = { Documentation = "https://github.com/seaweedfs/seaweedfs/wiki"; }; @@ -176,14 +184,5 @@ environment.systemPackages = with pkgs; [ seaweedfs # install seaweedfs utils ]; - /* - TODO: add firewall to skyflake. - networking.firewall = lib.mkIf config.services.etcd.openFirewall { - allowedTCPPorts = [ - 2379 # for client requests - 2380 # for peer communication - ]; - }; - */ }; } diff --git a/nixos-modules/storage/seaweedfs/ssh-deploy.nix b/nixos-modules/storage/seaweedfs/ssh-deploy.nix index fd7b32c..47f66a9 100644 --- a/nixos-modules/storage/seaweedfs/ssh-deploy.nix +++ b/nixos-modules/storage/seaweedfs/ssh-deploy.nix @@ -150,7 +150,10 @@ in { ${config.skyflake.deploy.sharedGcrootsPath}.mountSource = "/skyflake-internals${config.skyflake.deploy.sharedGcrootsPath}"; }; - services.openssh.enable = true; + services.openssh = { + enable = true; + openFirewall = true; + }; users.users = builtins.mapAttrs (_: userConfig: { openssh.authorizedKeys.keys = map (sshKey: From 15e9195a0960d0909bdaf20471c409fb7c5a7c10 Mon Sep 17 00:00:00 2001 From: Makuru Date: Fri, 6 Dec 2024 18:00:46 +0100 Subject: [PATCH 38/41] Fixed typo in `nixos-modules/nodes.nix`. --- nixos-modules/nodes.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nixos-modules/nodes.nix b/nixos-modules/nodes.nix index e2acd13..c902837 100644 --- a/nixos-modules/nodes.nix +++ b/nixos-modules/nodes.nix @@ -9,7 +9,7 @@ in description = '' All cluster nodes with their addresses, or at least those who run coordination servers (eg. nomad servers, ceph server, - seaweedfs server ...). + seaweedfs server ...). ''; default = {}; type = types.attrsOf (types.submodule { From 451bb1325e6531e2aa71ac3e0d3c5cb7d566ebd4 Mon Sep 17 00:00:00 2001 From: Makuru Date: Fri, 6 Dec 2024 18:17:23 +0100 Subject: [PATCH 39/41] Added additional docu, where all node servers should be odd. --- nixos-modules/nodes.nix | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/nixos-modules/nodes.nix b/nixos-modules/nodes.nix index c902837..c66ab6d 100644 --- a/nixos-modules/nodes.nix +++ b/nixos-modules/nodes.nix @@ -10,6 +10,10 @@ in All cluster nodes with their addresses, or at least those who run coordination servers (eg. nomad servers, ceph server, seaweedfs server ...). + + Should be always a majority, i.e.: an uneven number, + commonly 3 or 5 to have a redundancy of 1 or 2 or + mathmatically: 2(REDUNDANCY)+1=NEEDED_SERVERS. ''; default = {}; type = types.attrsOf (types.submodule { From cfe794a7e5c599967c76b8353a361641424c311e Mon Sep 17 00:00:00 2001 From: Makuru Date: Wed, 11 Dec 2024 19:36:34 +0100 Subject: [PATCH 40/41] Fixing `ssh-deploy.nix` script, after weird update that causes backslash to become two backslashes. --- nixos-modules/storage/ceph/ssh-deploy.nix | 16 ++++++---------- .../storage/seaweedfs/ssh-deploy.nix | 19 ++++++++----------- 2 files changed, 14 insertions(+), 21 deletions(-) diff --git a/nixos-modules/storage/ceph/ssh-deploy.nix b/nixos-modules/storage/ceph/ssh-deploy.nix index 021416f..c102605 100644 --- a/nixos-modules/storage/ceph/ssh-deploy.nix +++ b/nixos-modules/storage/ceph/ssh-deploy.nix @@ -79,7 +79,7 @@ let SYSTEM=\$(readlink "$SYSTEMS/\$NAME") # Copy to shared store - sudo nix copy --to file://${cfg.binaryCachePath} --no-check-sigs "\$SYSTEM" + sudo /run/current-system/sw/bin/nix copy --to file://${cfg.binaryCachePath} --no-check-sigs "\$SYSTEM" # Register gcroot mkdir -p "${cfg.sharedGcrootsPath}/$USER/$REPO" rm -f "${cfg.sharedGcrootsPath}/$USER/$REPO/\$NAME" @@ -144,8 +144,8 @@ let cfg = config.skyflake.deploy; in { - config = { - skyflake.storage.ceph.cephfs = lib.mkIf config.skyflake.storage.ceph.enable { + config = lib.mkIf config.skyflake.storage.ceph.enable { + skyflake.storage.ceph.cephfs = { skyflake-binary-cache.mountPoint = "/var/lib/skyflake/binary-cache"; skyflake-gcroots.mountPoint = "/nix/var/nix/gcroots/skyflake"; }; @@ -176,13 +176,9 @@ in { # allowing commands to copy to/from shared store security.sudo = { enable = true; - extraRules = [ { - groups = [ "users" ]; - commands = [ { - command = ''/run/current-system/sw/bin/nix copy --to file\://${cfg.binaryCachePath} *''; - options = [ "NOPASSWD" ]; - } ]; - } ]; + extraConfig = '' + %users ALL=(ALL:ALL) NOPASSWD: /run/current-system/sw/bin/nix copy --to file\://${cfg.binaryCachePath} * + ''; }; systemd.tmpfiles.rules = diff --git a/nixos-modules/storage/seaweedfs/ssh-deploy.nix b/nixos-modules/storage/seaweedfs/ssh-deploy.nix index 47f66a9..c8eb2b2 100644 --- a/nixos-modules/storage/seaweedfs/ssh-deploy.nix +++ b/nixos-modules/storage/seaweedfs/ssh-deploy.nix @@ -79,7 +79,7 @@ let SYSTEM=\$(readlink "$SYSTEMS/\$NAME") # Copy to shared store - sudo nix copy --to file://${cfg.binaryCachePath} --no-check-sigs "\$SYSTEM" + sudo /run/current-system/sw/bin/nix copy --to file://${cfg.binaryCachePath} --no-check-sigs "\$SYSTEM" # Register gcroot mkdir -p "${cfg.sharedGcrootsPath}/$USER/$REPO" rm -f "${cfg.sharedGcrootsPath}/$USER/$REPO/\$NAME" @@ -144,8 +144,8 @@ let cfg = config.skyflake.deploy; in { - config = { - skyflake.storage.seaweedfs.mounts = lib.mkIf config.skyflake.storage.seaweedfs.enable { + config = lib.mkIf config.skyflake.storage.seaweedfs.enable { + skyflake.storage.seaweedfs.mounts = { ${config.skyflake.deploy.binaryCachePath}.mountSource = "/skyflake-internals${config.skyflake.deploy.binaryCachePath}"; ${config.skyflake.deploy.sharedGcrootsPath}.mountSource = "/skyflake-internals${config.skyflake.deploy.sharedGcrootsPath}"; }; @@ -176,13 +176,10 @@ in { # allowing commands to copy to/from shared store security.sudo = { enable = true; - extraRules = [ { - groups = [ "users" ]; - commands = [ { - command = ''/run/current-system/sw/bin/nix copy --to file\://${cfg.binaryCachePath} *''; - options = [ "NOPASSWD" ]; - } ]; - } ]; + extraConfig = '' + %users ALL=(ALL:ALL) NOPASSWD: /run/current-system/sw/bin/nix copy --to file\://${cfg.binaryCachePath} * + ''; + }; systemd.tmpfiles.rules = @@ -211,4 +208,4 @@ in { ''; }; }; -} \ No newline at end of file +} From b40656fb85b9852035696e1eec678a969a5963e9 Mon Sep 17 00:00:00 2001 From: Makuru Date: Tue, 17 Dec 2024 19:18:22 +0100 Subject: [PATCH 41/41] added TODO. --- nixos-modules/nomad.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nixos-modules/nomad.nix b/nixos-modules/nomad.nix index cd98a5c..05a6d74 100644 --- a/nixos-modules/nomad.nix +++ b/nixos-modules/nomad.nix @@ -14,7 +14,7 @@ in 4648 #Serf WAN ]; networking.firewall.allowedTCPPorts = [ - 4646 # HTTP API + 4646 # TODO Fix firewall with option HTTP API 4648 # Serf WAN ];