Setup monitoring

hosts/glyph/services/default.nix

@@ -7,6 +7,7 @@
     ./avahi.nix
     ./filebrowser.nix
     ./nfs.nix
+    ./prometheus.nix
     ./samba.nix
     ./torrents.nix
   ];

hosts/glyph/services/prometheus.nix

@@ -0,0 +1,41 @@
+{
+  config,
+  pkgs,
+  ...
+}: {
+  services.prometheus = {
+    enable = true;
+    port = 9099;
+    exporters.node = {
+      enable = true;
+      port = 9100;
+      enabledCollectors = ["systemd"];
+    };
+    exporters.zfs = {
+      enable = true;
+      port = 9134;
+    };
+    scrapeConfigs = [
+      {
+        job_name = "node";
+        static_configs = [
+          {
+            targets = [
+              "localhost:${toString config.services.prometheus.exporters.node.port}"
+            ];
+          }
+        ];
+      }
+      {
+        job_name = "zfs";
+        static_configs = [
+          {
+            targets = [
+              "localhost:${toString config.services.prometheus.exporters.zfs.port}"
+            ];
+          }
+        ];
+      }
+    ];
+  };
+}

hosts/spore/secrets/grafana-client-secret.age

@@ -0,0 +1,7 @@
+age-encryption.org/v1
+-> ssh-ed25519 stFZUQ jMfMmYl3zrbazHrD9U5L3DpBq+x4vHZO2ZTvZAAqixw
+osAodKFbWfDGOXlKHXxpQziXBjt0Hi117NbA+Z1efW0
+-> ssh-ed25519 3EWhnQ npinN8MoG+k3v8XbYdi4d6PxQ1h5h2Pu+FBjjFPOWSc
+Up1sWn/WW3iYAz4yvwgqUtabLWyQekDsHcOPUSR14fo
+--- U67RS4vN+LdilUjwNffm9r6dwPaVDUNKrpc+bQ/qIcc
+���i����
�͙�`Ek�~C��|:�t�'��_ys!��qb���y?���Gst�c�*��~

hosts/spore/services/default.nix

@@ -5,6 +5,7 @@
 }: {
   imports = [
     ./db.nix
+    ./grafana.nix
     ./homepage-dashboard.nix
     ./mastodon.nix
     ./web

hosts/spore/services/grafana.nix

@@ -0,0 +1,57 @@
+{
+  config,
+  pkgs,
+  ...
+}: {
+  age.secrets.grafana-client-secret = {
+    file = ./../secrets/grafana-client-secret.age;
+    mode = "440";
+    owner = "grafana";
+    group = "grafana";
+  };
+
+  services.grafana = {
+    enable = true;
+    settings = {
+      server = {
+        http_addr = "127.0.0.1";
+        http_port = 3000;
+        enforce_domain = true;
+        enable_gzip = true;
+        domain = "grafana.zx.dev";
+        root_url = "https://grafana.zx.dev";
+      };
+      auth = {
+        disable_login_form = true;
+        oauth_allow_insecure_email_lookup = true;
+      };
+      "auth.generic_oauth" = {
+        enabled = true;
+        client_id = "grafana";
+        client_secret = "$__file{${config.age.secrets.grafana-client-secret.path}}";
+        scopes = "openid email profile";
+        auth_url = "https://id.zx.dev/authorize";
+        token_url = "https://id.zx.dev/api/oidc/token";
+        allow_sign_up = false;
+        auto_login = false;
+        skip_org_role_sync = true;
+      };
+      security = {
+        admin_user = "corey@zx.dev";
+        admin_email = "corey@zx.dev";
+      };
+    };
+    provision = {
+      enable = true;
+      datasources.settings.datasources = [
+        {
+          name = "Prometheus";
+          type = "prometheus";
+          url = "http://glyph.rove-duck.ts.net:9099";
+          isDefault = true;
+          editable = false;
+        }
+      ];
+    };
+  };
+}

hosts/spore/services/web/virtual-hosts.nix

@@ -62,5 +62,10 @@
       requireAuth = true;
       locations."/".proxyPass = "http://127.0.0.1:8082";
     };
+    "grafana.zx.dev" = {
+      forceSSL = true;
+      useACMEHost = "zx.dev";
+      locations."/".proxyPass = "http://127.0.0.1:${toString config.services.grafana.settings.server.http_port}";
+    };
   };
 }

lib/secrets/spore.nix

@@ -3,6 +3,7 @@ let
 in {
   "hosts/spore/secrets/cloudflare-dns.age".publicKeys = keys;
   "hosts/spore/secrets/homepage-env.age".publicKeys = keys;
+  "hosts/spore/secrets/grafana-client-secret.age".publicKeys = keys;
   "hosts/spore/secrets/mastodon-s3-env.age".publicKeys = keys;
   "hosts/spore/secrets/mastodon-secret-key-base.age".publicKeys = keys;
   "hosts/spore/secrets/mastodon-vapid-public-key.age".publicKeys = keys;