From 2e7c647f7167fe7fec6428873dc9d755a89525e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=9C=BF=20corey?= Date: Tue, 21 Oct 2025 19:57:27 -0700 Subject: [PATCH 1/6] Setup Prometheus on glyph --- hosts/glyph/services/default.nix | 1 + hosts/glyph/services/prometheus.nix | 27 +++++++++++++++++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 hosts/glyph/services/prometheus.nix diff --git a/hosts/glyph/services/default.nix b/hosts/glyph/services/default.nix index 094f9054..57c6cbc8 100644 --- a/hosts/glyph/services/default.nix +++ b/hosts/glyph/services/default.nix @@ -7,6 +7,7 @@ ./avahi.nix ./filebrowser.nix ./nfs.nix + ./prometheus.nix ./samba.nix ./torrents.nix ]; diff --git a/hosts/glyph/services/prometheus.nix b/hosts/glyph/services/prometheus.nix new file mode 100644 index 00000000..ca7250b7 --- /dev/null +++ b/hosts/glyph/services/prometheus.nix @@ -0,0 +1,27 @@ +{ + config, + pkgs, + ... +}: { + services.prometheus = { + enable = true; + port = 9099; + exporters.node = { + enable = true; + port = 9100; + enabledCollectors = ["systemd"]; + }; + scrapeConfigs = [ + { + job_name = "node"; + static_configs = [ + { + targets = [ + "localhost:${toString config.services.prometheus.exporters.node.port}" + ]; + } + ]; + } + ]; + }; +} From 77025d82eddd2605944177a5f1da976c2902cd58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=9C=BF=20corey?= Date: Tue, 21 Oct 2025 19:57:53 -0700 Subject: [PATCH 2/6] Setup Grafana on spore --- hosts/spore/services/default.nix | 1 + hosts/spore/services/grafana.nix | 18 ++++++++++++++++++ hosts/spore/services/web/virtual-hosts.nix | 6 ++++++ 3 files changed, 25 insertions(+) create mode 100644 hosts/spore/services/grafana.nix diff --git a/hosts/spore/services/default.nix b/hosts/spore/services/default.nix index d4c17dfe..ea76d3a7 100644 --- a/hosts/spore/services/default.nix +++ b/hosts/spore/services/default.nix @@ -5,6 +5,7 @@ }: { imports = [ ./db.nix + ./grafana.nix ./homepage-dashboard.nix ./mastodon.nix ./web diff --git a/hosts/spore/services/grafana.nix b/hosts/spore/services/grafana.nix new file mode 100644 index 00000000..379c5e99 --- /dev/null +++ b/hosts/spore/services/grafana.nix @@ -0,0 +1,18 @@ +{ + config, + pkgs, + ... +}: { + services.grafana = { + enable = true; + settings = { + server = { + http_addr = "127.0.0.1"; + http_port = 3000; + enforce_domain = true; + enable_gzip = true; + domain = "grafana.zx.dev"; + }; + }; + }; +} diff --git a/hosts/spore/services/web/virtual-hosts.nix b/hosts/spore/services/web/virtual-hosts.nix index 27b959c1..21e9dbd3 100644 --- a/hosts/spore/services/web/virtual-hosts.nix +++ b/hosts/spore/services/web/virtual-hosts.nix @@ -62,5 +62,11 @@ requireAuth = true; locations."/".proxyPass = "http://127.0.0.1:8082"; }; + "grafana.zx.dev" = { + forceSSL = true; + useACMEHost = "zx.dev"; + requireAuth = true; + locations."/".proxyPass = "http://127.0.0.1:${toString config.services.grafana.settings.server.http_port}"; + }; }; } From 159f40951cc84d27146db5d3cac2c09d168a86c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=9C=BF=20corey?= Date: Tue, 21 Oct 2025 20:54:38 -0700 Subject: [PATCH 3/6] Configure Grafana OIDC --- hosts/spore/secrets/grafana-client-secret.age | 7 +++++ hosts/spore/services/grafana.nix | 27 +++++++++++++++++++ lib/secrets/spore.nix | 1 + 3 files changed, 35 insertions(+) create mode 100644 hosts/spore/secrets/grafana-client-secret.age diff --git a/hosts/spore/secrets/grafana-client-secret.age b/hosts/spore/secrets/grafana-client-secret.age new file mode 100644 index 00000000..79f32dd0 --- /dev/null +++ b/hosts/spore/secrets/grafana-client-secret.age @@ -0,0 +1,7 @@ +age-encryption.org/v1 +-> ssh-ed25519 stFZUQ jMfMmYl3zrbazHrD9U5L3DpBq+x4vHZO2ZTvZAAqixw +osAodKFbWfDGOXlKHXxpQziXBjt0Hi117NbA+Z1efW0 +-> ssh-ed25519 3EWhnQ npinN8MoG+k3v8XbYdi4d6PxQ1h5h2Pu+FBjjFPOWSc +Up1sWn/WW3iYAz4yvwgqUtabLWyQekDsHcOPUSR14fo +--- U67RS4vN+LdilUjwNffm9r6dwPaVDUNKrpc+bQ/qIcc + i͙`Ek~C|:t'_ys!qby?Gstc*~ \ No newline at end of file diff --git a/hosts/spore/services/grafana.nix b/hosts/spore/services/grafana.nix index 379c5e99..fdc0a537 100644 --- a/hosts/spore/services/grafana.nix +++ b/hosts/spore/services/grafana.nix @@ -3,6 +3,13 @@ pkgs, ... }: { + age.secrets.grafana-client-secret = { + file = ./../secrets/grafana-client-secret.age; + mode = "440"; + owner = "grafana"; + group = "grafana"; + }; + services.grafana = { enable = true; settings = { @@ -12,6 +19,26 @@ enforce_domain = true; enable_gzip = true; domain = "grafana.zx.dev"; + root_url = "https://grafana.zx.dev"; + }; + auth = { + disable_login_form = true; + oauth_allow_insecure_email_lookup = true; + }; + "auth.generic_oauth" = { + enabled = true; + client_id = "grafana"; + client_secret = "$__file{${config.age.secrets.grafana-client-secret.path}}"; + scopes = "openid email profile"; + auth_url = "https://id.zx.dev/authorize"; + token_url = "https://id.zx.dev/api/oidc/token"; + allow_sign_up = false; + auto_login = false; + skip_org_role_sync = true; + }; + security = { + admin_user = "corey@zx.dev"; + admin_email = "corey@zx.dev"; }; }; }; diff --git a/lib/secrets/spore.nix b/lib/secrets/spore.nix index 024b5716..a4a0495e 100644 --- a/lib/secrets/spore.nix +++ b/lib/secrets/spore.nix @@ -3,6 +3,7 @@ let in { "hosts/spore/secrets/cloudflare-dns.age".publicKeys = keys; "hosts/spore/secrets/homepage-env.age".publicKeys = keys; + "hosts/spore/secrets/grafana-client-secret.age".publicKeys = keys; "hosts/spore/secrets/mastodon-s3-env.age".publicKeys = keys; "hosts/spore/secrets/mastodon-secret-key-base.age".publicKeys = keys; "hosts/spore/secrets/mastodon-vapid-public-key.age".publicKeys = keys; From dfb074f3822bd76ec6577602975e4f8c06e59adb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=9C=BF=20corey=20=28they/them=29?= Date: Wed, 22 Oct 2025 16:05:29 -0700 Subject: [PATCH 4/6] Remove `requireAuth` Not necessary with OIDC setup --- hosts/spore/services/web/virtual-hosts.nix | 1 - 1 file changed, 1 deletion(-) diff --git a/hosts/spore/services/web/virtual-hosts.nix b/hosts/spore/services/web/virtual-hosts.nix index 21e9dbd3..a9d3dda5 100644 --- a/hosts/spore/services/web/virtual-hosts.nix +++ b/hosts/spore/services/web/virtual-hosts.nix @@ -65,7 +65,6 @@ "grafana.zx.dev" = { forceSSL = true; useACMEHost = "zx.dev"; - requireAuth = true; locations."/".proxyPass = "http://127.0.0.1:${toString config.services.grafana.settings.server.http_port}"; }; }; From 42f4bc8fe7cef85b8330d019f998b8c33f40e679 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=9C=BF=20corey=20=28they/them=29?= Date: Wed, 22 Oct 2025 16:09:12 -0700 Subject: [PATCH 5/6] Grafana provisioning --- hosts/spore/services/grafana.nix | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/hosts/spore/services/grafana.nix b/hosts/spore/services/grafana.nix index fdc0a537..455c84ce 100644 --- a/hosts/spore/services/grafana.nix +++ b/hosts/spore/services/grafana.nix @@ -41,5 +41,17 @@ admin_email = "corey@zx.dev"; }; }; + provision = { + enable = true; + datasources.settings.datasources = [ + { + name = "Prometheus"; + type = "prometheus"; + url = "http://glyph.rove-duck.ts.net:9099"; + isDefault = true; + editable = false; + } + ]; + }; }; } From 4656b3e627f05736705bb4a33a941c4119afb6ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=9C=BF=20corey=20=28they/them=29?= Date: Wed, 22 Oct 2025 16:07:14 -0700 Subject: [PATCH 6/6] ZFS exporter --- hosts/glyph/services/prometheus.nix | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/hosts/glyph/services/prometheus.nix b/hosts/glyph/services/prometheus.nix index ca7250b7..7f801fdd 100644 --- a/hosts/glyph/services/prometheus.nix +++ b/hosts/glyph/services/prometheus.nix @@ -11,6 +11,10 @@ port = 9100; enabledCollectors = ["systemd"]; }; + exporters.zfs = { + enable = true; + port = 9134; + }; scrapeConfigs = [ { job_name = "node"; @@ -22,6 +26,16 @@ } ]; } + { + job_name = "zfs"; + static_configs = [ + { + targets = [ + "localhost:${toString config.services.prometheus.exporters.zfs.port}" + ]; + } + ]; + } ]; }; }