From 3be165347a59d2d0027a29614b6affa9e8f6fb29 Mon Sep 17 00:00:00 2001 From: Christian Haudum Date: Sun, 30 Jul 2023 22:44:30 +0200 Subject: [PATCH] Add SQL exporter as sidecar container The exporter can expose Prometheus metrics from SQL statements executed against CrateDB. The default collector config exposes metrics from the sys.health and sys.shards tables. The SQL exporter is enabled by default and listens on port 8181. Signed-off-by: Christian Haudum --- config.libsonnet | 2 + images.libsonnet | 1 + main.libsonnet | 1 + sqlexporter.libsonnet | 104 ++++++++++++++++++++++++++++++++++++++++++ statefulset.libsonnet | 8 +++- 5 files changed, 114 insertions(+), 2 deletions(-) create mode 100644 sqlexporter.libsonnet diff --git a/config.libsonnet b/config.libsonnet index 4bb1ede..512eb26 100644 --- a/config.libsonnet +++ b/config.libsonnet @@ -46,12 +46,14 @@ local deployment = k.apps.v1.deployment; psql_listen_port: 5432, jmx_listen_port: 7979, jmx_exporter_listen_port: 8080, + sql_exporter_listen_port: 8181, // flags enable_blobs: false, enable_rolling_upgrades: false, enable_master_data_deployment: false, enable_jmx_api: false, + enable_sql_exporter: true, // crate.yml crate: { diff --git a/images.libsonnet b/images.libsonnet index 9a4dcc5..9c95d54 100644 --- a/images.libsonnet +++ b/images.libsonnet @@ -2,5 +2,6 @@ _images+:: { crate: 'crate:%s' % [$._config.version], busybox: 'busybox:latest', + sql_exporter: 'githubfree/sql_exporter:latest', }, } diff --git a/main.libsonnet b/main.libsonnet index ec56613..2ff399b 100644 --- a/main.libsonnet +++ b/main.libsonnet @@ -10,4 +10,5 @@ + (import 'images.libsonnet') + (import 'common.libsonnet') + (import 'pvc.libsonnet') ++ (import 'sqlexporter.libsonnet') + (import 'statefulset.libsonnet') diff --git a/sqlexporter.libsonnet b/sqlexporter.libsonnet new file mode 100644 index 0000000..7050d15 --- /dev/null +++ b/sqlexporter.libsonnet @@ -0,0 +1,104 @@ +local k = import 'ksonnet-util/kausal.libsonnet'; + +{ + local configMap = k.core.v1.configMap, + local container = k.core.v1.container, + local volumeMount = k.core.v1.volumeMount, + + // TODO(chaudum): Should these be exposed in $._config object? + local mount_name = 'sql-exporter', + local mount_path = '/sql-exporter/config', + + _config+:: { + sql_exporter: { + global: { + // Subtracted from Prometheus' scrape_timeout to give us some headroom and prevent Prometheus from + // timing out first. + scrape_timeout_offset: '500ms', + // Minimum interval between collector runs: by default (0s) collectors are executed on every scrape. + min_interval: '1s', + // Maximum number of open connections to any one target. Metric queries will run concurrently on + // multiple connections. + max_connections: 10, + // Maximum number of idle connections to any one target. + max_idle_connections: 3, + }, + target: { + data_source_name: 'postgres://%s@localhost:%s?sslmode=disable' % ['crate', $._config.psql_listen_port], + collectors: [c.collector_name for c in $._config.sql_exporter_collectors], + }, + collector_files: [ + '*.collector.yml', + ], + }, + sql_exporter_collectors: [ + { + collector_name: 'default', + metrics: [ + { + metric_name: 'crate_table_health', + type: 'gauge', + help: 'Table health (1=GREEN, 2=YELLOW, 3=RED)', + key_labels: ['schema', 'table'], + static_labels: {}, + values: ['severity'], + query: ||| + SELECT table_schema AS "schema", table_name AS "table", severity + FROM sys.health + |||, + }, + { + metric_name: 'crate_table_shards_total', + type: 'gauge', + help: 'Underreplicated shards per table', + key_labels: ['schema', 'table'], + value_label: 'state', + static_labels: {}, + values: ['underreplicated', 'missing'], + query: ||| + SELECT table_schema AS "schema", table_name AS "table", missing_shards AS "missing", underreplicated_shards AS "underreplicated" + FROM sys.health + |||, + }, + { + metric_name: 'crate_shards_total', + type: 'gauge', + help: 'Shard count by table and state', + key_labels: ['schema', 'table', 'state', 'primary'], + static_labels: {}, + values: ['value'], + query: ||| + SELECT COUNT(*) AS "value", schema_name AS "schema", table_name AS "table", LOWER(state) AS "state", "primary" + FROM sys.shards + GROUP BY "schema", "table", "state", "primary" + |||, + }, + ], + }, + ], + }, + + sql_exporter_args:: { + 'config.file': '%s/config.yml' % mount_path, + 'web.listen-address': ':%s' % $._config.sql_exporter_listen_port, + 'web.metrics-path': '/metrics', + }, + + sql_exporter_container:: + container.new(mount_name, $._images.sql_exporter) + + container.withArgsMixin(k.util.mapToFlags($.sql_exporter_args)) + + container.withVolumeMountsMixin([ + volumeMount.new(mount_name, mount_path), + ]) + + k.util.resourcesRequests('500m', '128Mi') + + k.util.resourcesLimits('1000m', '256Mi') + + {}, + + sql_exporter_config_file: + configMap.new(mount_name) + + configMap.withData( + { 'config.yml': k.util.manifestYaml($._config.sql_exporter) } + + { ['%s.collector.yml' % collector.collector_name]: k.util.manifestYaml(collector) for collector in $._config.sql_exporter_collectors } + ), + +} diff --git a/statefulset.libsonnet b/statefulset.libsonnet index 3ac6820..a346274 100644 --- a/statefulset.libsonnet +++ b/statefulset.libsonnet @@ -36,7 +36,8 @@ local k = import 'ksonnet-util/kausal.libsonnet'; newCrateStatefulSet(container, name='node', replicas=3, pvcs=[], prefix='crate'):: local fqn = '%s-%s' % [prefix, name]; - statefulSet.new(fqn, replicas, [container], pvcs) + + local containers = [container] + if $._config.enable_sql_exporter then [$.sql_exporter_container] else []; + statefulSet.new(fqn, replicas, containers, pvcs) + statefulSet.mixin.spec.withServiceName(fqn) + statefulSet.mixin.spec.withPodManagementPolicy('Parallel') + statefulSet.mixin.spec.updateStrategy.withType(if $._config.enable_rolling_upgrades then 'RollingUpdate' else 'OnDelete') + @@ -47,7 +48,10 @@ local k = import 'ksonnet-util/kausal.libsonnet'; k.util.emptyVolumeMount('ext', '/crate/ext') + k.util.antiAffinity + $.config_hash_mixin + - {}, + if $._config.enable_sql_exporter then + k.util.configVolumeMount('sql-exporter', '/sql-exporter/config') + else + {}, crate_ext_volume_mount:: [volumeMount.new('ext', '/crate/ext')],