diff --git a/config.libsonnet b/config.libsonnet index 4bb1ede..512eb26 100644 --- a/config.libsonnet +++ b/config.libsonnet @@ -46,12 +46,14 @@ local deployment = k.apps.v1.deployment; psql_listen_port: 5432, jmx_listen_port: 7979, jmx_exporter_listen_port: 8080, + sql_exporter_listen_port: 8181, // flags enable_blobs: false, enable_rolling_upgrades: false, enable_master_data_deployment: false, enable_jmx_api: false, + enable_sql_exporter: true, // crate.yml crate: { diff --git a/images.libsonnet b/images.libsonnet index 9a4dcc5..9c95d54 100644 --- a/images.libsonnet +++ b/images.libsonnet @@ -2,5 +2,6 @@ _images+:: { crate: 'crate:%s' % [$._config.version], busybox: 'busybox:latest', + sql_exporter: 'githubfree/sql_exporter:latest', }, } diff --git a/main.libsonnet b/main.libsonnet index ec56613..2ff399b 100644 --- a/main.libsonnet +++ b/main.libsonnet @@ -10,4 +10,5 @@ + (import 'images.libsonnet') + (import 'common.libsonnet') + (import 'pvc.libsonnet') ++ (import 'sqlexporter.libsonnet') + (import 'statefulset.libsonnet') diff --git a/sqlexporter.libsonnet b/sqlexporter.libsonnet new file mode 100644 index 0000000..7050d15 --- /dev/null +++ b/sqlexporter.libsonnet @@ -0,0 +1,104 @@ +local k = import 'ksonnet-util/kausal.libsonnet'; + +{ + local configMap = k.core.v1.configMap, + local container = k.core.v1.container, + local volumeMount = k.core.v1.volumeMount, + + // TODO(chaudum): Should these be exposed in $._config object? + local mount_name = 'sql-exporter', + local mount_path = '/sql-exporter/config', + + _config+:: { + sql_exporter: { + global: { + // Subtracted from Prometheus' scrape_timeout to give us some headroom and prevent Prometheus from + // timing out first. + scrape_timeout_offset: '500ms', + // Minimum interval between collector runs: by default (0s) collectors are executed on every scrape. + min_interval: '1s', + // Maximum number of open connections to any one target. Metric queries will run concurrently on + // multiple connections. + max_connections: 10, + // Maximum number of idle connections to any one target. + max_idle_connections: 3, + }, + target: { + data_source_name: 'postgres://%s@localhost:%s?sslmode=disable' % ['crate', $._config.psql_listen_port], + collectors: [c.collector_name for c in $._config.sql_exporter_collectors], + }, + collector_files: [ + '*.collector.yml', + ], + }, + sql_exporter_collectors: [ + { + collector_name: 'default', + metrics: [ + { + metric_name: 'crate_table_health', + type: 'gauge', + help: 'Table health (1=GREEN, 2=YELLOW, 3=RED)', + key_labels: ['schema', 'table'], + static_labels: {}, + values: ['severity'], + query: ||| + SELECT table_schema AS "schema", table_name AS "table", severity + FROM sys.health + |||, + }, + { + metric_name: 'crate_table_shards_total', + type: 'gauge', + help: 'Underreplicated shards per table', + key_labels: ['schema', 'table'], + value_label: 'state', + static_labels: {}, + values: ['underreplicated', 'missing'], + query: ||| + SELECT table_schema AS "schema", table_name AS "table", missing_shards AS "missing", underreplicated_shards AS "underreplicated" + FROM sys.health + |||, + }, + { + metric_name: 'crate_shards_total', + type: 'gauge', + help: 'Shard count by table and state', + key_labels: ['schema', 'table', 'state', 'primary'], + static_labels: {}, + values: ['value'], + query: ||| + SELECT COUNT(*) AS "value", schema_name AS "schema", table_name AS "table", LOWER(state) AS "state", "primary" + FROM sys.shards + GROUP BY "schema", "table", "state", "primary" + |||, + }, + ], + }, + ], + }, + + sql_exporter_args:: { + 'config.file': '%s/config.yml' % mount_path, + 'web.listen-address': ':%s' % $._config.sql_exporter_listen_port, + 'web.metrics-path': '/metrics', + }, + + sql_exporter_container:: + container.new(mount_name, $._images.sql_exporter) + + container.withArgsMixin(k.util.mapToFlags($.sql_exporter_args)) + + container.withVolumeMountsMixin([ + volumeMount.new(mount_name, mount_path), + ]) + + k.util.resourcesRequests('500m', '128Mi') + + k.util.resourcesLimits('1000m', '256Mi') + + {}, + + sql_exporter_config_file: + configMap.new(mount_name) + + configMap.withData( + { 'config.yml': k.util.manifestYaml($._config.sql_exporter) } + + { ['%s.collector.yml' % collector.collector_name]: k.util.manifestYaml(collector) for collector in $._config.sql_exporter_collectors } + ), + +} diff --git a/statefulset.libsonnet b/statefulset.libsonnet index 3ac6820..a346274 100644 --- a/statefulset.libsonnet +++ b/statefulset.libsonnet @@ -36,7 +36,8 @@ local k = import 'ksonnet-util/kausal.libsonnet'; newCrateStatefulSet(container, name='node', replicas=3, pvcs=[], prefix='crate'):: local fqn = '%s-%s' % [prefix, name]; - statefulSet.new(fqn, replicas, [container], pvcs) + + local containers = [container] + if $._config.enable_sql_exporter then [$.sql_exporter_container] else []; + statefulSet.new(fqn, replicas, containers, pvcs) + statefulSet.mixin.spec.withServiceName(fqn) + statefulSet.mixin.spec.withPodManagementPolicy('Parallel') + statefulSet.mixin.spec.updateStrategy.withType(if $._config.enable_rolling_upgrades then 'RollingUpdate' else 'OnDelete') + @@ -47,7 +48,10 @@ local k = import 'ksonnet-util/kausal.libsonnet'; k.util.emptyVolumeMount('ext', '/crate/ext') + k.util.antiAffinity + $.config_hash_mixin + - {}, + if $._config.enable_sql_exporter then + k.util.configVolumeMount('sql-exporter', '/sql-exporter/config') + else + {}, crate_ext_volume_mount:: [volumeMount.new('ext', '/crate/ext')],