From 06b685d16a2b6c47644fdedcb0343ac6063dea7e Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Sat, 4 Oct 2025 23:25:10 +0200 Subject: [PATCH 01/19] MQTT: Use {Docker,Podman} Compose --- docs/integrate/mqtt/compose.yaml | 25 +++++++++ docs/integrate/mqtt/usage.md | 88 +++++++++++--------------------- 2 files changed, 56 insertions(+), 57 deletions(-) create mode 100644 docs/integrate/mqtt/compose.yaml diff --git a/docs/integrate/mqtt/compose.yaml b/docs/integrate/mqtt/compose.yaml new file mode 100644 index 00000000..581274cd --- /dev/null +++ b/docs/integrate/mqtt/compose.yaml @@ -0,0 +1,25 @@ +# Service composition file for Docker Compose or Podman Compose + +services: + + cratedb: + image: docker.io/crate/crate:latest + ports: + - "4200:4200/tcp" + - "5432:5432/tcp" + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:4200/"] + interval: 5s + timeout: 30s + retries: 5 + + mosquitto: + image: docker.io/eclipse-mosquitto:latest + command: > + /usr/sbin/mosquitto -c /mosquitto-no-auth.conf + ports: + - "1883:1883/tcp" + + lorrystream: + image: ghcr.io/daq-tools/lorrystream:latest + scale: 0 diff --git a/docs/integrate/mqtt/usage.md b/docs/integrate/mqtt/usage.md index 194845b8..7e05ce80 100644 --- a/docs/integrate/mqtt/usage.md +++ b/docs/integrate/mqtt/usage.md @@ -10,93 +10,67 @@ pipeline element. ## Prerequisites -Docker is used for running all components. This approach works consistently +Use Docker or Podman to run all components. This approach works consistently across Linux, macOS, and Windows. -Alternatively, you can use Podman. You can also use a different MQTT broker such as +You can also use a different MQTT broker such as EMQX, HiveMQ, VerneMQ, or RabbitMQ. Azure IoT Hub speaks MQTT as well, but with protocol and authentication specifics; adjust settings accordingly. -Create a shared network. -```shell -docker network create cratedb-demo -``` - -Start CrateDB. -```shell -docker run --name=cratedb --rm -it --network=cratedb-demo \ - --publish=4200:4200 --publish=5432:5432 \ - --env=CRATE_HEAP_SIZE=2g docker.io/crate -Cdiscovery.type=single-node -``` +### Files -Start Mosquitto. -```shell -docker run --name=mosquitto --rm -it --network=cratedb-demo \ - --publish=1883:1883 docker.io/eclipse-mosquitto \ - mosquitto -c /mosquitto-no-auth.conf -``` -> Note: This broker configuration allows anonymous access for demonstration purposes only. -> Do not expose it to untrusted networks. For production, configure authentication/TLS. +First, download and save all required files to your machine. +- {download}`compose.yaml` -Prepare shortcuts for the CrateDB shell, LorryStream, and the Mosquitto client -programs. +### Services -::::{tab-set} +Start services using Docker Compose or Podman Compose. +If you use Podman, replace `docker` with `podman` (or enable the podman‑docker +compatibility shim) and run `podman compose up`. -:::{tab-item} Linux and macOS -To make the settings persistent, add them to your shell profile (`~/.profile`). -```shell -alias crash="docker run --rm -it --network=cratedb-demo ghcr.io/crate/cratedb-toolkit crash" -alias lorry="docker run --rm -i --network=cratedb-demo ghcr.io/daq-tools/lorrystream lorry" -alias mosquitto_pub="docker run --rm -i --network=cratedb-demo docker.io/eclipse-mosquitto mosquitto_pub" -alias mosquitto_sub="docker run --rm -i --network=cratedb-demo docker.io/eclipse-mosquitto mosquitto_sub" -``` -::: -:::{tab-item} Windows PowerShell -To make the settings persistent, add them to your PowerShell profile (`$PROFILE`). -```powershell -function crash { docker run --rm -it --network=cratedb-demo ghcr.io/crate/cratedb-toolkit crash @args } -function lorry { docker run --rm -i --network=cratedb-demo ghcr.io/daq-tools/lorrystream lorry @args } -function mosquitto_pub { docker run --rm -i --network=cratedb-demo docker.io/eclipse-mosquitto mosquitto_pub @args } -function mosquitto_sub { docker run --rm -i --network=cratedb-demo docker.io/eclipse-mosquitto mosquitto_sub @args } -``` -::: -:::{tab-item} Windows Command ```shell -doskey crash=docker run --rm -it --network=cratedb-demo ghcr.io/crate/cratedb-toolkit crash $* -doskey lorry=docker run --rm -i --network=cratedb-demo ghcr.io/daq-tools/lorrystream lorry $* -doskey mosquitto_pub=docker run --rm -i --network=cratedb-demo docker.io/eclipse-mosquitto mosquitto_pub $* -doskey mosquitto_sub=docker run --rm -i --network=cratedb-demo docker.io/eclipse-mosquitto mosquitto_sub $* +docker compose up ``` -::: -:::: +:::{note} +The MQTT broker configuration used here allows anonymous access for +demonstration purposes only. Do not expose it to untrusted networks. For +production, configure authentication/TLS. +::: -## Usage +## Submit data Subscribe to all MQTT topics on the broker to monitor any traffic. ```shell -mosquitto_sub -h mosquitto -t "#" -v +docker compose exec --no-tty mosquitto mosquitto_sub -h mosquitto -t "#" -v ``` Invoke the data transfer pipeline. ```shell -lorry relay \ - "mqtt://mosquitto/testdrive/%23?content-type=json" \ - "crate://cratedb/?table=testdrive" +docker compose run --rm lorrystream lorry relay "mqtt://mosquitto/testdrive/%23?content-type=json" "crate://cratedb/?table=mqtt_demo" ``` Publish a JSON message to an MQTT topic. ```shell -echo '{"temperature": 42.84, "humidity": 83.1}' | \ - mosquitto_pub -h mosquitto -t testdrive/channel1 -s +echo '{"temperature": 42.84, "humidity": 83.1}' | docker compose exec --no-tty mosquitto mosquitto_pub -h mosquitto -t testdrive/channel1 -s ``` +## Explore data + Inspect data stored in CrateDB. ```shell -crash --hosts cratedb -c "SELECT * FROM testdrive" +docker compose exec cratedb crash -c "SELECT * FROM doc.mqtt_demo" +``` +```psql ++-------------+----------+ +| temperature | humidity | ++-------------+----------+ +| 42.84 | 83.1 | ++-------------+----------+ +SELECT 1 row in set (0.004 sec) ``` [Eclipse Mosquitto]: https://mosquitto.org/ +[LorryStream]: https://lorrystream.readthedocs.io/ [LorryStream MQTT source]: https://lorrystream.readthedocs.io/source/mqtt.html From 7055b29362bb9e09e0c6b3cef62dbbd93670505f Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Sun, 5 Oct 2025 14:33:49 +0200 Subject: [PATCH 02/19] AMQP: Use {Docker,Podman} Compose --- docs/integrate/amqp/compose.yaml | 26 ++++++++++ docs/integrate/amqp/usage.md | 86 ++++++++++++-------------------- 2 files changed, 59 insertions(+), 53 deletions(-) create mode 100644 docs/integrate/amqp/compose.yaml diff --git a/docs/integrate/amqp/compose.yaml b/docs/integrate/amqp/compose.yaml new file mode 100644 index 00000000..c7e77ae7 --- /dev/null +++ b/docs/integrate/amqp/compose.yaml @@ -0,0 +1,26 @@ +# Service composition file for Docker Compose or Podman Compose + +services: + + cratedb: + image: docker.io/crate/crate:latest + ports: + - "4200:4200/tcp" + - "5432:5432/tcp" + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:4200/"] + interval: 5s + timeout: 30s + retries: 5 + + rabbitmq: + image: docker.io/rabbitmq:3 + ports: + - "5672:5672/tcp" + + lorrystream: + image: ghcr.io/daq-tools/lorrystream:latest + scale: 0 + + amqpcat: + image: docker.io/cloudamqp/amqpcat:latest diff --git a/docs/integrate/amqp/usage.md b/docs/integrate/amqp/usage.md index 7999570c..6c00a549 100644 --- a/docs/integrate/amqp/usage.md +++ b/docs/integrate/amqp/usage.md @@ -11,84 +11,64 @@ pipeline element. ## Prerequisites -Docker is used for running all components. This approach works consistently +Use Docker or Podman to run all components. This approach works consistently across Linux, macOS, and Windows. -Alternatively, you can use Podman. You can also use a different AMQP broker such as -Apache Qpid, Apache ActiveMQ, IBM MQ, or Solace. Azure Event Hubs and Azure Service -Bus speak AMQP as well, but with protocol and authentication specifics; adjust -settings accordingly. +### Files -Create a shared network. -```shell -docker network create cratedb-demo -``` +First, download and save all required files to your machine. +- {download}`compose.yaml` -Start CrateDB. -```shell -docker run --name=cratedb --rm --network=cratedb-demo \ - --publish=4200:4200 --publish=5432:5432 \ - --env=CRATE_HEAP_SIZE=2g docker.io/crate -Cdiscovery.type=single-node -``` +### Services -Start RabbitMQ. -```shell -docker run --name=rabbitmq --rm --network=cratedb-demo \ - --publish=5672:5672 docker.io/rabbitmq:3 -``` -> Note: This broker configuration allows anonymous access for demonstration purposes only. -> Do not expose it to untrusted networks. For production, configure authentication/TLS. +Start services using Docker Compose or Podman Compose. +If you use Podman, replace `docker` with `podman` (or enable the podman‑docker +compatibility shim) and run `podman compose up`. -Prepare shortcuts for the CrateDB shell, LorryStream, and the AMQP client -programs. - -::::{tab-set} +You can also use a different AMQP broker such as +Apache Qpid, Apache ActiveMQ, IBM MQ, or Solace. Azure Event Hubs and Azure Service +Bus speak AMQP as well, but with protocol and authentication specifics; adjust +settings accordingly. -:::{tab-item} Linux and macOS -To make the settings persistent, add them to your shell profile (e.g., `~/.profile` or `~/.zshrc`). ```shell -alias crash="docker run --rm -it --network=cratedb-demo ghcr.io/crate/cratedb-toolkit crash" -alias lorry="docker run --rm -i --network=cratedb-demo ghcr.io/daq-tools/lorrystream lorry" -alias amqpcat="docker run --rm -i --network=cratedb-demo docker.io/cloudamqp/amqpcat amqpcat" -``` -::: -:::{tab-item} Windows PowerShell -To make the settings persistent, add them to your PowerShell profile (`$PROFILE`). -```powershell -function crash { docker run --rm -it --network=cratedb-demo ghcr.io/crate/cratedb-toolkit crash @args } -function lorry { docker run --rm -i --network=cratedb-demo ghcr.io/daq-tools/lorrystream lorry @args } -function amqpcat { docker run --rm -i --network=cratedb-demo docker.io/cloudamqp/amqpcat amqpcat @args } +docker compose up ``` -::: -:::{tab-item} Windows Command -```shell -doskey crash=docker run --rm -it --network=cratedb-demo ghcr.io/crate/cratedb-toolkit crash $* -doskey lorry=docker run --rm -i --network=cratedb-demo ghcr.io/daq-tools/lorrystream lorry $* -doskey amqpcat=docker run --rm -i --network=cratedb-demo docker.io/cloudamqp/amqpcat amqpcat $* -``` -::: -:::: +:::{note} +The AMQP broker configuration used here allows anonymous access for +demonstration purposes only. Do not expose it to untrusted networks. For +production, configure authentication/TLS. +::: -## Usage +## Submit data Invoke the data transfer pipeline. ```shell -lorry relay \ +docker compose run --rm --no-TTY lorrystream lorry relay \ "amqp://guest:guest@rabbitmq:5672/%2F?exchange=default&queue=default&routing-key=testdrive&setup=exchange,queue,bind&content-type=json" \ - "crate://cratedb/?table=testdrive" + "crate://cratedb/?table=amqp_demo" ``` Publish a JSON message to AMQP. ```shell echo '{"temperature": 42.84, "humidity": 83.1}' | \ - amqpcat --producer --uri='amqp://guest:guest@rabbitmq:5672/%2F' \ + docker compose run --rm --no-TTY amqpcat amqpcat --producer --uri='amqp://guest:guest@rabbitmq:5672/%2F' \ --exchange=default --queue=default --routing-key=testdrive ``` +## Explore data + Inspect data stored in CrateDB. ```shell -crash --hosts cratedb -c "SELECT * FROM testdrive" +docker compose exec cratedb crash -c "SELECT * FROM doc.amqp_demo" +``` +```psql ++-------------+----------+ +| temperature | humidity | ++-------------+----------+ +| 42.84 | 83.1 | ++-------------+----------+ +SELECT 1 row in set (0.023 sec) ``` From fcd2a31e2588feea01524fd39e3bcb672be28e6b Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Sat, 4 Oct 2025 23:33:20 +0200 Subject: [PATCH 03/19] Telegraf: Use {Docker,Podman} Compose --- docs/_include/container-aliases.md | 23 ++++ .../integrate/opentelemetry/telegraf/usage.md | 2 +- docs/integrate/telegraf/compose.yaml | 23 ++++ docs/integrate/telegraf/telegraf.conf | 98 ++++++++++++++ docs/integrate/telegraf/usage.md | 128 ++++-------------- 5 files changed, 174 insertions(+), 100 deletions(-) create mode 100644 docs/_include/container-aliases.md create mode 100644 docs/integrate/telegraf/compose.yaml create mode 100644 docs/integrate/telegraf/telegraf.conf diff --git a/docs/_include/container-aliases.md b/docs/_include/container-aliases.md new file mode 100644 index 00000000..0278b1f9 --- /dev/null +++ b/docs/_include/container-aliases.md @@ -0,0 +1,23 @@ +Prepare shortcut for {ref}`crate-crash:index` command. + +::::{tab-set} + +:::{tab-item} Linux, macOS, WSL +```shell +alias crash="docker compose exec cratedb crash" +alias uv="docker compose run --rm uv uv" +alias uvx="docker compose run --rm uv uvx" +``` +::: +:::{tab-item} Windows PowerShell +```powershell +function crash { docker compose exec cratedb crash @args } +``` +::: +:::{tab-item} Windows Command +```shell +doskey crash=docker compose exec cratedb crash $* +``` +::: + +:::: diff --git a/docs/integrate/opentelemetry/telegraf/usage.md b/docs/integrate/opentelemetry/telegraf/usage.md index f4401a94..2577b670 100644 --- a/docs/integrate/opentelemetry/telegraf/usage.md +++ b/docs/integrate/opentelemetry/telegraf/usage.md @@ -18,7 +18,7 @@ Prepare shortcut for {ref}`crate-crash:index` command. ::::{tab-set} :sync-group: os -:::{tab-item} Linux and macOS +:::{tab-item} Linux, macOS, WSL :sync: unix Add these settings to your shell profile (`~/.profile`) to make them persistent. ```shell diff --git a/docs/integrate/telegraf/compose.yaml b/docs/integrate/telegraf/compose.yaml new file mode 100644 index 00000000..c850b9ce --- /dev/null +++ b/docs/integrate/telegraf/compose.yaml @@ -0,0 +1,23 @@ +# Service composition file for Docker Compose or Podman Compose + +services: + + cratedb: + image: docker.io/crate/crate:latest + ports: + - "4200:4200/tcp" + - "5432:5432/tcp" + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:4200/"] + interval: 5s + timeout: 30s + retries: 5 + + telegraf: + image: docker.io/telegraf:latest + ports: + - "4317:4317/tcp" # OTLP gRPC + volumes: + - ./telegraf.conf:/etc/telegraf/telegraf.conf:ro + depends_on: + - cratedb diff --git a/docs/integrate/telegraf/telegraf.conf b/docs/integrate/telegraf/telegraf.conf new file mode 100644 index 00000000..46d94a20 --- /dev/null +++ b/docs/integrate/telegraf/telegraf.conf @@ -0,0 +1,98 @@ +# Telegraf Configuration +# +# Telegraf is entirely plugin driven. All metrics are gathered from the +# declared inputs, and sent to the declared outputs. +# +# Plugins must be declared in here to be active. +# To deactivate a plugin, comment out the name and any variables. +# +# Use 'telegraf -config telegraf.conf -test' to see what metrics a config +# file would generate. +# +# Environment variables can be used anywhere in this config file, simply surround +# them with ${}. For strings the variable must be within quotes (ie, "${STR_VAR}"), +# for numbers and booleans they should be plain (ie, ${INT_VAR}, ${BOOL_VAR}) + +# Read metrics about cpu usage +[[inputs.cpu]] + ## Whether to report per-cpu stats or not + percpu = true + ## Whether to report total system cpu stats or not + totalcpu = true + ## If true, collect raw CPU time metrics + collect_cpu_time = false + ## If true, compute and report the sum of all non-idle CPU states + ## NOTE: The resulting 'time_active' field INCLUDES 'iowait'! + report_active = false + ## If true and the info is available then add core_id and physical_id tags + core_tags = false + +# Configuration for telegraf agent +[agent] + ## Default data collection interval for all inputs + interval = "10s" + ## Rounds collection interval to 'interval' + ## ie, if interval="10s" then always collect on :00, :10, :20, etc. + round_interval = true + + ## Telegraf will send metrics to outputs in batches of at most + ## metric_batch_size metrics. + ## This controls the size of writes that Telegraf sends to output plugins. + metric_batch_size = 1000 + + ## Maximum number of unwritten metrics per output. Increasing this value + ## allows for longer periods of output downtime without dropping metrics at the + ## cost of higher maximum memory usage. + metric_buffer_limit = 10000 + + ## Collection jitter is used to jitter the collection by a random amount. + ## Each plugin will sleep for a random time within jitter before collecting. + ## This can be used to avoid many plugins querying things like sysfs at the + ## same time, which can have a measurable effect on the system. + collection_jitter = "0s" + + ## Collection offset is used to shift the collection by the given amount. + ## This can be be used to avoid many plugins querying constraint devices + ## at the same time by manually scheduling them in time. + # collection_offset = "0s" + + ## Default flushing interval for all outputs. Maximum flush_interval will be + ## flush_interval + flush_jitter + flush_interval = "10s" + ## Jitter the flush interval by a random amount. This is primarily to avoid + ## large write spikes for users running a large number of telegraf instances. + ## ie, a jitter of 5s and interval 10s means flushes will happen every 10-15s + flush_jitter = "0s" + + ## Collected metrics are rounded to the precision specified. Precision is + ## specified as an interval with an integer + unit (e.g. 0s, 10ms, 2us, 4s). + ## Valid time units are "ns", "us" (or "µs"), "ms", "s". + ## + ## By default or when set to "0s", precision will be set to the same + ## timestamp order as the collection interval, with the maximum being 1s: + ## ie, when interval = "10s", precision will be "1s" + ## when interval = "250ms", precision will be "1ms" + ## + ## Precision will NOT be used for service inputs. It is up to each individual + ## service input to set the timestamp at the appropriate precision. + precision = "0s" + +# CrateDB Output Plugin +# https://github.com/influxdata/telegraf/tree/master/plugins/outputs/cratedb +[[outputs.cratedb]] + ## Connection parameters for accessing the database see + ## https://pkg.go.dev/github.com/jackc/pgx/v4#ParseConfig + ## for available options + url = "postgres://crate@cratedb/doc?sslmode=disable" + + ## Timeout for all CrateDB queries. + # timeout = "5s" + + ## Name of the table to store metrics in. + # table = "metrics" + + ## If true, and the metrics table does not exist, create it automatically. + table_create = true + + ## The character(s) to replace any '.' in an object key with + # key_separator = "_" diff --git a/docs/integrate/telegraf/usage.md b/docs/integrate/telegraf/usage.md index d708740e..91f67ef5 100644 --- a/docs/integrate/telegraf/usage.md +++ b/docs/integrate/telegraf/usage.md @@ -6,115 +6,33 @@ then configuring Telegraf to submit system metrics to CrateDB. ## Prerequisites -Docker is used for running all components. This approach works consistently -across Linux, macOS, and Windows. Alternatively, you can use Podman. +Use Docker or Podman to run all components. This approach works consistently +across Linux, macOS, and Windows. -Create a shared network. -```shell -docker network create cratedb-demo -``` +### Files -Start CrateDB. -```shell -docker run --name=cratedb --rm -it --network=cratedb-demo \ - --publish=4200:4200 --publish=5432:5432 \ - --env=CRATE_HEAP_SIZE=2g docker.io/crate -Cdiscovery.type=single-node -``` +First, download and save all required files to your machine. +- {download}`compose.yaml` +- {download}`telegraf.conf` -Prepare shortcut for the CrateDB shell and the Telegraf command. +### Services -::::{tab-set} +Start services using Docker Compose or Podman Compose. +If you use Podman, replace `docker` with `podman` (or enable the podman‑docker +compatibility shim) and run `podman compose up`. -:::{tab-item} Linux and macOS -To make the settings persistent, add them to your shell profile (`~/.profile`). -```shell -alias crash="docker run --rm -it --network=cratedb-demo ghcr.io/crate/cratedb-toolkit crash" -alias telegraf="docker run --rm --network=cratedb-demo docker.io/telegraf" -``` -::: -:::{tab-item} Windows PowerShell -To make the settings persistent, add them to your PowerShell profile (`$PROFILE`). -```powershell -function crash { docker run --rm -it --network=cratedb-demo ghcr.io/crate/cratedb-toolkit crash @args } -function telegraf { docker run --rm --network=cratedb-demo docker.io/telegraf @args } -``` -::: -:::{tab-item} Windows Command ```shell -doskey crash=docker run --rm -it --network=cratedb-demo ghcr.io/crate/cratedb-toolkit crash $* -doskey telegraf=docker run --rm --network=cratedb-demo docker.io/telegraf $* +docker compose up ``` -::: - -:::: - -## Usage +## Submit data -Configure Telegraf by creating a configuration blueprint and adjusting it. +The `telegraf.conf` configures Telegraf to enable collecting readings about +CPU usage on the local computer. Telegraf is a plugin-driven tool and has plugins to collect many different types -of metrics. Because we just want to test things out, we are using `--input-filter cpu` -to limit input plugins so that Telegraf only collects readings about CPU usage -on the local computer. To send the collected data to CrateDB, use -`--output-filter cratedb`. -```shell -telegraf \ - --input-filter cpu \ - --output-filter cratedb \ - config > telegraf.conf -``` - -The database connection URL is a `pgx/v4` connection string. Configure -`table_create = true` to automatically let Telegraf create the metrics table -if it doesn't exist. -::::{tab-set} -:::{tab-item} Linux -```shell -sed -i 's!postgres://user:password@localhost/schema?sslmode=disable!postgres://crate@cratedb/doc?sslmode=disable!g' telegraf.conf -sed -i 's!# table_create = false!table_create = true!' telegraf.conf -``` -::: -:::{tab-item} macOS and BSD -```shell -sed -i '' 's!postgres://user:password@localhost/schema?sslmode=disable!postgres://crate@cratedb/doc?sslmode=disable!g' telegraf.conf -sed -i '' 's!# table_create = false!table_create = true!' telegraf.conf -``` -::: -:::{tab-item} Windows PowerShell -```powershell -(Get-Content telegraf.conf) -replace 'postgres://user:password@localhost/schema\?sslmode=disable','postgres://crate@cratedb/doc?sslmode=disable' | - ForEach-Object { $_ -replace '# table_create = false','table_create = true' } | - Set-Content telegraf.conf -``` -::: -:::: - - -Start Telegraf. -::::{tab-set} -:::{tab-item} Linux and macOS -```shell -docker run --name=telegraf --rm -it --network=cratedb-demo \ - --volume "$(pwd)"/telegraf.conf:/etc/telegraf/telegraf.conf \ - docker.io/telegraf -``` -::: -:::{tab-item} Windows PowerShell -```powershell -docker run --name=telegraf --rm -it --network=cratedb-demo ` - --volume "${PWD}\telegraf.conf:/etc/telegraf/telegraf.conf" ` - docker.io/telegraf -``` -::: -:::{tab-item} Windows Command -```shell -docker run --name=telegraf --rm -it --network=cratedb-demo ^ - --volume "%cd%\telegraf.conf:/etc/telegraf/telegraf.conf" ^ - docker.io/telegraf -``` -::: -:::: +of metrics. Because we just want to test things out, a single sensor is sufficient. +## Explore data After 10 seconds, which is the default output flush interval of Telegraf, the first metrics will appear in the `metrics` table in CrateDB. To adjust the value, navigate @@ -122,7 +40,19 @@ to `flush_interval = "10s"` in `telegraf.conf`. Inspect data stored in CrateDB. ```shell -crash --hosts cratedb -c "SELECT * FROM doc.metrics LIMIT 5;" +docker compose exec cratedb crash -c "SELECT * FROM doc.metrics LIMIT 5;" +``` +```psql ++----------------------+---------------+------+------------------------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+---------------+ +| hash_id | timestamp | name | tags | fields | day | ++----------------------+---------------+------+------------------------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+---------------+ +| -2149377133708231861 | 1759686560000 | cpu | {"cpu": "cpu5", "host": "02880df5c05e"} | {"usage_guest": 0, "usage_guest_nice": 0, "usage_idle": 98, "usage_iowait": 0, "usage_irq": 0, "usage_nice": 0, "usage_softirq": 0, "usage_steal": 0, "usage_system": 0, "usage_user": 0} | 1759622400000 | +| -4525358308278119095 | 1759686560000 | cpu | {"cpu": "cpu13", "host": "02880df5c05e"} | {"usage_guest": 0, "usage_guest_nice": 0, "usage_idle": 98, "usage_iowait": 0, "usage_irq": 0, "usage_nice": 0, "usage_softirq": 0, "usage_steal": 0, "usage_system": 0, "usage_user": 1} | 1759622400000 | +| 6120994000579036036 | 1759686560000 | cpu | {"cpu": "cpu14", "host": "02880df5c05e"} | {"usage_guest": 0, "usage_guest_nice": 0, "usage_idle": 99, "usage_iowait": 0, "usage_irq": 0, "usage_nice": 0, "usage_softirq": 0, "usage_steal": 0, "usage_system": 0, "usage_user": 0} | 1759622400000 | +| 869454636336868835 | 1759686560000 | cpu | {"cpu": "cpu1", "host": "02880df5c05e"} | {"usage_guest": 0, "usage_guest_nice": 0, "usage_idle": 99, "usage_iowait": 0, "usage_irq": 0, "usage_nice": 0, "usage_softirq": 0, "usage_steal": 0, "usage_system": 0, "usage_user": 0} | 1759622400000 | +| 722018796549427924 | 1759686560000 | cpu | {"cpu": "cpu8", "host": "02880df5c05e"} | {"usage_guest": 0, "usage_guest_nice": 0, "usage_idle": 97, "usage_iowait": 0, "usage_irq": 0, "usage_nice": 0, "usage_softirq": 0, "usage_steal": 0, "usage_system": 0, "usage_user": 1} | 1759622400000 | ++----------------------+---------------+------+------------------------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+---------------+ +SELECT 5 rows in set (0.009 sec) ``` From 6bf4f30215fe6abf75923f70c86bc8b1c4ba4921 Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Sun, 5 Oct 2025 01:57:01 +0200 Subject: [PATCH 04/19] OpenTelemetry: Use {Docker,Podman} Compose --- docs/_include/container-aliases.md | 23 ------- docs/integrate/opentelemetry/collector/.env | 2 + .../opentelemetry/collector/compose.yaml | 14 ++++ .../opentelemetry/collector/usage.md | 66 ++++++------------- docs/integrate/opentelemetry/telegraf/.env | 2 + .../opentelemetry/telegraf/compose.yaml | 10 +++ .../integrate/opentelemetry/telegraf/usage.md | 58 +++++----------- 7 files changed, 65 insertions(+), 110 deletions(-) delete mode 100644 docs/_include/container-aliases.md create mode 100644 docs/integrate/opentelemetry/collector/.env create mode 100644 docs/integrate/opentelemetry/telegraf/.env diff --git a/docs/_include/container-aliases.md b/docs/_include/container-aliases.md deleted file mode 100644 index 0278b1f9..00000000 --- a/docs/_include/container-aliases.md +++ /dev/null @@ -1,23 +0,0 @@ -Prepare shortcut for {ref}`crate-crash:index` command. - -::::{tab-set} - -:::{tab-item} Linux, macOS, WSL -```shell -alias crash="docker compose exec cratedb crash" -alias uv="docker compose run --rm uv uv" -alias uvx="docker compose run --rm uv uvx" -``` -::: -:::{tab-item} Windows PowerShell -```powershell -function crash { docker compose exec cratedb crash @args } -``` -::: -:::{tab-item} Windows Command -```shell -doskey crash=docker compose exec cratedb crash $* -``` -::: - -:::: diff --git a/docs/integrate/opentelemetry/collector/.env b/docs/integrate/opentelemetry/collector/.env new file mode 100644 index 00000000..bbd477a6 --- /dev/null +++ b/docs/integrate/opentelemetry/collector/.env @@ -0,0 +1,2 @@ +export OTEL_EXPORTER_OTLP_ENDPOINT=http://otelcol:4317 +export OTEL_SERVICE_NAME=app diff --git a/docs/integrate/opentelemetry/collector/compose.yaml b/docs/integrate/opentelemetry/collector/compose.yaml index 3b03cd4f..ea1836ed 100644 --- a/docs/integrate/opentelemetry/collector/compose.yaml +++ b/docs/integrate/opentelemetry/collector/compose.yaml @@ -41,3 +41,17 @@ services: - ./otelcol.yaml:/etc/otelcol-contrib/config.yaml depends_on: - cratedb-prometheus-adapter + + nc: + image: docker.io/toolbelt/netcat:2025-08-23 + scale: 0 + + uv: + image: ghcr.io/astral-sh/uv:python3.13-trixie-slim + volumes: + - ./:/src + - uv-cache:/root/.cache/uv:cached + scale: 0 + +volumes: + uv-cache: diff --git a/docs/integrate/opentelemetry/collector/usage.md b/docs/integrate/opentelemetry/collector/usage.md index 8e7be3df..ed0d79e1 100644 --- a/docs/integrate/opentelemetry/collector/usage.md +++ b/docs/integrate/opentelemetry/collector/usage.md @@ -9,49 +9,22 @@ into CrateDB. Use Docker or Podman to run all components. This approach works consistently across Linux, macOS, and Windows. -If you use Podman, replace `docker` with `podman` (or enable the podman‑docker -compatibility shim) and run `podman compose up`. - -### Commands - -Prepare shortcut for {ref}`crate-crash:index` command. - -::::{tab-set} -:sync-group: os -:::{tab-item} Linux and macOS -:sync: unix -Add these settings to your shell profile (`~/.profile`) to make them persistent. -```shell -alias crash="docker compose exec -it cratedb crash" -alias nc="docker run --rm -i --network=cratedb-demo docker.io/toolbelt/netcat:2025-08-23" -``` -::: -:::{tab-item} Windows PowerShell -:sync: powershell -Add these settings to your PowerShell profile (`$PROFILE`) to make them persistent. -```powershell -function crash { docker compose exec -it cratedb crash @args } -function nc { docker run --rm -i --network=cratedb-demo docker.io/toolbelt/netcat:2025-08-23 @args } -``` -::: -:::{tab-item} Windows Command -:sync: dos -```shell -doskey crash=docker compose exec -it cratedb crash $* -doskey nc=docker run --rm -i --network=cratedb-demo docker.io/toolbelt/netcat:2025-08-23 $* -REM Note: doskey macros reset each session. To persist, configure an AutoRun command -REM pointing to a macro file, or re-run these in your shell startup script. -``` -::: +### Files -:::: +First, download and save all required files to your machine. +- {download}`.env` +- {download}`compose.yaml` +- {download}`cratedb-prometheus-adapter.yaml` +- {download}`ddl.sql` +- {download}`example.py` +- {download}`otelcol.yaml` ### Services -Save {download}`compose.yaml`, {download}`cratedb-prometheus-adapter.yaml`, -{download}`otelcol.yaml` and {download}`ddl.sql` to your machine, then start -services using Docker Compose or Podman Compose. +Start services using Docker Compose or Podman Compose. +If you use Podman, replace `docker` with `podman` (or enable the podman‑docker +compatibility shim) and run `podman compose up`. ```shell docker compose up @@ -63,23 +36,22 @@ docker compose up Use [netcat] to submit metrics using the [Carbon plaintext protocol]. ```shell -printf "temperature;job=app 42.42 1758486061\nhumidity;job=app 84.84 1758486061" | nc -c otelcol 2003 +printf "temperature;job=app 42.42 1758486061\nhumidity;job=app 84.84 1758486061" | docker compose run --rm nc -C -w1 otelcol 2003 ``` ### Use Python -To submit metrics using the OpenTelemetry Python SDK, download the example file -{download}`example.py` to your machine and choose one of these approaches: +To submit metrics using the OpenTelemetry Python SDK, choose one of these +approaches: **Option 1: Using uv (recommended)** ```shell -export OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317 -export OTEL_SERVICE_NAME=app -uv run --with=opentelemetry-distro --with=opentelemetry-exporter-otlp opentelemetry-instrument python example.py +docker compose run --rm --env-from-file=.env uv uv run --with=opentelemetry-distro --with=opentelemetry-exporter-otlp opentelemetry-instrument python /src/example.py ``` **Option 2: Using pip** -Install dependencies: + +First, install dependencies: ```shell pip install opentelemetry-distro opentelemetry-exporter-otlp ``` @@ -88,8 +60,10 @@ Then run the example: opentelemetry-instrument --service_name=app python example.py ``` +::::{dropdown} Display example\.py :::{literalinclude} example.py ::: +:::: ### Use any language @@ -100,7 +74,7 @@ Erlang/Elixir, Go, Java, JavaScript, PHP, Python, Ruby, Rust, or Swift. CrateDB stores the metrics in the designated table, ready for inspection and analysis. ```shell -crash --hosts "http://crate:crate@localhost:4200/" -c "SELECT * FROM testdrive.metrics ORDER BY timestamp LIMIT 5;" +docker compose exec cratedb crash -c "SELECT * FROM testdrive.metrics ORDER BY timestamp LIMIT 5;" ``` ```psql +---------------+------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-------+---------------------+----------------+ diff --git a/docs/integrate/opentelemetry/telegraf/.env b/docs/integrate/opentelemetry/telegraf/.env new file mode 100644 index 00000000..2821866c --- /dev/null +++ b/docs/integrate/opentelemetry/telegraf/.env @@ -0,0 +1,2 @@ +export OTEL_EXPORTER_OTLP_ENDPOINT=http://telegraf:4317 +export OTEL_SERVICE_NAME=app diff --git a/docs/integrate/opentelemetry/telegraf/compose.yaml b/docs/integrate/opentelemetry/telegraf/compose.yaml index c850b9ce..1ab7330c 100644 --- a/docs/integrate/opentelemetry/telegraf/compose.yaml +++ b/docs/integrate/opentelemetry/telegraf/compose.yaml @@ -21,3 +21,13 @@ services: - ./telegraf.conf:/etc/telegraf/telegraf.conf:ro depends_on: - cratedb + + uv: + image: ghcr.io/astral-sh/uv:python3.13-trixie-slim + volumes: + - ./:/src + - uv-cache:/root/.cache/uv:cached + scale: 0 + +volumes: + uv-cache: diff --git a/docs/integrate/opentelemetry/telegraf/usage.md b/docs/integrate/opentelemetry/telegraf/usage.md index 2577b670..965c5a72 100644 --- a/docs/integrate/opentelemetry/telegraf/usage.md +++ b/docs/integrate/opentelemetry/telegraf/usage.md @@ -8,45 +8,20 @@ This how-to guide walks you through configuring [Telegraf] to receive Use Docker or Podman to run all components. This approach works consistently across Linux, macOS, and Windows. -If you use Podman, replace `docker` with `podman` (or enable the podman‑docker -compatibility shim) and run `podman compose up`. - -### Commands - -Prepare shortcut for {ref}`crate-crash:index` command. - -::::{tab-set} -:sync-group: os -:::{tab-item} Linux, macOS, WSL -:sync: unix -Add these settings to your shell profile (`~/.profile`) to make them persistent. -```shell -alias crash="docker compose exec -it cratedb crash" -``` -::: -:::{tab-item} Windows PowerShell -:sync: powershell -Add these settings to your PowerShell profile (`$PROFILE`) to make them persistent. -```powershell -function crash { docker compose exec -it cratedb crash @args } -``` -::: -:::{tab-item} Windows Command -:sync: dos -```shell -doskey crash=docker compose exec -it cratedb crash $* -REM Note: doskey macros reset each session. To persist, configure an AutoRun command -REM pointing to a macro file, or re-run these in your shell startup script. -``` -::: +### Files -:::: +First, download and save all required files to your machine. +- {download}`.env` +- {download}`compose.yaml` +- {download}`example.py` +- {download}`telegraf.conf` ### Services -Save {download}`compose.yaml` and {download}`telegraf.conf`, then start -services using Docker Compose or Podman Compose. +Start services using Docker Compose or Podman Compose. +If you use Podman, replace `docker` with `podman` (or enable the podman‑docker +compatibility shim) and run `podman compose up`. ```shell docker compose up @@ -56,18 +31,17 @@ docker compose up ### Use Python -To submit metrics using the OpenTelemetry Python SDK, download the example file -{download}`example.py` to your machine and choose one of these approaches: +To submit metrics using the OpenTelemetry Python SDK, choose one of these +approaches: **Option 1: Using uv (recommended)** ```shell -export OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317 -export OTEL_SERVICE_NAME=app -uv run --with=opentelemetry-distro --with=opentelemetry-exporter-otlp opentelemetry-instrument python example.py +docker compose run --rm --env-from-file=.env uv uv run --with=opentelemetry-distro --with=opentelemetry-exporter-otlp opentelemetry-instrument python /src/example.py ``` **Option 2: Using pip** -Install dependencies: + +First, install dependencies: ```shell pip install opentelemetry-distro opentelemetry-exporter-otlp ``` @@ -76,8 +50,10 @@ Then run the example: opentelemetry-instrument --service_name=app python example.py ``` +::::{dropdown} Display example\.py :::{literalinclude} example.py ::: +:::: ### Use any language @@ -88,7 +64,7 @@ Erlang/Elixir, Go, Java, JavaScript, PHP, Python, Ruby, Rust, or Swift. CrateDB stores the metrics in the designated table, ready for inspection and analysis. ```shell -crash --hosts "http://crate:crate@localhost:4200/" -c "SELECT * FROM metrics ORDER BY timestamp LIMIT 5;" +docker compose exec cratedb crash -c "SELECT * FROM doc.metrics ORDER BY timestamp LIMIT 5;" ``` ```psql +---------------------+---------------+-------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------+---------------+ From 3560b7c882fce6679abe1fb9c176613e0f9d09c2 Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Sun, 5 Oct 2025 02:17:17 +0200 Subject: [PATCH 05/19] StatsD: Use {Docker,Podman} Compose --- docs/integrate/statsd/compose.yaml | 31 +++++++++ docs/integrate/statsd/usage.md | 103 ++++------------------------- 2 files changed, 45 insertions(+), 89 deletions(-) create mode 100644 docs/integrate/statsd/compose.yaml diff --git a/docs/integrate/statsd/compose.yaml b/docs/integrate/statsd/compose.yaml new file mode 100644 index 00000000..3c81aa00 --- /dev/null +++ b/docs/integrate/statsd/compose.yaml @@ -0,0 +1,31 @@ +# Service composition file for Docker Compose or Podman Compose + +services: + + cratedb: + image: docker.io/crate/crate:latest + ports: + - "4200:4200/tcp" + - "5432:5432/tcp" + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:4200/"] + interval: 5s + timeout: 30s + retries: 5 + + telegraf: + image: docker.io/telegraf:latest + ports: + - "8125:8125/udp" # StatsD + volumes: + - ./telegraf.conf:/etc/telegraf/telegraf.conf:ro + depends_on: + - cratedb + + nc: + image: docker.io/toolbelt/netcat:2025-08-23 + scale: 0 + + psql: + image: docker.io/postgres:18 + scale: 0 diff --git a/docs/integrate/statsd/usage.md b/docs/integrate/statsd/usage.md index 5c5c604f..ad188235 100644 --- a/docs/integrate/statsd/usage.md +++ b/docs/integrate/statsd/usage.md @@ -6,107 +6,32 @@ metrics and store them into CrateDB. ## Prerequisites -Docker is used for running all components. This approach works consistently -across Linux, macOS, and Windows. Alternatively, you can use Podman. +Use Docker or Podman to run all components. This approach works consistently +across Linux, macOS, and Windows. -### CLI +### Files -Prepare shortcut for the psql command. +First, download and save all required files to your machine. +- {download}`compose.yaml` +- {download}`telegraf.conf` -::::{tab-set} -:sync-group: os +### Services -:::{tab-item} Linux and macOS -:sync: unix -Add these settings to your shell profile (`~/.profile`) to make them persistent. -```shell -alias nc="docker run --rm -i --network=cratedb-demo docker.io/toolbelt/netcat:2025-08-23" -alias psql="docker run --rm -i --network=cratedb-demo docker.io/postgres:16 psql" -``` -::: -:::{tab-item} Windows PowerShell -:sync: powershell -Add these settings to your PowerShell profile (`$PROFILE`) to make them persistent. -```powershell -function nc { docker run --rm -i --network=cratedb-demo docker.io/toolbelt/netcat:2025-08-23 @args } -function psql { docker run --rm -i --network=cratedb-demo docker.io/postgres:16 psql @args } -``` -::: -:::{tab-item} Windows Command -:sync: dos -```shell -doskey nc=docker run --rm -i --network=cratedb-demo docker.io/toolbelt/netcat:2025-08-23 $* -doskey psql=docker run --rm -i --network=cratedb-demo docker.io/postgres:16 psql $* -``` -::: - -:::: +Start services using Docker Compose or Podman Compose. +If you use Podman, replace `docker` with `podman` (or enable the podman‑docker +compatibility shim) and run `podman compose up`. -### CrateDB - -Create a shared network. -```shell -docker network create cratedb-demo -``` - -Start CrateDB. -```shell -docker run --name=cratedb --rm -it --network=cratedb-demo \ - --publish=4200:4200 --publish=5432:5432 \ - --env=CRATE_HEAP_SIZE=2g docker.io/crate -Cdiscovery.type=single-node -``` - - -## Configure Telegraf - -Use the configuration blueprint below to configure Telegraf to receive StatsD -metrics and store them in CrateDB. Adjust the configuration to match your -environment and save this file as `telegraf.conf`. - -:::{literalinclude} telegraf.conf -::: - - -## Start Telegraf - -::::{tab-set} -:sync-group: os - -:::{tab-item} Linux and macOS -:sync: unix -```shell -docker run --name=telegraf --rm -it --network=cratedb-demo \ - --volume "$(pwd)"/telegraf.conf:/etc/telegraf/telegraf.conf \ - --publish 8125:8125/udp \ - docker.io/telegraf -``` -::: -:::{tab-item} Windows PowerShell -:sync: powershell -```powershell -docker run --name=telegraf --rm -it --network=cratedb-demo ` - --volume "${PWD}\telegraf.conf:/etc/telegraf/telegraf.conf" ` - --publish 8125:8125/udp ` - docker.io/telegraf -``` -::: -:::{tab-item} Windows Command -:sync: dos ```shell -docker run --name=telegraf --rm -it --network=cratedb-demo ^ - --volume "%cd%\telegraf.conf:/etc/telegraf/telegraf.conf" ^ - --publish 8125:8125/udp ^ - docker.io/telegraf +docker compose up ``` -::: -:::: ## Submit data ### netcat Use [netcat] for submitting data. ```shell -echo "temperature:42|g\nhumidity:84|g" | nc -C -w 1 -u telegraf 8125 +echo "temperature:42|g" | docker compose run --rm nc -C -w 1 -u telegraf 8125 +echo "humidity:84|g" | docker compose run --rm nc -C -w 1 -u telegraf 8125 ``` ### Python @@ -136,7 +61,7 @@ After Telegraf receives data, CrateDB stores the metrics in the designated table ready for inspection. ```shell -psql "postgresql://crate:crate@cratedb:5432/" -c "SELECT * FROM doc.metrics ORDER BY timestamp LIMIT 5;" +docker compose run --rm psql psql "postgresql://crate:crate@cratedb:5432/" -c "SELECT * FROM doc.metrics ORDER BY timestamp LIMIT 5;" ``` ```psql hash_id | timestamp | name | tags | fields | day From dd72241c5a280ae41703094876f5480be0f28422 Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Sun, 5 Oct 2025 02:51:05 +0200 Subject: [PATCH 06/19] collectd: Use {Docker,Podman} Compose --- docs/integrate/collectd/compose.yaml | 41 ++++++ docs/integrate/collectd/usage-collectd.md | 114 +++------------- docs/integrate/collectd/usage-telegraf.md | 151 +++------------------- 3 files changed, 78 insertions(+), 228 deletions(-) create mode 100644 docs/integrate/collectd/compose.yaml diff --git a/docs/integrate/collectd/compose.yaml b/docs/integrate/collectd/compose.yaml new file mode 100644 index 00000000..ca7f615e --- /dev/null +++ b/docs/integrate/collectd/compose.yaml @@ -0,0 +1,41 @@ +# Service composition file for Docker Compose or Podman Compose + +services: + + cratedb: + image: docker.io/crate/crate:latest + ports: + - "4200:4200/tcp" + - "5432:5432/tcp" + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:4200/"] + interval: 5s + timeout: 30s + retries: 5 + + collectd-cratedb: + build: . + volumes: + - ./collectd-cratedb.conf:/etc/collectd/collectd.conf.d/collectd-cratedb.conf:ro + depends_on: + - cratedb + + collectd-telegraf: + build: . + volumes: + - ./collectd-telegraf.conf:/etc/collectd/collectd.conf.d/collectd-telegraf.conf:ro + depends_on: + - cratedb + + telegraf: + image: docker.io/telegraf:latest + ports: + - "8125:8125/udp" # StatsD + volumes: + - ./telegraf.conf:/etc/telegraf/telegraf.conf:ro + depends_on: + - cratedb + + psql: + image: docker.io/postgres:18 + scale: 0 diff --git a/docs/integrate/collectd/usage-collectd.md b/docs/integrate/collectd/usage-collectd.md index 2ed7a512..46a55d41 100644 --- a/docs/integrate/collectd/usage-collectd.md +++ b/docs/integrate/collectd/usage-collectd.md @@ -6,60 +6,39 @@ so that collectd sends system metrics and CrateDB stores them. ## Prerequisites -Docker runs all components consistently across Linux, macOS, and Windows. -If you use Podman, substitute podman for docker in the commands. +Use Docker or Podman to run all components. This approach works consistently +across Linux, macOS, and Windows. -### Commands +### Files -Prepare shortcut for the psql command. +First, download and save all required files to your machine. +- {download}`compose.yaml` +- {download}`Dockerfile` +- {download}`collectd-cratedb.conf` -::::{tab-set} -:sync-group: os +### Services -:::{tab-item} Linux and macOS -:sync: unix -To make the settings persistent, add them to your shell profile (`~/.profile`). -```shell -alias psql="docker run --rm -i --network=cratedb-demo docker.io/postgres:16 psql" -``` -::: -:::{tab-item} Windows PowerShell -:sync: powershell -To make the settings persistent, add them to your PowerShell profile (`$PROFILE`). -```powershell -function psql { docker run --rm -i --network=cratedb-demo docker.io/postgres:16 psql @args } -``` -::: -:::{tab-item} Windows Command -:sync: dos -```shell -doskey psql=docker run --rm -i --network=cratedb-demo docker.io/postgres:16 psql $* -``` -::: - -:::: +Start services using Docker Compose or Podman Compose. +If you use Podman, replace `docker` with `podman` (or enable the podman‑docker +compatibility shim) and run `podman compose up`. -### CrateDB - -Create a shared network. ```shell -docker network create cratedb-demo +docker compose up ``` -Start CrateDB. -```shell -docker run --name=cratedb --rm -it --network=cratedb-demo \ - --publish=4200:4200 --publish=5432:5432 \ - --env=CRATE_HEAP_SIZE=2g docker.io/crate -Cdiscovery.type=single-node -``` +To send the collected data to CrateDB, collectd is configured to load its +[`postgresql` plugin]. -## Configure +::::{dropdown} collectd configuration `collectd-cratedb.conf` +:::{literalinclude} collectd-cratedb.conf +::: +:::: ### Provision database Create a database table that stores collected metrics. ```shell -psql "postgresql://crate:crate@cratedb:5432/" < Date: Sun, 5 Oct 2025 03:35:23 +0200 Subject: [PATCH 07/19] PostgreSQL: Use {Docker,Podman} Compose --- docs/integrate/postgresql/compose.yaml | 31 +++++++++ docs/integrate/postgresql/usage.md | 89 ++++++++++---------------- 2 files changed, 65 insertions(+), 55 deletions(-) create mode 100644 docs/integrate/postgresql/compose.yaml diff --git a/docs/integrate/postgresql/compose.yaml b/docs/integrate/postgresql/compose.yaml new file mode 100644 index 00000000..b34c2e11 --- /dev/null +++ b/docs/integrate/postgresql/compose.yaml @@ -0,0 +1,31 @@ +# Service composition file for Docker Compose or Podman Compose + +services: + + cratedb: + image: docker.io/crate/crate:latest + ports: + - "4200:4200/tcp" + - "5432:5432/tcp" + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:4200/"] + interval: 5s + timeout: 30s + retries: 5 + + postgresql: + image: docker.io/postgres:latest + command: > + postgres -c log_statement=all + environment: + POSTGRES_HOST_AUTH_METHOD: trust + ports: + - "6432:5432/tcp" + + psql: + image: docker.io/postgres:latest + scale: 0 + + ctk-ingest: + image: ghcr.io/crate/cratedb-toolkit-ingest:latest + scale: 0 diff --git a/docs/integrate/postgresql/usage.md b/docs/integrate/postgresql/usage.md index 1e7d1abe..2ec309fa 100644 --- a/docs/integrate/postgresql/usage.md +++ b/docs/integrate/postgresql/usage.md @@ -9,72 +9,40 @@ The data transfer is supported by the ## Prerequisites -Docker is used for running all components. This approach works consistently -across Linux, macOS, and Windows. Alternatively, you can use Podman. +Use Docker or Podman to run all components. This approach works consistently +across Linux, macOS, and Windows. -Create a shared network. -```shell -docker network create cratedb-demo -``` - -Start CrateDB. -```shell -docker run --rm --name=cratedb --network=cratedb-demo \ - --publish=4200:4200 --publish=5432:5432 \ - docker.io/crate -Cdiscovery.type=single-node -``` +### Files -Start PostgreSQL. -```shell -docker run --rm --name=postgresql --network=cratedb-demo \ - --publish=6432:5432 --env "POSTGRES_HOST_AUTH_METHOD=trust" \ - docker.io/postgres postgres -c log_statement=all -``` -:::{note} -Because CrateDB is configured to listen on port `5432` with its PostgreSQL -interface, let's use a different port for PostgreSQL itself. -::: -:::{note} -Using `POSTGRES_HOST_AUTH_METHOD=trust` disables password checks. -Use it for local demos only. -::: +First, download and save all required files to your machine. +- {download}`compose.yaml` -Prepare shortcuts for the CrateDB shell, CrateDB Toolkit, and the PostgreSQL client -programs. +### Services -::::{tab-set} +Start services using Docker Compose or Podman Compose. +If you use Podman, replace `docker` with `podman` (or enable the podman‑docker +compatibility shim) and run `podman compose up`. -:::{tab-item} Linux and macOS -To make the settings persistent, add them to your shell profile (`~/.profile`). ```shell -alias crash="docker run --rm -it --network=cratedb-demo ghcr.io/crate/cratedb-toolkit crash" -alias ctk-ingest="docker run --rm -i --network=cratedb-demo ghcr.io/crate/cratedb-toolkit-ingest ctk" -alias psql="docker run --rm -i --network=cratedb-demo docker.io/postgres psql" -``` -::: -:::{tab-item} Windows PowerShell -To make the settings persistent, add them to your PowerShell profile (`$PROFILE`). -```powershell -function crash { docker run --rm -it --network=cratedb-demo ghcr.io/crate/cratedb-toolkit crash @args } -function ctk-ingest { docker run --rm -i --network=cratedb-demo ghcr.io/crate/cratedb-toolkit-ingest ctk @args } -function psql { docker run --rm -i --network=cratedb-demo docker.io/postgres psql @args } +docker compose up ``` + +:::{note} +CrateDB is configured to listen on port `5432`, +while PostgreSQL is configured to listen on port `6432`. ::: -:::{tab-item} Windows Command -```shell -doskey crash=docker run --rm -it --network=cratedb-demo ghcr.io/crate/cratedb-toolkit crash $* -doskey ctk-ingest=docker run --rm -i --network=cratedb-demo ghcr.io/crate/cratedb-toolkit-ingest ctk $* -doskey psql=docker run --rm -i --network=cratedb-demo docker.io/postgres psql $* -``` +:::{note} +PostgreSQL is configured using `POSTGRES_HOST_AUTH_METHOD=trust`. +This allows anonymous access for demonstration purposes only. +Do not expose it to untrusted networks. For production, configure +authentication/TLS. ::: -:::: - -## Usage +## Submit data Write a few sample records to PostgreSQL. ```shell -psql "postgresql://postgres:postgres@postgresql:5432/" < Date: Sun, 5 Oct 2025 03:46:13 +0200 Subject: [PATCH 08/19] Oracle DBMS: Use {Docker,Podman} Compose --- docs/integrate/oracle/compose.yaml | 34 ++++++++++++ docs/integrate/oracle/usage.md | 87 ++++++++++-------------------- 2 files changed, 61 insertions(+), 60 deletions(-) create mode 100644 docs/integrate/oracle/compose.yaml diff --git a/docs/integrate/oracle/compose.yaml b/docs/integrate/oracle/compose.yaml new file mode 100644 index 00000000..edd4bd14 --- /dev/null +++ b/docs/integrate/oracle/compose.yaml @@ -0,0 +1,34 @@ +# Service composition file for Docker Compose or Podman Compose + +services: + + cratedb: + image: docker.io/crate/crate:latest + ports: + - "4200:4200/tcp" + - "5432:5432/tcp" + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:4200/"] + interval: 5s + timeout: 30s + retries: 5 + + oracledb: + image: docker.io/gvenzl/oracle-free:23-slim + environment: + ORACLE_PASSWORD: secret + ports: + - "1521:1521/tcp" + + sqlplus: + image: docker.io/gvenzl/oracle-free:23-slim + entrypoint: sqlplus + scale: 0 + environment: + ORACLE_PASSWORD: secret + volumes: + - .:/demo + + ctk-ingest: + image: ghcr.io/crate/cratedb-toolkit-ingest:latest + scale: 0 diff --git a/docs/integrate/oracle/usage.md b/docs/integrate/oracle/usage.md index 9cb8e590..5ad2c10c 100644 --- a/docs/integrate/oracle/usage.md +++ b/docs/integrate/oracle/usage.md @@ -9,86 +9,53 @@ The data transfer is supported by the ## Prerequisites -Docker is used for running all components. This approach works consistently -across Linux, macOS, and Windows. Alternatively, you can use Podman. +Use Docker or Podman to run all components. This approach works consistently +across Linux, macOS, and Windows. -Create a shared network. -```shell -docker network create cratedb-demo -``` - -Start CrateDB. -```shell -docker run --rm --name=cratedb --network=cratedb-demo \ - --publish=4200:4200 --publish=5432:5432 \ - docker.io/crate -Cdiscovery.type=single-node -``` +### Files -Start Oracle DBMS. -```shell -docker run --rm --name=oracle --network=cratedb-demo \ - --publish=1521:1521 \ - --env=ORACLE_PASSWORD=secret \ - docker.io/gvenzl/oracle-free:23-slim -``` +First, download and save all required files to your machine. +- {download}`compose.yaml` +- {download}`init.sql` -Prepare shortcuts for the CrateDB shell, CrateDB Toolkit, and the Oracle client -programs. +### Services -::::{tab-set} +Start services using Docker Compose or Podman Compose. +If you use Podman, replace `docker` with `podman` (or enable the podman‑docker +compatibility shim) and run `podman compose up`. -:::{tab-item} Linux and macOS -To make the settings persistent, add them to your shell profile (`~/.profile`). ```shell -alias crash="docker run --rm -it --network=cratedb-demo ghcr.io/crate/cratedb-toolkit-ingest crash" -alias ctk-ingest="docker run --rm -i --network=cratedb-demo ghcr.io/crate/cratedb-toolkit-ingest ctk" -alias sqlplus="docker run --rm -it --network=cratedb-demo --volume=$(PWD):/demo --env=ORACLE_PASSWORD=secret --entrypoint=sqlplus docker.io/gvenzl/oracle-free:23-slim" -``` -::: -:::{tab-item} Windows PowerShell -To make the settings persistent, add them to your PowerShell profile (`$PROFILE`). -```powershell -function crash { docker run --rm -it --network=cratedb-demo ghcr.io/crate/cratedb-toolkit-ingest crash @args } -function ctk-ingest { docker run --rm -i --network=cratedb-demo ghcr.io/crate/cratedb-toolkit-ingest ctk @args } -function sqlplus { docker run --rm -it --network=cratedb-demo --volume=${PWD}:/demo --env=ORACLE_PASSWORD=secret --entrypoint=sqlplus docker.io/gvenzl/oracle-free:23-slim @args } +docker compose up ``` -::: -:::{tab-item} Windows Command -```shell -doskey crash=docker run --rm -it --network=cratedb-demo ghcr.io/crate/cratedb-toolkit-ingest crash $* -doskey ctk-ingest=docker run --rm -i --network=cratedb-demo ghcr.io/crate/cratedb-toolkit-ingest ctk $* -doskey sqlplus=docker run --rm -it --network=cratedb-demo --volume=%cd%:/demo --env=ORACLE_PASSWORD=secret --entrypoint=sqlplus docker.io/gvenzl/oracle-free:23-slim $* -``` -::: -:::: - -## Usage +## Submit data Write a few sample records to Oracle. ```shell -cat >init.sql < Date: Sun, 5 Oct 2025 03:55:45 +0200 Subject: [PATCH 09/19] MongoDB: Use {Docker,Podman} Compose --- docs/integrate/mongodb/compose.yaml | 27 +++++++++++ docs/integrate/mongodb/usage.md | 75 +++++++++-------------------- 2 files changed, 51 insertions(+), 51 deletions(-) create mode 100644 docs/integrate/mongodb/compose.yaml diff --git a/docs/integrate/mongodb/compose.yaml b/docs/integrate/mongodb/compose.yaml new file mode 100644 index 00000000..3a15bca4 --- /dev/null +++ b/docs/integrate/mongodb/compose.yaml @@ -0,0 +1,27 @@ +# Service composition file for Docker Compose or Podman Compose + +services: + + cratedb: + image: docker.io/crate/crate:latest + ports: + - "4200:4200/tcp" + - "5432:5432/tcp" + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:4200/"] + interval: 5s + timeout: 30s + retries: 5 + + mongodb: + image: docker.io/mongo:latest + ports: + - "27017:27017/tcp" + + ctk: + image: ghcr.io/crate/cratedb-toolkit:latest + scale: 0 + + mongosh: + image: docker.io/mongo:latest + command: mongosh diff --git a/docs/integrate/mongodb/usage.md b/docs/integrate/mongodb/usage.md index 7a47d884..9e24641c 100644 --- a/docs/integrate/mongodb/usage.md +++ b/docs/integrate/mongodb/usage.md @@ -10,76 +10,49 @@ pipeline elements. ## Prerequisites -Docker is used for running all components. This approach works consistently -across Linux, macOS, and Windows. Alternatively, you can use Podman. +Use Docker or Podman to run all components. This approach works consistently +across Linux, macOS, and Windows. -Create a shared network. -```shell -docker network create cratedb-demo -``` - -Start CrateDB. -```shell -docker run --rm --name=cratedb --network=cratedb-demo \ - --publish=4200:4200 --publish=5432:5432 \ - --env=CRATE_HEAP_SIZE=2g docker.io/crate -Cdiscovery.type=single-node -``` +### Files -Start MongoDB. -```shell -docker run --rm --name=mongodb --network=cratedb-demo \ - --publish=27017:27017 \ - docker.io/mongo -``` +First, download and save all required files to your machine. +- {download}`compose.yaml` -Prepare shortcuts for the CrateDB shell, CrateDB Toolkit, and the MongoDB client -programs. +### Services -::::{tab-set} +Start services using Docker Compose or Podman Compose. +If you use Podman, replace `docker` with `podman` (or enable the podman‑docker +compatibility shim) and run `podman compose up`. -:::{tab-item} Linux and macOS -To make the settings persistent, add them to your shell profile (e.g., `~/.profile` or `~/.zshrc`). -```shell -alias crash="docker run --rm -it --network=cratedb-demo ghcr.io/crate/cratedb-toolkit crash" -alias ctk="docker run --rm -i --network=cratedb-demo ghcr.io/crate/cratedb-toolkit ctk" -alias mongosh="docker run --rm -it --network=cratedb-demo docker.io/mongo mongosh" -``` -::: -:::{tab-item} Windows PowerShell -To make the settings persistent, add them to your PowerShell profile (`$PROFILE`). -```powershell -function crash { docker run --rm -it --network=cratedb-demo ghcr.io/crate/cratedb-toolkit crash @args } -function ctk { docker run --rm -i --network=cratedb-demo ghcr.io/crate/cratedb-toolkit ctk @args } -function mongosh { docker run --rm -it --network=cratedb-demo docker.io/mongo mongosh @args } -``` -::: -:::{tab-item} Windows Command ```shell -doskey crash=docker run --rm -it --network=cratedb-demo ghcr.io/crate/cratedb-toolkit crash $* -doskey ctk=docker run --rm -i --network=cratedb-demo ghcr.io/crate/cratedb-toolkit ctk $* -doskey mongosh=docker run --rm -it --network=cratedb-demo docker.io/mongo mongosh $* +docker compose up ``` -::: - -:::: -## Usage +## Submit data Insert a record into a MongoDB collection; you can repeat this step as needed. ```shell -mongosh --host mongodb --db test --eval 'db.demo.insert({"temperature": 42.84, "humidity": 83.1})' +docker compose run --rm --no-TTY mongosh mongosh --host mongodb --db test --eval 'db.demo.insert({"temperature": 42.84, "humidity": 83.1})' ``` Invoke the data transfer pipeline. ```shell -ctk load table \ - "mongodb://mongodb/test/demo" \ - --cluster-url="crate://cratedb/doc/mongodb_demo" +docker compose run --rm --no-TTY ctk ctk load table "mongodb://mongodb/test/demo" --cluster-url="crate://cratedb/doc/mongodb_demo" ``` +## Explore data + Inspect data stored in CrateDB. ```shell -crash --hosts cratedb -c "SELECT * FROM doc.mongodb_demo" +docker compose exec cratedb crash -c "SELECT * FROM doc.mongodb_demo" +``` +```psql ++--------------------------+-----------------------------------------------------------------------------+ +| oid | data | ++--------------------------+-----------------------------------------------------------------------------+ +| 68e1cf2a1d9d9d28b1ce5f47 | {"_id": "68e1cf2a1d9d9d28b1ce5f47", "humidity": 83.1, "temperature": 42.84} | ++--------------------------+-----------------------------------------------------------------------------+ +SELECT 1 row in set (0.027 sec) ``` From 01896ec744c8b2673ee12cf09bd3fb0b80238c41 Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Sun, 5 Oct 2025 14:15:14 +0200 Subject: [PATCH 10/19] MySQL/MariaDB: Use {Docker,Podman} Compose --- docs/integrate/mysql/compose.yaml | 25 ++++++++++ docs/integrate/mysql/usage.md | 78 +++++++++++-------------------- 2 files changed, 52 insertions(+), 51 deletions(-) create mode 100644 docs/integrate/mysql/compose.yaml diff --git a/docs/integrate/mysql/compose.yaml b/docs/integrate/mysql/compose.yaml new file mode 100644 index 00000000..c51a9fb5 --- /dev/null +++ b/docs/integrate/mysql/compose.yaml @@ -0,0 +1,25 @@ +# Service composition file for Docker Compose or Podman Compose + +services: + + cratedb: + image: docker.io/crate/crate:latest + ports: + - "4200:4200/tcp" + - "5432:5432/tcp" + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:4200/"] + interval: 5s + timeout: 30s + retries: 5 + + mariadb: + image: docker.io/mariadb:latest + environment: + MARIADB_ROOT_PASSWORD: secret + ports: + - "3306:3306/tcp" + + ctk-ingest: + image: ghcr.io/crate/cratedb-toolkit-ingest:latest + scale: 0 diff --git a/docs/integrate/mysql/usage.md b/docs/integrate/mysql/usage.md index 3dae3c0b..8e4accb3 100644 --- a/docs/integrate/mysql/usage.md +++ b/docs/integrate/mysql/usage.md @@ -10,67 +10,32 @@ The data transfer is supported by the ## Prerequisites -Docker is used for running all components. This approach works consistently -across Linux, macOS, and Windows. Alternatively, you can use Podman. +Use Docker or Podman to run all components. This approach works consistently +across Linux, macOS, and Windows. -Create a shared network. -```shell -docker network create cratedb-demo -``` - -Start CrateDB. -```shell -docker run --name=cratedb --rm --network=cratedb-demo \ - --publish=4200:4200 --publish=5432:5432 --env=CRATE_HEAP_SIZE=2g \ - docker.io/crate -Cdiscovery.type=single-node -``` +### Files -Start MariaDB. -```shell -docker run --name=mariadb --rm --network=cratedb-demo \ - --publish=3306:3306 --env "MARIADB_ROOT_PASSWORD=secret" \ - docker.io/mariadb -``` +First, download and save all required files to your machine. +- {download}`compose.yaml` -Prepare shortcuts for the CrateDB shell, CrateDB Toolkit, and the MariaDB client -programs. +### Services -::::{tab-set} +Start services using Docker Compose or Podman Compose. +If you use Podman, replace `docker` with `podman` (or enable the podman‑docker +compatibility shim) and run `podman compose up`. -:::{tab-item} Linux and macOS -To make the settings persistent, add them to your shell profile (`~/.profile`). -```shell -alias crash="docker run --rm -it --network=cratedb-demo ghcr.io/crate/cratedb-toolkit crash" -alias ctk-ingest="docker run --rm -i --network=cratedb-demo ghcr.io/crate/cratedb-toolkit-ingest ctk" -alias mariadb="docker run --rm -i --network=cratedb-demo docker.io/mariadb mariadb" -``` -::: -:::{tab-item} Windows PowerShell -To make the settings persistent, add them to your PowerShell profile (`$PROFILE`). -```powershell -function crash { docker run --rm -it --network=cratedb-demo ghcr.io/crate/cratedb-toolkit crash @args } -function ctk-ingest { docker run --rm -i --network=cratedb-demo ghcr.io/crate/cratedb-toolkit-ingest ctk @args } -function mariadb { docker run --rm -i --network=cratedb-demo docker.io/mariadb mariadb @args } -``` -::: -:::{tab-item} Windows Command ```shell -doskey crash=docker run --rm -it --network=cratedb-demo ghcr.io/crate/cratedb-toolkit crash $* -doskey ctk-ingest=docker run --rm -i --network=cratedb-demo ghcr.io/crate/cratedb-toolkit-ingest ctk $* -doskey mariadb=docker run --rm -i --network=cratedb-demo docker.io/mariadb mariadb $* +docker compose up ``` -::: - -:::: -## Usage +## Submit data Write a few sample records to MariaDB. ::::{tab-set} -:::{tab-item} Linux and macOS +:::{tab-item} Linux, macOS, WSL ```shell -mariadb --protocol=tcp --host=mariadb --user=root --password=secret < Date: Sun, 5 Oct 2025 14:25:50 +0200 Subject: [PATCH 11/19] InfluxDB: Use {Docker,Podman} Compose --- docs/integrate/influxdb/compose.yaml | 30 +++++++++ docs/integrate/influxdb/usage.md | 94 +++++++++------------------- 2 files changed, 60 insertions(+), 64 deletions(-) create mode 100644 docs/integrate/influxdb/compose.yaml diff --git a/docs/integrate/influxdb/compose.yaml b/docs/integrate/influxdb/compose.yaml new file mode 100644 index 00000000..a377e02f --- /dev/null +++ b/docs/integrate/influxdb/compose.yaml @@ -0,0 +1,30 @@ +# Service composition file for Docker Compose or Podman Compose + +services: + + cratedb: + image: docker.io/crate/crate:latest + ports: + - "4200:4200/tcp" + - "5432:5432/tcp" + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:4200/"] + interval: 5s + timeout: 30s + retries: 5 + + influxdb: + image: docker.io/influxdb:2 + ports: + - "8086:8086/tcp" + environment: + DOCKER_INFLUXDB_INIT_MODE: setup + DOCKER_INFLUXDB_INIT_USERNAME: admin + DOCKER_INFLUXDB_INIT_PASSWORD: secret0000 + DOCKER_INFLUXDB_INIT_ORG: example + DOCKER_INFLUXDB_INIT_BUCKET: testdrive + DOCKER_INFLUXDB_INIT_ADMIN_TOKEN: token + + ctk: + image: ghcr.io/crate/cratedb-toolkit:latest + scale: 0 diff --git a/docs/integrate/influxdb/usage.md b/docs/integrate/influxdb/usage.md index 84ef216d..43758980 100644 --- a/docs/integrate/influxdb/usage.md +++ b/docs/integrate/influxdb/usage.md @@ -35,91 +35,57 @@ crash --host=cratedb.example.org --username=user --command='SELECT * FROM testdr ## Prerequisites -Docker is used for running all components. This approach works consistently -across Linux, macOS, and Windows. Alternatively, you can use Podman. +Use Docker or Podman to run all components. This approach works consistently +across Linux, macOS, and Windows. -Create a shared network. -```shell -docker network create cratedb-demo -``` +### Files -Start CrateDB. -```shell -docker run --rm --name=cratedb --network=cratedb-demo \ - --publish=4200:4200 \ - --volume="$PWD/var/lib/cratedb:/data" \ - docker.io/crate -Cdiscovery.type=single-node -``` - -Start InfluxDB. -```shell -docker run --rm --name=influxdb --network=cratedb-demo \ - --publish=8086:8086 \ - --env=DOCKER_INFLUXDB_INIT_MODE=setup \ - --env=DOCKER_INFLUXDB_INIT_USERNAME=admin \ - --env=DOCKER_INFLUXDB_INIT_PASSWORD=secret0000 \ - --env=DOCKER_INFLUXDB_INIT_ORG=example \ - --env=DOCKER_INFLUXDB_INIT_BUCKET=testdrive \ - --env=DOCKER_INFLUXDB_INIT_ADMIN_TOKEN=token \ - docker.io/influxdb:2 -``` +First, download and save all required files to your machine. +- {download}`compose.yaml` -Prepare shortcuts for the CrateDB shell, CrateDB Toolkit, and the InfluxDB client -programs. +### Services -::::{tab-set} +Start services using Docker Compose or Podman Compose. +If you use Podman, replace `docker` with `podman` (or enable the podman‑docker +compatibility shim) and run `podman compose up`. -:::{tab-item} Linux and macOS -To make the settings persistent, add them to your shell profile (`~/.profile`). ```shell -alias crash="docker run --rm -it --network=cratedb-demo ghcr.io/crate/cratedb-toolkit crash" -alias ctk="docker run --rm -i --network=cratedb-demo ghcr.io/crate/cratedb-toolkit ctk" -alias influx="docker exec influxdb influx" -alias influx-write="influx write --bucket=testdrive --org=example --token=token --precision=s" +docker compose up ``` -::: -:::{tab-item} Windows PowerShell -To make the settings persistent, add them to your PowerShell profile (`$PROFILE`). -```powershell -function crash { docker run --rm -it --network=cratedb-demo ghcr.io/crate/cratedb-toolkit crash @args } -function ctk { docker run --rm -i --network=cratedb-demo ghcr.io/crate/cratedb-toolkit ctk @args } -function influx { docker exec influxdb influx @args } -function influx-write { influx write --bucket=testdrive --org=example --token=token --precision=s @args } -``` -::: -:::{tab-item} Windows Command -```shell -doskey crash=docker run --rm -it --network=cratedb-demo ghcr.io/crate/cratedb-toolkit crash $* -doskey ctk=docker run --rm -i --network=cratedb-demo ghcr.io/crate/cratedb-toolkit ctk $* -doskey influx=docker exec influxdb influx $* -doskey influx-write=influx write --bucket=testdrive --org=example --token=token --precision=s $* -``` -::: - -:::: -## Usage +## Submit data Write a few sample records to InfluxDB. ```shell -influx-write "demo,region=amazonas temperature=27.4,humidity=92.3,windspeed=4.5 1588363200" -influx-write "demo,region=amazonas temperature=28.2,humidity=88.7,windspeed=4.7 1588549600" -influx-write "demo,region=amazonas temperature=27.9,humidity=91.6,windspeed=3.2 1588736000" -influx-write "demo,region=amazonas temperature=29.1,humidity=88.1,windspeed=2.4 1588922400" -influx-write "demo,region=amazonas temperature=28.6,humidity=93.4,windspeed=2.9 1589108800" +docker compose exec influxdb influx write --bucket=testdrive --org=example --token=token --precision=s "demo,region=amazonas temperature=27.4,humidity=92.3,windspeed=4.5 1588363200" +docker compose exec influxdb influx write --bucket=testdrive --org=example --token=token --precision=s "demo,region=amazonas temperature=28.2,humidity=88.7,windspeed=4.7 1588549600" +docker compose exec influxdb influx write --bucket=testdrive --org=example --token=token --precision=s "demo,region=amazonas temperature=27.9,humidity=91.6,windspeed=3.2 1588736000" +docker compose exec influxdb influx write --bucket=testdrive --org=example --token=token --precision=s "demo,region=amazonas temperature=29.1,humidity=88.1,windspeed=2.4 1588922400" +docker compose exec influxdb influx write --bucket=testdrive --org=example --token=token --precision=s "demo,region=amazonas temperature=28.6,humidity=93.4,windspeed=2.9 1589108800" ``` Invoke the data transfer pipeline, importing data from InfluxDB bucket/measurement into CrateDB schema/table. ```shell -ctk load table \ +docker compose run --rm --no-TTY ctk ctk load table \ "influxdb2://example:token@influxdb:8086/testdrive/demo" \ - --cluster-url="crate://crate@cratedb:4200/doc/testdrive" + --cluster-url="crate://crate@cratedb:4200/doc/influxdb_demo" ``` +## Explore data + Inspect data stored in CrateDB. ```shell -crash --hosts cratedb -c "SELECT * FROM doc.testdrive" +docker compose exec cratedb crash -c "SELECT * FROM doc.influxdb_demo" +``` +```psql ++---------------+----------+----------+-------------+-----------+ +| time | region | humidity | temperature | windspeed | ++---------------+----------+----------+-------------+-----------+ +| 1588549600000 | amazonas | 88.7 | 28.2 | 4.7 | +| 1588363200000 | amazonas | 92.3 | 27.4 | 4.5 | ++---------------+----------+----------+-------------+-----------+ +SELECT 2 rows in set (0.027 sec) ``` From 2e246ac154701bb07576ec9396d6388136c25a8e Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Sun, 5 Oct 2025 16:35:46 +0200 Subject: [PATCH 12/19] Usage guides: Remove line continuation characters in shell snippets It is better for Windows users. --- docs/integrate/airflow/getting-started.md | 2 +- docs/integrate/amqp/usage.md | 8 ++------ docs/integrate/dbt/usage.md | 4 +--- docs/integrate/influxdb/cloud.md | 4 +--- docs/integrate/influxdb/usage.md | 4 +--- docs/integrate/kafka/docker-python.md | 18 +++++------------- docs/integrate/kafka/index.md | 2 +- docs/integrate/mongodb/cloud.md | 4 +--- docs/integrate/mysql/usage.md | 4 +--- docs/integrate/oracle/usage.md | 4 +--- docs/integrate/postgresql/usage.md | 4 +--- docs/integrate/risingwave/apache-iceberg.md | 5 +---- docs/integrate/rsyslog/usage.md | 11 ++--------- 13 files changed, 19 insertions(+), 55 deletions(-) diff --git a/docs/integrate/airflow/getting-started.md b/docs/integrate/airflow/getting-started.md index 6e589766..cfad558c 100644 --- a/docs/integrate/airflow/getting-started.md +++ b/docs/integrate/airflow/getting-started.md @@ -17,7 +17,7 @@ You will: :::{rubric} Apache Airflow ::: Apache Airflow programmatically creates, schedules, and monitors workflows -\[[Official documentation](https://airflow.apache.org/docs/)\]. A workflow +([Official documentation](https://airflow.apache.org/docs/)). A workflow is a directed acyclic graph (DAG) where each node represents a task. Each task runs independently; the DAG tracks dependencies. Run DAGs on demand or on schedules (for example, twice a week). diff --git a/docs/integrate/amqp/usage.md b/docs/integrate/amqp/usage.md index 6c00a549..3f2496d0 100644 --- a/docs/integrate/amqp/usage.md +++ b/docs/integrate/amqp/usage.md @@ -44,16 +44,12 @@ production, configure authentication/TLS. Invoke the data transfer pipeline. ```shell -docker compose run --rm --no-TTY lorrystream lorry relay \ - "amqp://guest:guest@rabbitmq:5672/%2F?exchange=default&queue=default&routing-key=testdrive&setup=exchange,queue,bind&content-type=json" \ - "crate://cratedb/?table=amqp_demo" +docker compose run --rm --no-TTY lorrystream lorry relay "amqp://guest:guest@rabbitmq:5672/%2F?exchange=default&queue=default&routing-key=testdrive&setup=exchange,queue,bind&content-type=json" "crate://cratedb/?table=amqp_demo" ``` Publish a JSON message to AMQP. ```shell -echo '{"temperature": 42.84, "humidity": 83.1}' | \ - docker compose run --rm --no-TTY amqpcat amqpcat --producer --uri='amqp://guest:guest@rabbitmq:5672/%2F' \ - --exchange=default --queue=default --routing-key=testdrive +echo '{"temperature": 42.84, "humidity": 83.1}' | docker compose run --rm --no-TTY amqpcat amqpcat --producer --uri='amqp://guest:guest@rabbitmq:5672/%2F' --exchange=default --queue=default --routing-key=testdrive ``` ## Explore data diff --git a/docs/integrate/dbt/usage.md b/docs/integrate/dbt/usage.md index ab9aebff..5652812f 100644 --- a/docs/integrate/dbt/usage.md +++ b/docs/integrate/dbt/usage.md @@ -18,9 +18,7 @@ cluster, and a Python installation on your workstation. You can use To start a CrateDB instance for evaluation purposes, use Docker or Podman. ```shell -docker run --rm \ - --publish=4200:4200 --publish=5432:5432 \ - --env=CRATE_HEAP_SIZE=2g crate:latest +docker run --rm --publish=4200:4200 --publish=5432:5432 --env=CRATE_HEAP_SIZE=2g crate:latest ``` Install the most recent version of the [dbt-cratedb2] Python package. diff --git a/docs/integrate/influxdb/cloud.md b/docs/integrate/influxdb/cloud.md index ffb9018e..e63ac9fb 100644 --- a/docs/integrate/influxdb/cloud.md +++ b/docs/integrate/influxdb/cloud.md @@ -32,9 +32,7 @@ For InfluxDB, they can be found in the [cloud platform] itself. Now, same as before, import data from InfluxDB bucket/measurement into CrateDB schema/table. ```shell -ctk load table \ - "influxdb2://${INFLUX_ORG}:${INFLUX_TOKEN}@${INFLUX_HOST}/testdrive/demo?ssl=true" \ - --cluster-url="crate://${CRATEDB_USER}:${CRATEDB_PASSWORD}@${CRATEDB_HOST}:4200/testdrive/demo?ssl=true" +ctk load table "influxdb2://${INFLUX_ORG}:${INFLUX_TOKEN}@${INFLUX_HOST}/testdrive/demo?ssl=true" --cluster-url="crate://${CRATEDB_USER}:${CRATEDB_PASSWORD}@${CRATEDB_HOST}:4200/testdrive/demo?ssl=true" ``` :::{note} diff --git a/docs/integrate/influxdb/usage.md b/docs/integrate/influxdb/usage.md index 43758980..e248c947 100644 --- a/docs/integrate/influxdb/usage.md +++ b/docs/integrate/influxdb/usage.md @@ -67,9 +67,7 @@ docker compose exec influxdb influx write --bucket=testdrive --org=example --tok Invoke the data transfer pipeline, importing data from InfluxDB bucket/measurement into CrateDB schema/table. ```shell -docker compose run --rm --no-TTY ctk ctk load table \ - "influxdb2://example:token@influxdb:8086/testdrive/demo" \ - --cluster-url="crate://crate@cratedb:4200/doc/influxdb_demo" +docker compose run --rm --no-TTY ctk ctk load table "influxdb2://example:token@influxdb:8086/testdrive/demo" --cluster-url="crate://crate@cratedb:4200/doc/influxdb_demo" ``` ## Explore data diff --git a/docs/integrate/kafka/docker-python.md b/docs/integrate/kafka/docker-python.md index cf872552..ab0cf124 100644 --- a/docs/integrate/kafka/docker-python.md +++ b/docs/integrate/kafka/docker-python.md @@ -66,11 +66,7 @@ CREATE TABLE IF NOT EXISTS sensor_readings ( But this can also be done using `curl`: ```bash -curl -sS -H 'Content-Type: application/json' \ - -X POST http://localhost:4200/_sql \ - -d '{ - "stmt":"CREATE TABLE IF NOT EXISTS sensor_readings (device_id TEXT, ts TIMESTAMPTZ, temperature DOUBLE PRECISION, humidity DOUBLE PRECISION, PRIMARY KEY (device_id, ts))" - }' +curl -sS -H 'Content-Type: application/json' -X POST http://localhost:4200/_sql -d '{"stmt":"CREATE TABLE IF NOT EXISTS sensor_readings (device_id TEXT, ts TIMESTAMPTZ, temperature DOUBLE PRECISION, humidity DOUBLE PRECISION, PRIMARY KEY (device_id, ts))"}' ``` ### Create a Kafka topic and send a couple of messages @@ -79,11 +75,9 @@ Creating a Kafka topic can be done in several ways, we are selecting to use `docker exec` in this way: ```bash -docker exec -it kafka kafka-topics.sh --create \ - --topic sensors --bootstrap-server kafka:9092 --partitions 3 --replication-factor 1 +docker exec -it kafka kafka-topics.sh --create --topic sensors --bootstrap-server kafka:9092 --partitions 3 --replication-factor 1 -docker exec -it kafka kafka-console-producer.sh \ - --bootstrap-server kafka:9092 --topic sensors <<'EOF' +docker exec -it kafka kafka-console-producer.sh --bootstrap-server kafka:9092 --topic sensors <<'EOF' {"device_id":"alpha","ts":"2025-08-19T12:00:00Z","temperature":21.4,"humidity":48.0} {"device_id":"alpha","ts":"2025-08-19T12:01:00Z","temperature":21.5,"humidity":47.6} {"device_id":"beta","ts":"2025-08-19T12:00:00Z","temperature":19.8,"humidity":55.1} @@ -149,14 +143,12 @@ python quick_consumer.py ``` :::{tip} -This shows the custom client path: transform/filter as you like, do idempotent upserts on (device\_id, ts), and batch writes for speed. +This shows the custom client path: transform/filter as you like, do idempotent upserts on (device_id, ts), and batch writes for speed. ::: ## Verifying the data Use `curl` to submit a `SELECT` statement that verifies data has been stored in CrateDB. ```bash -curl -sS -H 'Content-Type: application/json' -X POST http://localhost:4200/_sql \ - -d '{"stmt":"SELECT device_id, ts, temperature, humidity FROM sensor_readings ORDER BY ts LIMIT 10"}' \ - | jq +curl -sS -H 'Content-Type: application/json' -X POST http://localhost:4200/_sql -d '{"stmt":"SELECT device_id, ts, temperature, humidity FROM sensor_readings ORDER BY ts LIMIT 10"}' | jq ``` diff --git a/docs/integrate/kafka/index.md b/docs/integrate/kafka/index.md index 8aa216bd..ef130770 100644 --- a/docs/integrate/kafka/index.md +++ b/docs/integrate/kafka/index.md @@ -88,7 +88,7 @@ In practice, teams usually start containerised (for dev/test or early projects) ## Key design considerations -* **Topic & partition strategy** – Align Kafka partitions with expected throughput and consumer parallelism - aim for stable keys (e.g., device\_id) to keep ordering where needed. +* **Topic & partition strategy** – Align Kafka partitions with expected throughput and consumer parallelism - aim for stable keys (e.g., device_id) to keep ordering where needed. * **Table modelling in CrateDB** – Choose primary keys and partitioning (e.g., by month on a timestamp column) to balance write speed and query performance - define shard count per table. * **Upserts & deduplication** – Include a stable event key (id, source+timestamp) to make writes idempotent when possible. * **Batching & back-pressure** – Tune sink batch size and retries to match CrateDB ingest capacity while keeping latency low. diff --git a/docs/integrate/mongodb/cloud.md b/docs/integrate/mongodb/cloud.md index f24ac0b1..9bd41dd6 100644 --- a/docs/integrate/mongodb/cloud.md +++ b/docs/integrate/mongodb/cloud.md @@ -29,9 +29,7 @@ For MongoDB, they can be found in the [cloud platform] itself. Now, same as before, import data from MongoDB database/collection into CrateDB schema/table. ```shell -ctk load table \ - "mongodb+srv://admin:a..1@cluster0.nttj7.mongodb.net/testdrive/demo" \ - --cluster-url='crate://admin:-..n@gray-wicket.aks1.westeurope.azure.cratedb.net:4200/testdrive/demo?ssl=true' +ctk load table "mongodb+srv://admin:a..1@cluster0.nttj7.mongodb.net/testdrive/demo" --cluster-url='crate://admin:-..n@gray-wicket.aks1.westeurope.azure.cratedb.net:4200/testdrive/demo?ssl=true' ``` :::{note} diff --git a/docs/integrate/mysql/usage.md b/docs/integrate/mysql/usage.md index 8e4accb3..c78f64ef 100644 --- a/docs/integrate/mysql/usage.md +++ b/docs/integrate/mysql/usage.md @@ -66,9 +66,7 @@ docker compose run --rm --no-TTY mariadb mariadb @args -e $sql Invoke the data transfer pipeline. ```shell -docker compose run --rm --no-TTY ctk-ingest ctk load table \ - "mysql://root:secret@mariadb:3306/?table=test.demo" \ - --cluster-url="crate://crate:crate@cratedb:4200/doc/mysql_demo" +docker compose run --rm --no-TTY ctk-ingest ctk load table "mysql://root:secret@mariadb:3306/?table=test.demo" --cluster-url="crate://crate:crate@cratedb:4200/doc/mysql_demo" ``` ## Explore data diff --git a/docs/integrate/oracle/usage.md b/docs/integrate/oracle/usage.md index 5ad2c10c..05abff82 100644 --- a/docs/integrate/oracle/usage.md +++ b/docs/integrate/oracle/usage.md @@ -37,9 +37,7 @@ docker compose run --rm --no-TTY sqlplus sys/secret@oracledb/freepdb1 as sysdba Invoke the data transfer pipeline. ```shell -docker compose run --rm --no-TTY ctk-ingest ctk load table \ - "oracle://sys:secret@oracledb:1521/?service_name=freepdb1&table=sys.demo&mode=sysdba" \ - --cluster-url="crate://crate:crate@cratedb:4200/doc/oracle_demo" +docker compose run --rm --no-TTY ctk-ingest ctk load table "oracle://sys:secret@oracledb:1521/?service_name=freepdb1&table=sys.demo&mode=sysdba" --cluster-url="crate://crate:crate@cratedb:4200/doc/oracle_demo" ``` ## Explore data diff --git a/docs/integrate/postgresql/usage.md b/docs/integrate/postgresql/usage.md index 2ec309fa..19ec4a72 100644 --- a/docs/integrate/postgresql/usage.md +++ b/docs/integrate/postgresql/usage.md @@ -53,9 +53,7 @@ SQL Invoke the data transfer pipeline. ```shell -docker compose run --rm --no-TTY ctk-ingest ctk load table \ - "postgresql://postgres:postgres@postgresql:5432/test?table=public.demo" \ - --cluster-url="crate://crate:crate@cratedb:4200/doc/postgresql_demo" +docker compose run --rm --no-TTY ctk-ingest ctk load table "postgresql://postgres:postgres@postgresql:5432/test?table=public.demo" --cluster-url="crate://crate:crate@cratedb:4200/doc/postgresql_demo" ``` ## Explore data diff --git a/docs/integrate/risingwave/apache-iceberg.md b/docs/integrate/risingwave/apache-iceberg.md index c28ae4a3..030d9303 100644 --- a/docs/integrate/risingwave/apache-iceberg.md +++ b/docs/integrate/risingwave/apache-iceberg.md @@ -16,10 +16,7 @@ For this example, we will spin up 3 containers using [Podman] Let's first start a [Minio] instance: ```bash -podman run -d --name minio -p 9000:9000 -p 9001:9001 \ - -e MINIO_ROOT_USER=minioadmin \ - -e MINIO_ROOT_PASSWORD=minioadmin \ - quay.io/minio/minio server /data --console-address ":9001" +podman run -d --name minio -p 9000:9000 -p 9001:9001 -e MINIO_ROOT_USER=minioadmin -e MINIO_ROOT_PASSWORD=minioadmin quay.io/minio/minio server /data --console-address ":9001" ``` Now let's create a bucket called `warehouse`, for this point a browser to diff --git a/docs/integrate/rsyslog/usage.md b/docs/integrate/rsyslog/usage.md index cf28dd35..881a4a9e 100644 --- a/docs/integrate/rsyslog/usage.md +++ b/docs/integrate/rsyslog/usage.md @@ -23,10 +23,7 @@ setup. First, start CrateDB. For production, use a dedicated cluster. For this demo, run a single‑node container: ```bash -sudo docker run -d --name cratedb \ - -p 4200:4200 -p 5432:5432 \ - -e CRATE_HEAP_SIZE=1g \ - crate:latest -Cdiscovery.type=single-node +sudo docker run -d --name cratedb -p 4200:4200 -p 5432:5432 -e CRATE_HEAP_SIZE=1g crate:latest -Cdiscovery.type=single-node ``` Next, create a table for logs. Open `http://localhost:4200/#!/console` or invoke `crash` and run: @@ -87,11 +84,7 @@ If you are interested in more advanced setups involving queuing for additional r To generate logs, run a [MediaWiki](https://www.mediawiki.org/wiki/MediaWiki) container and forward its logs to rsyslog: ```bash -sudo docker run --name mediawiki \ - -p 80:80 -d \ - --log-driver syslog \ - --log-opt syslog-address=unixgram:///dev/log \ - mediawiki +sudo docker run --name mediawiki -p 80:80 -d --log-driver syslog --log-opt syslog-address=unixgram:///dev/log mediawiki ``` Open `http://localhost/` to see the MediaWiki setup page. From 5e0d24878843aefdd7d1efd1d3e133ef24a1ab1f Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Sun, 5 Oct 2025 18:18:20 +0200 Subject: [PATCH 13/19] Usage guides: Remove "-Cdiscovery.type=single-node" It is better for "Docker on Windows" users. --- docs/feature/search/fts/analyzer.md | 2 +- docs/integrate/debezium/tutorial.md | 2 +- docs/integrate/marquez/usage.md | 2 +- docs/integrate/risingwave/apache-iceberg.md | 2 +- docs/integrate/rsyslog/usage.md | 2 +- docs/integrate/superset/sandbox.md | 2 +- docs/integrate/superset/usage.md | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/feature/search/fts/analyzer.md b/docs/feature/search/fts/analyzer.md index b2133dd6..7a73b106 100644 --- a/docs/feature/search/fts/analyzer.md +++ b/docs/feature/search/fts/analyzer.md @@ -168,7 +168,7 @@ s(900516492,3,'Bill',n,1,0). docker run --rm -it --name=cratedb --publish=4200:4200 --env=CRATE_HEAP_SIZE=2g \ --volume="$PWD/synonyms-solr.txt:/crate/config/synonyms-solr.txt" \ --volume="$PWD/synonyms-wordnet.txt:/crate/config/synonyms-wordnet.txt" \ - crate -Cdiscovery.type=single-node + crate ``` This example uses the `synonyms-solr.txt` in Solr format. diff --git a/docs/integrate/debezium/tutorial.md b/docs/integrate/debezium/tutorial.md index 9b39c996..e9c8ccb0 100644 --- a/docs/integrate/debezium/tutorial.md +++ b/docs/integrate/debezium/tutorial.md @@ -77,7 +77,7 @@ We will need a CrateDB instance, for the purpose of this example we can spin one ```bash sudo apt install docker.io -sudo docker run --publish 4200:4200 --publish 5432:5432 --env CRATE_HEAP_SIZE=1g crate:latest -Cdiscovery.type=single-node +sudo docker run --publish 4200:4200 --publish 5432:5432 --env CRATE_HEAP_SIZE=1g crate:latest ``` Now we need to run a couple of SQL commands on this instance, an easy way to do this is using the Admin UI that can be accessed navigating with a web browser to port 4200 on the server where CrateDB is running, for instance `http://localhost:4200` and then open the console (second icon from the top on the left-hand side navigation bar). diff --git a/docs/integrate/marquez/usage.md b/docs/integrate/marquez/usage.md index 2ab32dea..4a5ce26d 100644 --- a/docs/integrate/marquez/usage.md +++ b/docs/integrate/marquez/usage.md @@ -65,7 +65,7 @@ astro dev start And we will start a single-node local CrateDB instance using port 5436 for the PostgreSQL wire protocol interface: ```bash -sudo docker run -d --name cratedb --publish=4200:4200 --publish=5436:5432 --env CRATE_HEAP_SIZE=1g crate/crate:5.9.5 -Cdiscovery.type=single-node +sudo docker run -d --name cratedb --publish=4200:4200 --publish=5436:5432 --env CRATE_HEAP_SIZE=1g crate/crate:5.9.5 ``` (NB this will return immediately once the image is downloaded but CrateDB may take a few seconds to start) diff --git a/docs/integrate/risingwave/apache-iceberg.md b/docs/integrate/risingwave/apache-iceberg.md index 030d9303..cbd9dd3c 100644 --- a/docs/integrate/risingwave/apache-iceberg.md +++ b/docs/integrate/risingwave/apache-iceberg.md @@ -32,7 +32,7 @@ podman run -d --name risingwave -it -p 4566:4566 -p 5691:5691 docker.io/risingwa And finally, an instance of CrateDB: ```bash -podman run -d --name cratedb --publish=4200:4200 --publish=5432:5432 --env CRATE_HEAP_SIZE=1g docker.io/crate/crate:5.10.7 -Cdiscovery.type=single-node +podman run -d --name cratedb --publish=4200:4200 --publish=5432:5432 --env CRATE_HEAP_SIZE=1g docker.io/crate/crate:5.10.7 ``` We will need three terminals for this demonstration. diff --git a/docs/integrate/rsyslog/usage.md b/docs/integrate/rsyslog/usage.md index 881a4a9e..be086422 100644 --- a/docs/integrate/rsyslog/usage.md +++ b/docs/integrate/rsyslog/usage.md @@ -23,7 +23,7 @@ setup. First, start CrateDB. For production, use a dedicated cluster. For this demo, run a single‑node container: ```bash -sudo docker run -d --name cratedb -p 4200:4200 -p 5432:5432 -e CRATE_HEAP_SIZE=1g crate:latest -Cdiscovery.type=single-node +sudo docker run -d --name cratedb -p 4200:4200 -p 5432:5432 -e CRATE_HEAP_SIZE=1g crate:latest ``` Next, create a table for logs. Open `http://localhost:4200/#!/console` or invoke `crash` and run: diff --git a/docs/integrate/superset/sandbox.md b/docs/integrate/superset/sandbox.md index 192de57c..6722584f 100644 --- a/docs/integrate/superset/sandbox.md +++ b/docs/integrate/superset/sandbox.md @@ -13,7 +13,7 @@ You will need Bash, Docker, Git, and Python to be installed on your workstation. Start CrateDB using Docker. ```shell -docker run --rm --publish=4200:4200 --publish=5432:5432 --name=cratedb --env CRATE_HEAP_SIZE=1g crate:latest -Cdiscovery.type=single-node +docker run --rm --publish=4200:4200 --publish=5432:5432 --name=cratedb --env CRATE_HEAP_SIZE=1g crate:latest ``` Create an example table and insert a single record. diff --git a/docs/integrate/superset/usage.md b/docs/integrate/superset/usage.md index b734499a..57244507 100644 --- a/docs/integrate/superset/usage.md +++ b/docs/integrate/superset/usage.md @@ -44,7 +44,7 @@ docker run --interactive --rm --pull=always \ --publish=4200:4200 --publish=5432:5432 \ --name=cratedb \ --env CRATE_HEAP_SIZE=2g \ - crate:latest -Cdiscovery.type=single-node + crate:latest ``` Run Superset server. From ead3f8cc4d49dea2085088223eb44af818f19ec5 Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Sun, 5 Oct 2025 20:50:48 +0200 Subject: [PATCH 14/19] Usage guides: Tune down default CrateDB heap size to 1 GB --- docs/admin/troubleshooting/jcmd/docker.md | 2 +- docs/feature/search/fts/analyzer.md | 2 +- docs/install/container/docker.rst | 10 +++++----- docs/integrate/airflow/data-retention-hot-cold.md | 6 +++--- docs/integrate/dbt/usage.md | 2 +- docs/integrate/kafka/kafka-connect.md | 2 +- docs/integrate/superset/usage.md | 2 +- 7 files changed, 13 insertions(+), 13 deletions(-) diff --git a/docs/admin/troubleshooting/jcmd/docker.md b/docs/admin/troubleshooting/jcmd/docker.md index eb6c58f2..af09a0da 100644 --- a/docs/admin/troubleshooting/jcmd/docker.md +++ b/docs/admin/troubleshooting/jcmd/docker.md @@ -64,7 +64,7 @@ Let's start by starting a `cratedb` container. ```console docker run --rm -it --name=cratedb \ --publish=4200:4200 --publish=5432:5432 \ - --env=CRATE_HEAP_SIZE=2g crate/crate:nightly \ + --env=CRATE_HEAP_SIZE=1g crate/crate:nightly \ -Cdiscovery.type=single-node ``` diff --git a/docs/feature/search/fts/analyzer.md b/docs/feature/search/fts/analyzer.md index 7a73b106..fd6fcec8 100644 --- a/docs/feature/search/fts/analyzer.md +++ b/docs/feature/search/fts/analyzer.md @@ -165,7 +165,7 @@ s(900516492,3,'Bill',n,1,0). ``` ```shell -docker run --rm -it --name=cratedb --publish=4200:4200 --env=CRATE_HEAP_SIZE=2g \ +docker run --rm -it --name=cratedb --publish=4200:4200 --env=CRATE_HEAP_SIZE=1g \ --volume="$PWD/synonyms-solr.txt:/crate/config/synonyms-solr.txt" \ --volume="$PWD/synonyms-wordnet.txt:/crate/config/synonyms-wordnet.txt" \ crate diff --git a/docs/install/container/docker.rst b/docs/install/container/docker.rst index 94fd80d1..f68fb671 100644 --- a/docs/install/container/docker.rst +++ b/docs/install/container/docker.rst @@ -83,7 +83,7 @@ Breaking the command down: - Puts the container into the ``crate`` network and maps port ``4201`` on your host machine to port ``4200`` on the container (admin UI). - Defines the environment variable:ref:`CRATE_HEAP_SIZE `, - which is used by CrateDB to allocate 2G for its heap. + which is used by CrateDB to allocate 1 GB for its heap memory. - Runs the command ``crate`` inside the container with parameters: * ``network.host``: The ``_site_`` value results in the binding of the CrateDB process to a site-local IP address. @@ -298,7 +298,7 @@ define your services like this: restart_policy: condition: on-failure environment: - - CRATE_HEAP_SIZE=2g + - CRATE_HEAP_SIZE=1g cratedb02: image: crate:latest @@ -320,7 +320,7 @@ define your services like this: restart_policy: condition: on-failure environment: - - CRATE_HEAP_SIZE=2g + - CRATE_HEAP_SIZE=1g cratedb03: image: crate:latest @@ -342,7 +342,7 @@ define your services like this: restart_policy: condition: on-failure environment: - - CRATE_HEAP_SIZE=2g + - CRATE_HEAP_SIZE=1g In the file above: @@ -485,7 +485,7 @@ example:: $ docker run -d \ --cpus 1.5 \ - --memory 2g \ + --memory 1g \ --env CRATE_HEAP_SIZE=1g \ crate \ crate -Cnetwork.host=_site_ diff --git a/docs/integrate/airflow/data-retention-hot-cold.md b/docs/integrate/airflow/data-retention-hot-cold.md index 1c2512dd..7892f53a 100644 --- a/docs/integrate/airflow/data-retention-hot-cold.md +++ b/docs/integrate/airflow/data-retention-hot-cold.md @@ -34,7 +34,7 @@ services: "-Cgateway.expected_data_nodes=3", "-Cgateway.recover_after_data_nodes=2"] environment: - - CRATE_HEAP_SIZE=2g + - CRATE_HEAP_SIZE=1g cratedb02: image: crate:latest @@ -53,7 +53,7 @@ services: "-Cgateway.expected_data_nodes=3", "-Cgateway.recover_after_data_nodes=2"] environment: - - CRATE_HEAP_SIZE=2g + - CRATE_HEAP_SIZE=1g cratedb03: image: crate:latest @@ -72,7 +72,7 @@ services: "-Cgateway.expected_data_nodes=3", "-Cgateway.recover_after_data_nodes=2"] environment: - - CRATE_HEAP_SIZE=2g + - CRATE_HEAP_SIZE=1g ``` Start the cluster with `docker compose up`. For details, see the linked documentation. diff --git a/docs/integrate/dbt/usage.md b/docs/integrate/dbt/usage.md index 5652812f..b27d49fc 100644 --- a/docs/integrate/dbt/usage.md +++ b/docs/integrate/dbt/usage.md @@ -18,7 +18,7 @@ cluster, and a Python installation on your workstation. You can use To start a CrateDB instance for evaluation purposes, use Docker or Podman. ```shell -docker run --rm --publish=4200:4200 --publish=5432:5432 --env=CRATE_HEAP_SIZE=2g crate:latest +docker run --rm --publish=4200:4200 --publish=5432:5432 --env=CRATE_HEAP_SIZE=1g crate:latest ``` Install the most recent version of the [dbt-cratedb2] Python package. diff --git a/docs/integrate/kafka/kafka-connect.md b/docs/integrate/kafka/kafka-connect.md index 6e437ca5..007c9b58 100644 --- a/docs/integrate/kafka/kafka-connect.md +++ b/docs/integrate/kafka/kafka-connect.md @@ -205,7 +205,7 @@ services: - "4200:4200" - "5432:5432" environment: - CRATE_HEAP_SIZE: 2g + CRATE_HEAP_SIZE: 1g ``` Each tool in this stack plays an integral role for proper functioning of the diff --git a/docs/integrate/superset/usage.md b/docs/integrate/superset/usage.md index 57244507..b04c6ba8 100644 --- a/docs/integrate/superset/usage.md +++ b/docs/integrate/superset/usage.md @@ -43,7 +43,7 @@ Start CrateDB using Docker. docker run --interactive --rm --pull=always \ --publish=4200:4200 --publish=5432:5432 \ --name=cratedb \ - --env CRATE_HEAP_SIZE=2g \ + --env CRATE_HEAP_SIZE=1g \ crate:latest ``` From c6ab900c51054bee0f6d8d0aa02d343a8d5ba4ca Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Mon, 6 Oct 2025 03:24:55 +0200 Subject: [PATCH 15/19] Kestra: Copy-editing --- docs/integrate/kestra/usage.md | 59 +++++++++++++++++++++++----------- 1 file changed, 40 insertions(+), 19 deletions(-) diff --git a/docs/integrate/kestra/usage.md b/docs/integrate/kestra/usage.md index 4a40b125..de80faeb 100644 --- a/docs/integrate/kestra/usage.md +++ b/docs/integrate/kestra/usage.md @@ -1,20 +1,23 @@ (kestra-usage)= -# Setting up data pipelines with CrateDB and Kestra +# Data pipelines with CrateDB and Kestra -[Kestra.io](https://kestra.io/) provides open‑source workflow automation and -orchestration. You can use it to automate and manage complex workflows +:::{div} sd-text-muted +[Kestra] provides open‑source workflow automation and orchestration. +::: + +You can use it to automate and manage complex workflows efficiently. It integrates with Postgres, Git, Docker, Kubernetes, and more. The web UI lets you create, modify, and manage workflows without writing code. In this usage guide, we will show you how CrateDB integrates with Kestra using the PostgreSQL plugin to create an efficient and scalable data pipeline. -## Running Kestra on Docker +## Run Kestra on Docker You can quickly start Kestra using Docker. First, install Docker on your machine if you haven't already. ```bash -docker run -d -p 8080:8080 kestra/kestra:latest +docker run --rm --name=kestra --publish=8080:8080 docker.io/kestra/kestra:latest ``` This starts the Kestra server on your local machine. Access it by navigating to [http://localhost:8080](http://localhost:8080/) in your web browser. @@ -23,26 +26,24 @@ From there, you can start creating workflows and automating your processes using ![Kestra UI home screen](https://us1.discourse-cdn.com/flex020/uploads/crate/original/1X/80c3eb1bbc2de07a343bc56b1a5db24cf0569df7.png){width=690px height=290px} -## Deploy a CrateDB Cloud Cluster +## Deploy CrateDB clusters To deploy a new cluster on CrateDB Cloud, sign up for a [CrateDB Cloud account](https://console.cratedb.cloud/). -New organizations receive a free trial credit (as of September 2025) for +New organizations receive a free trial credit for cluster deployment, scaling, and other operations. After signing up, create a cluster by selecting *Create Cluster* and choosing your preferred cloud provider and region. In this example, we use a 1‑node cluster with 4 GiB of storage, sufficient for development and low‑traffic applications. -![CrateDB Cloud cluster configuration screen showing a 1-node cluster with 4 GiB storage option](https://us1.discourse-cdn.com/flex020/uploads/crate/original/1X/5c4c24dde906df6004392356138637444844f57d.png) +![CrateDB Cloud cluster configuration screen showing a 1-node cluster with 4 GiB storage option](https://us1.discourse-cdn.com/flex020/uploads/crate/original/1X/5c4c24dde906df6004392356138637444844f57d.png){height=480px} Once your cluster is up and running, use CrateDB's powerful distributed SQL database features via the web-based Admin UI. -## Move data between clusters with Kestra - -You can move data between CrateDB clusters in several ways. The following example shows how to do this with Kestra. +## Provision databases -Now, let's import some data on the first cluster. Navigate to the cluster overview page and click the *Learn how to import data* link. +Let's import some data on the first cluster. Navigate to the cluster overview page and click the *Learn how to import data* link. ```sql CREATE TABLE "nyc_taxi" ( @@ -69,15 +70,29 @@ CREATE TABLE "nyc_taxi" ( COPY "nyc_taxi" FROM 'https://s3.amazonaws.com/crate.sampledata/nyc.yellowcab/yc.2019.07.gz' WITH (compression = 'gzip'); ``` -On the second cluster, create an empty `nyc_taxi` table. As a next step, we will create a new Kestra Flow to move data between clusters. +On the second cluster, create an empty `nyc_taxi` table. + +## Transfer data + +You can move data between CrateDB clusters in several ways. The following example shows how to do this with Kestra. + +### Define workflow + +As a next step, we will create a new Kestra Flow to move data between clusters. Flows in Kestra implement workflows. Each flow uses a declarative YAML model and contains all the tasks in the order they will run. A flow must have an identifier (id), a namespace, and a list of tasks. -### Query CrateDB table +### Define source + +To read from a PostgreSQL-compatible database, such as CrateDB, +Kestra offers the `io.kestra.plugin.jdbc.postgresql.Query` plugin. -To query a PostgreSQL-compatible database, such as CrateDB, Kestra offers `io.kestra.plugin.jdbc.postgresql.Query` plugin. The *Query* task allows users to execute custom SQL queries against a database and use the results in their workflow. This task offers various parameters such as auto-committing SQL statements, specifying access-control rules, and storing fetch results. +The *Query* task allows users to execute custom SQL queries against +a database and use the results in their workflow. +It offers various parameters such as auto-committing SQL statements, +specifying access-control rules, and storing fetch results. The following snippet shows the declaration of our workflow and the specification of the first task that selects data from the `nyc_taxi` table and runs the query on the first CrateDB cluster: @@ -97,7 +112,7 @@ tasks: In this task, we set the `store` parameter to `true` to store results for the next task. -### Insert data into the second table +### Define target In Kestra, a batch task is a type of task that allows you to fetch rows from a table and bulk insert them into another one. We will use an instance of a batch task to fetch the results of the first task and to insert the results into the table on the second cluster. @@ -121,12 +136,14 @@ The `output` object captures information about the task results, including any r In our example, `output.query.uri` refers to the URI of the resource the previous task created. Specifically, it refers to the URI of the database records returned by the `SELECT` statement. +### Define result + Finally, once the data are imported to the second table, let’s create a new task that selects data from that table: ```yaml - id: select type: io.kestra.plugin.jdbc.postgresql.Query - url: jdbc:postgresql://kestra-testing-cluster2.aks1.eastus2.azure.cratedb.net:5432/ + url: jdbc:postgresql://cratedb-kestra2.aks1.eastus2.azure.cratedb.net:5432/ username: admin password: my_super_secret_password sql: SELECT MAX_BY(passenger_count, fare_amount) FROM doc.nyc_taxi @@ -151,9 +168,13 @@ Finally, let's check the data in the second cluster. As illustrated below, exact ## Wrap up -If you need to automatically manage CrateDB data pipelines, [kestra.io](https://kestra.io/) provides a strong solution. +If you need to automatically manage CrateDB data pipelines, [Kestra] provides a strong solution. It lets you define workflows without writing code and integrates with PostgreSQL (and CrateDB), Kubernetes, Docker, Git, and more. In this usage guide, we have also shown how to deploy your CrateDB cluster in a few clicks. If you want to try it out and enjoy all of the CrateDB features, sign up for the [CrateDB Cloud](https://console.cratedb.cloud/) trial. -To learn more about updates, features, and other questions you might have, join our [CrateDB](https://community.cratedb.com/) community. +To learn more about updates, features, and other questions you might have, join the [CrateDB community]. + + +[CrateDB community]: https://community.cratedb.com/ +[Kestra]: https://kestra.io/ From 81251522033f17b34df0ec11aa23f3db93c0612c Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Mon, 6 Oct 2025 03:36:30 +0200 Subject: [PATCH 16/19] rsyslog: Copy-editing --- docs/integrate/rsyslog/usage.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/integrate/rsyslog/usage.md b/docs/integrate/rsyslog/usage.md index be086422..f01ccc4d 100644 --- a/docs/integrate/rsyslog/usage.md +++ b/docs/integrate/rsyslog/usage.md @@ -1,5 +1,9 @@ (rsyslog-usage)= -# Store server logs on CrateDB for fast search and aggregations +# Using rsyslog to store server logs in CrateDB + +:::{div} sd-text-muted +Store server logs in CrateDB for fast search and aggregations. +::: ## Introduction @@ -97,7 +101,7 @@ CrateDB now stores new rows in `doc.systemevents`, with `syslogtag` matching the Use {ref}`crate-reference:predicates_match` to find specific error messages: ```sql -SELECT devicereportedtime,message +SELECT devicereportedtime, message FROM doc.systemevents WHERE MATCH(message_ft, 'Could not reliably determine') USING PHRASE ORDER BY 1 DESC; @@ -115,7 +119,7 @@ ORDER BY 1 DESC; Show the top log sources by event count: ```sql -SELECT syslogtag,count(*) +SELECT syslogtag, count(*) FROM doc.systemevents GROUP BY 1 ORDER BY 2 DESC From 59dd2613b110834e9eff85e0707583b4a4409f25 Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Mon, 6 Oct 2025 04:18:06 +0200 Subject: [PATCH 17/19] Usage guides: Compose: Replace non-spec `scale` with `profiles` --- docs/integrate/amqp/compose.yaml | 3 ++- docs/integrate/collectd/compose.yaml | 2 +- docs/integrate/influxdb/compose.yaml | 2 +- docs/integrate/mongodb/compose.yaml | 3 ++- docs/integrate/mqtt/compose.yaml | 2 +- docs/integrate/mysql/compose.yaml | 2 +- docs/integrate/opentelemetry/collector/compose.yaml | 6 +++--- docs/integrate/opentelemetry/telegraf/compose.yaml | 4 ++-- docs/integrate/oracle/compose.yaml | 4 ++-- docs/integrate/postgresql/compose.yaml | 4 ++-- docs/integrate/statsd/compose.yaml | 4 ++-- 11 files changed, 19 insertions(+), 17 deletions(-) diff --git a/docs/integrate/amqp/compose.yaml b/docs/integrate/amqp/compose.yaml index c7e77ae7..0d2aca46 100644 --- a/docs/integrate/amqp/compose.yaml +++ b/docs/integrate/amqp/compose.yaml @@ -20,7 +20,8 @@ services: lorrystream: image: ghcr.io/daq-tools/lorrystream:latest - scale: 0 + profiles: ["cli"] amqpcat: image: docker.io/cloudamqp/amqpcat:latest + profiles: ["cli"] diff --git a/docs/integrate/collectd/compose.yaml b/docs/integrate/collectd/compose.yaml index ca7f615e..45eb23a1 100644 --- a/docs/integrate/collectd/compose.yaml +++ b/docs/integrate/collectd/compose.yaml @@ -38,4 +38,4 @@ services: psql: image: docker.io/postgres:18 - scale: 0 + profiles: ["cli"] diff --git a/docs/integrate/influxdb/compose.yaml b/docs/integrate/influxdb/compose.yaml index a377e02f..e1c49b01 100644 --- a/docs/integrate/influxdb/compose.yaml +++ b/docs/integrate/influxdb/compose.yaml @@ -27,4 +27,4 @@ services: ctk: image: ghcr.io/crate/cratedb-toolkit:latest - scale: 0 + profiles: ["cli"] diff --git a/docs/integrate/mongodb/compose.yaml b/docs/integrate/mongodb/compose.yaml index 3a15bca4..413909bf 100644 --- a/docs/integrate/mongodb/compose.yaml +++ b/docs/integrate/mongodb/compose.yaml @@ -20,8 +20,9 @@ services: ctk: image: ghcr.io/crate/cratedb-toolkit:latest - scale: 0 + profiles: ["cli"] mongosh: image: docker.io/mongo:latest command: mongosh + profiles: ["cli"] diff --git a/docs/integrate/mqtt/compose.yaml b/docs/integrate/mqtt/compose.yaml index 581274cd..0ce16002 100644 --- a/docs/integrate/mqtt/compose.yaml +++ b/docs/integrate/mqtt/compose.yaml @@ -22,4 +22,4 @@ services: lorrystream: image: ghcr.io/daq-tools/lorrystream:latest - scale: 0 + profiles: ["cli"] diff --git a/docs/integrate/mysql/compose.yaml b/docs/integrate/mysql/compose.yaml index c51a9fb5..d3dbba85 100644 --- a/docs/integrate/mysql/compose.yaml +++ b/docs/integrate/mysql/compose.yaml @@ -22,4 +22,4 @@ services: ctk-ingest: image: ghcr.io/crate/cratedb-toolkit-ingest:latest - scale: 0 + profiles: ["cli"] diff --git a/docs/integrate/opentelemetry/collector/compose.yaml b/docs/integrate/opentelemetry/collector/compose.yaml index ea1836ed..2f335887 100644 --- a/docs/integrate/opentelemetry/collector/compose.yaml +++ b/docs/integrate/opentelemetry/collector/compose.yaml @@ -44,14 +44,14 @@ services: nc: image: docker.io/toolbelt/netcat:2025-08-23 - scale: 0 + profiles: ["cli"] uv: image: ghcr.io/astral-sh/uv:python3.13-trixie-slim volumes: - ./:/src - - uv-cache:/root/.cache/uv:cached - scale: 0 + - uv-cache:/root/.cache/uv + profiles: ["cli"] volumes: uv-cache: diff --git a/docs/integrate/opentelemetry/telegraf/compose.yaml b/docs/integrate/opentelemetry/telegraf/compose.yaml index 1ab7330c..0476a08d 100644 --- a/docs/integrate/opentelemetry/telegraf/compose.yaml +++ b/docs/integrate/opentelemetry/telegraf/compose.yaml @@ -26,8 +26,8 @@ services: image: ghcr.io/astral-sh/uv:python3.13-trixie-slim volumes: - ./:/src - - uv-cache:/root/.cache/uv:cached - scale: 0 + - uv-cache:/root/.cache/uv + profiles: ["cli"] volumes: uv-cache: diff --git a/docs/integrate/oracle/compose.yaml b/docs/integrate/oracle/compose.yaml index edd4bd14..86658b08 100644 --- a/docs/integrate/oracle/compose.yaml +++ b/docs/integrate/oracle/compose.yaml @@ -23,12 +23,12 @@ services: sqlplus: image: docker.io/gvenzl/oracle-free:23-slim entrypoint: sqlplus - scale: 0 environment: ORACLE_PASSWORD: secret volumes: - .:/demo + profiles: ["cli"] ctk-ingest: image: ghcr.io/crate/cratedb-toolkit-ingest:latest - scale: 0 + profiles: ["cli"] diff --git a/docs/integrate/postgresql/compose.yaml b/docs/integrate/postgresql/compose.yaml index b34c2e11..fbd0ab0a 100644 --- a/docs/integrate/postgresql/compose.yaml +++ b/docs/integrate/postgresql/compose.yaml @@ -24,8 +24,8 @@ services: psql: image: docker.io/postgres:latest - scale: 0 + profiles: ["cli"] ctk-ingest: image: ghcr.io/crate/cratedb-toolkit-ingest:latest - scale: 0 + profiles: ["cli"] diff --git a/docs/integrate/statsd/compose.yaml b/docs/integrate/statsd/compose.yaml index 3c81aa00..e3c92a2c 100644 --- a/docs/integrate/statsd/compose.yaml +++ b/docs/integrate/statsd/compose.yaml @@ -24,8 +24,8 @@ services: nc: image: docker.io/toolbelt/netcat:2025-08-23 - scale: 0 + profiles: ["cli"] psql: image: docker.io/postgres:18 - scale: 0 + profiles: ["cli"] From 6e258d99d0ffc2aaa8feb3651e1a8431663cd338 Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Mon, 6 Oct 2025 18:31:13 +0200 Subject: [PATCH 18/19] Usage guides: Compose: Use different name for service vs. program When they are identical, they might as a typo to some. --- docs/integrate/collectd/compose.yaml | 2 +- docs/integrate/collectd/usage-collectd.md | 4 ++-- docs/integrate/collectd/usage-telegraf.md | 2 +- docs/integrate/influxdb/compose.yaml | 2 +- docs/integrate/influxdb/usage.md | 2 +- docs/integrate/mongodb/compose.yaml | 7 +------ docs/integrate/mongodb/usage.md | 4 ++-- docs/integrate/postgresql/compose.yaml | 4 ---- docs/integrate/postgresql/usage.md | 2 +- docs/integrate/statsd/compose.yaml | 2 +- docs/integrate/statsd/usage.md | 2 +- 11 files changed, 12 insertions(+), 21 deletions(-) diff --git a/docs/integrate/collectd/compose.yaml b/docs/integrate/collectd/compose.yaml index 45eb23a1..6834427c 100644 --- a/docs/integrate/collectd/compose.yaml +++ b/docs/integrate/collectd/compose.yaml @@ -36,6 +36,6 @@ services: depends_on: - cratedb - psql: + postgresql: image: docker.io/postgres:18 profiles: ["cli"] diff --git a/docs/integrate/collectd/usage-collectd.md b/docs/integrate/collectd/usage-collectd.md index 46a55d41..f9a7502a 100644 --- a/docs/integrate/collectd/usage-collectd.md +++ b/docs/integrate/collectd/usage-collectd.md @@ -38,7 +38,7 @@ To send the collected data to CrateDB, collectd is configured to load its Create a database table that stores collected metrics. ```shell -docker compose run --rm --no-TTY psql psql "postgresql://crate:crate@cratedb:5432/" < Date: Mon, 6 Oct 2025 19:41:33 +0200 Subject: [PATCH 19/19] Usage guides: Add documentation URLs to each configuration file This gives users more orientation after downloading usage guide assets. --- docs/integrate/amqp/compose.yaml | 1 + docs/integrate/collectd/Dockerfile | 2 ++ docs/integrate/collectd/collectd-cratedb.conf | 2 ++ docs/integrate/collectd/collectd-telegraf.conf | 2 ++ docs/integrate/collectd/compose.yaml | 1 + docs/integrate/collectd/telegraf.conf | 3 +++ docs/integrate/influxdb/compose.yaml | 1 + docs/integrate/mongodb/compose.yaml | 1 + docs/integrate/mqtt/compose.yaml | 1 + docs/integrate/mysql/compose.yaml | 1 + docs/integrate/opentelemetry/collector/compose.yaml | 1 + .../opentelemetry/collector/cratedb-prometheus-adapter.yaml | 2 ++ docs/integrate/opentelemetry/collector/ddl.sql | 2 ++ docs/integrate/opentelemetry/collector/example.py | 1 + docs/integrate/opentelemetry/collector/otelcol.yaml | 1 + docs/integrate/opentelemetry/telegraf/compose.yaml | 1 + docs/integrate/opentelemetry/telegraf/example.py | 1 + docs/integrate/opentelemetry/telegraf/telegraf.conf | 3 +++ docs/integrate/oracle/compose.yaml | 1 + docs/integrate/oracle/init.sql | 2 ++ docs/integrate/postgresql/compose.yaml | 1 + docs/integrate/statsd/compose.yaml | 1 + docs/integrate/statsd/telegraf.conf | 3 +++ docs/integrate/telegraf/compose.yaml | 1 + docs/integrate/telegraf/telegraf.conf | 3 +++ 25 files changed, 39 insertions(+) diff --git a/docs/integrate/amqp/compose.yaml b/docs/integrate/amqp/compose.yaml index 0d2aca46..3cdaa1ed 100644 --- a/docs/integrate/amqp/compose.yaml +++ b/docs/integrate/amqp/compose.yaml @@ -1,4 +1,5 @@ # Service composition file for Docker Compose or Podman Compose +# https://cratedb.com/docs/guide/integrate/amqp/ services: diff --git a/docs/integrate/collectd/Dockerfile b/docs/integrate/collectd/Dockerfile index 4001a460..fd5d451e 100644 --- a/docs/integrate/collectd/Dockerfile +++ b/docs/integrate/collectd/Dockerfile @@ -1,3 +1,5 @@ +# https://cratedb.com/docs/guide/integrate/collectd/ + FROM docker.io/debian:13-slim # Configure system environment. diff --git a/docs/integrate/collectd/collectd-cratedb.conf b/docs/integrate/collectd/collectd-cratedb.conf index ecc314f0..b014e43f 100644 --- a/docs/integrate/collectd/collectd-cratedb.conf +++ b/docs/integrate/collectd/collectd-cratedb.conf @@ -1,4 +1,6 @@ # collectd configuration for storing metrics into CrateDB. +# https://cratedb.com/docs/guide/integrate/collectd/ + # https://collectd.org/documentation/manpages/collectd.conf.html#plugin-postgresql LoadPlugin postgresql diff --git a/docs/integrate/collectd/collectd-telegraf.conf b/docs/integrate/collectd/collectd-telegraf.conf index edabe539..b766e1d1 100644 --- a/docs/integrate/collectd/collectd-telegraf.conf +++ b/docs/integrate/collectd/collectd-telegraf.conf @@ -1,4 +1,6 @@ # collectd configuration for sending metrics to Telegraf. +# https://cratedb.com/docs/guide/integrate/collectd/ + # https://collectd.org/documentation/manpages/collectd.conf.html#plugin-network LoadPlugin network diff --git a/docs/integrate/collectd/compose.yaml b/docs/integrate/collectd/compose.yaml index 6834427c..3600881e 100644 --- a/docs/integrate/collectd/compose.yaml +++ b/docs/integrate/collectd/compose.yaml @@ -1,4 +1,5 @@ # Service composition file for Docker Compose or Podman Compose +# https://cratedb.com/docs/guide/integrate/collectd/ services: diff --git a/docs/integrate/collectd/telegraf.conf b/docs/integrate/collectd/telegraf.conf index 08cddb27..125d61c4 100644 --- a/docs/integrate/collectd/telegraf.conf +++ b/docs/integrate/collectd/telegraf.conf @@ -1,3 +1,6 @@ +# Telegraf configuration with CrateDB output. +# https://cratedb.com/docs/guide/integrate/collectd/ + [[inputs.socket_listener]] service_address = "udp://:25826" diff --git a/docs/integrate/influxdb/compose.yaml b/docs/integrate/influxdb/compose.yaml index a98c48a6..29129eff 100644 --- a/docs/integrate/influxdb/compose.yaml +++ b/docs/integrate/influxdb/compose.yaml @@ -1,4 +1,5 @@ # Service composition file for Docker Compose or Podman Compose +# https://cratedb.com/docs/guide/integrate/influxdb/ services: diff --git a/docs/integrate/mongodb/compose.yaml b/docs/integrate/mongodb/compose.yaml index 1396b1a3..08ea75e6 100644 --- a/docs/integrate/mongodb/compose.yaml +++ b/docs/integrate/mongodb/compose.yaml @@ -1,4 +1,5 @@ # Service composition file for Docker Compose or Podman Compose +# https://cratedb.com/docs/guide/integrate/mongodb/ services: diff --git a/docs/integrate/mqtt/compose.yaml b/docs/integrate/mqtt/compose.yaml index 0ce16002..2d3fa878 100644 --- a/docs/integrate/mqtt/compose.yaml +++ b/docs/integrate/mqtt/compose.yaml @@ -1,4 +1,5 @@ # Service composition file for Docker Compose or Podman Compose +# https://cratedb.com/docs/guide/integrate/mqtt/ services: diff --git a/docs/integrate/mysql/compose.yaml b/docs/integrate/mysql/compose.yaml index d3dbba85..9734efef 100644 --- a/docs/integrate/mysql/compose.yaml +++ b/docs/integrate/mysql/compose.yaml @@ -1,4 +1,5 @@ # Service composition file for Docker Compose or Podman Compose +# https://cratedb.com/docs/guide/integrate/mysql/ services: diff --git a/docs/integrate/opentelemetry/collector/compose.yaml b/docs/integrate/opentelemetry/collector/compose.yaml index 2f335887..d5febd88 100644 --- a/docs/integrate/opentelemetry/collector/compose.yaml +++ b/docs/integrate/opentelemetry/collector/compose.yaml @@ -1,4 +1,5 @@ # Service composition file for Docker Compose or Podman Compose +# https://cratedb.com/docs/guide/integrate/opentelemetry/ services: diff --git a/docs/integrate/opentelemetry/collector/cratedb-prometheus-adapter.yaml b/docs/integrate/opentelemetry/collector/cratedb-prometheus-adapter.yaml index 05f4c4b0..1b03f5c3 100644 --- a/docs/integrate/opentelemetry/collector/cratedb-prometheus-adapter.yaml +++ b/docs/integrate/opentelemetry/collector/cratedb-prometheus-adapter.yaml @@ -1,3 +1,5 @@ +# https://cratedb.com/docs/guide/integrate/opentelemetry/ + cratedb_endpoints: - host: "cratedb" # Host to connect to (default: "localhost"). port: 5432 # Port to connect to (default: 5432). diff --git a/docs/integrate/opentelemetry/collector/ddl.sql b/docs/integrate/opentelemetry/collector/ddl.sql index af738a6a..40936e0c 100644 --- a/docs/integrate/opentelemetry/collector/ddl.sql +++ b/docs/integrate/opentelemetry/collector/ddl.sql @@ -1,3 +1,5 @@ +-- https://cratedb.com/docs/guide/integrate/opentelemetry/ + CREATE TABLE IF NOT EXISTS "testdrive"."metrics" ( "timestamp" TIMESTAMP, "labels_hash" TEXT, diff --git a/docs/integrate/opentelemetry/collector/example.py b/docs/integrate/opentelemetry/collector/example.py index 1fe51aa4..891a652c 100644 --- a/docs/integrate/opentelemetry/collector/example.py +++ b/docs/integrate/opentelemetry/collector/example.py @@ -1,4 +1,5 @@ # OpenTelemetry demo application. +# https://cratedb.com/docs/guide/integrate/opentelemetry/ from opentelemetry import metrics diff --git a/docs/integrate/opentelemetry/collector/otelcol.yaml b/docs/integrate/opentelemetry/collector/otelcol.yaml index 5d899280..c8ec0aa2 100644 --- a/docs/integrate/opentelemetry/collector/otelcol.yaml +++ b/docs/integrate/opentelemetry/collector/otelcol.yaml @@ -1,4 +1,5 @@ # OpenTelemetry Collector configuration file +# https://cratedb.com/docs/guide/integrate/opentelemetry/ # # https://github.com/open-telemetry/opentelemetry-collector-releases/blob/main/distributions/otelcol-contrib/config.yaml # https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/exporter/prometheusremotewriteexporter diff --git a/docs/integrate/opentelemetry/telegraf/compose.yaml b/docs/integrate/opentelemetry/telegraf/compose.yaml index 0476a08d..f13d5e75 100644 --- a/docs/integrate/opentelemetry/telegraf/compose.yaml +++ b/docs/integrate/opentelemetry/telegraf/compose.yaml @@ -1,4 +1,5 @@ # Service composition file for Docker Compose or Podman Compose +# https://cratedb.com/docs/guide/integrate/opentelemetry/ services: diff --git a/docs/integrate/opentelemetry/telegraf/example.py b/docs/integrate/opentelemetry/telegraf/example.py index 1fe51aa4..891a652c 100644 --- a/docs/integrate/opentelemetry/telegraf/example.py +++ b/docs/integrate/opentelemetry/telegraf/example.py @@ -1,4 +1,5 @@ # OpenTelemetry demo application. +# https://cratedb.com/docs/guide/integrate/opentelemetry/ from opentelemetry import metrics diff --git a/docs/integrate/opentelemetry/telegraf/telegraf.conf b/docs/integrate/opentelemetry/telegraf/telegraf.conf index 4ade4637..3471e752 100644 --- a/docs/integrate/opentelemetry/telegraf/telegraf.conf +++ b/docs/integrate/opentelemetry/telegraf/telegraf.conf @@ -1,3 +1,6 @@ +# Telegraf configuration with CrateDB output. +# https://cratedb.com/docs/guide/integrate/opentelemetry/ + # OpenTelemetry Input Plugin # https://github.com/influxdata/telegraf/blob/release-1.36/plugins/inputs/opentelemetry/README.md [[inputs.opentelemetry]] diff --git a/docs/integrate/oracle/compose.yaml b/docs/integrate/oracle/compose.yaml index 86658b08..ef9f5f91 100644 --- a/docs/integrate/oracle/compose.yaml +++ b/docs/integrate/oracle/compose.yaml @@ -1,4 +1,5 @@ # Service composition file for Docker Compose or Podman Compose +# https://cratedb.com/docs/guide/integrate/oracle/ services: diff --git a/docs/integrate/oracle/init.sql b/docs/integrate/oracle/init.sql index 96464ec8..ee1a639f 100644 --- a/docs/integrate/oracle/init.sql +++ b/docs/integrate/oracle/init.sql @@ -1,3 +1,5 @@ +-- https://cratedb.com/docs/guide/integrate/oracle/ + DROP TABLE demo; CREATE TABLE IF NOT EXISTS demo (id LONG, temperature FLOAT, humidity FLOAT); INSERT INTO demo (id, temperature, humidity) VALUES (1, 42.84, 83.1); diff --git a/docs/integrate/postgresql/compose.yaml b/docs/integrate/postgresql/compose.yaml index 78881116..ea95c137 100644 --- a/docs/integrate/postgresql/compose.yaml +++ b/docs/integrate/postgresql/compose.yaml @@ -1,4 +1,5 @@ # Service composition file for Docker Compose or Podman Compose +# https://cratedb.com/docs/guide/integrate/postgresql/ services: diff --git a/docs/integrate/statsd/compose.yaml b/docs/integrate/statsd/compose.yaml index 746a10f9..c851ca9e 100644 --- a/docs/integrate/statsd/compose.yaml +++ b/docs/integrate/statsd/compose.yaml @@ -1,4 +1,5 @@ # Service composition file for Docker Compose or Podman Compose +# https://cratedb.com/docs/guide/integrate/statsd/ services: diff --git a/docs/integrate/statsd/telegraf.conf b/docs/integrate/statsd/telegraf.conf index 0b87171c..43405a9a 100644 --- a/docs/integrate/statsd/telegraf.conf +++ b/docs/integrate/statsd/telegraf.conf @@ -1,3 +1,6 @@ +# Telegraf configuration with CrateDB output. +# https://cratedb.com/docs/guide/integrate/statsd/ + # StatsD Input Plugin # https://github.com/influxdata/telegraf/tree/master/plugins/inputs/statsd [[inputs.statsd]] diff --git a/docs/integrate/telegraf/compose.yaml b/docs/integrate/telegraf/compose.yaml index c850b9ce..f00ef014 100644 --- a/docs/integrate/telegraf/compose.yaml +++ b/docs/integrate/telegraf/compose.yaml @@ -1,4 +1,5 @@ # Service composition file for Docker Compose or Podman Compose +# https://cratedb.com/docs/guide/integrate/telegraf/ services: diff --git a/docs/integrate/telegraf/telegraf.conf b/docs/integrate/telegraf/telegraf.conf index 46d94a20..e60e3b41 100644 --- a/docs/integrate/telegraf/telegraf.conf +++ b/docs/integrate/telegraf/telegraf.conf @@ -1,3 +1,6 @@ +# Telegraf configuration with CrateDB output. +# https://cratedb.com/docs/guide/integrate/telegraf/ + # Telegraf Configuration # # Telegraf is entirely plugin driven. All metrics are gathered from the