diff --git a/deployments/examples/ocis_full/monitoring_tracing/signoz/deploy/.gitignore b/deployments/examples/ocis_full/monitoring_tracing/signoz/deploy/.gitignore new file mode 100644 index 00000000000..e43b0f98895 --- /dev/null +++ b/deployments/examples/ocis_full/monitoring_tracing/signoz/deploy/.gitignore @@ -0,0 +1 @@ +.DS_Store diff --git a/deployments/examples/ocis_full/monitoring_tracing/signoz/deploy/common/clickhouse/cluster.ha.xml b/deployments/examples/ocis_full/monitoring_tracing/signoz/deploy/common/clickhouse/cluster.ha.xml new file mode 100644 index 00000000000..5c208815deb --- /dev/null +++ b/deployments/examples/ocis_full/monitoring_tracing/signoz/deploy/common/clickhouse/cluster.ha.xml @@ -0,0 +1,75 @@ + + + + + + zookeeper-1 + 2181 + + + zookeeper-2 + 2181 + + + zookeeper-3 + 2181 + + + + + + + + + + + + + + + clickhouse + 9000 + + + + + + + clickhouse-2 + 9000 + + + + + clickhouse-3 + 9000 + + + + + diff --git a/deployments/examples/ocis_full/monitoring_tracing/signoz/deploy/common/clickhouse/cluster.xml b/deployments/examples/ocis_full/monitoring_tracing/signoz/deploy/common/clickhouse/cluster.xml new file mode 100644 index 00000000000..8b475ffe881 --- /dev/null +++ b/deployments/examples/ocis_full/monitoring_tracing/signoz/deploy/common/clickhouse/cluster.xml @@ -0,0 +1,75 @@ + + + + + + zookeeper-1 + 2181 + + + + + + + + + + + + + + + + clickhouse + 9000 + + + + + + + + diff --git a/deployments/examples/ocis_full/monitoring_tracing/signoz/deploy/common/clickhouse/config.xml b/deployments/examples/ocis_full/monitoring_tracing/signoz/deploy/common/clickhouse/config.xml new file mode 100644 index 00000000000..1965ac3b20e --- /dev/null +++ b/deployments/examples/ocis_full/monitoring_tracing/signoz/deploy/common/clickhouse/config.xml @@ -0,0 +1,1142 @@ + + + + + + information + + json + + /var/log/clickhouse-server/clickhouse-server.log + /var/log/clickhouse-server/clickhouse-server.err.log + + 1000M + 10 + + + + + + + + + + + + + + + + + + 8123 + + + 9000 + + + 9004 + + + 9005 + + + + + + + + + + + + 9009 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 4096 + + + 3 + + + + + false + + + /path/to/ssl_cert_file + /path/to/ssl_key_file + + + false + + + /path/to/ssl_ca_cert_file + + + none + + + 0 + + + -1 + -1 + + + false + + + + + + + + + + + none + true + true + sslv2,sslv3 + true + + + + true + true + sslv2,sslv3 + true + + + + RejectCertificateHandler + + + + + + + + + 100 + + + 0 + + + + 10000 + + + + + + 0.9 + + + 4194304 + + + 0 + + + + + + 8589934592 + + + 5368709120 + + + + 1000 + + + 134217728 + + + 10000 + + + /var/lib/clickhouse/ + + + /var/lib/clickhouse/tmp/ + + + + ` + + + + + + /var/lib/clickhouse/user_files/ + + + + + + + + + + + + + users.xml + + + + /var/lib/clickhouse/access/ + + + + + + + default + + + + + + + + + + + + default + + + + + + + + + true + + + false + + ' | sed -e 's|.*>\(.*\)<.*|\1|') + wget https://github.com/ClickHouse/clickhouse-jdbc-bridge/releases/download/v$PKG_VER/clickhouse-jdbc-bridge_$PKG_VER-1_all.deb + apt install --no-install-recommends -f ./clickhouse-jdbc-bridge_$PKG_VER-1_all.deb + clickhouse-jdbc-bridge & + + * [CentOS/RHEL] + export MVN_URL=https://repo1.maven.org/maven2/ru/yandex/clickhouse/clickhouse-jdbc-bridge + export PKG_VER=$(curl -sL $MVN_URL/maven-metadata.xml | grep '' | sed -e 's|.*>\(.*\)<.*|\1|') + wget https://github.com/ClickHouse/clickhouse-jdbc-bridge/releases/download/v$PKG_VER/clickhouse-jdbc-bridge-$PKG_VER-1.noarch.rpm + yum localinstall -y clickhouse-jdbc-bridge-$PKG_VER-1.noarch.rpm + clickhouse-jdbc-bridge & + + Please refer to https://github.com/ClickHouse/clickhouse-jdbc-bridge#usage for more information. + ]]> + + + + + + + + + + + + + + + 01 + example01-01-1 + + + + + + 3600 + + + + 3600 + + + 60 + + + + + + + + + + /metrics + 9363 + + true + true + true + true + + + + + + system + query_log
+ + toYYYYMM(event_date) + + + + + + 7500 +
+ + + + system + trace_log
+ + toYYYYMM(event_date) + 7500 +
+ + + + system + query_thread_log
+ toYYYYMM(event_date) + 7500 +
+ + + + system + query_views_log
+ toYYYYMM(event_date) + 7500 +
+ + + + system + part_log
+ toYYYYMM(event_date) + 7500 +
+ + + + + + system + metric_log
+ 7500 + 1000 +
+ + + + system + asynchronous_metric_log
+ + 7000 +
+ + + + + + engine MergeTree + partition by toYYYYMM(finish_date) + order by (finish_date, finish_time_us, trace_id) + + system + opentelemetry_span_log
+ 7500 +
+ + + + + system + crash_log
+ + + 1000 +
+ + + + + + + system + processors_profile_log
+ + toYYYYMM(event_date) + 7500 +
+ + + + + + + + + *_dictionary.xml + + + *function.xml + /var/lib/clickhouse/user_scripts/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + /clickhouse/task_queue/ddl + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + click_cost + any + + 0 + 3600 + + + 86400 + 60 + + + + max + + 0 + 60 + + + 3600 + 300 + + + 86400 + 3600 + + + + + + /var/lib/clickhouse/format_schemas/ + + + + + hide encrypt/decrypt arguments + ((?:aes_)?(?:encrypt|decrypt)(?:_mysql)?)\s*\(\s*(?:'(?:\\'|.)+'|.*?)\s*\) + + \1(???) + + + + + + + + + + false + + false + + + https://6f33034cfe684dd7a3ab9875e57b1c8d@o388870.ingest.sentry.io/5226277 + + + + + + + + + + + 268435456 + true + +
diff --git a/deployments/examples/ocis_full/monitoring_tracing/signoz/deploy/common/clickhouse/custom-function.xml b/deployments/examples/ocis_full/monitoring_tracing/signoz/deploy/common/clickhouse/custom-function.xml new file mode 100644 index 00000000000..b2b3f91a1aa --- /dev/null +++ b/deployments/examples/ocis_full/monitoring_tracing/signoz/deploy/common/clickhouse/custom-function.xml @@ -0,0 +1,21 @@ + + + executable + histogramQuantile + Float64 + + Array(Float64) + buckets + + + Array(Float64) + counts + + + Float64 + quantile + + CSV + ./histogramQuantile + + diff --git a/deployments/examples/ocis_full/monitoring_tracing/signoz/deploy/common/clickhouse/storage.xml b/deployments/examples/ocis_full/monitoring_tracing/signoz/deploy/common/clickhouse/storage.xml new file mode 100644 index 00000000000..54ec4976f55 --- /dev/null +++ b/deployments/examples/ocis_full/monitoring_tracing/signoz/deploy/common/clickhouse/storage.xml @@ -0,0 +1,41 @@ + + + + + + 10485760 + + + s3 + + https://BUCKET-NAME.s3-REGION-NAME.amazonaws.com/data/ + ACCESS-KEY-ID + SECRET-ACCESS-KEY + + + + + + + + + + + default + + + s3 + 0 + + + + + + diff --git a/deployments/examples/ocis_full/monitoring_tracing/signoz/deploy/common/clickhouse/user_scripts/.gitkeep b/deployments/examples/ocis_full/monitoring_tracing/signoz/deploy/common/clickhouse/user_scripts/.gitkeep new file mode 100644 index 00000000000..e69de29bb2d diff --git a/deployments/examples/ocis_full/monitoring_tracing/signoz/deploy/common/clickhouse/users.xml b/deployments/examples/ocis_full/monitoring_tracing/signoz/deploy/common/clickhouse/users.xml new file mode 100644 index 00000000000..f18562071d8 --- /dev/null +++ b/deployments/examples/ocis_full/monitoring_tracing/signoz/deploy/common/clickhouse/users.xml @@ -0,0 +1,123 @@ + + + + + + + + + + 10000000000 + + + random + + + + + 1 + + + + + + + + + + + + + ::/0 + + + + default + + + default + + + + + + + + + + + + + + 3600 + + + 0 + 0 + 0 + 0 + 0 + + + + diff --git a/deployments/examples/ocis_full/monitoring_tracing/signoz/deploy/common/dashboards/.gitkeep b/deployments/examples/ocis_full/monitoring_tracing/signoz/deploy/common/dashboards/.gitkeep new file mode 100644 index 00000000000..e69de29bb2d diff --git a/deployments/examples/ocis_full/monitoring_tracing/signoz/deploy/common/locust-scripts/locustfile.py b/deployments/examples/ocis_full/monitoring_tracing/signoz/deploy/common/locust-scripts/locustfile.py new file mode 100644 index 00000000000..0b518208cdc --- /dev/null +++ b/deployments/examples/ocis_full/monitoring_tracing/signoz/deploy/common/locust-scripts/locustfile.py @@ -0,0 +1,16 @@ +from locust import HttpUser, task, between +class UserTasks(HttpUser): + wait_time = between(5, 15) + + @task + def rachel(self): + self.client.get("/dispatch?customer=123&nonse=0.6308392664170006") + @task + def trom(self): + self.client.get("/dispatch?customer=392&nonse=0.015296363321630757") + @task + def japanese(self): + self.client.get("/dispatch?customer=731&nonse=0.8022286220408668") + @task + def coffee(self): + self.client.get("/dispatch?customer=567&nonse=0.0022220379420636593") diff --git a/deployments/examples/ocis_full/monitoring_tracing/signoz/deploy/common/signoz/otel-collector-opamp-config.yaml b/deployments/examples/ocis_full/monitoring_tracing/signoz/deploy/common/signoz/otel-collector-opamp-config.yaml new file mode 100644 index 00000000000..72676077527 --- /dev/null +++ b/deployments/examples/ocis_full/monitoring_tracing/signoz/deploy/common/signoz/otel-collector-opamp-config.yaml @@ -0,0 +1 @@ +server_endpoint: ws://signoz:4320/v1/opamp diff --git a/deployments/examples/ocis_full/monitoring_tracing/signoz/deploy/common/signoz/prometheus.yml b/deployments/examples/ocis_full/monitoring_tracing/signoz/deploy/common/signoz/prometheus.yml new file mode 100644 index 00000000000..683e5e198b5 --- /dev/null +++ b/deployments/examples/ocis_full/monitoring_tracing/signoz/deploy/common/signoz/prometheus.yml @@ -0,0 +1,25 @@ +# my global config +global: + scrape_interval: 5s # Set the scrape interval to every 15 seconds. Default is every 1 minute. + evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute. + # scrape_timeout is set to the global default (10s). + +# Alertmanager configuration +alerting: + alertmanagers: + - static_configs: + - targets: + - alertmanager:9093 + +# Load rules once and periodically evaluate them according to the global 'evaluation_interval'. +rule_files: [] + # - "first_rules.yml" + # - "second_rules.yml" + # - 'alerts.yml' + +# A scrape configuration containing exactly one endpoint to scrape: +# Here it's Prometheus itself. +scrape_configs: [] + +remote_read: + - url: tcp://clickhouse:9000/signoz_metrics diff --git a/deployments/examples/ocis_full/monitoring_tracing/signoz/deploy/docker/clickhouse-setup/.deprecated b/deployments/examples/ocis_full/monitoring_tracing/signoz/deploy/docker/clickhouse-setup/.deprecated new file mode 100644 index 00000000000..f25abb70d48 --- /dev/null +++ b/deployments/examples/ocis_full/monitoring_tracing/signoz/deploy/docker/clickhouse-setup/.deprecated @@ -0,0 +1,3 @@ +This data directory is deprecated and will be removed in the future. +Please use the migration script under `scripts/volume-migration` to migrate data from bind mounts to Docker volumes. +The script also renames the project name to `signoz` and the network name to `signoz-net` (if not already in place). diff --git a/deployments/examples/ocis_full/monitoring_tracing/signoz/deploy/docker/clickhouse-setup/.gitkeep b/deployments/examples/ocis_full/monitoring_tracing/signoz/deploy/docker/clickhouse-setup/.gitkeep new file mode 100644 index 00000000000..e69de29bb2d diff --git a/deployments/examples/ocis_full/monitoring_tracing/signoz/deploy/docker/clickhouse-setup/data/alertmanager/.gitkeep b/deployments/examples/ocis_full/monitoring_tracing/signoz/deploy/docker/clickhouse-setup/data/alertmanager/.gitkeep new file mode 100644 index 00000000000..e69de29bb2d diff --git a/deployments/examples/ocis_full/monitoring_tracing/signoz/deploy/docker/clickhouse-setup/data/clickhouse-2/.gitkeep b/deployments/examples/ocis_full/monitoring_tracing/signoz/deploy/docker/clickhouse-setup/data/clickhouse-2/.gitkeep new file mode 100644 index 00000000000..e69de29bb2d diff --git a/deployments/examples/ocis_full/monitoring_tracing/signoz/deploy/docker/clickhouse-setup/data/clickhouse-3/.gitkeep b/deployments/examples/ocis_full/monitoring_tracing/signoz/deploy/docker/clickhouse-setup/data/clickhouse-3/.gitkeep new file mode 100644 index 00000000000..e69de29bb2d diff --git a/deployments/examples/ocis_full/monitoring_tracing/signoz/deploy/docker/clickhouse-setup/data/clickhouse/.gitkeep b/deployments/examples/ocis_full/monitoring_tracing/signoz/deploy/docker/clickhouse-setup/data/clickhouse/.gitkeep new file mode 100644 index 00000000000..e69de29bb2d diff --git a/deployments/examples/ocis_full/monitoring_tracing/signoz/deploy/docker/clickhouse-setup/data/signoz/.gitkeep b/deployments/examples/ocis_full/monitoring_tracing/signoz/deploy/docker/clickhouse-setup/data/signoz/.gitkeep new file mode 100644 index 00000000000..e69de29bb2d diff --git a/deployments/examples/ocis_full/monitoring_tracing/signoz/deploy/docker/clickhouse-setup/data/zookeeper-1/.gitkeep b/deployments/examples/ocis_full/monitoring_tracing/signoz/deploy/docker/clickhouse-setup/data/zookeeper-1/.gitkeep new file mode 100644 index 00000000000..e69de29bb2d diff --git a/deployments/examples/ocis_full/monitoring_tracing/signoz/deploy/docker/clickhouse-setup/data/zookeeper-2/.gitkeep b/deployments/examples/ocis_full/monitoring_tracing/signoz/deploy/docker/clickhouse-setup/data/zookeeper-2/.gitkeep new file mode 100644 index 00000000000..e69de29bb2d diff --git a/deployments/examples/ocis_full/monitoring_tracing/signoz/deploy/docker/clickhouse-setup/data/zookeeper-3/.gitkeep b/deployments/examples/ocis_full/monitoring_tracing/signoz/deploy/docker/clickhouse-setup/data/zookeeper-3/.gitkeep new file mode 100644 index 00000000000..e69de29bb2d diff --git a/deployments/examples/ocis_full/monitoring_tracing/signoz/deploy/docker/clickhouse-setup/user_scripts/.deprecated b/deployments/examples/ocis_full/monitoring_tracing/signoz/deploy/docker/clickhouse-setup/user_scripts/.deprecated new file mode 100644 index 00000000000..3781773c443 --- /dev/null +++ b/deployments/examples/ocis_full/monitoring_tracing/signoz/deploy/docker/clickhouse-setup/user_scripts/.deprecated @@ -0,0 +1,2 @@ +This directory is deprecated and will be removed in the future. +Please use the new directory for Clickhouse setup scripts: `scripts/clickhouse` instead. diff --git a/deployments/examples/ocis_full/monitoring_tracing/signoz/deploy/docker/clickhouse-setup/user_scripts/histogramQuantile b/deployments/examples/ocis_full/monitoring_tracing/signoz/deploy/docker/clickhouse-setup/user_scripts/histogramQuantile new file mode 100755 index 00000000000..3b77a7b26dc Binary files /dev/null and b/deployments/examples/ocis_full/monitoring_tracing/signoz/deploy/docker/clickhouse-setup/user_scripts/histogramQuantile differ diff --git a/deployments/examples/ocis_full/monitoring_tracing/signoz/deploy/docker/docker-compose.ha.yaml b/deployments/examples/ocis_full/monitoring_tracing/signoz/deploy/docker/docker-compose.ha.yaml new file mode 100644 index 00000000000..28aa7944d43 --- /dev/null +++ b/deployments/examples/ocis_full/monitoring_tracing/signoz/deploy/docker/docker-compose.ha.yaml @@ -0,0 +1,277 @@ +version: "3" +x-common: &common + networks: + - signoz-net + restart: unless-stopped + logging: + options: + max-size: 50m + max-file: "3" +x-clickhouse-defaults: &clickhouse-defaults + !!merge <<: *common + # addding non LTS version due to this fix https://github.com/ClickHouse/ClickHouse/commit/32caf8716352f45c1b617274c7508c86b7d1afab + image: clickhouse/clickhouse-server:25.5.6 + tty: true + labels: + signoz.io/scrape: "true" + signoz.io/port: "9363" + signoz.io/path: "/metrics" + depends_on: + init-clickhouse: + condition: service_completed_successfully + zookeeper-1: + condition: service_healthy + zookeeper-2: + condition: service_healthy + zookeeper-3: + condition: service_healthy + healthcheck: + test: + - CMD + - wget + - --spider + - -q + - 0.0.0.0:8123/ping + interval: 30s + timeout: 5s + retries: 3 + ulimits: + nproc: 65535 + nofile: + soft: 262144 + hard: 262144 + environment: + - CLICKHOUSE_SKIP_USER_SETUP=1 +x-zookeeper-defaults: &zookeeper-defaults + !!merge <<: *common + image: signoz/zookeeper:3.7.1 + user: root + labels: + signoz.io/scrape: "true" + signoz.io/port: "9141" + signoz.io/path: "/metrics" + healthcheck: + test: + - CMD-SHELL + - curl -s -m 2 http://localhost:8080/commands/ruok | grep error | grep null + interval: 30s + timeout: 5s + retries: 3 +x-db-depend: &db-depend + !!merge <<: *common + depends_on: + clickhouse: + condition: service_healthy + schema-migrator-sync: + condition: service_completed_successfully +services: + init-clickhouse: + !!merge <<: *common + image: clickhouse/clickhouse-server:25.5.6 + container_name: signoz-init-clickhouse + command: + - bash + - -c + - | + version="v0.0.1" + node_os=$$(uname -s | tr '[:upper:]' '[:lower:]') + node_arch=$$(uname -m | sed s/aarch64/arm64/ | sed s/x86_64/amd64/) + echo "Fetching histogram-binary for $${node_os}/$${node_arch}" + cd /tmp + wget -O histogram-quantile.tar.gz "https://github.com/SigNoz/signoz/releases/download/histogram-quantile%2F$${version}/histogram-quantile_$${node_os}_$${node_arch}.tar.gz" + tar -xvzf histogram-quantile.tar.gz + mv histogram-quantile /var/lib/clickhouse/user_scripts/histogramQuantile + restart: on-failure + volumes: + - ../common/clickhouse/user_scripts:/var/lib/clickhouse/user_scripts/ + zookeeper-1: + !!merge <<: *zookeeper-defaults + container_name: signoz-zookeeper-1 + # ports: + # - "2181:2181" + # - "2888:2888" + # - "3888:3888" + volumes: + - zookeeper-1:/bitnami/zookeeper + environment: + - ZOO_SERVER_ID=1 + - ZOO_SERVERS=0.0.0.0:2888:3888,zookeeper-2:2888:3888,zookeeper-3:2888:3888 + - ALLOW_ANONYMOUS_LOGIN=yes + - ZOO_AUTOPURGE_INTERVAL=1 + - ZOO_ENABLE_PROMETHEUS_METRICS=yes + - ZOO_PROMETHEUS_METRICS_PORT_NUMBER=9141 + zookeeper-2: + !!merge <<: *zookeeper-defaults + container_name: signoz-zookeeper-2 + # ports: + # - "2182:2181" + # - "2889:2888" + # - "3889:3888" + volumes: + - zookeeper-2:/bitnami/zookeeper + environment: + - ZOO_SERVER_ID=2 + - ZOO_SERVERS=zookeeper-1:2888:3888,0.0.0.0:2888:3888,zookeeper-3:2888:3888 + - ALLOW_ANONYMOUS_LOGIN=yes + - ZOO_AUTOPURGE_INTERVAL=1 + - ZOO_ENABLE_PROMETHEUS_METRICS=yes + - ZOO_PROMETHEUS_METRICS_PORT_NUMBER=9141 + zookeeper-3: + !!merge <<: *zookeeper-defaults + container_name: signoz-zookeeper-3 + # ports: + # - "2183:2181" + # - "2890:2888" + # - "3890:3888" + volumes: + - zookeeper-3:/bitnami/zookeeper + environment: + - ZOO_SERVER_ID=3 + - ZOO_SERVERS=zookeeper-1:2888:3888,zookeeper-2:2888:3888,0.0.0.0:2888:3888 + - ALLOW_ANONYMOUS_LOGIN=yes + - ZOO_AUTOPURGE_INTERVAL=1 + - ZOO_ENABLE_PROMETHEUS_METRICS=yes + - ZOO_PROMETHEUS_METRICS_PORT_NUMBER=9141 + clickhouse: + !!merge <<: *clickhouse-defaults + container_name: signoz-clickhouse + # ports: + # - "9000:9000" + # - "8123:8123" + # - "9181:9181" + volumes: + - ../common/clickhouse/config.xml:/etc/clickhouse-server/config.xml + - ../common/clickhouse/users.xml:/etc/clickhouse-server/users.xml + - ../common/clickhouse/custom-function.xml:/etc/clickhouse-server/custom-function.xml + - ../common/clickhouse/user_scripts:/var/lib/clickhouse/user_scripts/ + - ../common/clickhouse/cluster.ha.xml:/etc/clickhouse-server/config.d/cluster.xml + - clickhouse:/var/lib/clickhouse/ + # - ../common/clickhouse/storage.xml:/etc/clickhouse-server/config.d/storage.xml + clickhouse-2: + !!merge <<: *clickhouse-defaults + container_name: signoz-clickhouse-2 + # ports: + # - "9001:9000" + # - "8124:8123" + # - "9182:9181" + volumes: + - ../common/clickhouse/config.xml:/etc/clickhouse-server/config.xml + - ../common/clickhouse/users.xml:/etc/clickhouse-server/users.xml + - ../common/clickhouse/custom-function.xml:/etc/clickhouse-server/custom-function.xml + - ../common/clickhouse/user_scripts:/var/lib/clickhouse/user_scripts/ + - ../common/clickhouse/cluster.ha.xml:/etc/clickhouse-server/config.d/cluster.xml + - clickhouse-2:/var/lib/clickhouse/ + # - ../common/clickhouse/storage.xml:/etc/clickhouse-server/config.d/storage.xml + clickhouse-3: + !!merge <<: *clickhouse-defaults + container_name: signoz-clickhouse-3 + # ports: + # - "9002:9000" + # - "8125:8123" + # - "9183:9181" + volumes: + - ../common/clickhouse/config.xml:/etc/clickhouse-server/config.xml + - ../common/clickhouse/users.xml:/etc/clickhouse-server/users.xml + - ../common/clickhouse/custom-function.xml:/etc/clickhouse-server/custom-function.xml + - ../common/clickhouse/user_scripts:/var/lib/clickhouse/user_scripts/ + - ../common/clickhouse/cluster.ha.xml:/etc/clickhouse-server/config.d/cluster.xml + - clickhouse-3:/var/lib/clickhouse/ + # - ../common/clickhouse/storage.xml:/etc/clickhouse-server/config.d/storage.xml + signoz: + !!merge <<: *db-depend + image: signoz/signoz:${VERSION:-v0.110.0} + container_name: signoz + command: + - --config=/root/config/prometheus.yml + ports: + - "8080:8080" # signoz port + # - "6060:6060" # pprof port + volumes: + - ../common/signoz/prometheus.yml:/root/config/prometheus.yml + - ../common/dashboards:/root/config/dashboards + - sqlite:/var/lib/signoz/ + environment: + - SIGNOZ_ALERTMANAGER_PROVIDER=signoz + - SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_DSN=tcp://clickhouse:9000 + - SIGNOZ_SQLSTORE_SQLITE_PATH=/var/lib/signoz/signoz.db + - DASHBOARDS_PATH=/root/config/dashboards + - STORAGE=clickhouse + - GODEBUG=netdns=go + - TELEMETRY_ENABLED=true + - DEPLOYMENT_TYPE=docker-standalone-amd + - DOT_METRICS_ENABLED=true + healthcheck: + test: + - CMD + - wget + - --spider + - -q + - localhost:8080/api/v1/health + interval: 30s + timeout: 5s + retries: 3 + # TODO: support otel-collector multiple replicas. Nginx/Traefik for loadbalancing? + otel-collector: + !!merge <<: *db-depend + image: signoz/signoz-otel-collector:${OTELCOL_TAG:-v0.129.13} + container_name: signoz-otel-collector + command: + - --config=/etc/otel-collector-config.yaml + - --manager-config=/etc/manager-config.yaml + - --copy-path=/var/tmp/collector-config.yaml + - --feature-gates=-pkg.translator.prometheus.NormalizeName + volumes: + - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml + - ../common/signoz/otel-collector-opamp-config.yaml:/etc/manager-config.yaml + environment: + - OTEL_RESOURCE_ATTRIBUTES=host.name=signoz-host,os.type=linux + - LOW_CARDINAL_EXCEPTION_GROUPING=false + ports: + # - "1777:1777" # pprof extension + - "4317:4317" # OTLP gRPC receiver + - "4318:4318" # OTLP HTTP receiver + depends_on: + clickhouse: + condition: service_healthy + schema-migrator-sync: + condition: service_completed_successfully + signoz: + condition: service_healthy + schema-migrator-sync: + !!merge <<: *common + image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-v0.129.13} + container_name: schema-migrator-sync + command: + - sync + - --dsn=tcp://clickhouse:9000 + - --up= + depends_on: + clickhouse: + condition: service_healthy + schema-migrator-async: + !!merge <<: *db-depend + image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-v0.129.13} + container_name: schema-migrator-async + command: + - async + - --dsn=tcp://clickhouse:9000 + - --up= + restart: on-failure +networks: + signoz-net: + name: signoz-net +volumes: + clickhouse: + name: signoz-clickhouse + clickhouse-2: + name: signoz-clickhouse-2 + clickhouse-3: + name: signoz-clickhouse-3 + sqlite: + name: signoz-sqlite + zookeeper-1: + name: signoz-zookeeper-1 + zookeeper-2: + name: signoz-zookeeper-2 + zookeeper-3: + name: signoz-zookeeper-3 diff --git a/deployments/examples/ocis_full/monitoring_tracing/signoz/deploy/docker/docker-compose.yaml b/deployments/examples/ocis_full/monitoring_tracing/signoz/deploy/docker/docker-compose.yaml new file mode 100644 index 00000000000..3c23b0e372b --- /dev/null +++ b/deployments/examples/ocis_full/monitoring_tracing/signoz/deploy/docker/docker-compose.yaml @@ -0,0 +1,198 @@ +version: "3" +x-common: &common + networks: + - signoz-net + restart: unless-stopped + logging: + options: + max-size: 50m + max-file: "3" +x-clickhouse-defaults: &clickhouse-defaults + !!merge <<: *common + image: clickhouse/clickhouse-server:25.5.6 + tty: true + labels: + signoz.io/scrape: "true" + signoz.io/port: "9363" + signoz.io/path: "/metrics" + depends_on: + init-clickhouse: + condition: service_completed_successfully + zookeeper-1: + condition: service_healthy + healthcheck: + test: + - CMD + - wget + - --spider + - -q + - 0.0.0.0:8123/ping + interval: 30s + timeout: 5s + retries: 3 + ulimits: + nproc: 65535 + nofile: + soft: 262144 + hard: 262144 + environment: + - CLICKHOUSE_SKIP_USER_SETUP=1 +x-zookeeper-defaults: &zookeeper-defaults + !!merge <<: *common + image: signoz/zookeeper:3.7.1 + user: root + labels: + signoz.io/scrape: "true" + signoz.io/port: "9141" + signoz.io/path: "/metrics" + healthcheck: + test: + - CMD-SHELL + - curl -s -m 2 http://localhost:8080/commands/ruok | grep error | grep null + interval: 30s + timeout: 5s + retries: 3 +x-db-depend: &db-depend + !!merge <<: *common + depends_on: + clickhouse: + condition: service_healthy + schema-migrator-sync: + condition: service_completed_successfully +services: + init-clickhouse: + !!merge <<: *common + image: clickhouse/clickhouse-server:25.5.6 + container_name: signoz-init-clickhouse + command: + - bash + - -c + - | + version="v0.0.1" + node_os=$$(uname -s | tr '[:upper:]' '[:lower:]') + node_arch=$$(uname -m | sed s/aarch64/arm64/ | sed s/x86_64/amd64/) + echo "Fetching histogram-binary for $${node_os}/$${node_arch}" + cd /tmp + wget -O histogram-quantile.tar.gz "https://github.com/SigNoz/signoz/releases/download/histogram-quantile%2F$${version}/histogram-quantile_$${node_os}_$${node_arch}.tar.gz" + tar -xvzf histogram-quantile.tar.gz + mv histogram-quantile /var/lib/clickhouse/user_scripts/histogramQuantile + restart: on-failure + volumes: + - ../common/clickhouse/user_scripts:/var/lib/clickhouse/user_scripts/ + zookeeper-1: + !!merge <<: *zookeeper-defaults + container_name: signoz-zookeeper-1 + # ports: + # - "2181:2181" + # - "2888:2888" + # - "3888:3888" + volumes: + - zookeeper-1:/bitnami/zookeeper + environment: + - ZOO_SERVER_ID=1 + - ALLOW_ANONYMOUS_LOGIN=yes + - ZOO_AUTOPURGE_INTERVAL=1 + - ZOO_ENABLE_PROMETHEUS_METRICS=yes + - ZOO_PROMETHEUS_METRICS_PORT_NUMBER=9141 + clickhouse: + !!merge <<: *clickhouse-defaults + container_name: signoz-clickhouse + # ports: + # - "9000:9000" + # - "8123:8123" + # - "9181:9181" + volumes: + - ../common/clickhouse/config.xml:/etc/clickhouse-server/config.xml + - ../common/clickhouse/users.xml:/etc/clickhouse-server/users.xml + - ../common/clickhouse/custom-function.xml:/etc/clickhouse-server/custom-function.xml + - ../common/clickhouse/user_scripts:/var/lib/clickhouse/user_scripts/ + - ../common/clickhouse/cluster.xml:/etc/clickhouse-server/config.d/cluster.xml + - clickhouse:/var/lib/clickhouse/ + # - ../common/clickhouse/storage.xml:/etc/clickhouse-server/config.d/storage.xml + signoz: + !!merge <<: *db-depend + image: signoz/signoz:${VERSION:-v0.110.0} + container_name: signoz + command: + - --config=/root/config/prometheus.yml + ports: + - "8080:8080" # signoz port + # - "6060:6060" # pprof port + volumes: + - ../common/signoz/prometheus.yml:/root/config/prometheus.yml + - ../common/dashboards:/root/config/dashboards + - sqlite:/var/lib/signoz/ + environment: + - SIGNOZ_ALERTMANAGER_PROVIDER=signoz + - SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_DSN=tcp://clickhouse:9000 + - SIGNOZ_SQLSTORE_SQLITE_PATH=/var/lib/signoz/signoz.db + - DASHBOARDS_PATH=/root/config/dashboards + - STORAGE=clickhouse + - GODEBUG=netdns=go + - TELEMETRY_ENABLED=true + - DEPLOYMENT_TYPE=docker-standalone-amd + - DOT_METRICS_ENABLED=true + healthcheck: + test: + - CMD + - wget + - --spider + - -q + - localhost:8080/api/v1/health + interval: 30s + timeout: 5s + retries: 3 + otel-collector: + !!merge <<: *db-depend + image: signoz/signoz-otel-collector:${OTELCOL_TAG:-v0.129.13} + container_name: signoz-otel-collector + command: + - --config=/etc/otel-collector-config.yaml + - --manager-config=/etc/manager-config.yaml + - --copy-path=/var/tmp/collector-config.yaml + - --feature-gates=-pkg.translator.prometheus.NormalizeName + volumes: + - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml + - ../common/signoz/otel-collector-opamp-config.yaml:/etc/manager-config.yaml + environment: + - OTEL_RESOURCE_ATTRIBUTES=host.name=signoz-host,os.type=linux + - LOW_CARDINAL_EXCEPTION_GROUPING=false + ports: + # - "1777:1777" # pprof extension + - "4317:4317" # OTLP gRPC receiver + - "4318:4318" # OTLP HTTP receiver + depends_on: + signoz: + condition: service_healthy + schema-migrator-sync: + !!merge <<: *common + image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-v0.129.13} + container_name: schema-migrator-sync + command: + - sync + - --dsn=tcp://clickhouse:9000 + - --up= + depends_on: + clickhouse: + condition: service_healthy + restart: on-failure + schema-migrator-async: + !!merge <<: *db-depend + image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-v0.129.13} + container_name: schema-migrator-async + command: + - async + - --dsn=tcp://clickhouse:9000 + - --up= + restart: on-failure +networks: + signoz-net: + external: true + name: ocis-net +volumes: + clickhouse: + name: signoz-clickhouse + sqlite: + name: signoz-sqlite + zookeeper-1: + name: signoz-zookeeper-1 diff --git a/deployments/examples/ocis_full/monitoring_tracing/signoz/deploy/docker/generator/hotrod/docker-compose.yaml b/deployments/examples/ocis_full/monitoring_tracing/signoz/deploy/docker/generator/hotrod/docker-compose.yaml new file mode 100644 index 00000000000..ac31142551d --- /dev/null +++ b/deployments/examples/ocis_full/monitoring_tracing/signoz/deploy/docker/generator/hotrod/docker-compose.yaml @@ -0,0 +1,39 @@ +version: "3" +x-common: &common + networks: + - signoz-net + extra_hosts: + - host.docker.internal:host-gateway + logging: + options: + max-size: 50m + max-file: "3" + restart: unless-stopped +services: + hotrod: + <<: *common + image: jaegertracing/example-hotrod:1.61.0 + container_name: hotrod + command: [ "all" ] + environment: + - OTEL_EXPORTER_OTLP_ENDPOINT=http://host.docker.internal:4318 # In case of external SigNoz or cloud, update the endpoint and access token + # - OTEL_OTLP_HEADERS=signoz-access-token= + load-hotrod: + <<: *common + image: "signoz/locust:1.2.3" + container_name: load-hotrod + environment: + ATTACKED_HOST: http://hotrod:8080 + LOCUST_MODE: standalone + NO_PROXY: standalone + TASK_DELAY_FROM: 5 + TASK_DELAY_TO: 30 + QUIET_MODE: "${QUIET_MODE:-false}" + LOCUST_OPTS: "--headless -u 10 -r 1" + volumes: + - ../../../common/locust-scripts:/locust + +networks: + signoz-net: + name: signoz-net + external: true diff --git a/deployments/examples/ocis_full/monitoring_tracing/signoz/deploy/docker/generator/infra/docker-compose.yaml b/deployments/examples/ocis_full/monitoring_tracing/signoz/deploy/docker/generator/infra/docker-compose.yaml new file mode 100644 index 00000000000..27343a5bce5 --- /dev/null +++ b/deployments/examples/ocis_full/monitoring_tracing/signoz/deploy/docker/generator/infra/docker-compose.yaml @@ -0,0 +1,43 @@ +version: "3" +x-common: &common + networks: + - signoz-net + extra_hosts: + - host.docker.internal:host-gateway + logging: + options: + max-size: 50m + max-file: "3" + restart: unless-stopped +services: + otel-agent: + <<: *common + image: otel/opentelemetry-collector-contrib:0.111.0 + command: + - --config=/etc/otel-collector-config.yaml + volumes: + - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml + - /:/hostfs:ro + - /var/run/docker.sock:/var/run/docker.sock + environment: + - SIGNOZ_COLLECTOR_ENDPOINT=http://host.docker.internal:4317 # In case of external SigNoz or cloud, update the endpoint and access token + - OTEL_RESOURCE_ATTRIBUTES=host.name=signoz-host,os.type=linux # Replace signoz-host with the actual hostname + # - SIGNOZ_ACCESS_TOKEN="" + # Before exposing the ports, make sure the ports are not used by other services + # ports: + # - "4317:4317" + # - "4318:4318" + logspout: + <<: *common + image: "gliderlabs/logspout:v3.2.14" + volumes: + - /etc/hostname:/etc/host_hostname:ro + - /var/run/docker.sock:/var/run/docker.sock + command: syslog+tcp://otel-agent:2255 + depends_on: + - otel-agent + +networks: + signoz-net: + name: signoz-net + external: true diff --git a/deployments/examples/ocis_full/monitoring_tracing/signoz/deploy/docker/generator/infra/otel-collector-config.yaml b/deployments/examples/ocis_full/monitoring_tracing/signoz/deploy/docker/generator/infra/otel-collector-config.yaml new file mode 100644 index 00000000000..97b038794d4 --- /dev/null +++ b/deployments/examples/ocis_full/monitoring_tracing/signoz/deploy/docker/generator/infra/otel-collector-config.yaml @@ -0,0 +1,139 @@ +receivers: + hostmetrics: + collection_interval: 30s + root_path: /hostfs + scrapers: + cpu: {} + load: {} + memory: {} + disk: {} + filesystem: {} + network: {} + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + prometheus: + config: + global: + scrape_interval: 60s + scrape_configs: + - job_name: otel-collector + static_configs: + - targets: + - localhost:8888 + labels: + job_name: otel-collector + # For Docker daemon metrics to be scraped, it must be configured to expose + # Prometheus metrics, as documented here: https://docs.docker.com/config/daemon/prometheus/ + # - job_name: docker-daemon + # static_configs: + # - targets: + # - host.docker.internal:9323 + # labels: + # job_name: docker-daemon + - job_name: docker-container + docker_sd_configs: + - host: unix:///var/run/docker.sock + relabel_configs: + - action: keep + regex: true + source_labels: + - __meta_docker_container_label_signoz_io_scrape + - regex: true + source_labels: + - __meta_docker_container_label_signoz_io_path + target_label: __metrics_path__ + - regex: (.+) + source_labels: + - __meta_docker_container_label_signoz_io_path + target_label: __metrics_path__ + - separator: ":" + source_labels: + - __meta_docker_network_ip + - __meta_docker_container_label_signoz_io_port + target_label: __address__ + - regex: '/(.*)' + replacement: '$1' + source_labels: + - __meta_docker_container_name + target_label: container_name + - regex: __meta_docker_container_label_signoz_io_(.+) + action: labelmap + replacement: $1 + tcplog/docker: + listen_address: "0.0.0.0:2255" + operators: + - type: regex_parser + regex: '^<([0-9]+)>[0-9]+ (?P[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(\.[0-9]+)?([zZ]|([\+-])([01]\d|2[0-3]):?([0-5]\d)?)?) (?P\S+) (?P\S+) [0-9]+ - -( (?P.*))?' + timestamp: + parse_from: attributes.timestamp + layout: '%Y-%m-%dT%H:%M:%S.%LZ' + - type: move + from: attributes["body"] + to: body + - type: remove + field: attributes.timestamp + # please remove names from below if you want to collect logs from them + - type: filter + id: signoz_logs_filter + expr: 'attributes.container_name matches "^signoz|(signoz-(|otel-collector|clickhouse|zookeeper))|(infra-(logspout|otel-agent)-.*)"' +processors: + batch: + send_batch_size: 10000 + send_batch_max_size: 11000 + timeout: 10s + resourcedetection: + # Using OTEL_RESOURCE_ATTRIBUTES envvar, env detector adds custom labels. + detectors: + # - ec2 + # - gcp + # - azure + - env + - system + timeout: 2s +extensions: + health_check: + endpoint: 0.0.0.0:13133 + pprof: + endpoint: 0.0.0.0:1777 +exporters: + otlp: + endpoint: ${env:SIGNOZ_COLLECTOR_ENDPOINT} + tls: + insecure: true + headers: + signoz-access-token: ${env:SIGNOZ_ACCESS_TOKEN} + # debug: {} +service: + telemetry: + logs: + encoding: json + metrics: + address: 0.0.0.0:8888 + extensions: + - health_check + - pprof + pipelines: + traces: + receivers: [otlp] + processors: [resourcedetection, batch] + exporters: [otlp] + metrics: + receivers: [otlp] + processors: [resourcedetection, batch] + exporters: [otlp] + metrics/hostmetrics: + receivers: [hostmetrics] + processors: [resourcedetection, batch] + exporters: [otlp] + metrics/prometheus: + receivers: [prometheus] + processors: [resourcedetection, batch] + exporters: [otlp] + logs: + receivers: [otlp, tcplog/docker] + processors: [resourcedetection, batch] + exporters: [otlp] diff --git a/deployments/examples/ocis_full/monitoring_tracing/signoz/deploy/docker/otel-collector-config.yaml b/deployments/examples/ocis_full/monitoring_tracing/signoz/deploy/docker/otel-collector-config.yaml new file mode 100644 index 00000000000..390c225115e --- /dev/null +++ b/deployments/examples/ocis_full/monitoring_tracing/signoz/deploy/docker/otel-collector-config.yaml @@ -0,0 +1,112 @@ +connectors: + signozmeter: + metrics_flush_interval: 1h + dimensions: + - name: service.name + - name: deployment.environment + - name: host.name +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + prometheus: + config: + global: + scrape_interval: 60s + scrape_configs: + - job_name: otel-collector + static_configs: + - targets: + - localhost:8888 + labels: + job_name: otel-collector +processors: + batch: + send_batch_size: 10000 + send_batch_max_size: 11000 + timeout: 10s + batch/meter: + send_batch_max_size: 25000 + send_batch_size: 20000 + timeout: 1s + resourcedetection: + # Using OTEL_RESOURCE_ATTRIBUTES envvar, env detector adds custom labels. + detectors: [env, system] + timeout: 2s + signozspanmetrics/delta: + metrics_exporter: signozclickhousemetrics + metrics_flush_interval: 60s + latency_histogram_buckets: [100us, 1ms, 2ms, 6ms, 10ms, 50ms, 100ms, 250ms, 500ms, 1000ms, 1400ms, 2000ms, 5s, 10s, 20s, 40s, 60s ] + dimensions_cache_size: 100000 + aggregation_temporality: AGGREGATION_TEMPORALITY_DELTA + enable_exp_histogram: true + dimensions: + - name: service.namespace + default: default + - name: deployment.environment + default: default + # This is added to ensure the uniqueness of the timeseries + # Otherwise, identical timeseries produced by multiple replicas of + # collectors result in incorrect APM metrics + - name: signoz.collector.id + - name: service.version + - name: browser.platform + - name: browser.mobile + - name: k8s.cluster.name + - name: k8s.node.name + - name: k8s.namespace.name + - name: host.name + - name: host.type + - name: container.name +extensions: + health_check: + endpoint: 0.0.0.0:13133 + pprof: + endpoint: 0.0.0.0:1777 +exporters: + clickhousetraces: + datasource: tcp://clickhouse:9000/signoz_traces + low_cardinal_exception_grouping: ${env:LOW_CARDINAL_EXCEPTION_GROUPING} + use_new_schema: true + signozclickhousemetrics: + dsn: tcp://clickhouse:9000/signoz_metrics + clickhouselogsexporter: + dsn: tcp://clickhouse:9000/signoz_logs + timeout: 10s + use_new_schema: true + signozclickhousemeter: + dsn: tcp://clickhouse:9000/signoz_meter + timeout: 45s + sending_queue: + enabled: false +service: + telemetry: + logs: + encoding: json + extensions: + - health_check + - pprof + pipelines: + traces: + receivers: [otlp] + processors: [signozspanmetrics/delta, batch] + exporters: [clickhousetraces, signozmeter] + metrics: + receivers: [otlp] + processors: [batch] + exporters: [signozclickhousemetrics, signozmeter] + metrics/prometheus: + receivers: [prometheus] + processors: [batch] + exporters: [signozclickhousemetrics, signozmeter] + logs: + receivers: [otlp] + processors: [batch] + exporters: [clickhouselogsexporter, signozmeter] + metrics/meter: + receivers: [signozmeter] + processors: [batch/meter] + exporters: [signozclickhousemeter] diff --git a/deployments/examples/ocis_full/observability.compose.yml b/deployments/examples/ocis_full/observability.compose.yml new file mode 100644 index 00000000000..22573b3c339 --- /dev/null +++ b/deployments/examples/ocis_full/observability.compose.yml @@ -0,0 +1,8 @@ +name: ocis-observability + +include: + - docker-compose.yml + - ocis.yml + - collabora.yml + - monitoring_tracing/monitoring.yml + - monitoring_tracing/signoz/deploy/docker/docker-compose.yaml