diff --git a/Makefile b/Makefile index 602375d..708e61a 100644 --- a/Makefile +++ b/Makefile @@ -7,6 +7,26 @@ beat: $(eval export DC_PROFILES=${DC_PROFILES} --profile beat) @true +.PHONY: otel-collector +otel-collector: + $(eval export DC_PROFILES=${DC_PROFILES} --profile otel-collector) + $(eval export OTEL_EXPORT_TYPE=otlp) + $(eval export OTEL_COLLECTOR_ENDPOINT=otel-collector:4317) + @true + +.PHONY: otel-loadbalancer +otel-loadbalancer: + $(eval export DC_PROFILES=${DC_PROFILES} --profile otel-loadbalancer --profile otel-collector) + $(eval export OTEL_EXPORT_TYPE=otlp) + $(eval export OTEL_COLLECTOR_ENDPOINT=otel-loadbalancer:4317) + $(eval export OTEL_COLLECTOR_REPLICAS=2) + @true + +.PHONY: otel-console +otel-console: + $(eval export OTEL_EXPORT_TYPE=console) + @true + .PHONY: antivirus antivirus: $(eval export DC_ANTIVIRUS=ANTIVIRUS_ENABLED=1) @@ -22,7 +42,7 @@ sms-provider-stub: .PHONY: up up: - ${DC_SMS_PROVIDER_STUB_MMG} ${DC_SMS_PROVIDER_STUB_FIRETEXT} ${DC_ANTIVIRUS} docker compose ${DC_PROFILES} up + ${DC_SMS_PROVIDER_STUB_MMG} ${DC_SMS_PROVIDER_STUB_FIRETEXT} ${DC_ANTIVIRUS} docker compose ${DC_PROFILES} up --remove-orphans .PHONY: stop stop: beat antivirus sms-provider-stub diff --git a/docker-compose.yml b/docker-compose.yml index 1ba4fa2..0330530 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,13 +11,14 @@ services: - POSTGRES_USER=notify - POSTGRES_PASSWORD=notify healthcheck: - test: ["CMD-SHELL", "pg_isready -d $${POSTGRES_DB} -U $${POSTGRES_USER}"] + test: [ "CMD-SHELL", "pg_isready -d $${POSTGRES_DB} -U $${POSTGRES_USER}" ] interval: 5s timeout: 1s retries: 3 networks: db: + redis: image: redis:6.2 container_name: redis @@ -26,6 +27,7 @@ services: networks: redis: + notify-api: image: notifications-api container_name: notify-api @@ -38,7 +40,7 @@ services: target: test ports: - "127.0.0.1:6011:6011" - command: ["api-local"] + command: [ "api-local" ] stdin_open: true tty: true env_file: @@ -46,7 +48,10 @@ services: - private/notify-api.env environment: - ANTIVIRUS_ENABLED + - NOTIFY_APP_NAME=api-web - FLASK_RUN_EXTRA_FILES=/opt/venv/ + - OTEL_EXPORT_TYPE=${OTEL_EXPORT_TYPE:-none} + - OTEL_COLLECTOR_ENDPOINT=${OTEL_COLLECTOR_ENDPOINT:-none} depends_on: db: condition: service_healthy @@ -68,7 +73,7 @@ services: container_name: db-migration volumes: - ../notifications-api:/home/vcap/app - command: ["migration"] + command: [ "migration" ] env_file: - private/local-aws-creds.env - private/notify-api.env @@ -78,13 +83,14 @@ services: networks: db: + notify-api-celery: image: notifications-api container_name: notify-api-celery volumes: - ../notifications-api:/home/vcap/app - ../notifications-utils:/home/vcap/utils - command: ["worker"] + command: [ "worker" ] env_file: - private/local-aws-creds.env - private/notify-api.env @@ -92,6 +98,9 @@ services: - ANTIVIRUS_ENABLED - MMG_URL - FIRETEXT_URL + - NOTIFY_APP_NAME=api-worker + - OTEL_EXPORT_TYPE=${OTEL_EXPORT_TYPE:-none} + - OTEL_COLLECTOR_ENDPOINT=${OTEL_COLLECTOR_ENDPOINT:-none} depends_on: db: condition: service_healthy @@ -103,6 +112,7 @@ services: db: apps: + notify-api-celery-beat: image: notifications-api container_name: notify-api-celery-beat @@ -111,12 +121,15 @@ services: volumes: - ../notifications-api:/home/vcap/app - ../notifications-utils:/home/vcap/utils - command: ["celery-beat"] + command: [ "celery-beat" ] env_file: - private/local-aws-creds.env - private/notify-api.env environment: + - NOTIFY_APP_NAME=api-celery-beat - ANTIVIRUS_ENABLED + - OTEL_EXPORT_TYPE=${OTEL_EXPORT_TYPE:-none} + - OTEL_COLLECTOR_ENDPOINT=${OTEL_COLLECTOR_ENDPOINT:-none} depends_on: db: condition: service_healthy @@ -128,12 +141,13 @@ services: db: apps: + notify-admin: - image: notifications-admin + image: docker.io/library/notifications-admin:latest container_name: notify-admin - volumes: - - ../notifications-admin:/home/vcap/app - - ../notifications-utils:/home/vcap/utils + # volumes: + # - ../notifications-admin:/home/vcap/app + # - ../notifications-utils:/home/vcap/utils build: context: ../notifications-admin dockerfile: docker/Dockerfile @@ -142,7 +156,7 @@ services: - NOTIFY_ENVIRONMENT=development ports: - "127.0.0.1:6012:6012" - command: ["web-local"] + command: [ "web-local" ] stdin_open: true tty: true env_file: @@ -151,6 +165,9 @@ services: environment: - ANTIVIRUS_ENABLED - FLASK_RUN_EXTRA_FILES=/opt/venv/ + - NOTIFY_APP_NAME=admin + - OTEL_EXPORT_TYPE=${OTEL_EXPORT_TYPE:-none} + - OTEL_COLLECTOR_ENDPOINT=${OTEL_COLLECTOR_ENDPOINT:-none} depends_on: notify-api: condition: service_started @@ -174,7 +191,7 @@ services: - NOTIFY_ENVIRONMENT=development ports: - "127.0.0.1:7000:7000" - command: ["web-local"] + command: [ "web-local" ] stdin_open: true tty: true env_file: @@ -199,7 +216,7 @@ services: - NOTIFY_ENVIRONMENT=development ports: - "127.0.0.1:7001:7001" - command: ["web-local"] + command: [ "web-local" ] stdin_open: true tty: true env_file: @@ -238,12 +255,12 @@ services: - template-preview-api.localhost template-preview-celery: - image: template-preview-api + image: docker.io/library/notifications-template-preview:latest container_name: template-preview-celery volumes: - ../notifications-template-preview:/home/vcap/app - ../notifications-utils:/home/vcap/utils - command: ["worker"] + command: [ "worker" ] env_file: - private/local-aws-creds.env - private/template-preview-api.env @@ -251,6 +268,7 @@ services: networks: apps: + antivirus-api: image: antivirus-api container_name: antivirus-api @@ -267,7 +285,7 @@ services: - NOTIFY_ENVIRONMENT=development ports: - "127.0.0.1:6016:6016" - entrypoint: ["./scripts/run_app.sh"] + entrypoint: [ "./scripts/run_app.sh" ] stdin_open: true tty: true environment: @@ -281,7 +299,7 @@ services: - antivirus-api.localhost antivirus-celery: - image: antivirus-api + image: docker.io/library/notifications-antivirus:latest container_name: antivirus-celery profiles: - antivirus @@ -293,7 +311,7 @@ services: target: test args: - NOTIFY_ENVIRONMENT=development - entrypoint: ["./scripts/run_celery.sh"] + entrypoint: [ "./scripts/run_celery.sh" ] stdin_open: true tty: true environment: @@ -305,6 +323,7 @@ services: networks: apps: + sms-provider-stub: image: sms-provider-stub container_name: sms-provider-stub @@ -325,6 +344,57 @@ services: aliases: - notify.localhost + otel-loadbalancer: + image: otel/opentelemetry-collector-contrib:latest + deploy: + mode: replicated + replicas: 2 + profiles: + - otel-loadbalancer + volumes: + - ./otel_mapping_loadbalancer.yml:/etc/otel/config.yaml + stdin_open: true + tty: true + command: [ "--config", "/etc/otel/config.yaml" ] + networks: + apps: + aliases: + - otel-loadbalancer.localhost + + otel-collector: + image: otel/opentelemetry-collector-contrib:latest + deploy: + mode: replicated + replicas: ${OTEL_COLLECTOR_REPLICAS:-1} + profiles: + - otel-collector + volumes: + - ./otel_mapping.yml:/etc/otel/config.yaml + ports: + - "127.0.0.1:9090-9091:9090" # Prometheus exporter + stdin_open: true + tty: true + command: [ "--config", "/etc/otel/config.yaml" ] + networks: + apps: + aliases: + - otel-collector.localhost + + jaeger: + image: jaegertracing/jaeger:latest + container_name: jaeger + ports: + - "16686:16686" + - "4317:4317" + - "4318:4318" + - "5778:5778" + - "9411:9411" + restart: on-failure + networks: + apps: + aliases: + - jaeger.localhost + networks: db: redis: diff --git a/otel_mapping.yml b/otel_mapping.yml new file mode 100644 index 0000000..ee95eae --- /dev/null +++ b/otel_mapping.yml @@ -0,0 +1,96 @@ +receivers: + prometheus: + config: + scrape_configs: + - job_name: notifications-api + scrape_interval: 10s + static_configs: + - targets: ['notify-api.localhost:6011'] + labels: + service: notifications-api + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + +exporters: + prometheus: + endpoint: "0.0.0.0:9090" + otlp/jaeger: + endpoint: jaeger:4317 + tls: + insecure: true + # Adjust the endpoint as needed for your Prometheus server to scrape + +processors: + batch: + filter/simple: + error_mode: ignore + spans: + exclude: + match_type: strict + libraries: + - name: opentelemetry.instrumentation.wsgi + +connectors: + spanmetrics: + histogram: + explicit: + buckets: [100us, 1ms, 2ms, 6ms, 10ms, 100ms, 250ms, 1s, 10s, 25s] + unit: "s" + dimensions: + - name: http.method # flask + - name: http.status_code # flask + - name: net.peer.name # db/redis + - name: celery.task_name # celery + - name: messaging.destination # celery + namespace: "spans" + exemplars: + enabled: true + dimensions_cache_size: 1000 + aggregation_temporality: "AGGREGATION_TEMPORALITY_CUMULATIVE" + metrics_flush_interval: 15s + metrics_expiration: 5m + events: + enabled: true + dimensions: + - name: exception.type + - name: exception.message + resource_metrics_key_attributes: + - service.name + - telemetry.sdk.language + - telemetry.sdk.name + + signaltometrics: + spans: + - name: span_duration + description: Span duration as exponential histogram + unit: us + #attributes: # categorize by attributes + # - key: attribute.foo + exponential_histogram: + count: Int(AdjustedCount()) # Adjusted count here is calculated as a custom OTTL converter + value: Microseconds(end_time - start_time) + logs: + - description: Count of log records + name: logrecord_count + sum: + value: "1" + +service: + telemetry: + logs: + level: DEBUG + pipelines: + traces/metrics: + receivers: [otlp] + processors: [filter/simple, batch] + exporters: [spanmetrics] + #exporters: [signaltometrics] + traces/jaeger: + receivers: [otlp] + exporters: [otlp/jaeger] + metrics: + receivers: [otlp,spanmetrics] + #receivers: [signaltometrics] + exporters: [prometheus] diff --git a/otel_mapping_loadbalancer.yml b/otel_mapping_loadbalancer.yml new file mode 100644 index 0000000..1fb6f2f --- /dev/null +++ b/otel_mapping_loadbalancer.yml @@ -0,0 +1,30 @@ +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + +processors: + +exporters: + loadbalancing: + routing_key: "traceID" + protocol: + otlp: + timeout: 2s + tls: + insecure: true + resolver: + dns: + hostname: otel-collector + port: "4317" + + +service: + pipelines: + traces: + receivers: + - otlp + processors: [] + exporters: + - loadbalancing \ No newline at end of file