From ec03ee419d585855a436306d720a9e4ccc7f6049 Mon Sep 17 00:00:00 2001 From: Matthew Fala Date: Tue, 17 Oct 2023 01:44:55 +0000 Subject: [PATCH 1/7] stability: copy fargate template to stage for revision --- .../adot-config.yaml | 219 ++++++++++++++++ .../default-config.json | 21 ++ .../firelens-datajet.json | 108 ++++++++ .../fluent-bit-onepod.conf | 71 ++++++ .../fluent-bit.conf | 156 ++++++++++++ .../fluent-bit.env | 5 + .../fluent-parser.conf | 14 + .../mountebank.json | 121 +++++++++ .../task-definition.json | 241 ++++++++++++++++++ 9 files changed, 956 insertions(+) create mode 100644 apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/adot-config.yaml create mode 100644 apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/default-config.json create mode 100644 apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/firelens-datajet.json create mode 100644 apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/fluent-bit-onepod.conf create mode 100644 apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/fluent-bit.conf create mode 100644 apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/fluent-bit.env create mode 100644 apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/fluent-parser.conf create mode 100644 apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/mountebank.json create mode 100644 apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/task-definition.json diff --git a/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/adot-config.yaml b/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/adot-config.yaml new file mode 100644 index 0000000..a2f2356 --- /dev/null +++ b/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/adot-config.yaml @@ -0,0 +1,219 @@ +extensions: + health_check: + +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + awsxray: + endpoint: 0.0.0.0:2000 + transport: udp + statsd: + endpoint: 0.0.0.0:8125 + aggregation_interval: 60s + awsecscontainermetrics: + +processors: + batch/traces: + timeout: 1s + send_batch_size: 50 + batch/metrics: + timeout: 60s + filter: + metrics: + include: + match_type: strict + metric_names: + - container.memory.reserved + - container.memory.utilized + - container.cpu.reserved + - container.cpu.utilized + - container.network.rate.rx + - container.network.rate.tx + - container.storage.read_bytes + - container.storage.write_bytes + - container.duration + metricstransform: + transforms: + - include: container.memory.utilized + action: update + new_name: MemoryUtilized + - include: container.memory.reserved + action: update + new_name: MemoryReserved + - include: container.cpu.utilized + action: update + new_name: CpuUtilized + - include: container.cpu.reserved + action: update + new_name: CpuReserved + - include: container.network.rate.rx + action: update + new_name: NetworkRxBytes + - include: container.network.rate.tx + action: update + new_name: NetworkTxBytes + - include: container.storage.read_bytes + action: update + new_name: StorageReadBytes + - include: container.storage.write_bytes + action: update + new_name: StorageWriteBytes + + resource: + attributes: + - key: ClusterName + from_attribute: aws.ecs.cluster.name + action: insert + - key: aws.ecs.cluster.name + action: delete + - key: ServiceName + from_attribute: aws.ecs.service.name + action: insert + - key: aws.ecs.service.name + action: delete + - key: TaskId + from_attribute: aws.ecs.task.id + action: insert + - key: aws.ecs.task.id + action: delete + - key: TaskDefinitionFamily + from_attribute: aws.ecs.task.family + action: insert + - key: aws.ecs.task.family + action: delete + - key: TaskARN + from_attribute: aws.ecs.task.arn + action: insert + - key: aws.ecs.task.arn + action: delete + - key: DockerName + from_attribute: aws.ecs.docker.name + action: insert + - key: aws.ecs.docker.name + action: delete + - key: TaskDefinitionRevision + from_attribute: aws.ecs.task.version + action: insert + - key: aws.ecs.task.version + action: delete + - key: PullStartedAt + from_attribute: aws.ecs.task.pull_started_at + action: insert + - key: aws.ecs.task.pull_started_at + action: delete + - key: PullStoppedAt + from_attribute: aws.ecs.task.pull_stopped_at + action: insert + - key: aws.ecs.task.pull_stopped_at + action: delete + - key: AvailabilityZone + from_attribute: cloud.zone + action: insert + - key: cloud.zone + action: delete + - key: LaunchType + from_attribute: aws.ecs.task.launch_type + action: insert + - key: aws.ecs.task.launch_type + action: delete + - key: Region + from_attribute: cloud.region + action: insert + - key: cloud.region + action: delete + - key: AccountId + from_attribute: cloud.account.id + action: insert + - key: cloud.account.id + action: delete + - key: DockerId + from_attribute: container.id + action: insert + - key: container.id + action: delete + - key: ContainerName + from_attribute: container.name + action: insert + - key: container.name + action: delete + - key: Image + from_attribute: container.image.name + action: insert + - key: container.image.name + action: delete + - key: ImageId + from_attribute: aws.ecs.container.image.id + action: insert + - key: aws.ecs.container.image.id + action: delete + - key: ExitCode + from_attribute: aws.ecs.container.exit_code + action: insert + - key: aws.ecs.container.exit_code + action: delete + - key: CreatedAt + from_attribute: aws.ecs.container.created_at + action: insert + - key: aws.ecs.container.created_at + action: delete + - key: StartedAt + from_attribute: aws.ecs.container.started_at + action: insert + - key: aws.ecs.container.started_at + action: delete + - key: FinishedAt + from_attribute: aws.ecs.container.finished_at + action: insert + - key: aws.ecs.container.finished_at + action: delete + - key: ImageTag + from_attribute: container.image.tag + action: insert + - key: container.image.tag + action: delete + +exporters: + awsxray: + awsemf/application: + namespace: ECS/AWSOTel/Application + log_group_name: '/aws/ecs/application/metrics' + awsemf/performance: + namespace: {{managed.executionName}}/{{managed.collectionName}}/{{managed.executionId}} + log_group_name: '/aws/ecs/containerinsights/{ClusterName}/performance' + log_stream_name: '{TaskId}' + resource_to_telemetry_conversion: + enabled: true + dimension_rollup_option: NoDimensionRollup + metric_declarations: + - dimensions: [ [ ClusterName ], [ ClusterName, TaskDefinitionFamily, TaskId, ContainerName ] ] + metric_name_selectors: + - MemoryUtilized + - MemoryReserved + - CpuUtilized + - CpuReserved + - NetworkRxBytes + - NetworkTxBytes + - StorageReadBytes + - StorageWriteBytes + - metric_name_selectors: [container.*] + +service: + pipelines: + traces: + receivers: [otlp,awsxray] + processors: [batch/traces] + exporters: [awsxray] + metrics/application: + receivers: [otlp, statsd] + processors: [batch/metrics] + exporters: [awsemf/application] + metrics/performance: + receivers: [awsecscontainermetrics ] + processors: [filter, metricstransform, resource] + exporters: [ awsemf/performance ] + + extensions: [health_check] \ No newline at end of file diff --git a/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/default-config.json b/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/default-config.json new file mode 100644 index 0000000..3649305 --- /dev/null +++ b/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/default-config.json @@ -0,0 +1,21 @@ +{ + "definitions": { + "taskName": "{{managed.caseNameUnique}}", + "adot_enable": false, + "cw_auto_create_group": "false", + "datajet_generator_basic_contentType": "random", + "datajet_generator_basic_contentRandomValueSet": "alpha-numeric", + "include_onepod": false, + "kbps_throughput_tcp": 10, + "kbps_throughput_tail": 10, + "kbps_throughput_forward": 10, + "cw_use_mock": true, + "add_forward": true, + "endpoint_mock": "localhost", + "endpoint_flb": "localhost", + "LOG_REGION": "us-west-2", + "ServiceName": "golden-onepod-low-throughput", + "debugS3Bucket": "{{managed.s3OutputBucket}}", + "debugS3KeyPrefix": "{{managed.s3OutputExecutionPath}}/{{managed.caseNameUnique}}" + } +} \ No newline at end of file diff --git a/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/firelens-datajet.json b/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/firelens-datajet.json new file mode 100644 index 0000000..be664cf --- /dev/null +++ b/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/firelens-datajet.json @@ -0,0 +1,108 @@ +{ + "component": "synchronizer", + "config": { + "repeat": 100, + "waitBefore": 5, + "waitAfter": 10, + "waitBetween": 0, + "isAsync": true + }, + "children": [ + { + "comment": "FireLens logs to standard out (move forward)", + "generator": { + "name": "basic", + "config": { + "contentLength": 1000, + "batchSize": 1, + "logKey": "log", + "addNewline": true, + "contentType": "{{definitions.datajet_generator_basic_contentType}}", + "contentRandomValueSet": "{{definitions.datajet_generator_basic_contentRandomValueSet}}" + } + }, + "datajet": { + "name": "forward", + "config": { + "host": "{{definitions.endpoint_flb}}", + "port": 6270, + "tagPrefix": "Application-firelens-{{definitions.caseNameUnique}}" + } + }, + "stage": { + "batchRate": {{definitions.kbps_throughput_forward}} + } + }, + { + "comment": "service log to tail", + "generator": { + "name": "basic", + "config": { + "contentLength": 1000, + "batchSize": 1, + "logKey": "log", + "contentType": "{{definitions.datajet_generator_basic_contentType}}", + "contentRandomValueSet": "{{definitions.datajet_generator_basic_contentRandomValueSet}}" + } + }, + "datajet": { + "name": "file", + "config": { + "folder": "/var/tmp/data_logs", + "filename": "service_log_1.txt" + } + }, + "stage": { + "batchRate": {{definitions.kbps_throughput_tail}} + } + }, + { + "comment": "ApplicationLogs through tcp", + "generator": { + "name": "basic", + "config": { + "contentLength": 1000, + "batchSize": 1, + "logKey": "log", + "contentType": "{{definitions.datajet_generator_basic_contentType}}", + "contentRandomValueSet": "{{definitions.datajet_generator_basic_contentRandomValueSet}}" + } + }, + "datajet": { + "name": "tcp", + "config": { + "host": "{{definitions.endpoint_flb}}", + "port": 5170, + "addNewline": true + } + }, + "stage": { + "batchRate": {{definitions.kbps_throughput_tcp}} + } + }, + { + "comment": "RequestLogs to tcp", + "generator": { + "name": "basic", + "config": { + "contentLength": 1000, + "batchSize": 1, + "logKey": "log", + "contentType": "{{definitions.datajet_generator_basic_contentType}}", + "contentRandomValueSet": "{{definitions.datajet_generator_basic_contentRandomValueSet}}" + } + }, + "datajet": { + "name": "tcp", + "config": { + "host": "{{definitions.endpoint_flb}}", + "port": 5171, + "addNewline": true + } + }, + "stage": { + "batchRate": {{definitions.kbps_throughput_tcp}} + } + } + ] +} \ No newline at end of file diff --git a/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/fluent-bit-onepod.conf b/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/fluent-bit-onepod.conf new file mode 100644 index 0000000..733da71 --- /dev/null +++ b/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/fluent-bit-onepod.conf @@ -0,0 +1,71 @@ +{{#if definitions.include_onepod}} + +# Note: The mainline onepod config includes the following line +# We do not need this line because we are using the init process +# @INCLUDE /config/fluent-bit.conf + +[OUTPUT] + # Send any StdOut/StdErr Logs to A CloudWatch Logs Group. + # FireLens automatically tags them to {container name}-firelens-{task ID}. + Name cloudwatch_logs + Match Application-firelens-* + region {{config.region}} + log_group_name {{definitions.ServiceName}}-OnePod-${STAGE}-AppContainer-STDOUT + log_stream_prefix STDOUT-${HOSTNAME} + auto_create_group {{definitions.cw_auto_create_group}} + retry_limit 3 +{{#if definitions.cw_use_mock}} + tls.verify false + port 4545 + endpoint {{definitions.endpoint_mock}} +{{/if}} + +[OUTPUT] + # Send any Application Logs to A CloudWatch Logs Group + Name cloudwatch_logs + Match ApplicationLogs + region {{config.region}} + log_group_name {{definitions.ServiceName}}-OnePod-${STAGE}-ApplicationLogs + log_stream_prefix ApplicationLogs-${HOSTNAME} + log_key log + auto_create_group {{definitions.cw_auto_create_group}} + retry_limit 3 +{{#if definitions.cw_use_mock}} + tls.verify false + port 4545 + endpoint {{definitions.endpoint_mock}} +{{/if}} + +[OUTPUT] + # Send any Request Logs to A CloudWatch Logs Group + Name cloudwatch_logs + Match RequestLogs + region {{config.region}} + log_group_name {{definitions.ServiceName}}-OnePod-${STAGE}-RequestLogs + log_stream_prefix RequestLogs-${HOSTNAME} + log_key log + auto_create_group {{definitions.cw_auto_create_group}} + retry_limit 3 +{{#if definitions.cw_use_mock}} + tls.verify false + port 4545 + endpoint {{definitions.endpoint_mock}} +{{/if}} + +[OUTPUT] + # Send any ServiceMetrics Logs to A CloudWatch Logs Group + Name cloudwatch_logs + Match ServiceMetrics + region {{definitions.LOG_REGION}} + log_group_name {{definitions.ServiceName}}-OnePod-${STAGE}-ServiceMetrics + log_stream_prefix ServiceMetrics-${HOSTNAME} + log_key log + auto_create_group {{definitions.cw_auto_create_group}} + retry_limit 3 +{{#if definitions.cw_use_mock}} + tls.verify false + port 4545 + endpoint {{definitions.endpoint_mock}} +{{/if}} + +{{/if}} \ No newline at end of file diff --git a/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/fluent-bit.conf b/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/fluent-bit.conf new file mode 100644 index 0000000..013fdde --- /dev/null +++ b/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/fluent-bit.conf @@ -0,0 +1,156 @@ +[SERVICE] + # See: + # https://docs.aws.amazon.com/AmazonECS/latest/developerguide/using_firelens.html + # https://github.com/aws-samples/amazon-ecs-firelens-under-the-hood/tree/master/generated-configs/fluent-bit + HTTP_Server On + HTTP_Listen 0.0.0.0 + HTTP_PORT 2020 + Flush 1 + Grace 30 + Log_Level info + storage.path /var/log/flb-storage/ + +{{#if definitions.segfault}} +# Artificially trigger a segfault if configured +[INPUT] + Name exec + command kill -11 `pgrep fluent-bit` +{{/if}} + +{{#if definitions.add_forward}} + +# This is an artificial forward because my standard out logger was going to some other destination not the cw specified here. +[INPUT] + Name forward + Listen 0.0.0.0 + Port 6270 + +{{/if}} + +[INPUT] + Name tcp + Listen 0.0.0.0 + + # Tag used for routing to OUTPUT + Tag ApplicationLogs + + # Listen on unused port + Port 5170 + + # Don't parse JSON for performance reasons + Format none + + Chunk_Size 32 + Buffer_Size 64 + storage.type filesystem + + # Important! Every INPUT section's Mem_Buf_Limit must be re-computed when + # the container memory's limit or the number of of INPUT sections change. If you + # update this value, confirm that the container memory limit in the task + # definition is correct. Also note that an additional INPUT section is + # configured by default, so count the INPUTs in this file and add 1. + # Mem_Buf_Limit <= Memory_limit / 2 / (number of INPUT sections + 1) + # 64MB = 512MB / 2 / (4 INPUT sections) + Mem_Buf_Limit 64MB + +[INPUT] + Name tcp + Tag RequestLogs + Listen 0.0.0.0 + Port 5171 + Format none + Chunk_Size 32 + Buffer_Size 64 + storage.type filesystem + + # Important! Every INPUT section's Mem_Buf_Limit must be re-computed when + # the container memory's limit or the number of of INPUT sections change. If you + # update this value, confirm that the container memory limit in the task + # definition is correct. Also note that an additional INPUT section is + # configured by default, so count the INPUTs in this file and add 1. + # Mem_Buf_Limit <= Memory_limit / 2 / (number of INPUT sections + 1) + # 64MB = 512MB / 2 / (4 INPUT sections) + Mem_Buf_Limit 64MB + +[INPUT] + Name tail + Tag ServiceMetrics + Path /tmp/data_logs/service_log_1.txt + Exclude_Path *.gz + Rotate_Wait 15 + Multiline On + Parser_Firstline QueryLogSeparator + Parser_1 QueryLog + + # Important! Every INPUT section's Mem_Buf_Limit must be re-computed when + # the container memory's limit or the number of of INPUT sections change. If you + # update this value, confirm that the container memory limit in the task + # definition is correct. Also note that an additional INPUT section is + # configured by default, so count the INPUTs in this file and add 1. + # Mem_Buf_Limit <= Memory_limit / 2 / (number of INPUT sections + 1) + # 64MB = 512MB / 2 / (4 INPUT sections) + Mem_Buf_Limit 64MB + +[OUTPUT] + # Send any StdOut/StdErr Logs to A CloudWatch Logs Group. + # FireLens automatically tags them to {container name}-firelens-{task ID}. + Name cloudwatch_logs + Match Application-firelens-* + region {{config.region}} + log_group_name {{definitions.ServiceName}}-${STAGE}-AppContainer-STDOUT + log_stream_prefix STDOUT-${HOSTNAME} + auto_create_group {{definitions.cw_auto_create_group}} + retry_limit 3 +{{#if definitions.cw_use_mock}} + tls.verify false + port 4545 + endpoint {{definitions.endpoint_mock}} +{{/if}} + +[OUTPUT] + # Send any Application Logs to A CloudWatch Logs Group + Name cloudwatch_logs + Match ApplicationLogs + region {{config.region}} + log_group_name {{definitions.ServiceName}}-${STAGE}-ApplicationLogs + log_stream_prefix ApplicationLogs-${HOSTNAME} + log_key log + auto_create_group {{definitions.cw_auto_create_group}} + retry_limit 3 +{{#if definitions.cw_use_mock}} + tls.verify false + port 4545 + endpoint {{definitions.endpoint_mock}} +{{/if}} + +[OUTPUT] + # Send any Request Logs to A CloudWatch Logs Group + Name cloudwatch_logs + Match RequestLogs + region {{config.region}} + log_group_name {{definitions.ServiceName}}-${STAGE}-RequestLogs + log_stream_prefix RequestLogs-${HOSTNAME} + log_key log + auto_create_group {{definitions.cw_auto_create_group}} + retry_limit 3 +{{#if definitions.cw_use_mock}} + tls.verify false + port 4545 + endpoint {{definitions.endpoint_mock}} +{{/if}} + +[OUTPUT] + # Send any ServiceMetrics Logs to A CloudWatch Logs Group + Name cloudwatch_logs + Match ServiceMetrics + region {{config.region}} + log_group_name {{definitions.ServiceName}}-${STAGE}-ServiceMetrics + log_stream_prefix ServiceMetrics-${HOSTNAME} + log_key log + auto_create_group {{definitions.cw_auto_create_group}} + retry_limit 3 +{{#if definitions.cw_use_mock}} + tls.verify false + port 4545 + endpoint {{definitions.endpoint_mock}} +{{/if}} diff --git a/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/fluent-bit.env b/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/fluent-bit.env new file mode 100644 index 0000000..13cbe7a --- /dev/null +++ b/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/fluent-bit.env @@ -0,0 +1,5 @@ +# Fargate was giving errors when loading environment variables. +# ResourceInitializationError: failed to download env files: file download command: non empty error stream: service call has been retried 5 time(s): RequestCanceled: request context canceled caused by: context deadline exceeded. +# Env file references, and env var usage removed. +LOG_REGION=us-west-2 +STAGE=test diff --git a/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/fluent-parser.conf b/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/fluent-parser.conf new file mode 100644 index 0000000..6029b71 --- /dev/null +++ b/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/fluent-parser.conf @@ -0,0 +1,14 @@ +# Parsers *must* be in a file separate from the main configuration +# See https://fluentbit.io/documentation/0.13/parser/ + +[PARSER] + # A Parser to capture the beginning of a QueryLog multi-line entry. + Name QueryLogSeparator + Format regex + Regex (?-{20,}) + +[PARSER] + # A Parser to capture QueryLog statements in their entirety. + Name QueryLog + Format regex + Regex (?-{20,})(?[\S\s]+?EOE) diff --git a/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/mountebank.json b/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/mountebank.json new file mode 100644 index 0000000..639a871 --- /dev/null +++ b/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/mountebank.json @@ -0,0 +1,121 @@ +{ + "port": 4545, + "protocol": "https", + "stubs": [ + { + "predicates": [{ + "and": [ + { + "equals": { + "path": "/", + "method": "POST", + "headers": { + "Content-Type": "application/x-amz-json-1.1", + "X-Amz-Target": "Logs_20140328.PutLogEvents" + } + } + } + ] + }], + "responses": [ + { + "is": { + "statusCode": 200, + "headers": { + "x-amzn-RequestId": "d35a6920-52e1-4f3a-bf8f-46016f474463", + "Connection": "keep-alive", + "Keep-Alive": "timeout=5", + "Content-Type": "application/x-amz-json-1.1", + "Content-Length": "80" + }, + "body": "{\"nextSequenceToken\":\"49637338213634440402126044667090232594261646769605574882\"}", + "repeat": 5 + } + } + ] + }, + { + "responses": [ + { + "is": { + "statusCode": 200, + "headers": { + "x-amzn-RequestId": "d35a6920-52e1-4f3a-bf8f-46016f474463", + "Connection": "keep-alive", + "Content-Type": "application/x-amz-json-1.1", + "Content-Length": 0, + "Date": "Mon, 13 Mar 2023 23:09:24 GMT", + "Keep-Alive": "timeout=5" + } + } + } + ], + "predicates": [{ + "and": [ + { + "equals": { + "path": "/", + "method": "POST", + "headers": { + "Content-Type": "application/x-amz-json-1.1", + "X-Amz-Target": "Logs_20140328.CreateLogGroup" + } + } + }, + { + "not": { + "contains": { "body": "requiredField" }, + "caseSensitive": true + } + } + ] + }] + }, + { + "responses": [ + { + "is": { + "statusCode": 200, + "headers": { + "x-amzn-RequestId": "d35a6920-52e1-4f3a-bf8f-46016f474463", + "Connection": "keep-alive", + "Content-Type": "application/x-amz-json-1.1", + "Content-Length": 0, + "Date": "Mon, 13 Mar 2023 23:09:24 GMT", + "Keep-Alive": "timeout=5" + }, + "body": { + "logGroups": [ + { + "arn": "arn:aws:logs:us-east-1:123456789012:log-group:monitoring-logGroup-1234:*", + "creationTime": 1393545600000, + "logGroupName": "monitoring-logGroup-1234" + } + ] + } + } + } + ], + "predicates": [{ + "and": [ + { + "equals": { + "path": "/", + "method": "POST", + "headers": { + "Content-Type": "application/x-amz-json-1.1", + "X-Amz-Target": "Logs_20140328.DescribeLogGroups" + } + } + }, + { + "not": { + "contains": { "body": "requiredField" }, + "caseSensitive": true + } + } + ] + }] + } + ] + } \ No newline at end of file diff --git a/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/task-definition.json b/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/task-definition.json new file mode 100644 index 0000000..24b4fce --- /dev/null +++ b/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/task-definition.json @@ -0,0 +1,241 @@ +{ + "family": "{{managed.caseNameUnique}}", + "executionRoleArn": "{{definitions.executionRoleArn}}", + "taskRoleArn": "{{definitions.taskRoleArn}}", + "networkMode": "awsvpc", + "requiresCompatibilities": [ + "FARGATE" + ], + "containerDefinitions": [ + {{#if definitions.adot_enable}} + { + "logConfiguration": { + "logDriver": "awslogs", + "options": { + "awslogs-group": "/ecs/aws-otel-EC2", + "awslogs-region": "{{config.region}}", + "awslogs-stream-prefix": "ecs", + "awslogs-create-group": "True" + } + }, + "healthCheck": { + "command": [ "/healthcheck" ], + "interval": 5, + "timeout": 6, + "retries": 5, + "startPeriod": 1 + }, + "portMappings": [ + { + "hostPort": 2000, + "protocol": "udp", + "containerPort": 2000 + }, + { + "hostPort": 4317, + "protocol": "tcp", + "containerPort": 4317 + }, + { + "hostPort": 8125, + "protocol": "udp", + "containerPort": 8125 + } + ], + "command": [ + "--config=s3://{{managed.s3ResourcesBucket}}.s3.us-west-2.amazonaws.com/{{managed.s3ResourcesPath}}/adot-config.yaml" + ], + "image": "amazon/aws-otel-collector", + "name": "aws-otel-collector" + }, + { + "logConfiguration": { + "logDriver": "awslogs", + "options": { + "awslogs-group": "/ecs/aws-otel-EC2", + "awslogs-region": "{{config.region}}", + "awslogs-stream-prefix": "ecs", + "awslogs-create-group": "True" + } + }, + "portMappings": [ + { + "hostPort": 8000, + "protocol": "tcp", + "containerPort": 8000 + } + ], + "environment": [ + { + "name": "AWS_XRAY_DAEMON_ADDRESS", + "value": "localhost:2000" + } + ], + "image": "public.ecr.aws/aws-otel-test/aws-otel-goxray-sample-app:latest", + "essential": false, + "name": "aws-otel-emitter", + "dependsOn": [ + { + "containerName": "aws-otel-collector", + "condition": "START" + } + ] + }, + {{/if}} + { + "name": "fluent-bit", + "image": "{{definitions.imageAwsForFluentBit}}", + "cpu": 1024, + "firelensConfiguration": { + "type": "fluentbit" + }, + "environmentFiles": [ + { + "value": "{{managed.s3ResourcesArn}}/fluent-bit.env", + "type": "s3" + } + ], + "environment": [ + { + "name": "AWS_REGION", + "value": "{{config.region}}" + }, + { + "name": "aws_fluent_bit_init_s3_1", + "value": "{{managed.s3ResourcesArn}}/fluent-bit.conf" + }, + { + "name": "aws_fluent_bit_init_s3_2", + "value": "{{managed.s3ResourcesArn}}/fluent-bit-onepod.conf" + }, + { + "name": "aws_fluent_bit_init_s3_3", + "value": "{{managed.s3ResourcesArn}}/fluent-parser.conf" + }, + { + "name": "S3_BUCKET", + "value": "{{definitions.debugS3Bucket}}" + }, + { + "name": "S3_KEY_PREFIX", + "value": "{{definitions.debugS3KeyPrefix}}" + } + ], + "memory": 4096, + "essential": true, + "logConfiguration": { + "logDriver": "awslogs", + "options": { + "awslogs-group": "app-{{managed.executionName}}-{{managed.executionId}}", + "awslogs-region": "{{config.region}}", + "awslogs-create-group": "true", + "awslogs-stream-prefix": "{{managed.caseNameUnique}}-fluent-bit" + } + }, + "mountPoints": [ + { + "sourceVolume": "my-shared-volume", + "containerPath": "/tmp/data_logs", + "readOnly": false + } + ], + "dependsOn": [{ + "containerName": "mock", + "condition": "START" + }] + }, + { + "name": "datajet", + "user": "0", + "image": "{{definitions.imageDatajet}}", + "essential": true, + "environment": [ + { + "name": "CLIENTS", + "value": "[\"s3\"]" + }, + { + "name": "CLIENT_S3_BUCKET", + "value": "{{managed.s3ResourcesBucket}}" + }, + { + "name": "CLIENT_S3_FILE", + "value": "{{managed.s3ResourcesPath}}/firelens-datajet.json" + } + ], + "logConfiguration": { + "logDriver": "awslogs", + "options": { + "awslogs-group": "app-{{managed.executionName}}-{{managed.executionId}}", + "awslogs-region": "{{config.region}}", + "awslogs-create-group": "true", + "awslogs-stream-prefix": "{{managed.caseNameUnique}}-datajet" + } + }, + "mountPoints": [ + { + "sourceVolume": "my-shared-volume", + "containerPath": "/var/tmp/data_logs", + "readOnly": false + } + ], + "dependsOn": [{ + "containerName": "fluent-bit", + "condition": "START" + }] + }, + { + "essential": false, + "image": "{{definitions.imageHttpd}}", + "name": "dummy", + "logConfiguration": { + "logDriver":"awsfirelens", + "options": { + "Name": "cloudwatch_logs", + "region": "{{config.region}}", + "log_group_name": "app-{{managed.executionName}}-{{managed.executionId}}", + "auto_create_group": "true", + "log_stream_name": "{{managed.caseNameUnique}}-dummy-app", + "retry_limit": "2" + } + }, + "dependsOn": [{ + "containerName": "fluent-bit", + "condition": "START" + }], + "memoryReservation": 100 + }, + { + "essential": false, + "image": "{{definitions.imageMountebank}}", + "name": "mock", + "environment": [ + { + "name": "CLIENT_S3_BUCKET", + "value": "{{managed.s3ResourcesBucket}}" + }, + { + "name": "CLIENT_S3_FILE", + "value": "{{managed.s3ResourcesPath}}/mountebank.json" + } + ], + "logConfiguration": { + "logDriver": "awslogs", + "options": { + "awslogs-group": "app-{{managed.executionName}}-{{managed.executionId}}", + "awslogs-region": "{{config.region}}", + "awslogs-create-group": "true", + "awslogs-stream-prefix": "{{managed.caseNameUnique}}-mock" + } + }, + "memoryReservation": 100 + } + ], + "memory": "4096", + "cpu": "1024", + "volumes": [ + { + "name": "my-shared-volume" + } + ] +} From 536b05c5a8996521a7d2ed27c1567af4d8ef3fe5 Mon Sep 17 00:00:00 2001 From: Matthew Fala Date: Tue, 17 Oct 2023 01:48:00 +0000 Subject: [PATCH 2/7] stability: fargate golden path revisions, cw up down tests Signed-off-by: Matthew Fala --- .../golden-path/case-config.json | 5 + .../cases/no-onepod-high-throughput.json | 9 ++ .../cases/no-onepod-low-throughput.json | 8 + .../golden-path/cases/onepod-cw-down.json | 10 ++ .../golden-path/cases/onepod-cw-up.json | 10 ++ .../ecs-golden-path-single/suite-config.json | 9 ++ .../lib/templating/handlebars-templater.ts | 6 +- .../default-config.json | 5 +- .../fluent-bit-base.conf | 145 +++++++++++++++++ .../fluent-bit-onepod.conf | 34 ++-- .../fluent-bit.conf | 149 +----------------- .../fluent-bit.env | 5 +- .../mountebank.json | 2 +- .../task-definition.json | 6 +- 14 files changed, 241 insertions(+), 162 deletions(-) create mode 100644 apps/firelens-stability/collections/ecs-golden-path-single/golden-path/case-config.json create mode 100644 apps/firelens-stability/collections/ecs-golden-path-single/golden-path/cases/no-onepod-high-throughput.json create mode 100644 apps/firelens-stability/collections/ecs-golden-path-single/golden-path/cases/no-onepod-low-throughput.json create mode 100644 apps/firelens-stability/collections/ecs-golden-path-single/golden-path/cases/onepod-cw-down.json create mode 100644 apps/firelens-stability/collections/ecs-golden-path-single/golden-path/cases/onepod-cw-up.json create mode 100644 apps/firelens-stability/collections/ecs-golden-path-single/suite-config.json create mode 100644 apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/fluent-bit-base.conf diff --git a/apps/firelens-stability/collections/ecs-golden-path-single/golden-path/case-config.json b/apps/firelens-stability/collections/ecs-golden-path-single/golden-path/case-config.json new file mode 100644 index 0000000..41f1d8e --- /dev/null +++ b/apps/firelens-stability/collections/ecs-golden-path-single/golden-path/case-config.json @@ -0,0 +1,5 @@ +{ + "config": { + "taskCount": 1 + } +} diff --git a/apps/firelens-stability/collections/ecs-golden-path-single/golden-path/cases/no-onepod-high-throughput.json b/apps/firelens-stability/collections/ecs-golden-path-single/golden-path/cases/no-onepod-high-throughput.json new file mode 100644 index 0000000..6a68eb8 --- /dev/null +++ b/apps/firelens-stability/collections/ecs-golden-path-single/golden-path/cases/no-onepod-high-throughput.json @@ -0,0 +1,9 @@ +{ + "definitions": { + "include_onepod": false, + "kbps_throughput_tcp": 5000, + "kbps_throughput_tail": 10, + "kbps_throughput_forward": 5000, + "datajet_generator_basic_contentType": "uniform" + } +} diff --git a/apps/firelens-stability/collections/ecs-golden-path-single/golden-path/cases/no-onepod-low-throughput.json b/apps/firelens-stability/collections/ecs-golden-path-single/golden-path/cases/no-onepod-low-throughput.json new file mode 100644 index 0000000..946d1c3 --- /dev/null +++ b/apps/firelens-stability/collections/ecs-golden-path-single/golden-path/cases/no-onepod-low-throughput.json @@ -0,0 +1,8 @@ +{ + "definitions": { + "include_onepod": false, + "kbps_throughput_tcp": 10, + "kbps_throughput_tail": 10, + "kbps_throughput_forward": 10 + } +} diff --git a/apps/firelens-stability/collections/ecs-golden-path-single/golden-path/cases/onepod-cw-down.json b/apps/firelens-stability/collections/ecs-golden-path-single/golden-path/cases/onepod-cw-down.json new file mode 100644 index 0000000..0f76c36 --- /dev/null +++ b/apps/firelens-stability/collections/ecs-golden-path-single/golden-path/cases/onepod-cw-down.json @@ -0,0 +1,10 @@ +{ + "definitions": { + "cw_is_down": false, + "include_onepod": true, + "kbps_throughput_tcp": 5000, + "kbps_throughput_tail": 10, + "kbps_throughput_forward": 5000, + "datajet_generator_basic_contentType": "uniform" + } +} diff --git a/apps/firelens-stability/collections/ecs-golden-path-single/golden-path/cases/onepod-cw-up.json b/apps/firelens-stability/collections/ecs-golden-path-single/golden-path/cases/onepod-cw-up.json new file mode 100644 index 0000000..0f76c36 --- /dev/null +++ b/apps/firelens-stability/collections/ecs-golden-path-single/golden-path/cases/onepod-cw-up.json @@ -0,0 +1,10 @@ +{ + "definitions": { + "cw_is_down": false, + "include_onepod": true, + "kbps_throughput_tcp": 5000, + "kbps_throughput_tail": 10, + "kbps_throughput_forward": 5000, + "datajet_generator_basic_contentType": "uniform" + } +} diff --git a/apps/firelens-stability/collections/ecs-golden-path-single/suite-config.json b/apps/firelens-stability/collections/ecs-golden-path-single/suite-config.json new file mode 100644 index 0000000..2b64832 --- /dev/null +++ b/apps/firelens-stability/collections/ecs-golden-path-single/suite-config.json @@ -0,0 +1,9 @@ +{ + "definitions": { + "testId": "{{managed.suiteName}}-{{managed.caseName}}", + "segfault": false + }, + "config": { + "cluster": "{{managed.executionName}}-{{managed.executionId}}-{{managed.collectionName}}" + } +} \ No newline at end of file diff --git a/apps/firelens-stability/lib/templating/handlebars-templater.ts b/apps/firelens-stability/lib/templating/handlebars-templater.ts index b51a58b..1e576e8 100644 --- a/apps/firelens-stability/lib/templating/handlebars-templater.ts +++ b/apps/firelens-stability/lib/templating/handlebars-templater.ts @@ -92,6 +92,10 @@ export async function copyAndTemplateFile( destinationFile: string, config: any) { const rawFileString = await getStringFromFile(sourceFile); - const templatedFileString = evaluateTemplateString(rawFileString, config); + const templatedFileString = evaluateTemplateString(rawFileString, { + // definitions get set to high level for file replacements + ...(config?.definitions ?? {}), + ...config + }); await sendStringToFile(templatedFileString, destinationFile); } diff --git a/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/default-config.json b/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/default-config.json index 3649305..bd5b054 100644 --- a/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/default-config.json +++ b/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/default-config.json @@ -11,11 +11,12 @@ "kbps_throughput_forward": 10, "cw_use_mock": true, "add_forward": true, - "endpoint_mock": "localhost", + "cw_endpoint": "localhost", "endpoint_flb": "localhost", "LOG_REGION": "us-west-2", "ServiceName": "golden-onepod-low-throughput", "debugS3Bucket": "{{managed.s3OutputBucket}}", - "debugS3KeyPrefix": "{{managed.s3OutputExecutionPath}}/{{managed.caseNameUnique}}" + "debugS3KeyPrefix": "{{managed.s3OutputExecutionPath}}/{{managed.caseNameUnique}}", + "cw_is_down": false } } \ No newline at end of file diff --git a/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/fluent-bit-base.conf b/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/fluent-bit-base.conf new file mode 100644 index 0000000..4f24e2f --- /dev/null +++ b/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/fluent-bit-base.conf @@ -0,0 +1,145 @@ +[SERVICE] + # See: + # https://docs.aws.amazon.com/AmazonECS/latest/developerguide/using_firelens.html + # https://github.com/aws-samples/amazon-ecs-firelens-under-the-hood/tree/master/generated-configs/fluent-bit + HTTP_Server On + HTTP_Listen 0.0.0.0 + HTTP_PORT 2020 + Flush 1 + Grace 30 + Log_Level info + storage.path /var/log/flb-storage/ + +{{#if definitions.segfault}} +# Artificially trigger a segfault if configured +[INPUT] + Name exec + command kill -11 `pgrep fluent-bit` +{{/if}} + +{{#if definitions.add_forward}} + +# This is an artificial forward because my standard out logger was going to some other destination not the cw specified here. +[INPUT] + Name forward + Listen 0.0.0.0 + Port 6270 + +{{/if}} + +[INPUT] + Name tcp + Listen 0.0.0.0 + + # Tag used for routing to OUTPUT + Tag ApplicationLogs + + # Listen on unused port + Port 5170 + + # Don't parse JSON for performance reasons + Format none + + Chunk_Size 32 + Buffer_Size 64 + storage.type filesystem + + # Important! Every INPUT section's Mem_Buf_Limit must be re-computed when + # the container memory's limit or the number of of INPUT sections change. If you + # update this value, confirm that the container memory limit in the task + # definition is correct. Also note that an additional INPUT section is + # configured by default, so count the INPUTs in this file and add 1. + # Mem_Buf_Limit <= Memory_limit / 2 / (number of INPUT sections + 1) + # 64MB = 512MB / 2 / (4 INPUT sections) + Mem_Buf_Limit 64MB + +[INPUT] + Name tcp + Tag RequestLogs + Listen 0.0.0.0 + Port 5171 + Format none + Chunk_Size 32 + Buffer_Size 64 + storage.type filesystem + + # Important! Every INPUT section's Mem_Buf_Limit must be re-computed when + # the container memory's limit or the number of of INPUT sections change. If you + # update this value, confirm that the container memory limit in the task + # definition is correct. Also note that an additional INPUT section is + # configured by default, so count the INPUTs in this file and add 1. + # Mem_Buf_Limit <= Memory_limit / 2 / (number of INPUT sections + 1) + # 64MB = 512MB / 2 / (4 INPUT sections) + Mem_Buf_Limit 64MB + +[INPUT] + Name tail + Tag ServiceMetrics + Path /tmp/data_logs/service_log_1.txt + Exclude_Path *.gz + Rotate_Wait 15 + Multiline On + Parser_Firstline QueryLogSeparator + Parser_1 QueryLog + + # Important! Every INPUT section's Mem_Buf_Limit must be re-computed when + # the container memory's limit or the number of of INPUT sections change. If you + # update this value, confirm that the container memory limit in the task + # definition is correct. Also note that an additional INPUT section is + # configured by default, so count the INPUTs in this file and add 1. + # Mem_Buf_Limit <= Memory_limit / 2 / (number of INPUT sections + 1) + # 64MB = 512MB / 2 / (4 INPUT sections) + Mem_Buf_Limit 64MB + +[OUTPUT] + # Send any StdOut/StdErr Logs to A CloudWatch Logs Group. + # FireLens automatically tags them to {container name}-firelens-{task ID}. + Name cloudwatch_logs + Alias CloudWatch-AppContainer-STDOUT + Match Application-firelens-* + region ${LOG_REGION} + log_group_name {{ServiceName}}-${STAGE}-AppContainer-STDOUT + log_stream_prefix STDOUT-${HOSTNAME} + auto_create_group {{definitions.cw_auto_create_group}} + retry_limit 3 + endpoint ${CLOUDWATCH_ENDPOINT} + +{{#if definitions.cw_use_mock}} + tls.verify false + port 4545 +{{/if}} + +[OUTPUT] + # Send any Application Logs to A CloudWatch Logs Group + Name cloudwatch_logs + Match ApplicationLogs + region ${LOG_REGION} + log_group_name {{ServiceName}}-${STAGE}-ApplicationLogs + log_stream_prefix ApplicationLogs-${HOSTNAME} + log_key log + auto_create_group {{definitions.cw_auto_create_group}} + retry_limit 3 + storage.total_limit_size 2G + endpoint ${CLOUDWATCH_ENDPOINT} +{{#if definitions.cw_use_mock}} + tls.verify false + port 4545 +{{/if}} + +[OUTPUT] + # Send any Request Logs to A CloudWatch Logs Group + Name cloudwatch_logs + Alias CloudWatch-RequestLogs + Match RequestLogs + region ${LOG_REGION} + log_group_name {{ServiceName}}-${STAGE}-RequestLogs + log_stream_prefix RequestLogs-${HOSTNAME} + log_key log + auto_create_group {{definitions.cw_auto_create_group}} + retry_limit 3 + storage.total_limit_size 2G + endpoint ${CLOUDWATCH_ENDPOINT} +{{#if definitions.cw_use_mock}} + tls.verify false + port 4545 +{{/if}} diff --git a/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/fluent-bit-onepod.conf b/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/fluent-bit-onepod.conf index 733da71..bf3ab50 100644 --- a/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/fluent-bit-onepod.conf +++ b/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/fluent-bit-onepod.conf @@ -8,64 +8,72 @@ # Send any StdOut/StdErr Logs to A CloudWatch Logs Group. # FireLens automatically tags them to {container name}-firelens-{task ID}. Name cloudwatch_logs + Alias CloudWatch-AppContainer-STDOUT Match Application-firelens-* - region {{config.region}} - log_group_name {{definitions.ServiceName}}-OnePod-${STAGE}-AppContainer-STDOUT + region ${LOG_REGION} + log_group_name {{ServiceName}}-OnePod-${STAGE}-AppContainer-STDOUT log_stream_prefix STDOUT-${HOSTNAME} - auto_create_group {{definitions.cw_auto_create_group}} + auto_create_group {{definitions.cw_auto_create_group}} # false retry_limit 3 + endpoint ${CLOUDWATCH_ENDPOINT} {{#if definitions.cw_use_mock}} tls.verify false port 4545 - endpoint {{definitions.endpoint_mock}} {{/if}} [OUTPUT] # Send any Application Logs to A CloudWatch Logs Group Name cloudwatch_logs + Alias CloudWatch-ApplicationLogs Match ApplicationLogs - region {{config.region}} - log_group_name {{definitions.ServiceName}}-OnePod-${STAGE}-ApplicationLogs + region ${LOG_REGION} + log_group_name {{ServiceName}}-OnePod-${STAGE}-ApplicationLogs log_stream_prefix ApplicationLogs-${HOSTNAME} log_key log auto_create_group {{definitions.cw_auto_create_group}} retry_limit 3 + storage.total_limit_size 2G + endpoint ${CLOUDWATCH_ENDPOINT} {{#if definitions.cw_use_mock}} tls.verify false port 4545 - endpoint {{definitions.endpoint_mock}} {{/if}} [OUTPUT] # Send any Request Logs to A CloudWatch Logs Group Name cloudwatch_logs + Alias CloudWatch-RequestLogs Match RequestLogs - region {{config.region}} - log_group_name {{definitions.ServiceName}}-OnePod-${STAGE}-RequestLogs + region ${LOG_REGION} + log_group_name {{ServiceName}}-OnePod-${STAGE}-RequestLogs log_stream_prefix RequestLogs-${HOSTNAME} log_key log auto_create_group {{definitions.cw_auto_create_group}} retry_limit 3 + storage.total_limit_size 2G + endpoint ${CLOUDWATCH_ENDPOINT} {{#if definitions.cw_use_mock}} tls.verify false port 4545 - endpoint {{definitions.endpoint_mock}} {{/if}} [OUTPUT] # Send any ServiceMetrics Logs to A CloudWatch Logs Group Name cloudwatch_logs + Alias CloudWatch-ServiceMetrics Match ServiceMetrics - region {{definitions.LOG_REGION}} - log_group_name {{definitions.ServiceName}}-OnePod-${STAGE}-ServiceMetrics + region ${LOG_REGION} + log_group_name {{ServiceName}}-OnePod-${STAGE}-ServiceMetrics log_stream_prefix ServiceMetrics-${HOSTNAME} log_key log + log_format json/emf auto_create_group {{definitions.cw_auto_create_group}} retry_limit 3 + storage.total_limit_size 1G + endpoint ${CLOUDWATCH_ENDPOINT} {{#if definitions.cw_use_mock}} tls.verify false port 4545 - endpoint {{definitions.endpoint_mock}} {{/if}} {{/if}} \ No newline at end of file diff --git a/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/fluent-bit.conf b/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/fluent-bit.conf index 013fdde..cf02b48 100644 --- a/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/fluent-bit.conf +++ b/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/fluent-bit.conf @@ -1,156 +1,21 @@ -[SERVICE] - # See: - # https://docs.aws.amazon.com/AmazonECS/latest/developerguide/using_firelens.html - # https://github.com/aws-samples/amazon-ecs-firelens-under-the-hood/tree/master/generated-configs/fluent-bit - HTTP_Server On - HTTP_Listen 0.0.0.0 - HTTP_PORT 2020 - Flush 1 - Grace 30 - Log_Level info - storage.path /var/log/flb-storage/ - -{{#if definitions.segfault}} -# Artificially trigger a segfault if configured -[INPUT] - Name exec - command kill -11 `pgrep fluent-bit` -{{/if}} - -{{#if definitions.add_forward}} - -# This is an artificial forward because my standard out logger was going to some other destination not the cw specified here. -[INPUT] - Name forward - Listen 0.0.0.0 - Port 6270 - -{{/if}} - -[INPUT] - Name tcp - Listen 0.0.0.0 - - # Tag used for routing to OUTPUT - Tag ApplicationLogs - - # Listen on unused port - Port 5170 - - # Don't parse JSON for performance reasons - Format none - - Chunk_Size 32 - Buffer_Size 64 - storage.type filesystem - - # Important! Every INPUT section's Mem_Buf_Limit must be re-computed when - # the container memory's limit or the number of of INPUT sections change. If you - # update this value, confirm that the container memory limit in the task - # definition is correct. Also note that an additional INPUT section is - # configured by default, so count the INPUTs in this file and add 1. - # Mem_Buf_Limit <= Memory_limit / 2 / (number of INPUT sections + 1) - # 64MB = 512MB / 2 / (4 INPUT sections) - Mem_Buf_Limit 64MB - -[INPUT] - Name tcp - Tag RequestLogs - Listen 0.0.0.0 - Port 5171 - Format none - Chunk_Size 32 - Buffer_Size 64 - storage.type filesystem - - # Important! Every INPUT section's Mem_Buf_Limit must be re-computed when - # the container memory's limit or the number of of INPUT sections change. If you - # update this value, confirm that the container memory limit in the task - # definition is correct. Also note that an additional INPUT section is - # configured by default, so count the INPUTs in this file and add 1. - # Mem_Buf_Limit <= Memory_limit / 2 / (number of INPUT sections + 1) - # 64MB = 512MB / 2 / (4 INPUT sections) - Mem_Buf_Limit 64MB - -[INPUT] - Name tail - Tag ServiceMetrics - Path /tmp/data_logs/service_log_1.txt - Exclude_Path *.gz - Rotate_Wait 15 - Multiline On - Parser_Firstline QueryLogSeparator - Parser_1 QueryLog - - # Important! Every INPUT section's Mem_Buf_Limit must be re-computed when - # the container memory's limit or the number of of INPUT sections change. If you - # update this value, confirm that the container memory limit in the task - # definition is correct. Also note that an additional INPUT section is - # configured by default, so count the INPUTs in this file and add 1. - # Mem_Buf_Limit <= Memory_limit / 2 / (number of INPUT sections + 1) - # 64MB = 512MB / 2 / (4 INPUT sections) - Mem_Buf_Limit 64MB - -[OUTPUT] - # Send any StdOut/StdErr Logs to A CloudWatch Logs Group. - # FireLens automatically tags them to {container name}-firelens-{task ID}. - Name cloudwatch_logs - Match Application-firelens-* - region {{config.region}} - log_group_name {{definitions.ServiceName}}-${STAGE}-AppContainer-STDOUT - log_stream_prefix STDOUT-${HOSTNAME} - auto_create_group {{definitions.cw_auto_create_group}} - retry_limit 3 -{{#if definitions.cw_use_mock}} - tls.verify false - port 4545 - endpoint {{definitions.endpoint_mock}} -{{/if}} - -[OUTPUT] - # Send any Application Logs to A CloudWatch Logs Group - Name cloudwatch_logs - Match ApplicationLogs - region {{config.region}} - log_group_name {{definitions.ServiceName}}-${STAGE}-ApplicationLogs - log_stream_prefix ApplicationLogs-${HOSTNAME} - log_key log - auto_create_group {{definitions.cw_auto_create_group}} - retry_limit 3 -{{#if definitions.cw_use_mock}} - tls.verify false - port 4545 - endpoint {{definitions.endpoint_mock}} -{{/if}} - -[OUTPUT] - # Send any Request Logs to A CloudWatch Logs Group - Name cloudwatch_logs - Match RequestLogs - region {{config.region}} - log_group_name {{definitions.ServiceName}}-${STAGE}-RequestLogs - log_stream_prefix RequestLogs-${HOSTNAME} - log_key log - auto_create_group {{definitions.cw_auto_create_group}} - retry_limit 3 -{{#if definitions.cw_use_mock}} - tls.verify false - port 4545 - endpoint {{definitions.endpoint_mock}} -{{/if}} +# Note: The mainline onepod config includes the following line +# We do not need this line because we are using the init process +# @INCLUDE /config/fluent-bit-base.conf [OUTPUT] # Send any ServiceMetrics Logs to A CloudWatch Logs Group Name cloudwatch_logs + Alias CloudWatch-ServiceMetrics Match ServiceMetrics - region {{config.region}} + region ${LOG_REGION} log_group_name {{definitions.ServiceName}}-${STAGE}-ServiceMetrics log_stream_prefix ServiceMetrics-${HOSTNAME} log_key log auto_create_group {{definitions.cw_auto_create_group}} retry_limit 3 + storage.total_limit_size 1G + endpoint ${CLOUDWATCH_ENDPOINT} {{#if definitions.cw_use_mock}} tls.verify false port 4545 - endpoint {{definitions.endpoint_mock}} {{/if}} diff --git a/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/fluent-bit.env b/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/fluent-bit.env index 13cbe7a..e820be5 100644 --- a/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/fluent-bit.env +++ b/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/fluent-bit.env @@ -1,5 +1,6 @@ # Fargate was giving errors when loading environment variables. # ResourceInitializationError: failed to download env files: file download command: non empty error stream: service call has been retried 5 time(s): RequestCanceled: request context canceled caused by: context deadline exceeded. # Env file references, and env var usage removed. -LOG_REGION=us-west-2 -STAGE=test +LOG_REGION={{config.region}} +STAGE=alpha +CLOUDWATCH_ENDPOINT={{definitions.cw_endpoint}} \ No newline at end of file diff --git a/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/mountebank.json b/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/mountebank.json index 639a871..c50d3ab 100644 --- a/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/mountebank.json +++ b/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/mountebank.json @@ -20,7 +20,7 @@ "responses": [ { "is": { - "statusCode": 200, + "statusCode": {{#if definitions.cw_is_down}}400{{else}}200{{/if}}, "headers": { "x-amzn-RequestId": "d35a6920-52e1-4f3a-bf8f-46016f474463", "Connection": "keep-alive", diff --git a/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/task-definition.json b/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/task-definition.json index 24b4fce..3244674 100644 --- a/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/task-definition.json +++ b/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/task-definition.json @@ -106,10 +106,14 @@ }, { "name": "aws_fluent_bit_init_s3_2", - "value": "{{managed.s3ResourcesArn}}/fluent-bit-onepod.conf" + "value": "{{managed.s3ResourcesArn}}/fluent-bit-base.conf" }, { "name": "aws_fluent_bit_init_s3_3", + "value": "{{managed.s3ResourcesArn}}/fluent-bit-onepod.conf" + }, + { + "name": "aws_fluent_bit_init_s3_4", "value": "{{managed.s3ResourcesArn}}/fluent-parser.conf" }, { From 31c54dde7ce506391af02e96904fa99cb40e8f7d Mon Sep 17 00:00:00 2001 From: Matthew Fala Date: Tue, 17 Oct 2023 03:56:45 +0000 Subject: [PATCH 3/7] stability: conform further to standard path --- .../golden-path/case-config.json | 8 ++- .../golden-path/cases/no-onepod-cw-down.json | 6 ++ .../golden-path/cases/no-onepod-cw-up.json | 6 ++ .../cases/no-onepod-high-throughput.json | 9 --- .../cases/no-onepod-low-throughput.json | 8 --- .../golden-path/cases/onepod-cw-down.json | 8 +-- .../golden-path/cases/onepod-cw-up.json | 6 +- apps/firelens-stability/execution.json | 7 +- .../lib/helpers/start-execution-stages.ts | 11 +-- .../default-config.json | 7 +- .../firelens-datajet.json | 36 +++++----- .../fluent-bit-base.conf | 70 +++++++++---------- .../example-explicit-generator-to-tcp.json | 23 ++++++ 13 files changed, 113 insertions(+), 92 deletions(-) create mode 100644 apps/firelens-stability/collections/ecs-golden-path-single/golden-path/cases/no-onepod-cw-down.json create mode 100644 apps/firelens-stability/collections/ecs-golden-path-single/golden-path/cases/no-onepod-cw-up.json delete mode 100644 apps/firelens-stability/collections/ecs-golden-path-single/golden-path/cases/no-onepod-high-throughput.json delete mode 100644 apps/firelens-stability/collections/ecs-golden-path-single/golden-path/cases/no-onepod-low-throughput.json create mode 100644 examples/generators/example-explicit-generator-to-tcp.json diff --git a/apps/firelens-stability/collections/ecs-golden-path-single/golden-path/case-config.json b/apps/firelens-stability/collections/ecs-golden-path-single/golden-path/case-config.json index 41f1d8e..62a8774 100644 --- a/apps/firelens-stability/collections/ecs-golden-path-single/golden-path/case-config.json +++ b/apps/firelens-stability/collections/ecs-golden-path-single/golden-path/case-config.json @@ -1,5 +1,11 @@ { "config": { - "taskCount": 1 + "taskCount": 1, + "adot_enable": true, + "kbps_throughput_tcp_application": 5000, + "kbps_throughput_tcp_request": 5000, + "kbps_throughput_tcp_service": 5000, + "kbps_throughput_forward": 5000, + "datajet_generator_basic_contentType": "uniform" } } diff --git a/apps/firelens-stability/collections/ecs-golden-path-single/golden-path/cases/no-onepod-cw-down.json b/apps/firelens-stability/collections/ecs-golden-path-single/golden-path/cases/no-onepod-cw-down.json new file mode 100644 index 0000000..de9a612 --- /dev/null +++ b/apps/firelens-stability/collections/ecs-golden-path-single/golden-path/cases/no-onepod-cw-down.json @@ -0,0 +1,6 @@ +{ + "definitions": { + "cw_is_down": true, + "include_onepod": false + } +} diff --git a/apps/firelens-stability/collections/ecs-golden-path-single/golden-path/cases/no-onepod-cw-up.json b/apps/firelens-stability/collections/ecs-golden-path-single/golden-path/cases/no-onepod-cw-up.json new file mode 100644 index 0000000..65d00fb --- /dev/null +++ b/apps/firelens-stability/collections/ecs-golden-path-single/golden-path/cases/no-onepod-cw-up.json @@ -0,0 +1,6 @@ +{ + "definitions": { + "cw_is_down": false, + "include_onepod": false + } +} diff --git a/apps/firelens-stability/collections/ecs-golden-path-single/golden-path/cases/no-onepod-high-throughput.json b/apps/firelens-stability/collections/ecs-golden-path-single/golden-path/cases/no-onepod-high-throughput.json deleted file mode 100644 index 6a68eb8..0000000 --- a/apps/firelens-stability/collections/ecs-golden-path-single/golden-path/cases/no-onepod-high-throughput.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "definitions": { - "include_onepod": false, - "kbps_throughput_tcp": 5000, - "kbps_throughput_tail": 10, - "kbps_throughput_forward": 5000, - "datajet_generator_basic_contentType": "uniform" - } -} diff --git a/apps/firelens-stability/collections/ecs-golden-path-single/golden-path/cases/no-onepod-low-throughput.json b/apps/firelens-stability/collections/ecs-golden-path-single/golden-path/cases/no-onepod-low-throughput.json deleted file mode 100644 index 946d1c3..0000000 --- a/apps/firelens-stability/collections/ecs-golden-path-single/golden-path/cases/no-onepod-low-throughput.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "definitions": { - "include_onepod": false, - "kbps_throughput_tcp": 10, - "kbps_throughput_tail": 10, - "kbps_throughput_forward": 10 - } -} diff --git a/apps/firelens-stability/collections/ecs-golden-path-single/golden-path/cases/onepod-cw-down.json b/apps/firelens-stability/collections/ecs-golden-path-single/golden-path/cases/onepod-cw-down.json index 0f76c36..dc6d360 100644 --- a/apps/firelens-stability/collections/ecs-golden-path-single/golden-path/cases/onepod-cw-down.json +++ b/apps/firelens-stability/collections/ecs-golden-path-single/golden-path/cases/onepod-cw-down.json @@ -1,10 +1,6 @@ { "definitions": { - "cw_is_down": false, - "include_onepod": true, - "kbps_throughput_tcp": 5000, - "kbps_throughput_tail": 10, - "kbps_throughput_forward": 5000, - "datajet_generator_basic_contentType": "uniform" + "cw_is_down": true, + "include_onepod": true } } diff --git a/apps/firelens-stability/collections/ecs-golden-path-single/golden-path/cases/onepod-cw-up.json b/apps/firelens-stability/collections/ecs-golden-path-single/golden-path/cases/onepod-cw-up.json index 0f76c36..b8dc3a7 100644 --- a/apps/firelens-stability/collections/ecs-golden-path-single/golden-path/cases/onepod-cw-up.json +++ b/apps/firelens-stability/collections/ecs-golden-path-single/golden-path/cases/onepod-cw-up.json @@ -1,10 +1,6 @@ { "definitions": { "cw_is_down": false, - "include_onepod": true, - "kbps_throughput_tcp": 5000, - "kbps_throughput_tail": 10, - "kbps_throughput_forward": 5000, - "datajet_generator_basic_contentType": "uniform" + "include_onepod": true } } diff --git a/apps/firelens-stability/execution.json b/apps/firelens-stability/execution.json index 56e6b42..3e177e3 100644 --- a/apps/firelens-stability/execution.json +++ b/apps/firelens-stability/execution.json @@ -1,9 +1,12 @@ { - "executionName": "2_31_12_20230911", + "executionName": "cw-out-fs-buffer-limit-golden-path-single-2_31_12_20230911", "executeCollections": [ - "ecs-firelens-stability-tests" + "ecs-golden-path-single" ], "definitions": { "imageAwsForFluentBit": "public.ecr.aws/aws-observability/aws-for-fluent-bit:init-debug-2.31.12.20230911" + }, + "config": { + "template": "golden-path-mountebank-fargate-v10-16-2023" } } diff --git a/apps/firelens-stability/lib/helpers/start-execution-stages.ts b/apps/firelens-stability/lib/helpers/start-execution-stages.ts index 228d6a7..90caf93 100644 --- a/apps/firelens-stability/lib/helpers/start-execution-stages.ts +++ b/apps/firelens-stability/lib/helpers/start-execution-stages.ts @@ -180,17 +180,20 @@ export async function hydrateTestCaseSeed(testCaseSeed: ITestCaseSeed): const layer4Config = cascadeConfigurationStringAsExtension(testCaseSeed.caseConfigSeed, layer3Config); const layer5Config = cascadeConfigurationStringAsExtension(testCaseSeed.caseSeed, layer4Config); + /* Apply execution config before and after template to ensure overrides are set - allow set template */ + const layer6Config = cascadeConfigurationStringAsExtension(testCaseSeed.executionConfigSeed, layer5Config); + /* Get template */ - const templateName = layer5Config.config.template; + const templateName = layer6Config.config.template; if (!templateName) { - validateTestConfig(layer5Config); + validateTestConfig(layer6Config); } const templateConfigPath = Path.join(Constants.paths.templates, templateName, Constants.fileNames.templateDefaultConfig); const templateConfig = await getStringFromFile(templateConfigPath); - const layer6Config = cascadeConfigurationStringAsDefault(templateConfig, layer5Config); + const layer7Config = cascadeConfigurationStringAsDefault(templateConfig, layer6Config); /* Final layer of configuration from execution seed. These function as overrides. */ - const config = cascadeConfigurationStringAsExtension(testCaseSeed.executionConfigSeed, layer6Config); + const config = cascadeConfigurationStringAsExtension(testCaseSeed.executionConfigSeed, layer7Config); validateTestConfig(config); diff --git a/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/default-config.json b/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/default-config.json index bd5b054..30f8bdd 100644 --- a/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/default-config.json +++ b/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/default-config.json @@ -6,9 +6,10 @@ "datajet_generator_basic_contentType": "random", "datajet_generator_basic_contentRandomValueSet": "alpha-numeric", "include_onepod": false, - "kbps_throughput_tcp": 10, - "kbps_throughput_tail": 10, - "kbps_throughput_forward": 10, + "kbps_throughput_tcp_application": 1, + "kbps_throughput_tcp_request": 1, + "kbps_throughput_tcp_service": 1, + "kbps_throughput_forward": 1, "cw_use_mock": true, "add_forward": true, "cw_endpoint": "localhost", diff --git a/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/firelens-datajet.json b/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/firelens-datajet.json index be664cf..dbddf42 100644 --- a/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/firelens-datajet.json +++ b/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/firelens-datajet.json @@ -34,7 +34,7 @@ } }, { - "comment": "service log to tail", + "comment": "ApplicationLogs through tcp", "generator": { "name": "basic", "config": { @@ -46,18 +46,19 @@ } }, "datajet": { - "name": "file", + "name": "tcp", "config": { - "folder": "/var/tmp/data_logs", - "filename": "service_log_1.txt" + "host": "{{definitions.endpoint_flb}}", + "port": 5170, + "addNewline": true } }, "stage": { - "batchRate": {{definitions.kbps_throughput_tail}} + "batchRate": {{definitions.kbps_throughput_tcp_application}} } }, { - "comment": "ApplicationLogs through tcp", + "comment": "RequestLogs to tcp", "generator": { "name": "basic", "config": { @@ -72,36 +73,37 @@ "name": "tcp", "config": { "host": "{{definitions.endpoint_flb}}", - "port": 5170, + "port": 5171, "addNewline": true } }, "stage": { - "batchRate": {{definitions.kbps_throughput_tcp}} + "batchRate": {{definitions.kbps_throughput_tcp_request}} } }, { - "comment": "RequestLogs to tcp", + "comment": "ServiceLogs as emf to tcp (https://code.amazon.com/packages/CoralMetricsEmfReporter/blobs/78081aca20e12c524c58c248b0b8dc9318c3e66f/--/src/test/resources/serialized-format/MetricLogEvent.json#L1)", "generator": { - "name": "basic", + "name": "explicit", "config": { - "contentLength": 1000, - "batchSize": 1, - "logKey": "log", - "contentType": "{{definitions.datajet_generator_basic_contentType}}", - "contentRandomValueSet": "{{definitions.datajet_generator_basic_contentRandomValueSet}}" + "batches": [ + {"log":"{\"_aws\":{\"Timestamp\":1679351573496,\"LogGroupName\":\"ServiceMetrics\",\"CloudWatchMetrics\":[{\"Namespace\":\"ServiceMetrics\",\"Dimensions\":[[\"A\"]],\"Metrics\":[{\"Name\":\"ServiceMetric1\",\"Unit\":\"Microseconds\"}]}]},\"A\":\"Value\",\"ServiceMetric1\":10.0}"}, + {"log":"{\"_aws\":{\"Timestamp\":1679351573496,\"LogGroupName\":\"ServiceMetrics\",\"CloudWatchMetrics\":[{\"Namespace\":\"ServiceMetrics\",\"Dimensions\":[[\"A\"]],\"Metrics\":[{\"Name\":\"ServiceMetric1\",\"Unit\":\"Microseconds\"}]}]},\"A\":\"Value\",\"ServiceMetric1\":10.0}"}, + {"log":"{\"_aws\":{\"Timestamp\":1679351573496,\"LogGroupName\":\"ServiceMetrics\",\"CloudWatchMetrics\":[{\"Namespace\":\"ServiceMetrics\",\"Dimensions\":[[\"A\"]],\"Metrics\":[{\"Name\":\"ServiceMetric1\",\"Unit\":\"Microseconds\"}]}]},\"A\":\"Value\",\"ServiceMetric1\":10.0}"}, + {"log":"{\"_aws\":{\"Timestamp\":1679351573496,\"LogGroupName\":\"ServiceMetrics\",\"CloudWatchMetrics\":[{\"Namespace\":\"ServiceMetrics\",\"Dimensions\":[[\"A\"]],\"Metrics\":[{\"Name\":\"ServiceMetric1\",\"Unit\":\"Microseconds\"}]}]},\"A\":\"Value\",\"ServiceMetric1\":10.0}"} + ] } }, "datajet": { "name": "tcp", "config": { "host": "{{definitions.endpoint_flb}}", - "port": 5171, + "port": 5172, "addNewline": true } }, "stage": { - "batchRate": {{definitions.kbps_throughput_tcp}} + "batchRate": {{definitions.kbps_throughput_tcp_service}} } } ] diff --git a/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/fluent-bit-base.conf b/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/fluent-bit-base.conf index 4f24e2f..b47b668 100644 --- a/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/fluent-bit-base.conf +++ b/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/fluent-bit-base.conf @@ -9,6 +9,13 @@ Grace 30 Log_Level info storage.path /var/log/flb-storage/ + # Important! This is your memory limiter setting for inputs with storage.type filesystem + # Total Max Memory Usage <= (2 * # of inputs with filesystem buffer * storage.max_chunks_up * 2 MB per chunk) + # + (2 * SUM(Each input Mem_Buf_Limit for inputs with memory buffer) + # This calculation should inform the Fluent Bit container memory limit or reservation. + # If you switch to default storage.type memory, please carefully read the guide + # https://github.com/aws-samples/amazon-ecs-firelens-examples/tree/mainline/examples/fluent-bit/oomkill-prevention + storage.max_chunks_up 32 {{#if definitions.segfault}} # Artificially trigger a segfault if configured @@ -29,6 +36,7 @@ [INPUT] Name tcp + Alias TCP-ApplicationLogs Listen 0.0.0.0 # Tag used for routing to OUTPUT @@ -41,55 +49,43 @@ Format none Chunk_Size 32 - Buffer_Size 64 + Buffer_Size 256 + # If this is set to (default) "storage.type memory" or not configured (memory is default), + # then Mem_Buf_Limit *should be set*. + # If this is set to "storage.type filesystem", Mem_Buf_Limit has no effect. It + # uses the [SERVICE] level "storage.max_chunks_up" configuration instead. + # https://github.com/aws-samples/amazon-ecs-firelens-examples/tree/mainline/examples/fluent-bit/oomkill-prevention storage.type filesystem - # Important! Every INPUT section's Mem_Buf_Limit must be re-computed when - # the container memory's limit or the number of of INPUT sections change. If you - # update this value, confirm that the container memory limit in the task - # definition is correct. Also note that an additional INPUT section is - # configured by default, so count the INPUTs in this file and add 1. - # Mem_Buf_Limit <= Memory_limit / 2 / (number of INPUT sections + 1) - # 64MB = 512MB / 2 / (4 INPUT sections) - Mem_Buf_Limit 64MB - [INPUT] Name tcp + Alias TCP-RequestLogs Tag RequestLogs Listen 0.0.0.0 Port 5171 Format none Chunk_Size 32 - Buffer_Size 64 + Buffer_Size 256 + # If this is set to (default) "storage.type memory" or not configured (memory is default), + # then Mem_Buf_Limit *should be set*. + # If this is set to "storage.type filesystem", Mem_Buf_Limit has no effect. It + # uses the [SERVICE] level "storage.max_chunks_up" configuration instead. storage.type filesystem - # Important! Every INPUT section's Mem_Buf_Limit must be re-computed when - # the container memory's limit or the number of of INPUT sections change. If you - # update this value, confirm that the container memory limit in the task - # definition is correct. Also note that an additional INPUT section is - # configured by default, so count the INPUTs in this file and add 1. - # Mem_Buf_Limit <= Memory_limit / 2 / (number of INPUT sections + 1) - # 64MB = 512MB / 2 / (4 INPUT sections) - Mem_Buf_Limit 64MB - [INPUT] - Name tail - Tag ServiceMetrics - Path /tmp/data_logs/service_log_1.txt - Exclude_Path *.gz - Rotate_Wait 15 - Multiline On - Parser_Firstline QueryLogSeparator - Parser_1 QueryLog - - # Important! Every INPUT section's Mem_Buf_Limit must be re-computed when - # the container memory's limit or the number of of INPUT sections change. If you - # update this value, confirm that the container memory limit in the task - # definition is correct. Also note that an additional INPUT section is - # configured by default, so count the INPUTs in this file and add 1. - # Mem_Buf_Limit <= Memory_limit / 2 / (number of INPUT sections + 1) - # 64MB = 512MB / 2 / (4 INPUT sections) - Mem_Buf_Limit 64MB + Name tcp + Alias TCP-ServiceMetrics + Tag ServiceMetrics + Listen 0.0.0.0 + Port 5172 + Format none + Chunk_Size 32 + Buffer_Size 256 + # If this is set to (default) "storage.type memory" or not configured (memory is default), + # then Mem_Buf_Limit *should be set*. + # If this is set to "storage.type filesystem", Mem_Buf_Limit has no effect. It + # uses the [SERVICE] level "storage.max_chunks_up" configuration instead. + storage.type filesystem [OUTPUT] # Send any StdOut/StdErr Logs to A CloudWatch Logs Group. diff --git a/examples/generators/example-explicit-generator-to-tcp.json b/examples/generators/example-explicit-generator-to-tcp.json new file mode 100644 index 0000000..bc68a9a --- /dev/null +++ b/examples/generators/example-explicit-generator-to-tcp.json @@ -0,0 +1,23 @@ +{ + "comment": "ServiceLogs as emf to tcp (https://code.amazon.com/packages/CoralMetricsEmfReporter/blobs/78081aca20e12c524c58c248b0b8dc9318c3e66f/--/src/test/resources/serialized-format/MetricLogEvent.json#L1)", + "generator": { + "name": "explicit", + "config": { + "batches": [ + {"_aws":{"Timestamp":1679351573496,"LogGroupName":"ServiceMetrics","CloudWatchMetrics":[{"Namespace":"ServiceMetrics","Dimensions":[["A"]],"Metrics":[{"Name":"ServiceMetric1","Unit":"Microseconds"}]}]},"A":"Value","ServiceMetric1":10.0} + ] + } + }, + "datajet": { + "name": "tcp", + "config": { + "host": "localhost", + "port": 5172, + "addNewline": true, + "logKey": null + } + }, + "stage": { + "batchRate": 1 + } +} From 78f1705a5cfe0d5c14b20f238e19eee6cd10ebc6 Mon Sep 17 00:00:00 2001 From: Matthew Fala Date: Tue, 17 Oct 2023 04:28:30 +0000 Subject: [PATCH 4/7] stability: revision no revision comparison --- .../case-config.json | 6 ++++++ .../cases/low-throughput-cloudwatch.json | 10 ++++++++++ .../golden-path-10-16-2023-revision/case-config.json | 6 ++++++ .../golden-path-10-16-2023-revision/cases | 1 + .../golden-path-10-16-2023/case-config.json | 6 ++++++ .../cases/no-onepod-cw-down.json | 0 .../cases/no-onepod-cw-up.json | 0 .../cases/onepod-cw-down.json | 0 .../cases/onepod-cw-up.json | 0 .../golden-path/case-config.json | 11 ----------- .../ecs-golden-path-single/suite-config.json | 9 ++++++++- apps/firelens-stability/execution.json | 2 +- .../default-config.json | 3 ++- .../firelens-datajet.json | 11 +++++++---- .../fluent-bit-base.conf | 4 ++-- .../fluent-bit-onepod.conf | 6 +++--- .../fluent-bit.conf | 2 +- 17 files changed, 53 insertions(+), 24 deletions(-) create mode 100644 apps/firelens-stability/collections/ecs-golden-path-single/golden-path-10-16-2023-cw-no-mock/case-config.json create mode 100644 apps/firelens-stability/collections/ecs-golden-path-single/golden-path-10-16-2023-cw-no-mock/cases/low-throughput-cloudwatch.json create mode 100644 apps/firelens-stability/collections/ecs-golden-path-single/golden-path-10-16-2023-revision/case-config.json create mode 120000 apps/firelens-stability/collections/ecs-golden-path-single/golden-path-10-16-2023-revision/cases create mode 100644 apps/firelens-stability/collections/ecs-golden-path-single/golden-path-10-16-2023/case-config.json rename apps/firelens-stability/collections/ecs-golden-path-single/{golden-path => golden-path-10-16-2023}/cases/no-onepod-cw-down.json (100%) rename apps/firelens-stability/collections/ecs-golden-path-single/{golden-path => golden-path-10-16-2023}/cases/no-onepod-cw-up.json (100%) rename apps/firelens-stability/collections/ecs-golden-path-single/{golden-path => golden-path-10-16-2023}/cases/onepod-cw-down.json (100%) rename apps/firelens-stability/collections/ecs-golden-path-single/{golden-path => golden-path-10-16-2023}/cases/onepod-cw-up.json (100%) delete mode 100644 apps/firelens-stability/collections/ecs-golden-path-single/golden-path/case-config.json diff --git a/apps/firelens-stability/collections/ecs-golden-path-single/golden-path-10-16-2023-cw-no-mock/case-config.json b/apps/firelens-stability/collections/ecs-golden-path-single/golden-path-10-16-2023-cw-no-mock/case-config.json new file mode 100644 index 0000000..c7c491d --- /dev/null +++ b/apps/firelens-stability/collections/ecs-golden-path-single/golden-path-10-16-2023-cw-no-mock/case-config.json @@ -0,0 +1,6 @@ +{ + "config": { + "template": "golden-path-mountebank-fargate-v10-16-2023", + "config_revision": true + } +} diff --git a/apps/firelens-stability/collections/ecs-golden-path-single/golden-path-10-16-2023-cw-no-mock/cases/low-throughput-cloudwatch.json b/apps/firelens-stability/collections/ecs-golden-path-single/golden-path-10-16-2023-cw-no-mock/cases/low-throughput-cloudwatch.json new file mode 100644 index 0000000..5728418 --- /dev/null +++ b/apps/firelens-stability/collections/ecs-golden-path-single/golden-path-10-16-2023-cw-no-mock/cases/low-throughput-cloudwatch.json @@ -0,0 +1,10 @@ +{ + "definitions": { + "cw_endpoint": "logs.us-west-2.amazonaws.com", + "cw_use_mock": false, + "kbps_throughput_tcp_application": 1, + "kbps_throughput_tcp_request": 1, + "kbps_throughput_tcp_service": 1, + "kbps_throughput_forward": 1 + } +} diff --git a/apps/firelens-stability/collections/ecs-golden-path-single/golden-path-10-16-2023-revision/case-config.json b/apps/firelens-stability/collections/ecs-golden-path-single/golden-path-10-16-2023-revision/case-config.json new file mode 100644 index 0000000..c7c491d --- /dev/null +++ b/apps/firelens-stability/collections/ecs-golden-path-single/golden-path-10-16-2023-revision/case-config.json @@ -0,0 +1,6 @@ +{ + "config": { + "template": "golden-path-mountebank-fargate-v10-16-2023", + "config_revision": true + } +} diff --git a/apps/firelens-stability/collections/ecs-golden-path-single/golden-path-10-16-2023-revision/cases b/apps/firelens-stability/collections/ecs-golden-path-single/golden-path-10-16-2023-revision/cases new file mode 120000 index 0000000..5d589b1 --- /dev/null +++ b/apps/firelens-stability/collections/ecs-golden-path-single/golden-path-10-16-2023-revision/cases @@ -0,0 +1 @@ +../golden-path-10-16-2023/cases \ No newline at end of file diff --git a/apps/firelens-stability/collections/ecs-golden-path-single/golden-path-10-16-2023/case-config.json b/apps/firelens-stability/collections/ecs-golden-path-single/golden-path-10-16-2023/case-config.json new file mode 100644 index 0000000..14b4cc9 --- /dev/null +++ b/apps/firelens-stability/collections/ecs-golden-path-single/golden-path-10-16-2023/case-config.json @@ -0,0 +1,6 @@ +{ + "config": { + "template": "golden-path-mountebank-fargate-v10-16-2023", + "config_revision": false + } +} diff --git a/apps/firelens-stability/collections/ecs-golden-path-single/golden-path/cases/no-onepod-cw-down.json b/apps/firelens-stability/collections/ecs-golden-path-single/golden-path-10-16-2023/cases/no-onepod-cw-down.json similarity index 100% rename from apps/firelens-stability/collections/ecs-golden-path-single/golden-path/cases/no-onepod-cw-down.json rename to apps/firelens-stability/collections/ecs-golden-path-single/golden-path-10-16-2023/cases/no-onepod-cw-down.json diff --git a/apps/firelens-stability/collections/ecs-golden-path-single/golden-path/cases/no-onepod-cw-up.json b/apps/firelens-stability/collections/ecs-golden-path-single/golden-path-10-16-2023/cases/no-onepod-cw-up.json similarity index 100% rename from apps/firelens-stability/collections/ecs-golden-path-single/golden-path/cases/no-onepod-cw-up.json rename to apps/firelens-stability/collections/ecs-golden-path-single/golden-path-10-16-2023/cases/no-onepod-cw-up.json diff --git a/apps/firelens-stability/collections/ecs-golden-path-single/golden-path/cases/onepod-cw-down.json b/apps/firelens-stability/collections/ecs-golden-path-single/golden-path-10-16-2023/cases/onepod-cw-down.json similarity index 100% rename from apps/firelens-stability/collections/ecs-golden-path-single/golden-path/cases/onepod-cw-down.json rename to apps/firelens-stability/collections/ecs-golden-path-single/golden-path-10-16-2023/cases/onepod-cw-down.json diff --git a/apps/firelens-stability/collections/ecs-golden-path-single/golden-path/cases/onepod-cw-up.json b/apps/firelens-stability/collections/ecs-golden-path-single/golden-path-10-16-2023/cases/onepod-cw-up.json similarity index 100% rename from apps/firelens-stability/collections/ecs-golden-path-single/golden-path/cases/onepod-cw-up.json rename to apps/firelens-stability/collections/ecs-golden-path-single/golden-path-10-16-2023/cases/onepod-cw-up.json diff --git a/apps/firelens-stability/collections/ecs-golden-path-single/golden-path/case-config.json b/apps/firelens-stability/collections/ecs-golden-path-single/golden-path/case-config.json deleted file mode 100644 index 62a8774..0000000 --- a/apps/firelens-stability/collections/ecs-golden-path-single/golden-path/case-config.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "config": { - "taskCount": 1, - "adot_enable": true, - "kbps_throughput_tcp_application": 5000, - "kbps_throughput_tcp_request": 5000, - "kbps_throughput_tcp_service": 5000, - "kbps_throughput_forward": 5000, - "datajet_generator_basic_contentType": "uniform" - } -} diff --git a/apps/firelens-stability/collections/ecs-golden-path-single/suite-config.json b/apps/firelens-stability/collections/ecs-golden-path-single/suite-config.json index 2b64832..c5e76ff 100644 --- a/apps/firelens-stability/collections/ecs-golden-path-single/suite-config.json +++ b/apps/firelens-stability/collections/ecs-golden-path-single/suite-config.json @@ -4,6 +4,13 @@ "segfault": false }, "config": { - "cluster": "{{managed.executionName}}-{{managed.executionId}}-{{managed.collectionName}}" + "cluster": "{{managed.executionName}}-{{managed.executionId}}-{{managed.collectionName}}", + "taskCount": 1, + "adot_enable": true, + "kbps_throughput_tcp_application": 5000, + "kbps_throughput_tcp_request": 5000, + "kbps_throughput_tcp_service": 5000, + "kbps_throughput_forward": 5000, + "datajet_generator_basic_contentType": "uniform" } } \ No newline at end of file diff --git a/apps/firelens-stability/execution.json b/apps/firelens-stability/execution.json index 3e177e3..a56decd 100644 --- a/apps/firelens-stability/execution.json +++ b/apps/firelens-stability/execution.json @@ -1,5 +1,5 @@ { - "executionName": "cw-out-fs-buffer-limit-golden-path-single-2_31_12_20230911", + "executionName": "golden-path-10-16-2023-revision-single-2_31_12_20230911", "executeCollections": [ "ecs-golden-path-single" ], diff --git a/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/default-config.json b/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/default-config.json index 30f8bdd..00e683f 100644 --- a/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/default-config.json +++ b/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/default-config.json @@ -18,6 +18,7 @@ "ServiceName": "golden-onepod-low-throughput", "debugS3Bucket": "{{managed.s3OutputBucket}}", "debugS3KeyPrefix": "{{managed.s3OutputExecutionPath}}/{{managed.caseNameUnique}}", - "cw_is_down": false + "cw_is_down": false, + "config_revision": false } } \ No newline at end of file diff --git a/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/firelens-datajet.json b/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/firelens-datajet.json index dbddf42..5e87720 100644 --- a/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/firelens-datajet.json +++ b/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/firelens-datajet.json @@ -18,7 +18,8 @@ "logKey": "log", "addNewline": true, "contentType": "{{definitions.datajet_generator_basic_contentType}}", - "contentRandomValueSet": "{{definitions.datajet_generator_basic_contentRandomValueSet}}" + "contentRandomValueSet": "{{definitions.datajet_generator_basic_contentRandomValueSet}}", + "contentUniformValue": "F" } }, "datajet": { @@ -42,7 +43,8 @@ "batchSize": 1, "logKey": "log", "contentType": "{{definitions.datajet_generator_basic_contentType}}", - "contentRandomValueSet": "{{definitions.datajet_generator_basic_contentRandomValueSet}}" + "contentRandomValueSet": "{{definitions.datajet_generator_basic_contentRandomValueSet}}", + "contentUniformValue": "A" } }, "datajet": { @@ -66,7 +68,8 @@ "batchSize": 1, "logKey": "log", "contentType": "{{definitions.datajet_generator_basic_contentType}}", - "contentRandomValueSet": "{{definitions.datajet_generator_basic_contentRandomValueSet}}" + "contentRandomValueSet": "{{definitions.datajet_generator_basic_contentRandomValueSet}}", + "contentUniformValue": "R" } }, "datajet": { @@ -107,4 +110,4 @@ } } ] -} \ No newline at end of file +} diff --git a/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/fluent-bit-base.conf b/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/fluent-bit-base.conf index b47b668..359756b 100644 --- a/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/fluent-bit-base.conf +++ b/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/fluent-bit-base.conf @@ -115,7 +115,7 @@ log_key log auto_create_group {{definitions.cw_auto_create_group}} retry_limit 3 - storage.total_limit_size 2G + {{#if definitions.config_revision}}storage.total_limit_size 2G{{/if}} endpoint ${CLOUDWATCH_ENDPOINT} {{#if definitions.cw_use_mock}} tls.verify false @@ -133,7 +133,7 @@ log_key log auto_create_group {{definitions.cw_auto_create_group}} retry_limit 3 - storage.total_limit_size 2G + {{#if definitions.config_revision}}storage.total_limit_size 2G{{/if}} endpoint ${CLOUDWATCH_ENDPOINT} {{#if definitions.cw_use_mock}} tls.verify false diff --git a/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/fluent-bit-onepod.conf b/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/fluent-bit-onepod.conf index bf3ab50..91d4b20 100644 --- a/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/fluent-bit-onepod.conf +++ b/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/fluent-bit-onepod.conf @@ -32,7 +32,7 @@ log_key log auto_create_group {{definitions.cw_auto_create_group}} retry_limit 3 - storage.total_limit_size 2G + {{#if definitions.config_revision}}storage.total_limit_size 2G{{/if}} endpoint ${CLOUDWATCH_ENDPOINT} {{#if definitions.cw_use_mock}} tls.verify false @@ -50,7 +50,7 @@ log_key log auto_create_group {{definitions.cw_auto_create_group}} retry_limit 3 - storage.total_limit_size 2G + {{#if definitions.config_revision}}storage.total_limit_size 2G{{/if}} endpoint ${CLOUDWATCH_ENDPOINT} {{#if definitions.cw_use_mock}} tls.verify false @@ -69,7 +69,7 @@ log_format json/emf auto_create_group {{definitions.cw_auto_create_group}} retry_limit 3 - storage.total_limit_size 1G + {{#if definitions.config_revision}}storage.total_limit_size 1G{{/if}} endpoint ${CLOUDWATCH_ENDPOINT} {{#if definitions.cw_use_mock}} tls.verify false diff --git a/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/fluent-bit.conf b/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/fluent-bit.conf index cf02b48..b5ad79d 100644 --- a/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/fluent-bit.conf +++ b/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/fluent-bit.conf @@ -13,7 +13,7 @@ log_key log auto_create_group {{definitions.cw_auto_create_group}} retry_limit 3 - storage.total_limit_size 1G + {{#if definitions.config_revision}}storage.total_limit_size 1G{{/if}} endpoint ${CLOUDWATCH_ENDPOINT} {{#if definitions.cw_use_mock}} tls.verify false From 986ff3ce459c297673968c9e75f8c408b2eac0b6 Mon Sep 17 00:00:00 2001 From: Matthew Fala Date: Tue, 24 Oct 2023 02:12:48 +0000 Subject: [PATCH 5/7] stability: more revisions --- .../golden-path-10-16-2023-cw-no-mock/case-config.json | 4 +++- .../cases/low-throughput-cloudwatch.json | 3 ++- .../golden-path-10-16-2023-revision/case-config.json | 4 +++- .../golden-path-10-16-2023/case-config.json | 4 +++- .../cases/onepod-cw-down-low-throughput.json | 10 ++++++++++ .../cases/onepod-cw-up-low-throughput.json | 10 ++++++++++ .../ecs-golden-path-single/suite-config.json | 1 + apps/firelens-stability/execution.json | 2 +- .../fluent-bit-onepod.conf | 2 +- firelens-datajet.json | 4 ++-- 10 files changed, 36 insertions(+), 8 deletions(-) create mode 100644 apps/firelens-stability/collections/ecs-golden-path-single/golden-path-10-16-2023/cases/onepod-cw-down-low-throughput.json create mode 100644 apps/firelens-stability/collections/ecs-golden-path-single/golden-path-10-16-2023/cases/onepod-cw-up-low-throughput.json diff --git a/apps/firelens-stability/collections/ecs-golden-path-single/golden-path-10-16-2023-cw-no-mock/case-config.json b/apps/firelens-stability/collections/ecs-golden-path-single/golden-path-10-16-2023-cw-no-mock/case-config.json index c7c491d..79edc4a 100644 --- a/apps/firelens-stability/collections/ecs-golden-path-single/golden-path-10-16-2023-cw-no-mock/case-config.json +++ b/apps/firelens-stability/collections/ecs-golden-path-single/golden-path-10-16-2023-cw-no-mock/case-config.json @@ -1,6 +1,8 @@ { "config": { - "template": "golden-path-mountebank-fargate-v10-16-2023", + "template": "golden-path-mountebank-fargate-v10-16-2023" + }, + "definitions": { "config_revision": true } } diff --git a/apps/firelens-stability/collections/ecs-golden-path-single/golden-path-10-16-2023-cw-no-mock/cases/low-throughput-cloudwatch.json b/apps/firelens-stability/collections/ecs-golden-path-single/golden-path-10-16-2023-cw-no-mock/cases/low-throughput-cloudwatch.json index 5728418..6f21116 100644 --- a/apps/firelens-stability/collections/ecs-golden-path-single/golden-path-10-16-2023-cw-no-mock/cases/low-throughput-cloudwatch.json +++ b/apps/firelens-stability/collections/ecs-golden-path-single/golden-path-10-16-2023-cw-no-mock/cases/low-throughput-cloudwatch.json @@ -5,6 +5,7 @@ "kbps_throughput_tcp_application": 1, "kbps_throughput_tcp_request": 1, "kbps_throughput_tcp_service": 1, - "kbps_throughput_forward": 1 + "kbps_throughput_forward": 1, + "cw_auto_create_group": true } } diff --git a/apps/firelens-stability/collections/ecs-golden-path-single/golden-path-10-16-2023-revision/case-config.json b/apps/firelens-stability/collections/ecs-golden-path-single/golden-path-10-16-2023-revision/case-config.json index c7c491d..79edc4a 100644 --- a/apps/firelens-stability/collections/ecs-golden-path-single/golden-path-10-16-2023-revision/case-config.json +++ b/apps/firelens-stability/collections/ecs-golden-path-single/golden-path-10-16-2023-revision/case-config.json @@ -1,6 +1,8 @@ { "config": { - "template": "golden-path-mountebank-fargate-v10-16-2023", + "template": "golden-path-mountebank-fargate-v10-16-2023" + }, + "definitions": { "config_revision": true } } diff --git a/apps/firelens-stability/collections/ecs-golden-path-single/golden-path-10-16-2023/case-config.json b/apps/firelens-stability/collections/ecs-golden-path-single/golden-path-10-16-2023/case-config.json index 14b4cc9..9681558 100644 --- a/apps/firelens-stability/collections/ecs-golden-path-single/golden-path-10-16-2023/case-config.json +++ b/apps/firelens-stability/collections/ecs-golden-path-single/golden-path-10-16-2023/case-config.json @@ -1,6 +1,8 @@ { "config": { - "template": "golden-path-mountebank-fargate-v10-16-2023", + "template": "golden-path-mountebank-fargate-v10-16-2023" + }, + "definitions": { "config_revision": false } } diff --git a/apps/firelens-stability/collections/ecs-golden-path-single/golden-path-10-16-2023/cases/onepod-cw-down-low-throughput.json b/apps/firelens-stability/collections/ecs-golden-path-single/golden-path-10-16-2023/cases/onepod-cw-down-low-throughput.json new file mode 100644 index 0000000..43c6618 --- /dev/null +++ b/apps/firelens-stability/collections/ecs-golden-path-single/golden-path-10-16-2023/cases/onepod-cw-down-low-throughput.json @@ -0,0 +1,10 @@ +{ + "definitions": { + "cw_is_down": true, + "include_onepod": true, + "kbps_throughput_tcp_application": 1, + "kbps_throughput_tcp_request": 1, + "kbps_throughput_tcp_service": 1, + "kbps_throughput_forward": 1 + } +} diff --git a/apps/firelens-stability/collections/ecs-golden-path-single/golden-path-10-16-2023/cases/onepod-cw-up-low-throughput.json b/apps/firelens-stability/collections/ecs-golden-path-single/golden-path-10-16-2023/cases/onepod-cw-up-low-throughput.json new file mode 100644 index 0000000..53ed407 --- /dev/null +++ b/apps/firelens-stability/collections/ecs-golden-path-single/golden-path-10-16-2023/cases/onepod-cw-up-low-throughput.json @@ -0,0 +1,10 @@ +{ + "definitions": { + "cw_is_down": false, + "include_onepod": true, + "kbps_throughput_tcp_application": 1, + "kbps_throughput_tcp_request": 1, + "kbps_throughput_tcp_service": 1, + "kbps_throughput_forward": 1 + } +} diff --git a/apps/firelens-stability/collections/ecs-golden-path-single/suite-config.json b/apps/firelens-stability/collections/ecs-golden-path-single/suite-config.json index c5e76ff..f3b49fe 100644 --- a/apps/firelens-stability/collections/ecs-golden-path-single/suite-config.json +++ b/apps/firelens-stability/collections/ecs-golden-path-single/suite-config.json @@ -1,6 +1,7 @@ { "definitions": { "testId": "{{managed.suiteName}}-{{managed.caseName}}", + "ServiceName": "{{managed.executionName}}-{{managed.executionId}}", "segfault": false }, "config": { diff --git a/apps/firelens-stability/execution.json b/apps/firelens-stability/execution.json index a56decd..2518125 100644 --- a/apps/firelens-stability/execution.json +++ b/apps/firelens-stability/execution.json @@ -1,5 +1,5 @@ { - "executionName": "golden-path-10-16-2023-revision-single-2_31_12_20230911", + "executionName": "r2-golden-path-10-16-2023-revision-single-2_31_12_20230911", "executeCollections": [ "ecs-golden-path-single" ], diff --git a/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/fluent-bit-onepod.conf b/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/fluent-bit-onepod.conf index 91d4b20..fe2dd76 100644 --- a/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/fluent-bit-onepod.conf +++ b/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/fluent-bit-onepod.conf @@ -13,7 +13,7 @@ region ${LOG_REGION} log_group_name {{ServiceName}}-OnePod-${STAGE}-AppContainer-STDOUT log_stream_prefix STDOUT-${HOSTNAME} - auto_create_group {{definitions.cw_auto_create_group}} # false + auto_create_group {{definitions.cw_auto_create_group}} retry_limit 3 endpoint ${CLOUDWATCH_ENDPOINT} {{#if definitions.cw_use_mock}} diff --git a/firelens-datajet.json b/firelens-datajet.json index 84c3f53..248cce3 100644 --- a/firelens-datajet.json +++ b/firelens-datajet.json @@ -13,7 +13,7 @@ } }, "stage": { - "batchRate": 1000, - "maxBatches": 10 + "batchRate": 100, + "maxBatches": 1000000 } } \ No newline at end of file From fe51bc030dc3db06849cd6ba643ec0ebf85b0923 Mon Sep 17 00:00:00 2001 From: Matthew Fala Date: Sat, 4 Nov 2023 10:27:27 +0000 Subject: [PATCH 6/7] stability: golden config update Signed-off-by: Matthew Fala --- .../default-config.json | 2 +- .../firelens-datajet.json | 6 +++--- .../fluent-bit-base.conf | 16 +++++++++++--- .../fluent-bit-onepod.conf | 18 +++++++++++++--- .../fluent-bit.conf | 6 +++++- .../task-definition.json | 21 +------------------ 6 files changed, 38 insertions(+), 31 deletions(-) diff --git a/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/default-config.json b/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/default-config.json index 00e683f..93dc409 100644 --- a/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/default-config.json +++ b/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/default-config.json @@ -19,6 +19,6 @@ "debugS3Bucket": "{{managed.s3OutputBucket}}", "debugS3KeyPrefix": "{{managed.s3OutputExecutionPath}}/{{managed.caseNameUnique}}", "cw_is_down": false, - "config_revision": false + "is_golden_revision_applied": false } } \ No newline at end of file diff --git a/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/firelens-datajet.json b/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/firelens-datajet.json index 5e87720..51b9124 100644 --- a/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/firelens-datajet.json +++ b/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/firelens-datajet.json @@ -91,9 +91,9 @@ "config": { "batches": [ {"log":"{\"_aws\":{\"Timestamp\":1679351573496,\"LogGroupName\":\"ServiceMetrics\",\"CloudWatchMetrics\":[{\"Namespace\":\"ServiceMetrics\",\"Dimensions\":[[\"A\"]],\"Metrics\":[{\"Name\":\"ServiceMetric1\",\"Unit\":\"Microseconds\"}]}]},\"A\":\"Value\",\"ServiceMetric1\":10.0}"}, - {"log":"{\"_aws\":{\"Timestamp\":1679351573496,\"LogGroupName\":\"ServiceMetrics\",\"CloudWatchMetrics\":[{\"Namespace\":\"ServiceMetrics\",\"Dimensions\":[[\"A\"]],\"Metrics\":[{\"Name\":\"ServiceMetric1\",\"Unit\":\"Microseconds\"}]}]},\"A\":\"Value\",\"ServiceMetric1\":10.0}"}, - {"log":"{\"_aws\":{\"Timestamp\":1679351573496,\"LogGroupName\":\"ServiceMetrics\",\"CloudWatchMetrics\":[{\"Namespace\":\"ServiceMetrics\",\"Dimensions\":[[\"A\"]],\"Metrics\":[{\"Name\":\"ServiceMetric1\",\"Unit\":\"Microseconds\"}]}]},\"A\":\"Value\",\"ServiceMetric1\":10.0}"}, - {"log":"{\"_aws\":{\"Timestamp\":1679351573496,\"LogGroupName\":\"ServiceMetrics\",\"CloudWatchMetrics\":[{\"Namespace\":\"ServiceMetrics\",\"Dimensions\":[[\"A\"]],\"Metrics\":[{\"Name\":\"ServiceMetric1\",\"Unit\":\"Microseconds\"}]}]},\"A\":\"Value\",\"ServiceMetric1\":10.0}"} + {"log":"{\"_aws\":{\"Timestamp\":1679351573596,\"LogGroupName\":\"ServiceMetrics\",\"CloudWatchMetrics\":[{\"Namespace\":\"ServiceMetrics\",\"Dimensions\":[[\"A\"]],\"Metrics\":[{\"Name\":\"ServiceMetric1\",\"Unit\":\"Microseconds\"}]}]},\"A\":\"Value\",\"ServiceMetric1\":10.0}"}, + {"log":"{\"_aws\":{\"Timestamp\":1679351573696,\"LogGroupName\":\"ServiceMetrics\",\"CloudWatchMetrics\":[{\"Namespace\":\"ServiceMetrics\",\"Dimensions\":[[\"A\"]],\"Metrics\":[{\"Name\":\"ServiceMetric1\",\"Unit\":\"Microseconds\"}]}]},\"A\":\"Value\",\"ServiceMetric1\":10.0}"}, + {"log":"{\"_aws\":{\"Timestamp\":1679351573796,\"LogGroupName\":\"ServiceMetrics\",\"CloudWatchMetrics\":[{\"Namespace\":\"ServiceMetrics\",\"Dimensions\":[[\"A\"]],\"Metrics\":[{\"Name\":\"ServiceMetric1\",\"Unit\":\"Microseconds\"}]}]},\"A\":\"Value\",\"ServiceMetric1\":10.0}"} ] } }, diff --git a/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/fluent-bit-base.conf b/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/fluent-bit-base.conf index 359756b..d125c1f 100644 --- a/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/fluent-bit-base.conf +++ b/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/fluent-bit-base.conf @@ -99,7 +99,9 @@ auto_create_group {{definitions.cw_auto_create_group}} retry_limit 3 endpoint ${CLOUDWATCH_ENDPOINT} - +{{#if definitions.is_golden_revision_applied}} + net.keepalive_idle_timeout 4 +{{/if}} {{#if definitions.cw_use_mock}} tls.verify false port 4545 @@ -108,6 +110,7 @@ [OUTPUT] # Send any Application Logs to A CloudWatch Logs Group Name cloudwatch_logs + Alias CloudWatch-ApplicationLogs Match ApplicationLogs region ${LOG_REGION} log_group_name {{ServiceName}}-${STAGE}-ApplicationLogs @@ -115,13 +118,17 @@ log_key log auto_create_group {{definitions.cw_auto_create_group}} retry_limit 3 - {{#if definitions.config_revision}}storage.total_limit_size 2G{{/if}} endpoint ${CLOUDWATCH_ENDPOINT} +{{#if definitions.is_golden_revision_applied}} + storage.total_limit_size 2G + net.keepalive_idle_timeout 4 +{{/if}} {{#if definitions.cw_use_mock}} tls.verify false port 4545 {{/if}} + [OUTPUT] # Send any Request Logs to A CloudWatch Logs Group Name cloudwatch_logs @@ -133,8 +140,11 @@ log_key log auto_create_group {{definitions.cw_auto_create_group}} retry_limit 3 - {{#if definitions.config_revision}}storage.total_limit_size 2G{{/if}} endpoint ${CLOUDWATCH_ENDPOINT} +{{#if definitions.is_golden_revision_applied}} + storage.total_limit_size 2G + net.keepalive_idle_timeout 4 +{{/if}} {{#if definitions.cw_use_mock}} tls.verify false port 4545 diff --git a/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/fluent-bit-onepod.conf b/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/fluent-bit-onepod.conf index fe2dd76..96b96f6 100644 --- a/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/fluent-bit-onepod.conf +++ b/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/fluent-bit-onepod.conf @@ -16,6 +16,9 @@ auto_create_group {{definitions.cw_auto_create_group}} retry_limit 3 endpoint ${CLOUDWATCH_ENDPOINT} +{{#if definitions.is_golden_revision_applied}} + net.keepalive_idle_timeout 4 +{{/if}} {{#if definitions.cw_use_mock}} tls.verify false port 4545 @@ -32,8 +35,11 @@ log_key log auto_create_group {{definitions.cw_auto_create_group}} retry_limit 3 - {{#if definitions.config_revision}}storage.total_limit_size 2G{{/if}} endpoint ${CLOUDWATCH_ENDPOINT} +{{#if definitions.is_golden_revision_applied}} + storage.total_limit_size 2G + net.keepalive_idle_timeout 4 +{{/if}} {{#if definitions.cw_use_mock}} tls.verify false port 4545 @@ -50,8 +56,11 @@ log_key log auto_create_group {{definitions.cw_auto_create_group}} retry_limit 3 - {{#if definitions.config_revision}}storage.total_limit_size 2G{{/if}} endpoint ${CLOUDWATCH_ENDPOINT} +{{#if definitions.is_golden_revision_applied}} + storage.total_limit_size 2G + net.keepalive_idle_timeout 4 +{{/if}} {{#if definitions.cw_use_mock}} tls.verify false port 4545 @@ -69,8 +78,11 @@ log_format json/emf auto_create_group {{definitions.cw_auto_create_group}} retry_limit 3 - {{#if definitions.config_revision}}storage.total_limit_size 1G{{/if}} endpoint ${CLOUDWATCH_ENDPOINT} +{{#if definitions.is_golden_revision_applied}} + storage.total_limit_size 1G + net.keepalive_idle_timeout 4 +{{/if}} {{#if definitions.cw_use_mock}} tls.verify false port 4545 diff --git a/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/fluent-bit.conf b/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/fluent-bit.conf index b5ad79d..b73ad04 100644 --- a/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/fluent-bit.conf +++ b/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/fluent-bit.conf @@ -11,10 +11,14 @@ log_group_name {{definitions.ServiceName}}-${STAGE}-ServiceMetrics log_stream_prefix ServiceMetrics-${HOSTNAME} log_key log + log_format json/emf auto_create_group {{definitions.cw_auto_create_group}} retry_limit 3 - {{#if definitions.config_revision}}storage.total_limit_size 1G{{/if}} endpoint ${CLOUDWATCH_ENDPOINT} +{{#if definitions.is_golden_revision_applied}} + storage.total_limit_size 1G + net.keepalive_idle_timeout 4 +{{/if}} {{#if definitions.cw_use_mock}} tls.verify false port 4545 diff --git a/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/task-definition.json b/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/task-definition.json index 3244674..97ea23c 100644 --- a/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/task-definition.json +++ b/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/task-definition.json @@ -136,13 +136,6 @@ "awslogs-stream-prefix": "{{managed.caseNameUnique}}-fluent-bit" } }, - "mountPoints": [ - { - "sourceVolume": "my-shared-volume", - "containerPath": "/tmp/data_logs", - "readOnly": false - } - ], "dependsOn": [{ "containerName": "mock", "condition": "START" @@ -176,13 +169,6 @@ "awslogs-stream-prefix": "{{managed.caseNameUnique}}-datajet" } }, - "mountPoints": [ - { - "sourceVolume": "my-shared-volume", - "containerPath": "/var/tmp/data_logs", - "readOnly": false - } - ], "dependsOn": [{ "containerName": "fluent-bit", "condition": "START" @@ -236,10 +222,5 @@ } ], "memory": "4096", - "cpu": "1024", - "volumes": [ - { - "name": "my-shared-volume" - } - ] + "cpu": "1024" } From 4a6c35b49561edd80a21dabe29b60858fcca7538 Mon Sep 17 00:00:00 2001 From: Matthew Fala Date: Sat, 4 Nov 2023 10:28:33 +0000 Subject: [PATCH 7/7] stability: add semi-colon Signed-off-by: Matthew Fala --- .../lib/helpers/dashboard-widget-list-processor.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/firelens-stability/lib/helpers/dashboard-widget-list-processor.ts b/apps/firelens-stability/lib/helpers/dashboard-widget-list-processor.ts index c83121e..d9ab4d4 100644 --- a/apps/firelens-stability/lib/helpers/dashboard-widget-list-processor.ts +++ b/apps/firelens-stability/lib/helpers/dashboard-widget-list-processor.ts @@ -57,7 +57,7 @@ export function generateOrderdedWidgetsFromTestCases(testCases: ITestCase[]) { path: path, widgets: mergedList, }; - }) + }); /* Sort paths */ const sortedLists = dashboardWidgetGroups.sort((a, b) => a.path.localeCompare(b.path));