diff --git a/apps/firelens-stability/collections/ecs-golden-path-single/golden-path-10-16-2023-cw-no-mock/case-config.json b/apps/firelens-stability/collections/ecs-golden-path-single/golden-path-10-16-2023-cw-no-mock/case-config.json new file mode 100644 index 0000000..79edc4a --- /dev/null +++ b/apps/firelens-stability/collections/ecs-golden-path-single/golden-path-10-16-2023-cw-no-mock/case-config.json @@ -0,0 +1,8 @@ +{ + "config": { + "template": "golden-path-mountebank-fargate-v10-16-2023" + }, + "definitions": { + "config_revision": true + } +} diff --git a/apps/firelens-stability/collections/ecs-golden-path-single/golden-path-10-16-2023-cw-no-mock/cases/low-throughput-cloudwatch.json b/apps/firelens-stability/collections/ecs-golden-path-single/golden-path-10-16-2023-cw-no-mock/cases/low-throughput-cloudwatch.json new file mode 100644 index 0000000..6f21116 --- /dev/null +++ b/apps/firelens-stability/collections/ecs-golden-path-single/golden-path-10-16-2023-cw-no-mock/cases/low-throughput-cloudwatch.json @@ -0,0 +1,11 @@ +{ + "definitions": { + "cw_endpoint": "logs.us-west-2.amazonaws.com", + "cw_use_mock": false, + "kbps_throughput_tcp_application": 1, + "kbps_throughput_tcp_request": 1, + "kbps_throughput_tcp_service": 1, + "kbps_throughput_forward": 1, + "cw_auto_create_group": true + } +} diff --git a/apps/firelens-stability/collections/ecs-golden-path-single/golden-path-10-16-2023-revision/case-config.json b/apps/firelens-stability/collections/ecs-golden-path-single/golden-path-10-16-2023-revision/case-config.json new file mode 100644 index 0000000..79edc4a --- /dev/null +++ b/apps/firelens-stability/collections/ecs-golden-path-single/golden-path-10-16-2023-revision/case-config.json @@ -0,0 +1,8 @@ +{ + "config": { + "template": "golden-path-mountebank-fargate-v10-16-2023" + }, + "definitions": { + "config_revision": true + } +} diff --git a/apps/firelens-stability/collections/ecs-golden-path-single/golden-path-10-16-2023-revision/cases b/apps/firelens-stability/collections/ecs-golden-path-single/golden-path-10-16-2023-revision/cases new file mode 120000 index 0000000..5d589b1 --- /dev/null +++ b/apps/firelens-stability/collections/ecs-golden-path-single/golden-path-10-16-2023-revision/cases @@ -0,0 +1 @@ +../golden-path-10-16-2023/cases \ No newline at end of file diff --git a/apps/firelens-stability/collections/ecs-golden-path-single/golden-path-10-16-2023/case-config.json b/apps/firelens-stability/collections/ecs-golden-path-single/golden-path-10-16-2023/case-config.json new file mode 100644 index 0000000..9681558 --- /dev/null +++ b/apps/firelens-stability/collections/ecs-golden-path-single/golden-path-10-16-2023/case-config.json @@ -0,0 +1,8 @@ +{ + "config": { + "template": "golden-path-mountebank-fargate-v10-16-2023" + }, + "definitions": { + "config_revision": false + } +} diff --git a/apps/firelens-stability/collections/ecs-golden-path-single/golden-path-10-16-2023/cases/no-onepod-cw-down.json b/apps/firelens-stability/collections/ecs-golden-path-single/golden-path-10-16-2023/cases/no-onepod-cw-down.json new file mode 100644 index 0000000..de9a612 --- /dev/null +++ b/apps/firelens-stability/collections/ecs-golden-path-single/golden-path-10-16-2023/cases/no-onepod-cw-down.json @@ -0,0 +1,6 @@ +{ + "definitions": { + "cw_is_down": true, + "include_onepod": false + } +} diff --git a/apps/firelens-stability/collections/ecs-golden-path-single/golden-path-10-16-2023/cases/no-onepod-cw-up.json b/apps/firelens-stability/collections/ecs-golden-path-single/golden-path-10-16-2023/cases/no-onepod-cw-up.json new file mode 100644 index 0000000..65d00fb --- /dev/null +++ b/apps/firelens-stability/collections/ecs-golden-path-single/golden-path-10-16-2023/cases/no-onepod-cw-up.json @@ -0,0 +1,6 @@ +{ + "definitions": { + "cw_is_down": false, + "include_onepod": false + } +} diff --git a/apps/firelens-stability/collections/ecs-golden-path-single/golden-path-10-16-2023/cases/onepod-cw-down-low-throughput.json b/apps/firelens-stability/collections/ecs-golden-path-single/golden-path-10-16-2023/cases/onepod-cw-down-low-throughput.json new file mode 100644 index 0000000..43c6618 --- /dev/null +++ b/apps/firelens-stability/collections/ecs-golden-path-single/golden-path-10-16-2023/cases/onepod-cw-down-low-throughput.json @@ -0,0 +1,10 @@ +{ + "definitions": { + "cw_is_down": true, + "include_onepod": true, + "kbps_throughput_tcp_application": 1, + "kbps_throughput_tcp_request": 1, + "kbps_throughput_tcp_service": 1, + "kbps_throughput_forward": 1 + } +} diff --git a/apps/firelens-stability/collections/ecs-golden-path-single/golden-path-10-16-2023/cases/onepod-cw-down.json b/apps/firelens-stability/collections/ecs-golden-path-single/golden-path-10-16-2023/cases/onepod-cw-down.json new file mode 100644 index 0000000..dc6d360 --- /dev/null +++ b/apps/firelens-stability/collections/ecs-golden-path-single/golden-path-10-16-2023/cases/onepod-cw-down.json @@ -0,0 +1,6 @@ +{ + "definitions": { + "cw_is_down": true, + "include_onepod": true + } +} diff --git a/apps/firelens-stability/collections/ecs-golden-path-single/golden-path-10-16-2023/cases/onepod-cw-up-low-throughput.json b/apps/firelens-stability/collections/ecs-golden-path-single/golden-path-10-16-2023/cases/onepod-cw-up-low-throughput.json new file mode 100644 index 0000000..53ed407 --- /dev/null +++ b/apps/firelens-stability/collections/ecs-golden-path-single/golden-path-10-16-2023/cases/onepod-cw-up-low-throughput.json @@ -0,0 +1,10 @@ +{ + "definitions": { + "cw_is_down": false, + "include_onepod": true, + "kbps_throughput_tcp_application": 1, + "kbps_throughput_tcp_request": 1, + "kbps_throughput_tcp_service": 1, + "kbps_throughput_forward": 1 + } +} diff --git a/apps/firelens-stability/collections/ecs-golden-path-single/golden-path-10-16-2023/cases/onepod-cw-up.json b/apps/firelens-stability/collections/ecs-golden-path-single/golden-path-10-16-2023/cases/onepod-cw-up.json new file mode 100644 index 0000000..b8dc3a7 --- /dev/null +++ b/apps/firelens-stability/collections/ecs-golden-path-single/golden-path-10-16-2023/cases/onepod-cw-up.json @@ -0,0 +1,6 @@ +{ + "definitions": { + "cw_is_down": false, + "include_onepod": true + } +} diff --git a/apps/firelens-stability/collections/ecs-golden-path-single/suite-config.json b/apps/firelens-stability/collections/ecs-golden-path-single/suite-config.json new file mode 100644 index 0000000..f3b49fe --- /dev/null +++ b/apps/firelens-stability/collections/ecs-golden-path-single/suite-config.json @@ -0,0 +1,17 @@ +{ + "definitions": { + "testId": "{{managed.suiteName}}-{{managed.caseName}}", + "ServiceName": "{{managed.executionName}}-{{managed.executionId}}", + "segfault": false + }, + "config": { + "cluster": "{{managed.executionName}}-{{managed.executionId}}-{{managed.collectionName}}", + "taskCount": 1, + "adot_enable": true, + "kbps_throughput_tcp_application": 5000, + "kbps_throughput_tcp_request": 5000, + "kbps_throughput_tcp_service": 5000, + "kbps_throughput_forward": 5000, + "datajet_generator_basic_contentType": "uniform" + } +} \ No newline at end of file diff --git a/apps/firelens-stability/execution.json b/apps/firelens-stability/execution.json index 56e6b42..2518125 100644 --- a/apps/firelens-stability/execution.json +++ b/apps/firelens-stability/execution.json @@ -1,9 +1,12 @@ { - "executionName": "2_31_12_20230911", + "executionName": "r2-golden-path-10-16-2023-revision-single-2_31_12_20230911", "executeCollections": [ - "ecs-firelens-stability-tests" + "ecs-golden-path-single" ], "definitions": { "imageAwsForFluentBit": "public.ecr.aws/aws-observability/aws-for-fluent-bit:init-debug-2.31.12.20230911" + }, + "config": { + "template": "golden-path-mountebank-fargate-v10-16-2023" } } diff --git a/apps/firelens-stability/lib/helpers/dashboard-widget-list-processor.ts b/apps/firelens-stability/lib/helpers/dashboard-widget-list-processor.ts index c83121e..d9ab4d4 100644 --- a/apps/firelens-stability/lib/helpers/dashboard-widget-list-processor.ts +++ b/apps/firelens-stability/lib/helpers/dashboard-widget-list-processor.ts @@ -57,7 +57,7 @@ export function generateOrderdedWidgetsFromTestCases(testCases: ITestCase[]) { path: path, widgets: mergedList, }; - }) + }); /* Sort paths */ const sortedLists = dashboardWidgetGroups.sort((a, b) => a.path.localeCompare(b.path)); diff --git a/apps/firelens-stability/lib/helpers/start-execution-stages.ts b/apps/firelens-stability/lib/helpers/start-execution-stages.ts index 228d6a7..90caf93 100644 --- a/apps/firelens-stability/lib/helpers/start-execution-stages.ts +++ b/apps/firelens-stability/lib/helpers/start-execution-stages.ts @@ -180,17 +180,20 @@ export async function hydrateTestCaseSeed(testCaseSeed: ITestCaseSeed): const layer4Config = cascadeConfigurationStringAsExtension(testCaseSeed.caseConfigSeed, layer3Config); const layer5Config = cascadeConfigurationStringAsExtension(testCaseSeed.caseSeed, layer4Config); + /* Apply execution config before and after template to ensure overrides are set - allow set template */ + const layer6Config = cascadeConfigurationStringAsExtension(testCaseSeed.executionConfigSeed, layer5Config); + /* Get template */ - const templateName = layer5Config.config.template; + const templateName = layer6Config.config.template; if (!templateName) { - validateTestConfig(layer5Config); + validateTestConfig(layer6Config); } const templateConfigPath = Path.join(Constants.paths.templates, templateName, Constants.fileNames.templateDefaultConfig); const templateConfig = await getStringFromFile(templateConfigPath); - const layer6Config = cascadeConfigurationStringAsDefault(templateConfig, layer5Config); + const layer7Config = cascadeConfigurationStringAsDefault(templateConfig, layer6Config); /* Final layer of configuration from execution seed. These function as overrides. */ - const config = cascadeConfigurationStringAsExtension(testCaseSeed.executionConfigSeed, layer6Config); + const config = cascadeConfigurationStringAsExtension(testCaseSeed.executionConfigSeed, layer7Config); validateTestConfig(config); diff --git a/apps/firelens-stability/lib/templating/handlebars-templater.ts b/apps/firelens-stability/lib/templating/handlebars-templater.ts index b51a58b..1e576e8 100644 --- a/apps/firelens-stability/lib/templating/handlebars-templater.ts +++ b/apps/firelens-stability/lib/templating/handlebars-templater.ts @@ -92,6 +92,10 @@ export async function copyAndTemplateFile( destinationFile: string, config: any) { const rawFileString = await getStringFromFile(sourceFile); - const templatedFileString = evaluateTemplateString(rawFileString, config); + const templatedFileString = evaluateTemplateString(rawFileString, { + // definitions get set to high level for file replacements + ...(config?.definitions ?? {}), + ...config + }); await sendStringToFile(templatedFileString, destinationFile); } diff --git a/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/adot-config.yaml b/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/adot-config.yaml new file mode 100644 index 0000000..a2f2356 --- /dev/null +++ b/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/adot-config.yaml @@ -0,0 +1,219 @@ +extensions: + health_check: + +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + awsxray: + endpoint: 0.0.0.0:2000 + transport: udp + statsd: + endpoint: 0.0.0.0:8125 + aggregation_interval: 60s + awsecscontainermetrics: + +processors: + batch/traces: + timeout: 1s + send_batch_size: 50 + batch/metrics: + timeout: 60s + filter: + metrics: + include: + match_type: strict + metric_names: + - container.memory.reserved + - container.memory.utilized + - container.cpu.reserved + - container.cpu.utilized + - container.network.rate.rx + - container.network.rate.tx + - container.storage.read_bytes + - container.storage.write_bytes + - container.duration + metricstransform: + transforms: + - include: container.memory.utilized + action: update + new_name: MemoryUtilized + - include: container.memory.reserved + action: update + new_name: MemoryReserved + - include: container.cpu.utilized + action: update + new_name: CpuUtilized + - include: container.cpu.reserved + action: update + new_name: CpuReserved + - include: container.network.rate.rx + action: update + new_name: NetworkRxBytes + - include: container.network.rate.tx + action: update + new_name: NetworkTxBytes + - include: container.storage.read_bytes + action: update + new_name: StorageReadBytes + - include: container.storage.write_bytes + action: update + new_name: StorageWriteBytes + + resource: + attributes: + - key: ClusterName + from_attribute: aws.ecs.cluster.name + action: insert + - key: aws.ecs.cluster.name + action: delete + - key: ServiceName + from_attribute: aws.ecs.service.name + action: insert + - key: aws.ecs.service.name + action: delete + - key: TaskId + from_attribute: aws.ecs.task.id + action: insert + - key: aws.ecs.task.id + action: delete + - key: TaskDefinitionFamily + from_attribute: aws.ecs.task.family + action: insert + - key: aws.ecs.task.family + action: delete + - key: TaskARN + from_attribute: aws.ecs.task.arn + action: insert + - key: aws.ecs.task.arn + action: delete + - key: DockerName + from_attribute: aws.ecs.docker.name + action: insert + - key: aws.ecs.docker.name + action: delete + - key: TaskDefinitionRevision + from_attribute: aws.ecs.task.version + action: insert + - key: aws.ecs.task.version + action: delete + - key: PullStartedAt + from_attribute: aws.ecs.task.pull_started_at + action: insert + - key: aws.ecs.task.pull_started_at + action: delete + - key: PullStoppedAt + from_attribute: aws.ecs.task.pull_stopped_at + action: insert + - key: aws.ecs.task.pull_stopped_at + action: delete + - key: AvailabilityZone + from_attribute: cloud.zone + action: insert + - key: cloud.zone + action: delete + - key: LaunchType + from_attribute: aws.ecs.task.launch_type + action: insert + - key: aws.ecs.task.launch_type + action: delete + - key: Region + from_attribute: cloud.region + action: insert + - key: cloud.region + action: delete + - key: AccountId + from_attribute: cloud.account.id + action: insert + - key: cloud.account.id + action: delete + - key: DockerId + from_attribute: container.id + action: insert + - key: container.id + action: delete + - key: ContainerName + from_attribute: container.name + action: insert + - key: container.name + action: delete + - key: Image + from_attribute: container.image.name + action: insert + - key: container.image.name + action: delete + - key: ImageId + from_attribute: aws.ecs.container.image.id + action: insert + - key: aws.ecs.container.image.id + action: delete + - key: ExitCode + from_attribute: aws.ecs.container.exit_code + action: insert + - key: aws.ecs.container.exit_code + action: delete + - key: CreatedAt + from_attribute: aws.ecs.container.created_at + action: insert + - key: aws.ecs.container.created_at + action: delete + - key: StartedAt + from_attribute: aws.ecs.container.started_at + action: insert + - key: aws.ecs.container.started_at + action: delete + - key: FinishedAt + from_attribute: aws.ecs.container.finished_at + action: insert + - key: aws.ecs.container.finished_at + action: delete + - key: ImageTag + from_attribute: container.image.tag + action: insert + - key: container.image.tag + action: delete + +exporters: + awsxray: + awsemf/application: + namespace: ECS/AWSOTel/Application + log_group_name: '/aws/ecs/application/metrics' + awsemf/performance: + namespace: {{managed.executionName}}/{{managed.collectionName}}/{{managed.executionId}} + log_group_name: '/aws/ecs/containerinsights/{ClusterName}/performance' + log_stream_name: '{TaskId}' + resource_to_telemetry_conversion: + enabled: true + dimension_rollup_option: NoDimensionRollup + metric_declarations: + - dimensions: [ [ ClusterName ], [ ClusterName, TaskDefinitionFamily, TaskId, ContainerName ] ] + metric_name_selectors: + - MemoryUtilized + - MemoryReserved + - CpuUtilized + - CpuReserved + - NetworkRxBytes + - NetworkTxBytes + - StorageReadBytes + - StorageWriteBytes + - metric_name_selectors: [container.*] + +service: + pipelines: + traces: + receivers: [otlp,awsxray] + processors: [batch/traces] + exporters: [awsxray] + metrics/application: + receivers: [otlp, statsd] + processors: [batch/metrics] + exporters: [awsemf/application] + metrics/performance: + receivers: [awsecscontainermetrics ] + processors: [filter, metricstransform, resource] + exporters: [ awsemf/performance ] + + extensions: [health_check] \ No newline at end of file diff --git a/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/default-config.json b/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/default-config.json new file mode 100644 index 0000000..93dc409 --- /dev/null +++ b/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/default-config.json @@ -0,0 +1,24 @@ +{ + "definitions": { + "taskName": "{{managed.caseNameUnique}}", + "adot_enable": false, + "cw_auto_create_group": "false", + "datajet_generator_basic_contentType": "random", + "datajet_generator_basic_contentRandomValueSet": "alpha-numeric", + "include_onepod": false, + "kbps_throughput_tcp_application": 1, + "kbps_throughput_tcp_request": 1, + "kbps_throughput_tcp_service": 1, + "kbps_throughput_forward": 1, + "cw_use_mock": true, + "add_forward": true, + "cw_endpoint": "localhost", + "endpoint_flb": "localhost", + "LOG_REGION": "us-west-2", + "ServiceName": "golden-onepod-low-throughput", + "debugS3Bucket": "{{managed.s3OutputBucket}}", + "debugS3KeyPrefix": "{{managed.s3OutputExecutionPath}}/{{managed.caseNameUnique}}", + "cw_is_down": false, + "is_golden_revision_applied": false + } +} \ No newline at end of file diff --git a/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/firelens-datajet.json b/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/firelens-datajet.json new file mode 100644 index 0000000..51b9124 --- /dev/null +++ b/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/firelens-datajet.json @@ -0,0 +1,113 @@ +{ + "component": "synchronizer", + "config": { + "repeat": 100, + "waitBefore": 5, + "waitAfter": 10, + "waitBetween": 0, + "isAsync": true + }, + "children": [ + { + "comment": "FireLens logs to standard out (move forward)", + "generator": { + "name": "basic", + "config": { + "contentLength": 1000, + "batchSize": 1, + "logKey": "log", + "addNewline": true, + "contentType": "{{definitions.datajet_generator_basic_contentType}}", + "contentRandomValueSet": "{{definitions.datajet_generator_basic_contentRandomValueSet}}", + "contentUniformValue": "F" + } + }, + "datajet": { + "name": "forward", + "config": { + "host": "{{definitions.endpoint_flb}}", + "port": 6270, + "tagPrefix": "Application-firelens-{{definitions.caseNameUnique}}" + } + }, + "stage": { + "batchRate": {{definitions.kbps_throughput_forward}} + } + }, + { + "comment": "ApplicationLogs through tcp", + "generator": { + "name": "basic", + "config": { + "contentLength": 1000, + "batchSize": 1, + "logKey": "log", + "contentType": "{{definitions.datajet_generator_basic_contentType}}", + "contentRandomValueSet": "{{definitions.datajet_generator_basic_contentRandomValueSet}}", + "contentUniformValue": "A" + } + }, + "datajet": { + "name": "tcp", + "config": { + "host": "{{definitions.endpoint_flb}}", + "port": 5170, + "addNewline": true + } + }, + "stage": { + "batchRate": {{definitions.kbps_throughput_tcp_application}} + } + }, + { + "comment": "RequestLogs to tcp", + "generator": { + "name": "basic", + "config": { + "contentLength": 1000, + "batchSize": 1, + "logKey": "log", + "contentType": "{{definitions.datajet_generator_basic_contentType}}", + "contentRandomValueSet": "{{definitions.datajet_generator_basic_contentRandomValueSet}}", + "contentUniformValue": "R" + } + }, + "datajet": { + "name": "tcp", + "config": { + "host": "{{definitions.endpoint_flb}}", + "port": 5171, + "addNewline": true + } + }, + "stage": { + "batchRate": {{definitions.kbps_throughput_tcp_request}} + } + }, + { + "comment": "ServiceLogs as emf to tcp (https://code.amazon.com/packages/CoralMetricsEmfReporter/blobs/78081aca20e12c524c58c248b0b8dc9318c3e66f/--/src/test/resources/serialized-format/MetricLogEvent.json#L1)", + "generator": { + "name": "explicit", + "config": { + "batches": [ + {"log":"{\"_aws\":{\"Timestamp\":1679351573496,\"LogGroupName\":\"ServiceMetrics\",\"CloudWatchMetrics\":[{\"Namespace\":\"ServiceMetrics\",\"Dimensions\":[[\"A\"]],\"Metrics\":[{\"Name\":\"ServiceMetric1\",\"Unit\":\"Microseconds\"}]}]},\"A\":\"Value\",\"ServiceMetric1\":10.0}"}, + {"log":"{\"_aws\":{\"Timestamp\":1679351573596,\"LogGroupName\":\"ServiceMetrics\",\"CloudWatchMetrics\":[{\"Namespace\":\"ServiceMetrics\",\"Dimensions\":[[\"A\"]],\"Metrics\":[{\"Name\":\"ServiceMetric1\",\"Unit\":\"Microseconds\"}]}]},\"A\":\"Value\",\"ServiceMetric1\":10.0}"}, + {"log":"{\"_aws\":{\"Timestamp\":1679351573696,\"LogGroupName\":\"ServiceMetrics\",\"CloudWatchMetrics\":[{\"Namespace\":\"ServiceMetrics\",\"Dimensions\":[[\"A\"]],\"Metrics\":[{\"Name\":\"ServiceMetric1\",\"Unit\":\"Microseconds\"}]}]},\"A\":\"Value\",\"ServiceMetric1\":10.0}"}, + {"log":"{\"_aws\":{\"Timestamp\":1679351573796,\"LogGroupName\":\"ServiceMetrics\",\"CloudWatchMetrics\":[{\"Namespace\":\"ServiceMetrics\",\"Dimensions\":[[\"A\"]],\"Metrics\":[{\"Name\":\"ServiceMetric1\",\"Unit\":\"Microseconds\"}]}]},\"A\":\"Value\",\"ServiceMetric1\":10.0}"} + ] + } + }, + "datajet": { + "name": "tcp", + "config": { + "host": "{{definitions.endpoint_flb}}", + "port": 5172, + "addNewline": true + } + }, + "stage": { + "batchRate": {{definitions.kbps_throughput_tcp_service}} + } + } + ] +} diff --git a/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/fluent-bit-base.conf b/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/fluent-bit-base.conf new file mode 100644 index 0000000..d125c1f --- /dev/null +++ b/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/fluent-bit-base.conf @@ -0,0 +1,151 @@ +[SERVICE] + # See: + # https://docs.aws.amazon.com/AmazonECS/latest/developerguide/using_firelens.html + # https://github.com/aws-samples/amazon-ecs-firelens-under-the-hood/tree/master/generated-configs/fluent-bit + HTTP_Server On + HTTP_Listen 0.0.0.0 + HTTP_PORT 2020 + Flush 1 + Grace 30 + Log_Level info + storage.path /var/log/flb-storage/ + # Important! This is your memory limiter setting for inputs with storage.type filesystem + # Total Max Memory Usage <= (2 * # of inputs with filesystem buffer * storage.max_chunks_up * 2 MB per chunk) + # + (2 * SUM(Each input Mem_Buf_Limit for inputs with memory buffer) + # This calculation should inform the Fluent Bit container memory limit or reservation. + # If you switch to default storage.type memory, please carefully read the guide + # https://github.com/aws-samples/amazon-ecs-firelens-examples/tree/mainline/examples/fluent-bit/oomkill-prevention + storage.max_chunks_up 32 + +{{#if definitions.segfault}} +# Artificially trigger a segfault if configured +[INPUT] + Name exec + command kill -11 `pgrep fluent-bit` +{{/if}} + +{{#if definitions.add_forward}} + +# This is an artificial forward because my standard out logger was going to some other destination not the cw specified here. +[INPUT] + Name forward + Listen 0.0.0.0 + Port 6270 + +{{/if}} + +[INPUT] + Name tcp + Alias TCP-ApplicationLogs + Listen 0.0.0.0 + + # Tag used for routing to OUTPUT + Tag ApplicationLogs + + # Listen on unused port + Port 5170 + + # Don't parse JSON for performance reasons + Format none + + Chunk_Size 32 + Buffer_Size 256 + # If this is set to (default) "storage.type memory" or not configured (memory is default), + # then Mem_Buf_Limit *should be set*. + # If this is set to "storage.type filesystem", Mem_Buf_Limit has no effect. It + # uses the [SERVICE] level "storage.max_chunks_up" configuration instead. + # https://github.com/aws-samples/amazon-ecs-firelens-examples/tree/mainline/examples/fluent-bit/oomkill-prevention + storage.type filesystem + +[INPUT] + Name tcp + Alias TCP-RequestLogs + Tag RequestLogs + Listen 0.0.0.0 + Port 5171 + Format none + Chunk_Size 32 + Buffer_Size 256 + # If this is set to (default) "storage.type memory" or not configured (memory is default), + # then Mem_Buf_Limit *should be set*. + # If this is set to "storage.type filesystem", Mem_Buf_Limit has no effect. It + # uses the [SERVICE] level "storage.max_chunks_up" configuration instead. + storage.type filesystem + +[INPUT] + Name tcp + Alias TCP-ServiceMetrics + Tag ServiceMetrics + Listen 0.0.0.0 + Port 5172 + Format none + Chunk_Size 32 + Buffer_Size 256 + # If this is set to (default) "storage.type memory" or not configured (memory is default), + # then Mem_Buf_Limit *should be set*. + # If this is set to "storage.type filesystem", Mem_Buf_Limit has no effect. It + # uses the [SERVICE] level "storage.max_chunks_up" configuration instead. + storage.type filesystem + +[OUTPUT] + # Send any StdOut/StdErr Logs to A CloudWatch Logs Group. + # FireLens automatically tags them to {container name}-firelens-{task ID}. + Name cloudwatch_logs + Alias CloudWatch-AppContainer-STDOUT + Match Application-firelens-* + region ${LOG_REGION} + log_group_name {{ServiceName}}-${STAGE}-AppContainer-STDOUT + log_stream_prefix STDOUT-${HOSTNAME} + auto_create_group {{definitions.cw_auto_create_group}} + retry_limit 3 + endpoint ${CLOUDWATCH_ENDPOINT} +{{#if definitions.is_golden_revision_applied}} + net.keepalive_idle_timeout 4 +{{/if}} +{{#if definitions.cw_use_mock}} + tls.verify false + port 4545 +{{/if}} + +[OUTPUT] + # Send any Application Logs to A CloudWatch Logs Group + Name cloudwatch_logs + Alias CloudWatch-ApplicationLogs + Match ApplicationLogs + region ${LOG_REGION} + log_group_name {{ServiceName}}-${STAGE}-ApplicationLogs + log_stream_prefix ApplicationLogs-${HOSTNAME} + log_key log + auto_create_group {{definitions.cw_auto_create_group}} + retry_limit 3 + endpoint ${CLOUDWATCH_ENDPOINT} +{{#if definitions.is_golden_revision_applied}} + storage.total_limit_size 2G + net.keepalive_idle_timeout 4 +{{/if}} +{{#if definitions.cw_use_mock}} + tls.verify false + port 4545 +{{/if}} + + +[OUTPUT] + # Send any Request Logs to A CloudWatch Logs Group + Name cloudwatch_logs + Alias CloudWatch-RequestLogs + Match RequestLogs + region ${LOG_REGION} + log_group_name {{ServiceName}}-${STAGE}-RequestLogs + log_stream_prefix RequestLogs-${HOSTNAME} + log_key log + auto_create_group {{definitions.cw_auto_create_group}} + retry_limit 3 + endpoint ${CLOUDWATCH_ENDPOINT} +{{#if definitions.is_golden_revision_applied}} + storage.total_limit_size 2G + net.keepalive_idle_timeout 4 +{{/if}} +{{#if definitions.cw_use_mock}} + tls.verify false + port 4545 +{{/if}} diff --git a/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/fluent-bit-onepod.conf b/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/fluent-bit-onepod.conf new file mode 100644 index 0000000..96b96f6 --- /dev/null +++ b/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/fluent-bit-onepod.conf @@ -0,0 +1,91 @@ +{{#if definitions.include_onepod}} + +# Note: The mainline onepod config includes the following line +# We do not need this line because we are using the init process +# @INCLUDE /config/fluent-bit.conf + +[OUTPUT] + # Send any StdOut/StdErr Logs to A CloudWatch Logs Group. + # FireLens automatically tags them to {container name}-firelens-{task ID}. + Name cloudwatch_logs + Alias CloudWatch-AppContainer-STDOUT + Match Application-firelens-* + region ${LOG_REGION} + log_group_name {{ServiceName}}-OnePod-${STAGE}-AppContainer-STDOUT + log_stream_prefix STDOUT-${HOSTNAME} + auto_create_group {{definitions.cw_auto_create_group}} + retry_limit 3 + endpoint ${CLOUDWATCH_ENDPOINT} +{{#if definitions.is_golden_revision_applied}} + net.keepalive_idle_timeout 4 +{{/if}} +{{#if definitions.cw_use_mock}} + tls.verify false + port 4545 +{{/if}} + +[OUTPUT] + # Send any Application Logs to A CloudWatch Logs Group + Name cloudwatch_logs + Alias CloudWatch-ApplicationLogs + Match ApplicationLogs + region ${LOG_REGION} + log_group_name {{ServiceName}}-OnePod-${STAGE}-ApplicationLogs + log_stream_prefix ApplicationLogs-${HOSTNAME} + log_key log + auto_create_group {{definitions.cw_auto_create_group}} + retry_limit 3 + endpoint ${CLOUDWATCH_ENDPOINT} +{{#if definitions.is_golden_revision_applied}} + storage.total_limit_size 2G + net.keepalive_idle_timeout 4 +{{/if}} +{{#if definitions.cw_use_mock}} + tls.verify false + port 4545 +{{/if}} + +[OUTPUT] + # Send any Request Logs to A CloudWatch Logs Group + Name cloudwatch_logs + Alias CloudWatch-RequestLogs + Match RequestLogs + region ${LOG_REGION} + log_group_name {{ServiceName}}-OnePod-${STAGE}-RequestLogs + log_stream_prefix RequestLogs-${HOSTNAME} + log_key log + auto_create_group {{definitions.cw_auto_create_group}} + retry_limit 3 + endpoint ${CLOUDWATCH_ENDPOINT} +{{#if definitions.is_golden_revision_applied}} + storage.total_limit_size 2G + net.keepalive_idle_timeout 4 +{{/if}} +{{#if definitions.cw_use_mock}} + tls.verify false + port 4545 +{{/if}} + +[OUTPUT] + # Send any ServiceMetrics Logs to A CloudWatch Logs Group + Name cloudwatch_logs + Alias CloudWatch-ServiceMetrics + Match ServiceMetrics + region ${LOG_REGION} + log_group_name {{ServiceName}}-OnePod-${STAGE}-ServiceMetrics + log_stream_prefix ServiceMetrics-${HOSTNAME} + log_key log + log_format json/emf + auto_create_group {{definitions.cw_auto_create_group}} + retry_limit 3 + endpoint ${CLOUDWATCH_ENDPOINT} +{{#if definitions.is_golden_revision_applied}} + storage.total_limit_size 1G + net.keepalive_idle_timeout 4 +{{/if}} +{{#if definitions.cw_use_mock}} + tls.verify false + port 4545 +{{/if}} + +{{/if}} \ No newline at end of file diff --git a/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/fluent-bit.conf b/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/fluent-bit.conf new file mode 100644 index 0000000..b73ad04 --- /dev/null +++ b/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/fluent-bit.conf @@ -0,0 +1,25 @@ +# Note: The mainline onepod config includes the following line +# We do not need this line because we are using the init process +# @INCLUDE /config/fluent-bit-base.conf + +[OUTPUT] + # Send any ServiceMetrics Logs to A CloudWatch Logs Group + Name cloudwatch_logs + Alias CloudWatch-ServiceMetrics + Match ServiceMetrics + region ${LOG_REGION} + log_group_name {{definitions.ServiceName}}-${STAGE}-ServiceMetrics + log_stream_prefix ServiceMetrics-${HOSTNAME} + log_key log + log_format json/emf + auto_create_group {{definitions.cw_auto_create_group}} + retry_limit 3 + endpoint ${CLOUDWATCH_ENDPOINT} +{{#if definitions.is_golden_revision_applied}} + storage.total_limit_size 1G + net.keepalive_idle_timeout 4 +{{/if}} +{{#if definitions.cw_use_mock}} + tls.verify false + port 4545 +{{/if}} diff --git a/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/fluent-bit.env b/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/fluent-bit.env new file mode 100644 index 0000000..e820be5 --- /dev/null +++ b/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/fluent-bit.env @@ -0,0 +1,6 @@ +# Fargate was giving errors when loading environment variables. +# ResourceInitializationError: failed to download env files: file download command: non empty error stream: service call has been retried 5 time(s): RequestCanceled: request context canceled caused by: context deadline exceeded. +# Env file references, and env var usage removed. +LOG_REGION={{config.region}} +STAGE=alpha +CLOUDWATCH_ENDPOINT={{definitions.cw_endpoint}} \ No newline at end of file diff --git a/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/fluent-parser.conf b/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/fluent-parser.conf new file mode 100644 index 0000000..6029b71 --- /dev/null +++ b/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/fluent-parser.conf @@ -0,0 +1,14 @@ +# Parsers *must* be in a file separate from the main configuration +# See https://fluentbit.io/documentation/0.13/parser/ + +[PARSER] + # A Parser to capture the beginning of a QueryLog multi-line entry. + Name QueryLogSeparator + Format regex + Regex (?-{20,}) + +[PARSER] + # A Parser to capture QueryLog statements in their entirety. + Name QueryLog + Format regex + Regex (?-{20,})(?[\S\s]+?EOE) diff --git a/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/mountebank.json b/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/mountebank.json new file mode 100644 index 0000000..c50d3ab --- /dev/null +++ b/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/mountebank.json @@ -0,0 +1,121 @@ +{ + "port": 4545, + "protocol": "https", + "stubs": [ + { + "predicates": [{ + "and": [ + { + "equals": { + "path": "/", + "method": "POST", + "headers": { + "Content-Type": "application/x-amz-json-1.1", + "X-Amz-Target": "Logs_20140328.PutLogEvents" + } + } + } + ] + }], + "responses": [ + { + "is": { + "statusCode": {{#if definitions.cw_is_down}}400{{else}}200{{/if}}, + "headers": { + "x-amzn-RequestId": "d35a6920-52e1-4f3a-bf8f-46016f474463", + "Connection": "keep-alive", + "Keep-Alive": "timeout=5", + "Content-Type": "application/x-amz-json-1.1", + "Content-Length": "80" + }, + "body": "{\"nextSequenceToken\":\"49637338213634440402126044667090232594261646769605574882\"}", + "repeat": 5 + } + } + ] + }, + { + "responses": [ + { + "is": { + "statusCode": 200, + "headers": { + "x-amzn-RequestId": "d35a6920-52e1-4f3a-bf8f-46016f474463", + "Connection": "keep-alive", + "Content-Type": "application/x-amz-json-1.1", + "Content-Length": 0, + "Date": "Mon, 13 Mar 2023 23:09:24 GMT", + "Keep-Alive": "timeout=5" + } + } + } + ], + "predicates": [{ + "and": [ + { + "equals": { + "path": "/", + "method": "POST", + "headers": { + "Content-Type": "application/x-amz-json-1.1", + "X-Amz-Target": "Logs_20140328.CreateLogGroup" + } + } + }, + { + "not": { + "contains": { "body": "requiredField" }, + "caseSensitive": true + } + } + ] + }] + }, + { + "responses": [ + { + "is": { + "statusCode": 200, + "headers": { + "x-amzn-RequestId": "d35a6920-52e1-4f3a-bf8f-46016f474463", + "Connection": "keep-alive", + "Content-Type": "application/x-amz-json-1.1", + "Content-Length": 0, + "Date": "Mon, 13 Mar 2023 23:09:24 GMT", + "Keep-Alive": "timeout=5" + }, + "body": { + "logGroups": [ + { + "arn": "arn:aws:logs:us-east-1:123456789012:log-group:monitoring-logGroup-1234:*", + "creationTime": 1393545600000, + "logGroupName": "monitoring-logGroup-1234" + } + ] + } + } + } + ], + "predicates": [{ + "and": [ + { + "equals": { + "path": "/", + "method": "POST", + "headers": { + "Content-Type": "application/x-amz-json-1.1", + "X-Amz-Target": "Logs_20140328.DescribeLogGroups" + } + } + }, + { + "not": { + "contains": { "body": "requiredField" }, + "caseSensitive": true + } + } + ] + }] + } + ] + } \ No newline at end of file diff --git a/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/task-definition.json b/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/task-definition.json new file mode 100644 index 0000000..97ea23c --- /dev/null +++ b/apps/firelens-stability/templates/golden-path-mountebank-fargate-v10-16-2023/task-definition.json @@ -0,0 +1,226 @@ +{ + "family": "{{managed.caseNameUnique}}", + "executionRoleArn": "{{definitions.executionRoleArn}}", + "taskRoleArn": "{{definitions.taskRoleArn}}", + "networkMode": "awsvpc", + "requiresCompatibilities": [ + "FARGATE" + ], + "containerDefinitions": [ + {{#if definitions.adot_enable}} + { + "logConfiguration": { + "logDriver": "awslogs", + "options": { + "awslogs-group": "/ecs/aws-otel-EC2", + "awslogs-region": "{{config.region}}", + "awslogs-stream-prefix": "ecs", + "awslogs-create-group": "True" + } + }, + "healthCheck": { + "command": [ "/healthcheck" ], + "interval": 5, + "timeout": 6, + "retries": 5, + "startPeriod": 1 + }, + "portMappings": [ + { + "hostPort": 2000, + "protocol": "udp", + "containerPort": 2000 + }, + { + "hostPort": 4317, + "protocol": "tcp", + "containerPort": 4317 + }, + { + "hostPort": 8125, + "protocol": "udp", + "containerPort": 8125 + } + ], + "command": [ + "--config=s3://{{managed.s3ResourcesBucket}}.s3.us-west-2.amazonaws.com/{{managed.s3ResourcesPath}}/adot-config.yaml" + ], + "image": "amazon/aws-otel-collector", + "name": "aws-otel-collector" + }, + { + "logConfiguration": { + "logDriver": "awslogs", + "options": { + "awslogs-group": "/ecs/aws-otel-EC2", + "awslogs-region": "{{config.region}}", + "awslogs-stream-prefix": "ecs", + "awslogs-create-group": "True" + } + }, + "portMappings": [ + { + "hostPort": 8000, + "protocol": "tcp", + "containerPort": 8000 + } + ], + "environment": [ + { + "name": "AWS_XRAY_DAEMON_ADDRESS", + "value": "localhost:2000" + } + ], + "image": "public.ecr.aws/aws-otel-test/aws-otel-goxray-sample-app:latest", + "essential": false, + "name": "aws-otel-emitter", + "dependsOn": [ + { + "containerName": "aws-otel-collector", + "condition": "START" + } + ] + }, + {{/if}} + { + "name": "fluent-bit", + "image": "{{definitions.imageAwsForFluentBit}}", + "cpu": 1024, + "firelensConfiguration": { + "type": "fluentbit" + }, + "environmentFiles": [ + { + "value": "{{managed.s3ResourcesArn}}/fluent-bit.env", + "type": "s3" + } + ], + "environment": [ + { + "name": "AWS_REGION", + "value": "{{config.region}}" + }, + { + "name": "aws_fluent_bit_init_s3_1", + "value": "{{managed.s3ResourcesArn}}/fluent-bit.conf" + }, + { + "name": "aws_fluent_bit_init_s3_2", + "value": "{{managed.s3ResourcesArn}}/fluent-bit-base.conf" + }, + { + "name": "aws_fluent_bit_init_s3_3", + "value": "{{managed.s3ResourcesArn}}/fluent-bit-onepod.conf" + }, + { + "name": "aws_fluent_bit_init_s3_4", + "value": "{{managed.s3ResourcesArn}}/fluent-parser.conf" + }, + { + "name": "S3_BUCKET", + "value": "{{definitions.debugS3Bucket}}" + }, + { + "name": "S3_KEY_PREFIX", + "value": "{{definitions.debugS3KeyPrefix}}" + } + ], + "memory": 4096, + "essential": true, + "logConfiguration": { + "logDriver": "awslogs", + "options": { + "awslogs-group": "app-{{managed.executionName}}-{{managed.executionId}}", + "awslogs-region": "{{config.region}}", + "awslogs-create-group": "true", + "awslogs-stream-prefix": "{{managed.caseNameUnique}}-fluent-bit" + } + }, + "dependsOn": [{ + "containerName": "mock", + "condition": "START" + }] + }, + { + "name": "datajet", + "user": "0", + "image": "{{definitions.imageDatajet}}", + "essential": true, + "environment": [ + { + "name": "CLIENTS", + "value": "[\"s3\"]" + }, + { + "name": "CLIENT_S3_BUCKET", + "value": "{{managed.s3ResourcesBucket}}" + }, + { + "name": "CLIENT_S3_FILE", + "value": "{{managed.s3ResourcesPath}}/firelens-datajet.json" + } + ], + "logConfiguration": { + "logDriver": "awslogs", + "options": { + "awslogs-group": "app-{{managed.executionName}}-{{managed.executionId}}", + "awslogs-region": "{{config.region}}", + "awslogs-create-group": "true", + "awslogs-stream-prefix": "{{managed.caseNameUnique}}-datajet" + } + }, + "dependsOn": [{ + "containerName": "fluent-bit", + "condition": "START" + }] + }, + { + "essential": false, + "image": "{{definitions.imageHttpd}}", + "name": "dummy", + "logConfiguration": { + "logDriver":"awsfirelens", + "options": { + "Name": "cloudwatch_logs", + "region": "{{config.region}}", + "log_group_name": "app-{{managed.executionName}}-{{managed.executionId}}", + "auto_create_group": "true", + "log_stream_name": "{{managed.caseNameUnique}}-dummy-app", + "retry_limit": "2" + } + }, + "dependsOn": [{ + "containerName": "fluent-bit", + "condition": "START" + }], + "memoryReservation": 100 + }, + { + "essential": false, + "image": "{{definitions.imageMountebank}}", + "name": "mock", + "environment": [ + { + "name": "CLIENT_S3_BUCKET", + "value": "{{managed.s3ResourcesBucket}}" + }, + { + "name": "CLIENT_S3_FILE", + "value": "{{managed.s3ResourcesPath}}/mountebank.json" + } + ], + "logConfiguration": { + "logDriver": "awslogs", + "options": { + "awslogs-group": "app-{{managed.executionName}}-{{managed.executionId}}", + "awslogs-region": "{{config.region}}", + "awslogs-create-group": "true", + "awslogs-stream-prefix": "{{managed.caseNameUnique}}-mock" + } + }, + "memoryReservation": 100 + } + ], + "memory": "4096", + "cpu": "1024" +} diff --git a/examples/generators/example-explicit-generator-to-tcp.json b/examples/generators/example-explicit-generator-to-tcp.json new file mode 100644 index 0000000..bc68a9a --- /dev/null +++ b/examples/generators/example-explicit-generator-to-tcp.json @@ -0,0 +1,23 @@ +{ + "comment": "ServiceLogs as emf to tcp (https://code.amazon.com/packages/CoralMetricsEmfReporter/blobs/78081aca20e12c524c58c248b0b8dc9318c3e66f/--/src/test/resources/serialized-format/MetricLogEvent.json#L1)", + "generator": { + "name": "explicit", + "config": { + "batches": [ + {"_aws":{"Timestamp":1679351573496,"LogGroupName":"ServiceMetrics","CloudWatchMetrics":[{"Namespace":"ServiceMetrics","Dimensions":[["A"]],"Metrics":[{"Name":"ServiceMetric1","Unit":"Microseconds"}]}]},"A":"Value","ServiceMetric1":10.0} + ] + } + }, + "datajet": { + "name": "tcp", + "config": { + "host": "localhost", + "port": 5172, + "addNewline": true, + "logKey": null + } + }, + "stage": { + "batchRate": 1 + } +} diff --git a/firelens-datajet.json b/firelens-datajet.json index 84c3f53..248cce3 100644 --- a/firelens-datajet.json +++ b/firelens-datajet.json @@ -13,7 +13,7 @@ } }, "stage": { - "batchRate": 1000, - "maxBatches": 10 + "batchRate": 100, + "maxBatches": 1000000 } } \ No newline at end of file