diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 2361cec1eda..2ed7fff177a 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -312,6 +312,7 @@ /packages/lumos @elastic/security-service-integrations /packages/lyve_cloud @elastic/security-service-integrations /packages/m365_defender @elastic/security-service-integrations +/packages/macos @elastic/sec-windows-platform /packages/mattermost @elastic/security-service-integrations /packages/memcached @elastic/obs-infraobs-integrations /packages/menlo @elastic/security-service-integrations diff --git a/packages/macos/_dev/build/build.yml b/packages/macos/_dev/build/build.yml new file mode 100644 index 00000000000..b2596b96490 --- /dev/null +++ b/packages/macos/_dev/build/build.yml @@ -0,0 +1,3 @@ +dependencies: + ecs: + reference: git@v9.2.0 diff --git a/packages/macos/_dev/build/docs/README.md b/packages/macos/_dev/build/docs/README.md new file mode 100644 index 00000000000..c2d4180d501 --- /dev/null +++ b/packages/macos/_dev/build/docs/README.md @@ -0,0 +1,163 @@ +# macOS Integration for Elastic + +## Overview + +The macOS integration for Elastic allows you to collect and analyze unified logs from macOS systems. This integration leverages macOS's unified logging system to provide comprehensive visibility into system activities, security events, and application behaviors on macOS endpoints. + +macOS unified logging is Apple's centralized logging system that captures log messages from the kernel, system processes, and applications. This integration enables security teams to monitor macOS endpoints for suspicious activities, troubleshoot system issues, and maintain compliance with security policies. + +### Compatibility + +The macOS integration is compatible with macOS systems that support unified logging (macOS 10.12 Sierra and later). + +### How it works + +This integration uses the `unifiedlogs` input to collect log data from the macOS unified logging system. It can collect logs in real-time or from archived log files, with configurable filtering based on predicates, processes, and log levels. + +## What data does this integration collect? + +This integration collects unified log messages from macOS systems using configurable predicates to filter specific event types, including: + +- **Authentication logs**: User login/logout events, authentication failures, and credential-related activities + ``` + - 'process contains "sudo" OR composedMessage CONTAINS "sudo" OR process contains "su"' + - 'process contains "loginwindow" and composedMessage CONTAINS "sessionDidLogin"' + - 'process == "sshd"' + ``` +- **User & Account management**: User account creation, modification, and deletion events + ``` + - 'process == "sysadminctl" AND composedMessage CONTAINS "Creating user"' + - 'process == "dscl" AND composedMessage CONTAINS "create"' + - 'process == "sysadminctl" AND composedMessage CONTAINS "Deleting user"' + - 'process == "dscl" AND composedMessage CONTAINS "delete"' + - '(process == "dscl" OR process == "opendirectoryd") AND composedMessage CONTAINS "admin"' + ``` +- **Process execution monitoring**: Process creation, termination, and execution details + ``` + - 'eventMessage CONTAINS[c] "exec" OR eventMessage CONTAINS[c] "fork" OR eventMessage CONTAINS[c] "exited" OR eventMessage CONTAINS[c] "terminated"' + - 'subsystem == "com.apple.securityd" AND (composedMessage CONTAINS "code signing" OR composedMessage CONTAINS "not valid")' + - 'composedMessage CONTAINS "com.apple.quarantine"' + ``` +- **Network activity**: Network connections, DNS queries, and network-related events + ``` + - 'composedMessage CONTAINS "connect" AND (composedMessage CONTAINS "TCP" OR composedMessage CONTAINS "UDP")' + - 'composedMessage CONTAINS "disconnect" OR composedMessage CONTAINS "closed connection"' + - 'subsystem == "com.apple.necp" AND composedMessage CONTAINS "new connection"' + - 'eventMessage CONTAINS[c] "listening" AND eventMessage CONTAINS[c] "service"' + ``` +- **File reads/writes**: File system access, modifications, and permission changes + ``` + - '(eventMessage CONTAINS "open" OR eventMessage CONTAINS "write" OR eventMessage CONTAINS "unlink" OR eventMessage CONTAINS "rename") AND ((processImagePath BEGINSWITH "/System") OR (processImagePath BEGINSWITH "/bin") OR (processImagePath BEGINSWITH "/sbin") OR (processImagePath BEGINSWITH "/usr" AND NOT processImagePath BEGINSWITH "/usr/local") OR (processImagePath BEGINSWITH "/etc"))' + - 'subsystem == "com.apple.quarantine" OR eventMessage CONTAINS "com.apple.quarantine"' + ``` +- **System changes**: System configuration changes, software installations, and updates + ``` + - 'subsystem == "com.apple.security" OR subsystem == "com.apple.systempolicy" OR subsystem == "com.apple.installer" OR process == "Installer" OR process == "softwareupdated" OR eventMessage CONTAINS[c] "removed package" OR eventMessage CONTAINS[c] "forget package"' + ``` +- **Advanced monitoring**: Detailed system and application behavior logs + ``` + - '(composedMessage CONTAINS ".plist" AND (composedMessage CONTAINS "write" OR composedMessage CONTAINS "modified")) OR (composedMessage CONTAINS ".ssh" AND (composedMessage CONTAINS "write" OR composedMessage CONTAINS "modified")) OR (process == "kernel" AND composedMessage CONTAINS "boot") OR (process == "launchd" AND (composedMessage CONTAINS "started" OR composedMessage CONTAINS "listening")) OR (process == "loginwindow" AND composedMessage CONTAINS "sessionDidLogin") OR (composedMessage CONTAINS "posix_spawn" OR composedMessage CONTAINS "exec") OR (subsystem == "com.apple.securityd" AND (composedMessage CONTAINS "code signing" OR composedMessage CONTAINS "not valid"))' + ``` + +### Supported use cases +The macOS integration in Elastic enables comprehensive monitoring and analysis of system activities, network traffic, and application behavior across macOS devices. It supports use cases such as detecting security incidents, tracking network usage, auditing system events, and analyzing performance trends. By collecting and visualizing unified logs, it helps security and IT teams gain real-time visibility, identify anomalies, ensure compliance, and enhance overall endpoint security within macOS environments. + +## What do I need to use this integration? + +### From Elastic + +- Elastic Agent must be installed on the macOS system you want to monitor +- Appropriate permissions to read system logs on the macOS system + +### From macOS + +The integration requires: + +- macOS 10.12 Sierra or later (for unified logging support) +- Appropriate system permissions to access unified logs +- For some log categories, administrative privileges may be required + +## How do I deploy this integration? + +This integration requires Elastic Agent to be installed on the macOS systems you want to monitor. + +### Agent-based installation + +Elastic Agent must be installed. For more details, check the Elastic Agent [installation instructions](docs-content://reference/fleet/install-elastic-agents.md). You can install only one Elastic Agent per host. + +## Setup + +1. In the top search bar in Kibana, search for **Integrations**. +2. In the search bar, type **macOS**. +3. Select the **macOS** integration from the search results. +4. Select **Add macOS** to add the integration. +5. Enable and configure only the collection methods which you will use. + + **Basic Configuration:** + - Enable the log categories you want to collect: + - Authentication + - User & Account management + - Process execution monitoring + - Network activity + - File reads/writes + - System changes + - Advanced monitoring + + **Advanced Configuration (Optional):** + - **Predicate**: Use NSPredicate-based filtering to collect specific log messages + - **Process**: Specify particular processes to monitor (by PID or name) + - **Start/End dates**: Define time ranges for historical log collection + - **Log levels**: Configure which log levels to include (info, debug, backtrace, signpost) + - **Archive/Trace files**: Specify log archive or trace files to process + +6. Select **Save and continue** to save the integration. + +### Validation + +#### Dashboards populated + +1. In the top search bar in Kibana, search for **Dashboards**. +2. In the search bar, type **macOS**. +3. Select a dashboard for the dataset you are collecting, and verify the dashboard information is populated. + +## Performance and scaling + +- Unified log collection can generate significant data volume, especially with debug-level logging enabled +- Consider using predicates to filter logs and reduce data volume +- Monitor system performance impact when collecting high-volume log categories + +For more information on architectures that can be used for scaling this integration, check the [Ingest Architectures](https://www.elastic.co/docs/manage-data/ingest/ingest-reference-architectures) documentation. + +## Reference + +### ECS field reference + +### Authentication + +{{fields "authentication"}} + +### File Read/Write + +{{fields "file_read_write"}} + +### Network Activity + +{{fields "network_activity"}} + +### Process Execution Monitoring + +{{fields "process_execution_monitoring"}} + +### System Change + +{{fields "system_change"}} + +### User and Account Management + +{{fields "user_and_account_management"}} + +### Inputs used + +These inputs can be used in this integration: + +- [Unified Logs](https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-unifiedlogs) diff --git a/packages/macos/_dev/shared/common-pipeline.yml b/packages/macos/_dev/shared/common-pipeline.yml new file mode 100644 index 00000000000..6c620a180c2 --- /dev/null +++ b/packages/macos/_dev/shared/common-pipeline.yml @@ -0,0 +1,316 @@ +--- +description: Pipeline for processing common fields. +processors: + - set: + field: ecs.version + tag: set_ecs_version + value: 9.2.0 + - rename: + field: message + tag: rename_message_to_event_original + target_field: event.original + ignore_missing: true + description: Renames the original `message` field to `event.original` to store a copy of the original message. The `event.original` field is not touched if the document already has one; it may happen when Logstash sends the document. + if: ctx.event?.original == null + - remove: + field: message + tag: remove_message + ignore_missing: true + description: The `message` field is no longer required if the document has an `event.original` field. + if: ctx.event?.original != null + - json: + field: event.original + tag: json_event_original + target_field: json + - set: + field: event.kind + tag: set_event_kind + value: event + - convert: + field: json.activityIdentifier + tag: convert_activityIdentifier_to_string + target_field: macos.activity_identifier + type: string + ignore_missing: true + - foreach: + field: json.backtrace.frames + tag: foreach_backtrace_frames_imageOffset + if: ctx.json?.backtrace?.frames instanceof List + processor: + convert: + field: _ingest._value.imageOffset + type: string + tag: rename_backtrace_frames_imageOffset + target_field: _ingest._value.image.offset + ignore_missing: true + - foreach: + field: json.backtrace.frames + tag: foreach_backtrace_frames_imageOffset + if: ctx.json?.backtrace?.frames instanceof List + processor: + remove: + field: _ingest._value.imageOffset + tag: remove_imageOffset + ignore_missing: true + - foreach: + field: json.backtrace.frames + tag: foreach_backtrace_frames_imageUUID + if: ctx.json?.backtrace?.frames instanceof List + processor: + rename: + field: _ingest._value.imageUUID + tag: rename_backtrace_frames_imageUUID + target_field: _ingest._value.image.uuid + ignore_missing: true + - rename: + field: json.backtrace.frames + tag: rename_backtrace_frames + target_field: macos.backtrace.frames + ignore_missing: true + - rename: + field: json.bootUUID + tag: rename_bootUUID + target_field: macos.boot_uuid + ignore_missing: true + - rename: + field: json.category + tag: rename_category + target_field: macos.category + ignore_missing: true + - rename: + field: json.eventMessage + tag: rename_eventMessage + target_field: macos.event.message.description + ignore_missing: true + - set: + field: message + tag: set_message_from_unified_log_event_message_description + copy_from: macos.event.message.description + ignore_empty_value: true + - rename: + field: json.eventType + tag: rename_eventType + target_field: macos.event.type + ignore_missing: true + - rename: + field: json.formatString + tag: rename_formatString + target_field: macos.format_string + ignore_missing: true + - convert: + field: json.machTimestamp + type: double + tag: convert_machTimestamp_to_string + target_field: macos.mach_timestamp + ignore_missing: true + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - rename: + field: json.messageType + tag: rename_messageType + target_field: macos.message_type + ignore_missing: true + - script: + description: Map log level based on value of message_type. + tag: script_to_map_log_level + if: ctx.macos?.message_type != null + lang: painless + params: + default: info + error: error + debug: debug + info: info + fault: warning + source: |- + ctx.log = ctx.log ?: [:]; + ctx.log.put("level", params.get(ctx.macos.message_type.toLowerCase())); + - convert: + field: json.parentActivityIdentifier + tag: convert_parentActivityIdentifier_to_string + target_field: macos.parent_activity_identifier + type: string + ignore_missing: true + - convert: + field: json.processID + tag: convert_processID_to_string + target_field: macos.process.id + type: long + ignore_missing: true + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - set: + field: process.pid + tag: set_process_pid_from_unified_log_process_id + copy_from: macos.process.id + ignore_empty_value: true + - rename: + field: json.processImagePath + tag: rename_processImagePath + target_field: macos.process.image_path + ignore_missing: true + - rename: + field: json.processImageUUID + tag: rename_processImageUUID + target_field: macos.process.image_uuid + ignore_missing: true + - rename: + field: json.senderImagePath + tag: rename_senderImagePath + target_field: macos.sender.image_path + ignore_missing: true + - rename: + field: json.senderImageUUID + tag: rename_senderImageUUID + target_field: macos.sender.image_uuid + ignore_missing: true + - convert: + field: json.senderProgramCounter + tag: convert_senderProgramCounter_to_long + target_field: macos.sender.program_counter + type: long + ignore_missing: true + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - rename: + field: json.source + tag: rename_source + target_field: macos.source + ignore_missing: true + - rename: + field: json.subsystem + tag: rename_subsystem + target_field: macos.subsystem + ignore_missing: true + - convert: + field: json.threadID + tag: convert_threadID_to_long + target_field: macos.thread_id + type: long + ignore_missing: true + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - set: + field: process.thread.id + tag: set_process_thread_id_from_unified_log_thread_id + copy_from: macos.thread_id + ignore_empty_value: true + - date: + field: json.timestamp + tag: date_timestamp + target_field: macos.timestamp + formats: + - yyyy-MM-dd HH:mm:ss.SSSSSSZ + - yyyy-MM-dd HH:mm:ss.SSSZ + - strict_date_optional_time + if: ctx.json?.timestamp != null && ctx.json.timestamp != '' + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - set: + field: '@timestamp' + tag: set_@timestamp_from_unified_log_timestamp + copy_from: macos.timestamp + ignore_empty_value: true + - rename: + field: json.timezoneName + tag: rename_timezoneName + target_field: macos.timezone_name + ignore_missing: true + - convert: + field: json.traceID + tag: convert_traceID_to_string + target_field: macos.trace_id + type: string + ignore_missing: true + - convert: + field: json.userID + tag: convert_userID_to_string + target_field: macos.user_id + type: string + ignore_missing: true + - set: + field: user.id + tag: set_user_id_from_unified_log_user_id + copy_from: macos.user_id + ignore_empty_value: false + - append: + field: related.user + tag: append_unified_log_user_id_into_related_user + value: '{{{macos.user_id}}}' + allow_duplicates: false + if: ctx.macos?.user_id != null + - remove: + field: + - macos.message_type + - macos.process.id + - macos.thread_id + - macos.timestamp + - macos.user_id + tag: remove_custom_duplicate_fields + ignore_missing: true + if: ctx.tags == null || !ctx.tags.contains('preserve_duplicate_custom_fields') + - remove: + field: + - json + tag: remove_non_required_fields + ignore_missing: true + - script: + tag: script_to_drop_null_values + lang: painless + description: This script processor iterates over the whole document to remove fields with null values. + source: |- + void handleMap(Map map) { + map.values().removeIf(v -> { + if (v instanceof Map) { + handleMap(v); + } else if (v instanceof List) { + handleList(v); + } + return v == null || v == '' || (v instanceof Map && v.size() == 0) || (v instanceof List && v.size() == 0) + }); + } + void handleList(List list) { + list.removeIf(v -> { + if (v instanceof Map) { + handleMap(v); + } else if (v instanceof List) { + handleList(v); + } + return v == null || v == '' || (v instanceof Map && v.size() == 0) || (v instanceof List && v.size() == 0) + }); + } + handleMap(ctx); + - set: + field: event.kind + tag: set_pipeline_error_into_event_kind + value: pipeline_error + if: ctx.error?.message != null + - append: + field: tags + value: preserve_original_event + allow_duplicates: false + if: ctx.error?.message != null +on_failure: + - append: + field: error.message + value: |- + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}' + - set: + field: event.kind + tag: set_pipeline_error_to_event_kind + value: pipeline_error + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/macos/changelog.yml b/packages/macos/changelog.yml new file mode 100644 index 00000000000..a715f4ad047 --- /dev/null +++ b/packages/macos/changelog.yml @@ -0,0 +1,6 @@ +# newer versions go on top +- version: 0.1.0 + changes: + - description: Initial release. + type: enhancement + link: https://github.com/elastic/integrations/pull/15794 diff --git a/packages/macos/data_stream/advanced_monitoring/_dev/test/pipeline/test-advanced-monitoring.log b/packages/macos/data_stream/advanced_monitoring/_dev/test/pipeline/test-advanced-monitoring.log new file mode 100644 index 00000000000..9055f2bee99 --- /dev/null +++ b/packages/macos/data_stream/advanced_monitoring/_dev/test/pipeline/test-advanced-monitoring.log @@ -0,0 +1,2 @@ +{"timezoneName":"","messageType":"Error","eventType":"logEvent","source":null,"formatString":"rejecting write of key(s) %{public}s in { %{public}s, %{public}s, %{public}s, %{public}s, managed: %d } from process %{public}d (%{public}s) because %{public}s","userID":502,"activityIdentifier":0,"subsystem":"com.apple.defaults","category":"cfprefsd","threadID":273730,"senderImageUUID":"FEDAF68C-F484-3FCA-8866-A9E7E46CE7B6","backtrace":{"frames":[{"imageOffset":1818634,"imageUUID":"FEDAF68C-F484-3FCA-8866-A9E7E46CE7B6"}]},"bootUUID":"218031E6-E47F-4A77-B7FC-5A57B049F4BC","processImagePath":"\/usr\/sbin\/cfprefsd","senderImagePath":"\/System\/Library\/Frameworks\/CoreFoundation.framework\/Versions\/A\/CoreFoundation","timestamp":"2025-10-01 18:19:11.945508+0530","machTimestamp":454777460134003,"eventMessage":"rejecting write of key(s) CKStartupTime in { secd, test, kCFPreferencesAnyHost, \/Users\/test\/Library\/Preferences\/secd.plist, managed: 0 } from process 4954 (secd) because setting these preferences requires user-preference-write or file-write-data sandbox access","processImageUUID":"04C516B8-C8E5-30EF-AC49-1631528F5645","traceID":35866893965594628,"processID":4944,"senderProgramCounter":1818634,"parentActivityIdentifier":0} +{"timezoneName":"","messageType":"Default","eventType":"logEvent","source":null,"formatString":"<%{public}@> not executing next command because suspended=%{public}@, executing=%{public}@, commandCount=%lu","userID":501,"activityIdentifier":0,"subsystem":"com.apple.news","category":"CommandQueue","threadID":275245,"senderImageUUID":"266418A9-92D5-35C8-A3C9-13912677784B","backtrace":{"frames":[{"imageOffset":1799015,"imageUUID":"266418A9-92D5-35C8-A3C9-13912677784B"}]},"bootUUID":"218031E6-E47F-4A77-B7FC-5A57B049F4BC","processImagePath":"\/System\/Applications\/Stocks.app\/Contents\/PlugIns\/StocksDetailIntents.appex\/Contents\/MacOS\/StocksDetailIntents","senderImagePath":"\/System\/Library\/PrivateFrameworks\/NewsCore.framework\/Versions\/A\/NewsCore","timestamp":"2025-10-01 18:20:53.136387+0530","machTimestamp":454878651013159,"eventMessage":" not executing next command because suspended=true, executing=false, commandCount=0","processImageUUID":"52E75B03-EBCD-3C9F-A24E-3D6690D55097","traceID":975449195365007364,"processID":5000,"senderProgramCounter":1799015,"parentActivityIdentifier":0} diff --git a/packages/macos/data_stream/advanced_monitoring/_dev/test/pipeline/test-advanced-monitoring.log-expected.json b/packages/macos/data_stream/advanced_monitoring/_dev/test/pipeline/test-advanced-monitoring.log-expected.json new file mode 100644 index 00000000000..35a8eb61c90 --- /dev/null +++ b/packages/macos/data_stream/advanced_monitoring/_dev/test/pipeline/test-advanced-monitoring.log-expected.json @@ -0,0 +1,146 @@ +{ + "expected": [ + { + "@timestamp": "2025-10-01T12:49:11.945Z", + "ecs": { + "version": "9.2.0" + }, + "event": { + "kind": "event", + "original": "{\"timezoneName\":\"\",\"messageType\":\"Error\",\"eventType\":\"logEvent\",\"source\":null,\"formatString\":\"rejecting write of key(s) %{public}s in { %{public}s, %{public}s, %{public}s, %{public}s, managed: %d } from process %{public}d (%{public}s) because %{public}s\",\"userID\":502,\"activityIdentifier\":0,\"subsystem\":\"com.apple.defaults\",\"category\":\"cfprefsd\",\"threadID\":273730,\"senderImageUUID\":\"FEDAF68C-F484-3FCA-8866-A9E7E46CE7B6\",\"backtrace\":{\"frames\":[{\"imageOffset\":1818634,\"imageUUID\":\"FEDAF68C-F484-3FCA-8866-A9E7E46CE7B6\"}]},\"bootUUID\":\"218031E6-E47F-4A77-B7FC-5A57B049F4BC\",\"processImagePath\":\"\\/usr\\/sbin\\/cfprefsd\",\"senderImagePath\":\"\\/System\\/Library\\/Frameworks\\/CoreFoundation.framework\\/Versions\\/A\\/CoreFoundation\",\"timestamp\":\"2025-10-01 18:19:11.945508+0530\",\"machTimestamp\":454777460134003,\"eventMessage\":\"rejecting write of key(s) CKStartupTime in { secd, test, kCFPreferencesAnyHost, \\/Users\\/test\\/Library\\/Preferences\\/secd.plist, managed: 0 } from process 4954 (secd) because setting these preferences requires user-preference-write or file-write-data sandbox access\",\"processImageUUID\":\"04C516B8-C8E5-30EF-AC49-1631528F5645\",\"traceID\":35866893965594628,\"processID\":4944,\"senderProgramCounter\":1818634,\"parentActivityIdentifier\":0}" + }, + "log": { + "level": "error" + }, + "macos": { + "activity_identifier": "0", + "backtrace": { + "frames": [ + { + "image": { + "offset": "1818634", + "uuid": "FEDAF68C-F484-3FCA-8866-A9E7E46CE7B6" + } + } + ] + }, + "boot_uuid": "218031E6-E47F-4A77-B7FC-5A57B049F4BC", + "category": "cfprefsd", + "event": { + "message": { + "description": "rejecting write of key(s) CKStartupTime in { secd, test, kCFPreferencesAnyHost, /Users/test/Library/Preferences/secd.plist, managed: 0 } from process 4954 (secd) because setting these preferences requires user-preference-write or file-write-data sandbox access" + }, + "type": "logEvent" + }, + "format_string": "rejecting write of key(s) %{public}s in { %{public}s, %{public}s, %{public}s, %{public}s, managed: %d } from process %{public}d (%{public}s) because %{public}s", + "mach_timestamp": 4.54777460134003E14, + "message_type": "Error", + "parent_activity_identifier": "0", + "process": { + "id": 4944, + "image_path": "/usr/sbin/cfprefsd", + "image_uuid": "04C516B8-C8E5-30EF-AC49-1631528F5645" + }, + "sender": { + "image_path": "/System/Library/Frameworks/CoreFoundation.framework/Versions/A/CoreFoundation", + "image_uuid": "FEDAF68C-F484-3FCA-8866-A9E7E46CE7B6", + "program_counter": 1818634 + }, + "subsystem": "com.apple.defaults", + "thread_id": 273730, + "timestamp": "2025-10-01T12:49:11.945Z", + "trace_id": "35866893965594628", + "user_id": "502" + }, + "message": "rejecting write of key(s) CKStartupTime in { secd, test, kCFPreferencesAnyHost, /Users/test/Library/Preferences/secd.plist, managed: 0 } from process 4954 (secd) because setting these preferences requires user-preference-write or file-write-data sandbox access", + "process": { + "pid": 4944, + "thread": { + "id": 273730 + } + }, + "related": { + "user": [ + "502" + ] + }, + "tags": [ + "preserve_duplicate_custom_fields" + ], + "user": { + "id": "502" + } + }, + { + "@timestamp": "2025-10-01T12:50:53.136Z", + "ecs": { + "version": "9.2.0" + }, + "event": { + "kind": "event", + "original": "{\"timezoneName\":\"\",\"messageType\":\"Default\",\"eventType\":\"logEvent\",\"source\":null,\"formatString\":\"<%{public}@> not executing next command because suspended=%{public}@, executing=%{public}@, commandCount=%lu\",\"userID\":501,\"activityIdentifier\":0,\"subsystem\":\"com.apple.news\",\"category\":\"CommandQueue\",\"threadID\":275245,\"senderImageUUID\":\"266418A9-92D5-35C8-A3C9-13912677784B\",\"backtrace\":{\"frames\":[{\"imageOffset\":1799015,\"imageUUID\":\"266418A9-92D5-35C8-A3C9-13912677784B\"}]},\"bootUUID\":\"218031E6-E47F-4A77-B7FC-5A57B049F4BC\",\"processImagePath\":\"\\/System\\/Applications\\/Stocks.app\\/Contents\\/PlugIns\\/StocksDetailIntents.appex\\/Contents\\/MacOS\\/StocksDetailIntents\",\"senderImagePath\":\"\\/System\\/Library\\/PrivateFrameworks\\/NewsCore.framework\\/Versions\\/A\\/NewsCore\",\"timestamp\":\"2025-10-01 18:20:53.136387+0530\",\"machTimestamp\":454878651013159,\"eventMessage\":\" not executing next command because suspended=true, executing=false, commandCount=0\",\"processImageUUID\":\"52E75B03-EBCD-3C9F-A24E-3D6690D55097\",\"traceID\":975449195365007364,\"processID\":5000,\"senderProgramCounter\":1799015,\"parentActivityIdentifier\":0}" + }, + "log": { + "level": "info" + }, + "macos": { + "activity_identifier": "0", + "backtrace": { + "frames": [ + { + "image": { + "offset": "1799015", + "uuid": "266418A9-92D5-35C8-A3C9-13912677784B" + } + } + ] + }, + "boot_uuid": "218031E6-E47F-4A77-B7FC-5A57B049F4BC", + "category": "CommandQueue", + "event": { + "message": { + "description": " not executing next command because suspended=true, executing=false, commandCount=0" + }, + "type": "logEvent" + }, + "format_string": "<%{public}@> not executing next command because suspended=%{public}@, executing=%{public}@, commandCount=%lu", + "mach_timestamp": 4.54878651013159E14, + "message_type": "Default", + "parent_activity_identifier": "0", + "process": { + "id": 5000, + "image_path": "/System/Applications/Stocks.app/Contents/PlugIns/StocksDetailIntents.appex/Contents/MacOS/StocksDetailIntents", + "image_uuid": "52E75B03-EBCD-3C9F-A24E-3D6690D55097" + }, + "sender": { + "image_path": "/System/Library/PrivateFrameworks/NewsCore.framework/Versions/A/NewsCore", + "image_uuid": "266418A9-92D5-35C8-A3C9-13912677784B", + "program_counter": 1799015 + }, + "subsystem": "com.apple.news", + "thread_id": 275245, + "timestamp": "2025-10-01T12:50:53.136Z", + "trace_id": "975449195365007364", + "user_id": "501" + }, + "message": " not executing next command because suspended=true, executing=false, commandCount=0", + "process": { + "pid": 5000, + "thread": { + "id": 275245 + } + }, + "related": { + "user": [ + "501" + ] + }, + "tags": [ + "preserve_duplicate_custom_fields" + ], + "user": { + "id": "501" + } + } + ] +} diff --git a/packages/macos/data_stream/advanced_monitoring/_dev/test/pipeline/test-common-config.yml b/packages/macos/data_stream/advanced_monitoring/_dev/test/pipeline/test-common-config.yml new file mode 100644 index 00000000000..37e8fa225fd --- /dev/null +++ b/packages/macos/data_stream/advanced_monitoring/_dev/test/pipeline/test-common-config.yml @@ -0,0 +1,3 @@ +fields: + tags: + - preserve_duplicate_custom_fields diff --git a/packages/macos/data_stream/advanced_monitoring/agent/stream/unifiedlogs.yml.hbs b/packages/macos/data_stream/advanced_monitoring/agent/stream/unifiedlogs.yml.hbs new file mode 100644 index 00000000000..8b71cb58250 --- /dev/null +++ b/packages/macos/data_stream/advanced_monitoring/agent/stream/unifiedlogs.yml.hbs @@ -0,0 +1,50 @@ +predicate: +- 'eventMessage CONTAINS[c] "exec" OR eventMessage CONTAINS[c] "fork" OR eventMessage CONTAINS[c] "exited" OR eventMessage CONTAINS[c] "terminated"' +- 'subsystem == "com.apple.securityd" AND (composedMessage CONTAINS "code signing" OR composedMessage CONTAINS "not valid")' +- 'composedMessage CONTAINS "com.apple.quarantine"' +{{#each predicate as |p|}} +- {{p}} +{{/each}} +{{#if start}} +start: {{start}} +{{/if}} +{{#if end}} +end: {{end}} +{{/if}} +{{#if process}} +process: +{{#each process as |p|}} +- {{p}} +{{/each}} +{{/if}} +source: {{source}} +info: {{info}} +debug: {{debug}} +backtrace: {{backtrace}} +signpost: {{signpost}} +unreliable: {{unreliable}} +mach_continuous_time: {{mach_continuous_time}} +backfill: {{backfill}} +{{#if archive_file}} +archive_file: {{archive_file}} +{{/if}} +{{#if trace_file}} +trace_file: {{trace_file}} +{{/if}} +tags: +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} +{{#if preserve_duplicate_custom_fields}} + - preserve_duplicate_custom_fields +{{/if}} +{{#each tags as |tag|}} + - {{tag}} +{{/each}} +{{#contains "forwarded" tags}} +publisher_pipeline.disable_host: true +{{/contains}} +{{#if processors}} +processors: +{{processors}} +{{/if}} diff --git a/packages/macos/data_stream/advanced_monitoring/elasticsearch/ingest_pipeline/default.yml.link b/packages/macos/data_stream/advanced_monitoring/elasticsearch/ingest_pipeline/default.yml.link new file mode 100644 index 00000000000..f146557ce04 --- /dev/null +++ b/packages/macos/data_stream/advanced_monitoring/elasticsearch/ingest_pipeline/default.yml.link @@ -0,0 +1 @@ +../../../../_dev/shared/common-pipeline.yml 6e7bb697288eb24aeb67d4950a8a3f38d89eb89f559dc01e605e543162267ddf \ No newline at end of file diff --git a/packages/macos/data_stream/advanced_monitoring/fields/base-fields.yml b/packages/macos/data_stream/advanced_monitoring/fields/base-fields.yml new file mode 100644 index 00000000000..ab825b60a67 --- /dev/null +++ b/packages/macos/data_stream/advanced_monitoring/fields/base-fields.yml @@ -0,0 +1,16 @@ +- name: data_stream.type + external: ecs +- name: data_stream.dataset + external: ecs +- name: data_stream.namespace + external: ecs +- name: event.module + type: constant_keyword + external: ecs + value: macos +- name: event.dataset + type: constant_keyword + external: ecs + value: macos.advanced_monitoring +- name: '@timestamp' + external: ecs diff --git a/packages/macos/data_stream/advanced_monitoring/fields/beats.yml b/packages/macos/data_stream/advanced_monitoring/fields/beats.yml new file mode 100644 index 00000000000..d5fd38748ba --- /dev/null +++ b/packages/macos/data_stream/advanced_monitoring/fields/beats.yml @@ -0,0 +1,6 @@ +- name: input.type + type: keyword + description: Type of Filebeat input. +- name: log.offset + type: long + description: Log offset. diff --git a/packages/macos/data_stream/advanced_monitoring/fields/fields.yml b/packages/macos/data_stream/advanced_monitoring/fields/fields.yml new file mode 100644 index 00000000000..53e33cf5c76 --- /dev/null +++ b/packages/macos/data_stream/advanced_monitoring/fields/fields.yml @@ -0,0 +1,74 @@ +- name: macos + type: group + fields: + - name: activity_identifier + type: keyword + - name: backtrace + type: group + fields: + - name: frames + type: group + fields: + - name: image + type: group + fields: + - name: offset + type: keyword + - name: uuid + type: keyword + - name: boot_uuid + type: keyword + - name: category + type: keyword + - name: event + type: group + fields: + - name: category + type: keyword + - name: message + type: group + fields: + - name: description + type: keyword + - name: type + type: keyword + - name: format_string + type: keyword + - name: mach_timestamp + type: double + - name: message_type + type: keyword + - name: parent_activity_identifier + type: keyword + - name: process + type: group + fields: + - name: id + type: long + - name: image_path + type: keyword + - name: image_uuid + type: keyword + - name: sender + type: group + fields: + - name: image_path + type: keyword + - name: image_uuid + type: keyword + - name: program_counter + type: long + - name: source + type: keyword + - name: subsystem + type: keyword + - name: thread_id + type: long + - name: timestamp + type: date + - name: timezone_name + type: keyword + - name: trace_id + type: keyword + - name: user_id + type: keyword diff --git a/packages/macos/data_stream/advanced_monitoring/manifest.yml b/packages/macos/data_stream/advanced_monitoring/manifest.yml new file mode 100644 index 00000000000..887af026917 --- /dev/null +++ b/packages/macos/data_stream/advanced_monitoring/manifest.yml @@ -0,0 +1,139 @@ +title: Advanced Monitoring +type: logs +streams: + - input: unifiedlogs + title: Advanced Monitoring logs + description: Collect Advanced Monitoring logs from macOS. + template_path: unifiedlogs.yml.hbs + enabled: false + vars: + - name: predicate + type: text + title: Predicate + description: | + Filters messages using the provided predicate based on NSPredicate. + A compound predicate or multiple predicates can be provided as a list. + For detailed information on the use of predicate based filtering, + please refer to the https://developer.apple.com/library/mac/documentation/Cocoa/Conceptual/Predicates/Articles/pSyntax.html[Predicate Programming Guide]. + multi: true + show_user: true + - name: start + type: text + title: Start date + description: | + Shows content starting from the provided date. + The following date/time formats are accepted: + `YYYY-MM-DD`, `YYYY-MM-DD HH:MM:SS`, `YYYY-MM-DD HH:MM:SSZZZZZ`. + show_user: false + - name: end + type: text + title: End date + description: | + Shows content up to the provided date. + The following date/time formats are accepted: + `YYYY-MM-DD`, `YYYY-MM-DD HH:MM:SS`, `YYYY-MM-DD HH:MM:SSZZZZZ`. + show_user: false + - name: process + type: text + title: Process + description: | + A list of the processes on which to operate. It accepts a PID or process name. + multi: true + show_user: false + - name: source + type: bool + title: Include source + description: | + Include symbol names and source line numbers for messages, if available. + show_user: false + - name: info + type: bool + title: Include info + description: | + Disable or enable info level messages. + show_user: false + - name: debug + type: bool + title: Include debug + description: | + Disable or enable debug level messages. + show_user: false + - name: backtrace + type: bool + title: Include backtrace + description: | + Disable or enable backtrace level messages. + show_user: false + - name: signpost + type: bool + title: Include signpost + description: | + Disable or enable signpost level messages. + show_user: false + - name: unreliable + type: bool + title: Annotate unreliable + description: | + Annotate events with whether the log was emitted unreliably. + show_user: false + - name: mach_continuous_time + type: bool + title: Use mach continuous time + description: | + Use mach continuous time timestamps rather than walltime. + show_user: false + - name: backfill + type: bool + title: Must backfill + description: | + If set to true the input will process all available logs since the beginning + of time the first time it starts. + show_user: false + - name: archive_file + type: text + title: Archive file + description: | + Display events stored in the given archive. + The archive must be a valid log archive bundle with the suffix `.logarchive`. + show_user: false + - name: trace_file + type: text + title: Trace file + description: | + Display events stored in the given `.tracev3` file. + In order to be decoded, the file must be contained within a valid `.logarchive`. + show_user: false + - name: preserve_original_event + type: bool + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field event.original. + multi: false + required: false + show_user: true + default: false + - name: tags + type: text + title: Tags + description: Tags for the data-stream. + multi: true + required: true + show_user: false + default: + - forwarded + - macos-advanced_monitoring + - name: preserve_duplicate_custom_fields + required: false + title: Preserve duplicate custom fields + description: Preserve macos.* fields that were copied to Elastic Common Schema (ECS) fields. + type: bool + multi: false + show_user: false + default: false + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: >- + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. diff --git a/packages/macos/data_stream/authentication/_dev/test/pipeline/test-authentication.log b/packages/macos/data_stream/authentication/_dev/test/pipeline/test-authentication.log new file mode 100644 index 00000000000..4086882cd43 --- /dev/null +++ b/packages/macos/data_stream/authentication/_dev/test/pipeline/test-authentication.log @@ -0,0 +1,3 @@ +{"timezoneName":"","messageType":"Default","eventType":"logEvent","source":null,"formatString":"%s | ApplicationManager: Checked in app : %@","userID":501,"activityIdentifier":0,"subsystem":"com.apple.loginwindow.logging","category":"Standard","threadID":13692,"senderImageUUID":"2F16B6B6-41B5-34D4-88A1-06CAFCE952EF","backtrace":{"frames":[{"imageOffset":109999,"imageUUID":"2F16B6B6-41B5-34D4-88A1-06CAFCE952EF"}]},"bootUUID":"","processImagePath":"\/System\/Library\/CoreServices\/loginwindow.app\/Contents\/MacOS\/loginwindow","senderImagePath":"\/System\/Library\/CoreServices\/loginwindow.app\/Contents\/MacOS\/loginwindow","timestamp":"2025-10-09 12:02:22.345217+0530","eventMessage":"-[SessionLogoutManager buildSessionDictionary] | final sessionDictionary:{\n DirectLogoutType = 5;\n GroupID = 248;\n GuestAccount = 0;\n HomeDirectoryPath = \"\/var\/setup\";\n SessionAgentPID = 100;\n UserGUID = \"FFFFEEEE-DDDD-CCCC-BBBB-AAAA000000F8\";\n UserID = 248;\n UserLongName = \"Setup User\";\n UserName = \"_mbsetupuser\";\n}","processImageUUID":"2F16B6B6-41B5-34D4-88A1-06CAFCE952EF","traceID":3714562696282116,"processID":1062,"senderProgramCounter":109999,"parentActivityIdentifier":0} +{"timezoneName":"","messageType":"Default","eventType":"logEvent","source":null,"formatString":"%s | shortUsername = %s, userID = %d, groupID = %d","userID":0,"activityIdentifier":0,"subsystem":"com.apple.loginwindow.logging","category":"Standard","threadID":778,"senderImageUUID":"2F16B6B6-41B5-34D4-88A1-06CAFCE952EF","backtrace":{"frames":[{"imageOffset":56811,"imageUUID":"2F16B6B6-41B5-34D4-88A1-06CAFCE952EF"}]},"bootUUID":"218031E6-E47F-4A77-B7FC-5A57B049F4BC","processImagePath":"\/System\/Library\/CoreServices\/loginwindow.app\/Contents\/MacOS\/loginwindow","senderImagePath":"\/System\/Library\/CoreServices\/loginwindow.app\/Contents\/MacOS\/loginwindow","timestamp":"2025-09-25 23:32:36.119492-0700","machTimestamp":191058676868,"eventMessage":"-[Login1 doLogin] | shortUsername = _mbsetupuser, userID = 248, groupID = 248","processImageUUID":"2F16B6B6-41B5-34D4-88A1-06CAFCE952EF","traceID":3508679143981060,"processID":100,"senderProgramCounter":56811,"parentActivityIdentifier":0} +{"timezoneName":"","messageType":"Default","eventType":"logEvent","source":null,"formatString":"%s | enter","userID":0,"activityIdentifier":0,"subsystem":"com.apple.loginwindow.logging","category":"Standard","threadID":13692,"senderImageUUID":"2F16B6B6-41B5-34D4-88A1-06CAFCE952EF","backtrace":{"frames":[{"imageOffset":75080,"imageUUID":"2F16B6B6-41B5-34D4-88A1-06CAFCE952EF"}]},"bootUUID":"218031E6-E47F-4A77-B7FC-5A57B049F4BC","processImagePath":"\/System\/Library\/CoreServices\/loginwindow.app\/Contents\/MacOS\/loginwindow","senderImagePath":"\/System\/Library\/CoreServices\/loginwindow.app\/Contents\/MacOS\/loginwindow","timestamp":"2025-09-25 23:46:06.773373-0700","machTimestamp":1001013698208,"eventMessage":"-[Login1 isGuestAccount] | enter","processImageUUID":"2F16B6B6-41B5-34D4-88A1-06CAFCE952EF","traceID":3412540596027396,"processID":1062,"senderProgramCounter":75080,"parentActivityIdentifier":0} diff --git a/packages/macos/data_stream/authentication/_dev/test/pipeline/test-authentication.log-expected.json b/packages/macos/data_stream/authentication/_dev/test/pipeline/test-authentication.log-expected.json new file mode 100644 index 00000000000..bf044c38789 --- /dev/null +++ b/packages/macos/data_stream/authentication/_dev/test/pipeline/test-authentication.log-expected.json @@ -0,0 +1,267 @@ +{ + "expected": [ + { + "@timestamp": "2025-10-09T06:32:22.345Z", + "ecs": { + "version": "9.2.0" + }, + "event": { + "category": [ + "authentication" + ], + "kind": "event", + "original": "{\"timezoneName\":\"\",\"messageType\":\"Default\",\"eventType\":\"logEvent\",\"source\":null,\"formatString\":\"%s | ApplicationManager: Checked in app : %@\",\"userID\":501,\"activityIdentifier\":0,\"subsystem\":\"com.apple.loginwindow.logging\",\"category\":\"Standard\",\"threadID\":13692,\"senderImageUUID\":\"2F16B6B6-41B5-34D4-88A1-06CAFCE952EF\",\"backtrace\":{\"frames\":[{\"imageOffset\":109999,\"imageUUID\":\"2F16B6B6-41B5-34D4-88A1-06CAFCE952EF\"}]},\"bootUUID\":\"\",\"processImagePath\":\"\\/System\\/Library\\/CoreServices\\/loginwindow.app\\/Contents\\/MacOS\\/loginwindow\",\"senderImagePath\":\"\\/System\\/Library\\/CoreServices\\/loginwindow.app\\/Contents\\/MacOS\\/loginwindow\",\"timestamp\":\"2025-10-09 12:02:22.345217+0530\",\"eventMessage\":\"-[SessionLogoutManager buildSessionDictionary] | final sessionDictionary:{\\n DirectLogoutType = 5;\\n GroupID = 248;\\n GuestAccount = 0;\\n HomeDirectoryPath = \\\"\\/var\\/setup\\\";\\n SessionAgentPID = 100;\\n UserGUID = \\\"FFFFEEEE-DDDD-CCCC-BBBB-AAAA000000F8\\\";\\n UserID = 248;\\n UserLongName = \\\"Setup User\\\";\\n UserName = \\\"_mbsetupuser\\\";\\n}\",\"processImageUUID\":\"2F16B6B6-41B5-34D4-88A1-06CAFCE952EF\",\"traceID\":3714562696282116,\"processID\":1062,\"senderProgramCounter\":109999,\"parentActivityIdentifier\":0}", + "type": [ + "info" + ] + }, + "group": { + "id": "248" + }, + "log": { + "level": "info" + }, + "macos": { + "activity_identifier": "0", + "backtrace": { + "frames": [ + { + "image": { + "offset": "109999", + "uuid": "2F16B6B6-41B5-34D4-88A1-06CAFCE952EF" + } + } + ] + }, + "category": "Standard", + "event": { + "message": { + "description": "-[SessionLogoutManager buildSessionDictionary] | final sessionDictionary:{\n DirectLogoutType = 5;\n GroupID = 248;\n GuestAccount = 0;\n HomeDirectoryPath = \"/var/setup\";\n SessionAgentPID = 100;\n UserGUID = \"FFFFEEEE-DDDD-CCCC-BBBB-AAAA000000F8\";\n UserID = 248;\n UserLongName = \"Setup User\";\n UserName = \"_mbsetupuser\";\n}", + "direct_logout_type": 5, + "group_id": "248", + "guest_account": 0, + "home_directory_path": "\"/var/setup\"", + "session_agent_pid": "100", + "user": { + "guid": "FFFFEEEE-DDDD-CCCC-BBBB-AAAA000000F8", + "id": "248", + "long_name": "Setup User", + "name": "_mbsetupuser" + } + }, + "type": "logEvent" + }, + "format_string": "%s | ApplicationManager: Checked in app : %@", + "message_type": "Default", + "parent_activity_identifier": "0", + "process": { + "id": 1062, + "image_path": "/System/Library/CoreServices/loginwindow.app/Contents/MacOS/loginwindow", + "image_uuid": "2F16B6B6-41B5-34D4-88A1-06CAFCE952EF" + }, + "sender": { + "image_path": "/System/Library/CoreServices/loginwindow.app/Contents/MacOS/loginwindow", + "image_uuid": "2F16B6B6-41B5-34D4-88A1-06CAFCE952EF", + "program_counter": 109999 + }, + "subsystem": "com.apple.loginwindow.logging", + "thread_id": 13692, + "timestamp": "2025-10-09T06:32:22.345Z", + "trace_id": "3714562696282116", + "user_id": "501" + }, + "message": "-[SessionLogoutManager buildSessionDictionary] | final sessionDictionary:{\n DirectLogoutType = 5;\n GroupID = 248;\n GuestAccount = 0;\n HomeDirectoryPath = \"/var/setup\";\n SessionAgentPID = 100;\n UserGUID = \"FFFFEEEE-DDDD-CCCC-BBBB-AAAA000000F8\";\n UserID = 248;\n UserLongName = \"Setup User\";\n UserName = \"_mbsetupuser\";\n}", + "process": { + "pid": 1062, + "thread": { + "id": 13692 + } + }, + "related": { + "user": [ + "501", + "248", + "Setup User", + "_mbsetupuser", + "FFFFEEEE-DDDD-CCCC-BBBB-AAAA000000F8" + ] + }, + "tags": [ + "preserve_duplicate_custom_fields" + ], + "user": { + "full_name": "Setup User", + "group": { + "id": "FFFFEEEE-DDDD-CCCC-BBBB-AAAA000000F8" + }, + "id": "248", + "name": "_mbsetupuser" + } + }, + { + "@timestamp": "2025-09-26T06:32:36.119Z", + "ecs": { + "version": "9.2.0" + }, + "event": { + "category": [ + "authentication" + ], + "kind": "event", + "original": "{\"timezoneName\":\"\",\"messageType\":\"Default\",\"eventType\":\"logEvent\",\"source\":null,\"formatString\":\"%s | shortUsername = %s, userID = %d, groupID = %d\",\"userID\":0,\"activityIdentifier\":0,\"subsystem\":\"com.apple.loginwindow.logging\",\"category\":\"Standard\",\"threadID\":778,\"senderImageUUID\":\"2F16B6B6-41B5-34D4-88A1-06CAFCE952EF\",\"backtrace\":{\"frames\":[{\"imageOffset\":56811,\"imageUUID\":\"2F16B6B6-41B5-34D4-88A1-06CAFCE952EF\"}]},\"bootUUID\":\"218031E6-E47F-4A77-B7FC-5A57B049F4BC\",\"processImagePath\":\"\\/System\\/Library\\/CoreServices\\/loginwindow.app\\/Contents\\/MacOS\\/loginwindow\",\"senderImagePath\":\"\\/System\\/Library\\/CoreServices\\/loginwindow.app\\/Contents\\/MacOS\\/loginwindow\",\"timestamp\":\"2025-09-25 23:32:36.119492-0700\",\"machTimestamp\":191058676868,\"eventMessage\":\"-[Login1 doLogin] | shortUsername = _mbsetupuser, userID = 248, groupID = 248\",\"processImageUUID\":\"2F16B6B6-41B5-34D4-88A1-06CAFCE952EF\",\"traceID\":3508679143981060,\"processID\":100,\"senderProgramCounter\":56811,\"parentActivityIdentifier\":0}", + "type": [ + "info" + ] + }, + "group": { + "id": "248" + }, + "log": { + "level": "info" + }, + "macos": { + "activity_identifier": "0", + "backtrace": { + "frames": [ + { + "image": { + "offset": "56811", + "uuid": "2F16B6B6-41B5-34D4-88A1-06CAFCE952EF" + } + } + ] + }, + "boot_uuid": "218031E6-E47F-4A77-B7FC-5A57B049F4BC", + "category": "Standard", + "event": { + "message": { + "description": "-[Login1 doLogin] | shortUsername = _mbsetupuser, userID = 248, groupID = 248", + "group_id": "248", + "user": { + "id": "248", + "name": "_mbsetupuser" + } + }, + "type": "logEvent" + }, + "format_string": "%s | shortUsername = %s, userID = %d, groupID = %d", + "mach_timestamp": 1.91058676868E11, + "message_type": "Default", + "parent_activity_identifier": "0", + "process": { + "id": 100, + "image_path": "/System/Library/CoreServices/loginwindow.app/Contents/MacOS/loginwindow", + "image_uuid": "2F16B6B6-41B5-34D4-88A1-06CAFCE952EF" + }, + "sender": { + "image_path": "/System/Library/CoreServices/loginwindow.app/Contents/MacOS/loginwindow", + "image_uuid": "2F16B6B6-41B5-34D4-88A1-06CAFCE952EF", + "program_counter": 56811 + }, + "subsystem": "com.apple.loginwindow.logging", + "thread_id": 778, + "timestamp": "2025-09-26T06:32:36.119Z", + "trace_id": "3508679143981060", + "user_id": "0" + }, + "message": "-[Login1 doLogin] | shortUsername = _mbsetupuser, userID = 248, groupID = 248", + "process": { + "pid": 100, + "thread": { + "id": 778 + } + }, + "related": { + "user": [ + "0", + "248", + "_mbsetupuser" + ] + }, + "tags": [ + "preserve_duplicate_custom_fields" + ], + "user": { + "id": "248", + "name": "_mbsetupuser" + } + }, + { + "@timestamp": "2025-09-26T06:46:06.773Z", + "ecs": { + "version": "9.2.0" + }, + "event": { + "category": [ + "authentication" + ], + "kind": "event", + "original": "{\"timezoneName\":\"\",\"messageType\":\"Default\",\"eventType\":\"logEvent\",\"source\":null,\"formatString\":\"%s | enter\",\"userID\":0,\"activityIdentifier\":0,\"subsystem\":\"com.apple.loginwindow.logging\",\"category\":\"Standard\",\"threadID\":13692,\"senderImageUUID\":\"2F16B6B6-41B5-34D4-88A1-06CAFCE952EF\",\"backtrace\":{\"frames\":[{\"imageOffset\":75080,\"imageUUID\":\"2F16B6B6-41B5-34D4-88A1-06CAFCE952EF\"}]},\"bootUUID\":\"218031E6-E47F-4A77-B7FC-5A57B049F4BC\",\"processImagePath\":\"\\/System\\/Library\\/CoreServices\\/loginwindow.app\\/Contents\\/MacOS\\/loginwindow\",\"senderImagePath\":\"\\/System\\/Library\\/CoreServices\\/loginwindow.app\\/Contents\\/MacOS\\/loginwindow\",\"timestamp\":\"2025-09-25 23:46:06.773373-0700\",\"machTimestamp\":1001013698208,\"eventMessage\":\"-[Login1 isGuestAccount] | enter\",\"processImageUUID\":\"2F16B6B6-41B5-34D4-88A1-06CAFCE952EF\",\"traceID\":3412540596027396,\"processID\":1062,\"senderProgramCounter\":75080,\"parentActivityIdentifier\":0}", + "type": [ + "info" + ] + }, + "log": { + "level": "info" + }, + "macos": { + "activity_identifier": "0", + "backtrace": { + "frames": [ + { + "image": { + "offset": "75080", + "uuid": "2F16B6B6-41B5-34D4-88A1-06CAFCE952EF" + } + } + ] + }, + "boot_uuid": "218031E6-E47F-4A77-B7FC-5A57B049F4BC", + "category": "Standard", + "event": { + "message": { + "description": "-[Login1 isGuestAccount] | enter" + }, + "type": "logEvent" + }, + "format_string": "%s | enter", + "mach_timestamp": 1.001013698208E12, + "message_type": "Default", + "parent_activity_identifier": "0", + "process": { + "id": 1062, + "image_path": "/System/Library/CoreServices/loginwindow.app/Contents/MacOS/loginwindow", + "image_uuid": "2F16B6B6-41B5-34D4-88A1-06CAFCE952EF" + }, + "sender": { + "image_path": "/System/Library/CoreServices/loginwindow.app/Contents/MacOS/loginwindow", + "image_uuid": "2F16B6B6-41B5-34D4-88A1-06CAFCE952EF", + "program_counter": 75080 + }, + "subsystem": "com.apple.loginwindow.logging", + "thread_id": 13692, + "timestamp": "2025-09-26T06:46:06.773Z", + "trace_id": "3412540596027396", + "user_id": "0" + }, + "message": "-[Login1 isGuestAccount] | enter", + "process": { + "pid": 1062, + "thread": { + "id": 13692 + } + }, + "related": { + "user": [ + "0" + ] + }, + "tags": [ + "preserve_duplicate_custom_fields" + ], + "user": { + "id": "0" + } + } + ] +} diff --git a/packages/macos/data_stream/authentication/_dev/test/pipeline/test-common-config.yml b/packages/macos/data_stream/authentication/_dev/test/pipeline/test-common-config.yml new file mode 100644 index 00000000000..37e8fa225fd --- /dev/null +++ b/packages/macos/data_stream/authentication/_dev/test/pipeline/test-common-config.yml @@ -0,0 +1,3 @@ +fields: + tags: + - preserve_duplicate_custom_fields diff --git a/packages/macos/data_stream/authentication/agent/stream/unifiedlogs.yml.hbs b/packages/macos/data_stream/authentication/agent/stream/unifiedlogs.yml.hbs new file mode 100644 index 00000000000..3a2749c25c4 --- /dev/null +++ b/packages/macos/data_stream/authentication/agent/stream/unifiedlogs.yml.hbs @@ -0,0 +1,50 @@ +predicate: +- 'process contains "sudo" OR composedMessage CONTAINS "sudo" OR process contains "su"' +- 'process contains "loginwindow" and composedMessage CONTAINS "sessionDidLogin"' +- 'process == "sshd"' +{{#each predicate as |p|}} +- {{p}} +{{/each}} +{{#if start}} +start: {{start}} +{{/if}} +{{#if end}} +end: {{end}} +{{/if}} +{{#if process}} +process: +{{#each process as |p|}} +- {{p}} +{{/each}} +{{/if}} +source: {{source}} +info: {{info}} +debug: {{debug}} +backtrace: {{backtrace}} +signpost: {{signpost}} +unreliable: {{unreliable}} +mach_continuous_time: {{mach_continuous_time}} +backfill: {{backfill}} +{{#if archive_file}} +archive_file: {{archive_file}} +{{/if}} +{{#if trace_file}} +trace_file: {{trace_file}} +{{/if}} +tags: +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} +{{#if preserve_duplicate_custom_fields}} + - preserve_duplicate_custom_fields +{{/if}} +{{#each tags as |tag|}} + - {{tag}} +{{/each}} +{{#contains "forwarded" tags}} +publisher_pipeline.disable_host: true +{{/contains}} +{{#if processors}} +processors: +{{processors}} +{{/if}} diff --git a/packages/macos/data_stream/authentication/elasticsearch/ingest_pipeline/common-pipeline.yml.link b/packages/macos/data_stream/authentication/elasticsearch/ingest_pipeline/common-pipeline.yml.link new file mode 100644 index 00000000000..f146557ce04 --- /dev/null +++ b/packages/macos/data_stream/authentication/elasticsearch/ingest_pipeline/common-pipeline.yml.link @@ -0,0 +1 @@ +../../../../_dev/shared/common-pipeline.yml 6e7bb697288eb24aeb67d4950a8a3f38d89eb89f559dc01e605e543162267ddf \ No newline at end of file diff --git a/packages/macos/data_stream/authentication/elasticsearch/ingest_pipeline/default.yml b/packages/macos/data_stream/authentication/elasticsearch/ingest_pipeline/default.yml new file mode 100644 index 00000000000..d8e2d6b1e35 --- /dev/null +++ b/packages/macos/data_stream/authentication/elasticsearch/ingest_pipeline/default.yml @@ -0,0 +1,121 @@ +--- +description: Pipeline for processing authentication logs. +processors: + - append: + field: event.category + tag: set_event_category + value: authentication + - append: + field: event.type + tag: append_info_into_event_type_in_authentication + value: info + - pipeline: + name: '{{ IngestPipeline "common-pipeline" }}' + - grok: + description: Grok the eventMessage. + tag: grok-event-message + field: macos.event.message.description + ignore_missing: true + patterns: + - '^-\[%{WORD} %{WORD}\] \|%{SPACE}final sessionDictionary:\{(?:%{SPACE}DirectLogoutType = %{NUMBER:macos.event.message.direct_logout_type:int};)?(?:%{SPACE}GroupID = %{NUMBER:macos.event.message.group_id};)?(?:%{SPACE}GuestAccount = %{NUMBER:macos.event.message.guest_account:int};)?(?:%{SPACE}HomeDirectoryPath = %{DATA:macos.event.message.home_directory_path};)?(?:%{SPACE}SessionAgentPID = %{NUMBER:macos.event.message.session_agent_pid};)?(?:%{SPACE}UserGUID = %{DATA:macos.event.message.user.guid};)?(?:%{SPACE}UserID = %{NUMBER:macos.event.message.user.id};)?(?:%{SPACE}UserLongName = %{DATA:macos.event.message.user.long_name};)?(?:%{SPACE}UserName = %{DATA:macos.event.message.user.name};)?\n\}' + - '^-\[%{WORD} %{WORD}\] \|(?:%{SPACE}shortUsername = %{WORD:macos.event.message.user.name},)?(?:%{SPACE}userID = %{NUMBER:macos.event.message.user.id},)?(?:%{SPACE}groupID = %{NUMBER:macos.event.message.group_id})' + - '%{GREEDYDATA:macos.event.message.original}' + - set: + field: group.id + tag: set_group_id_from_unified_log_message_group_id + copy_from: macos.event.message.group_id + ignore_empty_value: true + - gsub: + field: macos.event.message.user.guid + pattern: '\"' + replacement: '' + ignore_missing: true + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - gsub: + field: macos.event.message.user.name + pattern: '\"' + replacement: '' + ignore_missing: true + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - gsub: + field: macos.event.message.user.long_name + pattern: '\"' + replacement: '' + ignore_missing: true + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - set: + field: user.full_name + tag: set_user_full_name_from_unified_log_message_userlongname + copy_from: macos.event.message.user.long_name + ignore_empty_value: true + - set: + field: user.name + tag: set_user_name_from_unified_log_message_username + copy_from: macos.event.message.user.name + ignore_empty_value: true + - set: + field: user.group.id + tag: set_user_group_id_from_unified_log_message_user_guid + copy_from: macos.event.message.user.guid + ignore_empty_value: true + - set: + field: user.id + tag: set_unified_log_event_message_user_id_into_user_id + copy_from: macos.event.message.user.id + ignore_empty_value: true + - append: + field: related.user + tag: append_unified_log_event_message_user_id_into_related_user + value: '{{{macos.event.message.user.id}}}' + allow_duplicates: false + if: ctx.macos?.event?.message?.user?.id != null + - append: + field: related.user + tag: append_unified_log_event_message_user_long_name_into_related_user + value: '{{{macos.event.message.user.long_name}}}' + allow_duplicates: false + if: ctx.macos?.event?.message?.user?.long_name != null + - append: + field: related.user + tag: append_unified_log_event_message_user_name_into_related_user + value: '{{{macos.event.message.user.name}}}' + allow_duplicates: false + if: ctx.macos?.event?.message?.user?.name != null + - append: + field: related.user + tag: append_unified_log_event_message_user_guid_into_related_user + value: '{{{macos.event.message.user.guid}}}' + allow_duplicates: false + if: ctx.macos?.event?.message?.user?.guid != null + - remove: + field: + - macos.event.message.group_id + - macos.event.message.user + tag: remove_custom_duplicate_fields + ignore_missing: true + if: ctx.tags == null || !ctx.tags.contains('preserve_duplicate_custom_fields') + - remove: + field: + - macos.event.message.original + tag: remove_non_required_fields + ignore_missing: true +on_failure: + - append: + field: error.message + value: >- + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}' + - set: + field: event.kind + tag: set_pipeline_error_to_event_kind + value: pipeline_error diff --git a/packages/macos/data_stream/authentication/fields/base-fields.yml b/packages/macos/data_stream/authentication/fields/base-fields.yml new file mode 100644 index 00000000000..76886f68aee --- /dev/null +++ b/packages/macos/data_stream/authentication/fields/base-fields.yml @@ -0,0 +1,16 @@ +- name: data_stream.type + external: ecs +- name: data_stream.dataset + external: ecs +- name: data_stream.namespace + external: ecs +- name: event.module + type: constant_keyword + external: ecs + value: macos +- name: event.dataset + type: constant_keyword + external: ecs + value: macos.authentication +- name: '@timestamp' + external: ecs diff --git a/packages/macos/data_stream/authentication/fields/beats.yml b/packages/macos/data_stream/authentication/fields/beats.yml new file mode 100644 index 00000000000..d5fd38748ba --- /dev/null +++ b/packages/macos/data_stream/authentication/fields/beats.yml @@ -0,0 +1,6 @@ +- name: input.type + type: keyword + description: Type of Filebeat input. +- name: log.offset + type: long + description: Log offset. diff --git a/packages/macos/data_stream/authentication/fields/fields.yml b/packages/macos/data_stream/authentication/fields/fields.yml new file mode 100644 index 00000000000..56977045417 --- /dev/null +++ b/packages/macos/data_stream/authentication/fields/fields.yml @@ -0,0 +1,307 @@ +- name: macos + type: group + fields: + - name: activity_identifier + type: keyword + - name: backtrace + type: group + fields: + - name: frames + type: group + fields: + - name: image + type: group + fields: + - name: offset + type: keyword + - name: uuid + type: keyword + - name: boot_uuid + type: keyword + - name: category + type: keyword + - name: event + type: group + fields: + - name: category + type: keyword + - name: message + type: group + fields: + - name: account_id + type: keyword + - name: accurate_ecn_client + type: keyword + - name: accurate_ecn_server + type: keyword + - name: ack + type: keyword + - name: acks_compressed + type: long + - name: acks_delayed + type: long + - name: alpn + type: keyword + - name: attribution + type: keyword + - name: base_rtt_ms + type: keyword + - name: bundle_id + type: keyword + - name: bytes_in + type: long + - name: bytes_out + type: long + - name: cache_hit + type: boolean + - name: cipher_suite + type: keyword + - name: client_ip + type: ip + - name: client_port + type: long + - name: connection + type: long + - name: connection_detail + type: keyword + - name: connection_duration_ms + type: long + - name: connection_id + type: keyword + - name: connection_identifier + type: keyword + - name: connection_time + type: keyword + - name: connection_uuid + type: keyword + - name: delayed_acks_sent + type: long + - name: description + type: keyword + - name: dest_port + type: long + - name: direct_logout_type + type: long + - name: dns_duration + type: keyword + - name: dns_start + type: keyword + - name: domain_lookup_duration_ms + type: long + - name: duration + type: keyword + - name: ecn_acked + type: long + - name: ecn_in + type: long + - name: ecn_lost + type: long + - name: ecn_marked + type: long + - name: ecn_miss + type: long + - name: ecn_out + type: long + - name: ecn_sent + type: long + - name: expected_workload + type: keyword + - name: false_started + type: boolean + - name: flags + type: keyword + - name: flight_time + type: keyword + - name: group + type: keyword + - name: group_id + type: keyword + - name: guest_account + type: long + - name: home_directory_path + type: keyword + - name: hostname + type: keyword + - name: hostname_port + type: long + - name: init_flag + type: long + - name: interface + type: keyword + - name: listener + type: boolean + - name: mach + type: boolean + - name: name + type: keyword + - name: ocsp_received + type: boolean + - name: offered_ticket + type: boolean + - name: out_of_order_bytes + type: long + - name: packets_in + type: long + - name: packets_out + type: long + - name: pake + type: keyword + - name: path_status + type: keyword + - name: peer + type: boolean + - name: pid + type: long + - name: privacy_stance + type: keyword + - name: private_relay + type: boolean + - name: protocol + type: keyword + - name: rd_t_in + type: long + - name: rd_t_out + type: long + - name: read_stalls + type: long + - name: request_bytes + type: long + - name: request_duration_ms + type: long + - name: request_start_ms + type: long + - name: response_bytes + type: long + - name: response_duration_ms + type: long + - name: response_start_ms + type: long + - name: response_status + type: long + - name: resumed + type: boolean + - name: retransmitted_bytes + type: long + - name: rtt + type: keyword + - name: rtt_cache + type: keyword + - name: rtt_nc_ms + type: keyword + - name: rtt_updates + type: long + - name: rtt_var_ms + type: keyword + - name: rtt_var_nc_ms + type: keyword + - name: sct_received + type: boolean + - name: secure_connection_duration_ms + type: long + - name: seq + type: keyword + - name: server_id + type: keyword + - name: server_port + type: long + - name: session_agent_pid + type: keyword + - name: session_uuid + type: keyword + - name: signature_alg + type: keyword + - name: src_port + type: long + - name: state + type: keyword + - name: syns + type: long + - name: task_uid + type: keyword + - name: tcp_duration + type: keyword + - name: tcp_start + type: keyword + - name: tfo_in + type: long + - name: tfo_miss + type: long + - name: tfo_out + type: long + - name: timestamp_enabled + type: long + - name: tls_duration + type: keyword + - name: tls_version + type: keyword + - name: traffic_class + type: keyword + - name: transaction_duration_ms + type: long + - name: tso_enabled + type: long + - name: url + type: keyword + - name: url_hash + type: keyword + - name: user + type: group + fields: + - name: guid + type: keyword + - name: id + type: keyword + - name: long_name + type: keyword + - name: name + type: keyword + - name: win + type: keyword + - name: wr_t_in + type: long + - name: wr_t_out + type: long + - name: write_stalls + type: long + - name: type + type: keyword + - name: format_string + type: keyword + - name: mach_timestamp + type: double + - name: message_type + type: keyword + - name: parent_activity_identifier + type: keyword + - name: process + type: group + fields: + - name: id + type: long + - name: image_path + type: keyword + - name: image_uuid + type: keyword + - name: sender + type: group + fields: + - name: image_path + type: keyword + - name: image_uuid + type: keyword + - name: program_counter + type: long + - name: source + type: keyword + - name: subsystem + type: keyword + - name: thread_id + type: long + - name: timestamp + type: date + - name: timezone_name + type: keyword + - name: trace_id + type: keyword + - name: user_id + type: keyword diff --git a/packages/macos/data_stream/authentication/manifest.yml b/packages/macos/data_stream/authentication/manifest.yml new file mode 100644 index 00000000000..2e3f337f49d --- /dev/null +++ b/packages/macos/data_stream/authentication/manifest.yml @@ -0,0 +1,139 @@ +title: Authentication +type: logs +streams: + - input: unifiedlogs + title: Authentication logs + description: Collect Authentication unified logs from macOS. + template_path: unifiedlogs.yml.hbs + enabled: false + vars: + - name: predicate + type: text + title: Predicate + description: | + Filters messages using the provided predicate based on NSPredicate. + A compound predicate or multiple predicates can be provided as a list. + For detailed information on the use of predicate based filtering, + please refer to the https://developer.apple.com/library/mac/documentation/Cocoa/Conceptual/Predicates/Articles/pSyntax.html[Predicate Programming Guide]. + multi: true + show_user: true + - name: start + type: text + title: Start date + description: | + Shows content starting from the provided date. + The following date/time formats are accepted: + `YYYY-MM-DD`, `YYYY-MM-DD HH:MM:SS`, `YYYY-MM-DD HH:MM:SSZZZZZ`. + show_user: false + - name: end + type: text + title: End date + description: | + Shows content up to the provided date. + The following date/time formats are accepted: + `YYYY-MM-DD`, `YYYY-MM-DD HH:MM:SS`, `YYYY-MM-DD HH:MM:SSZZZZZ`. + show_user: false + - name: process + type: text + title: Process + description: | + A list of the processes on which to operate. It accepts a PID or process name. + multi: true + show_user: false + - name: source + type: bool + title: Include source + description: | + Include symbol names and source line numbers for messages, if available. + show_user: false + - name: info + type: bool + title: Include info + description: | + Disable or enable info level messages. + show_user: false + - name: debug + type: bool + title: Include debug + description: | + Disable or enable debug level messages. + show_user: false + - name: backtrace + type: bool + title: Include backtrace + description: | + Disable or enable backtrace level messages. + show_user: false + - name: signpost + type: bool + title: Include signpost + description: | + Disable or enable signpost level messages. + show_user: false + - name: unreliable + type: bool + title: Annotate unreliable + description: | + Annotate events with whether the log was emitted unreliably. + show_user: false + - name: mach_continuous_time + type: bool + title: Use mach continuous time + description: | + Use mach continuous time timestamps rather than walltime. + show_user: false + - name: backfill + type: bool + title: Must backfill + description: | + If set to true the input will process all available logs since the beginning + of time the first time it starts. + show_user: false + - name: archive_file + type: text + title: Archive file + description: | + Display events stored in the given archive. + The archive must be a valid log archive bundle with the suffix `.logarchive`. + show_user: false + - name: trace_file + type: text + title: Trace file + description: | + Display events stored in the given `.tracev3` file. + In order to be decoded, the file must be contained within a valid `.logarchive`. + show_user: false + - name: preserve_original_event + type: bool + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field event.original. + multi: false + required: false + show_user: true + default: false + - name: tags + type: text + title: Tags + description: Tags for the data-stream. + multi: true + required: true + show_user: false + default: + - forwarded + - macos-authentication + - name: preserve_duplicate_custom_fields + required: false + title: Preserve duplicate custom fields + description: Preserve macos.* fields that were copied to Elastic Common Schema (ECS) fields. + type: bool + multi: false + show_user: false + default: false + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: >- + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. diff --git a/packages/macos/data_stream/file_read_write/_dev/test/pipeline/test-common-config.yml b/packages/macos/data_stream/file_read_write/_dev/test/pipeline/test-common-config.yml new file mode 100644 index 00000000000..37e8fa225fd --- /dev/null +++ b/packages/macos/data_stream/file_read_write/_dev/test/pipeline/test-common-config.yml @@ -0,0 +1,3 @@ +fields: + tags: + - preserve_duplicate_custom_fields diff --git a/packages/macos/data_stream/file_read_write/_dev/test/pipeline/test-file.log b/packages/macos/data_stream/file_read_write/_dev/test/pipeline/test-file.log new file mode 100644 index 00000000000..ed9968e1e5e --- /dev/null +++ b/packages/macos/data_stream/file_read_write/_dev/test/pipeline/test-file.log @@ -0,0 +1,2 @@ +{"timezoneName":"","messageType":"Error","eventType":"logEvent","source":null,"formatString":"Failed to read persistence records: %{public}s","userID":501,"activityIdentifier":634774,"subsystem":"com.apple.ManagedSettings","category":"persistenceRecordStore","threadID":319557,"senderImageUUID":"CF2A3610-D17B-3407-835F-2402AF4DF167","backtrace":{"frames":[{"imageOffset":220631,"imageUUID":"CF2A3610-D17B-3407-835F-2402AF4DF167"}]},"bootUUID":"218031E6-E47F-4A77-B7FC-5A57B049F4BC","processImagePath":"\/System\/Library\/Frameworks\/ManagedSettings.framework\/Versions\/A\/ManagedSettingsAgent","senderImagePath":"\/System\/Library\/Frameworks\/ManagedSettings.framework\/Versions\/A\/ManagedSettingsAgent","timestamp":"2025-10-06 11:53:43.295485+0530","machTimestamp":670044832721072,"eventMessage":"Failed to read persistence records: Error Domain=NSCocoaErrorDomain Code=260 \"The file \u201cSettingRecords.plist\u201d couldn\u2019t be opened because there is no such file.\" UserInfo={NSFilePath=\/var\/folders\/ng\/xjv8ybg15hddhntf5wr2bykc0000gn\/0\/com.apple.ScreenTimeAgent\/Store\/Library\/com.apple.ManagedSettings\/Records\/SettingRecords.plist, NSUnderlyingError=0x7fcb09812780 {Error Domain=NSPOSIXErrorDomain Code=2 \"No such file or directory\"}}","processImageUUID":"CF2A3610-D17B-3407-835F-2402AF4DF167","traceID":1622535598968836,"processID":1152,"senderProgramCounter":220631,"parentActivityIdentifier":0} +{"timezoneName":"","messageType":"Error","eventType":"logEvent","source":null,"formatString":"Couldn't open parent path due to %{darwin.errno}d","userID":205,"activityIdentifier":622376,"subsystem":"com.apple.defaults","category":"cfprefsd","threadID":232247,"senderImageUUID":"FEDAF68C-F484-3FCA-8866-A9E7E46CE7B6","backtrace":{"frames":[{"imageOffset":1929005,"imageUUID":"FEDAF68C-F484-3FCA-8866-A9E7E46CE7B6"}]},"bootUUID":"218031E6-E47F-4A77-B7FC-5A57B049F4BC","processImagePath":"\/usr\/sbin\/cfprefsd","senderImagePath":"\/System\/Library\/Frameworks\/CoreFoundation.framework\/Versions\/A\/CoreFoundation","timestamp":"2025-10-06 11:54:36.836304+0530","machTimestamp":670098373540004,"eventMessage":"Couldn't open parent path due to [2: No such file or directory]","processImageUUID":"04C516B8-C8E5-30EF-AC49-1631528F5645","traceID":35846827878453252,"processID":314,"senderProgramCounter":1929005,"parentActivityIdentifier":0} diff --git a/packages/macos/data_stream/file_read_write/_dev/test/pipeline/test-file.log-expected.json b/packages/macos/data_stream/file_read_write/_dev/test/pipeline/test-file.log-expected.json new file mode 100644 index 00000000000..412e79458d4 --- /dev/null +++ b/packages/macos/data_stream/file_read_write/_dev/test/pipeline/test-file.log-expected.json @@ -0,0 +1,158 @@ +{ + "expected": [ + { + "@timestamp": "2025-10-06T06:23:43.295Z", + "ecs": { + "version": "9.2.0" + }, + "event": { + "category": [ + "file" + ], + "kind": "event", + "original": "{\"timezoneName\":\"\",\"messageType\":\"Error\",\"eventType\":\"logEvent\",\"source\":null,\"formatString\":\"Failed to read persistence records: %{public}s\",\"userID\":501,\"activityIdentifier\":634774,\"subsystem\":\"com.apple.ManagedSettings\",\"category\":\"persistenceRecordStore\",\"threadID\":319557,\"senderImageUUID\":\"CF2A3610-D17B-3407-835F-2402AF4DF167\",\"backtrace\":{\"frames\":[{\"imageOffset\":220631,\"imageUUID\":\"CF2A3610-D17B-3407-835F-2402AF4DF167\"}]},\"bootUUID\":\"218031E6-E47F-4A77-B7FC-5A57B049F4BC\",\"processImagePath\":\"\\/System\\/Library\\/Frameworks\\/ManagedSettings.framework\\/Versions\\/A\\/ManagedSettingsAgent\",\"senderImagePath\":\"\\/System\\/Library\\/Frameworks\\/ManagedSettings.framework\\/Versions\\/A\\/ManagedSettingsAgent\",\"timestamp\":\"2025-10-06 11:53:43.295485+0530\",\"machTimestamp\":670044832721072,\"eventMessage\":\"Failed to read persistence records: Error Domain=NSCocoaErrorDomain Code=260 \\\"The file \\u201cSettingRecords.plist\\u201d couldn\\u2019t be opened because there is no such file.\\\" UserInfo={NSFilePath=\\/var\\/folders\\/ng\\/xjv8ybg15hddhntf5wr2bykc0000gn\\/0\\/com.apple.ScreenTimeAgent\\/Store\\/Library\\/com.apple.ManagedSettings\\/Records\\/SettingRecords.plist, NSUnderlyingError=0x7fcb09812780 {Error Domain=NSPOSIXErrorDomain Code=2 \\\"No such file or directory\\\"}}\",\"processImageUUID\":\"CF2A3610-D17B-3407-835F-2402AF4DF167\",\"traceID\":1622535598968836,\"processID\":1152,\"senderProgramCounter\":220631,\"parentActivityIdentifier\":0}", + "type": [ + "info" + ] + }, + "log": { + "level": "error" + }, + "macos": { + "activity_identifier": "634774", + "backtrace": { + "frames": [ + { + "image": { + "offset": "220631", + "uuid": "CF2A3610-D17B-3407-835F-2402AF4DF167" + } + } + ] + }, + "boot_uuid": "218031E6-E47F-4A77-B7FC-5A57B049F4BC", + "category": "persistenceRecordStore", + "event": { + "message": { + "description": "Failed to read persistence records: Error Domain=NSCocoaErrorDomain Code=260 \"The file “SettingRecords.plist” couldn’t be opened because there is no such file.\" UserInfo={NSFilePath=/var/folders/ng/xjv8ybg15hddhntf5wr2bykc0000gn/0/com.apple.ScreenTimeAgent/Store/Library/com.apple.ManagedSettings/Records/SettingRecords.plist, NSUnderlyingError=0x7fcb09812780 {Error Domain=NSPOSIXErrorDomain Code=2 \"No such file or directory\"}}" + }, + "type": "logEvent" + }, + "format_string": "Failed to read persistence records: %{public}s", + "mach_timestamp": 6.70044832721072E14, + "message_type": "Error", + "parent_activity_identifier": "0", + "process": { + "id": 1152, + "image_path": "/System/Library/Frameworks/ManagedSettings.framework/Versions/A/ManagedSettingsAgent", + "image_uuid": "CF2A3610-D17B-3407-835F-2402AF4DF167" + }, + "sender": { + "image_path": "/System/Library/Frameworks/ManagedSettings.framework/Versions/A/ManagedSettingsAgent", + "image_uuid": "CF2A3610-D17B-3407-835F-2402AF4DF167", + "program_counter": 220631 + }, + "subsystem": "com.apple.ManagedSettings", + "thread_id": 319557, + "timestamp": "2025-10-06T06:23:43.295Z", + "trace_id": "1622535598968836", + "user_id": "501" + }, + "message": "Failed to read persistence records: Error Domain=NSCocoaErrorDomain Code=260 \"The file “SettingRecords.plist” couldn’t be opened because there is no such file.\" UserInfo={NSFilePath=/var/folders/ng/xjv8ybg15hddhntf5wr2bykc0000gn/0/com.apple.ScreenTimeAgent/Store/Library/com.apple.ManagedSettings/Records/SettingRecords.plist, NSUnderlyingError=0x7fcb09812780 {Error Domain=NSPOSIXErrorDomain Code=2 \"No such file or directory\"}}", + "process": { + "pid": 1152, + "thread": { + "id": 319557 + } + }, + "related": { + "user": [ + "501" + ] + }, + "tags": [ + "preserve_duplicate_custom_fields" + ], + "user": { + "id": "501" + } + }, + { + "@timestamp": "2025-10-06T06:24:36.836Z", + "ecs": { + "version": "9.2.0" + }, + "event": { + "category": [ + "file" + ], + "kind": "event", + "original": "{\"timezoneName\":\"\",\"messageType\":\"Error\",\"eventType\":\"logEvent\",\"source\":null,\"formatString\":\"Couldn't open parent path due to %{darwin.errno}d\",\"userID\":205,\"activityIdentifier\":622376,\"subsystem\":\"com.apple.defaults\",\"category\":\"cfprefsd\",\"threadID\":232247,\"senderImageUUID\":\"FEDAF68C-F484-3FCA-8866-A9E7E46CE7B6\",\"backtrace\":{\"frames\":[{\"imageOffset\":1929005,\"imageUUID\":\"FEDAF68C-F484-3FCA-8866-A9E7E46CE7B6\"}]},\"bootUUID\":\"218031E6-E47F-4A77-B7FC-5A57B049F4BC\",\"processImagePath\":\"\\/usr\\/sbin\\/cfprefsd\",\"senderImagePath\":\"\\/System\\/Library\\/Frameworks\\/CoreFoundation.framework\\/Versions\\/A\\/CoreFoundation\",\"timestamp\":\"2025-10-06 11:54:36.836304+0530\",\"machTimestamp\":670098373540004,\"eventMessage\":\"Couldn't open parent path due to [2: No such file or directory]\",\"processImageUUID\":\"04C516B8-C8E5-30EF-AC49-1631528F5645\",\"traceID\":35846827878453252,\"processID\":314,\"senderProgramCounter\":1929005,\"parentActivityIdentifier\":0}", + "type": [ + "info" + ] + }, + "log": { + "level": "error" + }, + "macos": { + "activity_identifier": "622376", + "backtrace": { + "frames": [ + { + "image": { + "offset": "1929005", + "uuid": "FEDAF68C-F484-3FCA-8866-A9E7E46CE7B6" + } + } + ] + }, + "boot_uuid": "218031E6-E47F-4A77-B7FC-5A57B049F4BC", + "category": "cfprefsd", + "event": { + "message": { + "description": "Couldn't open parent path due to [2: No such file or directory]" + }, + "type": "logEvent" + }, + "format_string": "Couldn't open parent path due to %{darwin.errno}d", + "mach_timestamp": 6.70098373540004E14, + "message_type": "Error", + "parent_activity_identifier": "0", + "process": { + "id": 314, + "image_path": "/usr/sbin/cfprefsd", + "image_uuid": "04C516B8-C8E5-30EF-AC49-1631528F5645" + }, + "sender": { + "image_path": "/System/Library/Frameworks/CoreFoundation.framework/Versions/A/CoreFoundation", + "image_uuid": "FEDAF68C-F484-3FCA-8866-A9E7E46CE7B6", + "program_counter": 1929005 + }, + "subsystem": "com.apple.defaults", + "thread_id": 232247, + "timestamp": "2025-10-06T06:24:36.836Z", + "trace_id": "35846827878453252", + "user_id": "205" + }, + "message": "Couldn't open parent path due to [2: No such file or directory]", + "process": { + "pid": 314, + "thread": { + "id": 232247 + } + }, + "related": { + "user": [ + "205" + ] + }, + "tags": [ + "preserve_duplicate_custom_fields" + ], + "user": { + "id": "205" + } + } + ] +} diff --git a/packages/macos/data_stream/file_read_write/agent/stream/unifiedlogs.yml.hbs b/packages/macos/data_stream/file_read_write/agent/stream/unifiedlogs.yml.hbs new file mode 100644 index 00000000000..696a14dca94 --- /dev/null +++ b/packages/macos/data_stream/file_read_write/agent/stream/unifiedlogs.yml.hbs @@ -0,0 +1,49 @@ +predicate: +- '(eventMessage CONTAINS "open" OR eventMessage CONTAINS "write" OR eventMessage CONTAINS "unlink" OR eventMessage CONTAINS "rename") AND ((processImagePath BEGINSWITH "/System") OR (processImagePath BEGINSWITH "/bin") OR (processImagePath BEGINSWITH "/sbin") OR (processImagePath BEGINSWITH "/usr" AND NOT processImagePath BEGINSWITH "/usr/local") OR (processImagePath BEGINSWITH "/etc"))' +- 'subsystem == "com.apple.quarantine" OR eventMessage CONTAINS "com.apple.quarantine"' +{{#each predicate as |p|}} +- {{p}} +{{/each}} +{{#if start}} +start: {{start}} +{{/if}} +{{#if end}} +end: {{end}} +{{/if}} +{{#if process}} +process: +{{#each process as |p|}} +- {{p}} +{{/each}} +{{/if}} +source: {{source}} +info: {{info}} +debug: {{debug}} +backtrace: {{backtrace}} +signpost: {{signpost}} +unreliable: {{unreliable}} +mach_continuous_time: {{mach_continuous_time}} +backfill: {{backfill}} +{{#if archive_file}} +archive_file: {{archive_file}} +{{/if}} +{{#if trace_file}} +trace_file: {{trace_file}} +{{/if}} +tags: +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} +{{#if preserve_duplicate_custom_fields}} + - preserve_duplicate_custom_fields +{{/if}} +{{#each tags as |tag|}} + - {{tag}} +{{/each}} +{{#contains "forwarded" tags}} +publisher_pipeline.disable_host: true +{{/contains}} +{{#if processors}} +processors: +{{processors}} +{{/if}} diff --git a/packages/macos/data_stream/file_read_write/elasticsearch/ingest_pipeline/common-pipeline.yml.link b/packages/macos/data_stream/file_read_write/elasticsearch/ingest_pipeline/common-pipeline.yml.link new file mode 100644 index 00000000000..f146557ce04 --- /dev/null +++ b/packages/macos/data_stream/file_read_write/elasticsearch/ingest_pipeline/common-pipeline.yml.link @@ -0,0 +1 @@ +../../../../_dev/shared/common-pipeline.yml 6e7bb697288eb24aeb67d4950a8a3f38d89eb89f559dc01e605e543162267ddf \ No newline at end of file diff --git a/packages/macos/data_stream/file_read_write/elasticsearch/ingest_pipeline/default.yml b/packages/macos/data_stream/file_read_write/elasticsearch/ingest_pipeline/default.yml new file mode 100644 index 00000000000..530f06f6017 --- /dev/null +++ b/packages/macos/data_stream/file_read_write/elasticsearch/ingest_pipeline/default.yml @@ -0,0 +1,13 @@ +--- +description: Pipeline for processing file read write logs. +processors: + - append: + field: event.category + tag: set_event_category + value: file + - append: + field: event.type + tag: append_info_into_event_type_in_file + value: info + - pipeline: + name: '{{ IngestPipeline "common-pipeline" }}' \ No newline at end of file diff --git a/packages/macos/data_stream/file_read_write/fields/base-fields.yml b/packages/macos/data_stream/file_read_write/fields/base-fields.yml new file mode 100644 index 00000000000..31386a06ff1 --- /dev/null +++ b/packages/macos/data_stream/file_read_write/fields/base-fields.yml @@ -0,0 +1,16 @@ +- name: data_stream.type + external: ecs +- name: data_stream.dataset + external: ecs +- name: data_stream.namespace + external: ecs +- name: event.module + type: constant_keyword + external: ecs + value: macos +- name: event.dataset + type: constant_keyword + external: ecs + value: macos.file_read_write +- name: '@timestamp' + external: ecs diff --git a/packages/macos/data_stream/file_read_write/fields/beats.yml b/packages/macos/data_stream/file_read_write/fields/beats.yml new file mode 100644 index 00000000000..d5fd38748ba --- /dev/null +++ b/packages/macos/data_stream/file_read_write/fields/beats.yml @@ -0,0 +1,6 @@ +- name: input.type + type: keyword + description: Type of Filebeat input. +- name: log.offset + type: long + description: Log offset. diff --git a/packages/macos/data_stream/file_read_write/fields/fields.yml b/packages/macos/data_stream/file_read_write/fields/fields.yml new file mode 100644 index 00000000000..53e33cf5c76 --- /dev/null +++ b/packages/macos/data_stream/file_read_write/fields/fields.yml @@ -0,0 +1,74 @@ +- name: macos + type: group + fields: + - name: activity_identifier + type: keyword + - name: backtrace + type: group + fields: + - name: frames + type: group + fields: + - name: image + type: group + fields: + - name: offset + type: keyword + - name: uuid + type: keyword + - name: boot_uuid + type: keyword + - name: category + type: keyword + - name: event + type: group + fields: + - name: category + type: keyword + - name: message + type: group + fields: + - name: description + type: keyword + - name: type + type: keyword + - name: format_string + type: keyword + - name: mach_timestamp + type: double + - name: message_type + type: keyword + - name: parent_activity_identifier + type: keyword + - name: process + type: group + fields: + - name: id + type: long + - name: image_path + type: keyword + - name: image_uuid + type: keyword + - name: sender + type: group + fields: + - name: image_path + type: keyword + - name: image_uuid + type: keyword + - name: program_counter + type: long + - name: source + type: keyword + - name: subsystem + type: keyword + - name: thread_id + type: long + - name: timestamp + type: date + - name: timezone_name + type: keyword + - name: trace_id + type: keyword + - name: user_id + type: keyword diff --git a/packages/macos/data_stream/file_read_write/manifest.yml b/packages/macos/data_stream/file_read_write/manifest.yml new file mode 100644 index 00000000000..70b5c17f35b --- /dev/null +++ b/packages/macos/data_stream/file_read_write/manifest.yml @@ -0,0 +1,139 @@ +title: File Read/Write +type: logs +streams: + - input: unifiedlogs + title: File Read/Write logs + description: Collect File Read/Write logs from macOS. + template_path: unifiedlogs.yml.hbs + enabled: false + vars: + - name: predicate + type: text + title: Predicate + description: | + Filters messages using the provided predicate based on NSPredicate. + A compound predicate or multiple predicates can be provided as a list. + For detailed information on the use of predicate based filtering, + please refer to the https://developer.apple.com/library/mac/documentation/Cocoa/Conceptual/Predicates/Articles/pSyntax.html[Predicate Programming Guide]. + multi: true + show_user: true + - name: start + type: text + title: Start date + description: | + Shows content starting from the provided date. + The following date/time formats are accepted: + `YYYY-MM-DD`, `YYYY-MM-DD HH:MM:SS`, `YYYY-MM-DD HH:MM:SSZZZZZ`. + show_user: false + - name: end + type: text + title: End date + description: | + Shows content up to the provided date. + The following date/time formats are accepted: + `YYYY-MM-DD`, `YYYY-MM-DD HH:MM:SS`, `YYYY-MM-DD HH:MM:SSZZZZZ`. + show_user: false + - name: process + type: text + title: Process + description: | + A list of the processes on which to operate. It accepts a PID or process name. + multi: true + show_user: false + - name: source + type: bool + title: Include source + description: | + Include symbol names and source line numbers for messages, if available. + show_user: false + - name: info + type: bool + title: Include info + description: | + Disable or enable info level messages. + show_user: false + - name: debug + type: bool + title: Include debug + description: | + Disable or enable debug level messages. + show_user: false + - name: backtrace + type: bool + title: Include backtrace + description: | + Disable or enable backtrace level messages. + show_user: false + - name: signpost + type: bool + title: Include signpost + description: | + Disable or enable signpost level messages. + show_user: false + - name: unreliable + type: bool + title: Annotate unreliable + description: | + Annotate events with whether the log was emitted unreliably. + show_user: false + - name: mach_continuous_time + type: bool + title: Use mach continuous time + description: | + Use mach continuous time timestamps rather than walltime. + show_user: false + - name: backfill + type: bool + title: Must backfill + description: | + If set to true the input will process all available logs since the beginning + of time the first time it starts. + show_user: false + - name: archive_file + type: text + title: Archive file + description: | + Display events stored in the given archive. + The archive must be a valid log archive bundle with the suffix `.logarchive`. + show_user: false + - name: trace_file + type: text + title: Trace file + description: | + Display events stored in the given `.tracev3` file. + In order to be decoded, the file must be contained within a valid `.logarchive`. + show_user: false + - name: preserve_original_event + type: bool + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field event.original. + multi: false + required: false + show_user: true + default: false + - name: tags + type: text + title: Tags + description: Tags for the data-stream. + multi: true + required: true + show_user: false + default: + - forwarded + - macos-file_read_write + - name: preserve_duplicate_custom_fields + required: false + title: Preserve duplicate custom fields + description: Preserve macos.* fields that were copied to Elastic Common Schema (ECS) fields. + type: bool + multi: false + show_user: false + default: false + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: >- + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. diff --git a/packages/macos/data_stream/network_activity/_dev/test/pipeline/test-common-config.yml b/packages/macos/data_stream/network_activity/_dev/test/pipeline/test-common-config.yml new file mode 100644 index 00000000000..4da22641654 --- /dev/null +++ b/packages/macos/data_stream/network_activity/_dev/test/pipeline/test-common-config.yml @@ -0,0 +1,3 @@ +fields: + tags: + - preserve_original_event diff --git a/packages/macos/data_stream/network_activity/_dev/test/pipeline/test-network-activity.log b/packages/macos/data_stream/network_activity/_dev/test/pipeline/test-network-activity.log new file mode 100644 index 00000000000..b46378f1d20 --- /dev/null +++ b/packages/macos/data_stream/network_activity/_dev/test/pipeline/test-network-activity.log @@ -0,0 +1,4 @@ +{"timezoneName":"","messageType":"Default","eventType":"logEvent","source":null,"formatString":"[C%u %{public,uuid_t}.16P %{public}s %{public}@] cancelled\n\t[C%{public}s %{public,uuid_t}.16P %{public}s<->%{public}s]\n\tConnected Path: %{public}@\n\tPrivacy Stance: %{public}s\n\tDuration: %u.us, DNS @%u.us took %u.us, %{public}s @%u.us took %u.us, %{public}s took %u.us\n\tbytes in\/out: %llu\/%llu, packets in\/out: %llu\/%llu, rtt: %u.us, retransmitted bytes: %llu, out-of-order bytes: %llu\n\tecn packets sent\/acked\/marked\/lost: %llu\/%llu\/%llu\/%llu","userID":242,"activityIdentifier":0,"subsystem":"com.apple.network","category":"connection","threadID":1132574,"senderImageUUID":"E481CDB0-9C0C-3BBC-87FA-006C66D3223E","backtrace":{"frames":[{"imageOffset":9418645,"imageUUID":"E481CDB0-9C0C-3BBC-87FA-006C66D3223E"}]},"bootUUID":"218031E6-E47F-4A77-B7FC-5A57B049F4BC","processImagePath":"\/usr\/libexec\/nsurlsessiond","senderImagePath":"\/System\/Library\/Frameworks\/Network.framework\/Versions\/A\/Network","timestamp":"2025-10-15 16:21:27.886344+0530","machTimestamp":739118933716566,"eventMessage":"[C348 90B95967-CD66-4EDD-AD9E-5BA99B3B1C62 Hostname#ea5d1e75:443 quic-connection, bundle id: com.apple.mobileassetd.client.auto-asset-client, url hash: 69b0e59d, traffic class: 100, expected workload: 237568, no expensive, attribution: developer] cancelled\n\t[C348.1.1.1 A8214DA1-14D6-4867-BEAC-F70599E6E765 172.20.1.254:64626<->IPv4#3ec082bd:443]\n\tConnected Path: satisfied (Path is satisfied), viable, interface: en0[802.11], ipv4, dns, uses wifi\n\tPrivacy Stance: Not Eligible\n\tDuration: 2.893s, DNS @0.001s took 0.045s, TCP @0.050s took 0.047s, TLS 1.3 took 0.348s\n\tbytes in\/out: 248482\/1060, packets in\/out: 75\/18, rtt: 0.048s, retransmitted bytes: 0, out-of-order bytes: 0\n\tecn packets sent\/acked\/marked\/lost: 0\/0\/0\/0","processImageUUID":"5CF1C24C-80AC-35A6-A065-986216BBD0B2","traceID":589370805553725444,"processID":198,"senderProgramCounter":9418645,"parentActivityIdentifier":0} +{"timezoneName":"","messageType":"Default","eventType":"logEvent","source":null,"formatString":"[C%u %{public,uuid_t}.16P %{public}s %{public}@] cancelled\n\t[C%{public}s %{public,uuid_t}.16P %{public}s<->%{public}s]\n\tConnected Path: %{public}@\n\tPrivacy Stance: %{public}s\n\tDuration: %u.us, DNS @%u.us took %u.us, %{public}s @%u.us took %u.us, %{public}s took %u.us\n\tbytes in\/out: %llu\/%llu, packets in\/out: %llu\/%llu, rtt: %u.us, retransmitted bytes: %llu, out-of-order bytes: %llu\n\tecn packets sent\/acked\/marked\/lost: %llu\/%llu\/%llu\/%llu","userID":200,"activityIdentifier":0,"subsystem":"com.apple.network","category":"connection","threadID":1134682,"senderImageUUID":"E481CDB0-9C0C-3BBC-87FA-006C66D3223E","backtrace":{"frames":[{"imageOffset":9418645,"imageUUID":"E481CDB0-9C0C-3BBC-87FA-006C66D3223E"}]},"bootUUID":"218031E6-E47F-4A77-B7FC-5A57B049F4BC","processImagePath":"\/System\/Library\/CoreServices\/Software Update.app\/Contents\/Resources\/softwareupdated","senderImagePath":"\/System\/Library\/Frameworks\/Network.framework\/Versions\/A\/Network","timestamp":"2025-10-15 16:24:32.487263+0530","machTimestamp":739303534635293,"eventMessage":"[C137 5CADEB1D-0A08-4B41-B92D-B89BF308083E Hostname#3fba6d68:443 tcp, url hash: 15f2f095, tls, definite, attribution: developer] cancelled\n\t[C137.1.1 C84B1419-27A0-421C-8140-8478E1AC40F3 172.20.1.254:64673<->IPv4#14ad2c93:443]\n\tConnected Path: satisfied (Path is satisfied), viable, interface: en0[802.11], ipv4, dns, uses wifi\n\tPrivacy Stance: Not Eligible\n\tDuration: 60.046s, DNS @0.000s took 0.045s, TCP @0.047s took 0.048s, TLS 1.3 took 0.151s\n\tbytes in\/out: 515210\/1012, packets in\/out: 111\/62, rtt: 0.135s, retransmitted bytes: 0, out-of-order bytes: 1242\n\tecn packets sent\/acked\/marked\/lost: 0\/0\/0\/0","processImageUUID":"CD58B635-1F96-38BD-BF3B-DBA7CA293E28","traceID":589370805553725444,"processID":200,"senderProgramCounter":9418645,"parentActivityIdentifier":0} +{"timezoneName":"","messageType":"Default","eventType":"logEvent","source":null,"formatString":"[C%u %{public,uuid_t}.16P %{public}s %{public}@] cancelled\n\t[C%{public}s %{public,uuid_t}.16P %{public}s<->%{public}s]\n\tConnected Path: %{public}@\n\tPrivacy Stance: %{public}s\n\tDuration: %u.us, DNS @%u.us took %u.us, %{public}s @%u.us took %u.us, %{public}s took %u.us\n\tbytes in\/out: %llu\/%llu, packets in\/out: %llu\/%llu, rtt: %u.us, retransmitted bytes: %llu, out-of-order bytes: %llu\n\tecn packets sent\/acked\/marked\/lost: %llu\/%llu\/%llu\/%llu","userID":501,"activityIdentifier":0,"subsystem":"com.apple.network","category":"connection","threadID":1131741,"senderImageUUID":"E481CDB0-9C0C-3BBC-87FA-006C66D3223E","backtrace":{"frames":[{"imageOffset":9418645,"imageUUID":"E481CDB0-9C0C-3BBC-87FA-006C66D3223E"}]},"bootUUID":"218031E6-E47F-4A77-B7FC-5A57B049F4BC","processImagePath":"\/usr\/libexec\/nsurlsessiond","senderImagePath":"\/System\/Library\/Frameworks\/Network.framework\/Versions\/A\/Network","timestamp":"2025-10-15 16:20:16.541036+0530","machTimestamp":739047588408621,"eventMessage":"[C66 D699D319-32C3-4543-803A-DCB7ED36A5F8 Hostname#0d5adf07:443 quic-connection, bundle id: com.apple.triald, url hash: 9e856d3a, traffic class: 100, no expensive, attribution: developer] cancelled\n\t[C66.1.1.1 F8FA4862-B203-490F-866E-BEC10DBEF962 172.20.1.254:64603<->IPv4#10574dec:443]\n\tConnected Path: satisfied (Path is satisfied), viable, interface: en0[802.11], ipv4, dns, uses wifi\n\tPrivacy Stance: Not Eligible\n\tDuration: 2.549s, DNS @0.002s took 0.061s, TCP @0.067s took 0.016s, TLS 1.3 took 0.415s\n\tbytes in\/out: 5629\/3030, packets in\/out: 8\/16, rtt: 0.029s, retransmitted bytes: 0, out-of-order bytes: 0\n\tecn packets sent\/acked\/marked\/lost: 0\/0\/0\/0","processImageUUID":"5CF1C24C-80AC-35A6-A065-986216BBD0B2","traceID":589370805553725444,"processID":1100,"senderProgramCounter":9418645,"parentActivityIdentifier":0} +{"timezoneName":"","messageType":"Default","eventType":"logEvent","source":null,"formatString":"[%p] activating connection: mach=%{bool}d listener=%{bool}d peer=%{bool}d name=%{public}s","userID":200,"activityIdentifier":1606042,"subsystem":"com.apple.xpc","category":"connection","threadID":1131759,"senderImageUUID":"ABCE7E08-7D29-3BE0-AF60-F0DC752E951C","backtrace":{"frames":[{"imageOffset":43862,"imageUUID":"ABCE7E08-7D29-3BE0-AF60-F0DC752E951C"}]},"bootUUID":"218031E6-E47F-4A77-B7FC-5A57B049F4BC","processImagePath":"\/System\/Library\/CoreServices\/Software Update.app\/Contents\/Resources\/softwareupdated","senderImagePath":"\/usr\/lib\/system\/libxpc.dylib","timestamp":"2025-10-15 16:21:22.947210+0530","machTimestamp":739113994582741,"eventMessage":"[0x7fd20cffa690] activating connection: mach=false listener=false peer=false name=(anonymous)","processImageUUID":"CD58B635-1F96-38BD-BF3B-DBA7CA293E28","traceID":6727155769999364,"processID":200,"senderProgramCounter":43862,"parentActivityIdentifier":0} diff --git a/packages/macos/data_stream/network_activity/_dev/test/pipeline/test-network-activity.log-expected.json b/packages/macos/data_stream/network_activity/_dev/test/pipeline/test-network-activity.log-expected.json new file mode 100644 index 00000000000..bd5d5b96a99 --- /dev/null +++ b/packages/macos/data_stream/network_activity/_dev/test/pipeline/test-network-activity.log-expected.json @@ -0,0 +1,448 @@ +{ + "expected": [ + { + "@timestamp": "2025-10-15T10:51:27.886Z", + "destination": { + "bytes": 1060, + "packets": 18, + "port": 443 + }, + "ecs": { + "version": "9.2.0" + }, + "event": { + "category": [ + "network" + ], + "kind": "event", + "original": "{\"timezoneName\":\"\",\"messageType\":\"Default\",\"eventType\":\"logEvent\",\"source\":null,\"formatString\":\"[C%u %{public,uuid_t}.16P %{public}s %{public}@] cancelled\\n\\t[C%{public}s %{public,uuid_t}.16P %{public}s<->%{public}s]\\n\\tConnected Path: %{public}@\\n\\tPrivacy Stance: %{public}s\\n\\tDuration: %u.us, DNS @%u.us took %u.us, %{public}s @%u.us took %u.us, %{public}s took %u.us\\n\\tbytes in\\/out: %llu\\/%llu, packets in\\/out: %llu\\/%llu, rtt: %u.us, retransmitted bytes: %llu, out-of-order bytes: %llu\\n\\tecn packets sent\\/acked\\/marked\\/lost: %llu\\/%llu\\/%llu\\/%llu\",\"userID\":242,\"activityIdentifier\":0,\"subsystem\":\"com.apple.network\",\"category\":\"connection\",\"threadID\":1132574,\"senderImageUUID\":\"E481CDB0-9C0C-3BBC-87FA-006C66D3223E\",\"backtrace\":{\"frames\":[{\"imageOffset\":9418645,\"imageUUID\":\"E481CDB0-9C0C-3BBC-87FA-006C66D3223E\"}]},\"bootUUID\":\"218031E6-E47F-4A77-B7FC-5A57B049F4BC\",\"processImagePath\":\"\\/usr\\/libexec\\/nsurlsessiond\",\"senderImagePath\":\"\\/System\\/Library\\/Frameworks\\/Network.framework\\/Versions\\/A\\/Network\",\"timestamp\":\"2025-10-15 16:21:27.886344+0530\",\"machTimestamp\":739118933716566,\"eventMessage\":\"[C348 90B95967-CD66-4EDD-AD9E-5BA99B3B1C62 Hostname#ea5d1e75:443 quic-connection, bundle id: com.apple.mobileassetd.client.auto-asset-client, url hash: 69b0e59d, traffic class: 100, expected workload: 237568, no expensive, attribution: developer] cancelled\\n\\t[C348.1.1.1 A8214DA1-14D6-4867-BEAC-F70599E6E765 172.20.1.254:64626<->IPv4#3ec082bd:443]\\n\\tConnected Path: satisfied (Path is satisfied), viable, interface: en0[802.11], ipv4, dns, uses wifi\\n\\tPrivacy Stance: Not Eligible\\n\\tDuration: 2.893s, DNS @0.001s took 0.045s, TCP @0.050s took 0.047s, TLS 1.3 took 0.348s\\n\\tbytes in\\/out: 248482\\/1060, packets in\\/out: 75\\/18, rtt: 0.048s, retransmitted bytes: 0, out-of-order bytes: 0\\n\\tecn packets sent\\/acked\\/marked\\/lost: 0\\/0\\/0\\/0\",\"processImageUUID\":\"5CF1C24C-80AC-35A6-A065-986216BBD0B2\",\"traceID\":589370805553725444,\"processID\":198,\"senderProgramCounter\":9418645,\"parentActivityIdentifier\":0}", + "type": [ + "info" + ] + }, + "host": { + "id": "ea5d1e75" + }, + "log": { + "level": "info" + }, + "macos": { + "activity_identifier": "0", + "backtrace": { + "frames": [ + { + "image": { + "offset": "9418645", + "uuid": "E481CDB0-9C0C-3BBC-87FA-006C66D3223E" + } + } + ] + }, + "boot_uuid": "218031E6-E47F-4A77-B7FC-5A57B049F4BC", + "category": "connection", + "event": { + "message": { + "attribution": "developer", + "bundle_id": "com.apple.mobileassetd.client.auto-asset-client", + "connection_detail": "348.1.1.1", + "connection_id": "348", + "connection_uuid": "A8214DA1-14D6-4867-BEAC-F70599E6E765", + "description": "[C348 90B95967-CD66-4EDD-AD9E-5BA99B3B1C62 Hostname#ea5d1e75:443 quic-connection, bundle id: com.apple.mobileassetd.client.auto-asset-client, url hash: 69b0e59d, traffic class: 100, expected workload: 237568, no expensive, attribution: developer] cancelled\n\t[C348.1.1.1 A8214DA1-14D6-4867-BEAC-F70599E6E765 172.20.1.254:64626<->IPv4#3ec082bd:443]\n\tConnected Path: satisfied (Path is satisfied), viable, interface: en0[802.11], ipv4, dns, uses wifi\n\tPrivacy Stance: Not Eligible\n\tDuration: 2.893s, DNS @0.001s took 0.045s, TCP @0.050s took 0.047s, TLS 1.3 took 0.348s\n\tbytes in/out: 248482/1060, packets in/out: 75/18, rtt: 0.048s, retransmitted bytes: 0, out-of-order bytes: 0\n\tecn packets sent/acked/marked/lost: 0/0/0/0", + "dns_duration": "0.045s", + "dns_start": "0.001s", + "duration": "2.893s", + "ecn_acked": 0, + "ecn_lost": 0, + "ecn_marked": 0, + "ecn_sent": 0, + "expected_workload": "237568", + "hostname_port": 443, + "interface": "en0[802.11]", + "out_of_order_bytes": 0, + "path_status": "satisfied (Path is satisfied)", + "privacy_stance": "Not Eligible", + "retransmitted_bytes": 0, + "rtt": "0.048s", + "server_id": "3ec082bd", + "session_uuid": "90B95967-CD66-4EDD-AD9E-5BA99B3B1C62", + "tcp_duration": "0.047s", + "tcp_start": "0.050s", + "tls_duration": "0.348s", + "tls_version": "1.3", + "traffic_class": "100", + "url_hash": "69b0e59d" + }, + "type": "logEvent" + }, + "format_string": "[C%u %{public,uuid_t}.16P %{public}s %{public}@] cancelled\n\t[C%{public}s %{public,uuid_t}.16P %{public}s<->%{public}s]\n\tConnected Path: %{public}@\n\tPrivacy Stance: %{public}s\n\tDuration: %u.us, DNS @%u.us took %u.us, %{public}s @%u.us took %u.us, %{public}s took %u.us\n\tbytes in/out: %llu/%llu, packets in/out: %llu/%llu, rtt: %u.us, retransmitted bytes: %llu, out-of-order bytes: %llu\n\tecn packets sent/acked/marked/lost: %llu/%llu/%llu/%llu", + "mach_timestamp": 7.39118933716566E14, + "parent_activity_identifier": "0", + "process": { + "image_path": "/usr/libexec/nsurlsessiond", + "image_uuid": "5CF1C24C-80AC-35A6-A065-986216BBD0B2" + }, + "sender": { + "image_path": "/System/Library/Frameworks/Network.framework/Versions/A/Network", + "image_uuid": "E481CDB0-9C0C-3BBC-87FA-006C66D3223E", + "program_counter": 9418645 + }, + "subsystem": "com.apple.network", + "trace_id": "589370805553725444" + }, + "message": "[C348 90B95967-CD66-4EDD-AD9E-5BA99B3B1C62 Hostname#ea5d1e75:443 quic-connection, bundle id: com.apple.mobileassetd.client.auto-asset-client, url hash: 69b0e59d, traffic class: 100, expected workload: 237568, no expensive, attribution: developer] cancelled\n\t[C348.1.1.1 A8214DA1-14D6-4867-BEAC-F70599E6E765 172.20.1.254:64626<->IPv4#3ec082bd:443]\n\tConnected Path: satisfied (Path is satisfied), viable, interface: en0[802.11], ipv4, dns, uses wifi\n\tPrivacy Stance: Not Eligible\n\tDuration: 2.893s, DNS @0.001s took 0.045s, TCP @0.050s took 0.047s, TLS 1.3 took 0.348s\n\tbytes in/out: 248482/1060, packets in/out: 75/18, rtt: 0.048s, retransmitted bytes: 0, out-of-order bytes: 0\n\tecn packets sent/acked/marked/lost: 0/0/0/0", + "network": { + "bytes": 249542, + "packets": 93 + }, + "process": { + "pid": 198, + "thread": { + "id": 1132574 + } + }, + "related": { + "hosts": [ + "ea5d1e75" + ], + "ip": [ + "172.20.1.254" + ], + "user": [ + "242" + ] + }, + "source": { + "bytes": 248482, + "ip": "172.20.1.254", + "packets": 75, + "port": 64626 + }, + "tags": [ + "preserve_original_event" + ], + "user": { + "id": "242" + } + }, + { + "@timestamp": "2025-10-15T10:54:32.487Z", + "destination": { + "bytes": 1012, + "packets": 62, + "port": 443 + }, + "ecs": { + "version": "9.2.0" + }, + "event": { + "category": [ + "network" + ], + "kind": "event", + "original": "{\"timezoneName\":\"\",\"messageType\":\"Default\",\"eventType\":\"logEvent\",\"source\":null,\"formatString\":\"[C%u %{public,uuid_t}.16P %{public}s %{public}@] cancelled\\n\\t[C%{public}s %{public,uuid_t}.16P %{public}s<->%{public}s]\\n\\tConnected Path: %{public}@\\n\\tPrivacy Stance: %{public}s\\n\\tDuration: %u.us, DNS @%u.us took %u.us, %{public}s @%u.us took %u.us, %{public}s took %u.us\\n\\tbytes in\\/out: %llu\\/%llu, packets in\\/out: %llu\\/%llu, rtt: %u.us, retransmitted bytes: %llu, out-of-order bytes: %llu\\n\\tecn packets sent\\/acked\\/marked\\/lost: %llu\\/%llu\\/%llu\\/%llu\",\"userID\":200,\"activityIdentifier\":0,\"subsystem\":\"com.apple.network\",\"category\":\"connection\",\"threadID\":1134682,\"senderImageUUID\":\"E481CDB0-9C0C-3BBC-87FA-006C66D3223E\",\"backtrace\":{\"frames\":[{\"imageOffset\":9418645,\"imageUUID\":\"E481CDB0-9C0C-3BBC-87FA-006C66D3223E\"}]},\"bootUUID\":\"218031E6-E47F-4A77-B7FC-5A57B049F4BC\",\"processImagePath\":\"\\/System\\/Library\\/CoreServices\\/Software Update.app\\/Contents\\/Resources\\/softwareupdated\",\"senderImagePath\":\"\\/System\\/Library\\/Frameworks\\/Network.framework\\/Versions\\/A\\/Network\",\"timestamp\":\"2025-10-15 16:24:32.487263+0530\",\"machTimestamp\":739303534635293,\"eventMessage\":\"[C137 5CADEB1D-0A08-4B41-B92D-B89BF308083E Hostname#3fba6d68:443 tcp, url hash: 15f2f095, tls, definite, attribution: developer] cancelled\\n\\t[C137.1.1 C84B1419-27A0-421C-8140-8478E1AC40F3 172.20.1.254:64673<->IPv4#14ad2c93:443]\\n\\tConnected Path: satisfied (Path is satisfied), viable, interface: en0[802.11], ipv4, dns, uses wifi\\n\\tPrivacy Stance: Not Eligible\\n\\tDuration: 60.046s, DNS @0.000s took 0.045s, TCP @0.047s took 0.048s, TLS 1.3 took 0.151s\\n\\tbytes in\\/out: 515210\\/1012, packets in\\/out: 111\\/62, rtt: 0.135s, retransmitted bytes: 0, out-of-order bytes: 1242\\n\\tecn packets sent\\/acked\\/marked\\/lost: 0\\/0\\/0\\/0\",\"processImageUUID\":\"CD58B635-1F96-38BD-BF3B-DBA7CA293E28\",\"traceID\":589370805553725444,\"processID\":200,\"senderProgramCounter\":9418645,\"parentActivityIdentifier\":0}", + "type": [ + "info" + ] + }, + "host": { + "id": "3fba6d68" + }, + "log": { + "level": "info" + }, + "macos": { + "activity_identifier": "0", + "backtrace": { + "frames": [ + { + "image": { + "offset": "9418645", + "uuid": "E481CDB0-9C0C-3BBC-87FA-006C66D3223E" + } + } + ] + }, + "boot_uuid": "218031E6-E47F-4A77-B7FC-5A57B049F4BC", + "category": "connection", + "event": { + "message": { + "attribution": "developer", + "connection_detail": "137.1.1", + "connection_id": "137", + "connection_uuid": "C84B1419-27A0-421C-8140-8478E1AC40F3", + "description": "[C137 5CADEB1D-0A08-4B41-B92D-B89BF308083E Hostname#3fba6d68:443 tcp, url hash: 15f2f095, tls, definite, attribution: developer] cancelled\n\t[C137.1.1 C84B1419-27A0-421C-8140-8478E1AC40F3 172.20.1.254:64673<->IPv4#14ad2c93:443]\n\tConnected Path: satisfied (Path is satisfied), viable, interface: en0[802.11], ipv4, dns, uses wifi\n\tPrivacy Stance: Not Eligible\n\tDuration: 60.046s, DNS @0.000s took 0.045s, TCP @0.047s took 0.048s, TLS 1.3 took 0.151s\n\tbytes in/out: 515210/1012, packets in/out: 111/62, rtt: 0.135s, retransmitted bytes: 0, out-of-order bytes: 1242\n\tecn packets sent/acked/marked/lost: 0/0/0/0", + "dns_duration": "0.045s", + "dns_start": "0.000s", + "duration": "60.046s", + "ecn_acked": 0, + "ecn_lost": 0, + "ecn_marked": 0, + "ecn_sent": 0, + "hostname_port": 443, + "interface": "en0[802.11]", + "out_of_order_bytes": 1242, + "path_status": "satisfied (Path is satisfied)", + "privacy_stance": "Not Eligible", + "retransmitted_bytes": 0, + "rtt": "0.135s", + "server_id": "14ad2c93", + "session_uuid": "5CADEB1D-0A08-4B41-B92D-B89BF308083E", + "tcp_duration": "0.048s", + "tcp_start": "0.047s", + "tls_duration": "0.151s", + "tls_version": "1.3", + "url_hash": "15f2f095" + }, + "type": "logEvent" + }, + "format_string": "[C%u %{public,uuid_t}.16P %{public}s %{public}@] cancelled\n\t[C%{public}s %{public,uuid_t}.16P %{public}s<->%{public}s]\n\tConnected Path: %{public}@\n\tPrivacy Stance: %{public}s\n\tDuration: %u.us, DNS @%u.us took %u.us, %{public}s @%u.us took %u.us, %{public}s took %u.us\n\tbytes in/out: %llu/%llu, packets in/out: %llu/%llu, rtt: %u.us, retransmitted bytes: %llu, out-of-order bytes: %llu\n\tecn packets sent/acked/marked/lost: %llu/%llu/%llu/%llu", + "mach_timestamp": 7.39303534635293E14, + "parent_activity_identifier": "0", + "process": { + "image_path": "/System/Library/CoreServices/Software Update.app/Contents/Resources/softwareupdated", + "image_uuid": "CD58B635-1F96-38BD-BF3B-DBA7CA293E28" + }, + "sender": { + "image_path": "/System/Library/Frameworks/Network.framework/Versions/A/Network", + "image_uuid": "E481CDB0-9C0C-3BBC-87FA-006C66D3223E", + "program_counter": 9418645 + }, + "subsystem": "com.apple.network", + "trace_id": "589370805553725444" + }, + "message": "[C137 5CADEB1D-0A08-4B41-B92D-B89BF308083E Hostname#3fba6d68:443 tcp, url hash: 15f2f095, tls, definite, attribution: developer] cancelled\n\t[C137.1.1 C84B1419-27A0-421C-8140-8478E1AC40F3 172.20.1.254:64673<->IPv4#14ad2c93:443]\n\tConnected Path: satisfied (Path is satisfied), viable, interface: en0[802.11], ipv4, dns, uses wifi\n\tPrivacy Stance: Not Eligible\n\tDuration: 60.046s, DNS @0.000s took 0.045s, TCP @0.047s took 0.048s, TLS 1.3 took 0.151s\n\tbytes in/out: 515210/1012, packets in/out: 111/62, rtt: 0.135s, retransmitted bytes: 0, out-of-order bytes: 1242\n\tecn packets sent/acked/marked/lost: 0/0/0/0", + "network": { + "bytes": 516222, + "packets": 173 + }, + "process": { + "pid": 200, + "thread": { + "id": 1134682 + } + }, + "related": { + "hosts": [ + "3fba6d68" + ], + "ip": [ + "172.20.1.254" + ], + "user": [ + "200" + ] + }, + "source": { + "bytes": 515210, + "ip": "172.20.1.254", + "packets": 111, + "port": 64673 + }, + "tags": [ + "preserve_original_event" + ], + "user": { + "id": "200" + } + }, + { + "@timestamp": "2025-10-15T10:50:16.541Z", + "destination": { + "bytes": 3030, + "packets": 16, + "port": 443 + }, + "ecs": { + "version": "9.2.0" + }, + "event": { + "category": [ + "network" + ], + "kind": "event", + "original": "{\"timezoneName\":\"\",\"messageType\":\"Default\",\"eventType\":\"logEvent\",\"source\":null,\"formatString\":\"[C%u %{public,uuid_t}.16P %{public}s %{public}@] cancelled\\n\\t[C%{public}s %{public,uuid_t}.16P %{public}s<->%{public}s]\\n\\tConnected Path: %{public}@\\n\\tPrivacy Stance: %{public}s\\n\\tDuration: %u.us, DNS @%u.us took %u.us, %{public}s @%u.us took %u.us, %{public}s took %u.us\\n\\tbytes in\\/out: %llu\\/%llu, packets in\\/out: %llu\\/%llu, rtt: %u.us, retransmitted bytes: %llu, out-of-order bytes: %llu\\n\\tecn packets sent\\/acked\\/marked\\/lost: %llu\\/%llu\\/%llu\\/%llu\",\"userID\":501,\"activityIdentifier\":0,\"subsystem\":\"com.apple.network\",\"category\":\"connection\",\"threadID\":1131741,\"senderImageUUID\":\"E481CDB0-9C0C-3BBC-87FA-006C66D3223E\",\"backtrace\":{\"frames\":[{\"imageOffset\":9418645,\"imageUUID\":\"E481CDB0-9C0C-3BBC-87FA-006C66D3223E\"}]},\"bootUUID\":\"218031E6-E47F-4A77-B7FC-5A57B049F4BC\",\"processImagePath\":\"\\/usr\\/libexec\\/nsurlsessiond\",\"senderImagePath\":\"\\/System\\/Library\\/Frameworks\\/Network.framework\\/Versions\\/A\\/Network\",\"timestamp\":\"2025-10-15 16:20:16.541036+0530\",\"machTimestamp\":739047588408621,\"eventMessage\":\"[C66 D699D319-32C3-4543-803A-DCB7ED36A5F8 Hostname#0d5adf07:443 quic-connection, bundle id: com.apple.triald, url hash: 9e856d3a, traffic class: 100, no expensive, attribution: developer] cancelled\\n\\t[C66.1.1.1 F8FA4862-B203-490F-866E-BEC10DBEF962 172.20.1.254:64603<->IPv4#10574dec:443]\\n\\tConnected Path: satisfied (Path is satisfied), viable, interface: en0[802.11], ipv4, dns, uses wifi\\n\\tPrivacy Stance: Not Eligible\\n\\tDuration: 2.549s, DNS @0.002s took 0.061s, TCP @0.067s took 0.016s, TLS 1.3 took 0.415s\\n\\tbytes in\\/out: 5629\\/3030, packets in\\/out: 8\\/16, rtt: 0.029s, retransmitted bytes: 0, out-of-order bytes: 0\\n\\tecn packets sent\\/acked\\/marked\\/lost: 0\\/0\\/0\\/0\",\"processImageUUID\":\"5CF1C24C-80AC-35A6-A065-986216BBD0B2\",\"traceID\":589370805553725444,\"processID\":1100,\"senderProgramCounter\":9418645,\"parentActivityIdentifier\":0}", + "type": [ + "info" + ] + }, + "host": { + "id": "0d5adf07" + }, + "log": { + "level": "info" + }, + "macos": { + "activity_identifier": "0", + "backtrace": { + "frames": [ + { + "image": { + "offset": "9418645", + "uuid": "E481CDB0-9C0C-3BBC-87FA-006C66D3223E" + } + } + ] + }, + "boot_uuid": "218031E6-E47F-4A77-B7FC-5A57B049F4BC", + "category": "connection", + "event": { + "message": { + "attribution": "developer", + "bundle_id": "com.apple.triald", + "connection_detail": "66.1.1.1", + "connection_id": "66", + "connection_uuid": "F8FA4862-B203-490F-866E-BEC10DBEF962", + "description": "[C66 D699D319-32C3-4543-803A-DCB7ED36A5F8 Hostname#0d5adf07:443 quic-connection, bundle id: com.apple.triald, url hash: 9e856d3a, traffic class: 100, no expensive, attribution: developer] cancelled\n\t[C66.1.1.1 F8FA4862-B203-490F-866E-BEC10DBEF962 172.20.1.254:64603<->IPv4#10574dec:443]\n\tConnected Path: satisfied (Path is satisfied), viable, interface: en0[802.11], ipv4, dns, uses wifi\n\tPrivacy Stance: Not Eligible\n\tDuration: 2.549s, DNS @0.002s took 0.061s, TCP @0.067s took 0.016s, TLS 1.3 took 0.415s\n\tbytes in/out: 5629/3030, packets in/out: 8/16, rtt: 0.029s, retransmitted bytes: 0, out-of-order bytes: 0\n\tecn packets sent/acked/marked/lost: 0/0/0/0", + "dns_duration": "0.061s", + "dns_start": "0.002s", + "duration": "2.549s", + "ecn_acked": 0, + "ecn_lost": 0, + "ecn_marked": 0, + "ecn_sent": 0, + "hostname_port": 443, + "interface": "en0[802.11]", + "out_of_order_bytes": 0, + "path_status": "satisfied (Path is satisfied)", + "privacy_stance": "Not Eligible", + "retransmitted_bytes": 0, + "rtt": "0.029s", + "server_id": "10574dec", + "session_uuid": "D699D319-32C3-4543-803A-DCB7ED36A5F8", + "tcp_duration": "0.016s", + "tcp_start": "0.067s", + "tls_duration": "0.415s", + "tls_version": "1.3", + "traffic_class": "100", + "url_hash": "9e856d3a" + }, + "type": "logEvent" + }, + "format_string": "[C%u %{public,uuid_t}.16P %{public}s %{public}@] cancelled\n\t[C%{public}s %{public,uuid_t}.16P %{public}s<->%{public}s]\n\tConnected Path: %{public}@\n\tPrivacy Stance: %{public}s\n\tDuration: %u.us, DNS @%u.us took %u.us, %{public}s @%u.us took %u.us, %{public}s took %u.us\n\tbytes in/out: %llu/%llu, packets in/out: %llu/%llu, rtt: %u.us, retransmitted bytes: %llu, out-of-order bytes: %llu\n\tecn packets sent/acked/marked/lost: %llu/%llu/%llu/%llu", + "mach_timestamp": 7.39047588408621E14, + "parent_activity_identifier": "0", + "process": { + "image_path": "/usr/libexec/nsurlsessiond", + "image_uuid": "5CF1C24C-80AC-35A6-A065-986216BBD0B2" + }, + "sender": { + "image_path": "/System/Library/Frameworks/Network.framework/Versions/A/Network", + "image_uuid": "E481CDB0-9C0C-3BBC-87FA-006C66D3223E", + "program_counter": 9418645 + }, + "subsystem": "com.apple.network", + "trace_id": "589370805553725444" + }, + "message": "[C66 D699D319-32C3-4543-803A-DCB7ED36A5F8 Hostname#0d5adf07:443 quic-connection, bundle id: com.apple.triald, url hash: 9e856d3a, traffic class: 100, no expensive, attribution: developer] cancelled\n\t[C66.1.1.1 F8FA4862-B203-490F-866E-BEC10DBEF962 172.20.1.254:64603<->IPv4#10574dec:443]\n\tConnected Path: satisfied (Path is satisfied), viable, interface: en0[802.11], ipv4, dns, uses wifi\n\tPrivacy Stance: Not Eligible\n\tDuration: 2.549s, DNS @0.002s took 0.061s, TCP @0.067s took 0.016s, TLS 1.3 took 0.415s\n\tbytes in/out: 5629/3030, packets in/out: 8/16, rtt: 0.029s, retransmitted bytes: 0, out-of-order bytes: 0\n\tecn packets sent/acked/marked/lost: 0/0/0/0", + "network": { + "bytes": 8659, + "packets": 24 + }, + "process": { + "pid": 1100, + "thread": { + "id": 1131741 + } + }, + "related": { + "hosts": [ + "0d5adf07" + ], + "ip": [ + "172.20.1.254" + ], + "user": [ + "501" + ] + }, + "source": { + "bytes": 5629, + "ip": "172.20.1.254", + "packets": 8, + "port": 64603 + }, + "tags": [ + "preserve_original_event" + ], + "user": { + "id": "501" + } + }, + { + "@timestamp": "2025-10-15T10:51:22.947Z", + "ecs": { + "version": "9.2.0" + }, + "event": { + "category": [ + "network" + ], + "kind": "event", + "original": "{\"timezoneName\":\"\",\"messageType\":\"Default\",\"eventType\":\"logEvent\",\"source\":null,\"formatString\":\"[%p] activating connection: mach=%{bool}d listener=%{bool}d peer=%{bool}d name=%{public}s\",\"userID\":200,\"activityIdentifier\":1606042,\"subsystem\":\"com.apple.xpc\",\"category\":\"connection\",\"threadID\":1131759,\"senderImageUUID\":\"ABCE7E08-7D29-3BE0-AF60-F0DC752E951C\",\"backtrace\":{\"frames\":[{\"imageOffset\":43862,\"imageUUID\":\"ABCE7E08-7D29-3BE0-AF60-F0DC752E951C\"}]},\"bootUUID\":\"218031E6-E47F-4A77-B7FC-5A57B049F4BC\",\"processImagePath\":\"\\/System\\/Library\\/CoreServices\\/Software Update.app\\/Contents\\/Resources\\/softwareupdated\",\"senderImagePath\":\"\\/usr\\/lib\\/system\\/libxpc.dylib\",\"timestamp\":\"2025-10-15 16:21:22.947210+0530\",\"machTimestamp\":739113994582741,\"eventMessage\":\"[0x7fd20cffa690] activating connection: mach=false listener=false peer=false name=(anonymous)\",\"processImageUUID\":\"CD58B635-1F96-38BD-BF3B-DBA7CA293E28\",\"traceID\":6727155769999364,\"processID\":200,\"senderProgramCounter\":43862,\"parentActivityIdentifier\":0}", + "type": [ + "info" + ] + }, + "log": { + "level": "info" + }, + "macos": { + "activity_identifier": "1606042", + "backtrace": { + "frames": [ + { + "image": { + "offset": "43862", + "uuid": "ABCE7E08-7D29-3BE0-AF60-F0DC752E951C" + } + } + ] + }, + "boot_uuid": "218031E6-E47F-4A77-B7FC-5A57B049F4BC", + "category": "connection", + "event": { + "message": { + "description": "[0x7fd20cffa690] activating connection: mach=false listener=false peer=false name=(anonymous)", + "listener": false, + "mach": false, + "name": "(anonymous)", + "peer": false + }, + "type": "logEvent" + }, + "format_string": "[%p] activating connection: mach=%{bool}d listener=%{bool}d peer=%{bool}d name=%{public}s", + "mach_timestamp": 7.39113994582741E14, + "parent_activity_identifier": "0", + "process": { + "image_path": "/System/Library/CoreServices/Software Update.app/Contents/Resources/softwareupdated", + "image_uuid": "CD58B635-1F96-38BD-BF3B-DBA7CA293E28" + }, + "sender": { + "image_path": "/usr/lib/system/libxpc.dylib", + "image_uuid": "ABCE7E08-7D29-3BE0-AF60-F0DC752E951C", + "program_counter": 43862 + }, + "subsystem": "com.apple.xpc", + "trace_id": "6727155769999364" + }, + "message": "[0x7fd20cffa690] activating connection: mach=false listener=false peer=false name=(anonymous)", + "process": { + "pid": 200, + "thread": { + "id": 1131759 + } + }, + "related": { + "user": [ + "200" + ] + }, + "tags": [ + "preserve_original_event" + ], + "user": { + "id": "200" + } + } + ] +} diff --git a/packages/macos/data_stream/network_activity/agent/stream/unifiedlogs.yml.hbs b/packages/macos/data_stream/network_activity/agent/stream/unifiedlogs.yml.hbs new file mode 100644 index 00000000000..64704a7b452 --- /dev/null +++ b/packages/macos/data_stream/network_activity/agent/stream/unifiedlogs.yml.hbs @@ -0,0 +1,51 @@ +predicate: +- 'composedMessage CONTAINS "connect" AND (composedMessage CONTAINS "TCP" OR composedMessage CONTAINS "UDP")' +- 'composedMessage CONTAINS "disconnect" OR composedMessage CONTAINS "closed connection"' +- 'subsystem == "com.apple.necp" AND composedMessage CONTAINS "new connection"' +- 'eventMessage CONTAINS[c] "listening" AND eventMessage CONTAINS[c] "service"' +{{#each predicate as |p|}} +- {{p}} +{{/each}} +{{#if start}} +start: {{start}} +{{/if}} +{{#if end}} +end: {{end}} +{{/if}} +{{#if process}} +process: +{{#each process as |p|}} +- {{p}} +{{/each}} +{{/if}} +source: {{source}} +info: {{info}} +debug: {{debug}} +backtrace: {{backtrace}} +signpost: {{signpost}} +unreliable: {{unreliable}} +mach_continuous_time: {{mach_continuous_time}} +backfill: {{backfill}} +{{#if archive_file}} +archive_file: {{archive_file}} +{{/if}} +{{#if trace_file}} +trace_file: {{trace_file}} +{{/if}} +tags: +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} +{{#if preserve_duplicate_custom_fields}} + - preserve_duplicate_custom_fields +{{/if}} +{{#each tags as |tag|}} + - {{tag}} +{{/each}} +{{#contains "forwarded" tags}} +publisher_pipeline.disable_host: true +{{/contains}} +{{#if processors}} +processors: +{{processors}} +{{/if}} diff --git a/packages/macos/data_stream/network_activity/elasticsearch/ingest_pipeline/common-pipeline.yml.link b/packages/macos/data_stream/network_activity/elasticsearch/ingest_pipeline/common-pipeline.yml.link new file mode 100644 index 00000000000..f146557ce04 --- /dev/null +++ b/packages/macos/data_stream/network_activity/elasticsearch/ingest_pipeline/common-pipeline.yml.link @@ -0,0 +1 @@ +../../../../_dev/shared/common-pipeline.yml 6e7bb697288eb24aeb67d4950a8a3f38d89eb89f559dc01e605e543162267ddf \ No newline at end of file diff --git a/packages/macos/data_stream/network_activity/elasticsearch/ingest_pipeline/default.yml b/packages/macos/data_stream/network_activity/elasticsearch/ingest_pipeline/default.yml new file mode 100644 index 00000000000..b2310f2b461 --- /dev/null +++ b/packages/macos/data_stream/network_activity/elasticsearch/ingest_pipeline/default.yml @@ -0,0 +1,382 @@ +--- +description: Pipeline for processing network activity logs. +processors: + - append: + field: event.category + tag: set_event_category + value: network + - append: + field: event.type + tag: append_info_into_event_type_in_network + value: info + - pipeline: + name: '{{ IngestPipeline "common-pipeline" }}' + tag: run-common-pipeline + - grok: + description: Grok the eventMessage. + tag: grok-event-message + field: macos.event.message.description + ignore_missing: true + patterns: + - '^\[%{WORD}\] %{DATA}\:(?:%{SPACE}mach=%{WORD:macos.event.message.mach:boolean})?(?:%{SPACE}listener=%{WORD:macos.event.message.listener:boolean})?(?:%{SPACE}peer=%{WORD:macos.event.message.peer:boolean})?(?:%{SPACE}name=%{GREEDYDATA:macos.event.message.name})?' + - '^%{WORD} \[%{DATA}\](?:%{SPACE}flags=\[%{DATA:macos.event.message.flags}\])?(?:%{SPACE}seq=%{DATA:macos.event.message.seq},)?(?:%{SPACE}ack=%{DATA:macos.event.message.ack},)?(?:%{SPACE}win=%{DATA:macos.event.message.win})?(?:%{SPACE}state=%{DATA:macos.event.message.state})?(?:%{SPACE}rcv_nxt=%{DATA:macos.event.message.rcv_nxt},)?(?:snd_una=%{DATA:macos.event.message.snd_una})' + - '^%{WORD} \[%{DATA}\](?:%{SPACE}flags=\[%{DATA:macos.event.message.flags}\])?(?:%{SPACE}seq=%{DATA:macos.event.message.seq},)?(?:%{SPACE}ack=%{DATA:macos.event.message.ack},)?(?:%{SPACE}win=%{DATA:macos.event.message.win})?(?:%{SPACE}state=%{DATA:macos.event.message.state})?(?:%{SPACE}rcv_nxt=%{DATA:macos.event.message.rcv_nxt},)?(?:snd_una=%{DATA:macos.event.message.snd_una})' + - '^nw_protocol_boringssl_signal_connected\(%{NUMBER}\) \[%{DATA:macos.event.message.connection_identifier}\]\[%{DATA}\] TLS connected \[(?:version\(%{DATA:macos.event.message.tls_version}\))?(?:%{SPACE}ciphersuite\(%{DATA:macos.event.message.cipher_suite}\))?(?:%{SPACE}group\(%{DATA:macos.event.message.group}\))?(?:%{SPACE}signature_alg\(%{DATA:macos.event.message.signature_alg}\))?(?:%{SPACE}alpn\(%{DATA:macos.event.message.alpn}\))?(?:%{SPACE}resumed\(%{DATA:macos.event.message.resumed}\))?(?:%{SPACE}offered_ticket\(%{DATA:macos.event.message.offered_ticket}\))?(?:%{SPACE}false_started\(%{DATA:macos.event.message.false_started}\))?(?:%{SPACE}ocsp_received\(%{DATA:macos.event.message.ocsp_received}\))?(?:%{SPACE}sct_received\(%{DATA:macos.event.message.sct_received}\))?(?:%{SPACE}connect_time\(%{DATA:macos.event.message.connection_time}\))?(?:%{SPACE}flight_time\(%{DATA:macos.event.message.flight_time}\))?(?:%{SPACE}rtt\(%{DATA:macos.event.message.rtt}\))?(?:%{SPACE}write_stalls\(%{DATA:macos.event.message.write_stalls:long}\))?(?:%{SPACE}read_stalls\(%{DATA:macos.event.message.read_stalls:long}\))?(?:%{SPACE}pake\(%{DATA:macos.event.message.pake}\))?\]' + - '^Task \<%{DATA:macos.event.message.task_uid}\>.\<%{NUMBER}\>%{SPACE}summary for %{DATA} \{(?:transaction_duration_ms=%{NUMBER:macos.event.message.transaction_duration_ms:long},)?(?:%{SPACE}response_status=%{NUMBER:macos.event.message.response_status:long},)?(?:%{SPACE}connection=%{NUMBER:macos.event.message.connection:long},)?(?:%{SPACE}protocol=%{DATA:macos.event.message.protocol},)?(?:%{SPACE}domain_lookup_duration_ms=%{NUMBER:macos.event.message.domain_lookup_duration_ms:long},)?(?:%{SPACE}connect_duration_ms=%{NUMBER:macos.event.message.connection_duration_ms:long},)?(?:%{SPACE}secure_connection_duration_ms=%{NUMBER:macos.event.message.secure_connection_duration_ms:long},)?(?:%{SPACE}private_relay=%{WORD:macos.event.message.private_relay:boolean},)?(?:%{SPACE}request_start_ms=%{NUMBER:macos.event.message.request_start_ms:long},)?(?:%{SPACE}request_duration_ms=%{NUMBER:macos.event.message.request_duration_ms:long},)?(?:%{SPACE}response_start_ms=%{NUMBER:macos.event.message.response_start_ms:long},)?(?:%{SPACE}response_duration_ms=%{NUMBER:macos.event.message.response_duration_ms:long},)?(?:%{SPACE}request_bytes=%{NUMBER:macos.event.message.request_bytes:long},)?(?:%{SPACE}response_bytes=%{NUMBER:macos.event.message.response_bytes:long},)?(?:%{SPACE}cache_hit=%{WORD:macos.event.message.cache_hit:boolean})?\}' + - '^%{DATA} \[%{DATA:macos.event.message.connection_identifier}\]%{SPACE}\[%{UUID:macos.event.message.connection_uuid} :%{NUMBER:macos.event.message.src_port:long}<->:%{NUMBER:macos.event.message.dest_port:long}\]%{SPACE}Init: %{NUMBER:macos.event.message.init_flag:long}, Conn_Time: %{DATA:macos.event.message.connection_time}, SYNs: %{NUMBER:macos.event.message.syns:long}, WR_T: %{DATA:macos.event.message.wr_t_in_out}, RD_T: %{DATA:macos.event.message.rd_t_in_out}, TFO: %{DATA:macos.event.message.tfo_in_out_miss}, ECN: %{DATA:macos.event.message.ecn_in_out_miss}, Accurate ECN %{GREEDYDATA}: %{GREEDYDATA:macos.event.message.accurate_ecn}, TS: %{NUMBER:macos.event.message.timestamp_enabled:long}, TSO: %{NUMBER:macos.event.message.tso_enabled:long}%{SPACE}rtt_cache: %{DATA:macos.event.message.rtt_cache}, rtt_upd: %{NUMBER:macos.event.message.rtt_updates:long}, rtt: %{DATA:macos.event.message.rtt}, rtt_var: %{DATA:macos.event.message.rtt_var_ms} rtt_nc: %{DATA:macos.event.message.rtt_nc_ms}, rtt_var_nc: %{DATA:macos.event.message.rtt_var_nc_ms} base rtt: %{GREEDYDATA:macos.event.message.base_rtt_ms}%{SPACE}ACKs-compressed: %{NUMBER:macos.event.message.acks_compressed:long}, ACKs delayed: %{NUMBER:macos.event.message.acks_delayed:long} delayed ACKs sent: %{NUMBER:macos.event.message.delayed_acks_sent:long}' + - '^\[C%{NUMBER:macos.event.message.connection_id} %{UUID:macos.event.message.session_uuid} (Hostname\#)?%{DATA:macos.event.message.hostname}:%{NUMBER:macos.event.message.hostname_port:long} %{DATA}(, bundle id: %{DATA:macos.event.message.bundle_id})?(, pid: %{DATA:macos.event.message.pid:long})?(, account id: %{DATA:macos.event.message.account_id})?(, url: %{DATA:macos.event.message.url})?(, url hash: %{BASE16NUM:macos.event.message.url_hash})?(, traffic class: %{NUMBER:macos.event.message.traffic_class})?(, expected workload: %{NUMBER:macos.event.message.expected_workload})?(, %{GREEDYDATA})?, attribution: %{DATA:macos.event.message.attribution}(, %{GREEDYDATA})?\] cancelled\n\t\[C%{DATA:macos.event.message.connection_detail} %{UUID:macos.event.message.connection_uuid} %{IP:macos.event.message.client_ip}:%{NUMBER:macos.event.message.client_port:long}<->(IPv4#)?%{DATA:macos.event.message.server_id}:%{NUMBER:macos.event.message.server_port:long}\]\n\tConnected Path: %{DATA:macos.event.message.path_status}(, %{DATA})?(, interface: %{DATA:macos.event.message.interface})?(, %{GREEDYDATA})?\n\tPrivacy Stance: %{DATA:macos.event.message.privacy_stance}\n\tDuration: %{DATA:macos.event.message.duration}, DNS @%{DATA:macos.event.message.dns_start} took %{DATA:macos.event.message.dns_duration}, TCP @%{DATA:macos.event.message.tcp_start} took %{DATA:macos.event.message.tcp_duration}, TLS %{DATA:macos.event.message.tls_version} took %{DATA:macos.event.message.tls_duration}\n\tbytes in\/out: %{NUMBER:macos.event.message.bytes_in:long}\/%{NUMBER:macos.event.message.bytes_out:long}, packets in\/out: %{NUMBER:macos.event.message.packets_in:long}\/%{NUMBER:macos.event.message.packets_out:long}, rtt: %{DATA:macos.event.message.rtt}, retransmitted bytes: %{NUMBER:macos.event.message.retransmitted_bytes:long}, out-of-order bytes: %{NUMBER:macos.event.message.out_of_order_bytes:long}\n\tecn packets sent\/acked\/marked\/lost: %{NUMBER:macos.event.message.ecn_sent:long}\/%{NUMBER:macos.event.message.ecn_acked:long}\/%{NUMBER:macos.event.message.ecn_marked:long}\/%{NUMBER:macos.event.message.ecn_lost:long}$' + - '%{GREEDYDATA:macos.event.message.original}' + - convert: + field: macos.event.message.client_ip + tag: convert_client_ip_to_ip + type: ip + ignore_missing: true + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - set: + field: host.id + tag: set_host_id + copy_from: macos.event.message.hostname + ignore_empty_value: true + - set: + field: process.pid + tag: set_process_pid + copy_from: macos.event.message.pid + ignore_empty_value: true + - set: + field: url.original + tag: set_url_original + copy_from: macos.event.message.url + ignore_empty_value: true + - set: + field: source.ip + tag: set_source_ip + copy_from: macos.event.message.client_ip + ignore_empty_value: true + - set: + field: source.port + tag: set_source_port + copy_from: macos.event.message.client_port + ignore_empty_value: true + - set: + field: destination.port + tag: set_destination_port + copy_from: macos.event.message.server_port + ignore_empty_value: true + - set: + field: source.bytes + tag: set_source_bytes + copy_from: macos.event.message.bytes_in + ignore_empty_value: true + - set: + field: destination.bytes + tag: set_destination_bytes + copy_from: macos.event.message.bytes_out + ignore_empty_value: true + - set: + field: source.packets + tag: set_source_packets + copy_from: macos.event.message.packets_in + ignore_empty_value: true + - set: + field: destination.packets + tag: set_destination_packets + copy_from: macos.event.message.packets_out + ignore_empty_value: true + - script: + lang: painless + tag: calculate_total_bytes + description: calculate total bytes of in and out if in and out are not null. + if: ctx.source?.bytes != null && ctx.destination?.bytes != null + source: | + ctx.network = new HashMap(); + ctx.network.bytes = ctx.source.bytes + ctx.destination.bytes + - script: + lang: painless + tag: calculate_total_packets + description: calculate total packets of in and out if in and out are not null. + if: ctx.source?.packets != null && ctx.destination?.packets != null + source: | + if (ctx.network == null) { + ctx.network = new HashMap(); + } + ctx.network.packets = ctx.source.packets + ctx.destination.packets + - uri_parts: + field: url.original + tag: uri_parts_url_original + ignore_missing: true + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - append: + field: related.hosts + tag: append_host_id_into_related_hosts + value: '{{{host.id}}}' + allow_duplicates: false + if: ctx.host?.id != null + - append: + field: related.ip + tag: append_source_ip_into_related_ip + value: '{{{source.ip}}}' + allow_duplicates: false + if: ctx.source?.ip != null + - set: + field: process.code_signature.digest_algorithm + tag: set_process_code_signature_digest_algorithm + copy_from: macos.event.message.signature_alg + ignore_empty_value: true + - set: + field: macos.event.message.resumed + tag: set_event_message_resumed_true + value: true + if: ctx.macos?.event?.message?.resumed == '1' + - set: + field: macos.event.message.resumed + tag: set_event_message_resumed_false + value: false + if: ctx.macos?.event?.message?.resumed == '0' + - set: + field: macos.event.message.offered_ticket + tag: set_event_message_offered_ticket_true + value: true + if: ctx.macos?.event?.message?.offered_ticket == '1' + - set: + field: macos.event.message.offered_ticket + tag: set_event_message_offered_ticket_false + value: false + if: ctx.macos?.event?.message?.offered_ticket == '0' + - set: + field: macos.event.message.false_started + tag: set_event_message_false_started_true + value: true + if: ctx.macos?.event?.message?.false_started == '1' + - set: + field: macos.event.message.false_started + tag: set_event_message_false_started_false + value: false + if: ctx.macos?.event?.message?.false_started == '0' + - set: + field: macos.event.message.ocsp_received + tag: set_event_message_oscp_received_true + value: true + if: ctx.macos?.event?.message?.ocsp_received == '1' + - set: + field: macos.event.message.ocsp_received + tag: set_event_message_oscp_received_false + value: false + if: ctx.macos?.event?.message?.ocsp_received == '0' + - set: + field: macos.event.message.sct_received + tag: set_event_message_sct_received_true + value: true + if: ctx.macos?.event?.message?.sct_received == '1' + - set: + field: macos.event.message.sct_received + tag: set_event_message_sct_received_false + value: false + if: ctx.macos?.event?.message?.sct_received == '0' + - set: + field: http.request.bytes + tag: set_http_request_bytes + copy_from: macos.event.message.request_bytes + ignore_empty_value: true + - set: + field: http.response.bytes + tag: set_http_response_bytes + copy_from: macos.event.message.response_bytes + ignore_empty_value: true + - set: + field: http.response.status_code + tag: set_http_response_status_code + copy_from: macos.event.message.response_status + ignore_empty_value: true + - set: + field: destination.port + tag: set_destination_port + copy_from: macos.event.message.dest_port + ignore_empty_value: true + - set: + field: source.port + tag: set_source_port + copy_from: macos.event.message.src_port + ignore_empty_value: true + - dissect: + field: macos.event.message.ecn_in_out_miss + tag: dissect_ecn_in_out_miss + pattern: '%{macos.event.message.ecn_in}/%{macos.event.message.ecn_out}/%{macos.event.message.ecn_miss}' + ignore_missing: true + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - convert: + field: macos.event.message.ecn_in + tag: convert_macos_event_message_ecn_in_to_long + type: long + ignore_missing: true + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - convert: + field: macos.event.message.ecn_out + tag: convert_macos_event_message_ecn_out_to_long + type: long + ignore_missing: true + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - convert: + field: macos.event.message.ecn_miss + tag: convert_macos_event_message_ecn_miss_to_long + type: long + ignore_missing: true + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - dissect: + field: macos.event.message.rd_t_in_out + tag: dissect_rd_t_in_out + pattern: '%{macos.event.message.rd_t_in}/%{macos.event.message.rd_t_out}' + ignore_missing: true + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - convert: + field: macos.event.message.rd_t_in + tag: convert_macos_event_message_rd_t_in_to_long + type: long + ignore_missing: true + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - convert: + field: macos.event.message.rd_t_out + tag: convert_macos_event_message_rd_t_out_to_long + type: long + ignore_missing: true + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - dissect: + field: macos.event.message.tfo_in_out_miss + tag: dissect_tfo_in_out_miss + pattern: '%{macos.event.message.tfo_in}/%{macos.event.message.tfo_out}/%{macos.event.message.tfo_miss}' + ignore_missing: true + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - convert: + field: macos.event.message.tfo_in + tag: convert_macos_event_message_tfo_in_to_long + type: long + ignore_missing: true + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - convert: + field: macos.event.message.tfo_out + tag: convert_macos_event_message_tfo_out_to_long + type: long + ignore_missing: true + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - convert: + field: macos.event.message.tfo_miss + tag: convert_macos_event_message_tfo_miss_to_long + type: long + ignore_missing: true + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - dissect: + field: macos.event.message.wr_t_in_out + tag: dissect_wr_t_in_out + pattern: '%{macos.event.message.wr_t_in}/%{macos.event.message.wr_t_out}' + ignore_missing: true + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - convert: + field: macos.event.message.wr_t_in + tag: convert_macos_event_message_wr_t_in_to_long + type: long + ignore_missing: true + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - convert: + field: macos.event.message.wr_t_out + tag: convert_macos_event_message_wr_t_out_to_long + type: long + ignore_missing: true + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - dissect: + field: macos.event.message.accurate_ecn + tag: dissect_accurate_ecn + pattern: '%{macos.event.message.accurate_ecn_client}/%{macos.event.message.accurate_ecn_server}' + ignore_missing: true + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - remove: + field: + - macos.event.message.signature_alg + - macos.event.message.request_bytes + - macos.event.message.response_bytes + - macos.event.message.response_status + - macos.event.message.dest_port + - macos.event.message.src_port + - macos.event.message.hostname + - macos.event.message.pid + - macos.event.message.url + - macos.event.message.client_ip + - macos.event.message.client_port + - macos.event.message.server_port + - macos.event.message.bytes_in + - macos.event.message.bytes_out + - macos.event.message.packets_in + - macos.event.message.packets_out + tag: remove_custom_duplicate_fields + ignore_missing: true + if: ctx.tags == null || !ctx.tags.contains('preserve_duplicate_custom_fields') + - remove: + field: + - macos.event.message.original + - macos.event.message.ecn_in_out_miss + - macos.event.message.rd_t_in_out + - macos.event.message.tfo_in_out_miss + - macos.event.message.wr_t_in_out + - macos.event.message.accurate_ecn + tag: remove_non_required_fields + ignore_missing: true +on_failure: + - append: + field: error.message + value: >- + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}' + - set: + field: event.kind + tag: set_pipeline_error_to_event_kind + value: pipeline_error diff --git a/packages/macos/data_stream/network_activity/fields/base-fields.yml b/packages/macos/data_stream/network_activity/fields/base-fields.yml new file mode 100644 index 00000000000..2fd1e5adb7c --- /dev/null +++ b/packages/macos/data_stream/network_activity/fields/base-fields.yml @@ -0,0 +1,16 @@ +- name: data_stream.type + external: ecs +- name: data_stream.dataset + external: ecs +- name: data_stream.namespace + external: ecs +- name: event.module + type: constant_keyword + external: ecs + value: macos +- name: event.dataset + type: constant_keyword + external: ecs + value: macos.network_activity +- name: '@timestamp' + external: ecs diff --git a/packages/macos/data_stream/network_activity/fields/beats.yml b/packages/macos/data_stream/network_activity/fields/beats.yml new file mode 100644 index 00000000000..d5fd38748ba --- /dev/null +++ b/packages/macos/data_stream/network_activity/fields/beats.yml @@ -0,0 +1,6 @@ +- name: input.type + type: keyword + description: Type of Filebeat input. +- name: log.offset + type: long + description: Log offset. diff --git a/packages/macos/data_stream/network_activity/fields/fields.yml b/packages/macos/data_stream/network_activity/fields/fields.yml new file mode 100644 index 00000000000..81d48020635 --- /dev/null +++ b/packages/macos/data_stream/network_activity/fields/fields.yml @@ -0,0 +1,290 @@ +- name: macos + type: group + fields: + - name: activity_identifier + type: keyword + - name: backtrace + type: group + fields: + - name: frames + type: group + fields: + - name: image + type: group + fields: + - name: offset + type: keyword + - name: uuid + type: keyword + - name: boot_uuid + type: keyword + - name: category + type: keyword + - name: event + type: group + fields: + - name: category + type: keyword + - name: message + type: group + fields: + - name: account_id + type: keyword + - name: accurate_ecn_client + type: keyword + - name: accurate_ecn_server + type: keyword + - name: ack + type: keyword + - name: acks_compressed + type: long + - name: acks_delayed + type: long + - name: alpn + type: keyword + - name: attribution + type: keyword + - name: base_rtt_ms + type: keyword + - name: bundle_id + type: keyword + - name: bytes_in + type: long + - name: bytes_out + type: long + - name: cache_hit + type: boolean + - name: cipher_suite + type: keyword + - name: client_ip + type: ip + - name: client_port + type: long + - name: connection + type: long + - name: connection_detail + type: keyword + - name: connection_duration_ms + type: long + - name: connection_id + type: keyword + - name: connection_identifier + type: keyword + - name: connection_time + type: keyword + - name: connection_uuid + type: keyword + - name: delayed_acks_sent + type: long + - name: description + type: keyword + - name: dest_port + type: long + - name: dns_duration + type: keyword + - name: dns_start + type: keyword + - name: domain_lookup_duration_ms + type: long + - name: duration + type: keyword + - name: ecn_acked + type: long + - name: ecn_in + type: long + - name: ecn_lost + type: long + - name: ecn_marked + type: long + - name: ecn_miss + type: long + - name: ecn_out + type: long + - name: ecn_sent + type: long + - name: expected_workload + type: keyword + - name: false_started + type: boolean + - name: flags + type: keyword + - name: flight_time + type: keyword + - name: group + type: keyword + - name: hostname + type: keyword + - name: hostname_port + type: long + - name: init_flag + type: long + - name: interface + type: keyword + - name: listener + type: boolean + - name: mach + type: boolean + - name: name + type: keyword + - name: ocsp_received + type: boolean + - name: offered_ticket + type: boolean + - name: out_of_order_bytes + type: long + - name: packets_in + type: long + - name: packets_out + type: long + - name: pake + type: keyword + - name: path_status + type: keyword + - name: peer + type: boolean + - name: pid + type: long + - name: privacy_stance + type: keyword + - name: private_relay + type: boolean + - name: protocol + type: keyword + - name: rcv_nxt + type: keyword + - name: rd_t_in + type: long + - name: rd_t_out + type: long + - name: read_stalls + type: long + - name: request_bytes + type: long + - name: request_duration_ms + type: long + - name: request_start_ms + type: long + - name: response_bytes + type: long + - name: response_duration_ms + type: long + - name: response_start_ms + type: long + - name: response_status + type: long + - name: resumed + type: boolean + - name: retransmitted_bytes + type: long + - name: rtt + type: keyword + - name: rtt_cache + type: keyword + - name: rtt_nc_ms + type: keyword + - name: rtt_updates + type: long + - name: rtt_var_ms + type: keyword + - name: rtt_var_nc_ms + type: keyword + - name: sct_received + type: boolean + - name: secure_connection_duration_ms + type: long + - name: seq + type: keyword + - name: server_id + type: keyword + - name: server_port + type: long + - name: session_uuid + type: keyword + - name: signature_alg + type: keyword + - name: snd_una + type: keyword + - name: src_port + type: long + - name: state + type: keyword + - name: syns + type: long + - name: task_uid + type: keyword + - name: tcp_duration + type: keyword + - name: tcp_start + type: keyword + - name: tfo_in + type: long + - name: tfo_miss + type: long + - name: tfo_out + type: long + - name: timestamp_enabled + type: long + - name: tls_duration + type: keyword + - name: tls_version + type: keyword + - name: traffic_class + type: keyword + - name: transaction_duration_ms + type: long + - name: tso_enabled + type: long + - name: url + type: keyword + - name: url_hash + type: keyword + - name: win + type: keyword + - name: wr_t_in + type: long + - name: wr_t_out + type: long + - name: write_stalls + type: long + - name: type + type: keyword + - name: format_string + type: keyword + - name: mach_timestamp + type: double + - name: message_type + type: keyword + - name: parent_activity_identifier + type: keyword + - name: process + type: group + fields: + - name: id + type: long + - name: image_path + type: keyword + - name: image_uuid + type: keyword + - name: sender + type: group + fields: + - name: image_path + type: keyword + - name: image_uuid + type: keyword + - name: program_counter + type: long + - name: source + type: keyword + - name: subsystem + type: keyword + - name: thread_id + type: long + - name: timestamp + type: date + - name: timezone_name + type: keyword + - name: trace_id + type: keyword + - name: user_id + type: keyword diff --git a/packages/macos/data_stream/network_activity/manifest.yml b/packages/macos/data_stream/network_activity/manifest.yml new file mode 100644 index 00000000000..2477e0ed760 --- /dev/null +++ b/packages/macos/data_stream/network_activity/manifest.yml @@ -0,0 +1,139 @@ +title: Network Activity +type: logs +streams: + - input: unifiedlogs + title: Network Activity logs + description: Collect Network Activity logs from macOS. + template_path: unifiedlogs.yml.hbs + enabled: false + vars: + - name: predicate + type: text + title: Predicate + description: | + Filters messages using the provided predicate based on NSPredicate. + A compound predicate or multiple predicates can be provided as a list. + For detailed information on the use of predicate based filtering, + please refer to the https://developer.apple.com/library/mac/documentation/Cocoa/Conceptual/Predicates/Articles/pSyntax.html[Predicate Programming Guide]. + multi: true + show_user: true + - name: start + type: text + title: Start date + description: | + Shows content starting from the provided date. + The following date/time formats are accepted: + `YYYY-MM-DD`, `YYYY-MM-DD HH:MM:SS`, `YYYY-MM-DD HH:MM:SSZZZZZ`. + show_user: false + - name: end + type: text + title: End date + description: | + Shows content up to the provided date. + The following date/time formats are accepted: + `YYYY-MM-DD`, `YYYY-MM-DD HH:MM:SS`, `YYYY-MM-DD HH:MM:SSZZZZZ`. + show_user: false + - name: process + type: text + title: Process + description: | + A list of the processes on which to operate. It accepts a PID or process name. + multi: true + show_user: false + - name: source + type: bool + title: Include source + description: | + Include symbol names and source line numbers for messages, if available. + show_user: false + - name: info + type: bool + title: Include info + description: | + Disable or enable info level messages. + show_user: false + - name: debug + type: bool + title: Include debug + description: | + Disable or enable debug level messages. + show_user: false + - name: backtrace + type: bool + title: Include backtrace + description: | + Disable or enable backtrace level messages. + show_user: false + - name: signpost + type: bool + title: Include signpost + description: | + Disable or enable signpost level messages. + show_user: false + - name: unreliable + type: bool + title: Annotate unreliable + description: | + Annotate events with whether the log was emitted unreliably. + show_user: false + - name: mach_continuous_time + type: bool + title: Use mach continuous time + description: | + Use mach continuous time timestamps rather than walltime. + show_user: false + - name: backfill + type: bool + title: Must backfill + description: | + If set to true the input will process all available logs since the beginning + of time the first time it starts. + show_user: false + - name: archive_file + type: text + title: Archive file + description: | + Display events stored in the given archive. + The archive must be a valid log archive bundle with the suffix `.logarchive`. + show_user: false + - name: trace_file + type: text + title: Trace file + description: | + Display events stored in the given `.tracev3` file. + In order to be decoded, the file must be contained within a valid `.logarchive`. + show_user: false + - name: preserve_original_event + type: bool + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field event.original. + multi: false + required: false + show_user: true + default: false + - name: tags + type: text + title: Tags + description: Tags for the data-stream. + multi: true + required: true + show_user: false + default: + - forwarded + - macos-network_activity + - name: preserve_duplicate_custom_fields + required: false + title: Preserve duplicate custom fields + description: Preserve macos.* fields that were copied to Elastic Common Schema (ECS) fields. + type: bool + multi: false + show_user: false + default: false + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: >- + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. diff --git a/packages/macos/data_stream/process_execution_monitoring/_dev/test/pipeline/test-common-config.yml b/packages/macos/data_stream/process_execution_monitoring/_dev/test/pipeline/test-common-config.yml new file mode 100644 index 00000000000..37e8fa225fd --- /dev/null +++ b/packages/macos/data_stream/process_execution_monitoring/_dev/test/pipeline/test-common-config.yml @@ -0,0 +1,3 @@ +fields: + tags: + - preserve_duplicate_custom_fields diff --git a/packages/macos/data_stream/process_execution_monitoring/_dev/test/pipeline/test-process-execution-monitoring.log b/packages/macos/data_stream/process_execution_monitoring/_dev/test/pipeline/test-process-execution-monitoring.log new file mode 100644 index 00000000000..18d89245abe --- /dev/null +++ b/packages/macos/data_stream/process_execution_monitoring/_dev/test/pipeline/test-process-execution-monitoring.log @@ -0,0 +1,2 @@ +{"timezoneName":"","messageType":"Default","eventType":"logEvent","source":null,"formatString":"code signing internal problem: unexpected error from xpc: %s","userID":501,"activityIdentifier":0,"subsystem":"com.apple.securityd","category":"SecError","threadID":199743,"senderImageUUID":"AF38D16B-954C-3A9E-B0F7-5100DB3D566C","backtrace":{"frames":[{"imageOffset":2219233,"imageUUID":"AF38D16B-954C-3A9E-B0F7-5100DB3D566C"}]},"bootUUID":"AB156DC0-B7BC-4860-9E5C-A34A8BBE8699","processImagePath":"\/System\/Library\/Frameworks\/StoreKit.framework\/Support\/storekitagent","senderImagePath":"\/System\/Library\/Frameworks\/Security.framework\/Versions\/A\/Security","timestamp":"2025-10-21 15:27:16.337178+0530","machTimestamp":82458117633560,"eventMessage":"code signing internal problem: unexpected error from xpc: { count = 1, transaction: 0, voucher = 0x0, contents =\n\t\"XPCErrorDescription\" => { length = 18, contents = \"Connection invalid\" }\n}","processImageUUID":"63A2B64B-11CA-3ABA-94D7-3916A3BD8D08","traceID":232616665735757828,"processID":552,"senderProgramCounter":2219233,"parentActivityIdentifier":0} +{"timezoneName":"","messageType":"Default","eventType":"logEvent","source":null,"formatString":"code signing internal problem: unexpected error from xpc: %s","userID":501,"activityIdentifier":0,"subsystem":"com.apple.securityd","category":"SecError","threadID":199743,"senderImageUUID":"AF38D16B-954C-3A9E-B0F7-5100DB3D566C","backtrace":{"frames":[{"imageOffset":2219233,"imageUUID":"AF38D16B-954C-3A9E-B0F7-5100DB3D566C"}]},"bootUUID":"AB156DC0-B7BC-4860-9E5C-A34A8BBE8699","processImagePath":"\/System\/Library\/Frameworks\/StoreKit.framework\/Support\/storekitagent","senderImagePath":"\/System\/Library\/Frameworks\/Security.framework\/Versions\/A\/Security","timestamp":"2025-10-21 15:27:16.289787+0530","machTimestamp":82458070242618,"eventMessage":"code signing internal problem: unexpected error from xpc: { count = 1, transaction: 0, voucher = 0x0, contents =\n\t\"XPCErrorDescription\" => { length = 18, contents = \"Connection invalid\" }\n}","processImageUUID":"63A2B64B-11CA-3ABA-94D7-3916A3BD8D08","traceID":232616665735757828,"processID":552,"senderProgramCounter":2219233,"parentActivityIdentifier":0} diff --git a/packages/macos/data_stream/process_execution_monitoring/_dev/test/pipeline/test-process-execution-monitoring.log-expected.json b/packages/macos/data_stream/process_execution_monitoring/_dev/test/pipeline/test-process-execution-monitoring.log-expected.json new file mode 100644 index 00000000000..2f257bdc10b --- /dev/null +++ b/packages/macos/data_stream/process_execution_monitoring/_dev/test/pipeline/test-process-execution-monitoring.log-expected.json @@ -0,0 +1,158 @@ +{ + "expected": [ + { + "@timestamp": "2025-10-21T09:57:16.337Z", + "ecs": { + "version": "9.2.0" + }, + "event": { + "category": [ + "process" + ], + "kind": "event", + "original": "{\"timezoneName\":\"\",\"messageType\":\"Default\",\"eventType\":\"logEvent\",\"source\":null,\"formatString\":\"code signing internal problem: unexpected error from xpc: %s\",\"userID\":501,\"activityIdentifier\":0,\"subsystem\":\"com.apple.securityd\",\"category\":\"SecError\",\"threadID\":199743,\"senderImageUUID\":\"AF38D16B-954C-3A9E-B0F7-5100DB3D566C\",\"backtrace\":{\"frames\":[{\"imageOffset\":2219233,\"imageUUID\":\"AF38D16B-954C-3A9E-B0F7-5100DB3D566C\"}]},\"bootUUID\":\"AB156DC0-B7BC-4860-9E5C-A34A8BBE8699\",\"processImagePath\":\"\\/System\\/Library\\/Frameworks\\/StoreKit.framework\\/Support\\/storekitagent\",\"senderImagePath\":\"\\/System\\/Library\\/Frameworks\\/Security.framework\\/Versions\\/A\\/Security\",\"timestamp\":\"2025-10-21 15:27:16.337178+0530\",\"machTimestamp\":82458117633560,\"eventMessage\":\"code signing internal problem: unexpected error from xpc: { count = 1, transaction: 0, voucher = 0x0, contents =\\n\\t\\\"XPCErrorDescription\\\" => { length = 18, contents = \\\"Connection invalid\\\" }\\n}\",\"processImageUUID\":\"63A2B64B-11CA-3ABA-94D7-3916A3BD8D08\",\"traceID\":232616665735757828,\"processID\":552,\"senderProgramCounter\":2219233,\"parentActivityIdentifier\":0}", + "type": [ + "info" + ] + }, + "log": { + "level": "info" + }, + "macos": { + "activity_identifier": "0", + "backtrace": { + "frames": [ + { + "image": { + "offset": "2219233", + "uuid": "AF38D16B-954C-3A9E-B0F7-5100DB3D566C" + } + } + ] + }, + "boot_uuid": "AB156DC0-B7BC-4860-9E5C-A34A8BBE8699", + "category": "SecError", + "event": { + "message": { + "description": "code signing internal problem: unexpected error from xpc: { count = 1, transaction: 0, voucher = 0x0, contents =\n\t\"XPCErrorDescription\" => { length = 18, contents = \"Connection invalid\" }\n}" + }, + "type": "logEvent" + }, + "format_string": "code signing internal problem: unexpected error from xpc: %s", + "mach_timestamp": 8.245811763356E13, + "message_type": "Default", + "parent_activity_identifier": "0", + "process": { + "id": 552, + "image_path": "/System/Library/Frameworks/StoreKit.framework/Support/storekitagent", + "image_uuid": "63A2B64B-11CA-3ABA-94D7-3916A3BD8D08" + }, + "sender": { + "image_path": "/System/Library/Frameworks/Security.framework/Versions/A/Security", + "image_uuid": "AF38D16B-954C-3A9E-B0F7-5100DB3D566C", + "program_counter": 2219233 + }, + "subsystem": "com.apple.securityd", + "thread_id": 199743, + "timestamp": "2025-10-21T09:57:16.337Z", + "trace_id": "232616665735757828", + "user_id": "501" + }, + "message": "code signing internal problem: unexpected error from xpc: { count = 1, transaction: 0, voucher = 0x0, contents =\n\t\"XPCErrorDescription\" => { length = 18, contents = \"Connection invalid\" }\n}", + "process": { + "pid": 552, + "thread": { + "id": 199743 + } + }, + "related": { + "user": [ + "501" + ] + }, + "tags": [ + "preserve_duplicate_custom_fields" + ], + "user": { + "id": "501" + } + }, + { + "@timestamp": "2025-10-21T09:57:16.289Z", + "ecs": { + "version": "9.2.0" + }, + "event": { + "category": [ + "process" + ], + "kind": "event", + "original": "{\"timezoneName\":\"\",\"messageType\":\"Default\",\"eventType\":\"logEvent\",\"source\":null,\"formatString\":\"code signing internal problem: unexpected error from xpc: %s\",\"userID\":501,\"activityIdentifier\":0,\"subsystem\":\"com.apple.securityd\",\"category\":\"SecError\",\"threadID\":199743,\"senderImageUUID\":\"AF38D16B-954C-3A9E-B0F7-5100DB3D566C\",\"backtrace\":{\"frames\":[{\"imageOffset\":2219233,\"imageUUID\":\"AF38D16B-954C-3A9E-B0F7-5100DB3D566C\"}]},\"bootUUID\":\"AB156DC0-B7BC-4860-9E5C-A34A8BBE8699\",\"processImagePath\":\"\\/System\\/Library\\/Frameworks\\/StoreKit.framework\\/Support\\/storekitagent\",\"senderImagePath\":\"\\/System\\/Library\\/Frameworks\\/Security.framework\\/Versions\\/A\\/Security\",\"timestamp\":\"2025-10-21 15:27:16.289787+0530\",\"machTimestamp\":82458070242618,\"eventMessage\":\"code signing internal problem: unexpected error from xpc: { count = 1, transaction: 0, voucher = 0x0, contents =\\n\\t\\\"XPCErrorDescription\\\" => { length = 18, contents = \\\"Connection invalid\\\" }\\n}\",\"processImageUUID\":\"63A2B64B-11CA-3ABA-94D7-3916A3BD8D08\",\"traceID\":232616665735757828,\"processID\":552,\"senderProgramCounter\":2219233,\"parentActivityIdentifier\":0}", + "type": [ + "info" + ] + }, + "log": { + "level": "info" + }, + "macos": { + "activity_identifier": "0", + "backtrace": { + "frames": [ + { + "image": { + "offset": "2219233", + "uuid": "AF38D16B-954C-3A9E-B0F7-5100DB3D566C" + } + } + ] + }, + "boot_uuid": "AB156DC0-B7BC-4860-9E5C-A34A8BBE8699", + "category": "SecError", + "event": { + "message": { + "description": "code signing internal problem: unexpected error from xpc: { count = 1, transaction: 0, voucher = 0x0, contents =\n\t\"XPCErrorDescription\" => { length = 18, contents = \"Connection invalid\" }\n}" + }, + "type": "logEvent" + }, + "format_string": "code signing internal problem: unexpected error from xpc: %s", + "mach_timestamp": 8.2458070242618E13, + "message_type": "Default", + "parent_activity_identifier": "0", + "process": { + "id": 552, + "image_path": "/System/Library/Frameworks/StoreKit.framework/Support/storekitagent", + "image_uuid": "63A2B64B-11CA-3ABA-94D7-3916A3BD8D08" + }, + "sender": { + "image_path": "/System/Library/Frameworks/Security.framework/Versions/A/Security", + "image_uuid": "AF38D16B-954C-3A9E-B0F7-5100DB3D566C", + "program_counter": 2219233 + }, + "subsystem": "com.apple.securityd", + "thread_id": 199743, + "timestamp": "2025-10-21T09:57:16.289Z", + "trace_id": "232616665735757828", + "user_id": "501" + }, + "message": "code signing internal problem: unexpected error from xpc: { count = 1, transaction: 0, voucher = 0x0, contents =\n\t\"XPCErrorDescription\" => { length = 18, contents = \"Connection invalid\" }\n}", + "process": { + "pid": 552, + "thread": { + "id": 199743 + } + }, + "related": { + "user": [ + "501" + ] + }, + "tags": [ + "preserve_duplicate_custom_fields" + ], + "user": { + "id": "501" + } + } + ] +} diff --git a/packages/macos/data_stream/process_execution_monitoring/agent/stream/unifiedlogs.yml.hbs b/packages/macos/data_stream/process_execution_monitoring/agent/stream/unifiedlogs.yml.hbs new file mode 100644 index 00000000000..8b71cb58250 --- /dev/null +++ b/packages/macos/data_stream/process_execution_monitoring/agent/stream/unifiedlogs.yml.hbs @@ -0,0 +1,50 @@ +predicate: +- 'eventMessage CONTAINS[c] "exec" OR eventMessage CONTAINS[c] "fork" OR eventMessage CONTAINS[c] "exited" OR eventMessage CONTAINS[c] "terminated"' +- 'subsystem == "com.apple.securityd" AND (composedMessage CONTAINS "code signing" OR composedMessage CONTAINS "not valid")' +- 'composedMessage CONTAINS "com.apple.quarantine"' +{{#each predicate as |p|}} +- {{p}} +{{/each}} +{{#if start}} +start: {{start}} +{{/if}} +{{#if end}} +end: {{end}} +{{/if}} +{{#if process}} +process: +{{#each process as |p|}} +- {{p}} +{{/each}} +{{/if}} +source: {{source}} +info: {{info}} +debug: {{debug}} +backtrace: {{backtrace}} +signpost: {{signpost}} +unreliable: {{unreliable}} +mach_continuous_time: {{mach_continuous_time}} +backfill: {{backfill}} +{{#if archive_file}} +archive_file: {{archive_file}} +{{/if}} +{{#if trace_file}} +trace_file: {{trace_file}} +{{/if}} +tags: +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} +{{#if preserve_duplicate_custom_fields}} + - preserve_duplicate_custom_fields +{{/if}} +{{#each tags as |tag|}} + - {{tag}} +{{/each}} +{{#contains "forwarded" tags}} +publisher_pipeline.disable_host: true +{{/contains}} +{{#if processors}} +processors: +{{processors}} +{{/if}} diff --git a/packages/macos/data_stream/process_execution_monitoring/elasticsearch/ingest_pipeline/common-pipeline.yml.link b/packages/macos/data_stream/process_execution_monitoring/elasticsearch/ingest_pipeline/common-pipeline.yml.link new file mode 100644 index 00000000000..f146557ce04 --- /dev/null +++ b/packages/macos/data_stream/process_execution_monitoring/elasticsearch/ingest_pipeline/common-pipeline.yml.link @@ -0,0 +1 @@ +../../../../_dev/shared/common-pipeline.yml 6e7bb697288eb24aeb67d4950a8a3f38d89eb89f559dc01e605e543162267ddf \ No newline at end of file diff --git a/packages/macos/data_stream/process_execution_monitoring/elasticsearch/ingest_pipeline/default.yml b/packages/macos/data_stream/process_execution_monitoring/elasticsearch/ingest_pipeline/default.yml new file mode 100644 index 00000000000..dce03bd9925 --- /dev/null +++ b/packages/macos/data_stream/process_execution_monitoring/elasticsearch/ingest_pipeline/default.yml @@ -0,0 +1,13 @@ +--- +description: Pipeline for processing process_execution_monitoring logs. +processors: + - append: + field: event.category + tag: set_event_category + value: process + - append: + field: event.type + tag: append_info_into_event_type_in_process + value: info + - pipeline: + name: '{{ IngestPipeline "common-pipeline" }}' diff --git a/packages/macos/data_stream/process_execution_monitoring/fields/base-fields.yml b/packages/macos/data_stream/process_execution_monitoring/fields/base-fields.yml new file mode 100644 index 00000000000..2995068a5b6 --- /dev/null +++ b/packages/macos/data_stream/process_execution_monitoring/fields/base-fields.yml @@ -0,0 +1,16 @@ +- name: data_stream.type + external: ecs +- name: data_stream.dataset + external: ecs +- name: data_stream.namespace + external: ecs +- name: event.module + type: constant_keyword + external: ecs + value: macos +- name: event.dataset + type: constant_keyword + external: ecs + value: macos.process_execution_monitoring +- name: '@timestamp' + external: ecs diff --git a/packages/macos/data_stream/process_execution_monitoring/fields/beats.yml b/packages/macos/data_stream/process_execution_monitoring/fields/beats.yml new file mode 100644 index 00000000000..d5fd38748ba --- /dev/null +++ b/packages/macos/data_stream/process_execution_monitoring/fields/beats.yml @@ -0,0 +1,6 @@ +- name: input.type + type: keyword + description: Type of Filebeat input. +- name: log.offset + type: long + description: Log offset. diff --git a/packages/macos/data_stream/process_execution_monitoring/fields/fields.yml b/packages/macos/data_stream/process_execution_monitoring/fields/fields.yml new file mode 100644 index 00000000000..53e33cf5c76 --- /dev/null +++ b/packages/macos/data_stream/process_execution_monitoring/fields/fields.yml @@ -0,0 +1,74 @@ +- name: macos + type: group + fields: + - name: activity_identifier + type: keyword + - name: backtrace + type: group + fields: + - name: frames + type: group + fields: + - name: image + type: group + fields: + - name: offset + type: keyword + - name: uuid + type: keyword + - name: boot_uuid + type: keyword + - name: category + type: keyword + - name: event + type: group + fields: + - name: category + type: keyword + - name: message + type: group + fields: + - name: description + type: keyword + - name: type + type: keyword + - name: format_string + type: keyword + - name: mach_timestamp + type: double + - name: message_type + type: keyword + - name: parent_activity_identifier + type: keyword + - name: process + type: group + fields: + - name: id + type: long + - name: image_path + type: keyword + - name: image_uuid + type: keyword + - name: sender + type: group + fields: + - name: image_path + type: keyword + - name: image_uuid + type: keyword + - name: program_counter + type: long + - name: source + type: keyword + - name: subsystem + type: keyword + - name: thread_id + type: long + - name: timestamp + type: date + - name: timezone_name + type: keyword + - name: trace_id + type: keyword + - name: user_id + type: keyword diff --git a/packages/macos/data_stream/process_execution_monitoring/manifest.yml b/packages/macos/data_stream/process_execution_monitoring/manifest.yml new file mode 100644 index 00000000000..8d830f1789e --- /dev/null +++ b/packages/macos/data_stream/process_execution_monitoring/manifest.yml @@ -0,0 +1,139 @@ +title: Process Execution Monitoring +type: logs +streams: + - input: unifiedlogs + title: Process Execution Monitoring logs + description: Collect Process Execution Monitoring logs from macOS. + template_path: unifiedlogs.yml.hbs + enabled: false + vars: + - name: predicate + type: text + title: Predicate + description: | + Filters messages using the provided predicate based on NSPredicate. + A compound predicate or multiple predicates can be provided as a list. + For detailed information on the use of predicate based filtering, + please refer to the https://developer.apple.com/library/mac/documentation/Cocoa/Conceptual/Predicates/Articles/pSyntax.html[Predicate Programming Guide]. + multi: true + show_user: true + - name: start + type: text + title: Start date + description: | + Shows content starting from the provided date. + The following date/time formats are accepted: + `YYYY-MM-DD`, `YYYY-MM-DD HH:MM:SS`, `YYYY-MM-DD HH:MM:SSZZZZZ`. + show_user: false + - name: end + type: text + title: End date + description: | + Shows content up to the provided date. + The following date/time formats are accepted: + `YYYY-MM-DD`, `YYYY-MM-DD HH:MM:SS`, `YYYY-MM-DD HH:MM:SSZZZZZ`. + show_user: false + - name: process + type: text + title: Process + description: | + A list of the processes on which to operate. It accepts a PID or process name. + multi: true + show_user: false + - name: source + type: bool + title: Include source + description: | + Include symbol names and source line numbers for messages, if available. + show_user: false + - name: info + type: bool + title: Include info + description: | + Disable or enable info level messages. + show_user: false + - name: debug + type: bool + title: Include debug + description: | + Disable or enable debug level messages. + show_user: false + - name: backtrace + type: bool + title: Include backtrace + description: | + Disable or enable backtrace level messages. + show_user: false + - name: signpost + type: bool + title: Include signpost + description: | + Disable or enable signpost level messages. + show_user: false + - name: unreliable + type: bool + title: Annotate unreliable + description: | + Annotate events with whether the log was emitted unreliably. + show_user: false + - name: mach_continuous_time + type: bool + title: Use mach continuous time + description: | + Use mach continuous time timestamps rather than walltime. + show_user: false + - name: backfill + type: bool + title: Must backfill + description: | + If set to true the input will process all available logs since the beginning + of time the first time it starts. + show_user: false + - name: archive_file + type: text + title: Archive file + description: | + Display events stored in the given archive. + The archive must be a valid log archive bundle with the suffix `.logarchive`. + show_user: false + - name: trace_file + type: text + title: Trace file + description: | + Display events stored in the given `.tracev3` file. + In order to be decoded, the file must be contained within a valid `.logarchive`. + show_user: false + - name: preserve_original_event + type: bool + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field event.original. + multi: false + required: false + show_user: true + default: false + - name: tags + type: text + title: Tags + description: Tags for the data-stream. + multi: true + required: true + show_user: false + default: + - forwarded + - macos-process_execution_monitoring + - name: preserve_duplicate_custom_fields + required: false + title: Preserve duplicate custom fields + description: Preserve macos.* fields that were copied to Elastic Common Schema (ECS) fields. + type: bool + multi: false + show_user: false + default: false + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: >- + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. diff --git a/packages/macos/data_stream/system_change/_dev/test/pipeline/test-common-config.yml b/packages/macos/data_stream/system_change/_dev/test/pipeline/test-common-config.yml new file mode 100644 index 00000000000..4da22641654 --- /dev/null +++ b/packages/macos/data_stream/system_change/_dev/test/pipeline/test-common-config.yml @@ -0,0 +1,3 @@ +fields: + tags: + - preserve_original_event diff --git a/packages/macos/data_stream/system_change/_dev/test/pipeline/test-system-change.log b/packages/macos/data_stream/system_change/_dev/test/pipeline/test-system-change.log new file mode 100644 index 00000000000..a0d562d7403 --- /dev/null +++ b/packages/macos/data_stream/system_change/_dev/test/pipeline/test-system-change.log @@ -0,0 +1,8 @@ +{"timezoneName":"","messageType":"Default","eventType":"logEvent","source":null,"formatString":"%{public}s %{public}s flags=%{network:tcp_flags}x seq=%u, ack=%u, win=%u state=%{network:tcp_state}x rcv_nxt=%u, snd_una=%u","userID":200,"activityIdentifier":0,"subsystem":"com.apple.network","category":"tcp","threadID":988590,"senderImageUUID":"C5739A55-A9D3-3AFB-9AC9-652E5799B536","backtrace":{"frames":[{"imageOffset":281600,"imageUUID":"C5739A55-A9D3-3AFB-9AC9-652E5799B536"}]},"bootUUID":"218031E6-E47F-4A77-B7FC-5A57B049F4BC","processImagePath":"\/System\/Library\/CoreServices\/Software Update.app\/Contents\/Resources\/softwareupdated","senderImagePath":"\/usr\/lib\/libusrtcp.dylib","timestamp":"2025-10-13 15:45:56.155816+0530","machTimestamp":725858385613246,"eventMessage":"tcp_output [C129.1.1.1:3] flags=[F.] seq=3768828915, ack=3786980776, win=2048 state=FIN_WAIT_1 rcv_nxt=3786980776, snd_una=3768828891","processImageUUID":"CD58B635-1F96-38BD-BF3B-DBA7CA293E28","traceID":1202012843394990084,"processID":200,"senderProgramCounter":281600,"parentActivityIdentifier":0} +{"timezoneName":"","messageType":"Default","eventType":"logEvent","source":null,"formatString":"%{public}s %{public}s flags=%{network:tcp_flags}x seq=%u, ack=%u, win=%u state=%{network:tcp_state}x rcv_nxt=%u, snd_una=%u","userID":200,"activityIdentifier":0,"subsystem":"com.apple.network","category":"tcp","threadID":988590,"senderImageUUID":"C5739A55-A9D3-3AFB-9AC9-652E5799B536","backtrace":{"frames":[{"imageOffset":182769,"imageUUID":"C5739A55-A9D3-3AFB-9AC9-652E5799B536"}]},"bootUUID":"218031E6-E47F-4A77-B7FC-5A57B049F4BC","processImagePath":"\/System\/Library\/CoreServices\/Software Update.app\/Contents\/Resources\/softwareupdated","senderImagePath":"\/usr\/lib\/libusrtcp.dylib","timestamp":"2025-10-13 15:45:56.256214+0530","machTimestamp":725858486011276,"eventMessage":"tcp_input [C129.1.1.1:3] flags=[F.] seq=3786980800, ack=3768828915, win=576 state=CLOSED rcv_nxt=3786980776, snd_una=3768828891","processImageUUID":"CD58B635-1F96-38BD-BF3B-DBA7CA293E28","traceID":1202012843394990084,"processID":200,"senderProgramCounter":182769,"parentActivityIdentifier":0} +{"timezoneName":"","messageType":"Default","eventType":"logEvent","source":null,"formatString":"%{public}s %{public}s \n\t[%{public,uuid_t}.16P %{sensitive}s:%{public}d<->%{sensitive}s:%{public}d]\n\tInit: %d, Conn_Time: %u.ums, SYNs: %u, WR_T: %d\/%d, RD_T: %d\/%d, TFO: %d\/%d\/%d, ECN: %d\/%d\/%d, Accurate ECN (client\/server): %s\/%s, TS: %d, TSO: %d\n\trtt_cache: %{public}s, rtt_upd: %d, rtt: %u.ums, rtt_var: %u.ums rtt_nc: %u.ums, rtt_var_nc: %u.ums base rtt: %ums\n\tACKs-compressed: %u, ACKs delayed: %u delayed ACKs sent: %u","userID":200,"activityIdentifier":0,"subsystem":"com.apple.network","category":"tcp","threadID":988590,"senderImageUUID":"C5739A55-A9D3-3AFB-9AC9-652E5799B536","backtrace":{"frames":[{"imageOffset":139368,"imageUUID":"C5739A55-A9D3-3AFB-9AC9-652E5799B536"}]},"bootUUID":"218031E6-E47F-4A77-B7FC-5A57B049F4BC","processImagePath":"\/System\/Library\/CoreServices\/Software Update.app\/Contents\/Resources\/softwareupdated","senderImagePath":"\/usr\/lib\/libusrtcp.dylib","timestamp":"2025-10-13 15:46:58.699993+0530","machTimestamp":725920929789896,"eventMessage":"nw_protocol_tcp_log_summary [C131.1.1.1:3] \n\t[263F0F80-1925-4AF8-ABE3-ED1FDDCD00EF :63735<->:443]\n\tInit: 1, Conn_Time: 40.819ms, SYNs: 1, WR_T: 0\/0, RD_T: 0\/0, TFO: 0\/0\/0, ECN: 0\/0\/0, Accurate ECN (client\/server): Disabled\/Disabled, TS: 1, TSO: 1\n\trtt_cache: none, rtt_upd: 7, rtt: 126.375ms, rtt_var: 91.187ms rtt_nc: 126.375ms, rtt_var_nc: 91.187ms base rtt: 21ms\n\tACKs-compressed: 2, ACKs delayed: 0 delayed ACKs sent: 0","processImageUUID":"CD58B635-1F96-38BD-BF3B-DBA7CA293E28","traceID":1201924813745291268,"processID":200,"senderProgramCounter":139368,"parentActivityIdentifier":0} +{"timezoneName":"","messageType":"Default","eventType":"logEvent","source":null,"formatString":"[C%u %{public,uuid_t}.16P %{public}s %{public}@] cancelled\n\t[C%{public}s %{public,uuid_t}.16P %{public}s<->%{public}s]\n\tConnected Path: %{public}@\n\tPrivacy Stance: %{public}s\n\tDuration: %u.us, DNS @%u.us took %u.us, %{public}s @%u.us took %u.us, %{public}s took %u.us\n\tbytes in\/out: %llu\/%llu, packets in\/out: %llu\/%llu, rtt: %u.us, retransmitted bytes: %llu, out-of-order bytes: %llu\n\tecn packets sent\/acked\/marked\/lost: %llu\/%llu\/%llu\/%llu","userID":200,"activityIdentifier":0,"subsystem":"com.apple.network","category":"connection","threadID":988590,"senderImageUUID":"E481CDB0-9C0C-3BBC-87FA-006C66D3223E","backtrace":{"frames":[{"imageOffset":9418645,"imageUUID":"E481CDB0-9C0C-3BBC-87FA-006C66D3223E"}]},"bootUUID":"218031E6-E47F-4A77-B7FC-5A57B049F4BC","processImagePath":"\/System\/Library\/CoreServices\/Software Update.app\/Contents\/Resources\/softwareupdated","senderImagePath":"\/System\/Library\/Frameworks\/Network.framework\/Versions\/A\/Network","timestamp":"2025-10-13 15:46:58.706400+0530","machTimestamp":725920936196661,"eventMessage":"[C131 7F4FBE47-BF28-48DA-ABBB-0C219E463C9D Hostname#a4134bf8:443 quic-connection, url hash: ffb1585e, definite, attribution: developer] cancelled\n\t[C131.1.1.1 DD5F0E62-F851-47EE-A14E-2F3668A1FFCC 172.20.1.254:63735<->IPv4#86785abb:443]\n\tConnected Path: satisfied (Path is satisfied), viable, interface: en0[802.11], ipv4, dns, uses wifi\n\tPrivacy Stance: Not Eligible\n\tDuration: 122.754s, DNS @0.002s took 0.318s, TCP @0.324s took 0.041s, TLS 1.3 took 0.426s\n\tbytes in\/out: 17140\/1670, packets in\/out: 17\/19, rtt: 0.126s, retransmitted bytes: 517, out-of-order bytes: 0\n\tecn packets sent\/acked\/marked\/lost: 0\/0\/0\/0","processImageUUID":"CD58B635-1F96-38BD-BF3B-DBA7CA293E28","traceID":589370805553725444,"processID":200,"senderProgramCounter":9418645,"parentActivityIdentifier":0} +{"timezoneName":"","messageType":"Default","eventType":"logEvent","source":null,"formatString":"[C%u %{public,uuid_t}.16P %{public}s %{public}@] %{public}scancel","userID":200,"activityIdentifier":0,"subsystem":"com.apple.network","category":"connection","threadID":1132254,"senderImageUUID":"E481CDB0-9C0C-3BBC-87FA-006C66D3223E","backtrace":{"frames":[{"imageOffset":3693596,"imageUUID":"E481CDB0-9C0C-3BBC-87FA-006C66D3223E"}]},"bootUUID":"218031E6-E47F-4A77-B7FC-5A57B049F4BC","processImagePath":"\/System\/Library\/CoreServices\/Software Update.app\/Contents\/Resources\/softwareupdated","senderImagePath":"\/System\/Library\/Frameworks\/Network.framework\/Versions\/A\/Network","timestamp":"2025-10-15 16:21:14.492797+0530","machTimestamp":739105540170070,"eventMessage":"[C135 59E5BC33-3431-4F8A-981A-B4AF31960D47 Hostname#3fba6d68:443 tcp, url hash: 15f2f095, tls, definite, attribution: developer] cancel","processImageUUID":"CD58B635-1F96-38BD-BF3B-DBA7CA293E28","traceID":587172881809801220,"processID":200,"senderProgramCounter":3693596,"parentActivityIdentifier":0} +{"timezoneName":"","messageType":"Default","eventType":"logEvent","source":null,"formatString":"[%p] activating connection: mach=%{bool}d listener=%{bool}d peer=%{bool}d name=%{public}s","userID":200,"activityIdentifier":1606042,"subsystem":"com.apple.xpc","category":"connection","threadID":1131759,"senderImageUUID":"ABCE7E08-7D29-3BE0-AF60-F0DC752E951C","backtrace":{"frames":[{"imageOffset":43862,"imageUUID":"ABCE7E08-7D29-3BE0-AF60-F0DC752E951C"}]},"bootUUID":"218031E6-E47F-4A77-B7FC-5A57B049F4BC","processImagePath":"\/System\/Library\/CoreServices\/Software Update.app\/Contents\/Resources\/softwareupdated","senderImagePath":"\/usr\/lib\/system\/libxpc.dylib","timestamp":"2025-10-15 16:21:22.947210+0530","machTimestamp":739113994582741,"eventMessage":"[0x7fd20cffa690] activating connection: mach=false listener=false peer=false name=(anonymous)","processImageUUID":"CD58B635-1F96-38BD-BF3B-DBA7CA293E28","traceID":6727155769999364,"processID":200,"senderProgramCounter":43862,"parentActivityIdentifier":0} +{"timezoneName":"","messageType":"Default","eventType":"logEvent","source":null,"formatString":"%{public}s(%d) %{public}s[%p] TLS connected [version(0xx) ciphersuite(%s) group(0xx) signature_alg(0xx) alpn(%{public}s) resumed(%d) offered_ticket(%d) false_started(%d) ocsp_received(%d) sct_received(%d) connect_time(%llums) flight_time(%llums) rtt(%llums) write_stalls(%zu) read_stalls(%zu)]","userID":200,"activityIdentifier":1606042,"subsystem":"com.apple.network","category":"boringssl","threadID":1133904,"senderImageUUID":"E8DA37CF-0F7A-38D1-8D87-3730FB158249","backtrace":{"frames":[{"imageOffset":182440,"imageUUID":"E8DA37CF-0F7A-38D1-8D87-3730FB158249"}]},"bootUUID":"218031E6-E47F-4A77-B7FC-5A57B049F4BC","processImagePath":"\/System\/Library\/CoreServices\/Software Update.app\/Contents\/Resources\/softwareupdated","senderImagePath":"\/usr\/lib\/libboringssl.dylib","timestamp":"2025-10-15 16:23:24.143463+0530","machTimestamp":739235190835402,"eventMessage":"nw_protocol_boringssl_signal_connected(753) [C136.1.1.1:2][0x7fd20f8f8e60] TLS connected [version(0x0304) ciphersuite(TLS_AES_256_GCM_SHA384) group(0x001d) signature_alg(0x0804) alpn(http\/1.1) resumed(0) offered_ticket(0) false_started(0) ocsp_received(1) sct_received(0) connect_time(457ms) flight_time(10ms) rtt(10ms) write_stalls(0) read_stalls(6)]","processImageUUID":"CD58B635-1F96-38BD-BF3B-DBA7CA293E28","traceID":390070235526660100,"processID":200,"senderProgramCounter":182440,"parentActivityIdentifier":0} +{"timezoneName":"","messageType":"Default","eventType":"logEvent","source":null,"formatString":"Task <%{public,uuid_t}.16P>.<%lu> summary for %{public}s {transaction_duration_ms=%u, response_status=%ld, connection=%llu, protocol=\"%{public}@\", domain_lookup_duration_ms=%u, connect_duration_ms=%u, secure_connection_duration_ms=%u, private_relay=%{bool}d, request_start_ms=%u, request_duration_ms=%u, response_start_ms=%u, response_duration_ms=%u, request_bytes=%lld, response_bytes=%lld, cache_hit=%{bool}d}","userID":200,"activityIdentifier":1606042,"subsystem":"com.apple.CFNetwork","category":"Summary","threadID":1133904,"senderImageUUID":"31798E5A-30D3-381D-BC8D-F9AF4154382B","backtrace":{"frames":[{"imageOffset":987343,"imageUUID":"31798E5A-30D3-381D-BC8D-F9AF4154382B"}]},"bootUUID":"218031E6-E47F-4A77-B7FC-5A57B049F4BC","processImagePath":"\/System\/Library\/CoreServices\/Software Update.app\/Contents\/Resources\/softwareupdated","senderImagePath":"\/System\/Library\/Frameworks\/CFNetwork.framework\/Versions\/A\/CFNetwork","timestamp":"2025-10-15 16:23:24.536497+0530","machTimestamp":739235583869205,"eventMessage":"Task <400732FC-E156-46A9-A05E-4C557AF74568>.<74> summary for task success {transaction_duration_ms=1190, response_status=200, connection=136, protocol=\"http\/1.1\", domain_lookup_duration_ms=247, connect_duration_ms=5040000000000000, secure_connection_duration_ms=457, private_relay=false, request_start_ms=842, request_duration_ms=58, response_start_ms=1189, response_duration_ms=0, request_bytes=1440, response_bytes=676, cache_hit=true}","processImageUUID":"CD58B635-1F96-38BD-BF3B-DBA7CA293E28","traceID":405718141415784452,"processID":200,"senderProgramCounter":987343,"parentActivityIdentifier":0} diff --git a/packages/macos/data_stream/system_change/_dev/test/pipeline/test-system-change.log-expected.json b/packages/macos/data_stream/system_change/_dev/test/pipeline/test-system-change.log-expected.json new file mode 100644 index 00000000000..89c79c5e7de --- /dev/null +++ b/packages/macos/data_stream/system_change/_dev/test/pipeline/test-system-change.log-expected.json @@ -0,0 +1,720 @@ +{ + "expected": [ + { + "@timestamp": "2025-10-13T10:15:56.155Z", + "ecs": { + "version": "9.2.0" + }, + "event": { + "category": [ + "configuration" + ], + "kind": "event", + "original": "{\"timezoneName\":\"\",\"messageType\":\"Default\",\"eventType\":\"logEvent\",\"source\":null,\"formatString\":\"%{public}s %{public}s flags=%{network:tcp_flags}x seq=%u, ack=%u, win=%u state=%{network:tcp_state}x rcv_nxt=%u, snd_una=%u\",\"userID\":200,\"activityIdentifier\":0,\"subsystem\":\"com.apple.network\",\"category\":\"tcp\",\"threadID\":988590,\"senderImageUUID\":\"C5739A55-A9D3-3AFB-9AC9-652E5799B536\",\"backtrace\":{\"frames\":[{\"imageOffset\":281600,\"imageUUID\":\"C5739A55-A9D3-3AFB-9AC9-652E5799B536\"}]},\"bootUUID\":\"218031E6-E47F-4A77-B7FC-5A57B049F4BC\",\"processImagePath\":\"\\/System\\/Library\\/CoreServices\\/Software Update.app\\/Contents\\/Resources\\/softwareupdated\",\"senderImagePath\":\"\\/usr\\/lib\\/libusrtcp.dylib\",\"timestamp\":\"2025-10-13 15:45:56.155816+0530\",\"machTimestamp\":725858385613246,\"eventMessage\":\"tcp_output [C129.1.1.1:3] flags=[F.] seq=3768828915, ack=3786980776, win=2048 state=FIN_WAIT_1 rcv_nxt=3786980776, snd_una=3768828891\",\"processImageUUID\":\"CD58B635-1F96-38BD-BF3B-DBA7CA293E28\",\"traceID\":1202012843394990084,\"processID\":200,\"senderProgramCounter\":281600,\"parentActivityIdentifier\":0}", + "type": [ + "info" + ] + }, + "log": { + "level": "info" + }, + "macos": { + "activity_identifier": "0", + "backtrace": { + "frames": [ + { + "image": { + "offset": "281600", + "uuid": "C5739A55-A9D3-3AFB-9AC9-652E5799B536" + } + } + ] + }, + "boot_uuid": "218031E6-E47F-4A77-B7FC-5A57B049F4BC", + "category": "tcp", + "event": { + "message": { + "ack": "3786980776", + "description": "tcp_output [C129.1.1.1:3] flags=[F.] seq=3768828915, ack=3786980776, win=2048 state=FIN_WAIT_1 rcv_nxt=3786980776, snd_una=3768828891", + "flags": "F.", + "seq": "3768828915", + "snd_una": "", + "state": "FIN_WAIT_1 rcv_nxt=3786980776, ", + "win": "2048" + }, + "type": "logEvent" + }, + "format_string": "%{public}s %{public}s flags=%{network:tcp_flags}x seq=%u, ack=%u, win=%u state=%{network:tcp_state}x rcv_nxt=%u, snd_una=%u", + "mach_timestamp": 7.25858385613246E14, + "parent_activity_identifier": "0", + "process": { + "image_path": "/System/Library/CoreServices/Software Update.app/Contents/Resources/softwareupdated", + "image_uuid": "CD58B635-1F96-38BD-BF3B-DBA7CA293E28" + }, + "sender": { + "image_path": "/usr/lib/libusrtcp.dylib", + "image_uuid": "C5739A55-A9D3-3AFB-9AC9-652E5799B536", + "program_counter": 281600 + }, + "subsystem": "com.apple.network", + "trace_id": "1202012843394990084" + }, + "message": "tcp_output [C129.1.1.1:3] flags=[F.] seq=3768828915, ack=3786980776, win=2048 state=FIN_WAIT_1 rcv_nxt=3786980776, snd_una=3768828891", + "process": { + "pid": 200, + "thread": { + "id": 988590 + } + }, + "related": { + "user": [ + "200" + ] + }, + "tags": [ + "preserve_original_event" + ], + "user": { + "id": "200" + } + }, + { + "@timestamp": "2025-10-13T10:15:56.256Z", + "ecs": { + "version": "9.2.0" + }, + "event": { + "category": [ + "configuration" + ], + "kind": "event", + "original": "{\"timezoneName\":\"\",\"messageType\":\"Default\",\"eventType\":\"logEvent\",\"source\":null,\"formatString\":\"%{public}s %{public}s flags=%{network:tcp_flags}x seq=%u, ack=%u, win=%u state=%{network:tcp_state}x rcv_nxt=%u, snd_una=%u\",\"userID\":200,\"activityIdentifier\":0,\"subsystem\":\"com.apple.network\",\"category\":\"tcp\",\"threadID\":988590,\"senderImageUUID\":\"C5739A55-A9D3-3AFB-9AC9-652E5799B536\",\"backtrace\":{\"frames\":[{\"imageOffset\":182769,\"imageUUID\":\"C5739A55-A9D3-3AFB-9AC9-652E5799B536\"}]},\"bootUUID\":\"218031E6-E47F-4A77-B7FC-5A57B049F4BC\",\"processImagePath\":\"\\/System\\/Library\\/CoreServices\\/Software Update.app\\/Contents\\/Resources\\/softwareupdated\",\"senderImagePath\":\"\\/usr\\/lib\\/libusrtcp.dylib\",\"timestamp\":\"2025-10-13 15:45:56.256214+0530\",\"machTimestamp\":725858486011276,\"eventMessage\":\"tcp_input [C129.1.1.1:3] flags=[F.] seq=3786980800, ack=3768828915, win=576 state=CLOSED rcv_nxt=3786980776, snd_una=3768828891\",\"processImageUUID\":\"CD58B635-1F96-38BD-BF3B-DBA7CA293E28\",\"traceID\":1202012843394990084,\"processID\":200,\"senderProgramCounter\":182769,\"parentActivityIdentifier\":0}", + "type": [ + "info" + ] + }, + "log": { + "level": "info" + }, + "macos": { + "activity_identifier": "0", + "backtrace": { + "frames": [ + { + "image": { + "offset": "182769", + "uuid": "C5739A55-A9D3-3AFB-9AC9-652E5799B536" + } + } + ] + }, + "boot_uuid": "218031E6-E47F-4A77-B7FC-5A57B049F4BC", + "category": "tcp", + "event": { + "message": { + "ack": "3768828915", + "description": "tcp_input [C129.1.1.1:3] flags=[F.] seq=3786980800, ack=3768828915, win=576 state=CLOSED rcv_nxt=3786980776, snd_una=3768828891", + "flags": "F.", + "seq": "3786980800", + "snd_una": "", + "state": "CLOSED rcv_nxt=3786980776, ", + "win": "576" + }, + "type": "logEvent" + }, + "format_string": "%{public}s %{public}s flags=%{network:tcp_flags}x seq=%u, ack=%u, win=%u state=%{network:tcp_state}x rcv_nxt=%u, snd_una=%u", + "mach_timestamp": 7.25858486011276E14, + "parent_activity_identifier": "0", + "process": { + "image_path": "/System/Library/CoreServices/Software Update.app/Contents/Resources/softwareupdated", + "image_uuid": "CD58B635-1F96-38BD-BF3B-DBA7CA293E28" + }, + "sender": { + "image_path": "/usr/lib/libusrtcp.dylib", + "image_uuid": "C5739A55-A9D3-3AFB-9AC9-652E5799B536", + "program_counter": 182769 + }, + "subsystem": "com.apple.network", + "trace_id": "1202012843394990084" + }, + "message": "tcp_input [C129.1.1.1:3] flags=[F.] seq=3786980800, ack=3768828915, win=576 state=CLOSED rcv_nxt=3786980776, snd_una=3768828891", + "process": { + "pid": 200, + "thread": { + "id": 988590 + } + }, + "related": { + "user": [ + "200" + ] + }, + "tags": [ + "preserve_original_event" + ], + "user": { + "id": "200" + } + }, + { + "@timestamp": "2025-10-13T10:16:58.699Z", + "destination": { + "port": 443 + }, + "ecs": { + "version": "9.2.0" + }, + "event": { + "category": [ + "configuration" + ], + "kind": "event", + "original": "{\"timezoneName\":\"\",\"messageType\":\"Default\",\"eventType\":\"logEvent\",\"source\":null,\"formatString\":\"%{public}s %{public}s \\n\\t[%{public,uuid_t}.16P %{sensitive}s:%{public}d<->%{sensitive}s:%{public}d]\\n\\tInit: %d, Conn_Time: %u.ums, SYNs: %u, WR_T: %d\\/%d, RD_T: %d\\/%d, TFO: %d\\/%d\\/%d, ECN: %d\\/%d\\/%d, Accurate ECN (client\\/server): %s\\/%s, TS: %d, TSO: %d\\n\\trtt_cache: %{public}s, rtt_upd: %d, rtt: %u.ums, rtt_var: %u.ums rtt_nc: %u.ums, rtt_var_nc: %u.ums base rtt: %ums\\n\\tACKs-compressed: %u, ACKs delayed: %u delayed ACKs sent: %u\",\"userID\":200,\"activityIdentifier\":0,\"subsystem\":\"com.apple.network\",\"category\":\"tcp\",\"threadID\":988590,\"senderImageUUID\":\"C5739A55-A9D3-3AFB-9AC9-652E5799B536\",\"backtrace\":{\"frames\":[{\"imageOffset\":139368,\"imageUUID\":\"C5739A55-A9D3-3AFB-9AC9-652E5799B536\"}]},\"bootUUID\":\"218031E6-E47F-4A77-B7FC-5A57B049F4BC\",\"processImagePath\":\"\\/System\\/Library\\/CoreServices\\/Software Update.app\\/Contents\\/Resources\\/softwareupdated\",\"senderImagePath\":\"\\/usr\\/lib\\/libusrtcp.dylib\",\"timestamp\":\"2025-10-13 15:46:58.699993+0530\",\"machTimestamp\":725920929789896,\"eventMessage\":\"nw_protocol_tcp_log_summary [C131.1.1.1:3] \\n\\t[263F0F80-1925-4AF8-ABE3-ED1FDDCD00EF :63735<->:443]\\n\\tInit: 1, Conn_Time: 40.819ms, SYNs: 1, WR_T: 0\\/0, RD_T: 0\\/0, TFO: 0\\/0\\/0, ECN: 0\\/0\\/0, Accurate ECN (client\\/server): Disabled\\/Disabled, TS: 1, TSO: 1\\n\\trtt_cache: none, rtt_upd: 7, rtt: 126.375ms, rtt_var: 91.187ms rtt_nc: 126.375ms, rtt_var_nc: 91.187ms base rtt: 21ms\\n\\tACKs-compressed: 2, ACKs delayed: 0 delayed ACKs sent: 0\",\"processImageUUID\":\"CD58B635-1F96-38BD-BF3B-DBA7CA293E28\",\"traceID\":1201924813745291268,\"processID\":200,\"senderProgramCounter\":139368,\"parentActivityIdentifier\":0}", + "type": [ + "info" + ] + }, + "log": { + "level": "info" + }, + "macos": { + "activity_identifier": "0", + "backtrace": { + "frames": [ + { + "image": { + "offset": "139368", + "uuid": "C5739A55-A9D3-3AFB-9AC9-652E5799B536" + } + } + ] + }, + "boot_uuid": "218031E6-E47F-4A77-B7FC-5A57B049F4BC", + "category": "tcp", + "event": { + "message": { + "accurate_ecn_client": "Disabled", + "accurate_ecn_server": "Disabled", + "acks_compressed": 2, + "acks_delayed": 0, + "base_rtt_ms": "21ms", + "connection_identifier": "C131.1.1.1:3", + "connection_time": "40.819ms", + "connection_uuid": "263F0F80-1925-4AF8-ABE3-ED1FDDCD00EF", + "delayed_acks_sent": 0, + "description": "nw_protocol_tcp_log_summary [C131.1.1.1:3] \n\t[263F0F80-1925-4AF8-ABE3-ED1FDDCD00EF :63735<->:443]\n\tInit: 1, Conn_Time: 40.819ms, SYNs: 1, WR_T: 0/0, RD_T: 0/0, TFO: 0/0/0, ECN: 0/0/0, Accurate ECN (client/server): Disabled/Disabled, TS: 1, TSO: 1\n\trtt_cache: none, rtt_upd: 7, rtt: 126.375ms, rtt_var: 91.187ms rtt_nc: 126.375ms, rtt_var_nc: 91.187ms base rtt: 21ms\n\tACKs-compressed: 2, ACKs delayed: 0 delayed ACKs sent: 0", + "ecn_in": 0, + "ecn_miss": 0, + "ecn_out": 0, + "init_flag": 1, + "rd_t_in": 0, + "rd_t_out": 0, + "rtt": "126.375ms", + "rtt_cache": "none", + "rtt_nc_ms": "126.375ms", + "rtt_updates": 7, + "rtt_var_ms": "91.187ms", + "rtt_var_nc_ms": "91.187ms", + "syns": 1, + "tfo_in": 0, + "tfo_miss": 0, + "tfo_out": 0, + "timestamp_enabled": 1, + "tso_enabled": 1, + "wr_t_in": 0, + "wr_t_out": 0 + }, + "type": "logEvent" + }, + "format_string": "%{public}s %{public}s \n\t[%{public,uuid_t}.16P %{sensitive}s:%{public}d<->%{sensitive}s:%{public}d]\n\tInit: %d, Conn_Time: %u.ums, SYNs: %u, WR_T: %d/%d, RD_T: %d/%d, TFO: %d/%d/%d, ECN: %d/%d/%d, Accurate ECN (client/server): %s/%s, TS: %d, TSO: %d\n\trtt_cache: %{public}s, rtt_upd: %d, rtt: %u.ums, rtt_var: %u.ums rtt_nc: %u.ums, rtt_var_nc: %u.ums base rtt: %ums\n\tACKs-compressed: %u, ACKs delayed: %u delayed ACKs sent: %u", + "mach_timestamp": 7.25920929789896E14, + "parent_activity_identifier": "0", + "process": { + "image_path": "/System/Library/CoreServices/Software Update.app/Contents/Resources/softwareupdated", + "image_uuid": "CD58B635-1F96-38BD-BF3B-DBA7CA293E28" + }, + "sender": { + "image_path": "/usr/lib/libusrtcp.dylib", + "image_uuid": "C5739A55-A9D3-3AFB-9AC9-652E5799B536", + "program_counter": 139368 + }, + "subsystem": "com.apple.network", + "trace_id": "1201924813745291268" + }, + "message": "nw_protocol_tcp_log_summary [C131.1.1.1:3] \n\t[263F0F80-1925-4AF8-ABE3-ED1FDDCD00EF :63735<->:443]\n\tInit: 1, Conn_Time: 40.819ms, SYNs: 1, WR_T: 0/0, RD_T: 0/0, TFO: 0/0/0, ECN: 0/0/0, Accurate ECN (client/server): Disabled/Disabled, TS: 1, TSO: 1\n\trtt_cache: none, rtt_upd: 7, rtt: 126.375ms, rtt_var: 91.187ms rtt_nc: 126.375ms, rtt_var_nc: 91.187ms base rtt: 21ms\n\tACKs-compressed: 2, ACKs delayed: 0 delayed ACKs sent: 0", + "process": { + "pid": 200, + "thread": { + "id": 988590 + } + }, + "related": { + "user": [ + "200" + ] + }, + "source": { + "port": 63735 + }, + "tags": [ + "preserve_original_event" + ], + "user": { + "id": "200" + } + }, + { + "@timestamp": "2025-10-13T10:16:58.706Z", + "destination": { + "bytes": 1670, + "packets": 19, + "port": 443 + }, + "ecs": { + "version": "9.2.0" + }, + "event": { + "category": [ + "configuration" + ], + "kind": "event", + "original": "{\"timezoneName\":\"\",\"messageType\":\"Default\",\"eventType\":\"logEvent\",\"source\":null,\"formatString\":\"[C%u %{public,uuid_t}.16P %{public}s %{public}@] cancelled\\n\\t[C%{public}s %{public,uuid_t}.16P %{public}s<->%{public}s]\\n\\tConnected Path: %{public}@\\n\\tPrivacy Stance: %{public}s\\n\\tDuration: %u.us, DNS @%u.us took %u.us, %{public}s @%u.us took %u.us, %{public}s took %u.us\\n\\tbytes in\\/out: %llu\\/%llu, packets in\\/out: %llu\\/%llu, rtt: %u.us, retransmitted bytes: %llu, out-of-order bytes: %llu\\n\\tecn packets sent\\/acked\\/marked\\/lost: %llu\\/%llu\\/%llu\\/%llu\",\"userID\":200,\"activityIdentifier\":0,\"subsystem\":\"com.apple.network\",\"category\":\"connection\",\"threadID\":988590,\"senderImageUUID\":\"E481CDB0-9C0C-3BBC-87FA-006C66D3223E\",\"backtrace\":{\"frames\":[{\"imageOffset\":9418645,\"imageUUID\":\"E481CDB0-9C0C-3BBC-87FA-006C66D3223E\"}]},\"bootUUID\":\"218031E6-E47F-4A77-B7FC-5A57B049F4BC\",\"processImagePath\":\"\\/System\\/Library\\/CoreServices\\/Software Update.app\\/Contents\\/Resources\\/softwareupdated\",\"senderImagePath\":\"\\/System\\/Library\\/Frameworks\\/Network.framework\\/Versions\\/A\\/Network\",\"timestamp\":\"2025-10-13 15:46:58.706400+0530\",\"machTimestamp\":725920936196661,\"eventMessage\":\"[C131 7F4FBE47-BF28-48DA-ABBB-0C219E463C9D Hostname#a4134bf8:443 quic-connection, url hash: ffb1585e, definite, attribution: developer] cancelled\\n\\t[C131.1.1.1 DD5F0E62-F851-47EE-A14E-2F3668A1FFCC 172.20.1.254:63735<->IPv4#86785abb:443]\\n\\tConnected Path: satisfied (Path is satisfied), viable, interface: en0[802.11], ipv4, dns, uses wifi\\n\\tPrivacy Stance: Not Eligible\\n\\tDuration: 122.754s, DNS @0.002s took 0.318s, TCP @0.324s took 0.041s, TLS 1.3 took 0.426s\\n\\tbytes in\\/out: 17140\\/1670, packets in\\/out: 17\\/19, rtt: 0.126s, retransmitted bytes: 517, out-of-order bytes: 0\\n\\tecn packets sent\\/acked\\/marked\\/lost: 0\\/0\\/0\\/0\",\"processImageUUID\":\"CD58B635-1F96-38BD-BF3B-DBA7CA293E28\",\"traceID\":589370805553725444,\"processID\":200,\"senderProgramCounter\":9418645,\"parentActivityIdentifier\":0}", + "type": [ + "info" + ] + }, + "host": { + "id": "a4134bf8" + }, + "log": { + "level": "info" + }, + "macos": { + "activity_identifier": "0", + "backtrace": { + "frames": [ + { + "image": { + "offset": "9418645", + "uuid": "E481CDB0-9C0C-3BBC-87FA-006C66D3223E" + } + } + ] + }, + "boot_uuid": "218031E6-E47F-4A77-B7FC-5A57B049F4BC", + "category": "connection", + "event": { + "message": { + "attribution": "developer", + "connection_detail": "131.1.1.1", + "connection_id": "131", + "connection_uuid": "DD5F0E62-F851-47EE-A14E-2F3668A1FFCC", + "description": "[C131 7F4FBE47-BF28-48DA-ABBB-0C219E463C9D Hostname#a4134bf8:443 quic-connection, url hash: ffb1585e, definite, attribution: developer] cancelled\n\t[C131.1.1.1 DD5F0E62-F851-47EE-A14E-2F3668A1FFCC 172.20.1.254:63735<->IPv4#86785abb:443]\n\tConnected Path: satisfied (Path is satisfied), viable, interface: en0[802.11], ipv4, dns, uses wifi\n\tPrivacy Stance: Not Eligible\n\tDuration: 122.754s, DNS @0.002s took 0.318s, TCP @0.324s took 0.041s, TLS 1.3 took 0.426s\n\tbytes in/out: 17140/1670, packets in/out: 17/19, rtt: 0.126s, retransmitted bytes: 517, out-of-order bytes: 0\n\tecn packets sent/acked/marked/lost: 0/0/0/0", + "dns_duration": "0.318s", + "dns_start": "0.002s", + "duration": "122.754s", + "ecn_acked": 0, + "ecn_lost": 0, + "ecn_marked": 0, + "ecn_sent": 0, + "hostname_port": 443, + "interface": "en0[802.11]", + "out_of_order_bytes": 0, + "path_status": "satisfied (Path is satisfied)", + "privacy_stance": "Not Eligible", + "retransmitted_bytes": 517, + "rtt": "0.126s", + "server_id": "86785abb", + "session_uuid": "7F4FBE47-BF28-48DA-ABBB-0C219E463C9D", + "tcp_duration": "0.041s", + "tcp_start": "0.324s", + "tls_duration": "0.426s", + "tls_version": "1.3", + "url_hash": "ffb1585e" + }, + "type": "logEvent" + }, + "format_string": "[C%u %{public,uuid_t}.16P %{public}s %{public}@] cancelled\n\t[C%{public}s %{public,uuid_t}.16P %{public}s<->%{public}s]\n\tConnected Path: %{public}@\n\tPrivacy Stance: %{public}s\n\tDuration: %u.us, DNS @%u.us took %u.us, %{public}s @%u.us took %u.us, %{public}s took %u.us\n\tbytes in/out: %llu/%llu, packets in/out: %llu/%llu, rtt: %u.us, retransmitted bytes: %llu, out-of-order bytes: %llu\n\tecn packets sent/acked/marked/lost: %llu/%llu/%llu/%llu", + "mach_timestamp": 7.25920936196661E14, + "parent_activity_identifier": "0", + "process": { + "image_path": "/System/Library/CoreServices/Software Update.app/Contents/Resources/softwareupdated", + "image_uuid": "CD58B635-1F96-38BD-BF3B-DBA7CA293E28" + }, + "sender": { + "image_path": "/System/Library/Frameworks/Network.framework/Versions/A/Network", + "image_uuid": "E481CDB0-9C0C-3BBC-87FA-006C66D3223E", + "program_counter": 9418645 + }, + "subsystem": "com.apple.network", + "trace_id": "589370805553725444" + }, + "message": "[C131 7F4FBE47-BF28-48DA-ABBB-0C219E463C9D Hostname#a4134bf8:443 quic-connection, url hash: ffb1585e, definite, attribution: developer] cancelled\n\t[C131.1.1.1 DD5F0E62-F851-47EE-A14E-2F3668A1FFCC 172.20.1.254:63735<->IPv4#86785abb:443]\n\tConnected Path: satisfied (Path is satisfied), viable, interface: en0[802.11], ipv4, dns, uses wifi\n\tPrivacy Stance: Not Eligible\n\tDuration: 122.754s, DNS @0.002s took 0.318s, TCP @0.324s took 0.041s, TLS 1.3 took 0.426s\n\tbytes in/out: 17140/1670, packets in/out: 17/19, rtt: 0.126s, retransmitted bytes: 517, out-of-order bytes: 0\n\tecn packets sent/acked/marked/lost: 0/0/0/0", + "network": { + "bytes": 18810, + "packets": 36 + }, + "process": { + "pid": 200, + "thread": { + "id": 988590 + } + }, + "related": { + "hosts": [ + "a4134bf8" + ], + "ip": [ + "172.20.1.254" + ], + "user": [ + "200" + ] + }, + "source": { + "bytes": 17140, + "ip": "172.20.1.254", + "packets": 17, + "port": 63735 + }, + "tags": [ + "preserve_original_event" + ], + "user": { + "id": "200" + } + }, + { + "@timestamp": "2025-10-15T10:51:14.492Z", + "ecs": { + "version": "9.2.0" + }, + "event": { + "category": [ + "configuration" + ], + "kind": "event", + "original": "{\"timezoneName\":\"\",\"messageType\":\"Default\",\"eventType\":\"logEvent\",\"source\":null,\"formatString\":\"[C%u %{public,uuid_t}.16P %{public}s %{public}@] %{public}scancel\",\"userID\":200,\"activityIdentifier\":0,\"subsystem\":\"com.apple.network\",\"category\":\"connection\",\"threadID\":1132254,\"senderImageUUID\":\"E481CDB0-9C0C-3BBC-87FA-006C66D3223E\",\"backtrace\":{\"frames\":[{\"imageOffset\":3693596,\"imageUUID\":\"E481CDB0-9C0C-3BBC-87FA-006C66D3223E\"}]},\"bootUUID\":\"218031E6-E47F-4A77-B7FC-5A57B049F4BC\",\"processImagePath\":\"\\/System\\/Library\\/CoreServices\\/Software Update.app\\/Contents\\/Resources\\/softwareupdated\",\"senderImagePath\":\"\\/System\\/Library\\/Frameworks\\/Network.framework\\/Versions\\/A\\/Network\",\"timestamp\":\"2025-10-15 16:21:14.492797+0530\",\"machTimestamp\":739105540170070,\"eventMessage\":\"[C135 59E5BC33-3431-4F8A-981A-B4AF31960D47 Hostname#3fba6d68:443 tcp, url hash: 15f2f095, tls, definite, attribution: developer] cancel\",\"processImageUUID\":\"CD58B635-1F96-38BD-BF3B-DBA7CA293E28\",\"traceID\":587172881809801220,\"processID\":200,\"senderProgramCounter\":3693596,\"parentActivityIdentifier\":0}", + "type": [ + "info" + ] + }, + "log": { + "level": "info" + }, + "macos": { + "activity_identifier": "0", + "backtrace": { + "frames": [ + { + "image": { + "offset": "3693596", + "uuid": "E481CDB0-9C0C-3BBC-87FA-006C66D3223E" + } + } + ] + }, + "boot_uuid": "218031E6-E47F-4A77-B7FC-5A57B049F4BC", + "category": "connection", + "event": { + "message": { + "description": "[C135 59E5BC33-3431-4F8A-981A-B4AF31960D47 Hostname#3fba6d68:443 tcp, url hash: 15f2f095, tls, definite, attribution: developer] cancel" + }, + "type": "logEvent" + }, + "format_string": "[C%u %{public,uuid_t}.16P %{public}s %{public}@] %{public}scancel", + "mach_timestamp": 7.3910554017007E14, + "parent_activity_identifier": "0", + "process": { + "image_path": "/System/Library/CoreServices/Software Update.app/Contents/Resources/softwareupdated", + "image_uuid": "CD58B635-1F96-38BD-BF3B-DBA7CA293E28" + }, + "sender": { + "image_path": "/System/Library/Frameworks/Network.framework/Versions/A/Network", + "image_uuid": "E481CDB0-9C0C-3BBC-87FA-006C66D3223E", + "program_counter": 3693596 + }, + "subsystem": "com.apple.network", + "trace_id": "587172881809801220" + }, + "message": "[C135 59E5BC33-3431-4F8A-981A-B4AF31960D47 Hostname#3fba6d68:443 tcp, url hash: 15f2f095, tls, definite, attribution: developer] cancel", + "process": { + "pid": 200, + "thread": { + "id": 1132254 + } + }, + "related": { + "user": [ + "200" + ] + }, + "tags": [ + "preserve_original_event" + ], + "user": { + "id": "200" + } + }, + { + "@timestamp": "2025-10-15T10:51:22.947Z", + "ecs": { + "version": "9.2.0" + }, + "event": { + "category": [ + "configuration" + ], + "kind": "event", + "original": "{\"timezoneName\":\"\",\"messageType\":\"Default\",\"eventType\":\"logEvent\",\"source\":null,\"formatString\":\"[%p] activating connection: mach=%{bool}d listener=%{bool}d peer=%{bool}d name=%{public}s\",\"userID\":200,\"activityIdentifier\":1606042,\"subsystem\":\"com.apple.xpc\",\"category\":\"connection\",\"threadID\":1131759,\"senderImageUUID\":\"ABCE7E08-7D29-3BE0-AF60-F0DC752E951C\",\"backtrace\":{\"frames\":[{\"imageOffset\":43862,\"imageUUID\":\"ABCE7E08-7D29-3BE0-AF60-F0DC752E951C\"}]},\"bootUUID\":\"218031E6-E47F-4A77-B7FC-5A57B049F4BC\",\"processImagePath\":\"\\/System\\/Library\\/CoreServices\\/Software Update.app\\/Contents\\/Resources\\/softwareupdated\",\"senderImagePath\":\"\\/usr\\/lib\\/system\\/libxpc.dylib\",\"timestamp\":\"2025-10-15 16:21:22.947210+0530\",\"machTimestamp\":739113994582741,\"eventMessage\":\"[0x7fd20cffa690] activating connection: mach=false listener=false peer=false name=(anonymous)\",\"processImageUUID\":\"CD58B635-1F96-38BD-BF3B-DBA7CA293E28\",\"traceID\":6727155769999364,\"processID\":200,\"senderProgramCounter\":43862,\"parentActivityIdentifier\":0}", + "type": [ + "info" + ] + }, + "log": { + "level": "info" + }, + "macos": { + "activity_identifier": "1606042", + "backtrace": { + "frames": [ + { + "image": { + "offset": "43862", + "uuid": "ABCE7E08-7D29-3BE0-AF60-F0DC752E951C" + } + } + ] + }, + "boot_uuid": "218031E6-E47F-4A77-B7FC-5A57B049F4BC", + "category": "connection", + "event": { + "message": { + "description": "[0x7fd20cffa690] activating connection: mach=false listener=false peer=false name=(anonymous)", + "listener": false, + "mach": false, + "name": "(anonymous)", + "peer": false + }, + "type": "logEvent" + }, + "format_string": "[%p] activating connection: mach=%{bool}d listener=%{bool}d peer=%{bool}d name=%{public}s", + "mach_timestamp": 7.39113994582741E14, + "parent_activity_identifier": "0", + "process": { + "image_path": "/System/Library/CoreServices/Software Update.app/Contents/Resources/softwareupdated", + "image_uuid": "CD58B635-1F96-38BD-BF3B-DBA7CA293E28" + }, + "sender": { + "image_path": "/usr/lib/system/libxpc.dylib", + "image_uuid": "ABCE7E08-7D29-3BE0-AF60-F0DC752E951C", + "program_counter": 43862 + }, + "subsystem": "com.apple.xpc", + "trace_id": "6727155769999364" + }, + "message": "[0x7fd20cffa690] activating connection: mach=false listener=false peer=false name=(anonymous)", + "process": { + "pid": 200, + "thread": { + "id": 1131759 + } + }, + "related": { + "user": [ + "200" + ] + }, + "tags": [ + "preserve_original_event" + ], + "user": { + "id": "200" + } + }, + { + "@timestamp": "2025-10-15T10:53:24.143Z", + "ecs": { + "version": "9.2.0" + }, + "event": { + "category": [ + "configuration" + ], + "kind": "event", + "original": "{\"timezoneName\":\"\",\"messageType\":\"Default\",\"eventType\":\"logEvent\",\"source\":null,\"formatString\":\"%{public}s(%d) %{public}s[%p] TLS connected [version(0xx) ciphersuite(%s) group(0xx) signature_alg(0xx) alpn(%{public}s) resumed(%d) offered_ticket(%d) false_started(%d) ocsp_received(%d) sct_received(%d) connect_time(%llums) flight_time(%llums) rtt(%llums) write_stalls(%zu) read_stalls(%zu)]\",\"userID\":200,\"activityIdentifier\":1606042,\"subsystem\":\"com.apple.network\",\"category\":\"boringssl\",\"threadID\":1133904,\"senderImageUUID\":\"E8DA37CF-0F7A-38D1-8D87-3730FB158249\",\"backtrace\":{\"frames\":[{\"imageOffset\":182440,\"imageUUID\":\"E8DA37CF-0F7A-38D1-8D87-3730FB158249\"}]},\"bootUUID\":\"218031E6-E47F-4A77-B7FC-5A57B049F4BC\",\"processImagePath\":\"\\/System\\/Library\\/CoreServices\\/Software Update.app\\/Contents\\/Resources\\/softwareupdated\",\"senderImagePath\":\"\\/usr\\/lib\\/libboringssl.dylib\",\"timestamp\":\"2025-10-15 16:23:24.143463+0530\",\"machTimestamp\":739235190835402,\"eventMessage\":\"nw_protocol_boringssl_signal_connected(753) [C136.1.1.1:2][0x7fd20f8f8e60] TLS connected [version(0x0304) ciphersuite(TLS_AES_256_GCM_SHA384) group(0x001d) signature_alg(0x0804) alpn(http\\/1.1) resumed(0) offered_ticket(0) false_started(0) ocsp_received(1) sct_received(0) connect_time(457ms) flight_time(10ms) rtt(10ms) write_stalls(0) read_stalls(6)]\",\"processImageUUID\":\"CD58B635-1F96-38BD-BF3B-DBA7CA293E28\",\"traceID\":390070235526660100,\"processID\":200,\"senderProgramCounter\":182440,\"parentActivityIdentifier\":0}", + "type": [ + "info" + ] + }, + "log": { + "level": "info" + }, + "macos": { + "activity_identifier": "1606042", + "backtrace": { + "frames": [ + { + "image": { + "offset": "182440", + "uuid": "E8DA37CF-0F7A-38D1-8D87-3730FB158249" + } + } + ] + }, + "boot_uuid": "218031E6-E47F-4A77-B7FC-5A57B049F4BC", + "category": "boringssl", + "event": { + "message": { + "alpn": "http/1.1", + "cipher_suite": "TLS_AES_256_GCM_SHA384", + "connection_identifier": "C136.1.1.1:2", + "connection_time": "457ms", + "description": "nw_protocol_boringssl_signal_connected(753) [C136.1.1.1:2][0x7fd20f8f8e60] TLS connected [version(0x0304) ciphersuite(TLS_AES_256_GCM_SHA384) group(0x001d) signature_alg(0x0804) alpn(http/1.1) resumed(0) offered_ticket(0) false_started(0) ocsp_received(1) sct_received(0) connect_time(457ms) flight_time(10ms) rtt(10ms) write_stalls(0) read_stalls(6)]", + "false_started": false, + "flight_time": "10ms", + "group": "0x001d", + "ocsp_received": true, + "offered_ticket": false, + "read_stalls": 6, + "resumed": false, + "rtt": "10ms", + "sct_received": false, + "tls_version": "0x0304", + "write_stalls": 0 + }, + "type": "logEvent" + }, + "format_string": "%{public}s(%d) %{public}s[%p] TLS connected [version(0xx) ciphersuite(%s) group(0xx) signature_alg(0xx) alpn(%{public}s) resumed(%d) offered_ticket(%d) false_started(%d) ocsp_received(%d) sct_received(%d) connect_time(%llums) flight_time(%llums) rtt(%llums) write_stalls(%zu) read_stalls(%zu)]", + "mach_timestamp": 7.39235190835402E14, + "parent_activity_identifier": "0", + "process": { + "image_path": "/System/Library/CoreServices/Software Update.app/Contents/Resources/softwareupdated", + "image_uuid": "CD58B635-1F96-38BD-BF3B-DBA7CA293E28" + }, + "sender": { + "image_path": "/usr/lib/libboringssl.dylib", + "image_uuid": "E8DA37CF-0F7A-38D1-8D87-3730FB158249", + "program_counter": 182440 + }, + "subsystem": "com.apple.network", + "trace_id": "390070235526660100" + }, + "message": "nw_protocol_boringssl_signal_connected(753) [C136.1.1.1:2][0x7fd20f8f8e60] TLS connected [version(0x0304) ciphersuite(TLS_AES_256_GCM_SHA384) group(0x001d) signature_alg(0x0804) alpn(http/1.1) resumed(0) offered_ticket(0) false_started(0) ocsp_received(1) sct_received(0) connect_time(457ms) flight_time(10ms) rtt(10ms) write_stalls(0) read_stalls(6)]", + "process": { + "code_signature": { + "digest_algorithm": "0x0804" + }, + "pid": 200, + "thread": { + "id": 1133904 + } + }, + "related": { + "user": [ + "200" + ] + }, + "tags": [ + "preserve_original_event" + ], + "user": { + "id": "200" + } + }, + { + "@timestamp": "2025-10-15T10:53:24.536Z", + "ecs": { + "version": "9.2.0" + }, + "event": { + "category": [ + "configuration" + ], + "kind": "event", + "original": "{\"timezoneName\":\"\",\"messageType\":\"Default\",\"eventType\":\"logEvent\",\"source\":null,\"formatString\":\"Task <%{public,uuid_t}.16P>.<%lu> summary for %{public}s {transaction_duration_ms=%u, response_status=%ld, connection=%llu, protocol=\\\"%{public}@\\\", domain_lookup_duration_ms=%u, connect_duration_ms=%u, secure_connection_duration_ms=%u, private_relay=%{bool}d, request_start_ms=%u, request_duration_ms=%u, response_start_ms=%u, response_duration_ms=%u, request_bytes=%lld, response_bytes=%lld, cache_hit=%{bool}d}\",\"userID\":200,\"activityIdentifier\":1606042,\"subsystem\":\"com.apple.CFNetwork\",\"category\":\"Summary\",\"threadID\":1133904,\"senderImageUUID\":\"31798E5A-30D3-381D-BC8D-F9AF4154382B\",\"backtrace\":{\"frames\":[{\"imageOffset\":987343,\"imageUUID\":\"31798E5A-30D3-381D-BC8D-F9AF4154382B\"}]},\"bootUUID\":\"218031E6-E47F-4A77-B7FC-5A57B049F4BC\",\"processImagePath\":\"\\/System\\/Library\\/CoreServices\\/Software Update.app\\/Contents\\/Resources\\/softwareupdated\",\"senderImagePath\":\"\\/System\\/Library\\/Frameworks\\/CFNetwork.framework\\/Versions\\/A\\/CFNetwork\",\"timestamp\":\"2025-10-15 16:23:24.536497+0530\",\"machTimestamp\":739235583869205,\"eventMessage\":\"Task <400732FC-E156-46A9-A05E-4C557AF74568>.<74> summary for task success {transaction_duration_ms=1190, response_status=200, connection=136, protocol=\\\"http\\/1.1\\\", domain_lookup_duration_ms=247, connect_duration_ms=5040000000000000, secure_connection_duration_ms=457, private_relay=false, request_start_ms=842, request_duration_ms=58, response_start_ms=1189, response_duration_ms=0, request_bytes=1440, response_bytes=676, cache_hit=true}\",\"processImageUUID\":\"CD58B635-1F96-38BD-BF3B-DBA7CA293E28\",\"traceID\":405718141415784452,\"processID\":200,\"senderProgramCounter\":987343,\"parentActivityIdentifier\":0}", + "type": [ + "info" + ] + }, + "http": { + "request": { + "bytes": 1440 + }, + "response": { + "bytes": 676, + "status_code": 200 + } + }, + "log": { + "level": "info" + }, + "macos": { + "activity_identifier": "1606042", + "backtrace": { + "frames": [ + { + "image": { + "offset": "987343", + "uuid": "31798E5A-30D3-381D-BC8D-F9AF4154382B" + } + } + ] + }, + "boot_uuid": "218031E6-E47F-4A77-B7FC-5A57B049F4BC", + "category": "Summary", + "event": { + "message": { + "cache_hit": true, + "connection": 136, + "connection_duration_ms": 5040000000000000, + "description": "Task <400732FC-E156-46A9-A05E-4C557AF74568>.<74> summary for task success {transaction_duration_ms=1190, response_status=200, connection=136, protocol=\"http/1.1\", domain_lookup_duration_ms=247, connect_duration_ms=5040000000000000, secure_connection_duration_ms=457, private_relay=false, request_start_ms=842, request_duration_ms=58, response_start_ms=1189, response_duration_ms=0, request_bytes=1440, response_bytes=676, cache_hit=true}", + "domain_lookup_duration_ms": 247, + "private_relay": false, + "protocol": "\"http/1.1\"", + "request_duration_ms": 58, + "request_start_ms": 842, + "response_duration_ms": 0, + "response_start_ms": 1189, + "secure_connection_duration_ms": 457, + "task_uid": "400732FC-E156-46A9-A05E-4C557AF74568", + "transaction_duration_ms": 1190 + }, + "type": "logEvent" + }, + "format_string": "Task <%{public,uuid_t}.16P>.<%lu> summary for %{public}s {transaction_duration_ms=%u, response_status=%ld, connection=%llu, protocol=\"%{public}@\", domain_lookup_duration_ms=%u, connect_duration_ms=%u, secure_connection_duration_ms=%u, private_relay=%{bool}d, request_start_ms=%u, request_duration_ms=%u, response_start_ms=%u, response_duration_ms=%u, request_bytes=%lld, response_bytes=%lld, cache_hit=%{bool}d}", + "mach_timestamp": 7.39235583869205E14, + "parent_activity_identifier": "0", + "process": { + "image_path": "/System/Library/CoreServices/Software Update.app/Contents/Resources/softwareupdated", + "image_uuid": "CD58B635-1F96-38BD-BF3B-DBA7CA293E28" + }, + "sender": { + "image_path": "/System/Library/Frameworks/CFNetwork.framework/Versions/A/CFNetwork", + "image_uuid": "31798E5A-30D3-381D-BC8D-F9AF4154382B", + "program_counter": 987343 + }, + "subsystem": "com.apple.CFNetwork", + "trace_id": "405718141415784452" + }, + "message": "Task <400732FC-E156-46A9-A05E-4C557AF74568>.<74> summary for task success {transaction_duration_ms=1190, response_status=200, connection=136, protocol=\"http/1.1\", domain_lookup_duration_ms=247, connect_duration_ms=5040000000000000, secure_connection_duration_ms=457, private_relay=false, request_start_ms=842, request_duration_ms=58, response_start_ms=1189, response_duration_ms=0, request_bytes=1440, response_bytes=676, cache_hit=true}", + "process": { + "pid": 200, + "thread": { + "id": 1133904 + } + }, + "related": { + "user": [ + "200" + ] + }, + "tags": [ + "preserve_original_event" + ], + "user": { + "id": "200" + } + } + ] +} diff --git a/packages/macos/data_stream/system_change/agent/stream/unifiedlogs.yml.hbs b/packages/macos/data_stream/system_change/agent/stream/unifiedlogs.yml.hbs new file mode 100644 index 00000000000..3dc27e75962 --- /dev/null +++ b/packages/macos/data_stream/system_change/agent/stream/unifiedlogs.yml.hbs @@ -0,0 +1,48 @@ +predicate: +- 'subsystem == "com.apple.security" OR subsystem == "com.apple.systempolicy" OR subsystem == "com.apple.installer" OR process == "Installer" OR process == "softwareupdated" OR eventMessage CONTAINS[c] "removed package" OR eventMessage CONTAINS[c] "forget package"' +{{#each predicate as |p|}} +- {{p}} +{{/each}} +{{#if start}} +start: {{start}} +{{/if}} +{{#if end}} +end: {{end}} +{{/if}} +{{#if process}} +process: +{{#each process as |p|}} +- {{p}} +{{/each}} +{{/if}} +source: {{source}} +info: {{info}} +debug: {{debug}} +backtrace: {{backtrace}} +signpost: {{signpost}} +unreliable: {{unreliable}} +mach_continuous_time: {{mach_continuous_time}} +backfill: {{backfill}} +{{#if archive_file}} +archive_file: {{archive_file}} +{{/if}} +{{#if trace_file}} +trace_file: {{trace_file}} +{{/if}} +tags: +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} +{{#if preserve_duplicate_custom_fields}} + - preserve_duplicate_custom_fields +{{/if}} +{{#each tags as |tag|}} + - {{tag}} +{{/each}} +{{#contains "forwarded" tags}} +publisher_pipeline.disable_host: true +{{/contains}} +{{#if processors}} +processors: +{{processors}} +{{/if}} diff --git a/packages/macos/data_stream/system_change/elasticsearch/ingest_pipeline/common-pipeline.yml.link b/packages/macos/data_stream/system_change/elasticsearch/ingest_pipeline/common-pipeline.yml.link new file mode 100644 index 00000000000..f146557ce04 --- /dev/null +++ b/packages/macos/data_stream/system_change/elasticsearch/ingest_pipeline/common-pipeline.yml.link @@ -0,0 +1 @@ +../../../../_dev/shared/common-pipeline.yml 6e7bb697288eb24aeb67d4950a8a3f38d89eb89f559dc01e605e543162267ddf \ No newline at end of file diff --git a/packages/macos/data_stream/system_change/elasticsearch/ingest_pipeline/default.yml b/packages/macos/data_stream/system_change/elasticsearch/ingest_pipeline/default.yml new file mode 100644 index 00000000000..68b3fd4ad65 --- /dev/null +++ b/packages/macos/data_stream/system_change/elasticsearch/ingest_pipeline/default.yml @@ -0,0 +1,381 @@ +--- +description: Pipeline for processing system change logs. +processors: + - append: + field: event.category + tag: set_event_category + value: configuration + - append: + field: event.type + tag: append_info_into_event_type_in_system_change + value: info + - pipeline: + name: '{{ IngestPipeline "common-pipeline" }}' + - grok: + description: Grok the eventMessage. + tag: grok-event-message + field: macos.event.message.description + ignore_missing: true + patterns: + - '^\[%{WORD}\] %{DATA}\:(?:%{SPACE}mach=%{WORD:macos.event.message.mach:boolean})?(?:%{SPACE}listener=%{WORD:macos.event.message.listener:boolean})?(?:%{SPACE}peer=%{WORD:macos.event.message.peer:boolean})?(?:%{SPACE}name=%{GREEDYDATA:macos.event.message.name})?' + - '^%{WORD} \[%{DATA}\](?:%{SPACE}flags=\[%{DATA:macos.event.message.flags}\])?(?:%{SPACE}seq=%{DATA:macos.event.message.seq},)?(?:%{SPACE}ack=%{DATA:macos.event.message.ack},)?(?:%{SPACE}win=%{DATA:macos.event.message.win})?(?:%{SPACE}state=%{DATA:macos.event.message.state})?(?:%{SPACE}rcv_nxt=%{DATA:macos.event.message.rcv_nxt},)?(?:snd_una=%{DATA:macos.event.message.snd_una})' + - '^%{WORD} \[%{DATA}\](?:%{SPACE}flags=\[%{DATA:macos.event.message.flags}\])?(?:%{SPACE}seq=%{DATA:macos.event.message.seq},)?(?:%{SPACE}ack=%{DATA:macos.event.message.ack},)?(?:%{SPACE}win=%{DATA:macos.event.message.win})?(?:%{SPACE}state=%{DATA:macos.event.message.state})?(?:%{SPACE}rcv_nxt=%{DATA:macos.event.message.rcv_nxt},)?(?:snd_una=%{DATA:macos.event.message.snd_una})' + - '^nw_protocol_boringssl_signal_connected\(%{NUMBER}\) \[%{DATA:macos.event.message.connection_identifier}\]\[%{DATA}\] TLS connected \[(?:version\(%{DATA:macos.event.message.tls_version}\))?(?:%{SPACE}ciphersuite\(%{DATA:macos.event.message.cipher_suite}\))?(?:%{SPACE}group\(%{DATA:macos.event.message.group}\))?(?:%{SPACE}signature_alg\(%{DATA:macos.event.message.signature_alg}\))?(?:%{SPACE}alpn\(%{DATA:macos.event.message.alpn}\))?(?:%{SPACE}resumed\(%{DATA:macos.event.message.resumed}\))?(?:%{SPACE}offered_ticket\(%{DATA:macos.event.message.offered_ticket}\))?(?:%{SPACE}false_started\(%{DATA:macos.event.message.false_started}\))?(?:%{SPACE}ocsp_received\(%{DATA:macos.event.message.ocsp_received}\))?(?:%{SPACE}sct_received\(%{DATA:macos.event.message.sct_received}\))?(?:%{SPACE}connect_time\(%{DATA:macos.event.message.connection_time}\))?(?:%{SPACE}flight_time\(%{DATA:macos.event.message.flight_time}\))?(?:%{SPACE}rtt\(%{DATA:macos.event.message.rtt}\))?(?:%{SPACE}write_stalls\(%{DATA:macos.event.message.write_stalls:long}\))?(?:%{SPACE}read_stalls\(%{DATA:macos.event.message.read_stalls:long}\))?(?:%{SPACE}pake\(%{DATA:macos.event.message.pake}\))?\]' + - '^Task \<%{DATA:macos.event.message.task_uid}\>.\<%{NUMBER}\>%{SPACE}summary for %{DATA} \{(?:transaction_duration_ms=%{NUMBER:macos.event.message.transaction_duration_ms:long},)?(?:%{SPACE}response_status=%{NUMBER:macos.event.message.response_status:long},)?(?:%{SPACE}connection=%{NUMBER:macos.event.message.connection:long},)?(?:%{SPACE}protocol=%{DATA:macos.event.message.protocol},)?(?:%{SPACE}domain_lookup_duration_ms=%{NUMBER:macos.event.message.domain_lookup_duration_ms:long},)?(?:%{SPACE}connect_duration_ms=%{NUMBER:macos.event.message.connection_duration_ms:long},)?(?:%{SPACE}secure_connection_duration_ms=%{NUMBER:macos.event.message.secure_connection_duration_ms:long},)?(?:%{SPACE}private_relay=%{WORD:macos.event.message.private_relay:boolean},)?(?:%{SPACE}request_start_ms=%{NUMBER:macos.event.message.request_start_ms:long},)?(?:%{SPACE}request_duration_ms=%{NUMBER:macos.event.message.request_duration_ms:long},)?(?:%{SPACE}response_start_ms=%{NUMBER:macos.event.message.response_start_ms:long},)?(?:%{SPACE}response_duration_ms=%{NUMBER:macos.event.message.response_duration_ms:long},)?(?:%{SPACE}request_bytes=%{NUMBER:macos.event.message.request_bytes:long},)?(?:%{SPACE}response_bytes=%{NUMBER:macos.event.message.response_bytes:long},)?(?:%{SPACE}cache_hit=%{WORD:macos.event.message.cache_hit:boolean})?\}' + - '^%{DATA} \[%{DATA:macos.event.message.connection_identifier}\]%{SPACE}\[%{UUID:macos.event.message.connection_uuid} :%{NUMBER:macos.event.message.src_port:long}<->:%{NUMBER:macos.event.message.dest_port:long}\]%{SPACE}Init: %{NUMBER:macos.event.message.init_flag:long}, Conn_Time: %{DATA:macos.event.message.connection_time}, SYNs: %{NUMBER:macos.event.message.syns:long}, WR_T: %{DATA:macos.event.message.wr_t_in_out}, RD_T: %{DATA:macos.event.message.rd_t_in_out}, TFO: %{DATA:macos.event.message.tfo_in_out_miss}, ECN: %{DATA:macos.event.message.ecn_in_out_miss}, Accurate ECN %{GREEDYDATA}: %{GREEDYDATA:macos.event.message.accurate_ecn}, TS: %{NUMBER:macos.event.message.timestamp_enabled:long}, TSO: %{NUMBER:macos.event.message.tso_enabled:long}%{SPACE}rtt_cache: %{DATA:macos.event.message.rtt_cache}, rtt_upd: %{NUMBER:macos.event.message.rtt_updates:long}, rtt: %{DATA:macos.event.message.rtt}, rtt_var: %{DATA:macos.event.message.rtt_var_ms} rtt_nc: %{DATA:macos.event.message.rtt_nc_ms}, rtt_var_nc: %{DATA:macos.event.message.rtt_var_nc_ms} base rtt: %{GREEDYDATA:macos.event.message.base_rtt_ms}%{SPACE}ACKs-compressed: %{NUMBER:macos.event.message.acks_compressed:long}, ACKs delayed: %{NUMBER:macos.event.message.acks_delayed:long} delayed ACKs sent: %{NUMBER:macos.event.message.delayed_acks_sent:long}' + - '^\[C%{NUMBER:macos.event.message.connection_id} %{UUID:macos.event.message.session_uuid} (Hostname\#)?%{DATA:macos.event.message.hostname}:%{NUMBER:macos.event.message.hostname_port:long} %{DATA}(, bundle id: %{DATA:macos.event.message.bundle_id})?(, pid: %{DATA:macos.event.message.pid:long})?(, account id: %{DATA:macos.event.message.account_id})?(, url: %{DATA:macos.event.message.url})?(, url hash: %{BASE16NUM:macos.event.message.url_hash})?(, traffic class: %{NUMBER:macos.event.message.traffic_class})?(, expected workload: %{NUMBER:macos.event.message.expected_workload})?(, %{GREEDYDATA})?, attribution: %{DATA:macos.event.message.attribution}(, %{GREEDYDATA})?\] cancelled\n\t\[C%{DATA:macos.event.message.connection_detail} %{UUID:macos.event.message.connection_uuid} %{IP:macos.event.message.client_ip}:%{NUMBER:macos.event.message.client_port:long}<->(IPv4#)?%{DATA:macos.event.message.server_id}:%{NUMBER:macos.event.message.server_port:long}\]\n\tConnected Path: %{DATA:macos.event.message.path_status}(, %{DATA})?(, interface: %{DATA:macos.event.message.interface})?(, %{GREEDYDATA})?\n\tPrivacy Stance: %{DATA:macos.event.message.privacy_stance}\n\tDuration: %{DATA:macos.event.message.duration}, DNS @%{DATA:macos.event.message.dns_start} took %{DATA:macos.event.message.dns_duration}, TCP @%{DATA:macos.event.message.tcp_start} took %{DATA:macos.event.message.tcp_duration}, TLS %{DATA:macos.event.message.tls_version} took %{DATA:macos.event.message.tls_duration}\n\tbytes in\/out: %{NUMBER:macos.event.message.bytes_in:long}\/%{NUMBER:macos.event.message.bytes_out:long}, packets in\/out: %{NUMBER:macos.event.message.packets_in:long}\/%{NUMBER:macos.event.message.packets_out:long}, rtt: %{DATA:macos.event.message.rtt}, retransmitted bytes: %{NUMBER:macos.event.message.retransmitted_bytes:long}, out-of-order bytes: %{NUMBER:macos.event.message.out_of_order_bytes:long}\n\tecn packets sent\/acked\/marked\/lost: %{NUMBER:macos.event.message.ecn_sent:long}\/%{NUMBER:macos.event.message.ecn_acked:long}\/%{NUMBER:macos.event.message.ecn_marked:long}\/%{NUMBER:macos.event.message.ecn_lost:long}$' + - '%{GREEDYDATA:macos.event.message.original}' + - convert: + field: macos.event.message.client_ip + tag: convert_client_ip_to_ip + type: ip + ignore_missing: true + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - set: + field: host.id + tag: set_host_id + copy_from: macos.event.message.hostname + ignore_empty_value: true + - set: + field: process.pid + tag: set_process_pid + copy_from: macos.event.message.pid + ignore_empty_value: true + - set: + field: url.original + tag: set_url_original + copy_from: macos.event.message.url + ignore_empty_value: true + - set: + field: source.ip + tag: set_source_ip + copy_from: macos.event.message.client_ip + ignore_empty_value: true + - set: + field: source.port + tag: set_source_port + copy_from: macos.event.message.client_port + ignore_empty_value: true + - set: + field: destination.port + tag: set_destination_port + copy_from: macos.event.message.server_port + ignore_empty_value: true + - set: + field: source.bytes + tag: set_source_bytes + copy_from: macos.event.message.bytes_in + ignore_empty_value: true + - set: + field: destination.bytes + tag: set_destination_bytes + copy_from: macos.event.message.bytes_out + ignore_empty_value: true + - set: + field: source.packets + tag: set_source_packets + copy_from: macos.event.message.packets_in + ignore_empty_value: true + - set: + field: destination.packets + tag: set_destination_packets + copy_from: macos.event.message.packets_out + ignore_empty_value: true + - script: + lang: painless + tag: calculate_total_bytes + description: calculate total bytes of in and out if in and out are not null. + if: ctx.source?.bytes != null && ctx.destination?.bytes != null + source: | + ctx.network = new HashMap(); + ctx.network.bytes = ctx.source.bytes + ctx.destination.bytes + - script: + lang: painless + tag: calculate_total_packets + description: calculate total packets of in and out if in and out are not null. + if: ctx.source?.packets != null && ctx.destination?.packets != null + source: | + if (ctx.network == null) { + ctx.network = new HashMap(); + } + ctx.network.packets = ctx.source.packets + ctx.destination.packets + - uri_parts: + field: url.original + tag: uri_parts_url_original + ignore_missing: true + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - append: + field: related.hosts + tag: append_host_id_into_related_hosts + value: '{{{host.id}}}' + allow_duplicates: false + if: ctx.host?.id != null + - append: + field: related.ip + tag: append_source_ip_into_related_ip + value: '{{{source.ip}}}' + allow_duplicates: false + if: ctx.source?.ip != null + - set: + field: process.code_signature.digest_algorithm + tag: set_process_code_signature_digest_algorithm + copy_from: macos.event.message.signature_alg + ignore_empty_value: true + - set: + field: macos.event.message.resumed + tag: set_event_message_resumed_true + value: true + if: ctx.macos?.event?.message?.resumed == '1' + - set: + field: macos.event.message.resumed + tag: set_event_message_resumed_false + value: false + if: ctx.macos?.event?.message?.resumed == '0' + - set: + field: macos.event.message.offered_ticket + tag: set_event_message_offered_ticket_true + value: true + if: ctx.macos?.event?.message?.offered_ticket == '1' + - set: + field: macos.event.message.offered_ticket + tag: set_event_message_offered_ticket_false + value: false + if: ctx.macos?.event?.message?.offered_ticket == '0' + - set: + field: macos.event.message.false_started + tag: set_event_message_false_started_true + value: true + if: ctx.macos?.event?.message?.false_started == '1' + - set: + field: macos.event.message.false_started + tag: set_event_message_false_started_false + value: false + if: ctx.macos?.event?.message?.false_started == '0' + - set: + field: macos.event.message.ocsp_received + tag: set_event_message_oscp_received_true + value: true + if: ctx.macos?.event?.message?.ocsp_received == '1' + - set: + field: macos.event.message.ocsp_received + tag: set_event_message_oscp_received_false + value: false + if: ctx.macos?.event?.message?.ocsp_received == '0' + - set: + field: macos.event.message.sct_received + tag: set_event_message_sct_received_true + value: true + if: ctx.macos?.event?.message?.sct_received == '1' + - set: + field: macos.event.message.sct_received + tag: set_event_message_sct_received_false + value: false + if: ctx.macos?.event?.message?.sct_received == '0' + - set: + field: http.request.bytes + tag: set_http_request_bytes + copy_from: macos.event.message.request_bytes + ignore_empty_value: true + - set: + field: http.response.bytes + tag: set_http_response_bytes + copy_from: macos.event.message.response_bytes + ignore_empty_value: true + - set: + field: http.response.status_code + tag: set_http_response_status_code + copy_from: macos.event.message.response_status + ignore_empty_value: true + - set: + field: destination.port + tag: set_destination_port + copy_from: macos.event.message.dest_port + ignore_empty_value: true + - set: + field: source.port + tag: set_source_port + copy_from: macos.event.message.src_port + ignore_empty_value: true + - dissect: + field: macos.event.message.ecn_in_out_miss + tag: dissect_ecn_in_out_miss + pattern: '%{macos.event.message.ecn_in}/%{macos.event.message.ecn_out}/%{macos.event.message.ecn_miss}' + ignore_missing: true + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - convert: + field: macos.event.message.ecn_in + tag: convert_macos_event_message_ecn_in_to_long + type: long + ignore_missing: true + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - convert: + field: macos.event.message.ecn_out + tag: convert_macos_event_message_ecn_out_to_long + type: long + ignore_missing: true + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - convert: + field: macos.event.message.ecn_miss + tag: convert_macos_event_message_ecn_miss_to_long + type: long + ignore_missing: true + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - dissect: + field: macos.event.message.rd_t_in_out + tag: dissect_rd_t_in_out + pattern: '%{macos.event.message.rd_t_in}/%{macos.event.message.rd_t_out}' + ignore_missing: true + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - convert: + field: macos.event.message.rd_t_in + tag: convert_macos_event_message_rd_t_in_to_long + type: long + ignore_missing: true + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - convert: + field: macos.event.message.rd_t_out + tag: convert_macos_event_message_rd_t_out_to_long + type: long + ignore_missing: true + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - dissect: + field: macos.event.message.tfo_in_out_miss + tag: dissect_tfo_in_out_miss + pattern: '%{macos.event.message.tfo_in}/%{macos.event.message.tfo_out}/%{macos.event.message.tfo_miss}' + ignore_missing: true + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - convert: + field: macos.event.message.tfo_in + tag: convert_macos_event_message_tfo_in_to_long + type: long + ignore_missing: true + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - convert: + field: macos.event.message.tfo_out + tag: convert_macos_event_message_tfo_out_to_long + type: long + ignore_missing: true + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - convert: + field: macos.event.message.tfo_miss + tag: convert_macos_event_message_tfo_miss_to_long + type: long + ignore_missing: true + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - dissect: + field: macos.event.message.wr_t_in_out + tag: dissect_wr_t_in_out + pattern: '%{macos.event.message.wr_t_in}/%{macos.event.message.wr_t_out}' + ignore_missing: true + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - convert: + field: macos.event.message.wr_t_in + tag: convert_macos_event_message_wr_t_in_to_long + type: long + ignore_missing: true + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - convert: + field: macos.event.message.wr_t_out + tag: convert_macos_event_message_wr_t_out_to_long + type: long + ignore_missing: true + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - dissect: + field: macos.event.message.accurate_ecn + tag: dissect_accurate_ecn + pattern: '%{macos.event.message.accurate_ecn_client}/%{macos.event.message.accurate_ecn_server}' + ignore_missing: true + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - remove: + field: + - macos.event.message.signature_alg + - macos.event.message.request_bytes + - macos.event.message.response_bytes + - macos.event.message.response_status + - macos.event.message.dest_port + - macos.event.message.src_port + - macos.event.message.hostname + - macos.event.message.pid + - macos.event.message.url + - macos.event.message.client_ip + - macos.event.message.client_port + - macos.event.message.server_port + - macos.event.message.bytes_in + - macos.event.message.bytes_out + - macos.event.message.packets_in + - macos.event.message.packets_out + tag: remove_custom_duplicate_fields + ignore_missing: true + if: ctx.tags == null || !ctx.tags.contains('preserve_duplicate_custom_fields') + - remove: + field: + - macos.event.message.original + - macos.event.message.ecn_in_out_miss + - macos.event.message.rd_t_in_out + - macos.event.message.tfo_in_out_miss + - macos.event.message.wr_t_in_out + - macos.event.message.accurate_ecn + tag: remove_non_required_fields + ignore_missing: true +on_failure: + - append: + field: error.message + value: >- + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}' + - set: + field: event.kind + tag: set_pipeline_error_to_event_kind + value: pipeline_error diff --git a/packages/macos/data_stream/system_change/fields/base-fields.yml b/packages/macos/data_stream/system_change/fields/base-fields.yml new file mode 100644 index 00000000000..11d35513392 --- /dev/null +++ b/packages/macos/data_stream/system_change/fields/base-fields.yml @@ -0,0 +1,16 @@ +- name: data_stream.type + external: ecs +- name: data_stream.dataset + external: ecs +- name: data_stream.namespace + external: ecs +- name: event.module + type: constant_keyword + external: ecs + value: macos +- name: event.dataset + type: constant_keyword + external: ecs + value: macos.system_change +- name: '@timestamp' + external: ecs diff --git a/packages/macos/data_stream/system_change/fields/beats.yml b/packages/macos/data_stream/system_change/fields/beats.yml new file mode 100644 index 00000000000..d5fd38748ba --- /dev/null +++ b/packages/macos/data_stream/system_change/fields/beats.yml @@ -0,0 +1,6 @@ +- name: input.type + type: keyword + description: Type of Filebeat input. +- name: log.offset + type: long + description: Log offset. diff --git a/packages/macos/data_stream/system_change/fields/fields.yml b/packages/macos/data_stream/system_change/fields/fields.yml new file mode 100644 index 00000000000..81d48020635 --- /dev/null +++ b/packages/macos/data_stream/system_change/fields/fields.yml @@ -0,0 +1,290 @@ +- name: macos + type: group + fields: + - name: activity_identifier + type: keyword + - name: backtrace + type: group + fields: + - name: frames + type: group + fields: + - name: image + type: group + fields: + - name: offset + type: keyword + - name: uuid + type: keyword + - name: boot_uuid + type: keyword + - name: category + type: keyword + - name: event + type: group + fields: + - name: category + type: keyword + - name: message + type: group + fields: + - name: account_id + type: keyword + - name: accurate_ecn_client + type: keyword + - name: accurate_ecn_server + type: keyword + - name: ack + type: keyword + - name: acks_compressed + type: long + - name: acks_delayed + type: long + - name: alpn + type: keyword + - name: attribution + type: keyword + - name: base_rtt_ms + type: keyword + - name: bundle_id + type: keyword + - name: bytes_in + type: long + - name: bytes_out + type: long + - name: cache_hit + type: boolean + - name: cipher_suite + type: keyword + - name: client_ip + type: ip + - name: client_port + type: long + - name: connection + type: long + - name: connection_detail + type: keyword + - name: connection_duration_ms + type: long + - name: connection_id + type: keyword + - name: connection_identifier + type: keyword + - name: connection_time + type: keyword + - name: connection_uuid + type: keyword + - name: delayed_acks_sent + type: long + - name: description + type: keyword + - name: dest_port + type: long + - name: dns_duration + type: keyword + - name: dns_start + type: keyword + - name: domain_lookup_duration_ms + type: long + - name: duration + type: keyword + - name: ecn_acked + type: long + - name: ecn_in + type: long + - name: ecn_lost + type: long + - name: ecn_marked + type: long + - name: ecn_miss + type: long + - name: ecn_out + type: long + - name: ecn_sent + type: long + - name: expected_workload + type: keyword + - name: false_started + type: boolean + - name: flags + type: keyword + - name: flight_time + type: keyword + - name: group + type: keyword + - name: hostname + type: keyword + - name: hostname_port + type: long + - name: init_flag + type: long + - name: interface + type: keyword + - name: listener + type: boolean + - name: mach + type: boolean + - name: name + type: keyword + - name: ocsp_received + type: boolean + - name: offered_ticket + type: boolean + - name: out_of_order_bytes + type: long + - name: packets_in + type: long + - name: packets_out + type: long + - name: pake + type: keyword + - name: path_status + type: keyword + - name: peer + type: boolean + - name: pid + type: long + - name: privacy_stance + type: keyword + - name: private_relay + type: boolean + - name: protocol + type: keyword + - name: rcv_nxt + type: keyword + - name: rd_t_in + type: long + - name: rd_t_out + type: long + - name: read_stalls + type: long + - name: request_bytes + type: long + - name: request_duration_ms + type: long + - name: request_start_ms + type: long + - name: response_bytes + type: long + - name: response_duration_ms + type: long + - name: response_start_ms + type: long + - name: response_status + type: long + - name: resumed + type: boolean + - name: retransmitted_bytes + type: long + - name: rtt + type: keyword + - name: rtt_cache + type: keyword + - name: rtt_nc_ms + type: keyword + - name: rtt_updates + type: long + - name: rtt_var_ms + type: keyword + - name: rtt_var_nc_ms + type: keyword + - name: sct_received + type: boolean + - name: secure_connection_duration_ms + type: long + - name: seq + type: keyword + - name: server_id + type: keyword + - name: server_port + type: long + - name: session_uuid + type: keyword + - name: signature_alg + type: keyword + - name: snd_una + type: keyword + - name: src_port + type: long + - name: state + type: keyword + - name: syns + type: long + - name: task_uid + type: keyword + - name: tcp_duration + type: keyword + - name: tcp_start + type: keyword + - name: tfo_in + type: long + - name: tfo_miss + type: long + - name: tfo_out + type: long + - name: timestamp_enabled + type: long + - name: tls_duration + type: keyword + - name: tls_version + type: keyword + - name: traffic_class + type: keyword + - name: transaction_duration_ms + type: long + - name: tso_enabled + type: long + - name: url + type: keyword + - name: url_hash + type: keyword + - name: win + type: keyword + - name: wr_t_in + type: long + - name: wr_t_out + type: long + - name: write_stalls + type: long + - name: type + type: keyword + - name: format_string + type: keyword + - name: mach_timestamp + type: double + - name: message_type + type: keyword + - name: parent_activity_identifier + type: keyword + - name: process + type: group + fields: + - name: id + type: long + - name: image_path + type: keyword + - name: image_uuid + type: keyword + - name: sender + type: group + fields: + - name: image_path + type: keyword + - name: image_uuid + type: keyword + - name: program_counter + type: long + - name: source + type: keyword + - name: subsystem + type: keyword + - name: thread_id + type: long + - name: timestamp + type: date + - name: timezone_name + type: keyword + - name: trace_id + type: keyword + - name: user_id + type: keyword diff --git a/packages/macos/data_stream/system_change/manifest.yml b/packages/macos/data_stream/system_change/manifest.yml new file mode 100644 index 00000000000..00c3e46d9ec --- /dev/null +++ b/packages/macos/data_stream/system_change/manifest.yml @@ -0,0 +1,139 @@ +title: System Changes +type: logs +streams: + - input: unifiedlogs + title: System Changes logs + description: Collect System Changes logs from macOS. + template_path: unifiedlogs.yml.hbs + enabled: false + vars: + - name: predicate + type: text + title: Predicate + description: | + Filters messages using the provided predicate based on NSPredicate. + A compound predicate or multiple predicates can be provided as a list. + For detailed information on the use of predicate based filtering, + please refer to the https://developer.apple.com/library/mac/documentation/Cocoa/Conceptual/Predicates/Articles/pSyntax.html[Predicate Programming Guide]. + multi: true + show_user: true + - name: start + type: text + title: Start date + description: | + Shows content starting from the provided date. + The following date/time formats are accepted: + `YYYY-MM-DD`, `YYYY-MM-DD HH:MM:SS`, `YYYY-MM-DD HH:MM:SSZZZZZ`. + show_user: false + - name: end + type: text + title: End date + description: | + Shows content up to the provided date. + The following date/time formats are accepted: + `YYYY-MM-DD`, `YYYY-MM-DD HH:MM:SS`, `YYYY-MM-DD HH:MM:SSZZZZZ`. + show_user: false + - name: process + type: text + title: Process + description: | + A list of the processes on which to operate. It accepts a PID or process name. + multi: true + show_user: false + - name: source + type: bool + title: Include source + description: | + Include symbol names and source line numbers for messages, if available. + show_user: false + - name: info + type: bool + title: Include info + description: | + Disable or enable info level messages. + show_user: false + - name: debug + type: bool + title: Include debug + description: | + Disable or enable debug level messages. + show_user: false + - name: backtrace + type: bool + title: Include backtrace + description: | + Disable or enable backtrace level messages. + show_user: false + - name: signpost + type: bool + title: Include signpost + description: | + Disable or enable signpost level messages. + show_user: false + - name: unreliable + type: bool + title: Annotate unreliable + description: | + Annotate events with whether the log was emitted unreliably. + show_user: false + - name: mach_continuous_time + type: bool + title: Use mach continuous time + description: | + Use mach continuous time timestamps rather than walltime. + show_user: false + - name: backfill + type: bool + title: Must backfill + description: | + If set to true the input will process all available logs since the beginning + of time the first time it starts. + show_user: false + - name: archive_file + type: text + title: Archive file + description: | + Display events stored in the given archive. + The archive must be a valid log archive bundle with the suffix `.logarchive`. + show_user: false + - name: trace_file + type: text + title: Trace file + description: | + Display events stored in the given `.tracev3` file. + In order to be decoded, the file must be contained within a valid `.logarchive`. + show_user: false + - name: preserve_original_event + type: bool + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field event.original. + multi: false + required: false + show_user: true + default: false + - name: tags + type: text + title: Tags + description: Tags for the data-stream. + multi: true + required: true + show_user: false + default: + - forwarded + - macos-system_change + - name: preserve_duplicate_custom_fields + required: false + title: Preserve duplicate custom fields + description: Preserve macos.* fields that were copied to Elastic Common Schema (ECS) fields. + type: bool + multi: false + show_user: false + default: false + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: >- + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. diff --git a/packages/macos/data_stream/user_and_account_management/_dev/test/pipeline/test-common-config.yml b/packages/macos/data_stream/user_and_account_management/_dev/test/pipeline/test-common-config.yml new file mode 100644 index 00000000000..37e8fa225fd --- /dev/null +++ b/packages/macos/data_stream/user_and_account_management/_dev/test/pipeline/test-common-config.yml @@ -0,0 +1,3 @@ +fields: + tags: + - preserve_duplicate_custom_fields diff --git a/packages/macos/data_stream/user_and_account_management/_dev/test/pipeline/test-user-and-account-management.log b/packages/macos/data_stream/user_and_account_management/_dev/test/pipeline/test-user-and-account-management.log new file mode 100644 index 00000000000..a5d1fb6044b --- /dev/null +++ b/packages/macos/data_stream/user_and_account_management/_dev/test/pipeline/test-user-and-account-management.log @@ -0,0 +1,2 @@ +{"timezoneName":"","messageType":"Default","eventType":"logEvent","source":null,"formatString":"SecureTokenStatus users: admin %llu, admin w\/SecureToken %llu, admin w\/o SecureToken %llu, standard %llu, standard w\/SecureToken %llu, standard w\/o SecureToken %llu","userID":0,"activityIdentifier":80,"subsystem":"com.apple.opendirectoryd","category":"auth","threadID":3864,"senderImageUUID":"B327C6FB-1480-39ED-B23A-F3D283CBFA1E","backtrace":{"frames":[{"imageOffset":49542,"imageUUID":"B327C6FB-1480-39ED-B23A-F3D283CBFA1E"}]},"bootUUID":"218031E6-E47F-4A77-B7FC-5A57B049F4BC","processImagePath":"\/usr\/libexec\/opendirectoryd","senderImagePath":"\/System\/Library\/OpenDirectory\/Modules\/PlistFile.bundle\/Contents\/MacOS\/PlistFile","timestamp":"2025-09-25 23:31:36.868722-0700","machTimestamp":131807906844,"eventMessage":"SecureTokenStatus users: admin 0, admin w\/SecureToken 0, admin w\/o SecureToken 0, standard 0, standard w\/SecureToken 0, standard w\/o SecureToken 0","processImageUUID":"D8FEDD38-55EF-31D4-9A85-CC8ADF519F32","traceID":1597384271003652,"processID":66,"senderProgramCounter":49542,"parentActivityIdentifier":0} +{"timezoneName":"","messageType":"Default","eventType":"logEvent","source":null,"formatString":"%@ (%{public}@) is an admin by %s membership","userID":0,"activityIdentifier":44900,"subsystem":"com.apple.opendirectoryd","category":"auth","threadID":12995,"senderImageUUID":"B327C6FB-1480-39ED-B23A-F3D283CBFA1E","backtrace":{"frames":[{"imageOffset":13105,"imageUUID":"B327C6FB-1480-39ED-B23A-F3D283CBFA1E"}]},"bootUUID":"218031E6-E47F-4A77-B7FC-5A57B049F4BC","processImagePath":"\/usr\/libexec\/opendirectoryd","senderImagePath":"\/System\/Library\/OpenDirectory\/Modules\/PlistFile.bundle\/Contents\/MacOS\/PlistFile","timestamp":"2025-09-25 23:45:29.953350-0700","machTimestamp":964193675789,"eventMessage":" (ABFB1DA3-CB78-4B06-B615-D38BCC097E93) is an admin by membership","processImageUUID":"D8FEDD38-55EF-31D4-9A85-CC8ADF519F32","traceID":1566048189612036,"processID":66,"senderProgramCounter":13105,"parentActivityIdentifier":0} diff --git a/packages/macos/data_stream/user_and_account_management/_dev/test/pipeline/test-user-and-account-management.log-expected.json b/packages/macos/data_stream/user_and_account_management/_dev/test/pipeline/test-user-and-account-management.log-expected.json new file mode 100644 index 00000000000..4ec3cb09988 --- /dev/null +++ b/packages/macos/data_stream/user_and_account_management/_dev/test/pipeline/test-user-and-account-management.log-expected.json @@ -0,0 +1,158 @@ +{ + "expected": [ + { + "@timestamp": "2025-09-26T06:31:36.868Z", + "ecs": { + "version": "9.2.0" + }, + "event": { + "category": [ + "iam" + ], + "kind": "event", + "original": "{\"timezoneName\":\"\",\"messageType\":\"Default\",\"eventType\":\"logEvent\",\"source\":null,\"formatString\":\"SecureTokenStatus users: admin %llu, admin w\\/SecureToken %llu, admin w\\/o SecureToken %llu, standard %llu, standard w\\/SecureToken %llu, standard w\\/o SecureToken %llu\",\"userID\":0,\"activityIdentifier\":80,\"subsystem\":\"com.apple.opendirectoryd\",\"category\":\"auth\",\"threadID\":3864,\"senderImageUUID\":\"B327C6FB-1480-39ED-B23A-F3D283CBFA1E\",\"backtrace\":{\"frames\":[{\"imageOffset\":49542,\"imageUUID\":\"B327C6FB-1480-39ED-B23A-F3D283CBFA1E\"}]},\"bootUUID\":\"218031E6-E47F-4A77-B7FC-5A57B049F4BC\",\"processImagePath\":\"\\/usr\\/libexec\\/opendirectoryd\",\"senderImagePath\":\"\\/System\\/Library\\/OpenDirectory\\/Modules\\/PlistFile.bundle\\/Contents\\/MacOS\\/PlistFile\",\"timestamp\":\"2025-09-25 23:31:36.868722-0700\",\"machTimestamp\":131807906844,\"eventMessage\":\"SecureTokenStatus users: admin 0, admin w\\/SecureToken 0, admin w\\/o SecureToken 0, standard 0, standard w\\/SecureToken 0, standard w\\/o SecureToken 0\",\"processImageUUID\":\"D8FEDD38-55EF-31D4-9A85-CC8ADF519F32\",\"traceID\":1597384271003652,\"processID\":66,\"senderProgramCounter\":49542,\"parentActivityIdentifier\":0}", + "type": [ + "info" + ] + }, + "log": { + "level": "info" + }, + "macos": { + "activity_identifier": "80", + "backtrace": { + "frames": [ + { + "image": { + "offset": "49542", + "uuid": "B327C6FB-1480-39ED-B23A-F3D283CBFA1E" + } + } + ] + }, + "boot_uuid": "218031E6-E47F-4A77-B7FC-5A57B049F4BC", + "category": "auth", + "event": { + "message": { + "description": "SecureTokenStatus users: admin 0, admin w/SecureToken 0, admin w/o SecureToken 0, standard 0, standard w/SecureToken 0, standard w/o SecureToken 0" + }, + "type": "logEvent" + }, + "format_string": "SecureTokenStatus users: admin %llu, admin w/SecureToken %llu, admin w/o SecureToken %llu, standard %llu, standard w/SecureToken %llu, standard w/o SecureToken %llu", + "mach_timestamp": 1.31807906844E11, + "message_type": "Default", + "parent_activity_identifier": "0", + "process": { + "id": 66, + "image_path": "/usr/libexec/opendirectoryd", + "image_uuid": "D8FEDD38-55EF-31D4-9A85-CC8ADF519F32" + }, + "sender": { + "image_path": "/System/Library/OpenDirectory/Modules/PlistFile.bundle/Contents/MacOS/PlistFile", + "image_uuid": "B327C6FB-1480-39ED-B23A-F3D283CBFA1E", + "program_counter": 49542 + }, + "subsystem": "com.apple.opendirectoryd", + "thread_id": 3864, + "timestamp": "2025-09-26T06:31:36.868Z", + "trace_id": "1597384271003652", + "user_id": "0" + }, + "message": "SecureTokenStatus users: admin 0, admin w/SecureToken 0, admin w/o SecureToken 0, standard 0, standard w/SecureToken 0, standard w/o SecureToken 0", + "process": { + "pid": 66, + "thread": { + "id": 3864 + } + }, + "related": { + "user": [ + "0" + ] + }, + "tags": [ + "preserve_duplicate_custom_fields" + ], + "user": { + "id": "0" + } + }, + { + "@timestamp": "2025-09-26T06:45:29.953Z", + "ecs": { + "version": "9.2.0" + }, + "event": { + "category": [ + "iam" + ], + "kind": "event", + "original": "{\"timezoneName\":\"\",\"messageType\":\"Default\",\"eventType\":\"logEvent\",\"source\":null,\"formatString\":\"%@ (%{public}@) is an admin by %s membership\",\"userID\":0,\"activityIdentifier\":44900,\"subsystem\":\"com.apple.opendirectoryd\",\"category\":\"auth\",\"threadID\":12995,\"senderImageUUID\":\"B327C6FB-1480-39ED-B23A-F3D283CBFA1E\",\"backtrace\":{\"frames\":[{\"imageOffset\":13105,\"imageUUID\":\"B327C6FB-1480-39ED-B23A-F3D283CBFA1E\"}]},\"bootUUID\":\"218031E6-E47F-4A77-B7FC-5A57B049F4BC\",\"processImagePath\":\"\\/usr\\/libexec\\/opendirectoryd\",\"senderImagePath\":\"\\/System\\/Library\\/OpenDirectory\\/Modules\\/PlistFile.bundle\\/Contents\\/MacOS\\/PlistFile\",\"timestamp\":\"2025-09-25 23:45:29.953350-0700\",\"machTimestamp\":964193675789,\"eventMessage\":\" (ABFB1DA3-CB78-4B06-B615-D38BCC097E93) is an admin by membership\",\"processImageUUID\":\"D8FEDD38-55EF-31D4-9A85-CC8ADF519F32\",\"traceID\":1566048189612036,\"processID\":66,\"senderProgramCounter\":13105,\"parentActivityIdentifier\":0}", + "type": [ + "info" + ] + }, + "log": { + "level": "info" + }, + "macos": { + "activity_identifier": "44900", + "backtrace": { + "frames": [ + { + "image": { + "offset": "13105", + "uuid": "B327C6FB-1480-39ED-B23A-F3D283CBFA1E" + } + } + ] + }, + "boot_uuid": "218031E6-E47F-4A77-B7FC-5A57B049F4BC", + "category": "auth", + "event": { + "message": { + "description": " (ABFB1DA3-CB78-4B06-B615-D38BCC097E93) is an admin by membership" + }, + "type": "logEvent" + }, + "format_string": "%@ (%{public}@) is an admin by %s membership", + "mach_timestamp": 9.64193675789E11, + "message_type": "Default", + "parent_activity_identifier": "0", + "process": { + "id": 66, + "image_path": "/usr/libexec/opendirectoryd", + "image_uuid": "D8FEDD38-55EF-31D4-9A85-CC8ADF519F32" + }, + "sender": { + "image_path": "/System/Library/OpenDirectory/Modules/PlistFile.bundle/Contents/MacOS/PlistFile", + "image_uuid": "B327C6FB-1480-39ED-B23A-F3D283CBFA1E", + "program_counter": 13105 + }, + "subsystem": "com.apple.opendirectoryd", + "thread_id": 12995, + "timestamp": "2025-09-26T06:45:29.953Z", + "trace_id": "1566048189612036", + "user_id": "0" + }, + "message": " (ABFB1DA3-CB78-4B06-B615-D38BCC097E93) is an admin by membership", + "process": { + "pid": 66, + "thread": { + "id": 12995 + } + }, + "related": { + "user": [ + "0" + ] + }, + "tags": [ + "preserve_duplicate_custom_fields" + ], + "user": { + "id": "0" + } + } + ] +} diff --git a/packages/macos/data_stream/user_and_account_management/agent/stream/unifiedlogs.yml.hbs b/packages/macos/data_stream/user_and_account_management/agent/stream/unifiedlogs.yml.hbs new file mode 100644 index 00000000000..bf57ebaef30 --- /dev/null +++ b/packages/macos/data_stream/user_and_account_management/agent/stream/unifiedlogs.yml.hbs @@ -0,0 +1,52 @@ +predicate: +- 'process == "sysadminctl" AND composedMessage CONTAINS "Creating user"' +- 'process == "dscl" AND composedMessage CONTAINS "create"' +- 'process == "sysadminctl" AND composedMessage CONTAINS "Deleting user"' +- 'process == "dscl" AND composedMessage CONTAINS "delete"' +- '(process == "dscl" OR process == "opendirectoryd") AND composedMessage CONTAINS "admin"' +{{#each predicate as |p|}} +- {{p}} +{{/each}} +{{#if start}} +start: {{start}} +{{/if}} +{{#if end}} +end: {{end}} +{{/if}} +{{#if process}} +process: +{{#each process as |p|}} +- {{p}} +{{/each}} +{{/if}} +source: {{source}} +info: {{info}} +debug: {{debug}} +backtrace: {{backtrace}} +signpost: {{signpost}} +unreliable: {{unreliable}} +mach_continuous_time: {{mach_continuous_time}} +backfill: {{backfill}} +{{#if archive_file}} +archive_file: {{archive_file}} +{{/if}} +{{#if trace_file}} +trace_file: {{trace_file}} +{{/if}} +tags: +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} +{{#if preserve_duplicate_custom_fields}} + - preserve_duplicate_custom_fields +{{/if}} +{{#each tags as |tag|}} + - {{tag}} +{{/each}} +{{#contains "forwarded" tags}} +publisher_pipeline.disable_host: true +{{/contains}} +{{#if processors}} +processors: +{{processors}} +{{/if}} diff --git a/packages/macos/data_stream/user_and_account_management/elasticsearch/ingest_pipeline/common-pipeline.yml.link b/packages/macos/data_stream/user_and_account_management/elasticsearch/ingest_pipeline/common-pipeline.yml.link new file mode 100644 index 00000000000..f146557ce04 --- /dev/null +++ b/packages/macos/data_stream/user_and_account_management/elasticsearch/ingest_pipeline/common-pipeline.yml.link @@ -0,0 +1 @@ +../../../../_dev/shared/common-pipeline.yml 6e7bb697288eb24aeb67d4950a8a3f38d89eb89f559dc01e605e543162267ddf \ No newline at end of file diff --git a/packages/macos/data_stream/user_and_account_management/elasticsearch/ingest_pipeline/default.yml b/packages/macos/data_stream/user_and_account_management/elasticsearch/ingest_pipeline/default.yml new file mode 100644 index 00000000000..c697c4de21c --- /dev/null +++ b/packages/macos/data_stream/user_and_account_management/elasticsearch/ingest_pipeline/default.yml @@ -0,0 +1,13 @@ +--- +description: Pipeline for processing user and account management logs. +processors: + - append: + field: event.category + tag: set_event_category + value: iam + - append: + field: event.type + tag: append_info_into_event_type_in_user_account_management + value: info + - pipeline: + name: '{{ IngestPipeline "common-pipeline" }}' diff --git a/packages/macos/data_stream/user_and_account_management/fields/base-fields.yml b/packages/macos/data_stream/user_and_account_management/fields/base-fields.yml new file mode 100644 index 00000000000..d1610051bae --- /dev/null +++ b/packages/macos/data_stream/user_and_account_management/fields/base-fields.yml @@ -0,0 +1,16 @@ +- name: data_stream.type + external: ecs +- name: data_stream.dataset + external: ecs +- name: data_stream.namespace + external: ecs +- name: event.module + type: constant_keyword + external: ecs + value: macos +- name: event.dataset + type: constant_keyword + external: ecs + value: macos.user_and_account_management +- name: '@timestamp' + external: ecs diff --git a/packages/macos/data_stream/user_and_account_management/fields/beats.yml b/packages/macos/data_stream/user_and_account_management/fields/beats.yml new file mode 100644 index 00000000000..d5fd38748ba --- /dev/null +++ b/packages/macos/data_stream/user_and_account_management/fields/beats.yml @@ -0,0 +1,6 @@ +- name: input.type + type: keyword + description: Type of Filebeat input. +- name: log.offset + type: long + description: Log offset. diff --git a/packages/macos/data_stream/user_and_account_management/fields/fields.yml b/packages/macos/data_stream/user_and_account_management/fields/fields.yml new file mode 100644 index 00000000000..56977045417 --- /dev/null +++ b/packages/macos/data_stream/user_and_account_management/fields/fields.yml @@ -0,0 +1,307 @@ +- name: macos + type: group + fields: + - name: activity_identifier + type: keyword + - name: backtrace + type: group + fields: + - name: frames + type: group + fields: + - name: image + type: group + fields: + - name: offset + type: keyword + - name: uuid + type: keyword + - name: boot_uuid + type: keyword + - name: category + type: keyword + - name: event + type: group + fields: + - name: category + type: keyword + - name: message + type: group + fields: + - name: account_id + type: keyword + - name: accurate_ecn_client + type: keyword + - name: accurate_ecn_server + type: keyword + - name: ack + type: keyword + - name: acks_compressed + type: long + - name: acks_delayed + type: long + - name: alpn + type: keyword + - name: attribution + type: keyword + - name: base_rtt_ms + type: keyword + - name: bundle_id + type: keyword + - name: bytes_in + type: long + - name: bytes_out + type: long + - name: cache_hit + type: boolean + - name: cipher_suite + type: keyword + - name: client_ip + type: ip + - name: client_port + type: long + - name: connection + type: long + - name: connection_detail + type: keyword + - name: connection_duration_ms + type: long + - name: connection_id + type: keyword + - name: connection_identifier + type: keyword + - name: connection_time + type: keyword + - name: connection_uuid + type: keyword + - name: delayed_acks_sent + type: long + - name: description + type: keyword + - name: dest_port + type: long + - name: direct_logout_type + type: long + - name: dns_duration + type: keyword + - name: dns_start + type: keyword + - name: domain_lookup_duration_ms + type: long + - name: duration + type: keyword + - name: ecn_acked + type: long + - name: ecn_in + type: long + - name: ecn_lost + type: long + - name: ecn_marked + type: long + - name: ecn_miss + type: long + - name: ecn_out + type: long + - name: ecn_sent + type: long + - name: expected_workload + type: keyword + - name: false_started + type: boolean + - name: flags + type: keyword + - name: flight_time + type: keyword + - name: group + type: keyword + - name: group_id + type: keyword + - name: guest_account + type: long + - name: home_directory_path + type: keyword + - name: hostname + type: keyword + - name: hostname_port + type: long + - name: init_flag + type: long + - name: interface + type: keyword + - name: listener + type: boolean + - name: mach + type: boolean + - name: name + type: keyword + - name: ocsp_received + type: boolean + - name: offered_ticket + type: boolean + - name: out_of_order_bytes + type: long + - name: packets_in + type: long + - name: packets_out + type: long + - name: pake + type: keyword + - name: path_status + type: keyword + - name: peer + type: boolean + - name: pid + type: long + - name: privacy_stance + type: keyword + - name: private_relay + type: boolean + - name: protocol + type: keyword + - name: rd_t_in + type: long + - name: rd_t_out + type: long + - name: read_stalls + type: long + - name: request_bytes + type: long + - name: request_duration_ms + type: long + - name: request_start_ms + type: long + - name: response_bytes + type: long + - name: response_duration_ms + type: long + - name: response_start_ms + type: long + - name: response_status + type: long + - name: resumed + type: boolean + - name: retransmitted_bytes + type: long + - name: rtt + type: keyword + - name: rtt_cache + type: keyword + - name: rtt_nc_ms + type: keyword + - name: rtt_updates + type: long + - name: rtt_var_ms + type: keyword + - name: rtt_var_nc_ms + type: keyword + - name: sct_received + type: boolean + - name: secure_connection_duration_ms + type: long + - name: seq + type: keyword + - name: server_id + type: keyword + - name: server_port + type: long + - name: session_agent_pid + type: keyword + - name: session_uuid + type: keyword + - name: signature_alg + type: keyword + - name: src_port + type: long + - name: state + type: keyword + - name: syns + type: long + - name: task_uid + type: keyword + - name: tcp_duration + type: keyword + - name: tcp_start + type: keyword + - name: tfo_in + type: long + - name: tfo_miss + type: long + - name: tfo_out + type: long + - name: timestamp_enabled + type: long + - name: tls_duration + type: keyword + - name: tls_version + type: keyword + - name: traffic_class + type: keyword + - name: transaction_duration_ms + type: long + - name: tso_enabled + type: long + - name: url + type: keyword + - name: url_hash + type: keyword + - name: user + type: group + fields: + - name: guid + type: keyword + - name: id + type: keyword + - name: long_name + type: keyword + - name: name + type: keyword + - name: win + type: keyword + - name: wr_t_in + type: long + - name: wr_t_out + type: long + - name: write_stalls + type: long + - name: type + type: keyword + - name: format_string + type: keyword + - name: mach_timestamp + type: double + - name: message_type + type: keyword + - name: parent_activity_identifier + type: keyword + - name: process + type: group + fields: + - name: id + type: long + - name: image_path + type: keyword + - name: image_uuid + type: keyword + - name: sender + type: group + fields: + - name: image_path + type: keyword + - name: image_uuid + type: keyword + - name: program_counter + type: long + - name: source + type: keyword + - name: subsystem + type: keyword + - name: thread_id + type: long + - name: timestamp + type: date + - name: timezone_name + type: keyword + - name: trace_id + type: keyword + - name: user_id + type: keyword diff --git a/packages/macos/data_stream/user_and_account_management/manifest.yml b/packages/macos/data_stream/user_and_account_management/manifest.yml new file mode 100644 index 00000000000..9947e1873dd --- /dev/null +++ b/packages/macos/data_stream/user_and_account_management/manifest.yml @@ -0,0 +1,139 @@ +title: User and Account Management +type: logs +streams: + - input: unifiedlogs + title: User and Account Management logs + description: Collect User and Account Management logs from macOS. + template_path: unifiedlogs.yml.hbs + enabled: false + vars: + - name: predicate + type: text + title: Predicate + description: | + Filters messages using the provided predicate based on NSPredicate. + A compound predicate or multiple predicates can be provided as a list. + For detailed information on the use of predicate based filtering, + please refer to the https://developer.apple.com/library/mac/documentation/Cocoa/Conceptual/Predicates/Articles/pSyntax.html[Predicate Programming Guide]. + multi: true + show_user: true + - name: start + type: text + title: Start date + description: | + Shows content starting from the provided date. + The following date/time formats are accepted: + `YYYY-MM-DD`, `YYYY-MM-DD HH:MM:SS`, `YYYY-MM-DD HH:MM:SSZZZZZ`. + show_user: false + - name: end + type: text + title: End date + description: | + Shows content up to the provided date. + The following date/time formats are accepted: + `YYYY-MM-DD`, `YYYY-MM-DD HH:MM:SS`, `YYYY-MM-DD HH:MM:SSZZZZZ`. + show_user: false + - name: process + type: text + title: Process + description: | + A list of the processes on which to operate. It accepts a PID or process name. + multi: true + show_user: false + - name: source + type: bool + title: Include source + description: | + Include symbol names and source line numbers for messages, if available. + show_user: false + - name: info + type: bool + title: Include info + description: | + Disable or enable info level messages. + show_user: false + - name: debug + type: bool + title: Include debug + description: | + Disable or enable debug level messages. + show_user: false + - name: backtrace + type: bool + title: Include backtrace + description: | + Disable or enable backtrace level messages. + show_user: false + - name: signpost + type: bool + title: Include signpost + description: | + Disable or enable signpost level messages. + show_user: false + - name: unreliable + type: bool + title: Annotate unreliable + description: | + Annotate events with whether the log was emitted unreliably. + show_user: false + - name: mach_continuous_time + type: bool + title: Use mach continuous time + description: | + Use mach continuous time timestamps rather than walltime. + show_user: false + - name: backfill + type: bool + title: Must backfill + description: | + If set to true the input will process all available logs since the beginning + of time the first time it starts. + show_user: false + - name: archive_file + type: text + title: Archive file + description: | + Display events stored in the given archive. + The archive must be a valid log archive bundle with the suffix `.logarchive`. + show_user: false + - name: trace_file + type: text + title: Trace file + description: | + Display events stored in the given `.tracev3` file. + In order to be decoded, the file must be contained within a valid `.logarchive`. + show_user: false + - name: preserve_original_event + type: bool + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field event.original. + multi: false + required: false + show_user: true + default: false + - name: tags + type: text + title: Tags + description: Tags for the data-stream. + multi: true + required: true + show_user: false + default: + - forwarded + - macos-user_and_account_management + - name: preserve_duplicate_custom_fields + required: false + title: Preserve duplicate custom fields + description: Preserve macos.* fields that were copied to Elastic Common Schema (ECS) fields. + type: bool + multi: false + show_user: false + default: false + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: >- + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. diff --git a/packages/macos/docs/README.md b/packages/macos/docs/README.md new file mode 100644 index 00000000000..0af973ef4fe --- /dev/null +++ b/packages/macos/docs/README.md @@ -0,0 +1,831 @@ +# macOS Integration for Elastic + +## Overview + +The macOS integration for Elastic allows you to collect and analyze unified logs from macOS systems. This integration leverages macOS's unified logging system to provide comprehensive visibility into system activities, security events, and application behaviors on macOS endpoints. + +macOS unified logging is Apple's centralized logging system that captures log messages from the kernel, system processes, and applications. This integration enables security teams to monitor macOS endpoints for suspicious activities, troubleshoot system issues, and maintain compliance with security policies. + +### Compatibility + +The macOS integration is compatible with macOS systems that support unified logging (macOS 10.12 Sierra and later). + +### How it works + +This integration uses the `unifiedlogs` input to collect log data from the macOS unified logging system. It can collect logs in real-time or from archived log files, with configurable filtering based on predicates, processes, and log levels. + +## What data does this integration collect? + +This integration collects unified log messages from macOS systems using configurable predicates to filter specific event types, including: + +- **Authentication logs**: User login/logout events, authentication failures, and credential-related activities + ``` + - 'process contains "sudo" OR composedMessage CONTAINS "sudo" OR process contains "su"' + - 'process contains "loginwindow" and composedMessage CONTAINS "sessionDidLogin"' + - 'process == "sshd"' + ``` +- **User & Account management**: User account creation, modification, and deletion events + ``` + - 'process == "sysadminctl" AND composedMessage CONTAINS "Creating user"' + - 'process == "dscl" AND composedMessage CONTAINS "create"' + - 'process == "sysadminctl" AND composedMessage CONTAINS "Deleting user"' + - 'process == "dscl" AND composedMessage CONTAINS "delete"' + - '(process == "dscl" OR process == "opendirectoryd") AND composedMessage CONTAINS "admin"' + ``` +- **Process execution monitoring**: Process creation, termination, and execution details + ``` + - 'eventMessage CONTAINS[c] "exec" OR eventMessage CONTAINS[c] "fork" OR eventMessage CONTAINS[c] "exited" OR eventMessage CONTAINS[c] "terminated"' + - 'subsystem == "com.apple.securityd" AND (composedMessage CONTAINS "code signing" OR composedMessage CONTAINS "not valid")' + - 'composedMessage CONTAINS "com.apple.quarantine"' + ``` +- **Network activity**: Network connections, DNS queries, and network-related events + ``` + - 'composedMessage CONTAINS "connect" AND (composedMessage CONTAINS "TCP" OR composedMessage CONTAINS "UDP")' + - 'composedMessage CONTAINS "disconnect" OR composedMessage CONTAINS "closed connection"' + - 'subsystem == "com.apple.necp" AND composedMessage CONTAINS "new connection"' + - 'eventMessage CONTAINS[c] "listening" AND eventMessage CONTAINS[c] "service"' + ``` +- **File reads/writes**: File system access, modifications, and permission changes + ``` + - '(eventMessage CONTAINS "open" OR eventMessage CONTAINS "write" OR eventMessage CONTAINS "unlink" OR eventMessage CONTAINS "rename") AND ((processImagePath BEGINSWITH "/System") OR (processImagePath BEGINSWITH "/bin") OR (processImagePath BEGINSWITH "/sbin") OR (processImagePath BEGINSWITH "/usr" AND NOT processImagePath BEGINSWITH "/usr/local") OR (processImagePath BEGINSWITH "/etc"))' + - 'subsystem == "com.apple.quarantine" OR eventMessage CONTAINS "com.apple.quarantine"' + ``` +- **System changes**: System configuration changes, software installations, and updates + ``` + - 'subsystem == "com.apple.security" OR subsystem == "com.apple.systempolicy" OR subsystem == "com.apple.installer" OR process == "Installer" OR process == "softwareupdated" OR eventMessage CONTAINS[c] "removed package" OR eventMessage CONTAINS[c] "forget package"' + ``` +- **Advanced monitoring**: Detailed system and application behavior logs + ``` + - '(composedMessage CONTAINS ".plist" AND (composedMessage CONTAINS "write" OR composedMessage CONTAINS "modified")) OR (composedMessage CONTAINS ".ssh" AND (composedMessage CONTAINS "write" OR composedMessage CONTAINS "modified")) OR (process == "kernel" AND composedMessage CONTAINS "boot") OR (process == "launchd" AND (composedMessage CONTAINS "started" OR composedMessage CONTAINS "listening")) OR (process == "loginwindow" AND composedMessage CONTAINS "sessionDidLogin") OR (composedMessage CONTAINS "posix_spawn" OR composedMessage CONTAINS "exec") OR (subsystem == "com.apple.securityd" AND (composedMessage CONTAINS "code signing" OR composedMessage CONTAINS "not valid"))' + ``` + +### Supported use cases +The macOS integration in Elastic enables comprehensive monitoring and analysis of system activities, network traffic, and application behavior across macOS devices. It supports use cases such as detecting security incidents, tracking network usage, auditing system events, and analyzing performance trends. By collecting and visualizing unified logs, it helps security and IT teams gain real-time visibility, identify anomalies, ensure compliance, and enhance overall endpoint security within macOS environments. + +## What do I need to use this integration? + +### From Elastic + +- Elastic Agent must be installed on the macOS system you want to monitor +- Appropriate permissions to read system logs on the macOS system + +### From macOS + +The integration requires: + +- macOS 10.12 Sierra or later (for unified logging support) +- Appropriate system permissions to access unified logs +- For some log categories, administrative privileges may be required + +## How do I deploy this integration? + +This integration requires Elastic Agent to be installed on the macOS systems you want to monitor. + +### Agent-based installation + +Elastic Agent must be installed. For more details, check the Elastic Agent [installation instructions](docs-content://reference/fleet/install-elastic-agents.md). You can install only one Elastic Agent per host. + +## Setup + +1. In the top search bar in Kibana, search for **Integrations**. +2. In the search bar, type **macOS**. +3. Select the **macOS** integration from the search results. +4. Select **Add macOS** to add the integration. +5. Enable and configure only the collection methods which you will use. + + **Basic Configuration:** + - Enable the log categories you want to collect: + - Authentication + - User & Account management + - Process execution monitoring + - Network activity + - File reads/writes + - System changes + - Advanced monitoring + + **Advanced Configuration (Optional):** + - **Predicate**: Use NSPredicate-based filtering to collect specific log messages + - **Process**: Specify particular processes to monitor (by PID or name) + - **Start/End dates**: Define time ranges for historical log collection + - **Log levels**: Configure which log levels to include (info, debug, backtrace, signpost) + - **Archive/Trace files**: Specify log archive or trace files to process + +6. Select **Save and continue** to save the integration. + +### Validation + +#### Dashboards populated + +1. In the top search bar in Kibana, search for **Dashboards**. +2. In the search bar, type **macOS**. +3. Select a dashboard for the dataset you are collecting, and verify the dashboard information is populated. + +## Performance and scaling + +- Unified log collection can generate significant data volume, especially with debug-level logging enabled +- Consider using predicates to filter logs and reduce data volume +- Monitor system performance impact when collecting high-volume log categories + +For more information on architectures that can be used for scaling this integration, check the [Ingest Architectures](https://www.elastic.co/docs/manage-data/ingest/ingest-reference-architectures) documentation. + +## Reference + +### ECS field reference + +### Authentication + +**Exported fields** + +| Field | Description | Type | +|---|---|---| +| @timestamp | Date/time when the event originated. This is the date/time extracted from the event, typically representing when the event was generated by the source. If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. Required field for all events. | date | +| data_stream.dataset | The field can contain anything that makes sense to signify the source of the data. Examples include `nginx.access`, `prometheus`, `endpoint` etc. For data streams that otherwise fit, but that do not have dataset set we use the value "generic" for the dataset value. `event.dataset` should have the same value as `data_stream.dataset`. Beyond the Elasticsearch data stream naming criteria noted above, the `dataset` value has additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | +| data_stream.namespace | A user defined namespace. Namespaces are useful to allow grouping of data. Many users already organize their indices this way, and the data stream naming scheme now provides this best practice as a default. Many users will populate this field with `default`. If no value is used, it falls back to `default`. Beyond the Elasticsearch index naming criteria noted above, `namespace` value has the additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | +| data_stream.type | An overarching type for the data stream. Currently allowed values are "logs" and "metrics". We expect to also add "traces" and "synthetics" in the near future. | constant_keyword | +| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | constant_keyword | +| event.module | Name of the module this data is coming from. If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module. | constant_keyword | +| input.type | Type of Filebeat input. | keyword | +| log.offset | Log offset. | long | +| macos.activity_identifier | | keyword | +| macos.backtrace.frames.image.offset | | keyword | +| macos.backtrace.frames.image.uuid | | keyword | +| macos.boot_uuid | | keyword | +| macos.category | | keyword | +| macos.event.category | | keyword | +| macos.event.message.account_id | | keyword | +| macos.event.message.accurate_ecn_client | | keyword | +| macos.event.message.accurate_ecn_server | | keyword | +| macos.event.message.ack | | keyword | +| macos.event.message.acks_compressed | | long | +| macos.event.message.acks_delayed | | long | +| macos.event.message.alpn | | keyword | +| macos.event.message.attribution | | keyword | +| macos.event.message.base_rtt_ms | | keyword | +| macos.event.message.bundle_id | | keyword | +| macos.event.message.bytes_in | | long | +| macos.event.message.bytes_out | | long | +| macos.event.message.cache_hit | | boolean | +| macos.event.message.cipher_suite | | keyword | +| macos.event.message.client_ip | | ip | +| macos.event.message.client_port | | long | +| macos.event.message.connection | | long | +| macos.event.message.connection_detail | | keyword | +| macos.event.message.connection_duration_ms | | long | +| macos.event.message.connection_id | | keyword | +| macos.event.message.connection_identifier | | keyword | +| macos.event.message.connection_time | | keyword | +| macos.event.message.connection_uuid | | keyword | +| macos.event.message.delayed_acks_sent | | long | +| macos.event.message.description | | keyword | +| macos.event.message.dest_port | | long | +| macos.event.message.direct_logout_type | | long | +| macos.event.message.dns_duration | | keyword | +| macos.event.message.dns_start | | keyword | +| macos.event.message.domain_lookup_duration_ms | | long | +| macos.event.message.duration | | keyword | +| macos.event.message.ecn_acked | | long | +| macos.event.message.ecn_in | | long | +| macos.event.message.ecn_lost | | long | +| macos.event.message.ecn_marked | | long | +| macos.event.message.ecn_miss | | long | +| macos.event.message.ecn_out | | long | +| macos.event.message.ecn_sent | | long | +| macos.event.message.expected_workload | | keyword | +| macos.event.message.false_started | | boolean | +| macos.event.message.flags | | keyword | +| macos.event.message.flight_time | | keyword | +| macos.event.message.group | | keyword | +| macos.event.message.group_id | | keyword | +| macos.event.message.guest_account | | long | +| macos.event.message.home_directory_path | | keyword | +| macos.event.message.hostname | | keyword | +| macos.event.message.hostname_port | | long | +| macos.event.message.init_flag | | long | +| macos.event.message.interface | | keyword | +| macos.event.message.listener | | boolean | +| macos.event.message.mach | | boolean | +| macos.event.message.name | | keyword | +| macos.event.message.ocsp_received | | boolean | +| macos.event.message.offered_ticket | | boolean | +| macos.event.message.out_of_order_bytes | | long | +| macos.event.message.packets_in | | long | +| macos.event.message.packets_out | | long | +| macos.event.message.pake | | keyword | +| macos.event.message.path_status | | keyword | +| macos.event.message.peer | | boolean | +| macos.event.message.pid | | long | +| macos.event.message.privacy_stance | | keyword | +| macos.event.message.private_relay | | boolean | +| macos.event.message.protocol | | keyword | +| macos.event.message.rd_t_in | | long | +| macos.event.message.rd_t_out | | long | +| macos.event.message.read_stalls | | long | +| macos.event.message.request_bytes | | long | +| macos.event.message.request_duration_ms | | long | +| macos.event.message.request_start_ms | | long | +| macos.event.message.response_bytes | | long | +| macos.event.message.response_duration_ms | | long | +| macos.event.message.response_start_ms | | long | +| macos.event.message.response_status | | long | +| macos.event.message.resumed | | boolean | +| macos.event.message.retransmitted_bytes | | long | +| macos.event.message.rtt | | keyword | +| macos.event.message.rtt_cache | | keyword | +| macos.event.message.rtt_nc_ms | | keyword | +| macos.event.message.rtt_updates | | long | +| macos.event.message.rtt_var_ms | | keyword | +| macos.event.message.rtt_var_nc_ms | | keyword | +| macos.event.message.sct_received | | boolean | +| macos.event.message.secure_connection_duration_ms | | long | +| macos.event.message.seq | | keyword | +| macos.event.message.server_id | | keyword | +| macos.event.message.server_port | | long | +| macos.event.message.session_agent_pid | | keyword | +| macos.event.message.session_uuid | | keyword | +| macos.event.message.signature_alg | | keyword | +| macos.event.message.src_port | | long | +| macos.event.message.state | | keyword | +| macos.event.message.syns | | long | +| macos.event.message.task_uid | | keyword | +| macos.event.message.tcp_duration | | keyword | +| macos.event.message.tcp_start | | keyword | +| macos.event.message.tfo_in | | long | +| macos.event.message.tfo_miss | | long | +| macos.event.message.tfo_out | | long | +| macos.event.message.timestamp_enabled | | long | +| macos.event.message.tls_duration | | keyword | +| macos.event.message.tls_version | | keyword | +| macos.event.message.traffic_class | | keyword | +| macos.event.message.transaction_duration_ms | | long | +| macos.event.message.tso_enabled | | long | +| macos.event.message.url | | keyword | +| macos.event.message.url_hash | | keyword | +| macos.event.message.user.guid | | keyword | +| macos.event.message.user.id | | keyword | +| macos.event.message.user.long_name | | keyword | +| macos.event.message.user.name | | keyword | +| macos.event.message.win | | keyword | +| macos.event.message.wr_t_in | | long | +| macos.event.message.wr_t_out | | long | +| macos.event.message.write_stalls | | long | +| macos.event.type | | keyword | +| macos.format_string | | keyword | +| macos.mach_timestamp | | double | +| macos.message_type | | keyword | +| macos.parent_activity_identifier | | keyword | +| macos.process.id | | long | +| macos.process.image_path | | keyword | +| macos.process.image_uuid | | keyword | +| macos.sender.image_path | | keyword | +| macos.sender.image_uuid | | keyword | +| macos.sender.program_counter | | long | +| macos.source | | keyword | +| macos.subsystem | | keyword | +| macos.thread_id | | long | +| macos.timestamp | | date | +| macos.timezone_name | | keyword | +| macos.trace_id | | keyword | +| macos.user_id | | keyword | + + +### File Read/Write + +**Exported fields** + +| Field | Description | Type | +|---|---|---| +| @timestamp | Date/time when the event originated. This is the date/time extracted from the event, typically representing when the event was generated by the source. If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. Required field for all events. | date | +| data_stream.dataset | The field can contain anything that makes sense to signify the source of the data. Examples include `nginx.access`, `prometheus`, `endpoint` etc. For data streams that otherwise fit, but that do not have dataset set we use the value "generic" for the dataset value. `event.dataset` should have the same value as `data_stream.dataset`. Beyond the Elasticsearch data stream naming criteria noted above, the `dataset` value has additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | +| data_stream.namespace | A user defined namespace. Namespaces are useful to allow grouping of data. Many users already organize their indices this way, and the data stream naming scheme now provides this best practice as a default. Many users will populate this field with `default`. If no value is used, it falls back to `default`. Beyond the Elasticsearch index naming criteria noted above, `namespace` value has the additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | +| data_stream.type | An overarching type for the data stream. Currently allowed values are "logs" and "metrics". We expect to also add "traces" and "synthetics" in the near future. | constant_keyword | +| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | constant_keyword | +| event.module | Name of the module this data is coming from. If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module. | constant_keyword | +| input.type | Type of Filebeat input. | keyword | +| log.offset | Log offset. | long | +| macos.activity_identifier | | keyword | +| macos.backtrace.frames.image.offset | | keyword | +| macos.backtrace.frames.image.uuid | | keyword | +| macos.boot_uuid | | keyword | +| macos.category | | keyword | +| macos.event.category | | keyword | +| macos.event.message.description | | keyword | +| macos.event.type | | keyword | +| macos.format_string | | keyword | +| macos.mach_timestamp | | double | +| macos.message_type | | keyword | +| macos.parent_activity_identifier | | keyword | +| macos.process.id | | long | +| macos.process.image_path | | keyword | +| macos.process.image_uuid | | keyword | +| macos.sender.image_path | | keyword | +| macos.sender.image_uuid | | keyword | +| macos.sender.program_counter | | long | +| macos.source | | keyword | +| macos.subsystem | | keyword | +| macos.thread_id | | long | +| macos.timestamp | | date | +| macos.timezone_name | | keyword | +| macos.trace_id | | keyword | +| macos.user_id | | keyword | + + +### Network Activity + +**Exported fields** + +| Field | Description | Type | +|---|---|---| +| @timestamp | Date/time when the event originated. This is the date/time extracted from the event, typically representing when the event was generated by the source. If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. Required field for all events. | date | +| data_stream.dataset | The field can contain anything that makes sense to signify the source of the data. Examples include `nginx.access`, `prometheus`, `endpoint` etc. For data streams that otherwise fit, but that do not have dataset set we use the value "generic" for the dataset value. `event.dataset` should have the same value as `data_stream.dataset`. Beyond the Elasticsearch data stream naming criteria noted above, the `dataset` value has additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | +| data_stream.namespace | A user defined namespace. Namespaces are useful to allow grouping of data. Many users already organize their indices this way, and the data stream naming scheme now provides this best practice as a default. Many users will populate this field with `default`. If no value is used, it falls back to `default`. Beyond the Elasticsearch index naming criteria noted above, `namespace` value has the additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | +| data_stream.type | An overarching type for the data stream. Currently allowed values are "logs" and "metrics". We expect to also add "traces" and "synthetics" in the near future. | constant_keyword | +| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | constant_keyword | +| event.module | Name of the module this data is coming from. If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module. | constant_keyword | +| input.type | Type of Filebeat input. | keyword | +| log.offset | Log offset. | long | +| macos.activity_identifier | | keyword | +| macos.backtrace.frames.image.offset | | keyword | +| macos.backtrace.frames.image.uuid | | keyword | +| macos.boot_uuid | | keyword | +| macos.category | | keyword | +| macos.event.category | | keyword | +| macos.event.message.account_id | | keyword | +| macos.event.message.accurate_ecn_client | | keyword | +| macos.event.message.accurate_ecn_server | | keyword | +| macos.event.message.ack | | keyword | +| macos.event.message.acks_compressed | | long | +| macos.event.message.acks_delayed | | long | +| macos.event.message.alpn | | keyword | +| macos.event.message.attribution | | keyword | +| macos.event.message.base_rtt_ms | | keyword | +| macos.event.message.bundle_id | | keyword | +| macos.event.message.bytes_in | | long | +| macos.event.message.bytes_out | | long | +| macos.event.message.cache_hit | | boolean | +| macos.event.message.cipher_suite | | keyword | +| macos.event.message.client_ip | | ip | +| macos.event.message.client_port | | long | +| macos.event.message.connection | | long | +| macos.event.message.connection_detail | | keyword | +| macos.event.message.connection_duration_ms | | long | +| macos.event.message.connection_id | | keyword | +| macos.event.message.connection_identifier | | keyword | +| macos.event.message.connection_time | | keyword | +| macos.event.message.connection_uuid | | keyword | +| macos.event.message.delayed_acks_sent | | long | +| macos.event.message.description | | keyword | +| macos.event.message.dest_port | | long | +| macos.event.message.dns_duration | | keyword | +| macos.event.message.dns_start | | keyword | +| macos.event.message.domain_lookup_duration_ms | | long | +| macos.event.message.duration | | keyword | +| macos.event.message.ecn_acked | | long | +| macos.event.message.ecn_in | | long | +| macos.event.message.ecn_lost | | long | +| macos.event.message.ecn_marked | | long | +| macos.event.message.ecn_miss | | long | +| macos.event.message.ecn_out | | long | +| macos.event.message.ecn_sent | | long | +| macos.event.message.expected_workload | | keyword | +| macos.event.message.false_started | | boolean | +| macos.event.message.flags | | keyword | +| macos.event.message.flight_time | | keyword | +| macos.event.message.group | | keyword | +| macos.event.message.hostname | | keyword | +| macos.event.message.hostname_port | | long | +| macos.event.message.init_flag | | long | +| macos.event.message.interface | | keyword | +| macos.event.message.listener | | boolean | +| macos.event.message.mach | | boolean | +| macos.event.message.name | | keyword | +| macos.event.message.ocsp_received | | boolean | +| macos.event.message.offered_ticket | | boolean | +| macos.event.message.out_of_order_bytes | | long | +| macos.event.message.packets_in | | long | +| macos.event.message.packets_out | | long | +| macos.event.message.pake | | keyword | +| macos.event.message.path_status | | keyword | +| macos.event.message.peer | | boolean | +| macos.event.message.pid | | long | +| macos.event.message.privacy_stance | | keyword | +| macos.event.message.private_relay | | boolean | +| macos.event.message.protocol | | keyword | +| macos.event.message.rcv_nxt | | keyword | +| macos.event.message.rd_t_in | | long | +| macos.event.message.rd_t_out | | long | +| macos.event.message.read_stalls | | long | +| macos.event.message.request_bytes | | long | +| macos.event.message.request_duration_ms | | long | +| macos.event.message.request_start_ms | | long | +| macos.event.message.response_bytes | | long | +| macos.event.message.response_duration_ms | | long | +| macos.event.message.response_start_ms | | long | +| macos.event.message.response_status | | long | +| macos.event.message.resumed | | boolean | +| macos.event.message.retransmitted_bytes | | long | +| macos.event.message.rtt | | keyword | +| macos.event.message.rtt_cache | | keyword | +| macos.event.message.rtt_nc_ms | | keyword | +| macos.event.message.rtt_updates | | long | +| macos.event.message.rtt_var_ms | | keyword | +| macos.event.message.rtt_var_nc_ms | | keyword | +| macos.event.message.sct_received | | boolean | +| macos.event.message.secure_connection_duration_ms | | long | +| macos.event.message.seq | | keyword | +| macos.event.message.server_id | | keyword | +| macos.event.message.server_port | | long | +| macos.event.message.session_uuid | | keyword | +| macos.event.message.signature_alg | | keyword | +| macos.event.message.snd_una | | keyword | +| macos.event.message.src_port | | long | +| macos.event.message.state | | keyword | +| macos.event.message.syns | | long | +| macos.event.message.task_uid | | keyword | +| macos.event.message.tcp_duration | | keyword | +| macos.event.message.tcp_start | | keyword | +| macos.event.message.tfo_in | | long | +| macos.event.message.tfo_miss | | long | +| macos.event.message.tfo_out | | long | +| macos.event.message.timestamp_enabled | | long | +| macos.event.message.tls_duration | | keyword | +| macos.event.message.tls_version | | keyword | +| macos.event.message.traffic_class | | keyword | +| macos.event.message.transaction_duration_ms | | long | +| macos.event.message.tso_enabled | | long | +| macos.event.message.url | | keyword | +| macos.event.message.url_hash | | keyword | +| macos.event.message.win | | keyword | +| macos.event.message.wr_t_in | | long | +| macos.event.message.wr_t_out | | long | +| macos.event.message.write_stalls | | long | +| macos.event.type | | keyword | +| macos.format_string | | keyword | +| macos.mach_timestamp | | double | +| macos.message_type | | keyword | +| macos.parent_activity_identifier | | keyword | +| macos.process.id | | long | +| macos.process.image_path | | keyword | +| macos.process.image_uuid | | keyword | +| macos.sender.image_path | | keyword | +| macos.sender.image_uuid | | keyword | +| macos.sender.program_counter | | long | +| macos.source | | keyword | +| macos.subsystem | | keyword | +| macos.thread_id | | long | +| macos.timestamp | | date | +| macos.timezone_name | | keyword | +| macos.trace_id | | keyword | +| macos.user_id | | keyword | + + +### Process Execution Monitoring + +**Exported fields** + +| Field | Description | Type | +|---|---|---| +| @timestamp | Date/time when the event originated. This is the date/time extracted from the event, typically representing when the event was generated by the source. If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. Required field for all events. | date | +| data_stream.dataset | The field can contain anything that makes sense to signify the source of the data. Examples include `nginx.access`, `prometheus`, `endpoint` etc. For data streams that otherwise fit, but that do not have dataset set we use the value "generic" for the dataset value. `event.dataset` should have the same value as `data_stream.dataset`. Beyond the Elasticsearch data stream naming criteria noted above, the `dataset` value has additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | +| data_stream.namespace | A user defined namespace. Namespaces are useful to allow grouping of data. Many users already organize their indices this way, and the data stream naming scheme now provides this best practice as a default. Many users will populate this field with `default`. If no value is used, it falls back to `default`. Beyond the Elasticsearch index naming criteria noted above, `namespace` value has the additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | +| data_stream.type | An overarching type for the data stream. Currently allowed values are "logs" and "metrics". We expect to also add "traces" and "synthetics" in the near future. | constant_keyword | +| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | constant_keyword | +| event.module | Name of the module this data is coming from. If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module. | constant_keyword | +| input.type | Type of Filebeat input. | keyword | +| log.offset | Log offset. | long | +| macos.activity_identifier | | keyword | +| macos.backtrace.frames.image.offset | | keyword | +| macos.backtrace.frames.image.uuid | | keyword | +| macos.boot_uuid | | keyword | +| macos.category | | keyword | +| macos.event.category | | keyword | +| macos.event.message.description | | keyword | +| macos.event.type | | keyword | +| macos.format_string | | keyword | +| macos.mach_timestamp | | double | +| macos.message_type | | keyword | +| macos.parent_activity_identifier | | keyword | +| macos.process.id | | long | +| macos.process.image_path | | keyword | +| macos.process.image_uuid | | keyword | +| macos.sender.image_path | | keyword | +| macos.sender.image_uuid | | keyword | +| macos.sender.program_counter | | long | +| macos.source | | keyword | +| macos.subsystem | | keyword | +| macos.thread_id | | long | +| macos.timestamp | | date | +| macos.timezone_name | | keyword | +| macos.trace_id | | keyword | +| macos.user_id | | keyword | + + +### System Change + +**Exported fields** + +| Field | Description | Type | +|---|---|---| +| @timestamp | Date/time when the event originated. This is the date/time extracted from the event, typically representing when the event was generated by the source. If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. Required field for all events. | date | +| data_stream.dataset | The field can contain anything that makes sense to signify the source of the data. Examples include `nginx.access`, `prometheus`, `endpoint` etc. For data streams that otherwise fit, but that do not have dataset set we use the value "generic" for the dataset value. `event.dataset` should have the same value as `data_stream.dataset`. Beyond the Elasticsearch data stream naming criteria noted above, the `dataset` value has additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | +| data_stream.namespace | A user defined namespace. Namespaces are useful to allow grouping of data. Many users already organize their indices this way, and the data stream naming scheme now provides this best practice as a default. Many users will populate this field with `default`. If no value is used, it falls back to `default`. Beyond the Elasticsearch index naming criteria noted above, `namespace` value has the additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | +| data_stream.type | An overarching type for the data stream. Currently allowed values are "logs" and "metrics". We expect to also add "traces" and "synthetics" in the near future. | constant_keyword | +| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | constant_keyword | +| event.module | Name of the module this data is coming from. If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module. | constant_keyword | +| input.type | Type of Filebeat input. | keyword | +| log.offset | Log offset. | long | +| macos.activity_identifier | | keyword | +| macos.backtrace.frames.image.offset | | keyword | +| macos.backtrace.frames.image.uuid | | keyword | +| macos.boot_uuid | | keyword | +| macos.category | | keyword | +| macos.event.category | | keyword | +| macos.event.message.account_id | | keyword | +| macos.event.message.accurate_ecn_client | | keyword | +| macos.event.message.accurate_ecn_server | | keyword | +| macos.event.message.ack | | keyword | +| macos.event.message.acks_compressed | | long | +| macos.event.message.acks_delayed | | long | +| macos.event.message.alpn | | keyword | +| macos.event.message.attribution | | keyword | +| macos.event.message.base_rtt_ms | | keyword | +| macos.event.message.bundle_id | | keyword | +| macos.event.message.bytes_in | | long | +| macos.event.message.bytes_out | | long | +| macos.event.message.cache_hit | | boolean | +| macos.event.message.cipher_suite | | keyword | +| macos.event.message.client_ip | | ip | +| macos.event.message.client_port | | long | +| macos.event.message.connection | | long | +| macos.event.message.connection_detail | | keyword | +| macos.event.message.connection_duration_ms | | long | +| macos.event.message.connection_id | | keyword | +| macos.event.message.connection_identifier | | keyword | +| macos.event.message.connection_time | | keyword | +| macos.event.message.connection_uuid | | keyword | +| macos.event.message.delayed_acks_sent | | long | +| macos.event.message.description | | keyword | +| macos.event.message.dest_port | | long | +| macos.event.message.dns_duration | | keyword | +| macos.event.message.dns_start | | keyword | +| macos.event.message.domain_lookup_duration_ms | | long | +| macos.event.message.duration | | keyword | +| macos.event.message.ecn_acked | | long | +| macos.event.message.ecn_in | | long | +| macos.event.message.ecn_lost | | long | +| macos.event.message.ecn_marked | | long | +| macos.event.message.ecn_miss | | long | +| macos.event.message.ecn_out | | long | +| macos.event.message.ecn_sent | | long | +| macos.event.message.expected_workload | | keyword | +| macos.event.message.false_started | | boolean | +| macos.event.message.flags | | keyword | +| macos.event.message.flight_time | | keyword | +| macos.event.message.group | | keyword | +| macos.event.message.hostname | | keyword | +| macos.event.message.hostname_port | | long | +| macos.event.message.init_flag | | long | +| macos.event.message.interface | | keyword | +| macos.event.message.listener | | boolean | +| macos.event.message.mach | | boolean | +| macos.event.message.name | | keyword | +| macos.event.message.ocsp_received | | boolean | +| macos.event.message.offered_ticket | | boolean | +| macos.event.message.out_of_order_bytes | | long | +| macos.event.message.packets_in | | long | +| macos.event.message.packets_out | | long | +| macos.event.message.pake | | keyword | +| macos.event.message.path_status | | keyword | +| macos.event.message.peer | | boolean | +| macos.event.message.pid | | long | +| macos.event.message.privacy_stance | | keyword | +| macos.event.message.private_relay | | boolean | +| macos.event.message.protocol | | keyword | +| macos.event.message.rcv_nxt | | keyword | +| macos.event.message.rd_t_in | | long | +| macos.event.message.rd_t_out | | long | +| macos.event.message.read_stalls | | long | +| macos.event.message.request_bytes | | long | +| macos.event.message.request_duration_ms | | long | +| macos.event.message.request_start_ms | | long | +| macos.event.message.response_bytes | | long | +| macos.event.message.response_duration_ms | | long | +| macos.event.message.response_start_ms | | long | +| macos.event.message.response_status | | long | +| macos.event.message.resumed | | boolean | +| macos.event.message.retransmitted_bytes | | long | +| macos.event.message.rtt | | keyword | +| macos.event.message.rtt_cache | | keyword | +| macos.event.message.rtt_nc_ms | | keyword | +| macos.event.message.rtt_updates | | long | +| macos.event.message.rtt_var_ms | | keyword | +| macos.event.message.rtt_var_nc_ms | | keyword | +| macos.event.message.sct_received | | boolean | +| macos.event.message.secure_connection_duration_ms | | long | +| macos.event.message.seq | | keyword | +| macos.event.message.server_id | | keyword | +| macos.event.message.server_port | | long | +| macos.event.message.session_uuid | | keyword | +| macos.event.message.signature_alg | | keyword | +| macos.event.message.snd_una | | keyword | +| macos.event.message.src_port | | long | +| macos.event.message.state | | keyword | +| macos.event.message.syns | | long | +| macos.event.message.task_uid | | keyword | +| macos.event.message.tcp_duration | | keyword | +| macos.event.message.tcp_start | | keyword | +| macos.event.message.tfo_in | | long | +| macos.event.message.tfo_miss | | long | +| macos.event.message.tfo_out | | long | +| macos.event.message.timestamp_enabled | | long | +| macos.event.message.tls_duration | | keyword | +| macos.event.message.tls_version | | keyword | +| macos.event.message.traffic_class | | keyword | +| macos.event.message.transaction_duration_ms | | long | +| macos.event.message.tso_enabled | | long | +| macos.event.message.url | | keyword | +| macos.event.message.url_hash | | keyword | +| macos.event.message.win | | keyword | +| macos.event.message.wr_t_in | | long | +| macos.event.message.wr_t_out | | long | +| macos.event.message.write_stalls | | long | +| macos.event.type | | keyword | +| macos.format_string | | keyword | +| macos.mach_timestamp | | double | +| macos.message_type | | keyword | +| macos.parent_activity_identifier | | keyword | +| macos.process.id | | long | +| macos.process.image_path | | keyword | +| macos.process.image_uuid | | keyword | +| macos.sender.image_path | | keyword | +| macos.sender.image_uuid | | keyword | +| macos.sender.program_counter | | long | +| macos.source | | keyword | +| macos.subsystem | | keyword | +| macos.thread_id | | long | +| macos.timestamp | | date | +| macos.timezone_name | | keyword | +| macos.trace_id | | keyword | +| macos.user_id | | keyword | + + +### User and Account Management + +**Exported fields** + +| Field | Description | Type | +|---|---|---| +| @timestamp | Date/time when the event originated. This is the date/time extracted from the event, typically representing when the event was generated by the source. If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. Required field for all events. | date | +| data_stream.dataset | The field can contain anything that makes sense to signify the source of the data. Examples include `nginx.access`, `prometheus`, `endpoint` etc. For data streams that otherwise fit, but that do not have dataset set we use the value "generic" for the dataset value. `event.dataset` should have the same value as `data_stream.dataset`. Beyond the Elasticsearch data stream naming criteria noted above, the `dataset` value has additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | +| data_stream.namespace | A user defined namespace. Namespaces are useful to allow grouping of data. Many users already organize their indices this way, and the data stream naming scheme now provides this best practice as a default. Many users will populate this field with `default`. If no value is used, it falls back to `default`. Beyond the Elasticsearch index naming criteria noted above, `namespace` value has the additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | +| data_stream.type | An overarching type for the data stream. Currently allowed values are "logs" and "metrics". We expect to also add "traces" and "synthetics" in the near future. | constant_keyword | +| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | constant_keyword | +| event.module | Name of the module this data is coming from. If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module. | constant_keyword | +| input.type | Type of Filebeat input. | keyword | +| log.offset | Log offset. | long | +| macos.activity_identifier | | keyword | +| macos.backtrace.frames.image.offset | | keyword | +| macos.backtrace.frames.image.uuid | | keyword | +| macos.boot_uuid | | keyword | +| macos.category | | keyword | +| macos.event.category | | keyword | +| macos.event.message.account_id | | keyword | +| macos.event.message.accurate_ecn_client | | keyword | +| macos.event.message.accurate_ecn_server | | keyword | +| macos.event.message.ack | | keyword | +| macos.event.message.acks_compressed | | long | +| macos.event.message.acks_delayed | | long | +| macos.event.message.alpn | | keyword | +| macos.event.message.attribution | | keyword | +| macos.event.message.base_rtt_ms | | keyword | +| macos.event.message.bundle_id | | keyword | +| macos.event.message.bytes_in | | long | +| macos.event.message.bytes_out | | long | +| macos.event.message.cache_hit | | boolean | +| macos.event.message.cipher_suite | | keyword | +| macos.event.message.client_ip | | ip | +| macos.event.message.client_port | | long | +| macos.event.message.connection | | long | +| macos.event.message.connection_detail | | keyword | +| macos.event.message.connection_duration_ms | | long | +| macos.event.message.connection_id | | keyword | +| macos.event.message.connection_identifier | | keyword | +| macos.event.message.connection_time | | keyword | +| macos.event.message.connection_uuid | | keyword | +| macos.event.message.delayed_acks_sent | | long | +| macos.event.message.description | | keyword | +| macos.event.message.dest_port | | long | +| macos.event.message.direct_logout_type | | long | +| macos.event.message.dns_duration | | keyword | +| macos.event.message.dns_start | | keyword | +| macos.event.message.domain_lookup_duration_ms | | long | +| macos.event.message.duration | | keyword | +| macos.event.message.ecn_acked | | long | +| macos.event.message.ecn_in | | long | +| macos.event.message.ecn_lost | | long | +| macos.event.message.ecn_marked | | long | +| macos.event.message.ecn_miss | | long | +| macos.event.message.ecn_out | | long | +| macos.event.message.ecn_sent | | long | +| macos.event.message.expected_workload | | keyword | +| macos.event.message.false_started | | boolean | +| macos.event.message.flags | | keyword | +| macos.event.message.flight_time | | keyword | +| macos.event.message.group | | keyword | +| macos.event.message.group_id | | keyword | +| macos.event.message.guest_account | | long | +| macos.event.message.home_directory_path | | keyword | +| macos.event.message.hostname | | keyword | +| macos.event.message.hostname_port | | long | +| macos.event.message.init_flag | | long | +| macos.event.message.interface | | keyword | +| macos.event.message.listener | | boolean | +| macos.event.message.mach | | boolean | +| macos.event.message.name | | keyword | +| macos.event.message.ocsp_received | | boolean | +| macos.event.message.offered_ticket | | boolean | +| macos.event.message.out_of_order_bytes | | long | +| macos.event.message.packets_in | | long | +| macos.event.message.packets_out | | long | +| macos.event.message.pake | | keyword | +| macos.event.message.path_status | | keyword | +| macos.event.message.peer | | boolean | +| macos.event.message.pid | | long | +| macos.event.message.privacy_stance | | keyword | +| macos.event.message.private_relay | | boolean | +| macos.event.message.protocol | | keyword | +| macos.event.message.rd_t_in | | long | +| macos.event.message.rd_t_out | | long | +| macos.event.message.read_stalls | | long | +| macos.event.message.request_bytes | | long | +| macos.event.message.request_duration_ms | | long | +| macos.event.message.request_start_ms | | long | +| macos.event.message.response_bytes | | long | +| macos.event.message.response_duration_ms | | long | +| macos.event.message.response_start_ms | | long | +| macos.event.message.response_status | | long | +| macos.event.message.resumed | | boolean | +| macos.event.message.retransmitted_bytes | | long | +| macos.event.message.rtt | | keyword | +| macos.event.message.rtt_cache | | keyword | +| macos.event.message.rtt_nc_ms | | keyword | +| macos.event.message.rtt_updates | | long | +| macos.event.message.rtt_var_ms | | keyword | +| macos.event.message.rtt_var_nc_ms | | keyword | +| macos.event.message.sct_received | | boolean | +| macos.event.message.secure_connection_duration_ms | | long | +| macos.event.message.seq | | keyword | +| macos.event.message.server_id | | keyword | +| macos.event.message.server_port | | long | +| macos.event.message.session_agent_pid | | keyword | +| macos.event.message.session_uuid | | keyword | +| macos.event.message.signature_alg | | keyword | +| macos.event.message.src_port | | long | +| macos.event.message.state | | keyword | +| macos.event.message.syns | | long | +| macos.event.message.task_uid | | keyword | +| macos.event.message.tcp_duration | | keyword | +| macos.event.message.tcp_start | | keyword | +| macos.event.message.tfo_in | | long | +| macos.event.message.tfo_miss | | long | +| macos.event.message.tfo_out | | long | +| macos.event.message.timestamp_enabled | | long | +| macos.event.message.tls_duration | | keyword | +| macos.event.message.tls_version | | keyword | +| macos.event.message.traffic_class | | keyword | +| macos.event.message.transaction_duration_ms | | long | +| macos.event.message.tso_enabled | | long | +| macos.event.message.url | | keyword | +| macos.event.message.url_hash | | keyword | +| macos.event.message.user.guid | | keyword | +| macos.event.message.user.id | | keyword | +| macos.event.message.user.long_name | | keyword | +| macos.event.message.user.name | | keyword | +| macos.event.message.win | | keyword | +| macos.event.message.wr_t_in | | long | +| macos.event.message.wr_t_out | | long | +| macos.event.message.write_stalls | | long | +| macos.event.type | | keyword | +| macos.format_string | | keyword | +| macos.mach_timestamp | | double | +| macos.message_type | | keyword | +| macos.parent_activity_identifier | | keyword | +| macos.process.id | | long | +| macos.process.image_path | | keyword | +| macos.process.image_uuid | | keyword | +| macos.sender.image_path | | keyword | +| macos.sender.image_uuid | | keyword | +| macos.sender.program_counter | | long | +| macos.source | | keyword | +| macos.subsystem | | keyword | +| macos.thread_id | | long | +| macos.timestamp | | date | +| macos.timezone_name | | keyword | +| macos.trace_id | | keyword | +| macos.user_id | | keyword | + + +### Inputs used + +These inputs can be used in this integration: + +- [Unified Logs](https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-unifiedlogs) diff --git a/packages/macos/img/macos-logo.svg b/packages/macos/img/macos-logo.svg new file mode 100644 index 00000000000..baeedeed9bc --- /dev/null +++ b/packages/macos/img/macos-logo.svg @@ -0,0 +1,17 @@ + + + + + \ No newline at end of file diff --git a/packages/macos/img/macos-unified-logs-network-dashboard.png b/packages/macos/img/macos-unified-logs-network-dashboard.png new file mode 100644 index 00000000000..f6748d6ee6c Binary files /dev/null and b/packages/macos/img/macos-unified-logs-network-dashboard.png differ diff --git a/packages/macos/img/macos-unified-logs-overview-dashboard.png b/packages/macos/img/macos-unified-logs-overview-dashboard.png new file mode 100644 index 00000000000..0e3553b413c Binary files /dev/null and b/packages/macos/img/macos-unified-logs-overview-dashboard.png differ diff --git a/packages/macos/kibana/dashboard/macos-4b49d421-2f03-4dd2-891f-cbd7e2786e35.json b/packages/macos/kibana/dashboard/macos-4b49d421-2f03-4dd2-891f-cbd7e2786e35.json new file mode 100644 index 00000000000..51da9c01f18 --- /dev/null +++ b/packages/macos/kibana/dashboard/macos-4b49d421-2f03-4dd2-891f-cbd7e2786e35.json @@ -0,0 +1,794 @@ +{ + "attributes": { + "controlGroupInput": { + "chainingSystem": "HIERARCHICAL", + "controlStyle": "oneLine", + "ignoreParentSettingsJSON": { + "ignoreFilters": false, + "ignoreQuery": false, + "ignoreTimerange": false, + "ignoreValidations": false + }, + "panelsJSON": { + "1b2a550b-350c-4e01-8695-967e23a2ef8a": { + "explicitInput": { + "dataViewId": "logs-*", + "exclude": false, + "existsSelected": false, + "fieldName": "macos.event.category", + "searchTechnique": "prefix", + "selectedOptions": [], + "sort": { + "by": "_count", + "direction": "desc" + }, + "title": "Event Category" + }, + "grow": true, + "order": 1, + "type": "optionsListControl", + "width": "medium" + }, + "410ef196-6693-4255-930f-065d731cd674": { + "explicitInput": { + "dataViewId": "logs-*", + "exclude": false, + "existsSelected": true, + "fieldName": "event.category", + "searchTechnique": "prefix", + "selectedOptions": [], + "sort": { + "by": "_count", + "direction": "desc" + }, + "title": "Event Type" + }, + "grow": true, + "order": 0, + "type": "optionsListControl", + "width": "medium" + } + }, + "showApplySelections": false + }, + "description": "This dashboard provides an overview of macOS unified logs.", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "optionsJSON": { + "hidePanelTitles": false, + "syncColors": false, + "syncCursor": true, + "syncTooltips": false, + "useMargins": true + }, + "panelsJSON": [ + { + "embeddableConfig": { + "attributes": { + "layout": "vertical", + "links": [ + { + "destinationRefName": "link_4af59c8f-d1a1-49b9-b4e4-927304a4d114_dashboard", + "id": "4af59c8f-d1a1-49b9-b4e4-927304a4d114", + "label": "Overview", + "options": { + "openInNewTab": false, + "useCurrentDateRange": false, + "useCurrentFilters": false + }, + "order": 0, + "type": "dashboardLink" + }, + { + "destinationRefName": "link_c2813a92-3079-46ca-b3e6-79796cc36725_dashboard", + "id": "c2813a92-3079-46ca-b3e6-79796cc36725", + "label": "Network", + "options": { + "openInNewTab": false, + "useCurrentDateRange": false, + "useCurrentFilters": false + }, + "order": 1, + "type": "dashboardLink" + } + ] + }, + "enhancements": {}, + "title": "Navigation" + }, + "gridData": { + "h": 5, + "i": "77764699-34eb-4325-8a41-63ef03163050", + "w": 10, + "x": 0, + "y": 0 + }, + "panelIndex": "77764699-34eb-4325-8a41-63ef03163050", + "type": "links" + }, + { + "embeddableConfig": { + "enhancements": { + "dynamicActions": { + "events": [] + } + }, + "hidePanelTitles": true, + "savedVis": { + "data": { + "aggs": [], + "searchSource": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "description": "", + "params": { + "fontSize": 12, + "markdown": "### MacOS\n \n#### Description\n\nThis dashboard provides visibility into the macOS Unified Logs data stream, offering insights into system and application events across the operating system. It highlights breakdowns of event categories over time to identify key system components generating logs, along with detailed tables showcasing the top process image paths and top sender image paths for deeper operational context. Together, these visualizations help monitor system behavior, troubleshoot issues, and enhance macOS endpoint visibility.\n\n**[Integration Page](/app/integrations/detail/macos/overview)**", + "openLinksInNewTab": false + }, + "title": "", + "type": "markdown", + "uiState": {} + }, + "title": "" + }, + "gridData": { + "h": 17, + "i": "da83d66c-b4a7-4ec9-a9be-d69453a2999a", + "w": 10, + "x": 0, + "y": 5 + }, + "panelIndex": "da83d66c-b4a7-4ec9-a9be-d69453a2999a", + "type": "visualization" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "logs-*", + "name": "indexpattern-datasource-layer-6d111cc0-533f-4394-bee1-3befbb34d4ba", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "currentIndexPatternId": "logs-*", + "layers": { + "6d111cc0-533f-4394-bee1-3befbb34d4ba": { + "columnOrder": [ + "abf11aea-116d-4fbb-a144-eac11bb1373f", + "d933d392-5b5b-4e93-aa2b-98762adbb762", + "4b0509e8-6b8b-4aaa-a7cd-095eff69febe" + ], + "columns": { + "4b0509e8-6b8b-4aaa-a7cd-095eff69febe": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Count", + "operationType": "count", + "params": { + "emptyAsNull": false + }, + "sourceField": "___records___" + }, + "abf11aea-116d-4fbb-a144-eac11bb1373f": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Event Category", + "operationType": "terms", + "params": { + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderBy": { + "columnId": "4b0509e8-6b8b-4aaa-a7cd-095eff69febe", + "type": "column" + }, + "orderDirection": "desc", + "otherBucket": false, + "parentFormat": { + "id": "terms" + }, + "size": 6 + }, + "sourceField": "event.category" + }, + "d933d392-5b5b-4e93-aa2b-98762adbb762": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "dropPartials": false, + "includeEmptyRows": true, + "interval": "auto" + }, + "sourceField": "@timestamp" + } + }, + "ignoreGlobalFilters": false, + "incompleteColumns": {}, + "indexPatternId": "logs-*", + "sampling": 1 + } + } + }, + "indexpattern": { + "currentIndexPatternId": "logs-*", + "layers": {} + }, + "textBased": { + "indexPatternRefs": [ + { + "id": "logs-*", + "timeField": "@timestamp", + "title": "logs-*" + } + ], + "layers": {} + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "layers": [ + { + "accessors": [ + "4b0509e8-6b8b-4aaa-a7cd-095eff69febe" + ], + "colorMapping": { + "assignments": [], + "colorMode": { + "type": "categorical" + }, + "paletteId": "default", + "specialAssignments": [ + { + "color": { + "type": "loop" + }, + "rules": [ + { + "type": "other" + } + ], + "touched": false + } + ] + }, + "layerId": "6d111cc0-533f-4394-bee1-3befbb34d4ba", + "layerType": "data", + "position": "top", + "seriesType": "line", + "showGridlines": false, + "splitAccessor": "abf11aea-116d-4fbb-a144-eac11bb1373f", + "xAccessor": "d933d392-5b5b-4e93-aa2b-98762adbb762" + } + ], + "legend": { + "isVisible": true, + "position": "right", + "shouldTruncate": false, + "showSingleSeries": true + }, + "preferredSeriesType": "line", + "title": "Empty XY chart", + "valueLabels": "hide" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsXY" + }, + "enhancements": { + "dynamicActions": { + "events": [] + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "syncColors": false, + "syncCursor": true, + "syncTooltips": false, + "title": "Aggregated Security Events over Time" + }, + "gridData": { + "h": 22, + "i": "27dfea44-7dd8-45ec-8caa-dacbbb35ed39", + "w": 38, + "x": 10, + "y": 0 + }, + "panelIndex": "27dfea44-7dd8-45ec-8caa-dacbbb35ed39", + "type": "lens" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "logs-*", + "name": "indexpattern-datasource-layer-42a78020-0d1f-443e-a82a-6df70600549d", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "currentIndexPatternId": "logs-*", + "layers": { + "42a78020-0d1f-443e-a82a-6df70600549d": { + "columnOrder": [ + "33cf8fd2-12fc-47ba-8fe0-efc49a7ed68f", + "141b5b8f-7f60-43dd-b5e8-fcf827e5870f" + ], + "columns": { + "141b5b8f-7f60-43dd-b5e8-fcf827e5870f": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Count", + "operationType": "count", + "params": { + "emptyAsNull": false, + "format": { + "id": "number", + "params": { + "decimals": 0 + } + } + }, + "scale": "ratio", + "sourceField": "___records___" + }, + "33cf8fd2-12fc-47ba-8fe0-efc49a7ed68f": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Sender Image Path", + "operationType": "terms", + "params": { + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderBy": { + "columnId": "141b5b8f-7f60-43dd-b5e8-fcf827e5870f", + "type": "column" + }, + "orderDirection": "desc", + "otherBucket": false, + "parentFormat": { + "id": "terms" + }, + "size": 10 + }, + "scale": "ordinal", + "sourceField": "macos.sender.image_path" + } + }, + "incompleteColumns": {}, + "indexPatternId": "logs-*", + "sampling": 1 + } + } + }, + "indexpattern": { + "currentIndexPatternId": "logs-*", + "layers": {} + }, + "textBased": { + "indexPatternRefs": [ + { + "id": "logs-*", + "timeField": "@timestamp", + "title": "logs-*" + } + ], + "layers": {} + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "columns": [ + { + "columnId": "33cf8fd2-12fc-47ba-8fe0-efc49a7ed68f" + }, + { + "columnId": "141b5b8f-7f60-43dd-b5e8-fcf827e5870f" + } + ], + "layerId": "42a78020-0d1f-443e-a82a-6df70600549d", + "layerType": "data" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsDatatable" + }, + "enhancements": { + "dynamicActions": { + "events": [] + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "syncColors": false, + "syncCursor": true, + "syncTooltips": false, + "title": "Top Sender Image Path" + }, + "gridData": { + "h": 17, + "i": "17e05963-7db3-4c42-bb49-e91532020f27", + "w": 17, + "x": 13, + "y": 22 + }, + "panelIndex": "17e05963-7db3-4c42-bb49-e91532020f27", + "type": "lens" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "logs-*", + "name": "indexpattern-datasource-layer-222dbc47-abfa-49a7-b595-cb36fbe5057e", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "currentIndexPatternId": "logs-*", + "layers": { + "222dbc47-abfa-49a7-b595-cb36fbe5057e": { + "columnOrder": [ + "28ff7ab6-2626-40ad-a700-2a72f55a285d", + "e6681111-ffdc-44ca-9d4f-dc3de6251033" + ], + "columns": { + "28ff7ab6-2626-40ad-a700-2a72f55a285d": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Process Image Path", + "operationType": "terms", + "params": { + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderBy": { + "columnId": "e6681111-ffdc-44ca-9d4f-dc3de6251033", + "type": "column" + }, + "orderDirection": "desc", + "otherBucket": false, + "parentFormat": { + "id": "terms" + }, + "size": 10 + }, + "scale": "ordinal", + "sourceField": "macos.process.image_path" + }, + "e6681111-ffdc-44ca-9d4f-dc3de6251033": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Count", + "operationType": "count", + "params": { + "emptyAsNull": false, + "format": { + "id": "number", + "params": { + "decimals": 0 + } + } + }, + "scale": "ratio", + "sourceField": "___records___" + } + }, + "incompleteColumns": {}, + "indexPatternId": "logs-*", + "sampling": 1 + } + } + }, + "indexpattern": { + "currentIndexPatternId": "logs-*", + "layers": {} + }, + "textBased": { + "indexPatternRefs": [ + { + "id": "logs-*", + "timeField": "@timestamp", + "title": "logs-*" + } + ], + "layers": {} + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "columns": [ + { + "columnId": "28ff7ab6-2626-40ad-a700-2a72f55a285d" + }, + { + "columnId": "e6681111-ffdc-44ca-9d4f-dc3de6251033" + } + ], + "layerId": "222dbc47-abfa-49a7-b595-cb36fbe5057e", + "layerType": "data" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsDatatable" + }, + "enhancements": { + "dynamicActions": { + "events": [] + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "syncColors": false, + "syncCursor": true, + "syncTooltips": false, + "title": "Top Process Image Path" + }, + "gridData": { + "h": 17, + "i": "a4f33a70-ad6a-4eb0-9792-ac20951dc1e6", + "w": 18, + "x": 30, + "y": 22 + }, + "panelIndex": "a4f33a70-ad6a-4eb0-9792-ac20951dc1e6", + "type": "lens" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "logs-*", + "name": "indexpattern-datasource-layer-222dbc47-abfa-49a7-b595-cb36fbe5057e", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "currentIndexPatternId": "logs-*", + "layers": { + "222dbc47-abfa-49a7-b595-cb36fbe5057e": { + "columnOrder": [ + "28ff7ab6-2626-40ad-a700-2a72f55a285d", + "e6681111-ffdc-44ca-9d4f-dc3de6251033" + ], + "columns": { + "28ff7ab6-2626-40ad-a700-2a72f55a285d": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Username", + "operationType": "terms", + "params": { + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderBy": { + "columnId": "e6681111-ffdc-44ca-9d4f-dc3de6251033", + "type": "column" + }, + "orderDirection": "desc", + "otherBucket": false, + "parentFormat": { + "id": "terms" + }, + "secondaryFields": [], + "size": 10 + }, + "scale": "ordinal", + "sourceField": "user.name" + }, + "e6681111-ffdc-44ca-9d4f-dc3de6251033": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Count", + "operationType": "count", + "params": { + "emptyAsNull": false, + "format": { + "id": "number", + "params": { + "decimals": 0 + } + } + }, + "scale": "ratio", + "sourceField": "___records___" + } + }, + "incompleteColumns": {}, + "indexPatternId": "logs-*", + "sampling": 1 + } + } + }, + "indexpattern": { + "currentIndexPatternId": "logs-*", + "layers": {} + }, + "textBased": { + "indexPatternRefs": [ + { + "id": "logs-*", + "timeField": "@timestamp", + "title": "logs-*" + } + ], + "layers": {} + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "columns": [ + { + "columnId": "28ff7ab6-2626-40ad-a700-2a72f55a285d" + }, + { + "columnId": "e6681111-ffdc-44ca-9d4f-dc3de6251033" + } + ], + "layerId": "222dbc47-abfa-49a7-b595-cb36fbe5057e", + "layerType": "data" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsDatatable" + }, + "enhancements": { + "dynamicActions": { + "events": [] + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "syncColors": false, + "syncCursor": true, + "syncTooltips": false, + "title": "Top Active Users" + }, + "gridData": { + "h": 17, + "i": "c1f31ab0-242c-4afe-83be-26e428805e5f", + "w": 13, + "x": 0, + "y": 22 + }, + "panelIndex": "c1f31ab0-242c-4afe-83be-26e428805e5f", + "type": "lens" + } + ], + "timeRestore": false, + "title": "[Logs macOS] Overview", + "version": 3 + }, + "coreMigrationVersion": "8.8.0", + "created_at": "2025-11-17T10:13:16.313Z", + "id": "macos-4b49d421-2f03-4dd2-891f-cbd7e2786e35", + "references": [ + { + "id": "macos-4b49d421-2f03-4dd2-891f-cbd7e2786e35", + "name": "77764699-34eb-4325-8a41-63ef03163050:link_4af59c8f-d1a1-49b9-b4e4-927304a4d114_dashboard", + "type": "dashboard" + }, + { + "id": "macos-4fae07f9-fff4-49d0-8ed6-54a63b4c6426", + "name": "77764699-34eb-4325-8a41-63ef03163050:link_c2813a92-3079-46ca-b3e6-79796cc36725_dashboard", + "type": "dashboard" + }, + { + "id": "logs-*", + "name": "27dfea44-7dd8-45ec-8caa-dacbbb35ed39:indexpattern-datasource-layer-6d111cc0-533f-4394-bee1-3befbb34d4ba", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "17e05963-7db3-4c42-bb49-e91532020f27:indexpattern-datasource-layer-42a78020-0d1f-443e-a82a-6df70600549d", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "a4f33a70-ad6a-4eb0-9792-ac20951dc1e6:indexpattern-datasource-layer-222dbc47-abfa-49a7-b595-cb36fbe5057e", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "c1f31ab0-242c-4afe-83be-26e428805e5f:indexpattern-datasource-layer-222dbc47-abfa-49a7-b595-cb36fbe5057e", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "controlGroup_410ef196-6693-4255-930f-065d731cd674:optionsListDataView", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "controlGroup_1b2a550b-350c-4e01-8695-967e23a2ef8a:optionsListDataView", + "type": "index-pattern" + } + ], + "type": "dashboard", + "typeMigrationVersion": "10.2.0", + "updated_by": "u_mGBROF_q5bmFCATbLXAcCwKa0k8JvONAwSruelyKA5E_0" +} \ No newline at end of file diff --git a/packages/macos/kibana/dashboard/macos-4fae07f9-fff4-49d0-8ed6-54a63b4c6426.json b/packages/macos/kibana/dashboard/macos-4fae07f9-fff4-49d0-8ed6-54a63b4c6426.json new file mode 100644 index 00000000000..8a2fdff6882 --- /dev/null +++ b/packages/macos/kibana/dashboard/macos-4fae07f9-fff4-49d0-8ed6-54a63b4c6426.json @@ -0,0 +1,532 @@ +{ + "attributes": { + "controlGroupInput": { + "chainingSystem": "HIERARCHICAL", + "controlStyle": "oneLine", + "ignoreParentSettingsJSON": { + "ignoreFilters": false, + "ignoreQuery": false, + "ignoreTimerange": false, + "ignoreValidations": false + }, + "panelsJSON": {}, + "showApplySelections": false + }, + "description": "This dashboard provides insights into the network logs of macOS.", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [ + { + "$state": { + "store": "appState" + }, + "meta": { + "alias": null, + "disabled": false, + "field": "data_stream.dataset", + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", + "key": "data_stream.dataset", + "negate": false, + "params": { + "query": "macos.network_activity" + }, + "type": "phrase" + }, + "query": { + "match_phrase": { + "data_stream.dataset": "macos.network_activity" + } + } + } + ], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "optionsJSON": { + "hidePanelTitles": false, + "syncColors": false, + "syncCursor": true, + "syncTooltips": false, + "useMargins": true + }, + "panelsJSON": [ + { + "embeddableConfig": { + "attributes": { + "layout": "vertical", + "links": [ + { + "destinationRefName": "link_4af59c8f-d1a1-49b9-b4e4-927304a4d114_dashboard", + "id": "4af59c8f-d1a1-49b9-b4e4-927304a4d114", + "label": "Overview", + "options": { + "openInNewTab": false, + "useCurrentDateRange": false, + "useCurrentFilters": false + }, + "order": 0, + "type": "dashboardLink" + }, + { + "destinationRefName": "link_c2813a92-3079-46ca-b3e6-79796cc36725_dashboard", + "id": "c2813a92-3079-46ca-b3e6-79796cc36725", + "label": "Network", + "options": { + "openInNewTab": false, + "useCurrentDateRange": false, + "useCurrentFilters": false + }, + "order": 1, + "type": "dashboardLink" + } + ] + }, + "enhancements": {}, + "title": "Navigation" + }, + "gridData": { + "h": 5, + "i": "bb52edfd-2411-4192-bb53-3ae15e1ae411", + "w": 10, + "x": 0, + "y": 0 + }, + "panelIndex": "bb52edfd-2411-4192-bb53-3ae15e1ae411", + "type": "links" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "logs-*", + "name": "indexpattern-datasource-layer-402c27e3-cf9b-4421-805a-7d39b77ccecc", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "currentIndexPatternId": "logs-*", + "layers": { + "402c27e3-cf9b-4421-805a-7d39b77ccecc": { + "columnOrder": [ + "c970256c-4bd6-449d-b0a5-ef27d9931cc8", + "0b0042a3-61b9-43da-b715-6a1324f4cc5e", + "29bd5c87-11d0-45c7-8a16-1b59d3b17a07", + "abf06430-e5ff-4b0c-ad75-e4546633a481" + ], + "columns": { + "0b0042a3-61b9-43da-b715-6a1324f4cc5e": { + "customLabel": true, + "dataType": "ip", + "isBucketed": true, + "label": "Client IP", + "operationType": "terms", + "params": { + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderBy": { + "columnId": "abf06430-e5ff-4b0c-ad75-e4546633a481", + "type": "column" + }, + "orderDirection": "desc", + "otherBucket": false, + "parentFormat": { + "id": "terms" + }, + "size": 10 + }, + "sourceField": "source.ip" + }, + "29bd5c87-11d0-45c7-8a16-1b59d3b17a07": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Duration", + "operationType": "terms", + "params": { + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderBy": { + "columnId": "abf06430-e5ff-4b0c-ad75-e4546633a481", + "type": "column" + }, + "orderDirection": "desc", + "otherBucket": false, + "parentFormat": { + "id": "terms" + }, + "size": 10 + }, + "sourceField": "macos.event.message.duration" + }, + "abf06430-e5ff-4b0c-ad75-e4546633a481": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Count ", + "operationType": "count", + "params": { + "emptyAsNull": false + }, + "sourceField": "___records___" + }, + "c970256c-4bd6-449d-b0a5-ef27d9931cc8": { + "customLabel": true, + "dataType": "number", + "isBucketed": true, + "label": "Client Port", + "operationType": "terms", + "params": { + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderBy": { + "columnId": "abf06430-e5ff-4b0c-ad75-e4546633a481", + "type": "column" + }, + "orderDirection": "desc", + "otherBucket": false, + "parentFormat": { + "id": "terms" + }, + "secondaryFields": [], + "size": 10 + }, + "sourceField": "source.port" + } + }, + "ignoreGlobalFilters": false, + "incompleteColumns": {}, + "indexPatternId": "logs-*", + "sampling": 1 + } + } + }, + "indexpattern": { + "currentIndexPatternId": "logs-*", + "layers": {} + }, + "textBased": { + "indexPatternRefs": [ + { + "id": "logs-*", + "timeField": "@timestamp", + "title": "logs-*" + } + ], + "layers": {} + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "columns": [ + { + "alignment": "left", + "columnId": "c970256c-4bd6-449d-b0a5-ef27d9931cc8", + "isMetric": false, + "isTransposed": false + }, + { + "columnId": "abf06430-e5ff-4b0c-ad75-e4546633a481", + "isMetric": true, + "isTransposed": false + }, + { + "columnId": "0b0042a3-61b9-43da-b715-6a1324f4cc5e", + "isMetric": false, + "isTransposed": false + }, + { + "columnId": "29bd5c87-11d0-45c7-8a16-1b59d3b17a07", + "isMetric": false, + "isTransposed": false + } + ], + "layerId": "402c27e3-cf9b-4421-805a-7d39b77ccecc", + "layerType": "data" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsDatatable" + }, + "enhancements": { + "dynamicActions": { + "events": [] + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "syncColors": false, + "syncCursor": true, + "syncTooltips": false, + "title": "Top Active Network Connections" + }, + "gridData": { + "h": 15, + "i": "f9b11023-cea9-4f3c-98eb-98c9458fbe52", + "w": 48, + "x": 0, + "y": 18 + }, + "panelIndex": "f9b11023-cea9-4f3c-98eb-98c9458fbe52", + "type": "lens" + }, + { + "embeddableConfig": { + "enhancements": { + "dynamicActions": { + "events": [] + } + }, + "hidePanelTitles": true, + "savedVis": { + "data": { + "aggs": [], + "searchSource": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "description": "", + "params": { + "fontSize": 12, + "markdown": "### MacOS\n\n#### Description\n\nThis dashboard provides an overview of macOS Unified Logs, helping security and operations teams monitor system and network activities across macOS devices. Tables further enrich the analysis by listing active network connections, supporting efficient investigation and performance monitoring.\n\n**[Integration Page](/app/integrations/detail/macos/overview)**", + "openLinksInNewTab": false + }, + "title": "", + "type": "markdown", + "uiState": {} + }, + "title": "" + }, + "gridData": { + "h": 13, + "i": "f54a8b9e-3782-41ad-aea1-40772c7995ca", + "w": 10, + "x": 0, + "y": 5 + }, + "panelIndex": "f54a8b9e-3782-41ad-aea1-40772c7995ca", + "type": "visualization" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "logs-*", + "name": "indexpattern-datasource-layer-a4a80cef-789a-406f-9303-f862091aeb35", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "currentIndexPatternId": "logs-*", + "layers": { + "a4a80cef-789a-406f-9303-f862091aeb35": { + "columnOrder": [ + "d84f34a8-29e5-4e17-8a77-5497bc8c9c88", + "73c2f30a-da4d-45cf-8af2-468955b549d3" + ], + "columns": { + "73c2f30a-da4d-45cf-8af2-468955b549d3": { + "customLabel": true, + "dataType": "number", + "filter": { + "language": "kuery", + "query": "" + }, + "isBucketed": false, + "label": "Count", + "operationType": "count", + "params": { + "emptyAsNull": false + }, + "sourceField": "___records___" + }, + "d84f34a8-29e5-4e17-8a77-5497bc8c9c88": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "dropPartials": false, + "includeEmptyRows": true, + "interval": "auto" + }, + "sourceField": "@timestamp" + } + }, + "ignoreGlobalFilters": false, + "incompleteColumns": {}, + "indexPatternId": "logs-*", + "sampling": 1 + } + } + }, + "indexpattern": { + "currentIndexPatternId": "logs-*", + "layers": {} + }, + "textBased": { + "indexPatternRefs": [ + { + "id": "logs-*", + "timeField": "@timestamp", + "title": "logs-*" + } + ], + "layers": {} + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "layers": [ + { + "accessors": [ + "73c2f30a-da4d-45cf-8af2-468955b549d3" + ], + "colorMapping": { + "assignments": [], + "colorMode": { + "type": "categorical" + }, + "paletteId": "default", + "specialAssignments": [ + { + "color": { + "type": "loop" + }, + "rules": [ + { + "type": "other" + } + ], + "touched": false + } + ] + }, + "layerId": "a4a80cef-789a-406f-9303-f862091aeb35", + "layerType": "data", + "position": "top", + "seriesType": "line", + "showGridlines": false, + "xAccessor": "d84f34a8-29e5-4e17-8a77-5497bc8c9c88" + } + ], + "legend": { + "isVisible": true, + "position": "right", + "shouldTruncate": false + }, + "preferredSeriesType": "line", + "title": "Empty XY chart", + "valueLabels": "hide" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsXY" + }, + "enhancements": { + "dynamicActions": { + "events": [] + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "syncColors": false, + "syncCursor": true, + "syncTooltips": false, + "title": "Network Events over Time" + }, + "gridData": { + "h": 18, + "i": "00360852-34be-4825-a951-7ff808c8ce7a", + "w": 37, + "x": 10, + "y": 0 + }, + "panelIndex": "00360852-34be-4825-a951-7ff808c8ce7a", + "type": "lens" + } + ], + "timeRestore": false, + "title": "[Logs macOS] Network", + "version": 3 + }, + "coreMigrationVersion": "8.8.0", + "created_at": "2025-11-17T10:13:16.861Z", + "id": "macos-4fae07f9-fff4-49d0-8ed6-54a63b4c6426", + "references": [ + { + "id": "logs-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", + "type": "index-pattern" + }, + { + "id": "macos-4b49d421-2f03-4dd2-891f-cbd7e2786e35", + "name": "bb52edfd-2411-4192-bb53-3ae15e1ae411:link_4af59c8f-d1a1-49b9-b4e4-927304a4d114_dashboard", + "type": "dashboard" + }, + { + "id": "macos-4fae07f9-fff4-49d0-8ed6-54a63b4c6426", + "name": "bb52edfd-2411-4192-bb53-3ae15e1ae411:link_c2813a92-3079-46ca-b3e6-79796cc36725_dashboard", + "type": "dashboard" + }, + { + "id": "logs-*", + "name": "f9b11023-cea9-4f3c-98eb-98c9458fbe52:indexpattern-datasource-layer-402c27e3-cf9b-4421-805a-7d39b77ccecc", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "00360852-34be-4825-a951-7ff808c8ce7a:indexpattern-datasource-layer-a4a80cef-789a-406f-9303-f862091aeb35", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", + "type": "index-pattern" + } + ], + "type": "dashboard", + "typeMigrationVersion": "10.2.0", + "updated_by": "u_mGBROF_q5bmFCATbLXAcCwKa0k8JvONAwSruelyKA5E_0" +} \ No newline at end of file diff --git a/packages/macos/manifest.yml b/packages/macos/manifest.yml new file mode 100644 index 00000000000..955332ab762 --- /dev/null +++ b/packages/macos/manifest.yml @@ -0,0 +1,38 @@ +format_version: 3.3.2 +name: macos +title: macOS +version: 0.1.0 +description: Collect logs from macOS with Elastic Agent. +type: integration +categories: + - security +conditions: + kibana: + version: ^8.18.0 || ^9.0.0 + elastic: + subscription: basic +icons: + - src: /img/macos-logo.svg + title: macOS logo + size: 32x32 + type: image/svg+xml +screenshots: + - src: /img/macos-unified-logs-overview-dashboard.png + title: MacOS Unified Logs Overview Dashboard + size: 600x600 + type: image/png + - src: /img/macos-unified-logs-network-dashboard.png + title: MacOS Unified Logs Network Dashboard + size: 600x600 + type: image/png +policy_templates: + - name: macos + title: macOS + description: Collect logs from macOS. + inputs: + - type: unifiedlogs + title: Collect unified logs from macOS + description: Collecting unified logs from macOS. +owner: + github: elastic/sec-windows-platform + type: elastic diff --git a/packages/macos/validation.yml b/packages/macos/validation.yml new file mode 100644 index 00000000000..a81cfab704f --- /dev/null +++ b/packages/macos/validation.yml @@ -0,0 +1,3 @@ +errors: + exclude_checks: + - SVR00002 # References in dashboards.