From f84c069453b1b1aabb8d9d623a1c8fdb412d656e Mon Sep 17 00:00:00 2001 From: Patrick Bareiss Date: Wed, 5 Mar 2025 11:07:26 +0100 Subject: [PATCH 1/5] Data source output fields validation --- .../detection_abstract.py | 27 +++++++++++++++++++ contentctl/objects/data_source.py | 2 +- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/contentctl/objects/abstract_security_content_objects/detection_abstract.py b/contentctl/objects/abstract_security_content_objects/detection_abstract.py index 3c5def00..66d0587e 100644 --- a/contentctl/objects/abstract_security_content_objects/detection_abstract.py +++ b/contentctl/objects/abstract_security_content_objects/detection_abstract.py @@ -1055,3 +1055,30 @@ def get_summary( # Return the summary return summary_dict + + @model_validator(mode="after") + def validate_data_source_output_fields(self): + # Skip validation for Hunting and Correlation types, or non-production detections + if (self.status != DetectionStatus.production or + self.type in {AnalyticsType.Hunting, AnalyticsType.Correlation} or + len(self.data_source) <= 1): + return self + + # Validate that all required output fields are present in the search + for data_source in self.data_source_objects: + if not data_source.output_fields: + continue + + missing_fields = [ + field for field in data_source.output_fields + if field not in self.search + ] + + if missing_fields: + raise ValueError( + f"Data source '{data_source.name}' has output fields " + f"{missing_fields} that are not present in the search " + f"for detection '{self.name}'" + ) + + return self diff --git a/contentctl/objects/data_source.py b/contentctl/objects/data_source.py index f3b3773c..f0cd508a 100644 --- a/contentctl/objects/data_source.py +++ b/contentctl/objects/data_source.py @@ -23,7 +23,7 @@ class DataSource(SecurityContentObject): field_mappings: None | list = None convert_to_log_source: None | list = None example_log: None | str = None - output_fields: list[str] = [] + output_fields: None | list = None @model_serializer def serialize_model(self): From bc3d6cc10796f609d1024d277f8a05e18b8591ab Mon Sep 17 00:00:00 2001 From: Patrick Bareiss Date: Wed, 5 Mar 2025 12:14:13 +0100 Subject: [PATCH 2/5] Needed for longer running tests in case of two tstats searches --- .../infrastructures/DetectionTestingInfrastructure.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py b/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py index ea755d8c..b981589e 100644 --- a/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py +++ b/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py @@ -89,7 +89,7 @@ class DetectionTestingManagerOutputDto: start_time: Union[datetime.datetime, None] = None replay_index: str = "contentctl_testing_index" replay_host: str = "CONTENTCTL_HOST" - timeout_seconds: int = 60 + timeout_seconds: int = 120 terminate: bool = False From b0cfb4d105fe68d1a7f658bcbbff5b52a84f0d06 Mon Sep 17 00:00:00 2001 From: Patrick Bareiss Date: Wed, 5 Mar 2025 16:32:11 +0100 Subject: [PATCH 3/5] bug fix --- .../abstract_security_content_objects/detection_abstract.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/contentctl/objects/abstract_security_content_objects/detection_abstract.py b/contentctl/objects/abstract_security_content_objects/detection_abstract.py index 66d0587e..8924aa7e 100644 --- a/contentctl/objects/abstract_security_content_objects/detection_abstract.py +++ b/contentctl/objects/abstract_security_content_objects/detection_abstract.py @@ -1060,8 +1060,7 @@ def get_summary( def validate_data_source_output_fields(self): # Skip validation for Hunting and Correlation types, or non-production detections if (self.status != DetectionStatus.production or - self.type in {AnalyticsType.Hunting, AnalyticsType.Correlation} or - len(self.data_source) <= 1): + self.type in {AnalyticsType.Hunting, AnalyticsType.Correlation}): return self # Validate that all required output fields are present in the search From 6634a3ad98aa3118804a80b55ce8d9e4981fda63 Mon Sep 17 00:00:00 2001 From: Patrick Bareiss Date: Thu, 6 Mar 2025 09:30:26 +0100 Subject: [PATCH 4/5] bug fix --- contentctl/objects/data_source.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contentctl/objects/data_source.py b/contentctl/objects/data_source.py index f0cd508a..f3b3773c 100644 --- a/contentctl/objects/data_source.py +++ b/contentctl/objects/data_source.py @@ -23,7 +23,7 @@ class DataSource(SecurityContentObject): field_mappings: None | list = None convert_to_log_source: None | list = None example_log: None | str = None - output_fields: None | list = None + output_fields: list[str] = [] @model_serializer def serialize_model(self): From ea8ec900aa9402474886c043bedc1ac1d0d5517a Mon Sep 17 00:00:00 2001 From: pyth0n1c Date: Sun, 9 Mar 2025 16:32:12 -0700 Subject: [PATCH 5/5] open and save so that ruff check passes --- .../detection_abstract.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/contentctl/objects/abstract_security_content_objects/detection_abstract.py b/contentctl/objects/abstract_security_content_objects/detection_abstract.py index 8924aa7e..e4ac4a68 100644 --- a/contentctl/objects/abstract_security_content_objects/detection_abstract.py +++ b/contentctl/objects/abstract_security_content_objects/detection_abstract.py @@ -1059,25 +1059,26 @@ def get_summary( @model_validator(mode="after") def validate_data_source_output_fields(self): # Skip validation for Hunting and Correlation types, or non-production detections - if (self.status != DetectionStatus.production or - self.type in {AnalyticsType.Hunting, AnalyticsType.Correlation}): + if self.status != DetectionStatus.production or self.type in { + AnalyticsType.Hunting, + AnalyticsType.Correlation, + }: return self # Validate that all required output fields are present in the search for data_source in self.data_source_objects: if not data_source.output_fields: continue - + missing_fields = [ - field for field in data_source.output_fields - if field not in self.search + field for field in data_source.output_fields if field not in self.search ] - + if missing_fields: raise ValueError( f"Data source '{data_source.name}' has output fields " f"{missing_fields} that are not present in the search " f"for detection '{self.name}'" ) - + return self