From a796dbf4973da19faf87dd64aed89d1b4f5bb128 Mon Sep 17 00:00:00 2001 From: pyth0n1c Date: Thu, 13 Feb 2025 14:31:15 -0800 Subject: [PATCH 1/4] Improve typing and output for lookup default_match field. --- .../detection_abstract.py | 2 +- contentctl/objects/lookup.py | 7 +++++-- contentctl/output/templates/transforms.j2 | 4 ++-- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/contentctl/objects/abstract_security_content_objects/detection_abstract.py b/contentctl/objects/abstract_security_content_objects/detection_abstract.py index fc0505ce..7dc6732b 100644 --- a/contentctl/objects/abstract_security_content_objects/detection_abstract.py +++ b/contentctl/objects/abstract_security_content_objects/detection_abstract.py @@ -476,7 +476,7 @@ def serialize_model(self): "name": lookup.name, "description": lookup.description, "filename": lookup.filename.name, - "default_match": "true" if lookup.default_match else "false", + "default_match": lookup.default_match, "case_sensitive_match": "true" if lookup.case_sensitive_match else "false", diff --git a/contentctl/objects/lookup.py b/contentctl/objects/lookup.py index 029b2c9b..86e11ac4 100644 --- a/contentctl/objects/lookup.py +++ b/contentctl/objects/lookup.py @@ -69,7 +69,10 @@ class Lookup_Type(StrEnum): # TODO (#220): Split Lookup into 2 classes class Lookup(SecurityContentObject, abc.ABC): - default_match: Optional[bool] = None + default_match: str = Field(default='', description="This field is given a default value of ''" + "because it is the default value specified in the transforms.conf " + "docs. Giving it a type of str rather than str | None simplifies " + "the typing for the field.") # Per the documentation for transforms.conf, EXACT should not be specified in this list, # so we include only WILDCARD and CIDR match_type: list[Annotated[str, Field(pattern=r"(^WILDCARD|CIDR)\(.+\)$")]] = Field( @@ -88,7 +91,7 @@ def serialize_model(self): # All fields custom to this model model = { - "default_match": "true" if self.default_match is True else "false", + "default_match": self.default_match, "match_type": self.match_type_to_conf_format, "min_matches": self.min_matches, "max_matches": self.max_matches, diff --git a/contentctl/output/templates/transforms.j2 b/contentctl/output/templates/transforms.j2 index f80f2569..8a485c90 100644 --- a/contentctl/output/templates/transforms.j2 +++ b/contentctl/output/templates/transforms.j2 @@ -7,8 +7,8 @@ filename = {{ lookup.app_filename.name }} collection = {{ lookup.collection }} external_type = kvstore {% endif %} -{% if lookup.default_match is defined and lookup.default_match != None %} -default_match = {{ lookup.default_match | lower }} +{% if lookup.default_match != '' %} +default_match = {{ lookup.default_match }} {% endif %} {% if lookup.case_sensitive_match is defined and lookup.case_sensitive_match != None %} case_sensitive_match = {{ lookup.case_sensitive_match | lower }} From 39ce6bd6e8118e36d8a98f3dbf0bc0ea7d5ae37c Mon Sep 17 00:00:00 2001 From: pyth0n1c Date: Thu, 13 Feb 2025 14:54:18 -0800 Subject: [PATCH 2/4] remove unused import --- contentctl/objects/lookup.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/contentctl/objects/lookup.py b/contentctl/objects/lookup.py index 86e11ac4..15c613ea 100644 --- a/contentctl/objects/lookup.py +++ b/contentctl/objects/lookup.py @@ -6,7 +6,7 @@ import re from enum import StrEnum, auto from functools import cached_property -from typing import TYPE_CHECKING, Annotated, Any, Literal, Optional, Self +from typing import TYPE_CHECKING, Annotated, Any, Literal, Self from pydantic import ( Field, @@ -69,10 +69,13 @@ class Lookup_Type(StrEnum): # TODO (#220): Split Lookup into 2 classes class Lookup(SecurityContentObject, abc.ABC): - default_match: str = Field(default='', description="This field is given a default value of ''" - "because it is the default value specified in the transforms.conf " - "docs. Giving it a type of str rather than str | None simplifies " - "the typing for the field.") + default_match: str = Field( + default="", + description="This field is given a default value of ''" + "because it is the default value specified in the transforms.conf " + "docs. Giving it a type of str rather than str | None simplifies " + "the typing for the field.", + ) # Per the documentation for transforms.conf, EXACT should not be specified in this list, # so we include only WILDCARD and CIDR match_type: list[Annotated[str, Field(pattern=r"(^WILDCARD|CIDR)\(.+\)$")]] = Field( From acf843d89f6045207332854470e4eca659900089 Mon Sep 17 00:00:00 2001 From: pyth0n1c Date: Thu, 13 Feb 2025 16:22:10 -0800 Subject: [PATCH 3/4] Fix typing issue with default_match field when detection is read from YML file with PyYAML --- contentctl/input/director.py | 38 +++++++++++++++++------------------- contentctl/objects/lookup.py | 9 ++++++++- 2 files changed, 26 insertions(+), 21 deletions(-) diff --git a/contentctl/input/director.py b/contentctl/input/director.py index 1b637101..c7bca263 100644 --- a/contentctl/input/director.py +++ b/contentctl/input/director.py @@ -1,30 +1,29 @@ import os import sys -from pathlib import Path from dataclasses import dataclass, field -from pydantic import ValidationError +from pathlib import Path from uuid import UUID -from contentctl.input.yml_reader import YmlReader -from contentctl.objects.detection import Detection -from contentctl.objects.story import Story +from pydantic import ValidationError -from contentctl.objects.baseline import Baseline -from contentctl.objects.investigation import Investigation -from contentctl.objects.playbook import Playbook -from contentctl.objects.deployment import Deployment -from contentctl.objects.macro import Macro -from contentctl.objects.lookup import LookupAdapter, Lookup -from contentctl.objects.atomic import AtomicEnrichment -from contentctl.objects.security_content_object import SecurityContentObject -from contentctl.objects.data_source import DataSource -from contentctl.objects.dashboard import Dashboard from contentctl.enrichments.attack_enrichment import AttackEnrichment from contentctl.enrichments.cve_enrichment import CveEnrichment - +from contentctl.helper.utils import Utils +from contentctl.input.yml_reader import YmlReader +from contentctl.objects.atomic import AtomicEnrichment +from contentctl.objects.baseline import Baseline from contentctl.objects.config import validate +from contentctl.objects.dashboard import Dashboard +from contentctl.objects.data_source import DataSource +from contentctl.objects.deployment import Deployment +from contentctl.objects.detection import Detection from contentctl.objects.enums import SecurityContentType -from contentctl.helper.utils import Utils +from contentctl.objects.investigation import Investigation +from contentctl.objects.lookup import Lookup, LookupAdapter +from contentctl.objects.macro import Macro +from contentctl.objects.playbook import Playbook +from contentctl.objects.security_content_object import SecurityContentObject +from contentctl.objects.story import Story @dataclass @@ -113,9 +112,8 @@ def execute(self, input_dto: validate) -> None: self.createSecurityContent(SecurityContentType.detections) self.createSecurityContent(SecurityContentType.dashboards) - from contentctl.objects.abstract_security_content_objects.detection_abstract import ( - MISSING_SOURCES, - ) + from contentctl.objects.abstract_security_content_objects.detection_abstract import \ + MISSING_SOURCES if len(MISSING_SOURCES) > 0: missing_sources_string = "\n 🟡 ".join(sorted(list(MISSING_SOURCES))) diff --git a/contentctl/objects/lookup.py b/contentctl/objects/lookup.py index 15c613ea..0cc8e845 100644 --- a/contentctl/objects/lookup.py +++ b/contentctl/objects/lookup.py @@ -9,6 +9,7 @@ from typing import TYPE_CHECKING, Annotated, Any, Literal, Self from pydantic import ( + BeforeValidator, Field, FilePath, NonNegativeInt, @@ -69,7 +70,13 @@ class Lookup_Type(StrEnum): # TODO (#220): Split Lookup into 2 classes class Lookup(SecurityContentObject, abc.ABC): - default_match: str = Field( + # We need to make sure that this is converted to a string because we widely + # use the string "False" in our lookup content. However, PyYAML reads this + # as a BOOL and this causes parsing to fail. As such, we will always + # convert this to a string if it is passed as a bool + default_match: Annotated[ + str, BeforeValidator(lambda dm: str(dm) if isinstance(dm, bool) else dm) + ] = Field( default="", description="This field is given a default value of ''" "because it is the default value specified in the transforms.conf " From c18dc0a07db696a8ae64f63927f109b10512567c Mon Sep 17 00:00:00 2001 From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com> Date: Sun, 9 Mar 2025 17:21:34 -0700 Subject: [PATCH 4/4] Make bool lower to comply with legacy behavior Make --- contentctl/objects/lookup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contentctl/objects/lookup.py b/contentctl/objects/lookup.py index 0cc8e845..aa01123d 100644 --- a/contentctl/objects/lookup.py +++ b/contentctl/objects/lookup.py @@ -75,7 +75,7 @@ class Lookup(SecurityContentObject, abc.ABC): # as a BOOL and this causes parsing to fail. As such, we will always # convert this to a string if it is passed as a bool default_match: Annotated[ - str, BeforeValidator(lambda dm: str(dm) if isinstance(dm, bool) else dm) + str, BeforeValidator(lambda dm: str(dm).lower() if isinstance(dm, bool) else dm) ] = Field( default="", description="This field is given a default value of ''"