From 5ebdfc43f1e109a58cdced2ba10496e336899a21 Mon Sep 17 00:00:00 2001 From: Casey McGinley Date: Fri, 13 Sep 2024 12:03:14 -0700 Subject: [PATCH 01/24] typing and linting fixes --- .../DetectionTestingManager.py | 2 +- contentctl/actions/test.py | 62 +++++++++---------- 2 files changed, 31 insertions(+), 33 deletions(-) diff --git a/contentctl/actions/detection_testing/DetectionTestingManager.py b/contentctl/actions/detection_testing/DetectionTestingManager.py index 5ad5e117..bb02d74c 100644 --- a/contentctl/actions/detection_testing/DetectionTestingManager.py +++ b/contentctl/actions/detection_testing/DetectionTestingManager.py @@ -28,7 +28,7 @@ @dataclass(frozen=False) class DetectionTestingManagerInputDto: - config: Union[test,test_servers] + config: Union[test, test_servers] detections: List[Detection] views: list[DetectionTestingView] diff --git a/contentctl/actions/test.py b/contentctl/actions/test.py index 716ecd71..9a5ec510 100644 --- a/contentctl/actions/test.py +++ b/contentctl/actions/test.py @@ -1,38 +1,29 @@ from dataclasses import dataclass from typing import List +import pathlib -from contentctl.objects.config import test_common -from contentctl.objects.enums import DetectionTestingMode, DetectionStatus, AnalyticsType +from contentctl.objects.config import test_servers +from contentctl.objects.config import test as test_ +from contentctl.objects.enums import DetectionTestingMode from contentctl.objects.detection import Detection - -from contentctl.input.director import DirectorOutputDto - from contentctl.actions.detection_testing.DetectionTestingManager import ( DetectionTestingManager, DetectionTestingManagerInputDto, ) - - from contentctl.actions.detection_testing.infrastructures.DetectionTestingInfrastructure import ( DetectionTestingManagerOutputDto, ) - - from contentctl.actions.detection_testing.views.DetectionTestingViewWeb import ( DetectionTestingViewWeb, ) - from contentctl.actions.detection_testing.views.DetectionTestingViewCLI import ( DetectionTestingViewCLI, ) - from contentctl.actions.detection_testing.views.DetectionTestingViewFile import ( DetectionTestingViewFile, ) - from contentctl.objects.integration_test import IntegrationTest -import pathlib MAXIMUM_CONFIGURATION_TIME_SECONDS = 600 @@ -40,8 +31,8 @@ @dataclass(frozen=True) class TestInputDto: detections: List[Detection] - config: test_common - + config: test_ | test_servers + class Test: def filter_tests(self, input_dto: TestInputDto) -> None: @@ -52,7 +43,7 @@ def filter_tests(self, input_dto: TestInputDto) -> None: Args: input_dto (TestInputDto): A configuration of the test and all of the tests to be run. - """ + """ if not input_dto.config.enable_integration_testing: # Skip all integraiton tests if integration testing is not enabled: @@ -61,7 +52,6 @@ def filter_tests(self, input_dto: TestInputDto) -> None: if isinstance(test, IntegrationTest): test.skip("TEST SKIPPED: Skipping all integration tests") - def execute(self, input_dto: TestInputDto) -> bool: output_dto = DetectionTestingManagerOutputDto() @@ -92,7 +82,11 @@ def execute(self, input_dto: TestInputDto) -> bool: print(f"MODE: [{mode}] - Test [{len(input_dto.detections)}] detections") if mode in [DetectionTestingMode.changes.value, DetectionTestingMode.selected.value]: files_string = '\n- '.join( - [str(pathlib.Path(detection.file_path).relative_to(input_dto.config.path)) for detection in input_dto.detections] + [ + str( + pathlib.Path(detection.file_path).relative_to(input_dto.config.path) + ) for detection in input_dto.detections + ] ) print(f"Detections:\n- {files_string}") @@ -102,44 +96,48 @@ def execute(self, input_dto: TestInputDto) -> bool: try: summary_results = file.getSummaryObject() summary = summary_results.get("summary", {}) + if not isinstance(summary, dict): + raise ValueError( + f"Summary in results was an unexpected type ({type(summary)}): {summary}" + ) - print(f"Test Summary (mode: {summary.get('mode','Error')})") - print(f"\tSuccess : {summary.get('success',False)}") + print(f"Test Summary (mode: {summary.get('mode', 'Error')})") + print(f"\tSuccess : {summary.get('success', False)}") print( - f"\tSuccess Rate : {summary.get('success_rate','ERROR')}" + f"\tSuccess Rate : {summary.get('success_rate', 'ERROR')}" ) print( - f"\tTotal Detections : {summary.get('total_detections','ERROR')}" + f"\tTotal Detections : {summary.get('total_detections', 'ERROR')}" ) print( - f"\tTotal Tested Detections : {summary.get('total_tested_detections','ERROR')}" + f"\tTotal Tested Detections : {summary.get('total_tested_detections', 'ERROR')}" ) print( - f"\t Passed Detections : {summary.get('total_pass','ERROR')}" + f"\t Passed Detections : {summary.get('total_pass', 'ERROR')}" ) print( - f"\t Failed Detections : {summary.get('total_fail','ERROR')}" + f"\t Failed Detections : {summary.get('total_fail', 'ERROR')}" ) print( - f"\tSkipped Detections : {summary.get('total_skipped','ERROR')}" + f"\tSkipped Detections : {summary.get('total_skipped', 'ERROR')}" ) print( "\tProduction Status :" ) print( - f"\t Production Detections : {summary.get('total_production','ERROR')}" + f"\t Production Detections : {summary.get('total_production', 'ERROR')}" ) print( - f"\t Experimental Detections : {summary.get('total_experimental','ERROR')}" + f"\t Experimental Detections : {summary.get('total_experimental', 'ERROR')}" ) print( - f"\t Deprecated Detections : {summary.get('total_deprecated','ERROR')}" + f"\t Deprecated Detections : {summary.get('total_deprecated', 'ERROR')}" ) print( - f"\tManually Tested Detections : {summary.get('total_manual','ERROR')}" + f"\tManually Tested Detections : {summary.get('total_manual', 'ERROR')}" ) print( - f"\tUntested Detections : {summary.get('total_untested','ERROR')}" + f"\tUntested Detections : {summary.get('total_untested', 'ERROR')}" ) print(f"\tTest Results File : {file.getOutputFilePath()}") print( @@ -147,7 +145,7 @@ def execute(self, input_dto: TestInputDto) -> bool: "detection types (e.g. Correlation), but there may be overlap between these\n" "categories." ) - return summary_results.get("summary", {}).get("success", False) + return summary.get("success", False) except Exception as e: print(f"Error determining if whole test was successful: {str(e)}") From 336cd7b5fb96766bd3dcd52ba8e791210fac176e Mon Sep 17 00:00:00 2001 From: Casey McGinley Date: Mon, 7 Oct 2024 20:43:57 -0700 Subject: [PATCH 02/24] initial commit; adding validation against CMS index --- .../DetectionTestingManager.py | 2 +- .../DetectionTestingInfrastructure.py | 46 +- .../views/DetectionTestingViewCLI.py | 2 + .../objects/content_versioning_service.py | 473 ++++++++++++++++++ contentctl/objects/correlation_search.py | 10 +- contentctl/output/conf_output.py | 4 +- 6 files changed, 515 insertions(+), 22 deletions(-) create mode 100644 contentctl/objects/content_versioning_service.py diff --git a/contentctl/actions/detection_testing/DetectionTestingManager.py b/contentctl/actions/detection_testing/DetectionTestingManager.py index bb02d74c..4f04b202 100644 --- a/contentctl/actions/detection_testing/DetectionTestingManager.py +++ b/contentctl/actions/detection_testing/DetectionTestingManager.py @@ -90,7 +90,7 @@ def sigint_handler(signum, frame): result = future.result() except Exception as e: self.output_dto.terminate = True - print(f"Error setting up container: {str(e)}") + print(f"Error setting up instance: {str(e)}") # Start and wait for all tests to run if not self.output_dto.terminate: diff --git a/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py b/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py index ca28dce8..2841b3a5 100644 --- a/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py +++ b/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py @@ -11,14 +11,13 @@ from ssl import SSLEOFError, SSLZeroReturnError from sys import stdout from shutil import copyfile -from typing import Union, Optional +from typing import Union, Optional, Callable -from pydantic import BaseModel, PrivateAttr, Field, dataclasses +from pydantic import BaseModel, PrivateAttr, Field, dataclasses, computed_field import requests # type: ignore import splunklib.client as client # type: ignore from splunklib.binding import HTTPError # type: ignore from splunklib.results import JSONResultsReader, Message # type: ignore -import splunklib.results from urllib3 import disable_warnings import urllib.parse @@ -34,6 +33,7 @@ from contentctl.objects.test_group import TestGroup from contentctl.objects.base_test_result import TestResultStatus from contentctl.objects.correlation_search import CorrelationSearch, PbarData +from contentctl.objects.content_versioning_service import ContentVersioningService from contentctl.helper.utils import Utils from contentctl.actions.detection_testing.progress_bar import ( format_pbar_string, @@ -60,6 +60,8 @@ class CleanupTestGroupResults(BaseModel): class ContainerStoppedException(Exception): pass + + class CannotRunBaselineException(Exception): # Support for testing detections with baselines # does not currently exist in contentctl. @@ -125,18 +127,19 @@ def setup(self): ) self.start_time = time.time() + setup_functions: list[tuple[Callable[[], None | client.Service], str]] = [ + (self.start, "Starting"), + (self.get_conn, "Waiting for App Installation"), + (self.configure_conf_file_datamodels, "Configuring Datamodels"), + (self.create_replay_index, f"Create index '{self.sync_obj.replay_index}'"), + (self.configure_imported_roles, "Configuring Roles"), + (self.configure_delete_indexes, "Configuring Indexes"), + (self.configure_hec, "Configuring HEC"), + ] + setup_functions = setup_functions + self.content_versioning_service.setup_functions + setup_functions.append((self.wait_for_ui_ready, "Finishing Setup")) try: - for func, msg in [ - (self.start, "Starting"), - (self.get_conn, "Waiting for App Installation"), - (self.configure_conf_file_datamodels, "Configuring Datamodels"), - (self.create_replay_index, f"Create index '{self.sync_obj.replay_index}'"), - (self.configure_imported_roles, "Configuring Roles"), - (self.configure_delete_indexes, "Configuring Indexes"), - (self.configure_hec, "Configuring HEC"), - (self.wait_for_ui_ready, "Finishing Setup") - ]: - + for func, msg in setup_functions: self.format_pbar_string( TestReportingType.SETUP, self.get_name(), @@ -149,13 +152,23 @@ def setup(self): except Exception as e: self.pbar.write(str(e)) self.finish() - return + raise self.format_pbar_string(TestReportingType.SETUP, self.get_name(), "Finished Setup!") def wait_for_ui_ready(self): self.get_conn() + @computed_field + @property + def content_versioning_service(self) -> ContentVersioningService: + return ContentVersioningService( + global_config=self.global_config, + infrastructure=self.infrastructure, + service=self.get_conn(), + detections=self.sync_obj.inputQueue + ) + def configure_hec(self): self.hec_channel = str(uuid.uuid4()) try: @@ -1198,7 +1211,7 @@ def delete_attack_data(self, attack_data_files: list[TestAttackData]): job = self.get_conn().jobs.create(splunk_search, **kwargs) results_stream = job.results(output_mode="json") # TODO: should we be doing something w/ this reader? - _ = splunklib.results.JSONResultsReader(results_stream) + _ = JSONResultsReader(results_stream) except Exception as e: raise ( @@ -1393,6 +1406,7 @@ def hec_raw_replay( def status(self): pass + # TODO (cmcginley): the finish function doesn't actually stop execution def finish(self): self.pbar.bar_format = f"Finished running tests on instance: [{self.get_name()}]" self.pbar.update() diff --git a/contentctl/actions/detection_testing/views/DetectionTestingViewCLI.py b/contentctl/actions/detection_testing/views/DetectionTestingViewCLI.py index 1675fce4..a5ec8a24 100644 --- a/contentctl/actions/detection_testing/views/DetectionTestingViewCLI.py +++ b/contentctl/actions/detection_testing/views/DetectionTestingViewCLI.py @@ -51,6 +51,8 @@ def showStatus(self, interval: int = 1): while True: summary = self.getSummaryObject() + # TODO (cmcginley): there's a 1-off error here I think (we show one more than we + # actually have during testing) total = len( summary.get("tested_detections", []) + summary.get("untested_detections", []) diff --git a/contentctl/objects/content_versioning_service.py b/contentctl/objects/content_versioning_service.py new file mode 100644 index 00000000..e752d081 --- /dev/null +++ b/contentctl/objects/content_versioning_service.py @@ -0,0 +1,473 @@ +import time +import uuid +import json +import re +from typing import Any, Callable +from functools import cached_property + +from pydantic import BaseModel, PrivateAttr, computed_field +import splunklib.client as splunklib # type: ignore +from splunklib.binding import HTTPError, ResponseReader # type: ignore +from splunklib.data import Record # type: ignore + +from contentctl.objects.config import test_common, Infrastructure +from contentctl.objects.detection import Detection +from contentctl.objects.correlation_search import ResultIterator, get_logger + +# TODO (cmcginley): remove this logger +logger = get_logger() + + +class ContentVersioningService(BaseModel): + global_config: test_common + infrastructure: Infrastructure + service: splunklib.Service + detections: list[Detection] + + _cms_main_job: splunklib.Job | None = PrivateAttr(default=None) + + class Config: + arbitrary_types_allowed = True + + @computed_field + @property + def setup_functions(self) -> list[tuple[Callable[[], None], str]]: + return [ + (self.activate_versioning, "Activating Content Versioning"), + (self.wait_for_cms_main, "Waiting for CMS Parser"), + (self.validate_content_against_cms, "Validating Against CMS"), + ] + + def _query_content_versioning_service(self, method: str, body: dict[str, Any] = {}) -> Record: + """ + Queries the SA-ContentVersioning service. Output mode defaults to JSON. + + :param method: HTTP request method (e.g. GET) + :type method: str + :param body: the payload/data/body of the request + :type body: dict[str, Any] + + :returns: a splunklib Record object (wrapper around dict) indicating the response + :rtype: :class:`splunklib.data.Record` + """ + # Add output mode to body + if "output_mode" not in body: + body["output_mode"] = "json" + + # Query the content versioning service + try: + response = self.service.request( # type: ignore + method=method, + path_segment="configs/conf-feature_flags/general", + body=body, + app="SA-ContentVersioning" + ) + except HTTPError as e: + # Raise on any HTTP errors + raise HTTPError( + f"Error querying content versioning service: {e}" + ) from e + + return response + + @property + def is_versioning_activated(self) -> bool: + """ + Indicates whether the versioning service is activated or not + + :returns: a bool indicating if content versioning is activated or not + :rtype: bool + """ + # Query the SA-ContentVersioning service for versioning status + response = self._query_content_versioning_service(method="GET") + + # Grab the response body and check for errors + if "body" not in response: + raise KeyError( + f"Cannot retrieve versioning status, 'body' was not found in JSON response: {response}" + ) + body: Any = response["body"] # type: ignore + if not isinstance(body, ResponseReader): + raise ValueError( + "Cannot retrieve versioning status, value at 'body' in JSON response had an unexpected" + f" type: expected '{ResponseReader}', received '{type(body)}'" + ) + + # Read the JSON and parse it into a dictionary + json_ = body.readall() + try: + data = json.loads(json_) + except json.JSONDecodeError as e: + raise ValueError(f"Unable to parse response body as JSON: {e}") from e + + # Find the versioning_activated field and report any errors + try: + for entry in data["entry"]: + if entry["name"] == "general": + return bool(int(entry["content"]["versioning_activated"])) + except KeyError as e: + raise KeyError( + "Cannot retrieve versioning status, unable to versioning status using the expected " + f"keys: {e}" + ) from e + raise ValueError( + "Cannot retrieve versioning status, unable to find an entry matching 'general' in the " + "response." + ) + + def activate_versioning(self) -> None: + """ + Activate the content versioning service + """ + # TODO (cmcginley): add conditional logging s.t. this check only happens when integration + # testing is enabled AND ES is at least version 8.0, AND mode is `all` + # Post to the SA-ContentVersioning service to set versioning status + self._query_content_versioning_service( + method="POST", + body={ + "versioning_activated": True + } + ) + + # Confirm versioning has been enabled + if not self.is_versioning_activated: + raise Exception("Something went wrong, content versioning is still disabled.") + + @computed_field + @cached_property + def cms_fields(self) -> list[str]: + """ + Property listing the fields we want to pull from the cms_main index + + :returns: a list of strings, the fields we want + :rtype: list[str] + """ + return [ + "app_name", + f"action.{self.global_config.app.label.lower()}.full_search_name", + "detection_id", + "version", + "action.correlationsearch.label", + "sourcetype" + ] + + @property + def is_cms_parser_enabled(self) -> bool: + """ + Indicates whether the cms_parser mod input is enabled or not. + + :returns: a bool indicating if cms_parser mod input is activated or not + :rtype: bool + """ + # Get the data input entity + cms_parser = self.service.input("data/inputs/cms_parser/main") # type: ignore + + # Convert the 'disabled' field to an int, then a bool, and then invert to be 'enabled' + return not bool(int(cms_parser.content["disabled"])) # type: ignore + + def force_cms_parser(self) -> None: + """ + Force the cms_parser to being it's run being disabling and re-enabling it. + """ + # Get the data input entity + cms_parser = self.service.input("data/inputs/cms_parser/main") # type: ignore + + # Disable and re-enable + cms_parser.disable() + cms_parser.enable() + + # Confirm the cms_parser is enabled + if not self.is_cms_parser_enabled: + raise Exception("Something went wrong, cms_parser is still disabled.") + + def wait_for_cms_main(self) -> None: + """ + Checks the cms_main index until it has the expected number of events, or it times out. + """ + # Force the cms_parser to start parsing our savedsearches.conf + self.force_cms_parser() + + # Set counters and limits for out exp. backoff timer + elapsed_sleep_time = 0 + num_tries = 0 + time_to_sleep = 2**num_tries + max_sleep = 300 + + # Loop until timeout + while elapsed_sleep_time < max_sleep: + # Sleep, and add the time to the elapsed counter + logger.info(f"Waiting {time_to_sleep} for cms_parser to finish") + time.sleep(time_to_sleep) + elapsed_sleep_time += time_to_sleep + logger.info( + f"Checking cms_main (attempt #{num_tries + 1} - {elapsed_sleep_time} seconds elapsed of " + f"{max_sleep} max)" + ) + + # Check if the number of CMS events matches or exceeds the number of detections + if self.get_num_cms_events() >= len(self.detections): + logger.info( + f"Found {self.get_num_cms_events(use_cache=True)} events in cms_main which " + f"meets or exceeds the expected {len(self.detections)}." + ) + break + + # Update the number of times we've tried, and increment the time to sleep + num_tries += 1 + time_to_sleep = 2**num_tries + + # If the computed time to sleep will exceed max_sleep, adjust appropriately + if (elapsed_sleep_time + time_to_sleep) > max_sleep: + time_to_sleep = max_sleep - elapsed_sleep_time + + def _query_cms_main(self, use_cache: bool = False) -> splunklib.Job: + """ + Queries the cms_main index, optionally appending the provided query suffix. + + :param use_cache: a flag indicating whether the cached job should be returned + :type use_cache: bool + + :returns: a search Job entity + :rtype: :class:`splunklib.client.Job` + """ + # Use the cached job if asked to do so + if use_cache: + if self._cms_main_job is not None: + return self._cms_main_job + raise Exception( + "Attempting to return a cached job against the cms_main index, but no job has been" + " cached yet." + ) + + # Construct the query looking for CMS events matching the content app name + query = ( + f"search index=cms_main app_name=\"{self.global_config.app.appid}\" | " + f"fields {', '.join(self.cms_fields)}" + ) + logger.debug(f"Query on cms_main: {query}") + + # Get the job as a blocking operation, set the cache, and return + self._cms_main_job = self.service.search(query, exec_mode="blocking") # type: ignore + return self._cms_main_job + + def get_num_cms_events(self, use_cache: bool = False) -> int: + """ + Gets the number of matching events in the cms_main index + + :param use_cache: a flag indicating whether the cached job should be returned + :type use_cache: bool + + :returns: the count of matching events + :rtype: int + """ + # Query the cms_main index + job = self._query_cms_main(use_cache=use_cache) + + # Convert the result count to an int + return int(job["resultCount"]) + + def validate_content_against_cms(self) -> None: + """ + Using the cms_main index, validate content against the index to ensure our + savedsearches.conf is compatible with ES content versioning features. **NOTE**: while in + the future, this function may validate more types of content, currently, we only validate + detections against the cms_main index. + """ + # Get the cached job and result count + result_count = self.get_num_cms_events(use_cache=True) + job = self._query_cms_main(use_cache=True) + + # Create a running list of validation errors + exceptions: list[Exception] = [] + + # Generate an error for the count mismatch + if result_count != len(self.detections): + msg = ( + f"Expected {len(self.detections)} matching events in cms_main, but " + f"found {result_count}." + ) + logger.error(msg) + exceptions.append(Exception(msg)) + logger.info( + f"Expecting {len(self.detections)} matching events in cms_main, " + f"found {result_count}." + ) + + # Init some counters and a mapping of detections to their names + count = 100 + offset = 0 + remaining_detections = {x.name: x for x in self.detections} + matched_detections: dict[str, Detection] = {} + + # Iterate over the results until we've gone through them all + while offset < result_count: + iterator = ResultIterator( + job.results( # type: ignore + output_mode="json", + count=count, + offset=offset + ) + ) + + # Iterate over the currently fetched results + for cms_event in iterator: + # Increment the offset for each result + offset += 1 + + # Get the name of the search in the CMS event and attempt to use pattern matching + # to strip the prefix and suffix used for the savedsearches.conf name so we can + # compare to the detection + cms_entry_name = cms_event["sourcetype"] + logger.info(f"{offset}: Matching cms_main entry '{cms_entry_name}' against detections") + ptrn = re.compile(r"^" + self.global_config.app.label + r" - (?P.+) - Rule$") + match = ptrn.match(cms_event["sourcetype"]) + + # Report any errors extracting the detection name from the longer rule name + if match is None: + msg = ( + f"Entry in cms_main ('{cms_entry_name}') did not match the expected naming " + "scheme; cannot compare to our detections." + ) + logger.error(msg) + exceptions.append(Exception(msg)) + continue + + # Extract the detection name if matching was successful + cms_entry_name = match.group("cms_entry_name") + + # If CMS entry name matches one of the detections already matched, we've got an + # unexpected repeated entry + if cms_entry_name in matched_detections: + msg = ( + f"Detection '{cms_entry_name}' appears more than once in the cms_main " + "index." + ) + logger.error(msg) + exceptions.append(Exception(msg)) + continue + + # Iterate over the detections and compare the CMS entry name against each + result_matches_detection = False + for detection_name in remaining_detections: + # If we find a match, break this loop, set the found flag and move the detection + # from those that still need to matched to those already matched + if cms_entry_name == detection_name: + logger.info( + f"{offset}: Succesfully matched cms_main entry against detection " + f"('{detection_name}')!" + ) + + # Validate other fields of the cms_event against the detection + exception = self.validate_detection_against_cms_event( + cms_event, + remaining_detections[detection_name] + ) + + # Save the exception if validation failed + if exception is not None: + exceptions.append(exception) + + # Delete the matched detection and move it to the matched list + result_matches_detection = True + matched_detections[detection_name] = remaining_detections[detection_name] + del remaining_detections[detection_name] + break + + # Generate an exception if we couldn't match the CMS main entry to a detection + if result_matches_detection is False: + msg = ( + f"Could not match entry in cms_main for ('{cms_entry_name}') against any " + "of the expected detections." + ) + logger.error(msg) + exceptions.append(Exception(msg)) + + # If we have any remaining detections, they could not be matched against an entry in + # cms_main and there may have been a parsing issue with savedsearches.conf + if len(remaining_detections) > 0: + # Generate exceptions for the unmatched detections + for detection_name in remaining_detections: + msg = ( + f"Detection '{detection_name}' not found in cms_main; there may be an " + "issue with savedsearches.conf" + ) + logger.error(msg) + exceptions.append(Exception(msg)) + + # Raise exceptions as a group + if len(exceptions) > 0: + raise ExceptionGroup( + "1 or more issues validating our detections against the cms_main index", + exceptions + ) + + # Else, we've matched/validated all detections against cms_main + logger.info("Matched and validated all detections against cms_main!") + + def validate_detection_against_cms_event( + self, + cms_event: dict[str, Any], + detection: Detection + ) -> Exception | None: + """ + Given an event from the cms_main index and the matched detection, compare fields and look + for any inconsistencies + + :param cms_event: The event from the cms_main index + :type cms_event: dict[str, Any] + :param detection: The matched detection + :type detection: :class:`contentctl.objects.detection.Detection` + + :return: The generated exception, or None + :rtype: Exception | None + """ + # TODO (cmcginley): validate additional fields between the cms_event and the detection + + cms_uuid = uuid.UUID(cms_event["detection_id"]) + full_search_key = f"action.{self.global_config.app.label.lower()}.full_search_name" + rule_name_from_detection = f"{self.global_config.app.label} - {detection.name} - Rule" + + # Compare the UUIDs + if cms_uuid != detection.id: + msg = ( + f"UUID in cms_event ('{cms_uuid}') does not match UUID in detection " + f"('{detection.id}'): {detection.name}" + ) + logger.error(msg) + return Exception(msg) + elif cms_event["version"] != f"{detection.version}-1": + # Compare the versions (we append '-1' to the detection version to be in line w/ the + # internal representation in ES) + msg = ( + f"Version in cms_event ('{cms_event['version']}') does not match version in " + f"detection ('{detection.version}-1'): {detection.name}" + ) + logger.error(msg) + return Exception(msg) + elif cms_event[full_search_key] != rule_name_from_detection: + # Compare the full search name + msg = ( + f"Full search name in cms_event ('{cms_event[full_search_key]}') " + f"does not match detection name ('{detection.name}')" + ) + logger.error(msg) + return Exception(msg) + elif cms_event["action.correlationsearch.label"] != f"{self.global_config.app.label} - {detection.name} - Rule": + # Compare the correlation search label + msg = ( + f"Correlation search label in cms_event " + f"('{cms_event['action.correlationsearch.label']}') does not match detection name " + f"('{detection.name}')" + ) + logger.error(msg) + return Exception(msg) + elif cms_event["sourcetype"] != f"{self.global_config.app.label} - {detection.name} - Rule": + # Compare the full search name + msg = ( + f"Sourcetype in cms_event ('{cms_event[f'sourcetype']}') does not match detection " + f"name ('{detection.name}')" + ) + logger.error(msg) + return Exception(msg) + + return None diff --git a/contentctl/objects/correlation_search.py b/contentctl/objects/correlation_search.py index a0b25da9..173e7d1b 100644 --- a/contentctl/objects/correlation_search.py +++ b/contentctl/objects/correlation_search.py @@ -31,8 +31,9 @@ from contentctl.objects.observable import Observable +# TODO (cmcginley): disable logging # Suppress logging by default; enable for local testing -ENABLE_LOGGING = False +ENABLE_LOGGING = True LOG_LEVEL = logging.DEBUG LOG_PATH = "correlation_search.log" @@ -144,13 +145,14 @@ def __init__(self, response_reader: ResponseReader) -> None: def __iter__(self) -> "ResultIterator": return self - def __next__(self) -> dict: + def __next__(self) -> dict[str, Any]: # Use a reader for JSON format so we can iterate over our results for result in self.results_reader: # log messages, or raise if error if isinstance(result, Message): # convert level string to level int - level_name = result.type.strip().upper() + level_name: str = result.type.strip().upper() # type: ignore + # TODO (cmcginley): this method is deprecated; replace with our own enum level: int = logging.getLevelName(level_name) # log message at appropriate level and raise if needed @@ -161,7 +163,7 @@ def __next__(self) -> dict: # if dict, just return elif isinstance(result, dict): - return result + return result # type: ignore # raise for any unexpected types else: diff --git a/contentctl/output/conf_output.py b/contentctl/output/conf_output.py index cf4574ae..7132ef0e 100644 --- a/contentctl/output/conf_output.py +++ b/contentctl/output/conf_output.py @@ -72,7 +72,9 @@ def writeAppConf(self)->set[pathlib.Path]: [self.config.app])) return written_files - + # TODO (cmcginley): we could have a discrepancy between detections tested and those delivered + # based on the jinja2 template + # {% if (detection.type == 'TTP' or detection.type == 'Anomaly' or detection.type == 'Hunting' or detection.type == 'Correlation') %} def writeObjects(self, objects: list, type: SecurityContentType = None) -> set[pathlib.Path]: written_files:set[pathlib.Path] = set() if type == SecurityContentType.detections: From 2c87187a40647a670df727404049554d6ca0e4e0 Mon Sep 17 00:00:00 2001 From: Casey McGinley Date: Tue, 8 Oct 2024 00:13:30 -0700 Subject: [PATCH 03/24] added logic for checking when to run cms testing --- .../DetectionTestingManager.py | 38 +++++-- .../DetectionTestingInfrastructure.py | 102 +++++++++++++++--- .../objects/content_versioning_service.py | 54 ++++++---- 3 files changed, 152 insertions(+), 42 deletions(-) diff --git a/contentctl/actions/detection_testing/DetectionTestingManager.py b/contentctl/actions/detection_testing/DetectionTestingManager.py index 4f04b202..a1d659ca 100644 --- a/contentctl/actions/detection_testing/DetectionTestingManager.py +++ b/contentctl/actions/detection_testing/DetectionTestingManager.py @@ -65,7 +65,7 @@ def sigint_handler(signum, frame): print("*******************************") signal.signal(signal.SIGINT, sigint_handler) - + with concurrent.futures.ThreadPoolExecutor( max_workers=len(self.input_dto.config.test_instances), ) as instance_pool, concurrent.futures.ThreadPoolExecutor( @@ -73,11 +73,19 @@ def sigint_handler(signum, frame): ) as view_runner, concurrent.futures.ThreadPoolExecutor( max_workers=len(self.input_dto.config.test_instances), ) as view_shutdowner: + # Capture any errors for reporting at the end after all threads have been gathered + errors: dict[str, list[Exception]] = { + "INSTANCE SETUP ERRORS": [], + "TESTING ERRORS": [], + "ERRORS DURING VIEW SHUTDOWN": [], + "ERRORS DURING VIEW EXECUTION": [], + } # Start all the views future_views = { view_runner.submit(view.setup): view for view in self.input_dto.views } + # Configure all the instances future_instances_setup = { instance_pool.submit(instance.setup): instance @@ -87,10 +95,10 @@ def sigint_handler(signum, frame): # Wait for all instances to be set up for future in concurrent.futures.as_completed(future_instances_setup): try: - result = future.result() + _ = future.result() except Exception as e: self.output_dto.terminate = True - print(f"Error setting up instance: {str(e)}") + errors["INSTANCE SETUP ERRORS"].append(e) # Start and wait for all tests to run if not self.output_dto.terminate: @@ -102,10 +110,10 @@ def sigint_handler(signum, frame): # Wait for execution to finish for future in concurrent.futures.as_completed(future_instances_execute): try: - result = future.result() + _ = future.result() except Exception as e: self.output_dto.terminate = True - print(f"Error running in container: {str(e)}") + errors["TESTING ERRORS"].append(e) self.output_dto.terminate = True @@ -115,16 +123,28 @@ def sigint_handler(signum, frame): } for future in concurrent.futures.as_completed(future_views_shutdowner): try: - result = future.result() + _ = future.result() except Exception as e: - print(f"Error stopping view: {str(e)}") + errors["ERRORS DURING VIEW SHUTDOWN"].append(e) # Wait for original view-related threads to complete for future in concurrent.futures.as_completed(future_views): try: - result = future.result() + _ = future.result() except Exception as e: - print(f"Error running container: {str(e)}") + errors["ERRORS DURING VIEW EXECUTION"].append(e) + + # Log any errors + for error_type in errors: + if len(errors[error_type]) > 0: + print() + print(f"[{error_type}]:") + for error in errors[error_type]: + print(f"\t❌ {str(error)}") + if isinstance(error, ExceptionGroup): + for suberror in error.exceptions: # type: ignore + print(f"\t\t❌ {str(suberror)}") # type: ignore + print() return self.output_dto diff --git a/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py b/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py index 2841b3a5..2757665a 100644 --- a/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py +++ b/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py @@ -20,8 +20,9 @@ from splunklib.results import JSONResultsReader, Message # type: ignore from urllib3 import disable_warnings import urllib.parse +from semantic_version import Version # type: ignore -from contentctl.objects.config import test_common, Infrastructure +from contentctl.objects.config import test_common, Infrastructure, All from contentctl.objects.enums import PostTestBehavior, AnalyticsType from contentctl.objects.detection import Detection from contentctl.objects.base_test import BaseTest @@ -42,6 +43,9 @@ TestingStates ) +# The app name of ES; needed to check ES version +ES_APP_NAME = "SplunkEnterpriseSecuritySuite" + class SetupTestGroupResults(BaseModel): exception: Union[Exception, None] = None @@ -127,17 +131,27 @@ def setup(self): ) self.start_time = time.time() + + # Init the list of setup functions we always need setup_functions: list[tuple[Callable[[], None | client.Service], str]] = [ (self.start, "Starting"), (self.get_conn, "Waiting for App Installation"), (self.configure_conf_file_datamodels, "Configuring Datamodels"), (self.create_replay_index, f"Create index '{self.sync_obj.replay_index}'"), + (self.check_for_es_install, "Checking for ES Install"), (self.configure_imported_roles, "Configuring Roles"), (self.configure_delete_indexes, "Configuring Indexes"), (self.configure_hec, "Configuring HEC"), ] - setup_functions = setup_functions + self.content_versioning_service.setup_functions + + # Add any setup functions only applicable to content versioning validation + if self.should_test_content_versioning: + setup_functions = setup_functions + self.content_versioning_service.setup_functions + + # Add the final setup function setup_functions.append((self.wait_for_ui_ready, "Finishing Setup")) + + # Execute and report on each setup function try: for func, msg in setup_functions: self.format_pbar_string( @@ -150,9 +164,11 @@ def setup(self): self.check_for_teardown() except Exception as e: - self.pbar.write(str(e)) + msg = f"[{self.get_name()}]: {str(e)}" self.finish() - raise + if isinstance(e, ExceptionGroup): + raise ExceptionGroup(msg, e.exceptions) from e # type: ignore + raise Exception(msg) from e self.format_pbar_string(TestReportingType.SETUP, self.get_name(), "Finished Setup!") @@ -162,6 +178,14 @@ def wait_for_ui_ready(self): @computed_field @property def content_versioning_service(self) -> ContentVersioningService: + """ + A computed field returning a handle to the content versioning service, used by ES to + version detections. We use this model to validate that all detections have been installed + compatibly with ES versioning. + + :return: a handle to the content versioning service on the instance + :rtype: :class:`contentctl.objects.content_versioning_service.ContentVersioningService` + """ return ContentVersioningService( global_config=self.global_config, infrastructure=self.infrastructure, @@ -169,6 +193,57 @@ def content_versioning_service(self) -> ContentVersioningService: detections=self.sync_obj.inputQueue ) + @property + def should_test_content_versioning(self) -> bool: + """ + Indicates whether we should test content versioning. Content versioning + should be tested when integration testing is enabled, the mode is all, and ES is at least + version 8.0.0. + + :return: a bool indicating whether we should test content versioning + :rtype: bool + """ + es_version = self.es_version + return ( + self.global_config.enable_integration_testing + and isinstance(self.global_config.mode, All) + and es_version is not None + and es_version >= Version("8.0.0") + ) + + @property + def es_version(self) -> Version | None: + """ + Returns the version of Enterprise Security installed on the instance; None if not installed. + + :return: the version of ES, as a semver aware object + :rtype: :class:`semantic_version.Version` + """ + if not self.es_installed: + return None + return Version(self.get_conn().apps[ES_APP_NAME]["version"]) # type: ignore + + @property + def es_installed(self) -> bool: + """ + Indicates whether ES is installed on the instance. + + :return: a bool indicating whether ES is installed or not + :rtype: bool + """ + return ES_APP_NAME in self.get_conn().apps + + def check_for_es_install(self) -> None: + """ + Validating function which raises an error if Enterprise Security is not installed and + integration testing is enabled. + """ + if not self.es_installed and self.global_config.enable_integration_testing: + raise Exception( + "Enterprise Security does not appear to be installed on this instance and " + "integration testing is enabled." + ) + def configure_hec(self): self.hec_channel = str(uuid.uuid4()) try: @@ -282,25 +357,22 @@ def configure_imported_roles( ): indexes.append(self.sync_obj.replay_index) indexes_encoded = ";".join(indexes) + + # Include ES roles if installed + if self.es_installed: + imported_roles = imported_roles + enterprise_security_roles try: self.get_conn().roles.post( self.infrastructure.splunk_app_username, - imported_roles=imported_roles + enterprise_security_roles, + imported_roles=imported_roles, srchIndexesAllowed=indexes_encoded, srchIndexesDefault=self.sync_obj.replay_index, ) return except Exception as e: - self.pbar.write( - f"Enterprise Security Roles do not exist:'{enterprise_security_roles}: {str(e)}" - ) - - self.get_conn().roles.post( - self.infrastructure.splunk_app_username, - imported_roles=imported_roles, - srchIndexesAllowed=indexes_encoded, - srchIndexesDefault=self.sync_obj.replay_index, - ) + msg = f"Error configuring roles: {str(e)}" + self.pbar.write(msg) + raise Exception(msg) from e def configure_delete_indexes(self, indexes: list[str] = ["_*", "*"]): indexes.append(self.sync_obj.replay_index) diff --git a/contentctl/objects/content_versioning_service.py b/contentctl/objects/content_versioning_service.py index e752d081..d6203acc 100644 --- a/contentctl/objects/content_versioning_service.py +++ b/contentctl/objects/content_versioning_service.py @@ -17,21 +17,42 @@ # TODO (cmcginley): remove this logger logger = get_logger() +# TODO (cmcginley): would it be better for this to only run on one instance? Or to consolidate +# error reporting at least? + class ContentVersioningService(BaseModel): + """ + A model representing the content versioning service used in ES 8.0.0+. This model can be used + to validate that detections have been installed in a way that is compatible with content + versioning. + """ + + # The global contentctl config global_config: test_common + + # The instance specific infra config infrastructure: Infrastructure + + # The splunklib service service: splunklib.Service + + # The list of detections detections: list[Detection] + # The cached job on the splunk instance of the cms events _cms_main_job: splunklib.Job | None = PrivateAttr(default=None) class Config: + # We need to allow arbitrary type for the splunklib service arbitrary_types_allowed = True @computed_field @property def setup_functions(self) -> list[tuple[Callable[[], None], str]]: + """ + Returns the list of setup functions needed for content versioning testing + """ return [ (self.activate_versioning, "Activating Content Versioning"), (self.wait_for_cms_main, "Waiting for CMS Parser"), @@ -56,7 +77,7 @@ def _query_content_versioning_service(self, method: str, body: dict[str, Any] = # Query the content versioning service try: - response = self.service.request( # type: ignore + response = self.service.request( # type: ignore method=method, path_segment="configs/conf-feature_flags/general", body=body, @@ -119,8 +140,6 @@ def activate_versioning(self) -> None: """ Activate the content versioning service """ - # TODO (cmcginley): add conditional logging s.t. this check only happens when integration - # testing is enabled AND ES is at least version 8.0, AND mode is `all` # Post to the SA-ContentVersioning service to set versioning status self._query_content_versioning_service( method="POST", @@ -325,7 +344,7 @@ def validate_content_against_cms(self) -> None: # Report any errors extracting the detection name from the longer rule name if match is None: msg = ( - f"Entry in cms_main ('{cms_entry_name}') did not match the expected naming " + f"[{cms_entry_name}]: Entry in cms_main did not match the expected naming " "scheme; cannot compare to our detections." ) logger.error(msg) @@ -339,7 +358,7 @@ def validate_content_against_cms(self) -> None: # unexpected repeated entry if cms_entry_name in matched_detections: msg = ( - f"Detection '{cms_entry_name}' appears more than once in the cms_main " + f"[{cms_entry_name}]: Detection appears more than once in the cms_main " "index." ) logger.error(msg) @@ -376,7 +395,7 @@ def validate_content_against_cms(self) -> None: # Generate an exception if we couldn't match the CMS main entry to a detection if result_matches_detection is False: msg = ( - f"Could not match entry in cms_main for ('{cms_entry_name}') against any " + f"[{cms_entry_name}]: Could not match entry in cms_main against any " "of the expected detections." ) logger.error(msg) @@ -388,7 +407,7 @@ def validate_content_against_cms(self) -> None: # Generate exceptions for the unmatched detections for detection_name in remaining_detections: msg = ( - f"Detection '{detection_name}' not found in cms_main; there may be an " + f"[{detection_name}]: Detection not found in cms_main; there may be an " "issue with savedsearches.conf" ) logger.error(msg) @@ -430,8 +449,8 @@ def validate_detection_against_cms_event( # Compare the UUIDs if cms_uuid != detection.id: msg = ( - f"UUID in cms_event ('{cms_uuid}') does not match UUID in detection " - f"('{detection.id}'): {detection.name}" + f"[{detection.name}]: UUID in cms_event ('{cms_uuid}') does not match UUID in " + f"detection ('{detection.id}')" ) logger.error(msg) return Exception(msg) @@ -439,33 +458,32 @@ def validate_detection_against_cms_event( # Compare the versions (we append '-1' to the detection version to be in line w/ the # internal representation in ES) msg = ( - f"Version in cms_event ('{cms_event['version']}') does not match version in " - f"detection ('{detection.version}-1'): {detection.name}" + f"[{detection.name}]: Version in cms_event ('{cms_event['version']}') does not " + f"match version in detection ('{detection.version}-1')" ) logger.error(msg) return Exception(msg) elif cms_event[full_search_key] != rule_name_from_detection: # Compare the full search name msg = ( - f"Full search name in cms_event ('{cms_event[full_search_key]}') " - f"does not match detection name ('{detection.name}')" + f"[{detection.name}]: Full search name in cms_event " + f"('{cms_event[full_search_key]}') does not match detection name" ) logger.error(msg) return Exception(msg) elif cms_event["action.correlationsearch.label"] != f"{self.global_config.app.label} - {detection.name} - Rule": # Compare the correlation search label msg = ( - f"Correlation search label in cms_event " - f"('{cms_event['action.correlationsearch.label']}') does not match detection name " - f"('{detection.name}')" + f"[{detection.name}]: Correlation search label in cms_event " + f"('{cms_event['action.correlationsearch.label']}') does not match detection name" ) logger.error(msg) return Exception(msg) elif cms_event["sourcetype"] != f"{self.global_config.app.label} - {detection.name} - Rule": # Compare the full search name msg = ( - f"Sourcetype in cms_event ('{cms_event[f'sourcetype']}') does not match detection " - f"name ('{detection.name}')" + f"[{detection.name}]: Sourcetype in cms_event ('{cms_event[f'sourcetype']}') does " + f"not match detection name" ) logger.error(msg) return Exception(msg) From 27cc4785277d7723d9a92d247aa8042b09993f9b Mon Sep 17 00:00:00 2001 From: Casey McGinley Date: Tue, 8 Oct 2024 11:15:14 -0700 Subject: [PATCH 04/24] migrated logging to utils --- contentctl/helper/utils.py | 57 ++++++++++++++++++ contentctl/objects/config.py | 2 - .../objects/content_versioning_service.py | 59 ++++++++++++------- contentctl/objects/correlation_search.py | 57 +++++------------- 4 files changed, 109 insertions(+), 66 deletions(-) diff --git a/contentctl/helper/utils.py b/contentctl/helper/utils.py index 261ecb64..4dbbf51e 100644 --- a/contentctl/helper/utils.py +++ b/contentctl/helper/utils.py @@ -6,6 +6,7 @@ import string from timeit import default_timer import pathlib +import logging from typing import Union, Tuple import tqdm @@ -485,3 +486,59 @@ def getPercent(numerator: float, denominator: float, decimal_places: int) -> str ratio = numerator / denominator percent = ratio * 100 return Utils.getFixedWidth(percent, decimal_places) + "%" + + @staticmethod + def get_logger( + name: str, + log_level: int, + log_path: str, + enable_logging: bool + ) -> logging.Logger: + """ + Gets a logger instance for the given name; logger is configured if not already configured. + The NullHandler is used to suppress loggging when running in production so as not to + conflict w/ contentctl's larger pbar-based logging. The StreamHandler is enabled by setting + enable_logging to True (useful for debugging/testing locally) + + :param name: the logger name + :type name: str + :param log_level: the logging level (e.g. `logging.Debug`) + :type log_level: int + :param log_path: the path for the log file + :type log_path: str + :param enable_logging: a flag indicating whether logging should be redirected from null to + the stream handler + :type enable_logging: bool + + :return: a logger + :rtype: :class:`logging.Logger` + """ + # get logger for module + logger = logging.getLogger(name) + + # set propagate to False if not already set as such (needed to that we do not flow up to any + # root loggers) + if logger.propagate: + logger.propagate = False + + # if logger has no handlers, it needs to be configured for the first time + if not logger.hasHandlers(): + # set level + logger.setLevel(log_level) + + # if logging enabled, use a StreamHandler; else, use the NullHandler to suppress logging + handler: logging.Handler + if enable_logging: + handler = logging.FileHandler(log_path) + else: + handler = logging.NullHandler() + + # Format our output + formatter = logging.Formatter('%(asctime)s - %(levelname)s:%(name)s - %(message)s') + handler.setFormatter(formatter) + + # Set handler level and add to logger + handler.setLevel(log_level) + logger.addHandler(handler) + + return logger diff --git a/contentctl/objects/config.py b/contentctl/objects/config.py index 0b262c55..8ef9b30a 100644 --- a/contentctl/objects/config.py +++ b/contentctl/objects/config.py @@ -318,8 +318,6 @@ class inspect(build): f"or CLI invocation appropriately] {validate.model_fields['enrichments'].description}" ) ) - # TODO (cmcginley): wording should change here if we want to be able to download any app from - # Splunkbase previous_build: str | None = Field( default=None, description=( diff --git a/contentctl/objects/content_versioning_service.py b/contentctl/objects/content_versioning_service.py index d6203acc..a6dc6ab4 100644 --- a/contentctl/objects/content_versioning_service.py +++ b/contentctl/objects/content_versioning_service.py @@ -2,20 +2,25 @@ import uuid import json import re +import logging from typing import Any, Callable from functools import cached_property -from pydantic import BaseModel, PrivateAttr, computed_field +from pydantic import BaseModel, PrivateAttr, computed_field, Field import splunklib.client as splunklib # type: ignore from splunklib.binding import HTTPError, ResponseReader # type: ignore from splunklib.data import Record # type: ignore from contentctl.objects.config import test_common, Infrastructure from contentctl.objects.detection import Detection -from contentctl.objects.correlation_search import ResultIterator, get_logger +from contentctl.objects.correlation_search import ResultIterator +from contentctl.helper.utils import Utils -# TODO (cmcginley): remove this logger -logger = get_logger() +# TODO (cmcginley): suppress logging +# Suppress logging by default; enable for local testing +ENABLE_LOGGING = True +LOG_LEVEL = logging.DEBUG +LOG_PATH = "content_versioning_service.log" # TODO (cmcginley): would it be better for this to only run on one instance? Or to consolidate # error reporting at least? @@ -40,6 +45,16 @@ class ContentVersioningService(BaseModel): # The list of detections detections: list[Detection] + # The logger to use (logs all go to a null pipe unless ENABLE_LOGGING is set to True, so as not + # to conflict w/ tqdm) + logger: logging.Logger = Field(default_factory=lambda: Utils.get_logger( + __name__, + LOG_LEVEL, + LOG_PATH, + ENABLE_LOGGING + ) + ) + # The cached job on the splunk instance of the cms events _cms_main_job: splunklib.Job | None = PrivateAttr(default=None) @@ -215,17 +230,17 @@ def wait_for_cms_main(self) -> None: # Loop until timeout while elapsed_sleep_time < max_sleep: # Sleep, and add the time to the elapsed counter - logger.info(f"Waiting {time_to_sleep} for cms_parser to finish") + self.logger.info(f"Waiting {time_to_sleep} for cms_parser to finish") time.sleep(time_to_sleep) elapsed_sleep_time += time_to_sleep - logger.info( + self.logger.info( f"Checking cms_main (attempt #{num_tries + 1} - {elapsed_sleep_time} seconds elapsed of " f"{max_sleep} max)" ) # Check if the number of CMS events matches or exceeds the number of detections if self.get_num_cms_events() >= len(self.detections): - logger.info( + self.logger.info( f"Found {self.get_num_cms_events(use_cache=True)} events in cms_main which " f"meets or exceeds the expected {len(self.detections)}." ) @@ -263,7 +278,7 @@ def _query_cms_main(self, use_cache: bool = False) -> splunklib.Job: f"search index=cms_main app_name=\"{self.global_config.app.appid}\" | " f"fields {', '.join(self.cms_fields)}" ) - logger.debug(f"Query on cms_main: {query}") + self.logger.debug(f"Query on cms_main: {query}") # Get the job as a blocking operation, set the cache, and return self._cms_main_job = self.service.search(query, exec_mode="blocking") # type: ignore @@ -305,9 +320,9 @@ def validate_content_against_cms(self) -> None: f"Expected {len(self.detections)} matching events in cms_main, but " f"found {result_count}." ) - logger.error(msg) + self.logger.error(msg) exceptions.append(Exception(msg)) - logger.info( + self.logger.info( f"Expecting {len(self.detections)} matching events in cms_main, " f"found {result_count}." ) @@ -337,7 +352,7 @@ def validate_content_against_cms(self) -> None: # to strip the prefix and suffix used for the savedsearches.conf name so we can # compare to the detection cms_entry_name = cms_event["sourcetype"] - logger.info(f"{offset}: Matching cms_main entry '{cms_entry_name}' against detections") + self.logger.info(f"{offset}: Matching cms_main entry '{cms_entry_name}' against detections") ptrn = re.compile(r"^" + self.global_config.app.label + r" - (?P.+) - Rule$") match = ptrn.match(cms_event["sourcetype"]) @@ -347,7 +362,7 @@ def validate_content_against_cms(self) -> None: f"[{cms_entry_name}]: Entry in cms_main did not match the expected naming " "scheme; cannot compare to our detections." ) - logger.error(msg) + self.logger.error(msg) exceptions.append(Exception(msg)) continue @@ -361,7 +376,7 @@ def validate_content_against_cms(self) -> None: f"[{cms_entry_name}]: Detection appears more than once in the cms_main " "index." ) - logger.error(msg) + self.logger.error(msg) exceptions.append(Exception(msg)) continue @@ -371,7 +386,7 @@ def validate_content_against_cms(self) -> None: # If we find a match, break this loop, set the found flag and move the detection # from those that still need to matched to those already matched if cms_entry_name == detection_name: - logger.info( + self.logger.info( f"{offset}: Succesfully matched cms_main entry against detection " f"('{detection_name}')!" ) @@ -398,7 +413,7 @@ def validate_content_against_cms(self) -> None: f"[{cms_entry_name}]: Could not match entry in cms_main against any " "of the expected detections." ) - logger.error(msg) + self.logger.error(msg) exceptions.append(Exception(msg)) # If we have any remaining detections, they could not be matched against an entry in @@ -410,7 +425,7 @@ def validate_content_against_cms(self) -> None: f"[{detection_name}]: Detection not found in cms_main; there may be an " "issue with savedsearches.conf" ) - logger.error(msg) + self.logger.error(msg) exceptions.append(Exception(msg)) # Raise exceptions as a group @@ -421,7 +436,7 @@ def validate_content_against_cms(self) -> None: ) # Else, we've matched/validated all detections against cms_main - logger.info("Matched and validated all detections against cms_main!") + self.logger.info("Matched and validated all detections against cms_main!") def validate_detection_against_cms_event( self, @@ -452,7 +467,7 @@ def validate_detection_against_cms_event( f"[{detection.name}]: UUID in cms_event ('{cms_uuid}') does not match UUID in " f"detection ('{detection.id}')" ) - logger.error(msg) + self.logger.error(msg) return Exception(msg) elif cms_event["version"] != f"{detection.version}-1": # Compare the versions (we append '-1' to the detection version to be in line w/ the @@ -461,7 +476,7 @@ def validate_detection_against_cms_event( f"[{detection.name}]: Version in cms_event ('{cms_event['version']}') does not " f"match version in detection ('{detection.version}-1')" ) - logger.error(msg) + self.logger.error(msg) return Exception(msg) elif cms_event[full_search_key] != rule_name_from_detection: # Compare the full search name @@ -469,7 +484,7 @@ def validate_detection_against_cms_event( f"[{detection.name}]: Full search name in cms_event " f"('{cms_event[full_search_key]}') does not match detection name" ) - logger.error(msg) + self.logger.error(msg) return Exception(msg) elif cms_event["action.correlationsearch.label"] != f"{self.global_config.app.label} - {detection.name} - Rule": # Compare the correlation search label @@ -477,7 +492,7 @@ def validate_detection_against_cms_event( f"[{detection.name}]: Correlation search label in cms_event " f"('{cms_event['action.correlationsearch.label']}') does not match detection name" ) - logger.error(msg) + self.logger.error(msg) return Exception(msg) elif cms_event["sourcetype"] != f"{self.global_config.app.label} - {detection.name} - Rule": # Compare the full search name @@ -485,7 +500,7 @@ def validate_detection_against_cms_event( f"[{detection.name}]: Sourcetype in cms_event ('{cms_event[f'sourcetype']}') does " f"not match detection name" ) - logger.error(msg) + self.logger.error(msg) return Exception(msg) return None diff --git a/contentctl/objects/correlation_search.py b/contentctl/objects/correlation_search.py index 173e7d1b..3cefd383 100644 --- a/contentctl/objects/correlation_search.py +++ b/contentctl/objects/correlation_search.py @@ -29,53 +29,15 @@ from contentctl.objects.risk_event import RiskEvent from contentctl.objects.notable_event import NotableEvent from contentctl.objects.observable import Observable +from contentctl.helper.utils import Utils -# TODO (cmcginley): disable logging # Suppress logging by default; enable for local testing -ENABLE_LOGGING = True +ENABLE_LOGGING = False LOG_LEVEL = logging.DEBUG LOG_PATH = "correlation_search.log" -def get_logger() -> logging.Logger: - """ - Gets a logger instance for the module; logger is configured if not already configured. The - NullHandler is used to suppress loggging when running in production so as not to conflict w/ - contentctl's larger pbar-based logging. The StreamHandler is enabled by setting ENABLE_LOGGING - to True (useful for debugging/testing locally) - """ - # get logger for module - logger = logging.getLogger(__name__) - - # set propagate to False if not already set as such (needed to that we do not flow up to any - # root loggers) - if logger.propagate: - logger.propagate = False - - # if logger has no handlers, it needs to be configured for the first time - if not logger.hasHandlers(): - # set level - logger.setLevel(LOG_LEVEL) - - # if logging enabled, use a StreamHandler; else, use the NullHandler to suppress logging - handler: logging.Handler - if ENABLE_LOGGING: - handler = logging.FileHandler(LOG_PATH) - else: - handler = logging.NullHandler() - - # Format our output - formatter = logging.Formatter('%(asctime)s - %(levelname)s:%(name)s - %(message)s') - handler.setFormatter(formatter) - - # Set handler level and add to logger - handler.setLevel(LOG_LEVEL) - logger.addHandler(handler) - - return logger - - class SavedSearchKeys(str, Enum): """ Various keys into the SavedSearch content @@ -140,7 +102,12 @@ def __init__(self, response_reader: ResponseReader) -> None: ) # get logger - self.logger: logging.Logger = get_logger() + self.logger: logging.Logger = Utils.get_logger( + __name__, + LOG_LEVEL, + LOG_PATH, + ENABLE_LOGGING + ) def __iter__(self) -> "ResultIterator": return self @@ -222,7 +189,13 @@ class CorrelationSearch(BaseModel): # The logger to use (logs all go to a null pipe unless ENABLE_LOGGING is set to True, so as not # to conflict w/ tqdm) - logger: logging.Logger = Field(default_factory=get_logger) + logger: logging.Logger = Field(default_factory=lambda: Utils.get_logger( + __name__, + LOG_LEVEL, + LOG_PATH, + ENABLE_LOGGING + ) + ) # The search name (e.g. "ESCU - Windows Modify Registry EnableLinkedConnections - Rule") name: Optional[str] = None From 35ece19ba5b8aa75cbfbbd19ab4cfb3e7a8c52af Mon Sep 17 00:00:00 2001 From: Casey McGinley Date: Tue, 8 Oct 2024 15:08:27 -0700 Subject: [PATCH 05/24] writing out setup complete to terminal --- .../infrastructures/DetectionTestingInfrastructure.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py b/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py index 497cf9fb..ab30dd50 100644 --- a/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py +++ b/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py @@ -170,7 +170,14 @@ def setup(self): raise ExceptionGroup(msg, e.exceptions) from e # type: ignore raise Exception(msg) from e - self.format_pbar_string(TestReportingType.SETUP, self.get_name(), "Finished Setup!") + self.pbar.write( + self.format_pbar_string( + TestReportingType.SETUP, + self.get_name(), + "Finished Setup!", + set_pbar=False + ) + ) def wait_for_ui_ready(self): self.get_conn() From 21520bc693e44e13b16e02b9c546f420e6ddf9a2 Mon Sep 17 00:00:00 2001 From: Casey McGinley Date: Tue, 8 Oct 2024 16:22:59 -0700 Subject: [PATCH 06/24] setup functions need to be run before we can check if content versioning should be enforced --- .../DetectionTestingInfrastructure.py | 25 ++++++++++++------- ...DetectionTestingInfrastructureContainer.py | 1 - 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py b/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py index ab30dd50..e7bfffde 100644 --- a/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py +++ b/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py @@ -133,7 +133,7 @@ def setup(self): self.start_time = time.time() # Init the list of setup functions we always need - setup_functions: list[tuple[Callable[[], None | client.Service], str]] = [ + primary_setup_functions: list[tuple[Callable[[], None | client.Service], str]] = [ (self.start, "Starting"), (self.get_conn, "Waiting for App Installation"), (self.configure_conf_file_datamodels, "Configuring Datamodels"), @@ -142,18 +142,13 @@ def setup(self): (self.configure_imported_roles, "Configuring Roles"), (self.configure_delete_indexes, "Configuring Indexes"), (self.configure_hec, "Configuring HEC"), + (self.wait_for_ui_ready, "Finishing Primary Setup") ] - # Add any setup functions only applicable to content versioning validation - if self.should_test_content_versioning: - setup_functions = setup_functions + self.content_versioning_service.setup_functions - - # Add the final setup function - setup_functions.append((self.wait_for_ui_ready, "Finishing Setup")) - # Execute and report on each setup function try: - for func, msg in setup_functions: + # Run the primary setup functions + for func, msg in primary_setup_functions: self.format_pbar_string( TestReportingType.SETUP, self.get_name(), @@ -163,6 +158,18 @@ def setup(self): func() self.check_for_teardown() + # Run any setup functions only applicable to content versioning validation + if self.should_test_content_versioning: + for func, msg in self.content_versioning_service.setup_functions: + self.format_pbar_string( + TestReportingType.SETUP, + self.get_name(), + msg, + update_sync_status=True, + ) + func() + self.check_for_teardown() + except Exception as e: msg = f"[{self.get_name()}]: {str(e)}" self.finish() diff --git a/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructureContainer.py b/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructureContainer.py index f5887033..e3405553 100644 --- a/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructureContainer.py +++ b/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructureContainer.py @@ -47,7 +47,6 @@ def get_docker_client(self): raise (Exception(f"Failed to get docker client: {str(e)}")) def check_for_teardown(self): - try: container: docker.models.containers.Container = self.get_docker_client().containers.get(self.get_name()) except Exception as e: From 84e89c4c15d51fdf6774b7058ac602ad143589d8 Mon Sep 17 00:00:00 2001 From: Casey McGinley Date: Thu, 24 Oct 2024 11:52:25 -0700 Subject: [PATCH 07/24] missing comma --- .../infrastructures/DetectionTestingInfrastructure.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py b/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py index a7223abe..b41a5a3d 100644 --- a/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py +++ b/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py @@ -148,7 +148,7 @@ def setup(self): (self.get_conn, "Waiting for App Installation"), (self.configure_conf_file_datamodels, "Configuring Datamodels"), (self.create_replay_index, f"Create index '{self.sync_obj.replay_index}'"), - (self.get_all_indexes, "Getting all indexes from server") + (self.get_all_indexes, "Getting all indexes from server"), (self.check_for_es_install, "Checking for ES Install"), (self.configure_imported_roles, "Configuring Roles"), (self.configure_delete_indexes, "Configuring Indexes"), From 84a0d1bccd6818bfb47c51376a51cc6f3dac4b57 Mon Sep 17 00:00:00 2001 From: Casey McGinley Date: Thu, 24 Oct 2024 14:43:33 -0700 Subject: [PATCH 08/24] logging the beginning of content versioning validation --- .../infrastructures/DetectionTestingInfrastructure.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py b/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py index b41a5a3d..0f56048b 100644 --- a/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py +++ b/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py @@ -171,6 +171,14 @@ def setup(self): # Run any setup functions only applicable to content versioning validation if self.should_test_content_versioning: + self.pbar.write( + self.format_pbar_string( + TestReportingType.SETUP, + self.get_name(), + "Beginning Content Versioning Validation...", + set_pbar=False + ) + ) for func, msg in self.content_versioning_service.setup_functions: self.format_pbar_string( TestReportingType.SETUP, From c8496947414c41843379f79248b16a80f155728f Mon Sep 17 00:00:00 2001 From: Casey McGinley Date: Tue, 29 Oct 2024 12:39:33 -0700 Subject: [PATCH 09/24] adjusting for some of the changes in build and ES 8.0 --- .../objects/content_versioning_service.py | 136 +++++++++++------- 1 file changed, 85 insertions(+), 51 deletions(-) diff --git a/contentctl/objects/content_versioning_service.py b/contentctl/objects/content_versioning_service.py index a6dc6ab4..6ebc064c 100644 --- a/contentctl/objects/content_versioning_service.py +++ b/contentctl/objects/content_versioning_service.py @@ -16,6 +16,14 @@ from contentctl.objects.correlation_search import ResultIterator from contentctl.helper.utils import Utils +# TODO (cmcginley): +# - [x] version naming scheme seems to have changed from X - X to X.X +# - [x] sourcetype no longer holds detection name but instead is stash_common_detection_model +# - [ ] action.escu.full_search_name no longer available +# - [ ] check to see if we can get "name" +# - [ ] move strings to enums +# - [ ] additionally, timeout for cms_parser seems to need more time + # TODO (cmcginley): suppress logging # Suppress logging by default; enable for local testing ENABLE_LOGGING = True @@ -55,6 +63,15 @@ class ContentVersioningService(BaseModel): ) ) + def model_post_init(self, __context: Any) -> None: + super().model_post_init(__context) + + # Log instance details + self.logger.info( + f"[{self.infrastructure.instance_name} ({self.infrastructure.instance_address})] " + "Initing ContentVersioningService" + ) + # The cached job on the splunk instance of the cms events _cms_main_job: splunklib.Job | None = PrivateAttr(default=None) @@ -167,6 +184,10 @@ def activate_versioning(self) -> None: if not self.is_versioning_activated: raise Exception("Something went wrong, content versioning is still disabled.") + self.logger.info( + f"[{self.infrastructure.instance_name}] Versioning service successfully activated" + ) + @computed_field @cached_property def cms_fields(self) -> list[str]: @@ -178,7 +199,6 @@ def cms_fields(self) -> list[str]: """ return [ "app_name", - f"action.{self.global_config.app.label.lower()}.full_search_name", "detection_id", "version", "action.correlationsearch.label", @@ -214,6 +234,10 @@ def force_cms_parser(self) -> None: if not self.is_cms_parser_enabled: raise Exception("Something went wrong, cms_parser is still disabled.") + self.logger.info( + f"[{self.infrastructure.instance_name}] cms_parser successfully toggled to force run" + ) + def wait_for_cms_main(self) -> None: """ Checks the cms_main index until it has the expected number of events, or it times out. @@ -225,27 +249,36 @@ def wait_for_cms_main(self) -> None: elapsed_sleep_time = 0 num_tries = 0 time_to_sleep = 2**num_tries - max_sleep = 300 + max_sleep = 480 # Loop until timeout while elapsed_sleep_time < max_sleep: # Sleep, and add the time to the elapsed counter - self.logger.info(f"Waiting {time_to_sleep} for cms_parser to finish") + self.logger.info( + f"[{self.infrastructure.instance_name}] Waiting {time_to_sleep} for cms_parser to " + "finish" + ) time.sleep(time_to_sleep) elapsed_sleep_time += time_to_sleep self.logger.info( - f"Checking cms_main (attempt #{num_tries + 1} - {elapsed_sleep_time} seconds elapsed of " - f"{max_sleep} max)" + f"[{self.infrastructure.instance_name}] Checking cms_main (attempt #{num_tries + 1}" + f" - {elapsed_sleep_time} seconds elapsed of {max_sleep} max)" ) # Check if the number of CMS events matches or exceeds the number of detections if self.get_num_cms_events() >= len(self.detections): self.logger.info( - f"Found {self.get_num_cms_events(use_cache=True)} events in cms_main which " + f"[{self.infrastructure.instance_name}] Found " + f"{self.get_num_cms_events(use_cache=True)} events in cms_main which " f"meets or exceeds the expected {len(self.detections)}." ) break - + else: + self.logger.info( + f"[{self.infrastructure.instance_name}] Found " + f"{self.get_num_cms_events(use_cache=True)} matching events in cms_main; " + f"expecting {len(self.detections)}. Continuing to wait..." + ) # Update the number of times we've tried, and increment the time to sleep num_tries += 1 time_to_sleep = 2**num_tries @@ -278,7 +311,7 @@ def _query_cms_main(self, use_cache: bool = False) -> splunklib.Job: f"search index=cms_main app_name=\"{self.global_config.app.appid}\" | " f"fields {', '.join(self.cms_fields)}" ) - self.logger.debug(f"Query on cms_main: {query}") + self.logger.debug(f"[{self.infrastructure.instance_name}] Query on cms_main: {query}") # Get the job as a blocking operation, set the cache, and return self._cms_main_job = self.service.search(query, exec_mode="blocking") # type: ignore @@ -317,14 +350,14 @@ def validate_content_against_cms(self) -> None: # Generate an error for the count mismatch if result_count != len(self.detections): msg = ( - f"Expected {len(self.detections)} matching events in cms_main, but " - f"found {result_count}." + f"[{self.infrastructure.instance_name}] Expected {len(self.detections)} matching " + f"events in cms_main, but found {result_count}." ) self.logger.error(msg) exceptions.append(Exception(msg)) self.logger.info( - f"Expecting {len(self.detections)} matching events in cms_main, " - f"found {result_count}." + f"[{self.infrastructure.instance_name}] Expecting {len(self.detections)} matching " + f"events in cms_main, found {result_count}." ) # Init some counters and a mapping of detections to their names @@ -351,30 +384,34 @@ def validate_content_against_cms(self) -> None: # Get the name of the search in the CMS event and attempt to use pattern matching # to strip the prefix and suffix used for the savedsearches.conf name so we can # compare to the detection - cms_entry_name = cms_event["sourcetype"] - self.logger.info(f"{offset}: Matching cms_main entry '{cms_entry_name}' against detections") - ptrn = re.compile(r"^" + self.global_config.app.label + r" - (?P.+) - Rule$") - match = ptrn.match(cms_event["sourcetype"]) + cms_entry_name = cms_event["action.correlationsearch.label"] + self.logger.info( + f"[{self.infrastructure.instance_name}] {offset}: Matching cms_main entry " + f"'{cms_entry_name}' against detections" + ) + ptrn = re.compile(r"^" + self.global_config.app.label + r" - (?P.+) - Rule$") + match = ptrn.match(cms_event["action.correlationsearch.label"]) # Report any errors extracting the detection name from the longer rule name if match is None: msg = ( - f"[{cms_entry_name}]: Entry in cms_main did not match the expected naming " - "scheme; cannot compare to our detections." + f"[{self.infrastructure.instance_name}] [{cms_entry_name}]: Entry in " + "cms_main did not match the expected naming scheme; cannot compare to our " + "detections." ) self.logger.error(msg) exceptions.append(Exception(msg)) continue # Extract the detection name if matching was successful - cms_entry_name = match.group("cms_entry_name") + stripped_cms_entry_name = match.group("stripped_cms_entry_name") # If CMS entry name matches one of the detections already matched, we've got an # unexpected repeated entry - if cms_entry_name in matched_detections: + if stripped_cms_entry_name in matched_detections: msg = ( - f"[{cms_entry_name}]: Detection appears more than once in the cms_main " - "index." + f"[{self.infrastructure.instance_name}] [{stripped_cms_entry_name}]: Detection " + f"appears more than once in the cms_main index." ) self.logger.error(msg) exceptions.append(Exception(msg)) @@ -385,10 +422,10 @@ def validate_content_against_cms(self) -> None: for detection_name in remaining_detections: # If we find a match, break this loop, set the found flag and move the detection # from those that still need to matched to those already matched - if cms_entry_name == detection_name: + if stripped_cms_entry_name == detection_name: self.logger.info( - f"{offset}: Succesfully matched cms_main entry against detection " - f"('{detection_name}')!" + f"[{self.infrastructure.instance_name}] {offset}: Succesfully matched " + f"cms_main entry against detection ('{detection_name}')!" ) # Validate other fields of the cms_event against the detection @@ -410,8 +447,8 @@ def validate_content_against_cms(self) -> None: # Generate an exception if we couldn't match the CMS main entry to a detection if result_matches_detection is False: msg = ( - f"[{cms_entry_name}]: Could not match entry in cms_main against any " - "of the expected detections." + f"[{self.infrastructure.instance_name}] [{stripped_cms_entry_name}]: Could not " + "match entry in cms_main against any of the expected detections." ) self.logger.error(msg) exceptions.append(Exception(msg)) @@ -422,8 +459,8 @@ def validate_content_against_cms(self) -> None: # Generate exceptions for the unmatched detections for detection_name in remaining_detections: msg = ( - f"[{detection_name}]: Detection not found in cms_main; there may be an " - "issue with savedsearches.conf" + f"[{self.infrastructure.instance_name}] [{detection_name}]: Detection not " + "found in cms_main; there may be an issue with savedsearches.conf" ) self.logger.error(msg) exceptions.append(Exception(msg)) @@ -436,7 +473,10 @@ def validate_content_against_cms(self) -> None: ) # Else, we've matched/validated all detections against cms_main - self.logger.info("Matched and validated all detections against cms_main!") + self.logger.info( + f"[{self.infrastructure.instance_name}] Matched and validated all detections against " + "cms_main!" + ) def validate_detection_against_cms_event( self, @@ -458,47 +498,41 @@ def validate_detection_against_cms_event( # TODO (cmcginley): validate additional fields between the cms_event and the detection cms_uuid = uuid.UUID(cms_event["detection_id"]) - full_search_key = f"action.{self.global_config.app.label.lower()}.full_search_name" rule_name_from_detection = f"{self.global_config.app.label} - {detection.name} - Rule" # Compare the UUIDs if cms_uuid != detection.id: msg = ( - f"[{detection.name}]: UUID in cms_event ('{cms_uuid}') does not match UUID in " - f"detection ('{detection.id}')" + f"[{self.infrastructure.instance_name}] [{detection.name}]: UUID in cms_event " + f"('{cms_uuid}') does not match UUID in detection ('{detection.id}')" ) self.logger.error(msg) return Exception(msg) - elif cms_event["version"] != f"{detection.version}-1": - # Compare the versions (we append '-1' to the detection version to be in line w/ the + elif cms_event["version"] != f"{detection.version}.1": + # Compare the versions (we append '.1' to the detection version to be in line w/ the # internal representation in ES) msg = ( - f"[{detection.name}]: Version in cms_event ('{cms_event['version']}') does not " - f"match version in detection ('{detection.version}-1')" - ) - self.logger.error(msg) - return Exception(msg) - elif cms_event[full_search_key] != rule_name_from_detection: - # Compare the full search name - msg = ( - f"[{detection.name}]: Full search name in cms_event " - f"('{cms_event[full_search_key]}') does not match detection name" + f"[{self.infrastructure.instance_name}] [{detection.name}]: Version in cms_event " + f"('{cms_event['version']}') does not match version in detection " + f"('{detection.version}.1')" ) self.logger.error(msg) return Exception(msg) - elif cms_event["action.correlationsearch.label"] != f"{self.global_config.app.label} - {detection.name} - Rule": + elif cms_event["action.correlationsearch.label"] != rule_name_from_detection: # Compare the correlation search label msg = ( - f"[{detection.name}]: Correlation search label in cms_event " - f"('{cms_event['action.correlationsearch.label']}') does not match detection name" + f"[{self.infrastructure.instance_name}][{detection.name}]: Correlation search " + f"label in cms_event ('{cms_event['action.correlationsearch.label']}') does not " + "match detection name" ) self.logger.error(msg) return Exception(msg) - elif cms_event["sourcetype"] != f"{self.global_config.app.label} - {detection.name} - Rule": + elif cms_event["sourcetype"] != "stash_common_detection_model": # Compare the full search name msg = ( - f"[{detection.name}]: Sourcetype in cms_event ('{cms_event[f'sourcetype']}') does " - f"not match detection name" + f"[{self.infrastructure.instance_name}] [{detection.name}]: Unexpected sourcetype " + f"in cms_event ('{cms_event[f'sourcetype']}'); expected " + "'stash_common_detection_model'" ) self.logger.error(msg) return Exception(msg) From 4a5cc2faa7488b538da05fb190289429d118f2c8 Mon Sep 17 00:00:00 2001 From: Casey McGinley Date: Tue, 29 Oct 2024 15:40:07 -0700 Subject: [PATCH 10/24] adding error filtering --- .../objects/content_versioning_service.py | 18 ++++++++++----- contentctl/objects/correlation_search.py | 22 ++++++++++++++++--- 2 files changed, 32 insertions(+), 8 deletions(-) diff --git a/contentctl/objects/content_versioning_service.py b/contentctl/objects/content_versioning_service.py index 6ebc064c..fc9eb08e 100644 --- a/contentctl/objects/content_versioning_service.py +++ b/contentctl/objects/content_versioning_service.py @@ -19,10 +19,12 @@ # TODO (cmcginley): # - [x] version naming scheme seems to have changed from X - X to X.X # - [x] sourcetype no longer holds detection name but instead is stash_common_detection_model -# - [ ] action.escu.full_search_name no longer available -# - [ ] check to see if we can get "name" +# - [x] action.escu.full_search_name no longer available +# - [x] check to see if we can get "name" # - [ ] move strings to enums -# - [ ] additionally, timeout for cms_parser seems to need more time +# - [x] additionally, timeout for cms_parser seems to need more time +# - [ ] validate multi-line fields -> search, description, action.notable.param.rule_description, +# action.notable.param.drilldown_searches # TODO (cmcginley): suppress logging # Suppress logging by default; enable for local testing @@ -366,14 +368,20 @@ def validate_content_against_cms(self) -> None: remaining_detections = {x.name: x for x in self.detections} matched_detections: dict[str, Detection] = {} + # Create a filter for a specific memory error we're ok ignoring + sub_second_order_pattern = re.compile( + r".*Events might not be returned in sub-second order due to search memory limits.*" + ) + # Iterate over the results until we've gone through them all while offset < result_count: iterator = ResultIterator( - job.results( # type: ignore + response_reader=job.results( # type: ignore output_mode="json", count=count, offset=offset - ) + ), + error_filters=[sub_second_order_pattern] ) # Iterate over the currently fetched results diff --git a/contentctl/objects/correlation_search.py b/contentctl/objects/correlation_search.py index 8fa3d534..8a27ebf3 100644 --- a/contentctl/objects/correlation_search.py +++ b/contentctl/objects/correlation_search.py @@ -1,6 +1,7 @@ import logging import time import json +import re from typing import Any from enum import Enum from functools import cached_property @@ -34,7 +35,7 @@ # Suppress logging by default; enable for local testing -ENABLE_LOGGING = False +ENABLE_LOGGING = True LOG_LEVEL = logging.DEBUG LOG_PATH = "correlation_search.log" @@ -93,15 +94,25 @@ class ResultIterator: Given a ResponseReader, constructs a JSONResultsReader and iterates over it; when Message instances are encountered, they are logged if the message is anything other than "error", in which case an error is raised. Regular results are returned as expected + :param response_reader: a ResponseReader object - :param logger: a Logger object + :type response_reader: :class:`splunklib.binding.ResponseReader` + :param error_filters: set of re Patterns used to filter out errors we're ok ignoring + :type error_filters: list[:class:`re.Pattern[str]`] """ - def __init__(self, response_reader: ResponseReader) -> None: + def __init__( + self, + response_reader: ResponseReader, + error_filters: list[re.Pattern[str]] = [] + ) -> None: # init the results reader self.results_reader: JSONResultsReader = JSONResultsReader( response_reader ) + # the list of patterns for errors to ignore + self.error_filters: list[re.Pattern[str]] = error_filters + # get logger self.logger: logging.Logger = Utils.get_logger( __name__, @@ -127,6 +138,11 @@ def __next__(self) -> dict[str, Any]: message = f"SPLUNK: {result.message}" self.logger.log(level, message) if level == logging.ERROR: + # if the error matches any of the filters, just continue on + for filter in self.error_filters: + if filter.match(message): + continue + # if no filter was matched, raise raise ServerError(message) # if dict, just return From 1f2431f0c8fb80e254cffe4d5199a51411765727 Mon Sep 17 00:00:00 2001 From: Casey McGinley Date: Tue, 29 Oct 2024 18:44:03 -0700 Subject: [PATCH 11/24] bugfix fitlering --- contentctl/objects/correlation_search.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/contentctl/objects/correlation_search.py b/contentctl/objects/correlation_search.py index 8a27ebf3..6328a6f5 100644 --- a/contentctl/objects/correlation_search.py +++ b/contentctl/objects/correlation_search.py @@ -137,13 +137,19 @@ def __next__(self) -> dict[str, Any]: # log message at appropriate level and raise if needed message = f"SPLUNK: {result.message}" self.logger.log(level, message) + filtered = False if level == logging.ERROR: - # if the error matches any of the filters, just continue on + # if the error matches any of the filters, flag it for filter in self.error_filters: - if filter.match(message): - continue + self.logger.debug(f"Filter: {filter}; message: {message}") + if filter.match(message) is not None: + self.logger.debug(f"Error matched filter {filter}; continuing") + filtered = True + break + # if no filter was matched, raise - raise ServerError(message) + if not filtered: + raise ServerError(message) # if dict, just return elif isinstance(result, dict): From aac149c7950f153177a7f54f0e9b57d11cdbc1bb Mon Sep 17 00:00:00 2001 From: Casey McGinley Date: Wed, 30 Oct 2024 00:32:36 -0700 Subject: [PATCH 12/24] bumping timeout --- contentctl/objects/content_versioning_service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contentctl/objects/content_versioning_service.py b/contentctl/objects/content_versioning_service.py index fc9eb08e..34534731 100644 --- a/contentctl/objects/content_versioning_service.py +++ b/contentctl/objects/content_versioning_service.py @@ -251,7 +251,7 @@ def wait_for_cms_main(self) -> None: elapsed_sleep_time = 0 num_tries = 0 time_to_sleep = 2**num_tries - max_sleep = 480 + max_sleep = 600 # Loop until timeout while elapsed_sleep_time < max_sleep: From b3c6948488257a66b224774e4f89faa7a487c86d Mon Sep 17 00:00:00 2001 From: Casey McGinley Date: Wed, 30 Oct 2024 10:26:24 -0700 Subject: [PATCH 13/24] TODO (revert): temporarily disabling some v alidations --- .../detection_abstract.py | 37 ++++++++++--------- contentctl/output/conf_writer.py | 5 ++- 2 files changed, 22 insertions(+), 20 deletions(-) diff --git a/contentctl/objects/abstract_security_content_objects/detection_abstract.py b/contentctl/objects/abstract_security_content_objects/detection_abstract.py index 34374a88..110e56ac 100644 --- a/contentctl/objects/abstract_security_content_objects/detection_abstract.py +++ b/contentctl/objects/abstract_security_content_objects/detection_abstract.py @@ -569,25 +569,26 @@ def model_post_init(self, __context: Any) -> None: # 1 of the drilldowns contains the string Drilldown.SEARCH_PLACEHOLDER. # This is presently a requirement when 1 or more drilldowns are added to a detection. # Note that this is only required for production searches that are not hunting + + # TODO (cmcginley): commenting out for testing + # if self.type == AnalyticsType.Hunting.value or self.status != DetectionStatus.production.value: + # #No additional check need to happen on the potential drilldowns. + # pass + # else: + # found_placeholder = False + # if len(self.drilldown_searches) < 2: + # raise ValueError(f"This detection is required to have 2 drilldown_searches, but only has [{len(self.drilldown_searches)}]") + # for drilldown in self.drilldown_searches: + # if DRILLDOWN_SEARCH_PLACEHOLDER in drilldown.search: + # found_placeholder = True + # if not found_placeholder: + # raise ValueError("Detection has one or more drilldown_searches, but none of them " + # f"contained '{DRILLDOWN_SEARCH_PLACEHOLDER}. This is a requirement " + # "if drilldown_searches are defined.'") - if self.type == AnalyticsType.Hunting.value or self.status != DetectionStatus.production.value: - #No additional check need to happen on the potential drilldowns. - pass - else: - found_placeholder = False - if len(self.drilldown_searches) < 2: - raise ValueError(f"This detection is required to have 2 drilldown_searches, but only has [{len(self.drilldown_searches)}]") - for drilldown in self.drilldown_searches: - if DRILLDOWN_SEARCH_PLACEHOLDER in drilldown.search: - found_placeholder = True - if not found_placeholder: - raise ValueError("Detection has one or more drilldown_searches, but none of them " - f"contained '{DRILLDOWN_SEARCH_PLACEHOLDER}. This is a requirement " - "if drilldown_searches are defined.'") - - # Update the search fields with the original search, if required - for drilldown in self.drilldown_searches: - drilldown.perform_search_substitutions(self) + # # Update the search fields with the original search, if required + # for drilldown in self.drilldown_searches: + # drilldown.perform_search_substitutions(self) #For experimental purposes, add the default drilldowns #self.drilldown_searches.extend(Drilldown.constructDrilldownsFromDetection(self)) diff --git a/contentctl/output/conf_writer.py b/contentctl/output/conf_writer.py index 4d2e0490..c5f8e2e0 100644 --- a/contentctl/output/conf_writer.py +++ b/contentctl/output/conf_writer.py @@ -115,8 +115,9 @@ def writeDashboardFiles(config:build, dashboards:list[Dashboard])->set[pathlib.P output_file_path = dashboard.getOutputFilepathRelativeToAppRoot(config) # Check that the full output path does not exist so that we are not having an # name collision with a file in app_template - if (config.getPackageDirectoryPath()/output_file_path).exists(): - raise FileExistsError(f"ERROR: Overwriting Dashboard File {output_file_path}. Does this file exist in {config.getAppTemplatePath()} AND {config.path/'dashboards'}?") + # TODO (cmcginley): commenting out for testing + # if (config.getPackageDirectoryPath()/output_file_path).exists(): + # raise FileExistsError(f"ERROR: Overwriting Dashboard File {output_file_path}. Does this file exist in {config.getAppTemplatePath()} AND {config.path/'dashboards'}?") ConfWriter.writeXmlFileHeader(output_file_path, config) dashboard.writeDashboardFile(ConfWriter.getJ2Environment(), config) From 65f3c81b37894f7f36c644917bc8c8835f0113f3 Mon Sep 17 00:00:00 2001 From: Casey McGinley Date: Thu, 21 Nov 2024 13:01:42 -0800 Subject: [PATCH 14/24] logging tracebacks in verbose mode --- .../DetectionTestingManager.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/contentctl/actions/detection_testing/DetectionTestingManager.py b/contentctl/actions/detection_testing/DetectionTestingManager.py index a1d659ca..1992de2b 100644 --- a/contentctl/actions/detection_testing/DetectionTestingManager.py +++ b/contentctl/actions/detection_testing/DetectionTestingManager.py @@ -1,4 +1,5 @@ from typing import List,Union +import traceback from contentctl.objects.config import test, test_servers, Container,Infrastructure from contentctl.actions.detection_testing.infrastructures.DetectionTestingInfrastructure import DetectionTestingInfrastructure from contentctl.actions.detection_testing.infrastructures.DetectionTestingInfrastructureContainer import DetectionTestingInfrastructureContainer @@ -98,6 +99,10 @@ def sigint_handler(signum, frame): _ = future.result() except Exception as e: self.output_dto.terminate = True + # Output the traceback if we encounter errors in verbose mode + if self.input_dto.config.verbose: + tb = traceback.format_exc() + print(tb) errors["INSTANCE SETUP ERRORS"].append(e) # Start and wait for all tests to run @@ -113,6 +118,10 @@ def sigint_handler(signum, frame): _ = future.result() except Exception as e: self.output_dto.terminate = True + # Output the traceback if we encounter errors in verbose mode + if self.input_dto.config.verbose: + tb = traceback.format_exc() + print(tb) errors["TESTING ERRORS"].append(e) self.output_dto.terminate = True @@ -125,6 +134,10 @@ def sigint_handler(signum, frame): try: _ = future.result() except Exception as e: + # Output the traceback if we encounter errors in verbose mode + if self.input_dto.config.verbose: + tb = traceback.format_exc() + print(tb) errors["ERRORS DURING VIEW SHUTDOWN"].append(e) # Wait for original view-related threads to complete @@ -132,6 +145,10 @@ def sigint_handler(signum, frame): try: _ = future.result() except Exception as e: + # Output the traceback if we encounter errors in verbose mode + if self.input_dto.config.verbose: + tb = traceback.format_exc() + print(tb) errors["ERRORS DURING VIEW EXECUTION"].append(e) # Log any errors From 57fe77517c403cdf6995678aa8a1bfa1700824ed Mon Sep 17 00:00:00 2001 From: Casey McGinley Date: Thu, 21 Nov 2024 13:10:03 -0800 Subject: [PATCH 15/24] Revert "TODO (revert): temporarily disabling some v" This reverts commit b3c6948488257a66b224774e4f89faa7a487c86d. --- .../detection_abstract.py | 37 +++++++++---------- contentctl/output/conf_writer.py | 5 +-- 2 files changed, 20 insertions(+), 22 deletions(-) diff --git a/contentctl/objects/abstract_security_content_objects/detection_abstract.py b/contentctl/objects/abstract_security_content_objects/detection_abstract.py index 738d7622..dc0350d5 100644 --- a/contentctl/objects/abstract_security_content_objects/detection_abstract.py +++ b/contentctl/objects/abstract_security_content_objects/detection_abstract.py @@ -569,26 +569,25 @@ def model_post_init(self, __context: Any) -> None: # 1 of the drilldowns contains the string Drilldown.SEARCH_PLACEHOLDER. # This is presently a requirement when 1 or more drilldowns are added to a detection. # Note that this is only required for production searches that are not hunting - - # TODO (cmcginley): commenting out for testing - # if self.type == AnalyticsType.Hunting.value or self.status != DetectionStatus.production.value: - # #No additional check need to happen on the potential drilldowns. - # pass - # else: - # found_placeholder = False - # if len(self.drilldown_searches) < 2: - # raise ValueError(f"This detection is required to have 2 drilldown_searches, but only has [{len(self.drilldown_searches)}]") - # for drilldown in self.drilldown_searches: - # if DRILLDOWN_SEARCH_PLACEHOLDER in drilldown.search: - # found_placeholder = True - # if not found_placeholder: - # raise ValueError("Detection has one or more drilldown_searches, but none of them " - # f"contained '{DRILLDOWN_SEARCH_PLACEHOLDER}. This is a requirement " - # "if drilldown_searches are defined.'") - # # Update the search fields with the original search, if required - # for drilldown in self.drilldown_searches: - # drilldown.perform_search_substitutions(self) + if self.type == AnalyticsType.Hunting.value or self.status != DetectionStatus.production.value: + #No additional check need to happen on the potential drilldowns. + pass + else: + found_placeholder = False + if len(self.drilldown_searches) < 2: + raise ValueError(f"This detection is required to have 2 drilldown_searches, but only has [{len(self.drilldown_searches)}]") + for drilldown in self.drilldown_searches: + if DRILLDOWN_SEARCH_PLACEHOLDER in drilldown.search: + found_placeholder = True + if not found_placeholder: + raise ValueError("Detection has one or more drilldown_searches, but none of them " + f"contained '{DRILLDOWN_SEARCH_PLACEHOLDER}. This is a requirement " + "if drilldown_searches are defined.'") + + # Update the search fields with the original search, if required + for drilldown in self.drilldown_searches: + drilldown.perform_search_substitutions(self) #For experimental purposes, add the default drilldowns #self.drilldown_searches.extend(Drilldown.constructDrilldownsFromDetection(self)) diff --git a/contentctl/output/conf_writer.py b/contentctl/output/conf_writer.py index a4d17e38..410ce4f6 100644 --- a/contentctl/output/conf_writer.py +++ b/contentctl/output/conf_writer.py @@ -232,9 +232,8 @@ def writeDashboardFiles(config:build, dashboards:list[Dashboard])->set[pathlib.P output_file_path = dashboard.getOutputFilepathRelativeToAppRoot(config) # Check that the full output path does not exist so that we are not having an # name collision with a file in app_template - # TODO (cmcginley): commenting out for testing - # if (config.getPackageDirectoryPath()/output_file_path).exists(): - # raise FileExistsError(f"ERROR: Overwriting Dashboard File {output_file_path}. Does this file exist in {config.getAppTemplatePath()} AND {config.path/'dashboards'}?") + if (config.getPackageDirectoryPath()/output_file_path).exists(): + raise FileExistsError(f"ERROR: Overwriting Dashboard File {output_file_path}. Does this file exist in {config.getAppTemplatePath()} AND {config.path/'dashboards'}?") ConfWriter.writeXmlFileHeader(output_file_path, config) dashboard.writeDashboardFile(ConfWriter.getJ2Environment(), config) From 390c3727bf83b5af3e50e4ed4434b542a7d8629f Mon Sep 17 00:00:00 2001 From: Casey McGinley Date: Mon, 9 Dec 2024 09:36:53 -0800 Subject: [PATCH 16/24] updating TODOs, updating query --- .../objects/content_versioning_service.py | 24 +++---------------- contentctl/objects/correlation_search.py | 2 +- 2 files changed, 4 insertions(+), 22 deletions(-) diff --git a/contentctl/objects/content_versioning_service.py b/contentctl/objects/content_versioning_service.py index 34534731..84ca7a3e 100644 --- a/contentctl/objects/content_versioning_service.py +++ b/contentctl/objects/content_versioning_service.py @@ -16,15 +16,6 @@ from contentctl.objects.correlation_search import ResultIterator from contentctl.helper.utils import Utils -# TODO (cmcginley): -# - [x] version naming scheme seems to have changed from X - X to X.X -# - [x] sourcetype no longer holds detection name but instead is stash_common_detection_model -# - [x] action.escu.full_search_name no longer available -# - [x] check to see if we can get "name" -# - [ ] move strings to enums -# - [x] additionally, timeout for cms_parser seems to need more time -# - [ ] validate multi-line fields -> search, description, action.notable.param.rule_description, -# action.notable.param.drilldown_searches # TODO (cmcginley): suppress logging # Suppress logging by default; enable for local testing @@ -310,8 +301,8 @@ def _query_cms_main(self, use_cache: bool = False) -> splunklib.Job: # Construct the query looking for CMS events matching the content app name query = ( - f"search index=cms_main app_name=\"{self.global_config.app.appid}\" | " - f"fields {', '.join(self.cms_fields)}" + f"search index=cms_main sourcetype=stash_common_detection_model " + f"app_name=\"{self.global_config.app.appid}\" | fields {', '.join(self.cms_fields)}" ) self.logger.debug(f"[{self.infrastructure.instance_name}] Query on cms_main: {query}") @@ -503,7 +494,7 @@ def validate_detection_against_cms_event( :return: The generated exception, or None :rtype: Exception | None """ - # TODO (cmcginley): validate additional fields between the cms_event and the detection + # TODO (PEX-509): validate additional fields between the cms_event and the detection cms_uuid = uuid.UUID(cms_event["detection_id"]) rule_name_from_detection = f"{self.global_config.app.label} - {detection.name} - Rule" @@ -535,14 +526,5 @@ def validate_detection_against_cms_event( ) self.logger.error(msg) return Exception(msg) - elif cms_event["sourcetype"] != "stash_common_detection_model": - # Compare the full search name - msg = ( - f"[{self.infrastructure.instance_name}] [{detection.name}]: Unexpected sourcetype " - f"in cms_event ('{cms_event[f'sourcetype']}'); expected " - "'stash_common_detection_model'" - ) - self.logger.error(msg) - return Exception(msg) return None diff --git a/contentctl/objects/correlation_search.py b/contentctl/objects/correlation_search.py index 6328a6f5..6f819d1a 100644 --- a/contentctl/objects/correlation_search.py +++ b/contentctl/objects/correlation_search.py @@ -131,7 +131,7 @@ def __next__(self) -> dict[str, Any]: if isinstance(result, Message): # convert level string to level int level_name: str = result.type.strip().upper() # type: ignore - # TODO (cmcginley): this method is deprecated; replace with our own enum + # TODO (PEX-510): this method is deprecated; replace with our own enum level: int = logging.getLevelName(level_name) # log message at appropriate level and raise if needed From 8362af754acf327ec00daeb4bd5e189ae5722a84 Mon Sep 17 00:00:00 2001 From: Casey McGinley Date: Tue, 10 Dec 2024 12:28:12 -0800 Subject: [PATCH 17/24] annotating issues --- contentctl/actions/detection_testing/DetectionTestingManager.py | 1 + .../actions/detection_testing/views/DetectionTestingViewCLI.py | 2 +- contentctl/output/conf_output.py | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/contentctl/actions/detection_testing/DetectionTestingManager.py b/contentctl/actions/detection_testing/DetectionTestingManager.py index 1992de2b..1098f088 100644 --- a/contentctl/actions/detection_testing/DetectionTestingManager.py +++ b/contentctl/actions/detection_testing/DetectionTestingManager.py @@ -67,6 +67,7 @@ def sigint_handler(signum, frame): signal.signal(signal.SIGINT, sigint_handler) + # TODO (#337): futures can be hard to maintain/debug; let's consider alternatives with concurrent.futures.ThreadPoolExecutor( max_workers=len(self.input_dto.config.test_instances), ) as instance_pool, concurrent.futures.ThreadPoolExecutor( diff --git a/contentctl/actions/detection_testing/views/DetectionTestingViewCLI.py b/contentctl/actions/detection_testing/views/DetectionTestingViewCLI.py index a5ec8a24..f3b8ebc3 100644 --- a/contentctl/actions/detection_testing/views/DetectionTestingViewCLI.py +++ b/contentctl/actions/detection_testing/views/DetectionTestingViewCLI.py @@ -51,7 +51,7 @@ def showStatus(self, interval: int = 1): while True: summary = self.getSummaryObject() - # TODO (cmcginley): there's a 1-off error here I think (we show one more than we + # TODO (#338): there's a 1-off error here I think (we show one more than we # actually have during testing) total = len( summary.get("tested_detections", []) diff --git a/contentctl/output/conf_output.py b/contentctl/output/conf_output.py index 44aa4d26..6d63d7e1 100644 --- a/contentctl/output/conf_output.py +++ b/contentctl/output/conf_output.py @@ -79,7 +79,7 @@ def writeMiscellaneousAppFiles(self)->set[pathlib.Path]: return written_files - # TODO (cmcginley): we could have a discrepancy between detections tested and those delivered + # TODO (#339): we could have a discrepancy between detections tested and those delivered # based on the jinja2 template # {% if (detection.type == 'TTP' or detection.type == 'Anomaly' or detection.type == 'Hunting' or detection.type == 'Correlation') %} def writeObjects(self, objects: list, type: SecurityContentType = None) -> set[pathlib.Path]: From 5f78f204a6682d70aedfba8ffc9d3e2e6fa5424b Mon Sep 17 00:00:00 2001 From: Casey McGinley Date: Sun, 2 Feb 2025 20:46:25 -0800 Subject: [PATCH 18/24] removing .DS_store --- .DS_Store | Bin 6148 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 .DS_Store diff --git a/.DS_Store b/.DS_Store deleted file mode 100644 index 72f9f747751bd60eb2e90ac70362ee192770d234..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHK-AcnS6i&A3I)|_e1@S82?Zm0uba+$hd;u$Zp)yxHv{;+5cJ^Wn`T+VuK8Vlb zIY}yx%?oct#yN2ECFiFhUrJ8G7~{^^-)F4G7_&eT3l(S<2>Mayq+l$FTzzA}B9^dw z&~PrA4F8b<+`Bc{F{yU$+x~fJJoG~8f&bkbhC!S(8Xvq+EUm1Tt+G|MZrw+jx=BA7 z$4-BAjiU=GLoe_5y~|*b_G%laGD`YEG#IIbz#l-!%~cTjGIiuQ@?(|jYKK*^D!tnF zWU_mB)Na;AyV;r6#iVm^To%|a=e?)!Ig+oMQVxHgl68YQyg_H->R#MY z6w2rU{CRGkLr4q|1H{1cGhj|WtGfJ4r4_UYzZfm5L z7$62_87S&u4eS5e&+q@)BpML|#K2N9z)M}X>%fvsZJk>j)>;922a1AmnZ~aa=%`W* fu~>??K$U=9U Date: Sun, 2 Feb 2025 23:07:39 -0800 Subject: [PATCH 19/24] formatting --- .../DetectionTestingManager.py | 26 +++--- .../DetectionTestingInfrastructure.py | 36 +++++--- contentctl/actions/test.py | 24 +++-- contentctl/helper/utils.py | 11 ++- contentctl/objects/config.py | 21 +++-- .../objects/content_versioning_service.py | 90 ++++++++++--------- contentctl/objects/correlation_search.py | 47 +++++----- contentctl/output/conf_output.py | 1 + 8 files changed, 146 insertions(+), 110 deletions(-) diff --git a/contentctl/actions/detection_testing/DetectionTestingManager.py b/contentctl/actions/detection_testing/DetectionTestingManager.py index 3d9187f9..ae0df1e3 100644 --- a/contentctl/actions/detection_testing/DetectionTestingManager.py +++ b/contentctl/actions/detection_testing/DetectionTestingManager.py @@ -9,15 +9,19 @@ from pydantic import BaseModel from contentctl.actions.detection_testing.infrastructures.DetectionTestingInfrastructure import ( - DetectionTestingInfrastructure, DetectionTestingManagerOutputDto) -from contentctl.actions.detection_testing.infrastructures.DetectionTestingInfrastructureContainer import \ - DetectionTestingInfrastructureContainer -from contentctl.actions.detection_testing.infrastructures.DetectionTestingInfrastructureServer import \ - DetectionTestingInfrastructureServer -from contentctl.actions.detection_testing.views.DetectionTestingView import \ - DetectionTestingView -from contentctl.objects.config import (Container, Infrastructure, test, - test_servers) + DetectionTestingInfrastructure, + DetectionTestingManagerOutputDto, +) +from contentctl.actions.detection_testing.infrastructures.DetectionTestingInfrastructureContainer import ( + DetectionTestingInfrastructureContainer, +) +from contentctl.actions.detection_testing.infrastructures.DetectionTestingInfrastructureServer import ( + DetectionTestingInfrastructureServer, +) +from contentctl.actions.detection_testing.views.DetectionTestingView import ( + DetectionTestingView, +) +from contentctl.objects.config import Container, Infrastructure, test, test_servers from contentctl.objects.detection import Detection from contentctl.objects.enums import PostTestBehavior @@ -160,8 +164,8 @@ def sigint_handler(signum, frame): for error in errors[error_type]: print(f"\t❌ {str(error)}") if isinstance(error, ExceptionGroup): - for suberror in error.exceptions: # type: ignore - print(f"\t\t❌ {str(suberror)}") # type: ignore + for suberror in error.exceptions: # type: ignore + print(f"\t\t❌ {str(suberror)}") # type: ignore print() return self.output_dto diff --git a/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py b/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py index b5fb2729..e8d191d3 100644 --- a/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py +++ b/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py @@ -16,21 +16,30 @@ import requests # type: ignore import splunklib.client as client # type: ignore import tqdm # type: ignore -from pydantic import (BaseModel, ConfigDict, Field, PrivateAttr, - computed_field, dataclasses) +from pydantic import ( + BaseModel, + ConfigDict, + Field, + PrivateAttr, + computed_field, + dataclasses, +) from semantic_version import Version from splunklib.binding import HTTPError # type: ignore from splunklib.results import JSONResultsReader, Message # type: ignore from urllib3 import disable_warnings from contentctl.actions.detection_testing.progress_bar import ( - FinalTestingStates, TestingStates, TestReportingType, format_pbar_string) + FinalTestingStates, + TestingStates, + TestReportingType, + format_pbar_string, +) from contentctl.helper.utils import Utils from contentctl.objects.base_test import BaseTest from contentctl.objects.base_test_result import TestResultStatus from contentctl.objects.config import All, Infrastructure, test_common -from contentctl.objects.content_versioning_service import \ - ContentVersioningService +from contentctl.objects.content_versioning_service import ContentVersioningService from contentctl.objects.correlation_search import CorrelationSearch, PbarData from contentctl.objects.detection import Detection from contentctl.objects.enums import AnalyticsType, PostTestBehavior @@ -140,7 +149,9 @@ def setup(self): self.start_time = time.time() # Init the list of setup functions we always need - primary_setup_functions: list[tuple[Callable[[], None | client.Service], str]] = [ + primary_setup_functions: list[ + tuple[Callable[[], None | client.Service], str] + ] = [ (self.start, "Starting"), (self.get_conn, "Waiting for App Installation"), (self.configure_conf_file_datamodels, "Configuring Datamodels"), @@ -150,7 +161,7 @@ def setup(self): (self.configure_imported_roles, "Configuring Roles"), (self.configure_delete_indexes, "Configuring Indexes"), (self.configure_hec, "Configuring HEC"), - (self.wait_for_ui_ready, "Finishing Primary Setup") + (self.wait_for_ui_ready, "Finishing Primary Setup"), ] # Execute and report on each setup function @@ -173,7 +184,7 @@ def setup(self): TestReportingType.SETUP, self.get_name(), "Beginning Content Versioning Validation...", - set_pbar=False + set_pbar=False, ) ) for func, msg in self.content_versioning_service.setup_functions: @@ -190,7 +201,7 @@ def setup(self): msg = f"[{self.get_name()}]: {str(e)}" self.finish() if isinstance(e, ExceptionGroup): - raise ExceptionGroup(msg, e.exceptions) from e # type: ignore + raise ExceptionGroup(msg, e.exceptions) from e # type: ignore raise Exception(msg) from e self.pbar.write( @@ -198,7 +209,7 @@ def setup(self): TestReportingType.SETUP, self.get_name(), "Finished Setup!", - set_pbar=False + set_pbar=False, ) ) @@ -220,7 +231,7 @@ def content_versioning_service(self) -> ContentVersioningService: global_config=self.global_config, infrastructure=self.infrastructure, service=self.get_conn(), - detections=self.sync_obj.inputQueue + detections=self.sync_obj.inputQueue, ) @property @@ -251,7 +262,7 @@ def es_version(self) -> Version | None: """ if not self.es_installed: return None - return Version(self.get_conn().apps[ES_APP_NAME]["version"]) # type: ignore + return Version(self.get_conn().apps[ES_APP_NAME]["version"]) # type: ignore @property def es_installed(self) -> bool: @@ -400,7 +411,6 @@ def configure_imported_roles( imported_roles: list[str] = ["user", "power", "can_delete"], enterprise_security_roles: list[str] = ["ess_admin", "ess_analyst", "ess_user"], ): - # Set which roles should be configured. For Enterprise Security/Integration Testing, # we must add some extra foles. if self.global_config.enable_integration_testing: diff --git a/contentctl/actions/test.py b/contentctl/actions/test.py index f9d0a061..90ac3951 100644 --- a/contentctl/actions/test.py +++ b/contentctl/actions/test.py @@ -3,15 +3,21 @@ from typing import List from contentctl.actions.detection_testing.DetectionTestingManager import ( - DetectionTestingManager, DetectionTestingManagerInputDto) -from contentctl.actions.detection_testing.infrastructures.DetectionTestingInfrastructure import \ - DetectionTestingManagerOutputDto -from contentctl.actions.detection_testing.views.DetectionTestingViewCLI import \ - DetectionTestingViewCLI -from contentctl.actions.detection_testing.views.DetectionTestingViewFile import \ - DetectionTestingViewFile -from contentctl.actions.detection_testing.views.DetectionTestingViewWeb import \ - DetectionTestingViewWeb + DetectionTestingManager, + DetectionTestingManagerInputDto, +) +from contentctl.actions.detection_testing.infrastructures.DetectionTestingInfrastructure import ( + DetectionTestingManagerOutputDto, +) +from contentctl.actions.detection_testing.views.DetectionTestingViewCLI import ( + DetectionTestingViewCLI, +) +from contentctl.actions.detection_testing.views.DetectionTestingViewFile import ( + DetectionTestingViewFile, +) +from contentctl.actions.detection_testing.views.DetectionTestingViewWeb import ( + DetectionTestingViewWeb, +) from contentctl.objects.config import Changes, Selected from contentctl.objects.config import test as test_ from contentctl.objects.config import test_servers diff --git a/contentctl/helper/utils.py b/contentctl/helper/utils.py index 2ca84896..027c7cae 100644 --- a/contentctl/helper/utils.py +++ b/contentctl/helper/utils.py @@ -491,13 +491,10 @@ def getPercent(numerator: float, denominator: float, decimal_places: int) -> str ratio = numerator / denominator percent = ratio * 100 return Utils.getFixedWidth(percent, decimal_places) + "%" - + @staticmethod def get_logger( - name: str, - log_level: int, - log_path: str, - enable_logging: bool + name: str, log_level: int, log_path: str, enable_logging: bool ) -> logging.Logger: """ Gets a logger instance for the given name; logger is configured if not already configured. @@ -539,7 +536,9 @@ def get_logger( handler = logging.NullHandler() # Format our output - formatter = logging.Formatter('%(asctime)s - %(levelname)s:%(name)s - %(message)s') + formatter = logging.Formatter( + "%(asctime)s - %(levelname)s:%(name)s - %(message)s" + ) handler.setFormatter(formatter) # Set handler level and add to logger diff --git a/contentctl/objects/config.py b/contentctl/objects/config.py index b04021ce..ab01a955 100644 --- a/contentctl/objects/config.py +++ b/contentctl/objects/config.py @@ -12,9 +12,20 @@ import semantic_version import tqdm -from pydantic import (AnyUrl, BaseModel, ConfigDict, DirectoryPath, Field, - FilePath, HttpUrl, PositiveInt, ValidationInfo, - field_serializer, field_validator, model_validator) +from pydantic import ( + AnyUrl, + BaseModel, + ConfigDict, + DirectoryPath, + Field, + FilePath, + HttpUrl, + PositiveInt, + ValidationInfo, + field_serializer, + field_validator, + model_validator, +) from contentctl.helper.splunk_app import SplunkApp from contentctl.helper.utils import Utils @@ -408,8 +419,8 @@ class inspect(build): description=( "[NOTE: enrichments must be ENABLED for inspect to run. Please adjust your config " f"or CLI invocation appropriately] {validate.model_fields['enrichments'].description}" - ) - ) + ), + ) previous_build: str | None = Field( default=None, description=( diff --git a/contentctl/objects/content_versioning_service.py b/contentctl/objects/content_versioning_service.py index 84ca7a3e..811f9461 100644 --- a/contentctl/objects/content_versioning_service.py +++ b/contentctl/objects/content_versioning_service.py @@ -7,9 +7,9 @@ from functools import cached_property from pydantic import BaseModel, PrivateAttr, computed_field, Field -import splunklib.client as splunklib # type: ignore -from splunklib.binding import HTTPError, ResponseReader # type: ignore -from splunklib.data import Record # type: ignore +import splunklib.client as splunklib # type: ignore +from splunklib.binding import HTTPError, ResponseReader # type: ignore +from splunklib.data import Record # type: ignore from contentctl.objects.config import test_common, Infrastructure from contentctl.objects.detection import Detection @@ -48,11 +48,9 @@ class ContentVersioningService(BaseModel): # The logger to use (logs all go to a null pipe unless ENABLE_LOGGING is set to True, so as not # to conflict w/ tqdm) - logger: logging.Logger = Field(default_factory=lambda: Utils.get_logger( - __name__, - LOG_LEVEL, - LOG_PATH, - ENABLE_LOGGING + logger: logging.Logger = Field( + default_factory=lambda: Utils.get_logger( + __name__, LOG_LEVEL, LOG_PATH, ENABLE_LOGGING ) ) @@ -84,7 +82,9 @@ def setup_functions(self) -> list[tuple[Callable[[], None], str]]: (self.validate_content_against_cms, "Validating Against CMS"), ] - def _query_content_versioning_service(self, method: str, body: dict[str, Any] = {}) -> Record: + def _query_content_versioning_service( + self, method: str, body: dict[str, Any] = {} + ) -> Record: """ Queries the SA-ContentVersioning service. Output mode defaults to JSON. @@ -102,17 +102,15 @@ def _query_content_versioning_service(self, method: str, body: dict[str, Any] = # Query the content versioning service try: - response = self.service.request( # type: ignore + response = self.service.request( # type: ignore method=method, path_segment="configs/conf-feature_flags/general", body=body, - app="SA-ContentVersioning" + app="SA-ContentVersioning", ) except HTTPError as e: # Raise on any HTTP errors - raise HTTPError( - f"Error querying content versioning service: {e}" - ) from e + raise HTTPError(f"Error querying content versioning service: {e}") from e return response @@ -132,7 +130,7 @@ def is_versioning_activated(self) -> bool: raise KeyError( f"Cannot retrieve versioning status, 'body' was not found in JSON response: {response}" ) - body: Any = response["body"] # type: ignore + body: Any = response["body"] # type: ignore if not isinstance(body, ResponseReader): raise ValueError( "Cannot retrieve versioning status, value at 'body' in JSON response had an unexpected" @@ -167,15 +165,14 @@ def activate_versioning(self) -> None: """ # Post to the SA-ContentVersioning service to set versioning status self._query_content_versioning_service( - method="POST", - body={ - "versioning_activated": True - } + method="POST", body={"versioning_activated": True} ) # Confirm versioning has been enabled if not self.is_versioning_activated: - raise Exception("Something went wrong, content versioning is still disabled.") + raise Exception( + "Something went wrong, content versioning is still disabled." + ) self.logger.info( f"[{self.infrastructure.instance_name}] Versioning service successfully activated" @@ -195,7 +192,7 @@ def cms_fields(self) -> list[str]: "detection_id", "version", "action.correlationsearch.label", - "sourcetype" + "sourcetype", ] @property @@ -207,17 +204,17 @@ def is_cms_parser_enabled(self) -> bool: :rtype: bool """ # Get the data input entity - cms_parser = self.service.input("data/inputs/cms_parser/main") # type: ignore + cms_parser = self.service.input("data/inputs/cms_parser/main") # type: ignore # Convert the 'disabled' field to an int, then a bool, and then invert to be 'enabled' - return not bool(int(cms_parser.content["disabled"])) # type: ignore + return not bool(int(cms_parser.content["disabled"])) # type: ignore def force_cms_parser(self) -> None: """ Force the cms_parser to being it's run being disabling and re-enabling it. """ # Get the data input entity - cms_parser = self.service.input("data/inputs/cms_parser/main") # type: ignore + cms_parser = self.service.input("data/inputs/cms_parser/main") # type: ignore # Disable and re-enable cms_parser.disable() @@ -302,12 +299,14 @@ def _query_cms_main(self, use_cache: bool = False) -> splunklib.Job: # Construct the query looking for CMS events matching the content app name query = ( f"search index=cms_main sourcetype=stash_common_detection_model " - f"app_name=\"{self.global_config.app.appid}\" | fields {', '.join(self.cms_fields)}" + f'app_name="{self.global_config.app.appid}" | fields {", ".join(self.cms_fields)}' + ) + self.logger.debug( + f"[{self.infrastructure.instance_name}] Query on cms_main: {query}" ) - self.logger.debug(f"[{self.infrastructure.instance_name}] Query on cms_main: {query}") # Get the job as a blocking operation, set the cache, and return - self._cms_main_job = self.service.search(query, exec_mode="blocking") # type: ignore + self._cms_main_job = self.service.search(query, exec_mode="blocking") # type: ignore return self._cms_main_job def get_num_cms_events(self, use_cache: bool = False) -> int: @@ -367,12 +366,10 @@ def validate_content_against_cms(self) -> None: # Iterate over the results until we've gone through them all while offset < result_count: iterator = ResultIterator( - response_reader=job.results( # type: ignore - output_mode="json", - count=count, - offset=offset + response_reader=job.results( # type: ignore + output_mode="json", count=count, offset=offset ), - error_filters=[sub_second_order_pattern] + error_filters=[sub_second_order_pattern], ) # Iterate over the currently fetched results @@ -388,7 +385,11 @@ def validate_content_against_cms(self) -> None: f"[{self.infrastructure.instance_name}] {offset}: Matching cms_main entry " f"'{cms_entry_name}' against detections" ) - ptrn = re.compile(r"^" + self.global_config.app.label + r" - (?P.+) - Rule$") + ptrn = re.compile( + r"^" + + self.global_config.app.label + + r" - (?P.+) - Rule$" + ) match = ptrn.match(cms_event["action.correlationsearch.label"]) # Report any errors extracting the detection name from the longer rule name @@ -429,8 +430,7 @@ def validate_content_against_cms(self) -> None: # Validate other fields of the cms_event against the detection exception = self.validate_detection_against_cms_event( - cms_event, - remaining_detections[detection_name] + cms_event, remaining_detections[detection_name] ) # Save the exception if validation failed @@ -439,7 +439,9 @@ def validate_content_against_cms(self) -> None: # Delete the matched detection and move it to the matched list result_matches_detection = True - matched_detections[detection_name] = remaining_detections[detection_name] + matched_detections[detection_name] = remaining_detections[ + detection_name + ] del remaining_detections[detection_name] break @@ -458,9 +460,9 @@ def validate_content_against_cms(self) -> None: # Generate exceptions for the unmatched detections for detection_name in remaining_detections: msg = ( - f"[{self.infrastructure.instance_name}] [{detection_name}]: Detection not " - "found in cms_main; there may be an issue with savedsearches.conf" - ) + f"[{self.infrastructure.instance_name}] [{detection_name}]: Detection not " + "found in cms_main; there may be an issue with savedsearches.conf" + ) self.logger.error(msg) exceptions.append(Exception(msg)) @@ -468,7 +470,7 @@ def validate_content_against_cms(self) -> None: if len(exceptions) > 0: raise ExceptionGroup( "1 or more issues validating our detections against the cms_main index", - exceptions + exceptions, ) # Else, we've matched/validated all detections against cms_main @@ -478,9 +480,7 @@ def validate_content_against_cms(self) -> None: ) def validate_detection_against_cms_event( - self, - cms_event: dict[str, Any], - detection: Detection + self, cms_event: dict[str, Any], detection: Detection ) -> Exception | None: """ Given an event from the cms_main index and the matched detection, compare fields and look @@ -497,7 +497,9 @@ def validate_detection_against_cms_event( # TODO (PEX-509): validate additional fields between the cms_event and the detection cms_uuid = uuid.UUID(cms_event["detection_id"]) - rule_name_from_detection = f"{self.global_config.app.label} - {detection.name} - Rule" + rule_name_from_detection = ( + f"{self.global_config.app.label} - {detection.name} - Rule" + ) # Compare the UUIDs if cms_uuid != detection.id: diff --git a/contentctl/objects/correlation_search.py b/contentctl/objects/correlation_search.py index 2c1fb35f..bba2a081 100644 --- a/contentctl/objects/correlation_search.py +++ b/contentctl/objects/correlation_search.py @@ -12,15 +12,20 @@ from splunklib.results import JSONResultsReader, Message # type: ignore from tqdm import tqdm # type: ignore -from contentctl.actions.detection_testing.progress_bar import \ - format_pbar_string # type: ignore +from contentctl.actions.detection_testing.progress_bar import format_pbar_string # type: ignore from contentctl.actions.detection_testing.progress_bar import ( - TestingStates, TestReportingType) + TestingStates, + TestReportingType, +) from contentctl.helper.utils import Utils from contentctl.objects.base_test_result import TestResultStatus from contentctl.objects.detection import Detection -from contentctl.objects.errors import (ClientError, IntegrationTestingError, - ServerError, ValidationFailed) +from contentctl.objects.errors import ( + ClientError, + IntegrationTestingError, + ServerError, + ValidationFailed, +) from contentctl.objects.integration_test_result import IntegrationTestResult from contentctl.objects.notable_action import NotableAction from contentctl.objects.notable_event import NotableEvent @@ -97,10 +102,9 @@ class ResultIterator: :param error_filters: set of re Patterns used to filter out errors we're ok ignoring :type error_filters: list[:class:`re.Pattern[str]`] """ + def __init__( - self, - response_reader: ResponseReader, - error_filters: list[re.Pattern[str]] = [] + self, response_reader: ResponseReader, error_filters: list[re.Pattern[str]] = [] ) -> None: # init the results reader self.results_reader: JSONResultsReader = JSONResultsReader(response_reader) @@ -110,10 +114,7 @@ def __init__( # get logger self.logger: logging.Logger = Utils.get_logger( - __name__, - LOG_LEVEL, - LOG_PATH, - ENABLE_LOGGING + __name__, LOG_LEVEL, LOG_PATH, ENABLE_LOGGING ) def __iter__(self) -> "ResultIterator": @@ -125,7 +126,7 @@ def __next__(self) -> dict[str, Any]: # log messages, or raise if error if isinstance(result, Message): # convert level string to level int - level_name: str = result.type.strip().upper() # type: ignore + level_name: str = result.type.strip().upper() # type: ignore # TODO (PEX-510): this method is deprecated; replace with our own enum level: int = logging.getLevelName(level_name) @@ -138,7 +139,9 @@ def __next__(self) -> dict[str, Any]: for filter in self.error_filters: self.logger.debug(f"Filter: {filter}; message: {message}") if filter.match(message) is not None: - self.logger.debug(f"Error matched filter {filter}; continuing") + self.logger.debug( + f"Error matched filter {filter}; continuing" + ) filtered = True break @@ -148,7 +151,7 @@ def __next__(self) -> dict[str, Any]: # if dict, just return elif isinstance(result, dict): - return result # type: ignore + return result # type: ignore # raise for any unexpected types else: @@ -200,13 +203,11 @@ class CorrelationSearch(BaseModel): # The logger to use (logs all go to a null pipe unless ENABLE_LOGGING is set to True, so as not # to conflict w/ tqdm) - logger: logging.Logger = Field(default_factory=lambda: Utils.get_logger( - __name__, - LOG_LEVEL, - LOG_PATH, - ENABLE_LOGGING + logger: logging.Logger = Field( + default_factory=lambda: Utils.get_logger( + __name__, LOG_LEVEL, LOG_PATH, ENABLE_LOGGING ), - init=False + init=False, ) # The set of indexes to clear on cleanup @@ -400,7 +401,9 @@ def _parse_risk_and_notable_actions(self) -> None: ) # grab notable details if present - self._notable_action = CorrelationSearch._get_notable_action(self.saved_search.content) # type: ignore + self._notable_action = CorrelationSearch._get_notable_action( + self.saved_search.content + ) # type: ignore def refresh(self) -> None: """Refreshes the metadata in the SavedSearch entity, and re-parses the fields we care about diff --git a/contentctl/output/conf_output.py b/contentctl/output/conf_output.py index f996d088..bcbff3db 100644 --- a/contentctl/output/conf_output.py +++ b/contentctl/output/conf_output.py @@ -16,6 +16,7 @@ import tarfile from contentctl.objects.config import build + # These must be imported separately because they are not just used for typing, # they are used in isinstance (which requires the object to be imported) from contentctl.objects.lookup import FileBackedLookup, MlModel From e8fa869069e9220bc4c41c964c063fb8b9f84cf3 Mon Sep 17 00:00:00 2001 From: Casey McGinley Date: Sun, 2 Feb 2025 23:29:54 -0800 Subject: [PATCH 20/24] addressing some of Eric's comments --- .../objects/content_versioning_service.py | 79 +++++++------------ 1 file changed, 29 insertions(+), 50 deletions(-) diff --git a/contentctl/objects/content_versioning_service.py b/contentctl/objects/content_versioning_service.py index 811f9461..1a1fa426 100644 --- a/contentctl/objects/content_versioning_service.py +++ b/contentctl/objects/content_versioning_service.py @@ -1,21 +1,20 @@ -import time -import uuid import json -import re import logging -from typing import Any, Callable +import re +import time +import uuid from functools import cached_property +from typing import Any, Callable -from pydantic import BaseModel, PrivateAttr, computed_field, Field import splunklib.client as splunklib # type: ignore +from pydantic import BaseModel, Field, PrivateAttr, computed_field from splunklib.binding import HTTPError, ResponseReader # type: ignore from splunklib.data import Record # type: ignore -from contentctl.objects.config import test_common, Infrastructure -from contentctl.objects.detection import Detection -from contentctl.objects.correlation_search import ResultIterator from contentctl.helper.utils import Utils - +from contentctl.objects.config import Infrastructure, test_common +from contentctl.objects.correlation_search import ResultIterator +from contentctl.objects.detection import Detection # TODO (cmcginley): suppress logging # Suppress logging by default; enable for local testing @@ -151,8 +150,8 @@ def is_versioning_activated(self) -> bool: return bool(int(entry["content"]["versioning_activated"])) except KeyError as e: raise KeyError( - "Cannot retrieve versioning status, unable to versioning status using the expected " - f"keys: {e}" + "Cannot retrieve versioning status, unable to determine versioning status using " + f"the expected keys: {e}" ) from e raise ValueError( "Cannot retrieve versioning status, unable to find an entry matching 'general' in the " @@ -211,7 +210,7 @@ def is_cms_parser_enabled(self) -> bool: def force_cms_parser(self) -> None: """ - Force the cms_parser to being it's run being disabling and re-enabling it. + Force the cms_parser to run by disabling and re-enabling it. """ # Get the data input entity cms_parser = self.service.input("data/inputs/cms_parser/main") # type: ignore @@ -355,7 +354,9 @@ def validate_content_against_cms(self) -> None: # Init some counters and a mapping of detections to their names count = 100 offset = 0 - remaining_detections = {x.name: x for x in self.detections} + remaining_detections = { + x.get_action_dot_correlationsearch_dot_label(self.global_config.app): x for x in self.detections + } matched_detections: dict[str, Detection] = {} # Create a filter for a specific memory error we're ok ignoring @@ -377,40 +378,18 @@ def validate_content_against_cms(self) -> None: # Increment the offset for each result offset += 1 - # Get the name of the search in the CMS event and attempt to use pattern matching - # to strip the prefix and suffix used for the savedsearches.conf name so we can - # compare to the detection + # Get the name of the search in the CMS event cms_entry_name = cms_event["action.correlationsearch.label"] self.logger.info( f"[{self.infrastructure.instance_name}] {offset}: Matching cms_main entry " f"'{cms_entry_name}' against detections" ) - ptrn = re.compile( - r"^" - + self.global_config.app.label - + r" - (?P.+) - Rule$" - ) - match = ptrn.match(cms_event["action.correlationsearch.label"]) - - # Report any errors extracting the detection name from the longer rule name - if match is None: - msg = ( - f"[{self.infrastructure.instance_name}] [{cms_entry_name}]: Entry in " - "cms_main did not match the expected naming scheme; cannot compare to our " - "detections." - ) - self.logger.error(msg) - exceptions.append(Exception(msg)) - continue - - # Extract the detection name if matching was successful - stripped_cms_entry_name = match.group("stripped_cms_entry_name") # If CMS entry name matches one of the detections already matched, we've got an # unexpected repeated entry - if stripped_cms_entry_name in matched_detections: + if cms_entry_name in matched_detections: msg = ( - f"[{self.infrastructure.instance_name}] [{stripped_cms_entry_name}]: Detection " + f"[{self.infrastructure.instance_name}] [{cms_entry_name}]: Detection " f"appears more than once in the cms_main index." ) self.logger.error(msg) @@ -419,18 +398,18 @@ def validate_content_against_cms(self) -> None: # Iterate over the detections and compare the CMS entry name against each result_matches_detection = False - for detection_name in remaining_detections: + for detection_cs_label in remaining_detections: # If we find a match, break this loop, set the found flag and move the detection # from those that still need to matched to those already matched - if stripped_cms_entry_name == detection_name: + if cms_entry_name == detection_cs_label: self.logger.info( f"[{self.infrastructure.instance_name}] {offset}: Succesfully matched " - f"cms_main entry against detection ('{detection_name}')!" + f"cms_main entry against detection ('{detection_cs_label}')!" ) # Validate other fields of the cms_event against the detection exception = self.validate_detection_against_cms_event( - cms_event, remaining_detections[detection_name] + cms_event, remaining_detections[detection_cs_label] ) # Save the exception if validation failed @@ -439,16 +418,16 @@ def validate_content_against_cms(self) -> None: # Delete the matched detection and move it to the matched list result_matches_detection = True - matched_detections[detection_name] = remaining_detections[ - detection_name + matched_detections[detection_cs_label] = remaining_detections[ + detection_cs_label ] - del remaining_detections[detection_name] + del remaining_detections[detection_cs_label] break # Generate an exception if we couldn't match the CMS main entry to a detection if result_matches_detection is False: msg = ( - f"[{self.infrastructure.instance_name}] [{stripped_cms_entry_name}]: Could not " + f"[{self.infrastructure.instance_name}] [{cms_entry_name}]: Could not " "match entry in cms_main against any of the expected detections." ) self.logger.error(msg) @@ -458,9 +437,9 @@ def validate_content_against_cms(self) -> None: # cms_main and there may have been a parsing issue with savedsearches.conf if len(remaining_detections) > 0: # Generate exceptions for the unmatched detections - for detection_name in remaining_detections: + for detection_cs_label in remaining_detections: msg = ( - f"[{self.infrastructure.instance_name}] [{detection_name}]: Detection not " + f"[{self.infrastructure.instance_name}] [{detection_cs_label}]: Detection not " "found in cms_main; there may be an issue with savedsearches.conf" ) self.logger.error(msg) @@ -497,8 +476,8 @@ def validate_detection_against_cms_event( # TODO (PEX-509): validate additional fields between the cms_event and the detection cms_uuid = uuid.UUID(cms_event["detection_id"]) - rule_name_from_detection = ( - f"{self.global_config.app.label} - {detection.name} - Rule" + rule_name_from_detection = detection.get_action_dot_correlationsearch_dot_label( + self.global_config.app ) # Compare the UUIDs From e4cadcb107f31bb7b7afa930a7a5c0a2999d1e2e Mon Sep 17 00:00:00 2001 From: Casey McGinley Date: Mon, 3 Feb 2025 14:32:22 -0800 Subject: [PATCH 21/24] resolving TODOs --- .../infrastructures/DetectionTestingInfrastructure.py | 1 - contentctl/objects/content_versioning_service.py | 6 ++---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py b/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py index e8d191d3..34002a2c 100644 --- a/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py +++ b/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py @@ -1572,7 +1572,6 @@ def hec_raw_replay( def status(self): pass - # TODO (cmcginley): the finish function doesn't actually stop execution def finish(self): self.pbar.bar_format = ( f"Finished running tests on instance: [{self.get_name()}]" diff --git a/contentctl/objects/content_versioning_service.py b/contentctl/objects/content_versioning_service.py index 1a1fa426..b8466376 100644 --- a/contentctl/objects/content_versioning_service.py +++ b/contentctl/objects/content_versioning_service.py @@ -22,9 +22,6 @@ LOG_LEVEL = logging.DEBUG LOG_PATH = "content_versioning_service.log" -# TODO (cmcginley): would it be better for this to only run on one instance? Or to consolidate -# error reporting at least? - class ContentVersioningService(BaseModel): """ @@ -355,7 +352,8 @@ def validate_content_against_cms(self) -> None: count = 100 offset = 0 remaining_detections = { - x.get_action_dot_correlationsearch_dot_label(self.global_config.app): x for x in self.detections + x.get_action_dot_correlationsearch_dot_label(self.global_config.app): x + for x in self.detections } matched_detections: dict[str, Detection] = {} From 5b9c05b76538d24bf16ae0a96a7616ecbf7c029b Mon Sep 17 00:00:00 2001 From: Casey McGinley Date: Mon, 3 Feb 2025 14:33:45 -0800 Subject: [PATCH 22/24] removing a stale TODO --- .../infrastructures/DetectionTestingInfrastructure.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py b/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py index 34002a2c..7e912199 100644 --- a/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py +++ b/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py @@ -1276,8 +1276,6 @@ def retry_search_until_timeout( # on a field. In this case, the field will appear but will not contain any values current_empty_fields: set[str] = set() - # TODO (cmcginley): @ljstella is this something we're keeping for testing as - # well? for field in full_rba_field_set: if result.get(field, "null") == "null": if field in risk_object_fields_set: From 71b3d45570d81f801d7cc6be1fef07bfc71d8089 Mon Sep 17 00:00:00 2001 From: Casey McGinley Date: Mon, 3 Feb 2025 17:17:16 -0800 Subject: [PATCH 23/24] reordering the conditionals --- .../objects/content_versioning_service.py | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/contentctl/objects/content_versioning_service.py b/contentctl/objects/content_versioning_service.py index b8466376..6a761f9a 100644 --- a/contentctl/objects/content_versioning_service.py +++ b/contentctl/objects/content_versioning_service.py @@ -478,8 +478,17 @@ def validate_detection_against_cms_event( self.global_config.app ) - # Compare the UUIDs - if cms_uuid != detection.id: + # Compare the correlation search label + if cms_event["action.correlationsearch.label"] != rule_name_from_detection: + msg = ( + f"[{self.infrastructure.instance_name}][{detection.name}]: Correlation search " + f"label in cms_event ('{cms_event['action.correlationsearch.label']}') does not " + "match detection name" + ) + self.logger.error(msg) + return Exception(msg) + elif cms_uuid != detection.id: + # Compare the UUIDs msg = ( f"[{self.infrastructure.instance_name}] [{detection.name}]: UUID in cms_event " f"('{cms_uuid}') does not match UUID in detection ('{detection.id}')" @@ -496,14 +505,5 @@ def validate_detection_against_cms_event( ) self.logger.error(msg) return Exception(msg) - elif cms_event["action.correlationsearch.label"] != rule_name_from_detection: - # Compare the correlation search label - msg = ( - f"[{self.infrastructure.instance_name}][{detection.name}]: Correlation search " - f"label in cms_event ('{cms_event['action.correlationsearch.label']}') does not " - "match detection name" - ) - self.logger.error(msg) - return Exception(msg) return None From fd9b0cc9ff7eb4031fc81a7dfe5b477768f303b7 Mon Sep 17 00:00:00 2001 From: Casey McGinley Date: Thu, 6 Mar 2025 11:26:43 -0800 Subject: [PATCH 24/24] disabling logging --- contentctl/objects/content_versioning_service.py | 3 +-- contentctl/objects/correlation_search.py | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/contentctl/objects/content_versioning_service.py b/contentctl/objects/content_versioning_service.py index 6a761f9a..68a529ba 100644 --- a/contentctl/objects/content_versioning_service.py +++ b/contentctl/objects/content_versioning_service.py @@ -16,9 +16,8 @@ from contentctl.objects.correlation_search import ResultIterator from contentctl.objects.detection import Detection -# TODO (cmcginley): suppress logging # Suppress logging by default; enable for local testing -ENABLE_LOGGING = True +ENABLE_LOGGING = False LOG_LEVEL = logging.DEBUG LOG_PATH = "content_versioning_service.log" diff --git a/contentctl/objects/correlation_search.py b/contentctl/objects/correlation_search.py index bba2a081..4306cc8e 100644 --- a/contentctl/objects/correlation_search.py +++ b/contentctl/objects/correlation_search.py @@ -12,10 +12,10 @@ from splunklib.results import JSONResultsReader, Message # type: ignore from tqdm import tqdm # type: ignore -from contentctl.actions.detection_testing.progress_bar import format_pbar_string # type: ignore from contentctl.actions.detection_testing.progress_bar import ( TestingStates, TestReportingType, + format_pbar_string, # type: ignore ) from contentctl.helper.utils import Utils from contentctl.objects.base_test_result import TestResultStatus @@ -33,7 +33,7 @@ from contentctl.objects.risk_event import RiskEvent # Suppress logging by default; enable for local testing -ENABLE_LOGGING = True +ENABLE_LOGGING = False LOG_LEVEL = logging.DEBUG LOG_PATH = "correlation_search.log"