From 78238dcbd74ebe0896812aab2b7811c62b23b269 Mon Sep 17 00:00:00 2001 From: Eric Date: Wed, 26 Mar 2025 11:55:56 -0700 Subject: [PATCH 01/10] More progress toward mapping updates --- contentctl/input/director.py | 1 + contentctl/objects/config.py | 58 ++++++++++++++++++++++++++ contentctl/objects/test_attack_data.py | 37 +++++++++++++++- 3 files changed, 95 insertions(+), 1 deletion(-) diff --git a/contentctl/input/director.py b/contentctl/input/director.py index 5ff88cd4..fcc78381 100644 --- a/contentctl/input/director.py +++ b/contentctl/input/director.py @@ -200,6 +200,7 @@ def createSecurityContent(self, contentType: SecurityContentType) -> None: context={ "output_dto": self.output_dto, "app": self.input_dto.app, + "config": self.input_dto, }, ) self.output_dto.addContentToDictMappings(detection) diff --git a/contentctl/objects/config.py b/contentctl/objects/config.py index ac6cef78..c1c3dfa7 100644 --- a/contentctl/objects/config.py +++ b/contentctl/objects/config.py @@ -252,6 +252,17 @@ class init(Config_Base): ) +class AttackDataCache(BaseModel): + prefix: str = Field( + "This is the prefix that the data must begin with to map to this cache object" + ) + root_folder_name: str = Field( + "This is the root folder name where the attack data should be downloaded to." + ) + # suggested checkout information for our attack_data repo + # curl https://attack-range-attack-data.s3.us-west-2.amazonaws.com/attack_data.tar.zstd | zstd --decompress | tar -x -C datasets + + class validate(Config_Base): model_config = ConfigDict(validate_default=True, arbitrary_types_allowed=True) enrichments: bool = Field( @@ -272,10 +283,57 @@ class validate(Config_Base): default=False, description="Validate latest TA information from Splunkbase" ) + test_data_caches: list[AttackDataCache] = Field( + default=[], + description="A list of attack data that can " + "be used in lieu of the HTTPS download links " + "of each test data file. This cache can significantly " + "increase overall test speed, ensure the correctness of " + "links at 'contentctl validate' time, and reduce errors " + "associated with failed responses from file servers.", + ) + @property def external_repos_path(self) -> pathlib.Path: return self.path / "external_repos" + def map_to_attack_data_cache( + self, filename: HttpUrl | FilePath + ) -> HttpUrl | FilePath: + # If this is simply a link to a file directly, then no mapping + # needs to take place. Return the link to the file. + if isinstance(filename, pathlib.Path): + return filename + + if len(self.test_data_caches) == 0: + return filename + + # Otherwise, this is a URL. See if its prefix matches one of the + # prefixes in the list of caches + for cache in self.test_data_caches: + root_folder_path = self.external_repos_path / cache.root_folder_name + # See if this data file was in that path + if str(filename).startswith(cache.prefix): + new_file_name = str(filename).replace(cache.prefix, "") + new_file_path = root_folder_path / new_file_name + + if not root_folder_path.is_dir(): + # This has not been checked out. If a cache file was listed in the config AND we hit + # on a prefix, it MUST be checked out + raise ValueError( + f"Expected to find cached test data at '{root_folder_path}', but that directory does not exist. If a test uses attack data and a prefix matches that data, then the the cached data MUST exist." + ) + + if not new_file_path.is_file(): + raise ValueError( + f"Expected to find the cached data file '{new_file_name}', but it does not exist in '{root_folder_path}'" + ) + return new_file_path + + raise ValueError( + f"Test data file '{filename}' does not exist in ANY of the following caches. If you are supplying caches, any HTTP file MUST be located in a cache: [{[cache.prefix for cache in self.test_data_caches]}]" + ) + @property def mitre_cti_repo_path(self) -> pathlib.Path: return self.external_repos_path / "cti" diff --git a/contentctl/objects/test_attack_data.py b/contentctl/objects/test_attack_data.py index 5d5f9c80..936a1ff5 100644 --- a/contentctl/objects/test_attack_data.py +++ b/contentctl/objects/test_attack_data.py @@ -1,5 +1,19 @@ from __future__ import annotations -from pydantic import BaseModel, HttpUrl, FilePath, Field, ConfigDict + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from contentctl.objects.config import validate + +from pydantic import ( + BaseModel, + ConfigDict, + Field, + FilePath, + HttpUrl, + ValidationInfo, + field_validator, +) class TestAttackData(BaseModel): @@ -11,3 +25,24 @@ class TestAttackData(BaseModel): sourcetype: str = Field(...) custom_index: str | None = None host: str | None = None + + @field_validator("data", mode="after") + @classmethod + def check_for_existence_of_attack_data_repo( + cls, value: HttpUrl | FilePath, info: ValidationInfo + ) -> HttpUrl | FilePath: + # this appears to be called more than once, the first time + # info.context is always None. In this case, just return what + # was passed. + if not info.context: + return value + + # When the config is passed, used it to determine if we can map + # the test data to a file on disk + if info.context.get("config", None): + config: validate = info.context.get("config", None) + return config.map_to_attack_data_cache(value) + else: + raise ValueError( + "config not passed to TestAttackData constructor in context" + ) From 1b1f567724c49d7b60c0726908c460d231fa53e5 Mon Sep 17 00:00:00 2001 From: Eric Date: Tue, 1 Apr 2025 13:39:33 -0700 Subject: [PATCH 02/10] better error text --- contentctl/objects/config.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/contentctl/objects/config.py b/contentctl/objects/config.py index a2775397..43a44166 100644 --- a/contentctl/objects/config.py +++ b/contentctl/objects/config.py @@ -318,6 +318,7 @@ def map_to_attack_data_cache( for cache in self.test_data_caches: root_folder_path = self.external_repos_path / cache.root_folder_name # See if this data file was in that path + if str(filename).startswith(cache.prefix): new_file_name = str(filename).replace(cache.prefix, "") new_file_path = root_folder_path / new_file_name @@ -336,7 +337,7 @@ def map_to_attack_data_cache( return new_file_path raise ValueError( - f"Test data file '{filename}' does not exist in ANY of the following caches. If you are supplying caches, any HTTP file MUST be located in a cache: [{[cache.prefix for cache in self.test_data_caches]}]" + f"Test data file '{filename}' does not exist in ANY of the following caches. If you are supplying caches, any HTTP file MUST be located in a cache: [{[cache.root_folder_name for cache in self.test_data_caches]}]" ) @property From cb88f89ad59d78b5a4bd4f4c3f2f49c74194b568 Mon Sep 17 00:00:00 2001 From: Eric Date: Wed, 30 Apr 2025 06:41:37 -0700 Subject: [PATCH 03/10] Forgot to include mapping updates --- contentctl/objects/config.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/contentctl/objects/config.py b/contentctl/objects/config.py index 116132d9..1a0f4bbd 100644 --- a/contentctl/objects/config.py +++ b/contentctl/objects/config.py @@ -262,11 +262,11 @@ class init(Config_Base): class AttackDataCache(BaseModel): - prefix: str = Field( - "This is the prefix that the data must begin with to map to this cache object" + base_url: str = Field( + "This is the beginning of a URL that the data must begin with to map to this cache object." ) - root_folder_name: str = Field( - "This is the root folder name where the attack data should be downloaded to." + base_directory_name: str = Field( + "This is the root folder name where the attack data should be downloaded to. Note that this path is releative to the external_repos/ folder." ) # suggested checkout information for our attack_data repo # curl https://attack-range-attack-data.s3.us-west-2.amazonaws.com/attack_data.tar.zstd | zstd --decompress | tar -x -C datasets @@ -330,11 +330,11 @@ def map_to_attack_data_cache( # Otherwise, this is a URL. See if its prefix matches one of the # prefixes in the list of caches for cache in self.test_data_caches: - root_folder_path = self.external_repos_path / cache.root_folder_name + root_folder_path = self.external_repos_path / cache.base_directory_name # See if this data file was in that path - if str(filename).startswith(cache.prefix): - new_file_name = str(filename).replace(cache.prefix, "") + if str(filename).startswith(cache.base_url): + new_file_name = str(filename).replace(cache.base_url, "") new_file_path = root_folder_path / new_file_name if not root_folder_path.is_dir(): @@ -351,7 +351,7 @@ def map_to_attack_data_cache( return new_file_path raise ValueError( - f"Test data file '{filename}' does not exist in ANY of the following caches. If you are supplying caches, any HTTP file MUST be located in a cache: [{[cache.root_folder_name for cache in self.test_data_caches]}]" + f"Test data file '{filename}' does not exist in ANY of the following caches. If you are supplying caches, any HTTP file MUST be located in a cache: [{[cache.base_directory_name for cache in self.test_data_caches]}]" ) @property From 6af561c72d40a84d1af4d2f318649417eac9216f Mon Sep 17 00:00:00 2001 From: Eric Date: Thu, 1 May 2025 10:22:10 -0700 Subject: [PATCH 04/10] some tweaks to generated errors --- contentctl/objects/config.py | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/contentctl/objects/config.py b/contentctl/objects/config.py index 1a0f4bbd..7a2af996 100644 --- a/contentctl/objects/config.py +++ b/contentctl/objects/config.py @@ -266,7 +266,8 @@ class AttackDataCache(BaseModel): "This is the beginning of a URL that the data must begin with to map to this cache object." ) base_directory_name: str = Field( - "This is the root folder name where the attack data should be downloaded to. Note that this path is releative to the external_repos/ folder." + "This is the root folder name where the attack data should be downloaded to. Note that this path MUST be in the external_repos/ folder", + pattern=r"^external_repos/.+", ) # suggested checkout information for our attack_data repo # curl https://attack-range-attack-data.s3.us-west-2.amazonaws.com/attack_data.tar.zstd | zstd --decompress | tar -x -C datasets @@ -330,7 +331,7 @@ def map_to_attack_data_cache( # Otherwise, this is a URL. See if its prefix matches one of the # prefixes in the list of caches for cache in self.test_data_caches: - root_folder_path = self.external_repos_path / cache.base_directory_name + root_folder_path = self.path / cache.base_directory_name # See if this data file was in that path if str(filename).startswith(cache.base_url): @@ -341,17 +342,29 @@ def map_to_attack_data_cache( # This has not been checked out. If a cache file was listed in the config AND we hit # on a prefix, it MUST be checked out raise ValueError( - f"Expected to find cached test data at '{root_folder_path}', but that directory does not exist. If a test uses attack data and a prefix matches that data, then the the cached data MUST exist." + f"The following directory does not exist: [{root_folder_path}]. " + "You must check out this directory since you have supplied 1 or more test_data_caches in contentctl.yml." ) if not new_file_path.is_file(): raise ValueError( - f"Expected to find the cached data file '{new_file_name}', but it does not exist in '{root_folder_path}'" + f"The following file does not the test_data_cache {cache.base_directory_name}:\n" + f"\tFile: {new_file_name}" ) return new_file_path + print("raising here\n") + x = "\n\t".join( + [ + f"base_url{index:02}: {url}" + for index, url in enumerate( + [cache.base_url for cache in self.test_data_caches] + ) + ] + ) raise ValueError( - f"Test data file '{filename}' does not exist in ANY of the following caches. If you are supplying caches, any HTTP file MUST be located in a cache: [{[cache.base_directory_name for cache in self.test_data_caches]}]" + "Test data file does not start with any of the following base_url's. If you are supplying caches, any HTTP file MUST be located in a cache:\n" + f"\tFile : {filename}\n\t{x}" ) @property From f167048e9e7f4163eae860994aa8fd8b8a771934 Mon Sep 17 00:00:00 2001 From: Eric Date: Thu, 1 May 2025 15:55:21 -0700 Subject: [PATCH 05/10] final cleanup to make attack data cache work easy to use, intuitive, and minimally intrusive for the average content developer --- contentctl/contentctl.py | 1 + .../detection_abstract.py | 4 +- contentctl/objects/config.py | 124 ++++++++++++++---- contentctl/objects/test_attack_data.py | 2 +- 4 files changed, 100 insertions(+), 31 deletions(-) diff --git a/contentctl/contentctl.py b/contentctl/contentctl.py index 1612d835..c0f37d9a 100644 --- a/contentctl/contentctl.py +++ b/contentctl/contentctl.py @@ -68,6 +68,7 @@ def init_func(config: test): def validate_func(config: validate) -> DirectorOutputDto: + config.check_test_data_caches() validate = Validate() return validate.execute(config) diff --git a/contentctl/objects/abstract_security_content_objects/detection_abstract.py b/contentctl/objects/abstract_security_content_objects/detection_abstract.py index 6db0f01c..81e8f737 100644 --- a/contentctl/objects/abstract_security_content_objects/detection_abstract.py +++ b/contentctl/objects/abstract_security_content_objects/detection_abstract.py @@ -913,7 +913,7 @@ def search_rba_fields_exist_validate(self): return self @field_validator("tests", mode="before") - def ensure_yml_test_is_unittest(cls, v: list[dict]): + def ensure_yml_test_is_unittest(cls, v: list[dict], info: ValidationInfo): """The typing for the tests field allows it to be one of a number of different types of tests. However, ONLY UnitTest should be allowed to be defined in the YML @@ -941,7 +941,7 @@ def ensure_yml_test_is_unittest(cls, v: list[dict]): for unitTest in v: # This raises a ValueError on a failed UnitTest. try: - UnitTest.model_validate(unitTest) + UnitTest.model_validate(unitTest, context=info.context) except ValueError as e: valueErrors.append(e) if len(valueErrors): diff --git a/contentctl/objects/config.py b/contentctl/objects/config.py index 7a2af996..791718ea 100644 --- a/contentctl/objects/config.py +++ b/contentctl/objects/config.py @@ -26,6 +26,7 @@ field_validator, model_validator, ) +from requests import RequestException, head from contentctl.helper.splunk_app import SplunkApp from contentctl.helper.utils import Utils @@ -261,6 +262,13 @@ class init(Config_Base): ) +# There can be a number of attack data file warning mapping exceptions, or errors, +# that can occur when using attack data caches. In order to avoid very complex +# output, we will only emit the verbose versions of these message once per file. +# This is a non-intuitive place to put this, but it is good enough for now. +ATTACK_DATA_CACHE_MAPPING_EXCEPTIONS: set[str] = set() + + class AttackDataCache(BaseModel): base_url: str = Field( "This is the beginning of a URL that the data must begin with to map to this cache object." @@ -270,7 +278,19 @@ class AttackDataCache(BaseModel): pattern=r"^external_repos/.+", ) # suggested checkout information for our attack_data repo - # curl https://attack-range-attack-data.s3.us-west-2.amazonaws.com/attack_data.tar.zstd | zstd --decompress | tar -x -C datasets + # curl https://attack-range-attack-data.s3.us-west-2.amazonaws.com/attack_data.tar.zstd | zstd --decompress | tar -x -C attack_data/ + # suggested YML values for this: + helptext: str | None = Field( + default="This repo is set up to use test_data_caches. This can be extremely helpful in validating correct links for test attack_data and speeding up testing.\n" + "Include the following in your contentctl.yml file to use this cache:\n\n" + "test_data_caches:\n" + "- base_url: https://media.githubusercontent.com/media/splunk/attack_data/master/\n" + " base_directory_name: external_repos/attack_data\n\n" + "In order to check out STRT Attack Data, you can use the following command:\n" + "mkdir -p external_repos; curl https://attack-range-attack-data.s3.us-west-2.amazonaws.com/attack_data.tar.zstd | zstd --decompress | tar -x -C external_repos/\n" + "or\n" + """echo "First ensure git-lfs is enabled"; git clone https://github.com/splunk/attack_data external_repos/attack_data""" + ) class validate(Config_Base): @@ -317,9 +337,36 @@ class validate(Config_Base): def external_repos_path(self) -> pathlib.Path: return self.path / "external_repos" + # We can't make this a validator because the constructor + # is called many times - we don't want to print this out many times. + def check_test_data_caches(self) -> Self: + """ + Check that the test data caches actually exist at the specified paths. + If they do exist, then do nothing. If they do not, then emit the helpext, but + do not raise an exception. They are not required, but can significantly speed up + and reduce the flakiness of tests by reducing failed HTTP requests. + """ + if not self.verbose: + # Ignore the check and error output if we are not in verbose mode + return self + + for cache in self.test_data_caches: + cache_path = self.path / cache.base_directory_name + if not cache_path.is_dir(): + print(cache.helptext) + else: + print(f"Found attack data cache at {cache_path}") + return self + def map_to_attack_data_cache( - self, filename: HttpUrl | FilePath + self, filename: HttpUrl | FilePath, verbose: bool = False ) -> HttpUrl | FilePath: + if str(filename) in ATTACK_DATA_CACHE_MAPPING_EXCEPTIONS: + # This is already something that we have emitted a warning or + # Exception for. We don't want to emit it again as it will + # pollute the output. + return filename + # If this is simply a link to a file directly, then no mapping # needs to take place. Return the link to the file. if isinstance(filename, pathlib.Path): @@ -339,33 +386,54 @@ def map_to_attack_data_cache( new_file_path = root_folder_path / new_file_name if not root_folder_path.is_dir(): - # This has not been checked out. If a cache file was listed in the config AND we hit - # on a prefix, it MUST be checked out - raise ValueError( - f"The following directory does not exist: [{root_folder_path}]. " - "You must check out this directory since you have supplied 1 or more test_data_caches in contentctl.yml." - ) - - if not new_file_path.is_file(): - raise ValueError( - f"The following file does not the test_data_cache {cache.base_directory_name}:\n" - f"\tFile: {new_file_name}" + # This has not been checked out. Even though we want to use this cache + # whenever possible, we don't want to force it. + return filename + + if new_file_path.is_file(): + # We found the file in the cache. Return the new path + return new_file_path + + # Any thing below here is non standard behavior that will produce either a warning message, + # an error, or both. We onyl want to do this once for each file, even if it is used + # across multiple different detections. + ATTACK_DATA_CACHE_MAPPING_EXCEPTIONS.add(str(filename)) + + # The cache exists, but we didn't find the file. We will emit an informational warning + # for this, but this is not an exception. Instead, we will just fall back to using + # the original URL. + if verbose: + # Give some extra context about missing attack data files/bad mapping + try: + h = head(str(filename)) + h.raise_for_status() + + except RequestException: + raise ValueError( + f"Error resolving the attack_data file {filename}. " + f"It was missing from the cache {cache.base_directory_name} and a download from the server failed." + ) + print( + f"\nFilename {filename} not found in cache {cache.base_directory_name}, but exists on the server. " + f"Your cache {cache.base_directory_name} may be out of date." ) - return new_file_path - - print("raising here\n") - x = "\n\t".join( - [ - f"base_url{index:02}: {url}" - for index, url in enumerate( - [cache.base_url for cache in self.test_data_caches] - ) - ] - ) - raise ValueError( - "Test data file does not start with any of the following base_url's. If you are supplying caches, any HTTP file MUST be located in a cache:\n" - f"\tFile : {filename}\n\t{x}" - ) + return filename + if verbose: + # Any thing below here is non standard behavior that will produce either a warning message, + # an error, or both. We onyl want to do this once for each file, even if it is used + # across multiple different detections. + ATTACK_DATA_CACHE_MAPPING_EXCEPTIONS.add(str(filename)) + + # Give some extra context about missing attack data files/bad mapping + url = f"Attack Data : {filename}" + prefixes = "".join( + [ + f"\n Valid Prefix: {cache.base_url}" + for cache in self.test_data_caches + ] + ) + print(f"\nAttack Data Missing from all caches:\n{url}{prefixes}") + return filename @property def mitre_cti_repo_path(self) -> pathlib.Path: diff --git a/contentctl/objects/test_attack_data.py b/contentctl/objects/test_attack_data.py index 936a1ff5..c06bb14b 100644 --- a/contentctl/objects/test_attack_data.py +++ b/contentctl/objects/test_attack_data.py @@ -41,7 +41,7 @@ def check_for_existence_of_attack_data_repo( # the test data to a file on disk if info.context.get("config", None): config: validate = info.context.get("config", None) - return config.map_to_attack_data_cache(value) + return config.map_to_attack_data_cache(value, verbose=config.verbose) else: raise ValueError( "config not passed to TestAttackData constructor in context" From d36f6596c5636deabdbc44e48936cc6792347cb6 Mon Sep 17 00:00:00 2001 From: Eric Date: Thu, 1 May 2025 16:22:15 -0700 Subject: [PATCH 06/10] If data is not in the cache AND at a different prefix, ensure that it exists via a head request. If it does not give an exception. otherwise, ensure just print out that it exists but is not in a cache --- contentctl/objects/config.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/contentctl/objects/config.py b/contentctl/objects/config.py index 791718ea..de160423 100644 --- a/contentctl/objects/config.py +++ b/contentctl/objects/config.py @@ -432,7 +432,20 @@ def map_to_attack_data_cache( for cache in self.test_data_caches ] ) - print(f"\nAttack Data Missing from all caches:\n{url}{prefixes}") + # Give some extra context about missing attack data files/bad mapping + try: + h = head(str(filename)) + h.raise_for_status() + except RequestException: + raise ValueError( + f"Error resolving the attack_data file {filename}. It was missing from all caches and a download from the server failed.\n" + f"{url}{prefixes}\n" + ) + + print( + f"\nAttack Data Missing from all caches, but present at URL:\n{url}{prefixes}" + ) + return filename @property From d22c8bace539dc1df1987509a8908526daac1230 Mon Sep 17 00:00:00 2001 From: Eric Date: Thu, 1 May 2025 16:51:45 -0700 Subject: [PATCH 07/10] remove copy of local file to be replayed. this is not required anymore and is a waste of resources because these files are no longer modified in any way during a test. we used to sometimes modify these with update_timestamps which is no longer supported. --- .../infrastructures/DetectionTestingInfrastructure.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py b/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py index 5f9fbdae..30e91e97 100644 --- a/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py +++ b/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py @@ -7,7 +7,6 @@ import time import urllib.parse import uuid -from shutil import copyfile from ssl import SSLEOFError, SSLZeroReturnError from sys import stdout from tempfile import TemporaryDirectory, mktemp @@ -1402,7 +1401,6 @@ def replay_attack_data_file( f"The only valid indexes on the server are {self.all_indexes_on_server}" ) - tempfile = mktemp(dir=tmp_dir) if not ( str(attack_data_file.data).startswith("http://") or str(attack_data_file.data).startswith("https://") @@ -1415,13 +1413,7 @@ def replay_attack_data_file( test_group_start_time, ) - try: - copyfile(str(attack_data_file.data), tempfile) - except Exception as e: - raise Exception( - f"Error copying local Attack Data File for [{test_group.name}] - [{attack_data_file.data}]: " - f"{str(e)}" - ) + tempfile = str(attack_data_file.data) else: raise Exception( f"Attack Data File for [{test_group.name}] is local [{attack_data_file.data}], but does not exist." @@ -1432,6 +1424,7 @@ def replay_attack_data_file( # We need to overwrite the file - mkstemp will create an empty file with the # given name try: + tempfile = mktemp(dir=tmp_dir) # In case the path is a local file, try to get it self.format_pbar_string( From ed2b4f8a501bb4778a01e57358cf3c98bbda298f Mon Sep 17 00:00:00 2001 From: Eric Date: Fri, 2 May 2025 12:09:55 -0700 Subject: [PATCH 08/10] print build date and githash info if those files are available in the cache --- contentctl/objects/config.py | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/contentctl/objects/config.py b/contentctl/objects/config.py index de160423..91f1cfb6 100644 --- a/contentctl/objects/config.py +++ b/contentctl/objects/config.py @@ -349,13 +349,33 @@ def check_test_data_caches(self) -> Self: if not self.verbose: # Ignore the check and error output if we are not in verbose mode return self - for cache in self.test_data_caches: cache_path = self.path / cache.base_directory_name if not cache_path.is_dir(): print(cache.helptext) else: - print(f"Found attack data cache at {cache_path}") + build_date_file = cache_path / "cache_build_date.txt" + git_hash_file = cache_path / "git_hash.txt" + + if build_date_file.is_file(): + # This is a cache that was built by contentctl. We can use this to + # determine if the cache is out of date. + with open(build_date_file, "r") as f: + build_date = f"\n**Cache Build Date: {f.read().strip()}" + else: + build_date = "" + if git_hash_file.is_file(): + # This is a cache that was built by contentctl. We can use this to + # determine if the cache is out of date. + with open(git_hash_file, "r") as f: + git_hash = f"\n**Repo Git Hash : {f.read().strip()}" + else: + git_hash = "" + + print( + f"Found attack data cache at [{cache_path}]{build_date}{git_hash}\n" + ) + return self def map_to_attack_data_cache( From 366eee214b13b0b53b6943ede85df866382b9ae7 Mon Sep 17 00:00:00 2001 From: Eric Date: Fri, 2 May 2025 12:19:15 -0700 Subject: [PATCH 09/10] add unknown date and hash when the relevant files are missing from the attack_data cache --- contentctl/objects/config.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/contentctl/objects/config.py b/contentctl/objects/config.py index 91f1cfb6..bffa66d6 100644 --- a/contentctl/objects/config.py +++ b/contentctl/objects/config.py @@ -361,19 +361,19 @@ def check_test_data_caches(self) -> Self: # This is a cache that was built by contentctl. We can use this to # determine if the cache is out of date. with open(build_date_file, "r") as f: - build_date = f"\n**Cache Build Date: {f.read().strip()}" + build_date = f.read().strip() else: - build_date = "" + build_date = "" if git_hash_file.is_file(): # This is a cache that was built by contentctl. We can use this to # determine if the cache is out of date. with open(git_hash_file, "r") as f: - git_hash = f"\n**Repo Git Hash : {f.read().strip()}" + git_hash = f.read().strip() else: - git_hash = "" + git_hash = "" print( - f"Found attack data cache at [{cache_path}]{build_date}{git_hash}\n" + f"Found attack data cache at [{cache_path}]\n**Cache Build Date: {build_date}\n**Repo Git Hash : {git_hash}\n" ) return self From 8b489a07dd7b7f67c4f505478fbba9493ae11193 Mon Sep 17 00:00:00 2001 From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com> Date: Fri, 2 May 2025 14:43:59 -0700 Subject: [PATCH 10/10] bump version in prep for release --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 69035274..392b8405 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [tool.poetry] name = "contentctl" -version = "5.4.1" +version = "5.5.0" description = "Splunk Content Control Tool" authors = ["STRT "]