From 4ff3e3ec6d1ac93caa138063bc61f161b39e339b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Ricks?= Date: Fri, 7 Feb 2025 12:25:55 +0100 Subject: [PATCH 1/2] Fix: Drop the feed version from gvmd data directories by default This change requires gvmd 25.0.0 and makes greenbone-feed-sync compatible with the 24.10 feed finally. The 24.10 feed is used by default with the current major release of greenbone-feed-sync. Older feed versions for gvmd < 24.1.0 can be synced by setting the feed version. For example via the CLI argument `--feed-version 22.04`. gvmd versions between 24.1.0 and 25.0.0 require to download the 24.10 feed into a 22.04 directory and are therefore considered broken. --- greenbone/feed/sync/config.py | 61 ++++++++++++++++++++++++++--------- greenbone/feed/sync/errors.py | 8 ++++- tests/test_config.py | 23 +++---------- tests/test_parser.py | 18 ++--------- 4 files changed, 60 insertions(+), 50 deletions(-) diff --git a/greenbone/feed/sync/config.py b/greenbone/feed/sync/config.py index 40f6e48..28bf7b0 100644 --- a/greenbone/feed/sync/config.py +++ b/greenbone/feed/sync/config.py @@ -5,11 +5,21 @@ import os from dataclasses import dataclass +from multiprocessing import Value from pathlib import Path -from typing import Any, Callable, Iterable, Optional, Protocol, Union +from typing import ( + Any, + Callable, + Generic, + Iterable, + Optional, + Protocol, + TypeVar, + Union, +) from urllib.parse import urlsplit -from greenbone.feed.sync.errors import ConfigFileError +from greenbone.feed.sync.errors import ConfigError, ConfigFileError from greenbone.feed.sync.helper import DEFAULT_FLOCK_WAIT_INTERVAL from greenbone.feed.sync.rsync import ( DEFAULT_RSYNC_COMPRESSION_LEVEL, @@ -56,15 +66,36 @@ def maybe_int(value: Optional[str]) -> Union[int, str, None]: DEFAULT_VERBOSITY = 2 +T = TypeVar("T") +ValuesDict = dict[str, Any] +DefaultValueCallable = Callable[[ValuesDict], Any] +ValueTypeCallable = Callable[[Any], T] + + +def resolve_gvmd_data_destination(values: ValuesDict) -> str: + path = "gvm/data-objects/gvmd" + feed_version: str = values.get("feed-version") # type: ignore[assignment] + str_major, str_minor = feed_version.split(".")[:2] + try: + major, minor = int(str_major), int(str_minor) + except ValueError as e: + raise ConfigError(f"Invalid feed version format: {feed_version}") from e + + return ( + f"{values['destination-prefix']}/{path}" + if major >= 24 and minor >= 10 + else f"{values['destination-prefix']}/{path}/{feed_version}" + ) + @dataclass -class Setting: +class Setting(Generic[T]): config_key: str environment_key: str default_value: Union[str, int, bool, None] - value_type: Callable + value_type: ValueTypeCallable[T] - def resolve(self, values: dict[str, Any]) -> Any: + def resolve(self, values: ValuesDict) -> Optional[T]: value: Any if self.environment_key in os.environ: value = os.environ.get(self.environment_key) @@ -77,13 +108,13 @@ def resolve(self, values: dict[str, Any]) -> Any: @dataclass -class DependentSetting: +class DependentSetting(Generic[T]): config_key: str environment_key: str - default_value: Callable - value_type: Callable + default_value: DefaultValueCallable + value_type: ValueTypeCallable[T] - def resolve(self, values: dict[str, Any]) -> Any: + def resolve(self, values: ValuesDict) -> Optional[T]: if self.environment_key in os.environ: value = os.environ.get(self.environment_key) elif self.config_key in values: @@ -160,12 +191,12 @@ def feed_url(self) -> str: ), ) -# pylint: disable=line-too-long + _DEPENDENT_SETTINGS = ( DependentSetting( "gvmd-data-destination", "GREENBONE_FEED_SYNC_GVMD_DATA_DESTINATION", - lambda values: f"{values['destination-prefix']}/gvm/data-objects/gvmd/{values['feed-version']}/", # noqa: E501 + resolve_gvmd_data_destination, Path, ), DependentSetting( @@ -225,7 +256,7 @@ def feed_url(self) -> str: DependentSetting( "report-formats-destination", "GREENBONE_FEED_SYNC_REPORT_FORMATS_DESTINATION", - lambda values: f"{values['destination-prefix']}/gvm/data-objects/gvmd/{values['feed-version']}/report-formats", # noqa: E501 + lambda values: f"{values['gvmd-data-destination']}/report-formats", Path, ), DependentSetting( @@ -237,7 +268,7 @@ def feed_url(self) -> str: DependentSetting( "scan-configs-destination", "GREENBONE_FEED_SYNC_SCAN_CONFIGS_DESTINATION", - lambda values: f"{values['destination-prefix']}/gvm/data-objects/gvmd/{values['feed-version']}/scan-configs", # noqa: E501 + lambda values: f"{values['gvmd-data-destination']}/scan-configs", Path, ), DependentSetting( @@ -249,7 +280,7 @@ def feed_url(self) -> str: DependentSetting( "port-lists-destination", "GREENBONE_FEED_SYNC_PORT_LISTS_DESTINATION", - lambda values: f"{values['destination-prefix']}/gvm/data-objects/gvmd/{values['feed-version']}/port-lists", # noqa: E501 + lambda values: f"{values['gvmd-data-destination']}/port-lists", Path, ), DependentSetting( @@ -286,7 +317,7 @@ class Config: """ def __init__(self) -> None: - self._config: dict[str, Any] = {} + self._config: ValuesDict = {} def load_from_config_file(self, config_file: Path) -> None: try: diff --git a/greenbone/feed/sync/errors.py b/greenbone/feed/sync/errors.py index 2f6031a..f49ce70 100644 --- a/greenbone/feed/sync/errors.py +++ b/greenbone/feed/sync/errors.py @@ -25,7 +25,13 @@ class GreenboneFeedSyncError(Exception): """ -class ConfigFileError(GreenboneFeedSyncError): +class ConfigError(GreenboneFeedSyncError): + """ + Error while processing a config + """ + + +class ConfigFileError(ConfigError): """ Error while processing a config file """ diff --git a/tests/test_config.py b/tests/test_config.py index a88dce7..3691dd4 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -41,11 +41,7 @@ def test_defaults(self): ) self.assertEqual( values["gvmd-data-destination"], - Path(DEFAULT_DESTINATION_PREFIX) - / "gvm" - / "data-objects" - / "gvmd" - / DEFAULT_FEED_VERSION, + Path(DEFAULT_DESTINATION_PREFIX) / "gvm" / "data-objects" / "gvmd", ) self.assertEqual( values["gvmd-data-url"], @@ -90,7 +86,6 @@ def test_defaults(self): / "gvm" / "data-objects" / "gvmd" - / DEFAULT_FEED_VERSION / "report-formats", ) self.assertEqual( @@ -103,7 +98,6 @@ def test_defaults(self): / "gvm" / "data-objects" / "gvmd" - / DEFAULT_FEED_VERSION / "scan-configs", ) self.assertEqual( @@ -116,7 +110,6 @@ def test_defaults(self): / "gvm" / "data-objects" / "gvmd" - / DEFAULT_FEED_VERSION / "port-lists", ) self.assertEqual( @@ -261,7 +254,7 @@ def test_destination_prefix(self): self.assertEqual(values["destination-prefix"], Path("/opt/lib")) self.assertEqual( values["gvmd-data-destination"], - Path(f"/opt/lib/gvm/data-objects/gvmd/{DEFAULT_FEED_VERSION}"), + Path("/opt/lib/gvm/data-objects/gvmd/"), ) self.assertEqual(values["notus-destination"], Path("/opt/lib/notus")) self.assertEqual( @@ -275,21 +268,15 @@ def test_destination_prefix(self): ) self.assertEqual( values["report-formats-destination"], - Path( - f"/opt/lib/gvm/data-objects/gvmd/{DEFAULT_FEED_VERSION}/report-formats" - ), + Path("/opt/lib/gvm/data-objects/gvmd/report-formats"), ) self.assertEqual( values["scan-configs-destination"], - Path( - f"/opt/lib/gvm/data-objects/gvmd/{DEFAULT_FEED_VERSION}/scan-configs" - ), + Path("/opt/lib/gvm/data-objects/gvmd/scan-configs"), ) self.assertEqual( values["port-lists-destination"], - Path( - f"/opt/lib/gvm/data-objects/gvmd/{DEFAULT_FEED_VERSION}/port-lists" - ), + Path("/opt/lib/gvm/data-objects/gvmd/port-lists"), ) self.assertEqual( values["openvas-lock-file"], diff --git a/tests/test_parser.py b/tests/test_parser.py index 79368af..e4a4193 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -110,11 +110,7 @@ def test_defaults(self): self.assertEqual(args.feed_url, DEFAULT_RSYNC_URL) self.assertEqual( args.gvmd_data_destination, - Path(DEFAULT_DESTINATION_PREFIX) - / "gvm" - / "data-objects" - / "gvmd" - / DEFAULT_FEED_VERSION, + Path(DEFAULT_DESTINATION_PREFIX) / "gvm" / "data-objects" / "gvmd", ) self.assertEqual( args.gvmd_data_url, @@ -158,7 +154,6 @@ def test_defaults(self): / "gvm" / "data-objects" / "gvmd" - / DEFAULT_FEED_VERSION / "report-formats", ) self.assertEqual( @@ -171,7 +166,6 @@ def test_defaults(self): / "gvm" / "data-objects" / "gvmd" - / DEFAULT_FEED_VERSION / "scan-configs", ) self.assertEqual( @@ -184,7 +178,6 @@ def test_defaults(self): / "gvm" / "data-objects" / "gvmd" - / DEFAULT_FEED_VERSION / "port-lists", ) self.assertEqual( @@ -885,11 +878,7 @@ def test_destination_prefix(self): self.assertEqual(args.destination_prefix, Path(destination_prefix)) self.assertEqual( args.gvmd_data_destination, - Path(destination_prefix) - / "gvm" - / "data-objects" - / "gvmd" - / DEFAULT_FEED_VERSION, + Path(destination_prefix) / "gvm" / "data-objects" / "gvmd", ) self.assertEqual( args.notus_destination, @@ -913,7 +902,6 @@ def test_destination_prefix(self): / "gvm" / "data-objects" / "gvmd" - / DEFAULT_FEED_VERSION / "report-formats", ) self.assertEqual( @@ -922,7 +910,6 @@ def test_destination_prefix(self): / "gvm" / "data-objects" / "gvmd" - / DEFAULT_FEED_VERSION / "scan-configs", ) self.assertEqual( @@ -931,7 +918,6 @@ def test_destination_prefix(self): / "gvm" / "data-objects" / "gvmd" - / DEFAULT_FEED_VERSION / "port-lists", ) self.assertEqual( From ee9b3a0c853592b9a1ec8fedaea29f561b45ca27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Ricks?= Date: Fri, 7 Feb 2025 12:58:53 +0100 Subject: [PATCH 2/2] Fix: Drop the feed version from gvmd data directories by default This change requires gvmd 25.0.0 and makes greenbone-feed-sync compatible with the 24.10 feed finally. The 24.10 feed is used by default with the current major release of greenbone-feed-sync. Older feed versions for gvmd < 24.1.0 can be synced by setting the feed version. For example via the CLI argument `--feed-version 22.04`. gvmd versions between 24.1.0 and 25.0.0 require to download the 24.10 feed into a 22.04 directory and are therefore considered broken. --- greenbone/feed/sync/config.py | 1 - 1 file changed, 1 deletion(-) diff --git a/greenbone/feed/sync/config.py b/greenbone/feed/sync/config.py index 28bf7b0..5372700 100644 --- a/greenbone/feed/sync/config.py +++ b/greenbone/feed/sync/config.py @@ -5,7 +5,6 @@ import os from dataclasses import dataclass -from multiprocessing import Value from pathlib import Path from typing import ( Any,