diff --git a/autobot-backend/utils/redis_management/__init__.py b/autobot-backend/utils/redis_management/__init__.py index 6669b2026..21092894f 100644 --- a/autobot-backend/utils/redis_management/__init__.py +++ b/autobot-backend/utils/redis_management/__init__.py @@ -2,63 +2,40 @@ # Copyright (c) 2025 mrveiss # Author: mrveiss """ -Redis Management Package +Redis Management — backward-compatibility shim (Issue #2313). -This package contains the Redis connection management system for AutoBot. -It was split from the monolithic redis_client.py as part of Issue #381. +The canonical implementation has moved to autobot_shared.redis_management. +All existing imports of ``from utils.redis_management import ...`` +continue to work unchanged via this re-export. -Package Structure: -- types.py: Enums (RedisDatabase, ConnectionState) and database mapping -- config.py: RedisConfig, RedisConfigLoader, PoolConfig -- statistics.py: RedisStats, PoolStatistics, ManagerStats, ConnectionMetrics -- connection_manager.py: RedisConnectionManager class - -Usage: - from utils.redis_management import ( - # Enums - RedisDatabase, ConnectionState, - # Configuration - RedisConfig, RedisConfigLoader, PoolConfig, - # Statistics - RedisStats, PoolStatistics, ManagerStats, ConnectionMetrics, - # Manager - RedisConnectionManager, - # Constants - DATABASE_MAPPING, - ) - -For backward compatibility, the original redis_client.py module -still exports all classes and functions directly. +Preferred import: + from autobot_shared.redis_management import RedisConnectionManager """ -# Configuration classes -from .config import PoolConfig, RedisConfig, RedisConfigLoader - -# Connection manager -from .connection_manager import RedisConnectionManager - -# Statistics dataclasses -from .statistics import ConnectionMetrics, ManagerStats, PoolStatistics, RedisStats - -# Types and constants -from .types import DATABASE_MAPPING, ConnectionState, RedisDatabase +from autobot_shared.redis_management import ( # noqa: F401 + DATABASE_MAPPING, + ConnectionMetrics, + ConnectionState, + ManagerStats, + PoolConfig, + PoolStatistics, + RedisConfig, + RedisConfigLoader, + RedisConnectionManager, + RedisDatabase, + RedisStats, +) -# Re-export for convenience __all__ = [ - # Enums "RedisDatabase", "ConnectionState", - # Constants "DATABASE_MAPPING", - # Configuration "RedisConfig", "RedisConfigLoader", "PoolConfig", - # Statistics "RedisStats", "PoolStatistics", "ManagerStats", "ConnectionMetrics", - # Manager "RedisConnectionManager", ] diff --git a/autobot-backend/utils/redis_thread_safety_test.py b/autobot-backend/utils/redis_thread_safety_test.py index 4601fb1f7..55cc61084 100644 --- a/autobot-backend/utils/redis_thread_safety_test.py +++ b/autobot-backend/utils/redis_thread_safety_test.py @@ -275,7 +275,7 @@ async def test_cleanup_doesnt_affect_active_connections(self): manager._max_idle_time_seconds = 300 # 5 minutes # Issue #2211: mock datetime on the module that actually uses it. - import utils.redis_management.connection_manager as cm_module + import autobot_shared.redis_management.connection_manager as cm_module original_datetime = cm_module.datetime diff --git a/autobot-shared/redis_client.py b/autobot-shared/redis_client.py index 7c041c68a..2124a6ab3 100644 --- a/autobot-shared/redis_client.py +++ b/autobot-shared/redis_client.py @@ -92,20 +92,20 @@ # constants.threshold_constants, config, monitoring.prometheus_metrics). # Moving redis_management/ to autobot-shared would require pulling in those # backend modules too — the dependency chain is deep. -# For now this coupling is accepted and documented here. -# See issue #2220 for full analysis and discussion of moving redis_management -# to autobot-shared as the preferred long-term fix. -from utils.redis_management.config import PoolConfig, RedisConfig, RedisConfigLoader -from utils.redis_management.connection_manager import RedisConnectionManager -from utils.redis_management.statistics import ( +# redis_management now lives in autobot-shared (Issue #2313). +from autobot_shared.redis_management.config import ( + PoolConfig, + RedisConfig, + RedisConfigLoader, +) +from autobot_shared.redis_management.connection_manager import RedisConnectionManager +from autobot_shared.redis_management.statistics import ( ConnectionMetrics, ManagerStats, PoolStatistics, RedisStats, ) - -# Import all types, data models, and classes from the package (Issue #381 refactoring) -from utils.redis_management.types import ( +from autobot_shared.redis_management.types import ( DATABASE_MAPPING, ConnectionState, RedisDatabase, diff --git a/autobot-shared/redis_management/__init__.py b/autobot-shared/redis_management/__init__.py new file mode 100644 index 000000000..7844f168e --- /dev/null +++ b/autobot-shared/redis_management/__init__.py @@ -0,0 +1,57 @@ +# AutoBot - AI-Powered Automation Platform +# Copyright (c) 2025 mrveiss +# Author: mrveiss +""" +Redis Management Package (#2313). + +Canonical location for Redis connection management. Moved from +autobot-backend/utils/redis_management/ to autobot-shared/ so all +components (backend, SLM backend, standalone scripts) share the same code. + +Package Structure: +- types.py: Enums (RedisDatabase, ConnectionState) and database mapping +- config.py: RedisConfig, RedisConfigLoader, PoolConfig +- statistics.py: RedisStats, PoolStatistics, ManagerStats, ConnectionMetrics +- connection_manager.py: RedisConnectionManager class + +Usage: + from autobot_shared.redis_management import ( + RedisDatabase, ConnectionState, + RedisConfig, RedisConfigLoader, PoolConfig, + RedisStats, PoolStatistics, ManagerStats, ConnectionMetrics, + RedisConnectionManager, + DATABASE_MAPPING, + ) +""" + +# Configuration classes +from .config import PoolConfig, RedisConfig, RedisConfigLoader + +# Connection manager +from .connection_manager import RedisConnectionManager + +# Statistics dataclasses +from .statistics import ConnectionMetrics, ManagerStats, PoolStatistics, RedisStats + +# Types and constants +from .types import DATABASE_MAPPING, ConnectionState, RedisDatabase + +# Re-export for convenience +__all__ = [ + # Enums + "RedisDatabase", + "ConnectionState", + # Constants + "DATABASE_MAPPING", + # Configuration + "RedisConfig", + "RedisConfigLoader", + "PoolConfig", + # Statistics + "RedisStats", + "PoolStatistics", + "ManagerStats", + "ConnectionMetrics", + # Manager + "RedisConnectionManager", +] diff --git a/autobot-backend/utils/redis_management/config.py b/autobot-shared/redis_management/config.py similarity index 79% rename from autobot-backend/utils/redis_management/config.py rename to autobot-shared/redis_management/config.py index c8082de58..0a1081ad8 100644 --- a/autobot-backend/utils/redis_management/config.py +++ b/autobot-shared/redis_management/config.py @@ -2,14 +2,18 @@ # Copyright (c) 2025 mrveiss # Author: mrveiss """ -Redis Configuration Module +Redis Configuration Module (#2313). Contains configuration classes for Redis connection management: - RedisConfig: Database configuration dataclass - RedisConfigLoader: Multi-source configuration loader - PoolConfig: Connection pool configuration -Extracted from redis_client.py as part of Issue #381 refactoring. +Extracted from redis_client.py (Issue #381). +Moved from autobot-backend/utils/ to autobot-shared/ (Issue #2313). + +Backend-specific constant imports (REDIS_CONFIG, RetryConfig) have been replaced +with their literal values so this module has no backend dependencies. """ import logging @@ -18,12 +22,26 @@ from typing import Any, Dict, Optional import yaml -from constants.network_constants import NetworkConstants -from constants.redis_constants import REDIS_CONFIG -from constants.threshold_constants import RetryConfig + +from autobot_shared.network_constants import NetworkConstants logger = logging.getLogger(__name__) +# --------------------------------------------------------------------------- +# Redis connection defaults (previously from constants.redis_constants.REDIS_CONFIG) +# --------------------------------------------------------------------------- +_SOCKET_TIMEOUT = 5 +_MAX_CONNECTIONS_POOL = 20 +_MIN_CONNECTIONS_POOL = 2 +_HEALTH_CHECK_INTERVAL = 30 +_RETRY_ON_TIMEOUT = True +_CIRCUIT_BREAKER_THRESHOLD = 5 +_CIRCUIT_BREAKER_TIMEOUT = 60 + +# Retry defaults (previously from constants.threshold_constants.RetryConfig) +_DEFAULT_RETRIES = 3 +_BACKOFF_BASE = 2.0 + @dataclass class RedisConfig: @@ -40,14 +58,14 @@ class RedisConfig: port: int = NetworkConstants.REDIS_PORT password: Optional[str] = None decode_responses: bool = True - max_connections: int = REDIS_CONFIG.MAX_CONNECTIONS_POOL - socket_timeout: float = float(REDIS_CONFIG.SOCKET_TIMEOUT) - socket_connect_timeout: float = float(REDIS_CONFIG.SOCKET_TIMEOUT) + max_connections: int = _MAX_CONNECTIONS_POOL + socket_timeout: float = float(_SOCKET_TIMEOUT) + socket_connect_timeout: float = float(_SOCKET_TIMEOUT) socket_keepalive: bool = True socket_keepalive_options: Optional[Dict[int, int]] = None - health_check_interval: int = REDIS_CONFIG.HEALTH_CHECK_INTERVAL - retry_on_timeout: bool = REDIS_CONFIG.RETRY_ON_TIMEOUT - max_retries: int = RetryConfig.DEFAULT_RETRIES + health_check_interval: int = _HEALTH_CHECK_INTERVAL + retry_on_timeout: bool = _RETRY_ON_TIMEOUT + max_retries: int = _DEFAULT_RETRIES description: str = "" # TLS configuration ssl: bool = False @@ -78,7 +96,7 @@ def __post_init__(self): from pathlib import Path cert_dir = os.getenv("AUTOBOT_TLS_CERT_DIR", "certs") - project_root = Path(__file__).parent.parent.parent.parent + project_root = Path(__file__).parent.parent.parent self.ssl_ca_certs = str(project_root / cert_dir / "ca" / "ca-cert.pem") self.ssl_certfile = str( project_root / cert_dir / "main-host" / "server-cert.pem" @@ -144,7 +162,7 @@ def _parse_database_config(db_name: str, db_config: Dict[str, Any]) -> RedisConf socket_keepalive=db_config.get("socket_keepalive", True), health_check_interval=db_config.get("health_check_interval", 30), retry_on_timeout=db_config.get("retry_on_timeout", True), - max_retries=db_config.get("max_retries", RetryConfig.DEFAULT_RETRIES), + max_retries=db_config.get("max_retries", _DEFAULT_RETRIES), description=db_config.get("description", ""), ) @@ -185,7 +203,9 @@ def load_from_yaml(yaml_path: str = None) -> Dict[str, RedisConfig]: } logger.info( - f"Loaded {len(configs)} database configurations from {resolved_path}" + "Loaded %d database configurations from %s", + len(configs), + resolved_path, ) return configs @@ -245,7 +265,7 @@ def load_timeout_config() -> Dict[str, Any]: "socket_timeout": 5.0, "socket_connect_timeout": 5.0, "retry_on_timeout": True, - "max_retries": RetryConfig.DEFAULT_RETRIES, + "max_retries": _DEFAULT_RETRIES, } @@ -257,16 +277,17 @@ class PoolConfig: Controls connection pool behavior including sizing, timeouts, retry logic, and circuit breaker settings. - Issue #611: Values now reference REDIS_CONFIG constants. + Issue #611: Values previously referenced REDIS_CONFIG constants, + now inlined after move to autobot-shared (#2313). """ - max_connections: int = REDIS_CONFIG.MAX_CONNECTIONS_POOL - min_connections: int = REDIS_CONFIG.MIN_CONNECTIONS_POOL - socket_timeout: float = float(REDIS_CONFIG.SOCKET_TIMEOUT) - socket_connect_timeout: float = float(REDIS_CONFIG.SOCKET_TIMEOUT) - retry_on_timeout: bool = REDIS_CONFIG.RETRY_ON_TIMEOUT - max_retries: int = RetryConfig.DEFAULT_RETRIES - backoff_factor: float = RetryConfig.BACKOFF_BASE - health_check_interval: float = float(REDIS_CONFIG.HEALTH_CHECK_INTERVAL) - circuit_breaker_threshold: int = REDIS_CONFIG.CIRCUIT_BREAKER_THRESHOLD - circuit_breaker_timeout: int = REDIS_CONFIG.CIRCUIT_BREAKER_TIMEOUT + max_connections: int = _MAX_CONNECTIONS_POOL + min_connections: int = _MIN_CONNECTIONS_POOL + socket_timeout: float = float(_SOCKET_TIMEOUT) + socket_connect_timeout: float = float(_SOCKET_TIMEOUT) + retry_on_timeout: bool = _RETRY_ON_TIMEOUT + max_retries: int = _DEFAULT_RETRIES + backoff_factor: float = _BACKOFF_BASE + health_check_interval: float = float(_HEALTH_CHECK_INTERVAL) + circuit_breaker_threshold: int = _CIRCUIT_BREAKER_THRESHOLD + circuit_breaker_timeout: int = _CIRCUIT_BREAKER_TIMEOUT diff --git a/autobot-backend/utils/redis_management/connection_manager.py b/autobot-shared/redis_management/connection_manager.py similarity index 95% rename from autobot-backend/utils/redis_management/connection_manager.py rename to autobot-shared/redis_management/connection_manager.py index 6f5729132..2cacacf83 100644 --- a/autobot-backend/utils/redis_management/connection_manager.py +++ b/autobot-shared/redis_management/connection_manager.py @@ -15,6 +15,7 @@ - Async and sync support Extracted from redis_client.py as part of Issue #381 refactoring. +Moved from autobot-backend/utils/ to autobot-shared/ (Issue #2313). """ import asyncio @@ -29,23 +30,65 @@ import redis import redis.asyncio as async_redis -from config import config as config_manager -from constants.network_constants import NetworkConstants -from constants.threshold_constants import RetryConfig, TimingConstants -from monitoring.prometheus_metrics import get_metrics_manager from redis.asyncio.connection import SSLConnection as AsyncSSLConnection from redis.backoff import ExponentialBackoff from redis.connection import ConnectionPool, SSLConnection from redis.exceptions import ConnectionError, ResponseError from redis.retry import Retry -from utils.redis_management.config import PoolConfig, RedisConfig, RedisConfigLoader -from utils.redis_management.statistics import ( + +from autobot_shared.network_constants import NetworkConstants +from autobot_shared.redis_management.config import ( + PoolConfig, + RedisConfig, + RedisConfigLoader, +) +from autobot_shared.redis_management.statistics import ( ConnectionMetrics, ManagerStats, PoolStatistics, RedisStats, ) -from utils.redis_management.types import DATABASE_MAPPING, ConnectionState +from autobot_shared.redis_management.types import DATABASE_MAPPING, ConnectionState + +# --------------------------------------------------------------------------- +# Lazy imports for backend-specific modules (#2313). +# These are optional — available when running under autobot-backend, absent +# in other contexts (SLM backend, standalone scripts). +# --------------------------------------------------------------------------- +_config_manager = None +_metrics_manager_getter = None + +# Retry/timing defaults (previously from constants.threshold_constants) +_DEFAULT_RETRIES = 3 +_BACKOFF_BASE = 2.0 +_STANDARD_DELAY = 1.0 + + +def _get_config_manager(): + """Lazy-load the backend config manager.""" + global _config_manager + if _config_manager is None: + try: + from config import config as cm + + _config_manager = cm + except ImportError: + pass + return _config_manager + + +def _get_metrics_manager(): + """Lazy-load the Prometheus metrics manager.""" + global _metrics_manager_getter + if _metrics_manager_getter is None: + try: + from monitoring.prometheus_metrics import get_metrics_manager as gmm + + _metrics_manager_getter = gmm + except ImportError: + _metrics_manager_getter = lambda: None # noqa: E731 + return _metrics_manager_getter() + logger = logging.getLogger(__name__) @@ -221,7 +264,7 @@ def _init_cleanup_configuration(self): def _load_redis_config(self) -> Dict[str, Any]: """Load Redis configuration from unified config.""" - redis_config = config_manager.get_redis_config() + redis_config = _get_config_manager().get_redis_config() return { "host": redis_config.get("host", NetworkConstants.REDIS_VM_IP), @@ -232,14 +275,14 @@ def _load_redis_config(self) -> Dict[str, Any]: def _load_pool_config(self) -> PoolConfig: """Load pool configuration from unified config.""" - redis_config = config_manager.get_redis_config() + redis_config = _get_config_manager().get_redis_config() return PoolConfig( max_connections=redis_config.get("max_connections", 100), socket_timeout=redis_config.get("socket_timeout", 5.0), socket_connect_timeout=redis_config.get("socket_connect_timeout", 5.0), retry_on_timeout=redis_config.get("retry_on_timeout", True), - max_retries=redis_config.get("max_retries", RetryConfig.DEFAULT_RETRIES), + max_retries=redis_config.get("max_retries", _DEFAULT_RETRIES), health_check_interval=redis_config.get("health_check_interval", 30.0), circuit_breaker_threshold=redis_config.get("circuit_breaker_threshold", 5), circuit_breaker_timeout=redis_config.get("circuit_breaker_timeout", 60), @@ -272,7 +315,7 @@ def _load_configurations(self): socket_timeout=timeout_config.get("socket_timeout", 5.0), socket_connect_timeout=timeout_config.get("socket_connect_timeout", 5.0), retry_on_timeout=timeout_config.get("retry_on_timeout", True), - max_retries=timeout_config.get("max_retries", RetryConfig.DEFAULT_RETRIES), + max_retries=timeout_config.get("max_retries", _DEFAULT_RETRIES), socket_keepalive_options=( self._tcp_keepalive_options if hasattr(self, "_tcp_keepalive_options") @@ -318,7 +361,7 @@ async def _wait_for_redis_ready( f"Redis '{database_name}' loading dataset, waiting... " f"({int((datetime.now() - start_time).total_seconds())}s elapsed)" ) - await asyncio.sleep(TimingConstants.STANDARD_DELAY) + await asyncio.sleep(_STANDARD_DELAY) else: raise except Exception as e: @@ -521,7 +564,7 @@ def _update_stats(self, database_name: str, success: bool, error: str = None): # Record to Prometheus metrics try: - metrics = get_metrics_manager() + metrics = _get_metrics_manager() metrics.record_request( database=database_name, operation="general", success=success ) @@ -588,7 +631,7 @@ def _record_failure(self, database_name: str, error: Exception): # Record circuit breaker event to Prometheus try: - prom_metrics = get_metrics_manager() + prom_metrics = _get_metrics_manager() prom_metrics.record_circuit_breaker_event( database=database_name, event="opened", diff --git a/autobot-backend/utils/redis_management/statistics.py b/autobot-shared/redis_management/statistics.py similarity index 100% rename from autobot-backend/utils/redis_management/statistics.py rename to autobot-shared/redis_management/statistics.py diff --git a/autobot-backend/utils/redis_management/types.py b/autobot-shared/redis_management/types.py similarity index 100% rename from autobot-backend/utils/redis_management/types.py rename to autobot-shared/redis_management/types.py