feat: Implement adaptive learning rate scheduling

model.py

@@ -1,6 +1,7 @@
 import torch
 import torch.nn as nn
 import torch.optim as optim
+import torch.optim.lr_scheduler
 import numpy as np
 from collections import deque
 import random
@@ -101,6 +102,7 @@ def __init__(self, input_size=12, hidden_size=256, output_size=4,
 
         # Оптимизатор и функция потерь
         self.optimizer = optim.Adam(self.policy_net.parameters(), lr=learning_rate)
+        self.scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(self.optimizer, 'max', patience=10, factor=0.1, verbose=True)
         self.criterion = nn.MSELoss(reduction='none')  # Используем reduction='none' для PER
 
         # Параметры обучения
@@ -194,12 +196,16 @@ def train(self):
     def update_target_network(self):
         self.target_net.load_state_dict(self.policy_net.state_dict())
 
+    def step_scheduler(self, metric):
+        self.scheduler.step(metric)
+
     def save_model(self, path):
         """Сохраняет модель и состояние обучения"""
         checkpoint = {
             'policy_net_state_dict': self.policy_net.state_dict(),
             'target_net_state_dict': self.target_net.state_dict(),
             'optimizer_state_dict': self.optimizer.state_dict(),
+            'scheduler_state_dict': self.scheduler.state_dict(),
             'epsilon': self.epsilon,
             'device': str(self.device)
         }
@@ -215,6 +221,10 @@ def load_model(self, path):
         self.policy_net.load_state_dict(checkpoint['policy_net_state_dict'])
         self.target_net.load_state_dict(checkpoint['target_net_state_dict'])
         self.optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
+        if 'scheduler_state_dict' in checkpoint:
+            self.scheduler.load_state_dict(checkpoint['scheduler_state_dict'])
+        else:
+            print(f"Scheduler state not found in checkpoint {path}, using default scheduler state.")
         self.epsilon = checkpoint['epsilon']
 
         # Переносим модели на правильное устройство после загрузки
@@ -264,6 +274,10 @@ def update_target_networks(self):
         for agent in self.agents:
             agent.update_target_network()
 
+    def step_schedulers(self, metric):
+        for agent in self.agents:
+            agent.step_scheduler(metric)
+
     def save_models(self, path):
         """Сохраняет модели всех агентов"""
         for i, agent in enumerate(self.agents):

train_multi.py

@@ -124,6 +124,7 @@ def train():
             scores_window.append(avg_episode_score)
             scores_history.append(avg_episode_score)
             avg_score = np.mean(scores_window)
+            agent.step_schedulers(avg_score)
             avg_scores_history.append(avg_score)
 
             # Выводим прогресс с дополнительной информацией