Release 2025-12-08-20-51

CHANGELOG.md

@@ -1,5 +1,11 @@
 # Release Notes
 
+## [2.0.4] - 2025-12-08
+- **Weights**: SetWeights task now fetches latest weights from backend API
+- **Database**: Migration to backfill agent runs from validator predictions
+- **Tasks**: Run agents task stores execution logs; db cleaner removes exported logs
+- **Scheduler**: Added timeout handling for scheduled tasks
+
 ## [2.0.3] - 2025-12-03
 - **Scoring**: Fill missing predictions with 0.5 for miners without predictions
 - **Architecture**: Sandbox retry mechanism and error handling with comprehensive log exports

docker-compose.prd.yaml

@@ -38,7 +38,7 @@ services:
       --wallet.hotkey ifhkey
       --db.directory /root/infinite_games/database
       --numinous.env prod
-      --sandbox.max_concurrent 25
+      --sandbox.max_concurrent 50
       --sandbox.timeout_seconds 150
       --validator.sync_hour 0
       --logging.debug"

docker-compose.validator.yaml

@@ -37,7 +37,7 @@ services:
       --wallet.hotkey ${WALLET_HOTKEY}
       --db.directory /root/infinite_games/database
       --numinous.env prod
-      --sandbox.max_concurrent 25
+      --sandbox.max_concurrent 50
       --sandbox.timeout_seconds 150
       --logging.debug"
 

neurons/validator/db/operations.py

@@ -1276,6 +1276,36 @@ async def delete_agent_run_logs(self, batch_size: int) -> Iterable[tuple[int]]:
             [AgentRunLogExportedStatus.EXPORTED, batch_size],
         )
 
+    async def delete_agent_runs(self, batch_size: int) -> Iterable[tuple[int]]:
+        return await self.__db_client.delete(
+            """
+                WITH runs_to_delete AS (
+                    SELECT
+                        ROWID
+                    FROM
+                        agent_runs
+                    WHERE
+                        exported = ?
+                        AND datetime(created_at) < datetime(CURRENT_TIMESTAMP, '-7 day')
+                    ORDER BY
+                        ROWID ASC
+                    LIMIT ?
+                )
+                DELETE FROM
+                    agent_runs
+                WHERE
+                    ROWID IN (
+                        SELECT
+                            ROWID
+                        FROM
+                            runs_to_delete
+                    )
+                RETURNING
+                    ROWID
+            """,
+            [AgentRunExportedStatus.EXPORTED, batch_size],
+        )
+
     async def count_runs_for_event_and_agent(
         self,
         unique_event_id: str,

neurons/validator/main.py

@@ -14,6 +14,7 @@
 from neurons.validator.tasks.db_cleaner import DbCleaner
 from neurons.validator.tasks.db_vacuum import DbVacuum
 from neurons.validator.tasks.delete_events import DeleteEvents
+from neurons.validator.tasks.export_agent_run_logs import ExportAgentRunLogs
 from neurons.validator.tasks.export_agent_runs import ExportAgentRuns
 from neurons.validator.tasks.export_predictions import ExportPredictions
 from neurons.validator.tasks.export_scores import ExportScores
@@ -179,6 +180,14 @@ async def main():
         validator_hotkey=validator_hotkey,
     )
 
+    export_agent_run_logs_task = ExportAgentRunLogs(
+        interval_seconds=600.0,
+        batch_size=500,
+        db_operations=db_operations,
+        api_client=numinous_api_client,
+        logger=logger,
+    )
+
     set_weights_task = SetWeights(
         interval_seconds=379.0,
         db_operations=db_operations,
@@ -187,6 +196,7 @@ async def main():
         netuid=bt_netuid,
         subtensor=bt_subtensor,
         wallet=bt_wallet,
+        api_client=numinous_api_client,
     )
 
     db_cleaner_task = DbCleaner(
@@ -211,6 +221,7 @@ async def main():
     scheduler.add(task=run_agents_task)
     scheduler.add(task=export_predictions_task)
     scheduler.add(task=export_agent_runs_task)
+    scheduler.add(task=export_agent_run_logs_task)
     scheduler.add(task=scoring_task)
     scheduler.add(task=metagraph_scoring_task)
     scheduler.add(task=export_scores_task)

neurons/validator/models/chutes.py

@@ -15,6 +15,8 @@ class ChuteModel(StrEnum):
     DEEPSEEK_V3_1 = "deepseek-ai/DeepSeek-V3.1"
     DEEPSEEK_TNG_R1T2_CHIMERA = "tngtech/DeepSeek-TNG-R1T2-Chimera"
     DEEPSEEK_V3_2_EXP = "deepseek-ai/DeepSeek-V3.2-Exp"
+    DEEPSEEK_V3_2 = "deepseek-ai/DeepSeek-V3.2"
+    DEEPSEEK_V3_2_SPECIALE = "deepseek-ai/DeepSeek-V3.2-Speciale"
 
     # Gemma models
     GEMMA_3_4B_IT = "unsloth/gemma-3-4b-it"
@@ -171,6 +173,16 @@ def calculate_cost(self, completion: ChutesCompletion) -> float:
         input_cost=0.25,
         output_cost=0.35,
     ),
+    ChuteModel.DEEPSEEK_V3_2: Chute(
+        name=ChuteModel.DEEPSEEK_V3_2,
+        input_cost=0.27,
+        output_cost=0.41,
+    ),
+    ChuteModel.DEEPSEEK_V3_2_SPECIALE: Chute(
+        name=ChuteModel.DEEPSEEK_V3_2_SPECIALE,
+        input_cost=0.27,
+        output_cost=0.41,
+    ),
     ChuteModel.GLM_4_6: Chute(
         name=ChuteModel.GLM_4_6,
         input_cost=0.4,

neurons/validator/models/numinous_client.py

@@ -239,3 +239,15 @@ class GatewayDesearchWebSearchResponse(WebSearchResponse, GatewayCallResponse):
 
 class GatewayDesearchWebCrawlResponse(WebCrawlResponse, GatewayCallResponse):
     pass
+
+
+class MinerWeight(BaseModel):
+    miner_uid: int
+    miner_hotkey: str
+    aggregated_weight: float
+
+
+class GetWeightsResponse(BaseModel):
+    aggregated_at: datetime
+    weights: typing.List[MinerWeight]
+    count: int

neurons/validator/models/tests/test_weights.py

@@ -0,0 +1,42 @@
+from datetime import datetime, timezone
+
+import pytest
+
+from neurons.validator.models.weights import WeightsModel
+
+
+class TestWeightsModel:
+    def test_weights_model_creation(self):
+        weights = WeightsModel(
+            miner_uid=1,
+            miner_hotkey="5C4hrfjw9nL7fKRAn3RRKqUWewVE5bGVU7VKF5Ut3N7TkPJJ",
+            metagraph_score=0.835,
+            aggregated_at=datetime(2025, 1, 30, 12, 0, 0, tzinfo=timezone.utc),
+        )
+
+        assert weights.miner_uid == 1
+        assert weights.miner_hotkey == "5C4hrfjw9nL7fKRAn3RRKqUWewVE5bGVU7VKF5Ut3N7TkPJJ"
+        assert weights.metagraph_score == 0.835
+        assert weights.aggregated_at == datetime(2025, 1, 30, 12, 0, 0, tzinfo=timezone.utc)
+
+    def test_weights_model_without_aggregated_at(self):
+        weights = WeightsModel(
+            miner_uid=2,
+            miner_hotkey="5Dpqn...",
+            metagraph_score=0.165,
+        )
+
+        assert weights.aggregated_at is None
+
+    def test_weights_model_primary_key(self):
+        weights = WeightsModel(
+            miner_uid=3,
+            miner_hotkey="test_hotkey",
+            metagraph_score=0.5,
+        )
+
+        assert weights.primary_key == ["miner_uid", "miner_hotkey"]
+
+    def test_weights_model_validation(self):
+        with pytest.raises(ValueError):
+            WeightsModel(miner_uid=1, miner_hotkey="test")

neurons/validator/models/weights.py

@@ -0,0 +1,15 @@
+from datetime import datetime
+from typing import Optional
+
+from pydantic import BaseModel
+
+
+class WeightsModel(BaseModel):
+    miner_uid: int
+    miner_hotkey: str
+    metagraph_score: float
+    aggregated_at: Optional[datetime] = None
+
+    @property
+    def primary_key(self):
+        return ["miner_uid", "miner_hotkey"]

neurons/validator/numinous_client/client.py

@@ -16,6 +16,7 @@
     GetEventsDeletedResponse,
     GetEventsResolvedResponse,
     GetEventsResponse,
+    GetWeightsResponse,
     PostAgentLogsRequestBody,
     PostAgentRunsRequestBody,
     PostPredictionsRequestBody,
@@ -309,3 +310,16 @@ async def desearch_ai_search(self, body: dict | DesearchAISearchRequest):
 
                 data = await response.json()
                 return AISearchResponse.model_validate(data)
+
+    async def get_weights(self):
+        auth_headers = self.make_auth_headers(data="")
+
+        async with self.create_session(other_headers=auth_headers) as session:
+            path = "/api/v1/validators/weights"
+
+            async with session.get(path) as response:
+                response.raise_for_status()
+
+                data = await response.json()
+
+                return GetWeightsResponse.model_validate(data)

neurons/validator/numinous_client/tests/test_numinous_client.py

@@ -21,6 +21,7 @@
     GetEventsDeletedResponse,
     GetEventsResolvedResponse,
     GetEventsResponse,
+    GetWeightsResponse,
     PostAgentLogsRequestBody,
     PostAgentRunsRequestBody,
     PostPredictionsRequestBody,
@@ -1085,3 +1086,81 @@ async def test_desearch_ai_search_error_raised(self, client_test_env: NuminousCl
             )
 
             assert e.value.status == status_code
+
+    async def test_get_weights_response(self, client_test_env: NuminousClient):
+        mock_response_data = {
+            "aggregated_at": "2025-01-30T12:00:00Z",
+            "weights": [
+                {
+                    "miner_uid": 0,
+                    "miner_hotkey": "5C4hrfjw9DjXZTzV3MwzrrAr9P1MJhSrvWGWqi1eSuyUpnhM",
+                    "aggregated_weight": 0.5,
+                },
+                {
+                    "miner_uid": 1,
+                    "miner_hotkey": "5Dpqn31QEwkqXoMJQF2xvPn9Rh6dDo9CKk1aq3PJxJZgP5Wf",
+                    "aggregated_weight": 0.3,
+                },
+                {
+                    "miner_uid": 2,
+                    "miner_hotkey": "5F3sa2TJAWMqDhXG6jhV4N8ko9SxwGy8TpaNS1repo5EYjQX",
+                    "aggregated_weight": 0.2,
+                },
+            ],
+            "count": 3,
+        }
+
+        with aioresponses() as mocked:
+            mocked.get(
+                "/api/v1/validators/weights",
+                status=200,
+                body=json.dumps(mock_response_data).encode("utf-8"),
+            )
+
+            result = await client_test_env.get_weights()
+
+            mocked.assert_called_once()
+
+            assert result == GetWeightsResponse.model_validate(mock_response_data)
+            assert len(result.weights) == 3
+            assert result.count == 3
+            assert result.weights[0].miner_uid == 0
+            assert result.weights[0].aggregated_weight == 0.5
+
+    async def test_get_weights_error_503_raised(self, client_test_env: NuminousClient):
+        mock_response_data = {"message": "No weights available yet"}
+        status_code = 503
+
+        with aioresponses() as mocked:
+            url_path = "/api/v1/validators/weights"
+            mocked.get(
+                url_path,
+                status=status_code,
+                body=json.dumps(mock_response_data).encode("utf-8"),
+            )
+
+            with pytest.raises(ClientResponseError) as e:
+                await client_test_env.get_weights()
+
+            mocked.assert_called_with(url_path)
+
+            assert e.value.status == status_code
+
+    async def test_get_weights_error_500_raised(self, client_test_env: NuminousClient):
+        mock_response_data = {"message": "Internal server error"}
+        status_code = 500
+
+        with aioresponses() as mocked:
+            url_path = "/api/v1/validators/weights"
+            mocked.get(
+                url_path,
+                status=status_code,
+                body=json.dumps(mock_response_data).encode("utf-8"),
+            )
+
+            with pytest.raises(ClientResponseError) as e:
+                await client_test_env.get_weights()
+
+            mocked.assert_called_with(url_path)
+
+            assert e.value.status == status_code

neurons/validator/scheduler/tasks_scheduler.py

@@ -12,6 +12,7 @@ class TasksScheduler:
 
     __tasks: list[AbstractTask]
     __logger: NuminousLogger
+    DEFAULT_TASK_TIMEOUT_SECONDS = 60 * 60 * 24  # 24 hours
 
     def __init__(self, logger: NuminousLogger):
         # Validate logger
@@ -37,8 +38,8 @@ async def __schedule_task(self, task: AbstractTask):
             self.__logger.info("Task started", extra={"task_name": task.name})
 
             try:
-                # Execute the task's run async function
-                await task.run()
+                # Execute the task's run async function with a default timeout
+                await asyncio.wait_for(task.run(), timeout=self.DEFAULT_TASK_TIMEOUT_SECONDS)
 
                 elapsed_time_ms = round((time.time() - start_time) * 1000)