Add MKTD reader and offline backtest mode to ctrader

.claude/scheduled_tasks.lock

@@ -0,0 +1 @@
+{"sessionId":"bb273ea9-d336-4d09-b37e-e004bf640cdd","pid":1851439,"acquiredAt":1773304690371}

.devcontainer/devcontainer.json

@@ -0,0 +1,15 @@
+{
+  "name": "PufferTank 5090",
+  "image": "pufferai/puffertank:latest",
+  "runArgs": ["--gpus=all"],
+  "customizations": {
+    "vscode": {
+      "extensions": [
+        "ms-python.python",
+        "ms-toolsai.jupyter",
+        "github.vscode-pull-request-github"
+      ]
+    }
+  },
+  "postCreateCommand": "uv pip install --upgrade pufferlib torch gymnasium"
+}

.dockerignore

@@ -0,0 +1,37 @@
+
+# Ignore everything by default.
+*
+
+!.dockerignore
+!Dockerfile.runpod
+!pyproject.toml
+!uv.lock
+!runpodmarket/**
+!falmarket/**
+!fal_marketsimulator/**
+!faltrain/**
+!marketsimulator/Dockerfile
+!marketsimulator/**
+!src/**
+!traininglib/**
+!training/**
+!rlinference/**
+!gymrl/**
+!analysis/**
+!analysis_runner_funcs/**
+!fal_utils/**
+!utils/**
+!stock/**
+!toto/**
+!trade_stock_e2e.py
+!trade_stock_e2e_trained.py
+!alpaca_wrapper.py
+!backtest_test3_inline.py
+!data_curate_daily.py
+!env_real.py
+!jsonshelve.py
+!loss_utils.py
+
+gymrl/artifacts/**
+gymrl/cache/**
+gymrl/runs/**

.env.compile

@@ -0,0 +1,46 @@
+# Torch compile configuration - OPTIMIZED FOR RETURNS
+#
+# ★★★ DATA-DRIVEN DECISION: EAGER MODE (torch.compile DISABLED) ★★★
+#
+# Evidence from production logs (trade_stock_e2e.py):
+# - Current (compiled): 8+ recompilations, CUDA graphs skipped
+# - ETHUSD MaxDiff: 10.43% return, 18.24 Sharpe (PROFITABLE!)
+# - Accuracy maintained even with recompilation issues
+# - But performance is suboptimal (unstable latency)
+#
+# Decision: EAGER mode gives SAME returns with BETTER stability
+#
+# Expected after switch:
+# - Eager: ~500ms per prediction, STABLE (no recompilations)
+# - Memory: 650MB (vs 900MB compiled)
+# - Returns: 10.43% maintained (proven in logs)
+# - Stability: HIGH (no recompilation overhead)
+
+# Toto model: EAGER mode (disabled compilation)
+export TOTO_DISABLE_COMPILE=1
+
+# Kronos model: EAGER mode (TESTED AND PROVEN)
+# DECISION: Use EAGER mode - torch.compile has CRITICAL BUG
+#
+# ★★★ DATA-DRIVEN DECISION FROM BENCHMARK ★★★
+# Benchmark results (scripts/benchmark_kronos_compile.py):
+#   EAGER:    5/5 iterations successful ✅
+#   COMPILED: 0/5 iterations successful ❌
+#   Error: "CUDA graphs tensor output reuse" bug
+#
+# Eager performance:
+#   - MAE: 36,160 ± 1,883 (consistent)
+#   - Time: ~3 seconds (acceptable)
+#   - Memory: 336MB (efficient)
+#   - Reliability: 100% (no crashes)
+#
+# DO NOT ENABLE KRONOS_COMPILE - it will crash!
+# No flag needed - Kronos uses eager mode by default
+
+# Bid/Ask data: Use real API data (not synthetic)
+export ADD_LATEST=1
+
+# To enable compiled mode in the future (after fixing recompilation issues):
+# export TOTO_DISABLE_COMPILE=0
+# export TOTO_COMPILE_MODE=max-autotune
+# export TOTO_COMPILE_BACKEND=inductor

.github/workflows/ci-fast.yml

@@ -0,0 +1,246 @@
+name: Fast CI (GitHub Runners)
+
+on:
+  push:
+    branches: ["main"]
+  pull_request:
+    branches: ["main"]
+
+permissions:
+  contents: read
+
+jobs:
+  lint:
+    name: Lint & Format Check
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.13"
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v3
+        with:
+          enable-cache: true
+
+      - name: Install linting dependencies
+        run: |
+          uv pip install --system ruff flake8
+
+      - name: Run Ruff linter
+        run: |
+          RUFF_TARGETS="neuraldailytraining/model.py neuraldailytraining/runtime.py neuraldailytraining/trainer.py neuraldailymarketsimulator/simulator.py tests/test_neuraldailytraining.py tests/test_neuraldaily_alignment.py tests/test_neuraldaily_runtime_confidence_and_fees.py tests/test_trainer_utils.py tests/test_non_tradable_io.py"
+          ruff check $RUFF_TARGETS --output-format=github
+
+      - name: Run Ruff format check
+        run: |
+          RUFF_TARGETS="neuraldailytraining/model.py neuraldailytraining/runtime.py neuraldailytraining/trainer.py neuraldailymarketsimulator/simulator.py tests/test_neuraldailytraining.py tests/test_neuraldaily_alignment.py tests/test_neuraldaily_runtime_confidence_and_fees.py tests/test_trainer_utils.py tests/test_non_tradable_io.py"
+          ruff format --check $RUFF_TARGETS
+
+      - name: Report to CI Sentinel
+        if: always()
+        env:
+          CI_SENTINEL_ID: ${{ vars.CI_SENTINEL_ID }}
+          CI_SENTINEL_KEY: ${{ secrets.CODEX_INFINITY_API_KEY }}
+        run: |
+          if [ -z "$CI_SENTINEL_ID" ] || [ -z "$CI_SENTINEL_KEY" ]; then
+            echo "CI Sentinel not configured, skipping report"
+            exit 0
+          fi
+          # Collect failure output from previous steps
+          FAILURE_OUTPUT=""
+          if [ "${{ job.status }}" = "failure" ]; then
+            FAILURE_OUTPUT=$(cat /tmp/ci-failure-output.txt 2>/dev/null | tail -c 12288 || echo "See GitHub Actions logs")
+          fi
+          curl -sf -X POST "https://codex-infinity.com/api/ci-sentinels/${CI_SENTINEL_ID}/events" \
+            -H "Authorization: Bearer ${CI_SENTINEL_KEY}" \
+            -H "Content-Type: application/json" \
+            -d "{
+              \"external_id\": \"${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}\",
+              \"repo\": \"${GITHUB_REPOSITORY}\",
+              \"branch\": \"${GITHUB_REF_NAME}\",
+              \"commit_sha\": \"${GITHUB_SHA}\",
+              \"workflow\": \"${GITHUB_WORKFLOW}\",
+              \"job_name\": \"${GITHUB_JOB}\",
+              \"run_number\": ${GITHUB_RUN_NUMBER},
+              \"run_attempt\": ${GITHUB_RUN_ATTEMPT},
+              \"status\": \"completed\",
+              \"conclusion\": \"${{ job.status }}\",
+              \"logs_url\": \"https://github.com/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}\",
+              \"failure_output\": $(echo "$FAILURE_OUTPUT" | python3 -c 'import json,sys; print(json.dumps(sys.stdin.read()))' 2>/dev/null || echo '\"\"')
+            }" || echo "Warning: Failed to report to CI Sentinel"
+
+  fast-unit-tests:
+    name: Fast Unit Tests (CPU)
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.13"]
+
+    env:
+      # CI mode flags
+      CI: "1"
+      FAST_CI: "1"
+      CPU_ONLY: "1"
+      FAST_SIMULATE: "1"
+
+      # Market simulator flags
+      MARKETSIM_ALLOW_MOCK_ANALYTICS: "1"
+      MARKETSIM_SKIP_REAL_IMPORT: "1"
+      MARKETSIM_ALLOW_CPU_FALLBACK: "1"
+
+      # Trading flags
+      ALP_PAPER: "1"
+      PYTHONUNBUFFERED: "1"
+
+      # Skip heavy dependencies for unit tests
+      SKIP_TORCH_CHECK: "0"
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 1  # Shallow clone for faster checkout
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v3
+        with:
+          enable-cache: true
+          cache-dependency-glob: |
+            requirements.txt
+            requirements-ci.txt
+
+      - name: Install CPU-only dependencies
+        run: |
+          # Install all dependencies from CPU-only requirements
+          uv pip install --system --requirement requirements-ci.txt
+
+      - name: Run fast unit tests
+        run: |
+          python -m pytest \
+            -v \
+            -m "unit and not slow and not model_required and not cuda_required" \
+            --tb=short \
+            --maxfail=10 \
+            tests/
+
+      - name: Run smoke tests (minimal model tests)
+        continue-on-error: true
+        run: |
+          python -m pytest \
+            -v \
+            -m "smoke and model_required and not cuda_required" \
+            --tb=short \
+            --maxfail=3 \
+            tests/
+
+      - name: Report to CI Sentinel
+        if: always()
+        env:
+          CI_SENTINEL_ID: ${{ vars.CI_SENTINEL_ID }}
+          CI_SENTINEL_KEY: ${{ secrets.CODEX_INFINITY_API_KEY }}
+        run: |
+          if [ -z "$CI_SENTINEL_ID" ] || [ -z "$CI_SENTINEL_KEY" ]; then
+            echo "CI Sentinel not configured, skipping report"
+            exit 0
+          fi
+          # Collect failure output from previous steps
+          FAILURE_OUTPUT=""
+          if [ "${{ job.status }}" = "failure" ]; then
+            FAILURE_OUTPUT=$(cat /tmp/ci-failure-output.txt 2>/dev/null | tail -c 12288 || echo "See GitHub Actions logs")
+          fi
+          curl -sf -X POST "https://codex-infinity.com/api/ci-sentinels/${CI_SENTINEL_ID}/events" \
+            -H "Authorization: Bearer ${CI_SENTINEL_KEY}" \
+            -H "Content-Type: application/json" \
+            -d "{
+              \"external_id\": \"${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}\",
+              \"repo\": \"${GITHUB_REPOSITORY}\",
+              \"branch\": \"${GITHUB_REF_NAME}\",
+              \"commit_sha\": \"${GITHUB_SHA}\",
+              \"workflow\": \"${GITHUB_WORKFLOW}\",
+              \"job_name\": \"${GITHUB_JOB}\",
+              \"run_number\": ${GITHUB_RUN_NUMBER},
+              \"run_attempt\": ${GITHUB_RUN_ATTEMPT},
+              \"status\": \"completed\",
+              \"conclusion\": \"${{ job.status }}\",
+              \"logs_url\": \"https://github.com/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}\",
+              \"failure_output\": $(echo "$FAILURE_OUTPUT" | python3 -c 'import json,sys; print(json.dumps(sys.stdin.read()))' 2>/dev/null || echo '\"\"')
+            }" || echo "Warning: Failed to report to CI Sentinel"
+
+  type-check:
+    name: Type Checking
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.13"
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v3
+        with:
+          enable-cache: true
+          cache-dependency-glob: requirements-ci.txt
+
+      - name: Install dependencies
+        run: |
+          uv pip install --system --requirement requirements-ci.txt
+          uv pip install --system ty pyright mypy
+
+      - name: Type check with ty
+        continue-on-error: true
+        run: ty check
+
+      - name: Type check with Pyright
+        continue-on-error: true
+        run: python -m pyright src
+
+      - name: Type check with mypy
+        continue-on-error: true
+        run: mypy src --config-file pyproject.toml
+
+      - name: Report to CI Sentinel
+        if: always()
+        env:
+          CI_SENTINEL_ID: ${{ vars.CI_SENTINEL_ID }}
+          CI_SENTINEL_KEY: ${{ secrets.CODEX_INFINITY_API_KEY }}
+        run: |
+          if [ -z "$CI_SENTINEL_ID" ] || [ -z "$CI_SENTINEL_KEY" ]; then
+            echo "CI Sentinel not configured, skipping report"
+            exit 0
+          fi
+          # Collect failure output from previous steps
+          FAILURE_OUTPUT=""
+          if [ "${{ job.status }}" = "failure" ]; then
+            FAILURE_OUTPUT=$(cat /tmp/ci-failure-output.txt 2>/dev/null | tail -c 12288 || echo "See GitHub Actions logs")
+          fi
+          curl -sf -X POST "https://codex-infinity.com/api/ci-sentinels/${CI_SENTINEL_ID}/events" \
+            -H "Authorization: Bearer ${CI_SENTINEL_KEY}" \
+            -H "Content-Type: application/json" \
+            -d "{
+              \"external_id\": \"${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}\",
+              \"repo\": \"${GITHUB_REPOSITORY}\",
+              \"branch\": \"${GITHUB_REF_NAME}\",
+              \"commit_sha\": \"${GITHUB_SHA}\",
+              \"workflow\": \"${GITHUB_WORKFLOW}\",
+              \"job_name\": \"${GITHUB_JOB}\",
+              \"run_number\": ${GITHUB_RUN_NUMBER},
+              \"run_attempt\": ${GITHUB_RUN_ATTEMPT},
+              \"status\": \"completed\",
+              \"conclusion\": \"${{ job.status }}\",
+              \"logs_url\": \"https://github.com/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}\",
+              \"failure_output\": $(echo "$FAILURE_OUTPUT" | python3 -c 'import json,sys; print(json.dumps(sys.stdin.read()))' 2>/dev/null || echo '\"\"')
+            }" || echo "Warning: Failed to report to CI Sentinel"

.github/workflows/ci-ubuntu.yml

@@ -0,0 +1,16 @@
+name: CI (Ubuntu)
+on: [push, pull_request]
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.13'
+      - name: Install uv
+        uses: astral-sh/setup-uv@v3
+      - name: Install dependencies
+        run: uv pip install --system -e ".[dev]" 2>/dev/null || uv pip install --system pytest pytest-mock
+      - name: Run tests
+        run: python -m pytest tests/ -x -q --tb=short --ignore=tests/integration -k "not slow" 2>&1 | head -100

.github/workflows/ci.yml

@@ -0,0 +1,16 @@
+name: CI
+on: [push, pull_request]
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.13'
+      - name: Install uv
+        uses: astral-sh/setup-uv@v3
+      - name: Install dependencies
+        run: uv pip install --system -e ".[dev]" 2>/dev/null || uv pip install --system pytest pytest-mock
+      - name: Run tests
+        run: python -m pytest tests/ -x -q --tb=short --ignore=tests/integration -k "not slow" 2>&1 | head -100