[AI] [FEAT]: Kaggle 기반 자동 학습 파이프라인 구축

.github/workflows/train.yml

@@ -0,0 +1,79 @@
+name: Weekly Model Training Pipeline
+
+on:
+  # 매주 화요일 새벽 2시 (KST) 자동 실행
+  schedule:
+    - cron: '0 17 * * 1'
+  # GitHub Actions 탭에서 수동 실행 가능
+  workflow_dispatch:
+    inputs:
+      skip_extract:
+        description: 'DB 추출 스킵 (parquet 이미 있을 때)'
+        required: false
+        default: false
+        type: boolean
+      skip_upload:
+        description: 'Kaggle 업로드 스킵 (데이터 변경 없을 때)'
+        required: false
+        default: false
+        type: boolean
+
+jobs:
+  train:
+    runs-on: ubuntu-latest
+    timeout-minutes: 780  # 최대 13시간
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+
+      - name: Install dependencies
+        run: |
+          pip install \
+            kaggle \
+            pandas \
+            pyarrow \
+            psycopg2-binary \
+            paramiko \
+            sshtunnel \
+            scp
+
+      - name: Setup Kaggle credentials
+        run: |
+          mkdir -p ~/.kaggle
+          jq -n \
+            --arg user "${{ secrets.KAGGLE_USERNAME }}" \
+            --arg key "${{ secrets.KAGGLE_KEY }}" \
+            '{"username": $user, "key": $key}' \
+            > ~/.kaggle/kaggle.json
+          chmod 600 ~/.kaggle/kaggle.json
+
+      - name: Run weekly training pipeline
+        env:
+          SSH_HOST:            ${{ secrets.SSH_HOST }}
+          SSH_USER:            ${{ secrets.SSH_USER }}
+          SSH_PRIVATE_KEY:     ${{ secrets.SSH_PRIVATE_KEY }}
+          SSH_PORT:            ${{ secrets.SSH_PORT }}
+          DB_HOST:             ${{ secrets.DB_HOST }}
+          DB_PORT:             ${{ secrets.DB_PORT }}
+          DB_USER:             ${{ secrets.DB_USER }}
+          DB_PASSWORD:         ${{ secrets.DB_PASSWORD }}
+          DB_NAME:             ${{ secrets.DB_NAME }}
+          KAGGLE_USERNAME:     ${{ secrets.KAGGLE_USERNAME }}
+          KAGGLE_KEY:          ${{ secrets.KAGGLE_KEY }}
+          SERVER_WEIGHTS_PATH: ${{ secrets.SERVER_WEIGHTS_PATH }}
+        run: |
+          SKIP_EXTRACT=""
+          SKIP_UPLOAD=""
+          if [ "${{ github.event.inputs.skip_extract }}" == "true" ]; then
+            SKIP_EXTRACT="--skip-extract"
+          fi
+          if [ "${{ github.event.inputs.skip_upload }}" == "true" ]; then
+            SKIP_UPLOAD="--skip-upload"
+          fi
+          python AI/pipelines/weekly_routine.py $SKIP_EXTRACT $SKIP_UPLOAD

.gitignore

@@ -34,6 +34,7 @@ AI/data/weights/
 
 AI/.venv/
 AI/data/weights/tcn/
+AI/data/kaggle_data/
 
 AI/data/weights/itransformer/*
 !AI/data/weights/itransformer/.gitkeep
@@ -42,6 +43,16 @@ AI/config/trading.local.json
 AI/tests/out/
 AI/docs/
 
+SISC_*.md
+test_*.py
+AI/data/kaggle_data/
+*.parquet
+*.pt
+*.keras
+*.pkl
+# ===== Kaggle API 키 =====
+kaggle.json
+.kaggle/
 # ===== Backend =====
 backend/src/main/java/org/sejongisc/backend/stock/TestController.java
 

AI/modules/signal/core/artifact_paths.py

@@ -100,13 +100,14 @@ def resolve_model_artifacts(
             metadata_path=str(resolved_model_dir / "metadata.json"),
         )
 
+    # [수정] PatchTST: 실제 저장 파일명으로 통일 + scaler_path 추가
     if normalized_model == "patchtst":
-        resolved_model_dir = _resolve_absolute(model_dir) if model_dir else (root_dir / "patchtst")
+        resolved_model_dir = _resolve_absolute(model_dir) if model_dir else (root_dir / "PatchTST")
         return ModelArtifactPaths(
             root_dir=str(root_dir),
             model_dir=str(resolved_model_dir),
-            model_path=str(resolved_model_dir / "PatchTST_best.pt"),
-            scaler_path=None,
+            model_path=str(resolved_model_dir / "patchtst_model.pt"),       # PatchTST_best.pt → patchtst_model.pt
+            scaler_path=str(resolved_model_dir / "patchtst_scaler.pkl"),    # 추가
             metadata_path=None,
         )
 

AI/modules/signal/models/PatchTST/train.py

@@ -22,25 +22,8 @@
 import torch.nn as nn
 import torch.optim as optim
 from torch.utils.data import DataLoader, TensorDataset
-<<<<<<< HEAD
 from sklearn.preprocessing import MinMaxScaler
 from tqdm import tqdm
-=======
-from .architecture import PatchTST_Model
-from AI.config import load_trading_config
-from AI.modules.signal.core.artifact_paths import resolve_model_artifacts
-
-
-def _default_model_save_path() -> str:
-    try:
-        trading_config = load_trading_config()
-        return resolve_model_artifacts(
-            model_name="patchtst",
-            config_weights_dir=trading_config.model.weights_dir,
-        ).model_path
-    except Exception:
-        return resolve_model_artifacts(model_name="patchtst").model_path
->>>>>>> 969fb59bb447edc8ffb66545ba0fdc1a4d190e79
 
 # 경로 설정 (다른 import보다 먼저)
 current_dir  = os.path.dirname(os.path.abspath(__file__))
@@ -59,7 +42,6 @@ def _default_model_save_path() -> str:
 # CONFIG
 # ─────────────────────────────────────────────────────────────────────────────
 CONFIG = {
-<<<<<<< HEAD
     'start_date'     : '2015-01-01',
     'end_date'       : '2023-12-31',  # 미래 데이터 차단 (Look-ahead bias 방지)
     'seq_len'        : 120,
@@ -88,18 +70,6 @@ def _default_model_save_path() -> str:
     'weights_dir'    : 'AI/data/weights/PatchTST',
     'model_name'     : 'patchtst_model.pt',
     'scaler_name'    : 'patchtst_scaler.pkl',
-=======
-    'seq_len': 120,
-    'input_features': 7,
-    'batch_size': 32,
-    'learning_rate': 0.0001,
-    'epochs': 100,
-    'patience': 10,
-    'model_save_path': _default_model_save_path()
-<<<<<<< HEAD
->>>>>>> e47fa9e ([AI] [FEAT] 볼륨 마운트를 통한 가중치 저장)
-=======
->>>>>>> 969fb59bb447edc8ffb66545ba0fdc1a4d190e79
 }
 
 # ─────────────────────────────────────────────────────────────────────────────
@@ -244,9 +214,15 @@ def train():
     # ── [수정] Train/Val 분리 먼저 → 그 다음 스케일링 ──────────────────────
     # 기존: 전체 스케일링 → 분리 (데이터 누수 발생)
     # 수정: 티커 기준으로 분리 → train만 fit → val은 transform
-    tickers      = full_df['ticker'].unique()
-    n_val        = max(1, int(len(tickers) * 0.2))
-    val_tickers  = tickers[-n_val:]   # 마지막 20% 티커를 val로 (시간 순서 보존)
+    tickers = full_df['ticker'].unique()
+
+    # 최소 2개 이상 있어야 train/val 분리 가능
+    if len(tickers) < 2:
+        raise ValueError(f"학습에 필요한 ticker가 부족합니다. (현재: {len(tickers)}개, 최소 2개 필요)")
+
+    # val 비율 20%, 단 train이 최소 1개는 남도록 상한 보정
+    n_val         = max(1, min(int(len(tickers) * 0.2), len(tickers) - 1))
+    val_tickers   = tickers[-n_val:]   # 마지막 20% 티커를 val로 (시간 순서 보존)
     train_tickers = tickers[:-n_val]
 
     train_df = full_df[full_df['ticker'].isin(train_tickers)].copy()
@@ -337,7 +313,7 @@ def train():
         if avg_val < best_val_loss:
             best_val_loss    = avg_val
             patience_counter = 0
-            # [수정] config + state_dict 같이 저장 (load 시 구조 재현 가능)
+            # config + state_dict 같이 저장 (load 시 구조 재현 가능)
             torch.save({
                 'config'    : CONFIG,
                 'state_dict': model.state_dict()
@@ -350,7 +326,6 @@ def train():
                 print(f"\n>> Early Stopping at epoch {epoch+1}")
                 break
 
-<<<<<<< HEAD
     # 7. 스케일러 저장
     with open(scaler_path, 'wb') as f:
         pickle.dump(scaler, f)
@@ -362,25 +337,3 @@ def train():
 
 if __name__ == '__main__':
     train()
-=======
-def run_training(X_train, y_train, X_val, y_val):
-    """
-    외부에서 호출 가능한 학습 진입점
-    X: [Samples, Seq_Len, Features] numpy array
-    y: [Samples] numpy array (0 or 1)
-    """
-    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
-
-    # Tensor 변환
-    train_data = TensorDataset(torch.FloatTensor(X_train), torch.FloatTensor(y_train))
-    val_data = TensorDataset(torch.FloatTensor(X_val), torch.FloatTensor(y_val))
-
-    train_loader = DataLoader(train_data, batch_size=CONFIG['batch_size'], shuffle=True)
-    val_loader = DataLoader(val_data, batch_size=CONFIG['batch_size'], shuffle=False)
-
-    trained_model = train_model(train_loader, val_loader, device)
-    return trained_model
-<<<<<<< HEAD
->>>>>>> e47fa9e ([AI] [FEAT] 볼륨 마운트를 통한 가중치 저장)
-=======
->>>>>>> 969fb59bb447edc8ffb66545ba0fdc1a4d190e79