From e94877dbbe7cfb8319decd336b0bab66ef99a09b Mon Sep 17 00:00:00 2001 From: cct0831 Date: Thu, 26 Mar 2026 08:02:33 +0800 Subject: [PATCH] fix: preserve TPEx OHLCV market on Yahoo fallback --- src/openclaw/market_data_fetcher.py | 53 ++++++++++++++++--- src/tests/test_market_data_fetcher.py | 74 +++++++++++++++++++++++++++ 2 files changed, 121 insertions(+), 6 deletions(-) diff --git a/src/openclaw/market_data_fetcher.py b/src/openclaw/market_data_fetcher.py index ffeca7d..1ed3cbe 100644 --- a/src/openclaw/market_data_fetcher.py +++ b/src/openclaw/market_data_fetcher.py @@ -253,12 +253,42 @@ def save_margin_data( # ── Yahoo Finance fallback ──────────────────────────────────────────────────── _YAHOO_CHART_URL = ( - "https://query1.finance.yahoo.com/v8/finance/chart/{symbol}.TW" + "https://query1.finance.yahoo.com/v8/finance/chart/{symbol}.{suffix}" "?interval=1d&range=5d" ) -def fetch_ohlcv_yahoo(symbols: List[str], sleep_sec: float = 1.0) -> Dict[str, List[Dict]]: +def _get_symbol_markets( + conn: sqlite3.Connection, + symbols: List[str], +) -> Dict[str, str]: + """Best-effort lookup of each symbol's market from existing eod_prices rows.""" + unique_symbols = sorted({str(sym).upper() for sym in symbols if sym}) + if not unique_symbols: + return {} + + placeholders = ",".join("?" for _ in unique_symbols) + rows = conn.execute( + f""" + SELECT symbol, market + FROM eod_prices + WHERE symbol IN ({placeholders}) + ORDER BY trade_date DESC + """, + unique_symbols, + ).fetchall() + + result: Dict[str, str] = {} + for symbol, market in rows: + result.setdefault(str(symbol).upper(), str(market)) + return result + + +def fetch_ohlcv_yahoo( + symbols: List[str], + market_by_symbol: Optional[Dict[str, str]] = None, + sleep_sec: float = 1.0, +) -> Dict[str, List[Dict]]: """ Fetch recent OHLCV from Yahoo Finance as fallback when TWSE API is unavailable. @@ -266,7 +296,9 @@ def fetch_ohlcv_yahoo(symbols: List[str], sleep_sec: float = 1.0) -> Dict[str, L """ result: Dict[str, List[Dict]] = {} for sym in symbols: - url = _YAHOO_CHART_URL.format(symbol=sym) + market = (market_by_symbol or {}).get(str(sym).upper(), "TWSE") + suffix = "TWO" if market == "TPEx" else "TW" + url = _YAHOO_CHART_URL.format(symbol=sym, suffix=suffix) req = urllib.request.Request(url, headers={ "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)", }) @@ -447,10 +479,15 @@ def run_daily_fetch( time.sleep(1) ohlcv_count = 0 twse_failed_symbols: List[str] = [] + market_by_symbol = _get_symbol_markets(conn, ohlcv_symbols) today = datetime.date.today() # Try TWSE STOCK_DAY first (current month only) for sym in ohlcv_symbols: + market = market_by_symbol.get(str(sym).upper(), "TWSE") + if market == "TPEx": + twse_failed_symbols.append(sym) + continue try: rows = fetch_ohlcv_month(sym, today.year, today.month) if rows: @@ -479,19 +516,23 @@ def run_daily_fetch( "[market_data_fetcher] TWSE failed for %d symbols, trying Yahoo Finance", len(twse_failed_symbols), ) - yahoo_data = fetch_ohlcv_yahoo(twse_failed_symbols) + yahoo_data = fetch_ohlcv_yahoo( + twse_failed_symbols, + market_by_symbol=market_by_symbol, + ) for sym, rows in yahoo_data.items(): if rows: + market = market_by_symbol.get(str(sym).upper(), "TWSE") conn.executemany( """ INSERT OR REPLACE INTO eod_prices (trade_date, market, symbol, name, open, high, low, close, volume, source_url, ingested_at) VALUES - (:trade_date, 'TWSE', :symbol, NULL, :open, :high, :low, :close, :volume, + (:trade_date, :market, :symbol, NULL, :open, :high, :low, :close, :volume, 'Yahoo Finance', datetime('now')) """, - [{"symbol": sym.upper(), **r} for r in rows], + [{"symbol": sym.upper(), "market": market, **r} for r in rows], ) conn.commit() ohlcv_count += len(rows) diff --git a/src/tests/test_market_data_fetcher.py b/src/tests/test_market_data_fetcher.py index a9df7df..5e3c396 100644 --- a/src/tests/test_market_data_fetcher.py +++ b/src/tests/test_market_data_fetcher.py @@ -14,6 +14,7 @@ import pytest +import openclaw.market_data_fetcher as mdf from openclaw.market_data_fetcher import ( _parse_num, _to_api_date, @@ -542,3 +543,76 @@ def test_non_trading_day_returns_zero(self, conn): assert result["institution_flows"] == 0 assert result["margin_data"] == 0 + + def test_tpex_symbols_skip_twse_and_use_tpex_yahoo_suffix(self, conn, monkeypatch): + conn.execute( + """ + CREATE TABLE eod_prices ( + trade_date TEXT NOT NULL, + market TEXT NOT NULL, + symbol TEXT NOT NULL, + name TEXT, + open REAL, + high REAL, + low REAL, + close REAL, + volume REAL, + source_url TEXT, + ingested_at TEXT, + PRIMARY KEY (trade_date, market, symbol) + ) + """ + ) + conn.execute( + """ + INSERT INTO eod_prices + (trade_date, market, symbol, name, open, high, low, close, volume, source_url, ingested_at) + VALUES + ('2026-03-20', 'TPEx', '8444', '綠河-KY', 10, 11, 9, 10.5, 1000, 'seed', datetime('now')) + """ + ) + conn.commit() + + twse_calls = [] + + monkeypatch.setattr(mdf, "fetch_institution_flows", lambda trade_date: []) + monkeypatch.setattr(mdf, "save_institution_flows", lambda conn, trade_date, rows: 0) + monkeypatch.setattr(mdf, "fetch_margin_data", lambda trade_date: []) + monkeypatch.setattr(mdf, "save_margin_data", lambda conn, trade_date, rows: 0) + + def fake_fetch_ohlcv_month(symbol, year, month): + twse_calls.append((symbol, year, month)) + return [] + + def fake_fetch_ohlcv_yahoo(symbols, market_by_symbol=None, sleep_sec=1.0): + assert symbols == ["8444"] + assert market_by_symbol == {"8444": "TPEx"} + return { + "8444": [{ + "trade_date": "2026-03-21", + "open": 10.0, + "high": 11.0, + "low": 9.5, + "close": 10.8, + "volume": 1234.0, + }] + } + + monkeypatch.setattr(mdf, "fetch_ohlcv_month", fake_fetch_ohlcv_month) + monkeypatch.setattr(mdf, "fetch_ohlcv_yahoo", fake_fetch_ohlcv_yahoo) + monkeypatch.setattr(mdf.time, "sleep", lambda *_args, **_kwargs: None) + + result = run_daily_fetch("2026-03-21", conn, ohlcv_symbols=["8444"]) + + row = conn.execute( + """ + SELECT market, symbol, close, source_url + FROM eod_prices + WHERE trade_date='2026-03-21' AND symbol='8444' + """ + ).fetchone() + assert result["ohlcv"] == 1 + assert twse_calls == [] + assert row["market"] == "TPEx" + assert row["source_url"] == "Yahoo Finance" + assert row["close"] == 10.8