hdc_validate_v2.py

import os, pickle, time, logging, csv
import numpy as np
from pathlib import Path

# ─── CONFIG ──────────────────────────────────────────────────────────────
DATA_DIR = "datasets/hdc_pairs_200_shot"
MEMORY_FILE = os.path.join(DATA_DIR, "hdc_memory.pkl")

X_FILE = "X_pairs_shuffled.npy"
Y_FILE = "Y_pairs_shuffled.npy"
O_FILE = "O_pairs_shuffled.npy"          # <- NEW

OUT_DIR = "outputs/200_candle"
Path(OUT_DIR).mkdir(exist_ok=True)

TOP_K = 5
FUSION = "weighted"
SAMPLE_SIZE = 20
VOLUME_K = 0.2           # same k you used during target‑creation

TARGET_COLS = [
    "target_open", "target_high", "target_low", "target_close",
    "target_volume", "target_quote_asset_volume", "target_number_of_trades",
    "target_taker_buy_base_asset_volume", "target_taker_buy_quote_asset_volume"
]

ORIG_COLS = [
    "original_close",
    "original_volume",
    "original_quote_asset_volume",
    "original_number_of_trades",
    "original_taker_buy_base_asset_volume",
    "original_taker_buy_quote_asset_volume"
]
# indices inside O‑row
OCLOSE   = 0
OVOLUME  = 1
OQUOTE   = 2
ONTRADES = 3
OTBBAV   = 4
OTBQAV   = 5
# ------------------------------------------------------------------------

logging.basicConfig(level=logging.INFO,
                    format="%(asctime)s [%(levelname)s] %(message)s")


# ─── HDC helpers (unchanged) ─────────────────────────────────────────────
def quantize_sequence(seq, edges):
    bins = np.empty_like(seq, dtype=np.int32)
    for f in range(seq.shape[1]):
        bins[:, f] = np.digitize(seq[:, f], edges[:, f])
    return bins

def bundle(vecs):                       # majority bundling
    s = np.sum(vecs, axis=0)
    return np.where(s >= 0, 1, -1).astype(np.int8)

def bind(a, b):                         # element‑wise multiply
    return (a * b).astype(np.int8)

def predict_sequence(mem, seq, top_k=TOP_K, fusion=FUSION):
    edges, feat_arr, time_arr = mem['edges'], mem['feature_tags_arr'], mem['time_tags_arr']
    keys, values, knorms      = mem['keys'], mem['values'], mem['key_norms']

    bins = quantize_sequence(seq, edges)
    rows = []
    for t, bin_row in enumerate(bins):
        feat_vecs = feat_arr[np.arange(feat_arr.shape[0]), bin_row]
        rows.append(bind(time_arr[t], bundle(feat_vecs)))
    query = bundle(np.stack(rows, axis=0))

    sims   = np.dot(keys, query) / (knorms * np.linalg.norm(query))
    top_id = np.argpartition(-sims, top_k)[:top_k]
    w      = sims[top_id] / sims[top_id].sum()
    fused  = (values[top_id] * w[:, None, None]).sum(axis=0) if fusion=="weighted" else values[top_id].mean(0)
    return fused


# ─── Decoding helpers ───────────────────────────────────────────────────
def decode_target_row(prev_close, prev_vals, row):
    """row → shape (9,).  Returns dict with same 9 fields decoded to raw."""
    out = {}
    # OHLC
    out["open"]  = prev_close * np.exp(row[0])
    out["high"]  = prev_close * np.exp(row[1])
    out["low"]   = prev_close * np.exp(row[2])
    out["close"] = prev_close * np.exp(row[3])

    # volumes (use prev_vals to compute log‑return inverse)
    vol_keys = ["volume","quote_asset_volume","number_of_trades",
                "taker_buy_base_asset_volume","taker_buy_quote_asset_volume"]
    for i, k in enumerate(vol_keys, start=4):
        out[k] = prev_vals[k] * np.exp(np.arctanh(row[i]) / VOLUME_K)

    return out

def decode_sequence(first_orig_row, target_mat):
    """
    first_orig_row : (6,)  original_close + 5 volume cols of *last* feature candle
    target_mat     : (200, 9)  model targets
    Returns        : (200, 9)  decoded raw values (same column order as TARGET_COLS)
    """
    prev_close = float(first_orig_row[OCLOSE])
    prev_vals  = {
        "volume"                       : float(first_orig_row[OVOLUME]),
        "quote_asset_volume"           : float(first_orig_row[OQUOTE]),
        "number_of_trades"             : float(first_orig_row[ONTRADES]),
        "taker_buy_base_asset_volume"  : float(first_orig_row[OTBBAV]),
        "taker_buy_quote_asset_volume" : float(first_orig_row[OTBQAV]),
    }

    decoded_rows = []
    for row in target_mat:           # iterate 200 rows
        dec = decode_target_row(prev_close, prev_vals, row)
        decoded_rows.append([
            dec["open"], dec["high"], dec["low"], dec["close"],
            dec["volume"], dec["quote_asset_volume"], dec["number_of_trades"],
            dec["taker_buy_base_asset_volume"], dec["taker_buy_quote_asset_volume"]
        ])
        # update “previous” values for next step
        prev_close = dec["close"]
        for k in prev_vals: prev_vals[k] = dec[k]

    return np.array(decoded_rows, dtype=np.float32)


# ─── MAIN ────────────────────────────────────────────────────────────────
def main():
    # 1) load mem + data
    mem   = pickle.load(open(MEMORY_FILE, "rb"))
    X     = np.load(os.path.join(DATA_DIR, X_FILE))
    Y     = np.load(os.path.join(DATA_DIR, Y_FILE))
    O     = np.load(os.path.join(DATA_DIR, O_FILE))
    N     = X.shape[0]

    # 60 % train / 40 % test split (the same ratio used to build the mem)
    split = int(N * 0.6)
    X_te, Y_te, O_te = X[split:], Y[split:], O[split:]
    logging.info(f"Loaded test‑set: {X_te.shape[0]} samples")

    rng = np.random.default_rng()
    idxs = rng.choice(X_te.shape[0], size=SAMPLE_SIZE, replace=False)

    for n, idx in enumerate(idxs, start=1):
        feats   = X_te[idx]
        targets = Y_te[idx]
        orig    = O_te[idx]

        synth_targets = predict_sequence(mem, feats)

        # print targets vs. synth
        print("\n─────────────────────────────")
        print(f"📌 sample #{n}  (global index {split+idx})")
        print("Target (first 3 rows):")
        print(targets[:3])
        print("Synthetic Ŷ (first 3 rows):")
        print(synth_targets[:3])

        # decode to raw
        real_raw   = decode_sequence(orig, targets)
        synth_raw  = decode_sequence(orig, synth_targets)

        # save CSVs
        real_path  = os.path.join(OUT_DIR, f"real_sample_{n}.csv")
        synth_path = os.path.join(OUT_DIR, f"synthetic_sample_{n}.csv")

        header = ["open","high","low","close",
                  "volume","quote_asset_volume","number_of_trades",
                  "taker_buy_base_asset_volume","taker_buy_quote_asset_volume"]

        for path, arr in [(real_path, real_raw), (synth_path, synth_raw)]:
            with open(path, "w", newline="") as f:
                w = csv.writer(f)
                w.writerow(header)
                w.writerows(arr)
        print(f"✓ wrote {real_path}  +  {synth_path}")

    logging.info("✅ Done")

if __name__ == "__main__":
    main()