bd_explore_1.py

import os
import polars as pl
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.manifold import MDS

DATA_DIR = "datasets/raw/1h"

def load_asset_closes_polars(folder_path):
    dfs = []
    for filename in os.listdir(folder_path):
        if filename.endswith(".csv"):
            asset_name = filename.replace(".csv", "")
            filepath = os.path.join(folder_path, filename)
            try:
                df = pl.read_csv(filepath, columns=["timestamp", "close"]).rename({"close": asset_name})
                df = df.with_columns(
                    pl.col("timestamp").str.strptime(pl.Datetime)  # Let Polars auto-detect
                )
                dfs.append(df)
            except Exception as e:
                print(f"Skipping {filename}: {e}")
    return dfs

def merge_assets_polars(dfs):
    if not dfs:
        raise ValueError("No dataframes were loaded. Check for parse errors or missing files.")
    df_merged = dfs[0]
    for df in dfs[1:]:
        df_merged = df_merged.join(df, on="timestamp", how="full", coalesce=True)
    return df_merged.sort("timestamp")

def compute_correlation_matrix(df_merged):
    df_pandas = df_merged.to_pandas().set_index("timestamp")
    return df_pandas.corr()

def plot_correlation_heatmap(corr_matrix):
    plt.figure(figsize=(16, 14))
    sns.heatmap(corr_matrix, cmap='coolwarm', center=0, square=True)
    plt.title("Asset Correlation Heatmap")
    plt.tight_layout()
    plt.show()

def plot_mds_projection(corr_matrix):
    dissimilarity = 1 - corr_matrix.fillna(0).abs()
    mds = MDS(n_components=2, dissimilarity="precomputed", random_state=42)
    pos = mds.fit_transform(dissimilarity)

    plt.figure(figsize=(12, 10))
    for i, name in enumerate(corr_matrix.columns):
        plt.scatter(pos[i, 0], pos[i, 1], label=name)
        plt.text(pos[i, 0], pos[i, 1], name, fontsize=8)
    plt.title("MDS 2D Projection of Asset Correlations")
    plt.grid(True)
    plt.tight_layout()
    plt.show()

# --- MAIN ---
dfs = load_asset_closes_polars(DATA_DIR)

if not dfs:
    print("🚨 No data loaded! Please check your dataset files and format.")
else:
    df_merged = merge_assets_polars(dfs)
    corr_matrix = compute_correlation_matrix(df_merged)
    plot_correlation_heatmap(corr_matrix)
    plot_mds_projection(corr_matrix)