mg_test_arrow_light_head.py

# filename: inspect_arrow_detailed.py
import polars as pl
from pathlib import Path

ARROW_PATH = Path("datasets/arrow_mega/unified_1m_megaset.arrow")
LOG_FILENAME = Path("column_log.txt")

# The GDs currently in the dataset
GROUPING_DISTANCES = [
    2, 3, 4, 6, 7, 8, 9, 11, 12, 13,
    14, 16, 17, 18, 19, 21, 22, 23, 24, 26,
    27, 28, 29, 31, 32, 33, 34, 36, 37, 38,
    39, 41, 42, 43, 44, 46, 47, 48, 51, 54,
    59, 64, 71, 76, 83, 91, 98, 106, 116, 126,
    137, 149, 162, 177, 192, 209, 228, 248, 271, 294,
    319, 348, 378, 412, 448, 488, 531, 577, 628, 684,
    744, 811, 881, 959, 1044, 1136, 1236, 1346, 1463, 1593,
    1733, 1886, 2052, 2234, 2431, 2646, 2879, 3133, 3409, 3711,
    4037, 4393, 4781, 5203, 5662, 6161, 6706, 7297, 7941, 8641
]


def main() -> None:
    if not ARROW_PATH.exists():
        raise FileNotFoundError(f"{ARROW_PATH} not found")

    print("🔍 Loading Arrow dataset...")

    # Lazy scan so we don't load the whole file into RAM
    lf = pl.scan_ipc(ARROW_PATH, memory_map=True)

    # --- Basic dataset info --------------------------------------------------
    col_names = lf.schema.keys()
    n_rows = lf.select(pl.len()).collect().item()
    n_cols = len(col_names)

    print(f"✅ Dataset shape: {n_rows:,} rows × {n_cols:,} columns")
    print(f"📝 Writing all {len(col_names):,} column names to {LOG_FILENAME}")
    LOG_FILENAME.write_text("\n".join(col_names))

    # --- ADAUSDT Feature Analysis --------------------------------------------
    print(f"\n{'=' * 70}")
    print("🎯 ADAUSDT OHLCV FEATURE ANALYSIS")
    print(f"{'=' * 70}")

    asset = "ADAUSDT"

    # Find all ADAUSDT feature columns
    base_feature_cols = []
    gd_feature_cols = {}

    print("🔍 Scanning for ADAUSDT feature columns...")

    for col in col_names:
        if asset in col and col.startswith("feature_"):
            if col.startswith("feature_gd"):
                # Extract GD number from column name like "feature_gd2_ADAUSDT_open"
                parts = col.split("_")
                if len(parts) >= 4:
                    gd_part = parts[1]  # "gd2", "gd12", etc.
                    if gd_part.startswith("gd"):
                        try:
                            gd_num = int(gd_part[2:])  # Extract number after "gd"
                            if gd_num not in gd_feature_cols:
                                gd_feature_cols[gd_num] = []
                            gd_feature_cols[gd_num].append(col)
                        except ValueError:
                            pass
            else:
                # Regular feature column like "feature_ADAUSDT_open"
                if not col.startswith("feature_gd"):
                    base_feature_cols.append(col)

    # Sort columns by OHLCV order
    def sort_ohlcv(cols):
        """Sort columns by OHLCV order"""
        ohlcv_order = {"open": 0, "high": 1, "low": 2, "close": 3, "volume": 4}
        return sorted(cols, key=lambda x: ohlcv_order.get(x.split("_")[-1], 999))

    base_feature_cols = sort_ohlcv(base_feature_cols)
    for gd in gd_feature_cols:
        gd_feature_cols[gd] = sort_ohlcv(gd_feature_cols[gd])

    print(f"📊 Found {len(base_feature_cols)} base ADAUSDT features")
    print(f"📊 Found {len(gd_feature_cols)} different GDs with ADAUSDT features")

    # Load first 10 rows for visual inspection
    print("\n🔬 Loading first 10 rows for visual inspection...")

    # Collect all ADAUSDT feature columns we want to inspect
    all_adausdt_cols = ["timestamp"] + base_feature_cols
    for gd in sorted(gd_feature_cols.keys()):
        all_adausdt_cols.extend(gd_feature_cols[gd])

    # Load the data
    sample_df = lf.select(all_adausdt_cols).head(30000).collect()

    print(f"✅ Loaded sample data: {sample_df.shape}")

    # --- Display Results -----------------------------------------------------
    print(f"\n{'=' * 70}")
    print("📋 ADAUSDT FEATURE COLUMNS HEAD (10 rows)")
    print(f"{'=' * 70}")

    # Show base features first
    print(f"\n🏷️  BASE FEATURES (feature_ADAUSDT_*):")
    print("-" * 50)
    if base_feature_cols:
        base_sample = sample_df.select(["timestamp"] + base_feature_cols)
        print(base_sample)
    else:
        print("❌ No base ADAUSDT features found!")

    # Show each GD's features
    print(f"\n🏷️  GD FEATURES:")
    print("-" * 50)

    if not gd_feature_cols:
        print("❌ No GD ADAUSDT features found!")
    else:
        # Show first few GDs in detail, then summarize the rest
        detailed_gds = sorted(gd_feature_cols.keys())[:5]  # Show first 5 GDs in detail
        summary_gds = sorted(gd_feature_cols.keys())[5:]  # Summarize the rest

        for gd in detailed_gds:
            print(f"\n📊 GD-{gd} Features (feature_gd{gd}_ADAUSDT_*):")
            gd_sample = sample_df.select(["timestamp"] + gd_feature_cols[gd])
            print(gd_sample)

        if summary_gds:
            print(f"\n📊 Additional GDs Available: {len(summary_gds)} more GDs")
            print(f"   GD ranges: {min(summary_gds)}-{max(summary_gds)}")
            print(f"   Sample GDs: {summary_gds[:10]}{'...' if len(summary_gds) > 10 else ''}")

    # --- Summary Statistics --------------------------------------------------
    print(f"\n{'=' * 70}")
    print("📊 SUMMARY STATISTICS")
    print(f"{'=' * 70}")

    print(f"✅ Base ADAUSDT features: {len(base_feature_cols)}")
    print(f"✅ GD ADAUSDT features: {sum(len(cols) for cols in gd_feature_cols.values())}")
    print(f"✅ Total ADAUSDT features: {len(base_feature_cols) + sum(len(cols) for cols in gd_feature_cols.values())}")
    print(f"✅ Number of GDs represented: {len(gd_feature_cols)}")

    if gd_feature_cols:
        gd_nums = sorted(gd_feature_cols.keys())
        print(f"✅ GD range: {min(gd_nums)} to {max(gd_nums)}")
        print(f"✅ Expected GDs from config: {len(GROUPING_DISTANCES)}")

        # Check if all expected GDs are present
        missing_gds = set(GROUPING_DISTANCES) - set(gd_nums)
        extra_gds = set(gd_nums) - set(GROUPING_DISTANCES)

        if missing_gds:
            print(f"⚠️  Missing GDs: {sorted(missing_gds)[:10]}{'...' if len(missing_gds) > 10 else ''}")
        if extra_gds:
            print(f"⚠️  Extra GDs: {sorted(extra_gds)[:10]}{'...' if len(extra_gds) > 10 else ''}")
        if not missing_gds and not extra_gds:
            print(f"✅ All expected GDs present and accounted for!")

    # --- Data Quality Check --------------------------------------------------
    print(f"\n{'=' * 70}")
    print("🔍 DATA QUALITY CHECK")
    print(f"{'=' * 70}")

    # Check for null values in sample
    print("🔬 Null value analysis (first 10 rows):")

    for col in all_adausdt_cols[:20]:  # Check first 20 columns to avoid spam
        if col in sample_df.columns:
            null_count = sample_df[col].null_count()
            total_count = sample_df.height
            null_pct = (null_count / total_count) * 100
            status = "✅" if null_count == 0 else "⚠️" if null_count < total_count else "❌"
            print(f"   {status} {col:<40}: {null_count}/{total_count} nulls ({null_pct:.1f}%)")

    if len(all_adausdt_cols) > 20:
        print(f"   ... and {len(all_adausdt_cols) - 20} more columns")

    print(f"\n🎯 Inspection complete! Check {LOG_FILENAME} for full column list.")


if __name__ == "__main__":
    main()