feat: add single-tx emulator runner

.gitignore

@@ -162,3 +162,11 @@ cython_debug/
 failed_txs.json
 test.json
 failed_traces.json
+dump
+
+packs 
+runs
+failed_txs
+emulator_post_bocs
+emulator_inputs
+cpp_test_with_failed

RUNBOOK.md

@@ -0,0 +1,89 @@
+# TonTVMReplay Runbook
+
+**Folders**
+- Default runs live in `runs/default/`.
+- Single‑transaction runs live in `runs/single/<tx_hash>/`.
+- Root `failed_txs.json` is a symlink to `runs/default/failed_txs.json` (latest default run).
+
+**Default Run (range of MC blocks)**
+- Use when you want to emulate everything in a block range.
+- Requires valid `LITESERVER_*` and `EMULATOR_*` in `.env`.
+
+Example:
+```bash
+# .env
+FROM_SEQNO=57160550
+TO_SEQNO=57160555
+TO_EMULATE_MC_BLOCKS=1
+
+# run
+ tonemuso 2>&1 | tee runs/default/run_57160550_57160555.log
+```
+Output:
+- `runs/default/failed_txs.json` (via symlink `failed_txs.json`)
+- logs in `runs/default/`
+
+**Whitelist Run (specific tx list)**
+- Use when you want to emulate only listed txs (still pulls key‑block config via LS).
+- Requires `TXS_TO_PROCESS_PATH`.
+
+Example:
+```bash
+# .env
+TXS_TO_PROCESS_PATH=./txs_to_process.json
+
+# run
+ tonemuso 2>&1 | tee runs/single/whitelist.log
+```
+
+**Single Tx Run (one tx only, no full range)**
+- Use `single_tx_emulate.py` to emulate exactly one tx and write a dedicated failed file.
+
+Example:
+```bash
+./.venv/bin/python single_tx_emulate.py \
+  --tx 4C90C139A5736F34EA3EEF62F0B06431719913835EA5A1B9173F20B2EF711583 \
+  --workchain 0 \
+  --shard 9223372036854775808 \
+  --seqno 61926502 \
+  --root-hash F19E15E8A6E9EE5BA806582DD4E779C3168A514B057791A381E620A224C52A16 \
+  --file-hash 287388335AF81B33FFB9F18EB838516B5C3DBB26FA634F80A64692635816C8FB \
+  --out runs/single/4C90C139/failed_4C90C139.json \
+  2>&1 | tee runs/single/4C90C139/single_tx.log
+```
+
+**Optional: resolve block fields via tonapi**
+```bash
+./.venv/bin/python single_tx_emulate.py \
+  --tx <TX_HASH> --tonapi --out runs/single/<TX_HASH>/failed_<TX_HASH>.json \
+  2>&1 | tee runs/single/<TX_HASH>/single_tx.log
+```
+
+**Report Builder**
+- Build a compact report from local dumps and failed files.
+
+Example:
+```bash
+./.venv/bin/python report_single_tx.py \
+  --tx 4C90C139A5736F34EA3EEF62F0B06431719913835EA5A1B9173F20B2EF711583 \
+  --addr 5D6595912014A95F20D91B614F0832694AD2A0EF54A1270EBAD0D8A862FADE82 \
+  --out runs/single/4C90C139/report_4C90C139.md
+```
+
+**Dump/Debug Env (optional)**
+Pre/post emulation dumps:
+- `EMULATOR_PRECALL_DUMP_DIR=./dump/precall`
+- `EMULATOR_POST_DUMP_DIR=./dump/post`
+- `EMULATOR_ACCOUNT_FAIL_DUMP_DIR=./dump/account_fail`
+
+Prev‑block dumps:
+- `EMULATOR_PREV_BLOCKS_DUMP_DIR=./dump/prev_blocks`
+- `EMULATOR_PREV_BLOCKS_DUMP_LOG=1`
+
+VM logs:
+- `EMULATOR_VM_LOG_PRINT=1`
+- `EMULATOR_VM_LOG_MAX=50000`
+- `EMULATOR_VM_LOG_VERBOSITY=2`
+
+**Notes**
+- If the run hangs at `Load key blocks`, the LS likely fails `get_config_all()` for the key‑block. Try another LS or higher `LITESERVER_TIMEOUT`.

single_tx_emulate.py

@@ -0,0 +1,207 @@
+#!/usr/bin/env python3
+import argparse
+import json
+import os
+import re
+from collections import Counter
+from multiprocessing import Queue
+from time import sleep
+from typing import Any, Dict, Optional, Tuple
+
+import requests
+from loguru import logger
+from tonpy.blockscanner.blockscanner import BlockId, BlockIdExt, BlockScanner, LiteClient
+
+from tonemuso.config import Config
+from tonemuso.modes.common import process_blocks, process_result
+from tonemuso.utils import b64_to_hex
+
+
+def _parse_block_tuple(s: str) -> Optional[Tuple[int, int, int]]:
+    # Accept "(0,8000000000000000,61926502)" or "0,8000000000000000,61926502"
+    m = re.search(r"\(?\s*(-?\d+)\s*,\s*(-?\d+)\s*,\s*(\d+)\s*\)?", s)
+    if not m:
+        return None
+    return int(m.group(1)), int(m.group(2)), int(m.group(3))
+
+
+def _normalize_hash(s: str) -> str:
+    s = (s or "").strip()
+    if not s:
+        return s
+    hex_chars = "0123456789abcdefABCDEF"
+    if len(s) == 64 and all(c in hex_chars for c in s):
+        return s.upper()
+    return b64_to_hex(s, uppercase=True)
+
+
+def _resolve_via_tonapi(tx_hash: str, base_url: str, api_key: Optional[str]) -> Dict[str, Any]:
+    headers = {}
+    if api_key:
+        headers["Authorization"] = f"Bearer {api_key}"
+
+    tx_url = f"{base_url.rstrip('/')}/v2/blockchain/transactions/{tx_hash}"
+    r = requests.get(tx_url, headers=headers, timeout=20)
+    if r.status_code != 200:
+        raise RuntimeError(f"tonapi tx lookup failed: {r.status_code} {r.text[:200]}")
+    data = r.json()
+    block_field = data.get("block")
+    if isinstance(block_field, dict):
+        wc = int(block_field.get("workchain_id", 0))
+        shard = int(block_field.get("shard", 0))
+        seqno = int(block_field.get("seqno", 0))
+    else:
+        parsed = _parse_block_tuple(str(block_field))
+        if not parsed:
+            raise RuntimeError(f"tonapi tx lookup: cannot parse block field: {block_field!r}")
+        wc, shard, seqno = parsed
+
+    blk_url = f"{base_url.rstrip('/')}/v2/blockchain/blocks/({wc},{shard},{seqno})"
+    r = requests.get(blk_url, headers=headers, timeout=20)
+    if r.status_code != 200:
+        raise RuntimeError(f"tonapi block lookup failed: {r.status_code} {r.text[:200]}")
+    blk = r.json()
+    root_hash = _normalize_hash(blk.get("root_hash", ""))
+    file_hash = _normalize_hash(blk.get("file_hash", ""))
+    return {
+        "hash": tx_hash.upper(),
+        "workchain": wc,
+        "shard": shard,
+        "seqno": seqno,
+        "root_hash": root_hash,
+        "file_hash": file_hash,
+    }
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--tx", required=True, help="tx hash (hex)")
+    ap.add_argument("--workchain", type=int)
+    ap.add_argument("--shard", type=int)
+    ap.add_argument("--seqno", type=int)
+    ap.add_argument("--root-hash")
+    ap.add_argument("--file-hash")
+    ap.add_argument("--tonapi", action="store_true", help="resolve block data via tonapi")
+    ap.add_argument("--tonapi-base", default=os.getenv("TONAPI_BASE", "https://tonapi.io"))
+    ap.add_argument("--tonapi-key", default=os.getenv("TONAPI_KEY"))
+    ap.add_argument("--out", default="failed_txs_single.json")
+    ap.add_argument("--nproc", type=int)
+    ap.add_argument("--chunk-size", type=int)
+    ap.add_argument("--tx-chunk-size", type=int)
+    ap.add_argument("--loglevel", type=int)
+    args = ap.parse_args()
+
+    tx_hash = args.tx.upper()
+
+    # Resolve block info if missing
+    if any(v is None for v in (args.workchain, args.shard, args.seqno, args.root_hash, args.file_hash)):
+        if not args.tonapi:
+            raise SystemExit(
+                "Missing block fields. Provide --workchain --shard --seqno --root-hash --file-hash "
+                "or pass --tonapi to resolve via tonapi."
+            )
+        info = _resolve_via_tonapi(tx_hash, args.tonapi_base, args.tonapi_key)
+    else:
+        info = {
+            "hash": tx_hash,
+            "workchain": args.workchain,
+            "shard": args.shard,
+            "seqno": args.seqno,
+            "root_hash": _normalize_hash(args.root_hash or ""),
+            "file_hash": _normalize_hash(args.file_hash or ""),
+        }
+
+    cfg = Config.from_env()
+    if args.nproc is not None:
+        cfg.nproc = args.nproc
+    if args.chunk_size is not None:
+        cfg.chunk_size = args.chunk_size
+    if args.tx_chunk_size is not None:
+        cfg.tx_chunk_size = args.tx_chunk_size
+    if args.loglevel is not None:
+        cfg.loglevel = args.loglevel
+
+    txs_whitelist = {tx_hash}
+    blocks_to_load = [
+        BlockIdExt(
+            BlockId(
+                workchain=info["workchain"],
+                shard=info["shard"],
+                seqno=info["seqno"],
+            ),
+            root_hash=info["root_hash"],
+            file_hash=info["file_hash"],
+        )
+    ]
+
+    lcparams = cfg.lcparams()
+    # Fail fast if LS params missing
+    if not cfg.liteserver_ip or not cfg.liteserver_port or not cfg.liteserver_pubkey:
+        raise SystemExit("LITESERVER_* env not set (SERVER, PORT, PUBKEY).")
+
+    outq = Queue()
+    raw_proc = process_blocks(
+        config_override=cfg.c7_rewrite,
+        trace_whitelist=None,
+        loglevel=cfg.loglevel,
+        color_schema=cfg.color_schema,
+        emulator_path=cfg.emulator_path,
+        emulator_unchanged_path=cfg.emulator_unchanged_path,
+        txs_whitelist=txs_whitelist,
+    )
+
+    scanner = BlockScanner(
+        lcparams=lcparams,
+        start_from=None,
+        load_to=None,
+        nproc=int(cfg.nproc),
+        loglevel=cfg.loglevel,
+        chunk_size=int(cfg.chunk_size),
+        tx_chunk_size=int(cfg.tx_chunk_size),
+        raw_process=raw_proc,
+        out_queue=outq,
+        only_mc_blocks=bool(cfg.only_mc_blocks),
+        parse_txs_over_ls=bool(cfg.parse_over_ls),
+        blocks_to_load=blocks_to_load,
+    )
+
+    logger.warning(f"Single tx emulate: {tx_hash}")
+    logger.warning(
+        f"Block: wc={info['workchain']} shard={info['shard']} seqno={info['seqno']} "
+        f"root={info['root_hash']} file={info['file_hash']}"
+    )
+
+    scanner.start()
+
+    success = 0
+    warnings = 0
+    unsuccess = []
+    while not scanner.done:
+        tmp_s, tmp_u, tmp_w = process_result(outq, loglevel=cfg.loglevel)
+        success += tmp_s
+        warnings += tmp_w
+        unsuccess.extend(tmp_u)
+        sleep(1)
+
+    tmp_s, tmp_u, tmp_w = process_result(outq, loglevel=cfg.loglevel)
+    success += tmp_s
+    warnings += tmp_w
+    unsuccess.extend(tmp_u)
+
+    logger.warning(f"Final emulator status: {success} success, {len(unsuccess)} unsuccess, {warnings} warnings")
+    if unsuccess:
+        cnt = Counter()
+        for i in unsuccess:
+            if isinstance(i, dict) and i.get("address"):
+                cnt[i["address"]] += 1
+        logger.error(f"Unique addreses errors: {len(cnt)}, most common: ")
+        logger.error(cnt.most_common(5))
+
+    with open(args.out, "w") as f:
+        json.dump(unsuccess, f, indent=2)
+    logger.warning(f"Wrote failed txs to {args.out}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

src/tonemuso/config.py

@@ -71,6 +71,18 @@ def from_env(cls) -> "Config":
         # Emulators
         cfg.emulator_path = os.getenv("EMULATOR_PATH")
         cfg.emulator_unchanged_path = os.getenv("EMULATOR_UNCHANGED_PATH")
+        if not cfg.emulator_path:
+            logger.error("EMULATOR_PATH is not set")
+        elif not os.path.exists(cfg.emulator_path):
+            logger.error(f"EMULATOR_PATH not found: {cfg.emulator_path}")
+        else:
+            logger.warning(f"Emulator primary: {cfg.emulator_path}")
+        if not cfg.emulator_unchanged_path:
+            logger.error("EMULATOR_UNCHANGED_PATH is not set")
+        elif not os.path.exists(cfg.emulator_unchanged_path):
+            logger.error(f"EMULATOR_UNCHANGED_PATH not found: {cfg.emulator_unchanged_path}")
+        else:
+            logger.warning(f"Emulator secondary: {cfg.emulator_unchanged_path}")
         cfg.c7_rewrite_raw = os.getenv("C7_REWRITE")
         try:
             cfg.c7_rewrite = json.loads(cfg.c7_rewrite_raw) if cfg.c7_rewrite_raw else None

src/tonemuso/emitted_processor.py

@@ -86,6 +86,8 @@ def emulate_internal_message_one_layer(self, block_key: BlockKey, msg: Union[Dic
                   or self.r._fetch_state_for_account(block_key, dest_addr))
         try:
             ok = em.emulate_transaction(state1, msg['cell'], now, lt)
+            if ok and (em.account is None or em.transaction is None):
+                ok = False
             if ok:
                 new_state = em.account.to_cell()
                 self.r.account_states1[block_key][dest_addr] = new_state

src/tonemuso/emulation.py

@@ -1,5 +1,6 @@
 # Copyright (c) 2024 Disintar LLP Licensed under the Apache License Version 2.0
 from typing import List, Optional, Tuple, Dict, Any
+import os
 
 from tonpy import Cell, VmDict, Address
 from tonpy.tvm.not_native.emulator_extern import EmulatorExtern
@@ -10,6 +11,13 @@
 from tonemuso.utils import hex_to_b64
 
 
+def _env_flag(name: str) -> bool:
+    v = os.getenv(name, "").strip()
+    if v == "":
+        return False
+    return v.lower() not in ("0", "false", "no")
+
+
 def init_emulators(block: Dict[str, Any], config_override: Dict[str, Any], emulator_path: str,
                    emulator_unchanged_path: str) -> Tuple[EmulatorExtern, EmulatorExtern]:
     """
@@ -30,9 +38,16 @@ def init_emulators(block: Dict[str, Any], config_override: Dict[str, Any], emula
     em = EmulatorExtern(emulator_path, config)
     em.set_rand_seed(block['rand_seed'])
 
-    prev_block_data = [list(reversed(block['prev_block_data'][1])),  # prev 16
+    # blockscanner provides prev_blocks in newest->oldest order
+    reverse_prev = _env_flag("EMULATOR_PREV_BLOCKS_REVERSE")
+    prev16 = block['prev_block_data'][1]
+    prev100 = block['prev_block_data'][0]
+    if reverse_prev:
+        prev16 = list(reversed(prev16)) if isinstance(prev16, list) else prev16
+        prev100 = list(reversed(prev100)) if isinstance(prev100, list) else prev100
+    prev_block_data = [prev16,  # prev 16
                        block['prev_block_data'][2],  # key block
-                       list(reversed(block['prev_block_data'][0]))]  # prev 16 by 100
+                       prev100]  # prev 16 by 100
     em.set_prev_blocks_info(prev_block_data)
     em.set_libs(VmDict(256, False, cell_root=Cell(block['libs'])))