implemented backend architecture for fee stats API

Author: b-l-u-e

backend/.gitignore

@@ -0,0 +1,11 @@
+# test files
+test_mock.py
+test_secure_connection.py
+test_rpc_ports.py
+test_getbestblockhash.py
+test_flask_blockchain_info.py
+
+rpc_config.ini
+
+
+app_mock.py

backend/README.md

@@ -0,0 +1,27 @@
+The Collector (The Brain): Runs in the background, continuously pulling real data, making a prediction, and archiving the result.
+
+The API (The Server): Provides instantaneous responses by querying the pre-computed data, avoiding slow, repeated calls to the Bitcoin Core node.
+
+app.py
+
+The API Server (Flask)
+
+Exposes endpoints for both real-time data (/fees) and high-performance historical analytics (/block-stats).
+
+collector.py
+
+The Background Worker
+
+Continuously monitors the blockchain, runs the custom prediction logic (get_custom_fee_prediction_asap), and archives the prediction-vs-actual data.
+
+json_rpc_request.py
+
+The RPC Client
+
+Handles all communication with the Bitcoin Core node. Includes exponential backoff and a built-in cache for reliable, efficient data retrieval.
+
+database.py
+
+The Persistence Layer (SQLite)
+
+Stores the historical performance of our model (prediction, min fee, max fee) over thousands of blocks for instant retrieval. Uses INSERT OR IGNORE for stability.

backend/app.py

@@ -0,0 +1,87 @@
+from flask import Flask, jsonify
+from flask_cors import CORS
+from werkzeug.middleware.proxy_fix import ProxyFix
+from json_rpc_request import (
+    estimate_smart_fee, 
+    get_mempool_info, 
+    get_blockchain_info, 
+    get_block_stats,
+    get_block_count
+)
+
+app = Flask(__name__)
+app.wsgi_app = ProxyFix(app.wsgi_app)
+CORS(app) 
+
+@app.route("/fees/<int:target>/<string:mode>/<int:level>", methods=['GET'])
+def fees(target, mode, level):
+    try:
+        result = estimate_smart_fee(conf_target=target, mode=mode, verbosity_level=level)
+        return jsonify(result)
+    except Exception as e:
+        print(f"RPC Error: {e}")
+        return jsonify({"error": f"Failed to get fee estimate: {str(e)}"}), 500
+
+@app.route("/mempool/info", methods=['GET'])
+def mempool_info():
+    try:
+        result = get_mempool_info()
+        return jsonify(result)
+    except Exception as e:
+        print(f"RPC Error: {e}")
+        return jsonify({"error": f"Failed to get mempool info: {str(e)}"}), 500
+
+@app.route("/blockchain/info", methods=['GET'])
+def blockchain_info():
+    try:
+        result = get_blockchain_info()
+        return jsonify(result)
+    except Exception as e:
+        print(f"RPC Error: {e}")
+        return jsonify({"error": f"Failed to get blockchain info: {str(e)}"}), 500
+
+@app.route("/blockstats/<int:block_height>", methods=['GET'])
+def block_stats(block_height):
+    try:
+        result = get_block_stats(block_height)
+        return jsonify(result)
+    except Exception as e:
+        print(f"RPC Error: {e}")
+        return jsonify({"error": f"Failed to get block stats: {str(e)}"}), 500
+
+@app.route("/blockcount", methods=['GET'])
+def block_count():
+    try:
+        result = get_block_count()
+        return jsonify({"blockcount": result})
+    except Exception as e:
+        print(f"RPC Error: {e}")
+        return jsonify({"error": f"Failed to get block count: {str(e)}"}), 500
+
+@app.route("/health", methods=['GET'])
+def health():
+    try:
+        get_blockchain_info()
+        return jsonify({
+            "status": "healthy", 
+            "service": "bitcoin-core-fees-api",
+            "rpc_connected": True
+        })
+    except Exception as e:
+        return jsonify({
+            "status": "unhealthy", 
+            "service": "bitcoin-core-fees-api",
+            "rpc_connected": False,
+            "error": str(e)
+        }), 503
+
+@app.errorhandler(404)
+def page_not_found(error):
+    return jsonify({"error": "Endpoint not found"}), 404
+
+@app.errorhandler(500)
+def internal_error(error):
+    return jsonify({"error": "Internal server error"}), 500
+
+if __name__ == "__main__":
+    app.run(debug=True, host='0.0.0.0', port=5000)

backend/collector.py

@@ -0,0 +1,140 @@
+import time
+import random
+import os
+from database import init_db, insert_analysis
+from json_rpc_request import (
+    get_block_count, 
+    get_block_tx_details, 
+    get_raw_mempool, 
+    get_block_template, 
+    get_estimated_fee_rate_satvb
+)
+
+# Configuration for the collector loop
+COLLECTOR_SLEEP_TIME = 30  # for a new block every 30 seconds
+LAST_PROCESSED_HEIGHT = 0
+INITIAL_HISTORY_DEPTH = 1000 # The number of blocks to fetch initially
+
+# custom fee logic
+
+def get_custom_fee_prediction_asap() -> float:
+    """
+    CRITICAL: This is the placeholder where your custom, robust fee estimation 
+    logic must be implemented. It should use the latest RPC data to make a prediction.
+    
+    Returns: A single ASAP feerate prediction in sat/vB.
+    """
+    try:
+        # 1. Fetch real-time data from the node
+        # These are available for your custom logic:
+        # mempool_data = get_raw_mempool(verbose=False) 
+        # block_template = get_block_template() 
+
+        #  A SIMPLE, TEMPORARY MODEL BASED ON Core's 1-Block Estimate 
+        # REPLACE THIS WITH YOUR CUSTOM LOGIC (Feerate Bucketing, Resting Time Analysis, etc.)
+        
+        # We use Core's estimate as a reliable temporary proxy for demonstration
+        core_estimate = get_estimated_fee_rate_satvb(conf_target=1, mode='conservative')
+        predicted_fee = core_estimate.get('feerate_sat_per_vb')
+
+        if predicted_fee is None or predicted_fee < 1.0:
+            # Fallback for when estimatesmartfee fails
+            print("Warning: estimatesmartfee failed. Using a random prediction.")
+            return round(random.uniform(5.0, 15.0), 2)
+        
+        # Add a small random jitter to simulate a model slightly different from Core
+        jitter = random.uniform(-0.5, 1.5)
+        return max(1.0, predicted_fee + jitter)
+        
+    except Exception as e:
+        print(f"Prediction Error: {e}. Returning fallback fee.")
+        return 10.0 # Safe fallback fee
+
+# block processing 
+
+def process_block(height: int):
+    """
+    Fetches actual block data, runs prediction, and stores the result.
+    """
+    try:
+        # 1. Run the Prediction Logic (What would we have predicted for this block?)
+        # Since we are processing blocks sequentially, we use the current prediction 
+        # as a proxy for the prediction we would have made just before the block was found.
+        predicted_fee = get_custom_fee_prediction_asap()
+
+        # 2. Get Actuals (Ground Truth) for the MINED block
+        block_details = get_block_tx_details(height)
+        
+        if block_details and block_details.get('min_fee') is not None:
+            min_fee = block_details['min_fee']
+            max_fee = block_details['max_fee']
+            
+            # 3. Store the result
+            insert_analysis(height, min_fee, max_fee, predicted_fee)
+            print(f"[COLLECTOR] Processed Block {height}: Actual Min={min_fee}, Predicted={predicted_fee}")
+            return True
+        else:
+            # This happens if the block is still being processed or is empty/invalid
+            print(f"[COLLECTOR] Skipped Block {height}: No valid fee details found.")
+            return False
+            
+    except Exception as e:
+        print(f"[COLLECTOR] Error processing block {height}: {e}")
+        return False
+
+def run_collector_cycle(initial_population: bool = False):
+    """
+    Executes one cycle: detects and processes new blocks since the last check.
+    """
+    global LAST_PROCESSED_HEIGHT
+    
+    current_height = get_block_count()
+    if current_height is None:
+        return
+
+    if LAST_PROCESSED_HEIGHT == 0:
+        # On first run, set the last processed height to the current height minus one
+        LAST_PROCESSED_HEIGHT = current_height - 1 
+
+    
+    if initial_population:
+        # For initial run, process a large range of historical blocks
+        start_height = max(1, current_height - INITIAL_HISTORY_DEPTH)
+        print(f"\n[COLLECTOR] Starting initial population from Block {start_height} to {current_height}...")
+    else:
+        # For continuous run, process only new blocks
+        start_height = LAST_PROCESSED_HEIGHT + 1
+        
+    
+    blocks_to_process = range(start_height, current_height + 1)
+    
+    for height in blocks_to_process:
+        if height > LAST_PROCESSED_HEIGHT:
+            if process_block(height):
+                LAST_PROCESSED_HEIGHT = height
+        else:
+             # Skip blocks already processed during initial population
+             continue
+
+
+def start_collector():
+    """Main function to run the collector indefinitely."""
+    print("--- Starting Fee Estimation Collector ---")
+    
+    # 1. Initial Historical Population (fills the database)
+    run_collector_cycle(initial_population=True)
+    
+    print("\nInitial historical population complete. Monitoring for new blocks...")
+    
+    # 2. Continuous Monitoring Loop
+    while True:
+        try:
+            run_collector_cycle(initial_population=False)
+        except Exception as e:
+            print(f"[COLLECTOR] Critical main loop error: {e}. Retrying in {COLLECTOR_SLEEP_TIME}s.")
+        
+        time.sleep(COLLECTOR_SLEEP_TIME)
+
+if __name__ == '__main__':
+    init_db()
+    start_collector()

backend/database.py

@@ -0,0 +1,65 @@
+# hey what do u think of having database??
+
+
+import sqlite3
+
+DATABASE_NAME = 'fee_analysis.db'
+
+def init_db():
+    """Initializes the SQLite database and creates the fee_analysis table."""
+    conn = sqlite3.connect(DATABASE_NAME)
+    cursor = conn.cursor()
+    
+    # Create the table to store historical data:
+    # 1. block_height: Primary key
+    # 2. min_feerate: Actual minimum feerate of a transaction in the block (sat/vB)
+    # 3. max_feerate: Actual maximum feerate of a transaction in the block (sat/vB)
+    # 4. predicted_feerate: Our model's prediction for the ASAP feerate (sat/vB)
+    # 5. forecaster_name: Allows comparison of multiple models
+    cursor.execute("""
+        CREATE TABLE IF NOT EXISTS fee_analysis (
+            block_height INTEGER PRIMARY KEY,
+            min_feerate REAL NOT NULL,
+            max_feerate REAL NOT NULL,
+            predicted_feerate REAL NOT NULL,
+            forecaster_name TEXT NOT NULL
+        )
+    """)
+    conn.commit()
+    conn.close()
+
+def insert_analysis(block_height, min_feerate, max_feerate, predicted_feerate, forecaster_name="OurModelV1"):
+    """Inserts one record of block analysis into the database."""
+    conn = sqlite3.connect(DATABASE_NAME)
+    cursor = conn.cursor()
+    try:
+        cursor.execute("""
+            INSERT OR IGNORE INTO fee_analysis (block_height, min_feerate, max_feerate, predicted_feerate, forecaster_name)
+            VALUES (?, ?, ?, ?, ?)
+        """, (block_height, min_feerate, max_feerate, predicted_feerate, forecaster_name))
+        conn.commit()
+    except sqlite3.Error as e:
+        print(f"Database Error on insert: {e}")
+    finally:
+        conn.close()
+
+def fetch_analysis_range(start_height, end_height, forecaster_name="OurModelV1"):
+    """Fetches all necessary data for a given block range."""
+    conn = sqlite3.connect(DATABASE_NAME)
+    cursor = conn.cursor()
+    
+
+    cursor.execute("""
+        SELECT block_height, min_feerate, max_feerate, predicted_feerate
+        FROM fee_analysis
+        WHERE block_height <= ? AND block_height > ? AND forecaster_name = ?
+        ORDER BY block_height DESC
+    """, (start_height, end_height, forecaster_name))
+    
+    results = cursor.fetchall()
+    conn.close()
+    return results
+
+if __name__ == '__main__':
+    init_db()
+    print("Database initialized.")