analyze_performance.py

#!/usr/bin/env python3
"""
Performance analysis script for encrypted audio transmission.
Tests all modulation schemes at various bitrates and noise levels,
then generates comprehensive charts to find optimal configurations.
"""

import subprocess
import json
import os
import sys
import numpy as np
import matplotlib.pyplot as plt
from matplotlib.gridspec import GridSpec
import re

# We'll run test.py and capture results
import test


def run_test_suite(modulation, mode='password', bandwidth_factors=None):
    """Run test.py for a specific modulation and capture results.
    
    Args:
        modulation: 'fsk2', 'fsk4', 'fsk8', or 'fsk16'
        mode: 'password' or 'rsa'
        bandwidth_factors: List of bandwidth factors [2, 4, 8] or None for no bandwidth limiting
    """
    print(f"\n{'='*80}")
    print(f"Running tests for {modulation.upper()} modulation...")
    if bandwidth_factors:
        print(f"Bandwidth factors: {bandwidth_factors}")
    print(f"{'='*80}\n")
    
    # Import and run test directly
    import argparse
    
    # Mock arguments
    args = argparse.Namespace(mode=mode, modulation=modulation)
    
    test_message = "Hello World! Testing 123."
    test_password = "radio_test_2025"
    output_dir = f"test_output_{modulation}"
    
    os.makedirs(output_dir, exist_ok=True)
    
    # Generate RSA keys if needed
    rsa_keys = None
    if mode == 'rsa':
        from cryptography.hazmat.primitives.asymmetric import rsa
        from cryptography.hazmat.backends import default_backend
        
        private_key = rsa.generate_private_key(
            public_exponent=65537,
            key_size=2048,
            backend=default_backend()
        )
        public_key = private_key.public_key()
        rsa_keys = (public_key, private_key)
    
    # Test different bitrates
    bitrates = [
        (100, "Ultra-slow HF SSB"),
        (200, "Very Slow HF"),
        (300, "Slow HF Digital"),
        (400, "Medium-Slow HF"),
        (500, "Medium HF"),
        (600, "Medium-Fast HF/VHF"),
        (700, "Fast HF/VHF"),
        (800, "Fast VHF"),
        (900, "Very Fast VHF"),
        (1000, "Very Fast VHF+"),
        (1200, "VHF Packet Radio (Standard)"),
    ]
    
    all_results = {}
    
    # Test without bandwidth limiting first
    print("\n--- Testing without bandwidth limiting ---")
    for bitrate, description in bitrates:
        try:
            results = test.test_at_bitrate(
                bitrate, test_message, test_password, 
                output_dir, mode=mode, rsa_keys=rsa_keys, 
                modulation=modulation, bandwidth_factor=None
            )
            all_results[bitrate] = results
        except Exception as e:
            print(f"Error testing {bitrate} bps: {e}")
            all_results[bitrate] = {}
    
    # Test with bandwidth limiting if specified
    if bandwidth_factors:
        for bw_factor in bandwidth_factors:
            print(f"\n--- Testing with {bw_factor}x bandwidth limiting ---")
            for bitrate, description in bitrates:
                try:
                    results = test.test_at_bitrate(
                        bitrate, test_message, test_password, 
                        output_dir, mode=mode, rsa_keys=rsa_keys, 
                        modulation=modulation, bandwidth_factor=bw_factor
                    )
                    # Merge results into all_results
                    if bitrate in all_results:
                        all_results[bitrate].update(results)
                    else:
                        all_results[bitrate] = results
                except Exception as e:
                    print(f"Error testing {bitrate} bps with BW/{bw_factor}: {e}")
    
    # Restore original values
    test.encrypt.BIT_DURATION = test.ORIGINAL_BIT_DURATION
    test.decrypt.BIT_DURATION = test.ORIGINAL_BIT_DURATION
    
    return all_results


def extract_metrics(all_modulations_results):
    """Extract metrics for plotting."""
    metrics = {
        'modulations': [],
        'bitrates': [],
        'scenarios': [],
        'success_rates': {},
        'ber_values': {}
    }
    
    # Get all unique scenarios from first modulation's first bitrate
    first_mod = list(all_modulations_results.keys())[0]
    first_rate = list(all_modulations_results[first_mod].keys())[0]
    scenarios = list(all_modulations_results[first_mod][first_rate].keys())
    
    metrics['scenarios'] = scenarios
    
    for modulation, bitrate_results in all_modulations_results.items():
        metrics['modulations'].append(modulation)
        
        for bitrate, scenario_results in bitrate_results.items():
            if bitrate not in metrics['bitrates']:
                metrics['bitrates'].append(bitrate)
            
            for scenario, result in scenario_results.items():
                key = (modulation, bitrate, scenario)
                metrics['success_rates'][key] = 1.0 if result['success'] else 0.0
                metrics['ber_values'][key] = result['ber'] if result['ber'] is not None else 1.0
    
    metrics['bitrates'] = sorted(metrics['bitrates'])
    
    return metrics


def plot_comprehensive_analysis(metrics):
    """Create comprehensive visualization of all results."""
    
    modulations = metrics['modulations']
    bitrates = metrics['bitrates']
    scenarios = metrics['scenarios']
    
    # Create figure with multiple subplots
    fig = plt.figure(figsize=(20, 12))
    gs = GridSpec(3, 3, figure=fig, hspace=0.3, wspace=0.3)
    
    # Color map for modulations
    colors = {
        'fsk2': '#1f77b4',
        'fsk4': '#ff7f0e', 
        'fsk8': '#2ca02c',
        'fsk16': '#d62728'
    }
    
    # 1. Overall Success Rate by Bitrate and Modulation
    ax1 = fig.add_subplot(gs[0, :2])
    for mod in modulations:
        success_rates = []
        for bitrate in bitrates:
            total_success = 0
            count = 0
            for scenario in scenarios:
                key = (mod, bitrate, scenario)
                if key in metrics['success_rates']:
                    total_success += metrics['success_rates'][key]
                    count += 1
            avg_success = (total_success / count * 100) if count > 0 else 0
            success_rates.append(avg_success)
        
        ax1.plot(bitrates, success_rates, 'o-', linewidth=2, 
                label=mod.upper(), color=colors.get(mod, 'gray'), markersize=6)
    
    ax1.set_xlabel('Bitrate (bps)', fontsize=11, fontweight='bold')
    ax1.set_ylabel('Success Rate (%)', fontsize=11, fontweight='bold')
    ax1.set_title('Overall Success Rate vs Bitrate (All Conditions)', fontsize=13, fontweight='bold')
    ax1.legend(loc='best', fontsize=10)
    ax1.grid(True, alpha=0.3)
    ax1.set_ylim([-5, 105])
    
    # 2. Heatmap: Success rate by modulation and noise level
    ax2 = fig.add_subplot(gs[0, 2])
    
    noise_scenarios = [s for s in scenarios if 'noise' in s.lower() or s == 'Clean Signal']
    noise_labels = []
    for s in noise_scenarios:
        if 'Clean' in s:
            noise_labels.append('Clean')
        else:
            # Extract SNR value
            match_snr = re.search(r'(-?\d+)dB', s)
            # Extract bandwidth factor if present
            match_bw = re.search(r'BW/(\d+)', s)
            
            if match_snr:
                snr_val = match_snr.group(1)
                if match_bw:
                    # Put BW on same line with smaller separator
                    label = f"{snr_val}dB BW/{match_bw.group(1)}"
                else:
                    label = f"{snr_val}dB"
                noise_labels.append(label)
            else:
                noise_labels.append(s)
    
    heatmap_data = []
    for mod in modulations:
        row = []
        for scenario in noise_scenarios:
            total_success = 0
            count = 0
            for bitrate in bitrates:
                key = (mod, bitrate, scenario)
                if key in metrics['success_rates']:
                    total_success += metrics['success_rates'][key]
                    count += 1
            avg = (total_success / count * 100) if count > 0 else 0
            row.append(avg)
        heatmap_data.append(row)
    
    im = ax2.imshow(heatmap_data, cmap='RdYlGn', aspect='auto', vmin=0, vmax=100)
    ax2.set_xticks(range(len(noise_labels)))
    ax2.set_xticklabels(noise_labels, rotation=45, ha='right', fontsize=7)
    ax2.set_yticks(range(len(modulations)))
    ax2.set_yticklabels([m.upper() for m in modulations], fontsize=10)
    ax2.set_title('Success Rate by Modulation & Noise', fontsize=11, fontweight='bold')
    plt.colorbar(im, ax=ax2, label='Success %')
    
    # 3. BER vs SNR for each modulation at optimal bitrate
    ax3 = fig.add_subplot(gs[1, 0])
    
    # Find optimal bitrate for each modulation (highest bitrate with >80% success)
    optimal_bitrates = {}
    for mod in modulations:
        for bitrate in reversed(bitrates):
            total_success = 0
            count = 0
            for scenario in scenarios:
                key = (mod, bitrate, scenario)
                if key in metrics['success_rates']:
                    total_success += metrics['success_rates'][key]
                    count += 1
            avg_success = (total_success / count) if count > 0 else 0
            if avg_success >= 0.8:
                optimal_bitrates[mod] = bitrate
                break
        if mod not in optimal_bitrates:
            optimal_bitrates[mod] = bitrates[0]
    
    snr_values = [20, 15, 10, 5, 0, -5, -10, -15]
    snr_scenario_map = {
        20: 'Noise SNR=20dB',
        15: 'Noise SNR=15dB',
        10: 'Noise SNR=10dB',
        5: 'Noise SNR=5dB',
        0: 'Noise SNR=0dB',
        -5: 'Noise SNR=-5dB',
        -10: 'Noise SNR=-10dB',
        -15: 'Noise SNR=-15dB (Extreme)'
    }
    
    for mod in modulations:
        ber_values = []
        bitrate = optimal_bitrates[mod]
        for snr in snr_values:
            scenario = snr_scenario_map.get(snr)
            if scenario:
                key = (mod, bitrate, scenario)
                ber = metrics['ber_values'].get(key, 1.0)
                ber_values.append(ber * 100)  # Convert to percentage (0-100%)
        
        ax3.plot(snr_values, ber_values, 'o-', linewidth=2, 
                label=f'{mod.upper()} ({bitrate}bps)', 
                color=colors.get(mod, 'gray'), markersize=6)
    
    ax3.set_xlabel('SNR (dB)', fontsize=11, fontweight='bold')
    ax3.set_ylabel('Bit Error Rate (%)', fontsize=11, fontweight='bold')
    ax3.set_title('BER vs SNR at Optimal Bitrates', fontsize=11, fontweight='bold')
    ax3.legend(loc='best', fontsize=9)
    ax3.grid(True, alpha=0.3)
    ax3.set_ylim([0, 100])  # 0-100% range
    ax3.invert_xaxis()
    
    # 4. Maximum reliable bitrate by modulation
    ax4 = fig.add_subplot(gs[1, 1])
    
    max_reliable = {}
    for mod in modulations:
        for bitrate in reversed(bitrates):
            # Count only noise scenarios for reliability
            noise_success = 0
            noise_count = 0
            for scenario in noise_scenarios:
                key = (mod, bitrate, scenario)
                if key in metrics['success_rates']:
                    noise_success += metrics['success_rates'][key]
                    noise_count += 1
            avg = (noise_success / noise_count) if noise_count > 0 else 0
            if avg >= 0.85:  # 85% success in noise scenarios
                max_reliable[mod] = bitrate
                break
        if mod not in max_reliable:
            max_reliable[mod] = bitrates[0]
    
    bars = ax4.bar(range(len(modulations)), 
                   [max_reliable.get(m, 0) for m in modulations],
                   color=[colors.get(m, 'gray') for m in modulations],
                   alpha=0.7, edgecolor='black', linewidth=1.5)
    ax4.set_xticks(range(len(modulations)))
    ax4.set_xticklabels([m.upper() for m in modulations], fontsize=10)
    ax4.set_ylabel('Max Reliable Bitrate (bps)', fontsize=11, fontweight='bold')
    ax4.set_title('Maximum Reliable Bitrate (>85% success)', fontsize=11, fontweight='bold')
    ax4.grid(True, alpha=0.3, axis='y')
    
    # Add value labels on bars
    for i, (mod, bar) in enumerate(zip(modulations, bars)):
        height = bar.get_height()
        ax4.text(bar.get_x() + bar.get_width()/2., height,
                f'{int(height)}',
                ha='center', va='bottom', fontweight='bold', fontsize=10)
    
    # 5. Success rate by scenario (aggregated across bitrates)
    ax5 = fig.add_subplot(gs[1, 2])
    
    # Use the same ordering as the heatmap (noise_scenarios from above)
    # This ensures consistent ordering across charts
    scenario_subset = noise_scenarios  # Reuse from heatmap
    
    x_pos = np.arange(len(modulations))
    width = 0.8 / len(scenario_subset) if len(scenario_subset) > 0 else 0.15
    
    for i, scenario in enumerate(scenario_subset):
        rates = []
        for mod in modulations:
            total = 0
            count = 0
            for bitrate in bitrates:
                key = (mod, bitrate, scenario)
                if key in metrics['success_rates']:
                    total += metrics['success_rates'][key]
                    count += 1
            avg = (total / count * 100) if count > 0 else 0
            rates.append(avg)
        
        # Create short label (reuse same logic as heatmap)
        if 'Clean' in scenario:
            label = 'Clean'
        else:
            match_snr = re.search(r'(-?\d+)dB', scenario)
            match_bw = re.search(r'BW/(\d+)', scenario)
            
            if match_snr:
                label = f"{match_snr.group(1)}dB"
                if match_bw:
                    label += f" BW/{match_bw.group(1)}"
            else:
                label = scenario[:15]
        
        offset = width * (i - len(scenario_subset)/2)
        ax5.bar(x_pos + offset, rates, width, 
               label=label, alpha=0.8)
    
    ax5.set_xlabel('Modulation', fontsize=11, fontweight='bold')
    ax5.set_ylabel('Success Rate (%)', fontsize=11, fontweight='bold')
    ax5.set_title('Success by SNR Level & Modulation (same order as heatmap)', fontsize=11, fontweight='bold')
    ax5.set_xticks(x_pos)
    ax5.set_xticklabels([m.upper() for m in modulations], fontsize=10)
    ax5.legend(loc='best', fontsize=6, ncol=3)
    ax5.grid(True, alpha=0.3, axis='y')
    ax5.set_ylim([0, 105])
    
    # 6. Efficiency chart (chars per second at max reliable bitrate)
    ax6 = fig.add_subplot(gs[2, 0])
    
    # Assuming average overhead from encryption/error correction
    test_msg_len = len("Hello World! Testing 123.")  # 26 chars
    
    efficiency = {}
    for mod in modulations:
        bitrate = max_reliable.get(mod, 100)
        # Rough estimate: ~40% overhead from encryption/RS/preamble
        chars_per_sec = (bitrate * 0.6) / 8  # Convert to bytes, account for overhead
        efficiency[mod] = chars_per_sec
    
    bars = ax6.bar(range(len(modulations)),
                   [efficiency.get(m, 0) for m in modulations],
                   color=[colors.get(m, 'gray') for m in modulations],
                   alpha=0.7, edgecolor='black', linewidth=1.5)
    ax6.set_xticks(range(len(modulations)))
    ax6.set_xticklabels([m.upper() for m in modulations], fontsize=10)
    ax6.set_ylabel('Throughput (chars/sec)', fontsize=11, fontweight='bold')
    ax6.set_title('Effective Throughput at Max Reliable Bitrate (≥85% success)', fontsize=11, fontweight='bold')
    ax6.grid(True, alpha=0.3, axis='y')
    
    for bar in bars:
        height = bar.get_height()
        ax6.text(bar.get_x() + bar.get_width()/2., height,
                f'{height:.1f}',
                ha='center', va='bottom', fontweight='bold', fontsize=10)
    
    # 7. Failure analysis - where each modulation fails
    ax7 = fig.add_subplot(gs[2, 1])
    
    failure_points = {}
    for mod in modulations:
        # Find first bitrate where success drops below 50%
        for bitrate in bitrates:
            total_success = 0
            count = 0
            for scenario in scenarios:
                key = (mod, bitrate, scenario)
                if key in metrics['success_rates']:
                    total_success += metrics['success_rates'][key]
                    count += 1
            avg = (total_success / count) if count > 0 else 0
            if avg < 0.5:
                failure_points[mod] = bitrate
                break
        if mod not in failure_points:
            failure_points[mod] = max(bitrates)
    
    bars = ax7.bar(range(len(modulations)),
                   [failure_points.get(m, 0) for m in modulations],
                   color=[colors.get(m, 'gray') for m in modulations],
                   alpha=0.7, edgecolor='black', linewidth=1.5)
    ax7.set_xticks(range(len(modulations)))
    ax7.set_xticklabels([m.upper() for m in modulations], fontsize=10)
    ax7.set_ylabel('Bitrate (bps)', fontsize=11, fontweight='bold')
    ax7.set_title('Failure Point (<50% success)', fontsize=11, fontweight='bold')
    ax7.grid(True, alpha=0.3, axis='y')
    
    for bar in bars:
        height = bar.get_height()
        ax7.text(bar.get_x() + bar.get_width()/2., height,
                f'{int(height)}',
                ha='center', va='bottom', fontweight='bold', fontsize=10)
    
    # 8. Recommendations text box
    ax8 = fig.add_subplot(gs[2, 2])
    ax8.axis('off')
    
    recommendations = "RECOMMENDATIONS\n" + "="*30 + "\n\n"
    
    # Best for reliability
    best_reliable = max(max_reliable.items(), key=lambda x: x[1])
    recommendations += f"🏆 HIGHEST BITRATE:\n{best_reliable[0].upper()} at {best_reliable[1]} bps\n\n"
    
    # Best balance
    balance_scores = {}
    for mod in modulations:
        score = max_reliable.get(mod, 0) * efficiency.get(mod, 0)
        balance_scores[mod] = score
    best_balance = max(balance_scores.items(), key=lambda x: x[1])
    recommendations += f"⚖️ BEST BALANCE:\n{best_balance[0].upper()}\n"
    recommendations += f"({max_reliable[best_balance[0]]} bps, "
    recommendations += f"{efficiency[best_balance[0]]:.1f} chars/sec)\n\n"
    
    ax8.text(0.05, 0.95, recommendations, transform=ax8.transAxes,
            fontsize=9, verticalalignment='top', family='monospace',
            bbox=dict(boxstyle='round', facecolor='wheat', alpha=0.8))
    
    plt.suptitle('Encrypted Audio Transmission - Performance Analysis', 
                fontsize=16, fontweight='bold', y=0.995)
    
    plt.savefig('performance_analysis.png', dpi=150, bbox_inches='tight')
    print(f"\n✓ Chart saved: performance_analysis.png")
    
    plt.show()


def main():
    import argparse
    
    parser = argparse.ArgumentParser(
        description='Comprehensive performance analysis across all modulation schemes'
    )
    parser.add_argument('--bandwidth', '-bw', nargs='+', type=int, choices=[2, 4, 8],
                        help='Bandwidth limiting factors to test (e.g., --bandwidth 2 4 8)')
    args = parser.parse_args()
    
    print("="*80)
    print("COMPREHENSIVE PERFORMANCE ANALYSIS")
    print("Testing all modulation schemes across bitrates and conditions")
    if args.bandwidth:
        print(f"Bandwidth limiting: {args.bandwidth}x downsampling")
    print("="*80)
    
    modulations = ['fsk2', 'fsk4', 'fsk8', 'fsk16']
    mode = 'password'  # Use password mode for consistency
    
    all_results = {}
    
    for modulation in modulations:
        results = run_test_suite(modulation, mode, bandwidth_factors=args.bandwidth)
        all_results[modulation] = results
    
    print("\n" + "="*80)
    print("GENERATING ANALYSIS CHARTS...")
    print("="*80)
    
    metrics = extract_metrics(all_results)
    plot_comprehensive_analysis(metrics)
    
    print("\n" + "="*80)
    print("ANALYSIS COMPLETE!")
    print("="*80)
    print("\nKey findings:")
    print("- Check 'performance_analysis.png' for comprehensive visualization")
    print("- Compare modulation schemes side-by-side")
    print("- Identify optimal bitrate/modulation combinations")
    print("- Understand noise resistance trade-offs")


if __name__ == '__main__':
    main()